From 61aedf201c2c4bf0e5aa4db32e74f4d860b88593 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Tue, 19 Dec 2017 23:39:21 +0000 Subject: New upstream version 1.9.0+dfsg --- .codeclimate.yml | 99 +- ChangeLog | 6 + Dockerfile | 3 + Dockerfile.armv7hf | 19 + LICENSE-REDISTRIBUTED.md | 13 + Makefile.am | 1 + Makefile.in | 1 + README.md | 44 +- charts.d/apache.chart.sh | 4 +- charts.d/cpufreq.chart.sh | 2 +- charts.d/example.chart.sh | 2 +- charts.d/hddtemp.chart.sh | 4 +- charts.d/nginx.chart.sh | 2 +- charts.d/phpfpm.chart.sh | 16 +- charts.d/sensors.chart.sh | 2 +- charts.d/tomcat.chart.sh | 4 +- conf.d/Makefile.am | 11 +- conf.d/Makefile.in | 11 +- conf.d/apps_groups.conf | 18 +- conf.d/health.d/beanstalkd.conf | 36 + conf.d/health.d/bind_rndc.conf | 2 +- conf.d/health.d/couchdb.conf | 13 + conf.d/health.d/redis.conf | 20 + conf.d/health.d/tcp_conn.conf | 19 + conf.d/health.d/tcp_listen.conf | 27 + conf.d/health.d/tcp_mem.conf | 20 + conf.d/health.d/tcp_orphans.conf | 21 + conf.d/health.d/tcp_resets.conf | 4 +- conf.d/health_alarm_notify.conf | 66 +- conf.d/python.d.conf | 14 +- conf.d/python.d/apache.conf | 21 +- conf.d/python.d/apache_cache.conf | 76 - conf.d/python.d/beanstalk.conf | 80 + conf.d/python.d/bind_rndc.conf | 29 +- conf.d/python.d/chrony.conf | 21 +- conf.d/python.d/couchdb.conf | 91 + conf.d/python.d/cpufreq.conf | 8 +- conf.d/python.d/dns_query_time.conf | 39 +- conf.d/python.d/dnsdist.conf | 85 + conf.d/python.d/dovecot.conf | 21 +- conf.d/python.d/elasticsearch.conf | 21 +- conf.d/python.d/example.conf | 21 +- conf.d/python.d/exim.conf | 21 +- conf.d/python.d/fail2ban.conf | 29 +- conf.d/python.d/freeradius.conf | 42 +- conf.d/python.d/go_expvar.conf | 24 +- conf.d/python.d/haproxy.conf | 21 +- conf.d/python.d/hddtemp.conf | 24 +- conf.d/python.d/ipfs.conf | 21 +- conf.d/python.d/isc_dhcpd.conf | 32 +- conf.d/python.d/mdstat.conf | 8 +- conf.d/python.d/memcached.conf | 21 +- conf.d/python.d/mongodb.conf | 21 +- conf.d/python.d/mysql.conf | 21 +- conf.d/python.d/nginx.conf | 24 +- conf.d/python.d/nsd.conf | 21 +- conf.d/python.d/ovpn_status_log.conf | 23 +- conf.d/python.d/phpfpm.conf | 21 +- conf.d/python.d/postfix.conf | 21 +- conf.d/python.d/postgres.conf | 23 +- conf.d/python.d/powerdns.conf | 78 + conf.d/python.d/rabbitmq.conf | 21 +- conf.d/python.d/redis.conf | 21 +- conf.d/python.d/retroshare.conf | 21 +- conf.d/python.d/samba.conf | 24 +- conf.d/python.d/sensors.conf | 11 +- conf.d/python.d/smartd_log.conf | 33 +- conf.d/python.d/squid.conf | 21 +- conf.d/python.d/tomcat.conf | 21 +- conf.d/python.d/varnish.conf | 23 +- conf.d/python.d/web_log.conf | 21 +- conf.d/stream.conf | 35 +- configs.signatures | 87 +- configure | 22 +- configure.ac | 2 +- contrib/debian/changelog | 3 - contrib/debian/compat | 1 - contrib/debian/control | 25 - contrib/debian/control.wheezy | 25 - contrib/debian/copyright | 10 - contrib/debian/netdata.conf | 16 - contrib/debian/netdata.default | 5 - contrib/debian/netdata.docs | 1 - contrib/debian/netdata.init | 56 - contrib/debian/netdata.install | 1 - contrib/debian/netdata.lintian-overrides | 16 - contrib/debian/netdata.postinst.in | 41 - contrib/debian/netdata.postrm | 43 - contrib/debian/netdata.service | 14 - contrib/debian/rules | 87 - contrib/debian/source/format | 1 - diagrams/netdata-overview.xml | 2 +- docker-build.sh | 3 - installer/functions.sh | 102 +- makeself/install-or-update.sh | 2 + makeself/jobs/70-netdata-git.install.sh | 8 + makeself/jobs/99-makeself.install.sh | 95 +- makeself/makeself-header.sh | 5 +- makeself/makeself.lsm | 2 +- makeself/makeself.sh | 8 +- netdata-installer.sh | 53 +- netdata.spec | 26 +- netdata.spec.in | 20 + node.d/named.node.js | 2 +- node.d/node_modules/netdata.js | 9 +- plugins.d/Makefile.am | 1 + plugins.d/Makefile.in | 1 + plugins.d/alarm-notify.sh | 200 +- plugins.d/cgroup-name.sh | 5 + plugins.d/cgroup-network-helper.sh | 256 ++ plugins.d/python.d.plugin | 869 +++--- python.d/Makefile.am | 34 +- python.d/Makefile.in | 154 +- python.d/README.md | 353 ++- python.d/apache.chart.py | 37 +- python.d/apache_cache.chart.py | 60 - python.d/beanstalk.chart.py | 250 ++ python.d/bind_rndc.chart.py | 26 +- python.d/chrony.chart.py | 4 +- python.d/couchdb.chart.py | 410 +++ python.d/cpufreq.chart.py | 23 +- python.d/cpuidle.chart.py | 27 +- python.d/dns_query_time.chart.py | 15 +- python.d/dnsdist.chart.py | 101 + python.d/dovecot.chart.py | 19 +- python.d/elasticsearch.chart.py | 3 +- python.d/example.chart.py | 51 +- python.d/exim.chart.py | 2 +- python.d/fail2ban.chart.py | 13 +- python.d/freeradius.chart.py | 101 +- python.d/go_expvar.chart.py | 31 +- python.d/haproxy.chart.py | 4 +- python.d/hddtemp.chart.py | 66 +- python.d/ipfs.chart.py | 85 +- python.d/isc_dhcpd.chart.py | 22 +- python.d/mdstat.chart.py | 7 +- python.d/memcached.chart.py | 11 +- python.d/mongodb.chart.py | 4 +- python.d/mysql.chart.py | 84 +- python.d/nginx.chart.py | 14 +- python.d/nsd.chart.py | 4 +- python.d/ovpn_status_log.chart.py | 41 +- python.d/phpfpm.chart.py | 9 +- python.d/postfix.chart.py | 2 +- python.d/postgres.chart.py | 33 +- python.d/powerdns.chart.py | 58 + python.d/python_modules/base.py | 1126 +------- .../bases/FrameworkServices/ExecutableService.py | 85 + .../bases/FrameworkServices/LogService.py | 78 + .../bases/FrameworkServices/MySQLService.py | 158 + .../bases/FrameworkServices/SimpleService.py | 252 ++ .../bases/FrameworkServices/SocketService.py | 250 ++ .../bases/FrameworkServices/UrlService.py | 115 + .../bases/FrameworkServices/__init__.py | 0 python.d/python_modules/bases/__init__.py | 0 python.d/python_modules/bases/charts.py | 376 +++ python.d/python_modules/bases/collection.py | 144 + python.d/python_modules/bases/loaders.py | 66 + python.d/python_modules/bases/loggers.py | 205 ++ python.d/python_modules/lm_sensors.py | 257 -- python.d/python_modules/msg.py | 101 - python.d/python_modules/third_party/__init__.py | 0 python.d/python_modules/third_party/lm_sensors.py | 257 ++ python.d/python_modules/third_party/ordereddict.py | 128 + python.d/rabbitmq.chart.py | 2 +- python.d/redis.chart.py | 33 +- python.d/retroshare.chart.py | 10 +- python.d/samba.chart.py | 8 +- python.d/sensors.chart.py | 68 +- python.d/smartd_log.chart.py | 508 ++-- python.d/squid.chart.py | 11 +- python.d/tomcat.chart.py | 15 +- python.d/varnish.chart.py | 406 +-- python.d/web_log.chart.py | 344 +-- src/Makefile.am | 10 +- src/Makefile.in | 15 +- src/adaptive_resortable_list.c | 11 +- src/adaptive_resortable_list.h | 3 + src/appconfig.c | 6 +- src/apps_plugin.c | 8 + src/avl.c | 5 +- src/backends.c | 10 +- src/cgroup-network.c | 184 +- src/clocks.c | 6 +- src/common.c | 7 +- src/common.h | 17 +- src/eval.c | 30 +- src/eval.h | 2 +- src/freebsd_devstat.c | 177 +- src/freebsd_getifaddrs.c | 14 + src/freebsd_getmntinfo.c | 4 + src/freebsd_ipfw.c | 10 + src/freebsd_kstat_zfs.c | 98 +- src/freebsd_sysctl.c | 1412 +++++---- src/freeipmi_plugin.c | 56 + src/global_statistics.c | 381 ++- src/health.c | 13 +- src/health.h | 88 +- src/health_config.c | 14 +- src/health_log.c | 4 +- src/ipc.c | 81 +- src/macos_fw.c | 260 +- src/macos_mach_smi.c | 83 +- src/macos_sysctl.c | 519 +++- src/main.c | 225 +- src/main.h | 6 - src/plugin_checks.c | 52 +- src/plugin_checks.h | 2 + src/plugin_freebsd.c | 19 +- src/plugin_freebsd.h | 2 + src/plugin_idlejitter.c | 2 + src/plugin_nfacct.c | 14 + src/plugin_proc.c | 19 +- src/plugin_proc.h | 2 + src/plugin_proc_diskspace.c | 70 +- src/plugin_tc.c | 170 +- src/plugins_d.c | 292 +- src/plugins_d.h | 9 + src/proc_diskstats.c | 312 +- src/proc_interrupts.c | 4 + src/proc_loadavg.c | 4 + src/proc_meminfo.c | 14 + src/proc_net_dev.c | 93 +- src/proc_net_ip_vs_stats.c | 63 +- src/proc_net_netstat.c | 76 +- src/proc_net_rpc_nfs.c | 403 +-- src/proc_net_rpc_nfsd.c | 947 +++--- src/proc_net_snmp.c | 1217 ++++---- src/proc_net_snmp6.c | 917 ++++-- src/proc_net_sockstat.c | 514 ++++ src/proc_net_sockstat6.c | 269 ++ src/proc_net_softnet_stat.c | 32 +- src/proc_net_stat_conntrack.c | 215 +- src/proc_net_stat_synproxy.c | 78 +- src/proc_softirqs.c | 4 + src/proc_spl_kstat_zfs.c | 16 +- src/proc_stat.c | 16 + src/proc_sys_kernel_random_entropy_avail.c | 2 + src/proc_uptime.c | 2 + src/proc_vmstat.c | 14 +- src/registry.c | 58 +- src/rrd.h | 97 +- src/rrd2json.c | 82 +- src/rrd2json.h | 1 + src/rrdcalc.c | 56 +- src/rrdcalctemplate.c | 40 +- src/rrddim.c | 8 +- src/rrddimvar.c | 42 +- src/rrdfamily.c | 19 +- src/rrdhost.c | 105 +- src/rrdpush.c | 860 +++--- src/rrdpush.h | 2 + src/rrdset.c | 75 +- src/rrdsetvar.c | 113 +- src/rrdvar.c | 93 +- src/signals.h | 2 +- src/simple_pattern.c | 145 +- src/simple_pattern.h | 5 +- src/socket.c | 51 +- src/socket.h | 7 +- src/statsd.c | 562 ++-- src/storage_number.c | 2 + src/storage_number.h | 3 + src/sys_devices_system_edac_mc.c | 4 + src/sys_devices_system_node.c | 2 + src/sys_fs_cgroup.c | 87 +- src/sys_kernel_mm_ksm.c | 6 + src/unit_test.c | 66 +- src/web_api_v1.c | 359 ++- src/web_api_v1.h | 2 + src/web_buffer_svg.c | 72 +- src/web_buffer_svg.h | 2 +- src/web_client.c | 155 +- src/web_client.h | 30 +- src/web_server.c | 152 +- src/zfs_common.c | 56 +- src/zfs_common.h | 10 +- system/netdata-openrc.in | 16 +- system/netdata.service.in | 3 + web/Makefile.am | 14 +- web/Makefile.in | 14 +- web/css/bootstrap-slider-10.0.0.min.css | 41 + web/dashboard.css | 13 +- web/dashboard.js | 2569 +++++++++++++---- web/dashboard.slate.css | 13 +- web/dashboard_info.js | 229 +- web/index.html | 3031 ++++++++++++++++---- web/lib/bootstrap-slider-10.0.0.min.js | 5 + web/lib/fontawesome-all-5.0.1.min.js | 5 + web/lib/lz-string-1.4.4.min.js | 1 + web/lib/pako-1.0.6.min.js | 1 + web/netdata-swagger.json | 1 + web/netdata-swagger.yaml | 2 +- web/version.txt | 2 +- 294 files changed, 20939 insertions(+), 9127 deletions(-) create mode 100644 Dockerfile.armv7hf create mode 100644 conf.d/health.d/beanstalkd.conf create mode 100644 conf.d/health.d/couchdb.conf create mode 100644 conf.d/health.d/tcp_conn.conf create mode 100644 conf.d/health.d/tcp_listen.conf create mode 100644 conf.d/health.d/tcp_mem.conf create mode 100644 conf.d/health.d/tcp_orphans.conf delete mode 100644 conf.d/python.d/apache_cache.conf create mode 100644 conf.d/python.d/beanstalk.conf create mode 100644 conf.d/python.d/couchdb.conf create mode 100644 conf.d/python.d/dnsdist.conf create mode 100644 conf.d/python.d/powerdns.conf delete mode 100644 contrib/debian/changelog delete mode 100644 contrib/debian/compat delete mode 100644 contrib/debian/control delete mode 100644 contrib/debian/control.wheezy delete mode 100644 contrib/debian/copyright delete mode 100644 contrib/debian/netdata.conf delete mode 100644 contrib/debian/netdata.default delete mode 100644 contrib/debian/netdata.docs delete mode 100755 contrib/debian/netdata.init delete mode 100644 contrib/debian/netdata.install delete mode 100644 contrib/debian/netdata.lintian-overrides delete mode 100644 contrib/debian/netdata.postinst.in delete mode 100644 contrib/debian/netdata.postrm delete mode 100644 contrib/debian/netdata.service delete mode 100755 contrib/debian/rules delete mode 100644 contrib/debian/source/format create mode 100755 plugins.d/cgroup-network-helper.sh delete mode 100644 python.d/apache_cache.chart.py create mode 100644 python.d/beanstalk.chart.py create mode 100644 python.d/couchdb.chart.py create mode 100644 python.d/dnsdist.chart.py create mode 100644 python.d/powerdns.chart.py create mode 100644 python.d/python_modules/bases/FrameworkServices/ExecutableService.py create mode 100644 python.d/python_modules/bases/FrameworkServices/LogService.py create mode 100644 python.d/python_modules/bases/FrameworkServices/MySQLService.py create mode 100644 python.d/python_modules/bases/FrameworkServices/SimpleService.py create mode 100644 python.d/python_modules/bases/FrameworkServices/SocketService.py create mode 100644 python.d/python_modules/bases/FrameworkServices/UrlService.py create mode 100644 python.d/python_modules/bases/FrameworkServices/__init__.py create mode 100644 python.d/python_modules/bases/__init__.py create mode 100644 python.d/python_modules/bases/charts.py create mode 100644 python.d/python_modules/bases/collection.py create mode 100644 python.d/python_modules/bases/loaders.py create mode 100644 python.d/python_modules/bases/loggers.py delete mode 100644 python.d/python_modules/lm_sensors.py delete mode 100644 python.d/python_modules/msg.py create mode 100644 python.d/python_modules/third_party/__init__.py create mode 100644 python.d/python_modules/third_party/lm_sensors.py create mode 100644 python.d/python_modules/third_party/ordereddict.py create mode 100644 src/proc_net_sockstat.c create mode 100644 src/proc_net_sockstat6.c create mode 100644 web/css/bootstrap-slider-10.0.0.min.css create mode 100644 web/lib/bootstrap-slider-10.0.0.min.js create mode 100644 web/lib/fontawesome-all-5.0.1.min.js create mode 100644 web/lib/lz-string-1.4.4.min.js create mode 100644 web/lib/pako-1.0.6.min.js diff --git a/.codeclimate.yml b/.codeclimate.yml index c3d4daa45..f53287ad7 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,15 +1,53 @@ ---- -engines: +version: "2" +checks: + argument-count: + enabled: false + config: + threshold: 10 + complex-logic: + enabled: false + config: + threshold: 10 + file-lines: + enabled: false + config: + threshold: 5000 + method-complexity: + enabled: false + config: + threshold: 20 + method-count: + enabled: false + config: + threshold: 50 + method-lines: + enabled: false + config: + threshold: 250 + nested-control-flow: + enabled: false + config: + threshold: 4 + return-statements: + enabled: false + config: + threshold: 4 + similar-code: + enabled: false + identical-code: + enabled: false +plugins: csslint: enabled: true duplication: - enabled: true + enabled: false config: languages: - - ruby - - javascript - - python - - php + - javascript: + mass_threshold: 100 + - python: + python_version: 3 + mass_threshold: 100 checks: Similar code: enabled: false @@ -41,31 +79,22 @@ engines: checks: Complexity: enabled: false -ratings: - paths: - - "**.css" - - "**.inc" - - "**.js" - - "**.jsx" - - "**.module" - - "**.php" - - "**.py" - - "**.rb" -exclude_paths: -- .gitignore -- conf.d/ -- hooks/ -- tests/ -- m4/ -- web/css/ -- web/lib/ -- web/fonts/ -- web/old/ -- python.d/python_modules/pyyaml2/ -- python.d/python_modules/pyyaml3/ -- python.d/python_modules/urllib3/ -- node.d/node_modules/lib/ -- node.d/node_modules/asn1-ber.js -- node.d/node_modules/extend.js -- node.d/node_modules/pixl-xml.js -- node.d/node_modules/net-snmp.js +exclude_patterns: + - ".gitignore" + - "conf.d/" + - "hooks/" + - "tests/" + - "m4/" + - "web/css/" + - "web/lib/" + - "web/fonts/" + - "web/old/" + - "python.d/python_modules/pyyaml2/" + - "python.d/python_modules/pyyaml3/" + - "python.d/python_modules/urllib3/" + - "node.d/node_modules/lib/" + - "node.d/node_modules/asn1-ber.js" + - "node.d/node_modules/extend.js" + - "node.d/node_modules/pixl-xml.js" + - "node.d/node_modules/net-snmp.js" + diff --git a/ChangeLog b/ChangeLog index 5d74b8aa3..b5b18c910 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +netdata (1.9.0) - 2017-12-17 + + Please check full changelog at github. + https://github.com/firehol/netdata/releases + + netdata (1.8.0) - 2017-09-17 This is mainly a bugfix release. diff --git a/Dockerfile b/Dockerfile index 5a123fc8d..05a83a748 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,9 @@ FROM debian:jessie ADD . /netdata.git +RUN echo "deb http://ftp.nl.debian.org/debian/ jessie main" > /etc/apt/sources.list +RUN echo "deb http://security.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list + RUN cd ./netdata.git && chmod +x ./docker-build.sh && sync && sleep 1 && ./docker-build.sh WORKDIR / diff --git a/Dockerfile.armv7hf b/Dockerfile.armv7hf new file mode 100644 index 000000000..2591748a9 --- /dev/null +++ b/Dockerfile.armv7hf @@ -0,0 +1,19 @@ +# author : titpetric +# original: https://github.com/titpetric/netdata + +FROM resin/armv7hf-debian:jessie + +RUN [ "cross-build-start"] + +ADD . /netdata.git + +RUN cd ./netdata.git && chmod +x ./docker-build.sh && sync && sleep 1 && ./docker-build.sh + +WORKDIR / + +ENV NETDATA_PORT 19999 +EXPOSE $NETDATA_PORT + +CMD /usr/sbin/netdata -D -s /host -p ${NETDATA_PORT} + +RUN [ "cross-build-end"] diff --git a/LICENSE-REDISTRIBUTED.md b/LICENSE-REDISTRIBUTED.md index 513d53f5d..b67c5acc1 100644 --- a/LICENSE-REDISTRIBUTED.md +++ b/LICENSE-REDISTRIBUTED.md @@ -54,6 +54,10 @@ connectivity is not available. Copyright 2015, Twitter [MIT License](http://getbootstrap.com/getting-started/#license-faqs) +- [Bootstrap-slider](http://seiyria.com/bootstrap-slider/) + + Copyright 2017 Kyle Kemp, Rohit Kalkur, and contributors + [MIT License](https://github.com/seiyria/bootstrap-slider/blob/master/LICENSE.md) - [Bootstrap Toggle](http://www.bootstraptoggle.com/) @@ -154,3 +158,12 @@ connectivity is not available. Copyright 2008-2016 Andrey Petrov and [contributors](https://github.com/shazow/urllib3/blob/master/CONTRIBUTORS.txt) [MIT License](https://github.com/shazow/urllib3/blob/master/LICENSE.txt) +- [lz-string](http://pieroxy.net/blog/pages/lz-string/index.html) + + Copyright 2013 Pieroxy + [WTFPL License](http://pieroxy.net/blog/pages/lz-string/index.html#inline_menu_10) + +- [pako](http://nodeca.github.io/pako/) + + Copyright 2014-2017 Vitaly Puzrin and Andrei Tuputcyn + [MIT License](https://github.com/nodeca/pako/blob/master/LICENSE) diff --git a/Makefile.am b/Makefile.am index 2ea9e40f9..f87a6bb30 100644 --- a/Makefile.am +++ b/Makefile.am @@ -63,6 +63,7 @@ SUBDIRS = \ dist_noinst_DATA= \ configs.signatures \ Dockerfile \ + Dockerfile.armv7hf \ netdata.spec \ package.json \ $(NULL) diff --git a/Makefile.in b/Makefile.in index 5abcfe6f0..a93c84ba6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -412,6 +412,7 @@ SUBDIRS = \ dist_noinst_DATA = \ configs.signatures \ Dockerfile \ + Dockerfile.armv7hf \ netdata.spec \ package.json \ $(NULL) diff --git a/README.md b/README.md index f33010e59..c6498165f 100644 --- a/README.md +++ b/README.md @@ -29,26 +29,12 @@ netdata runs on **Linux**, **FreeBSD**, and **MacOS**. ## News -

-Netdata is featured at GitHub's State Of The Octoverse 2016
- -

+`Sep 17th, 2017` - **[netdata v1.8.0 released!](https://github.com/firehol/netdata/releases)** -`Jul 16th, 2017` - **[netdata v1.7.0 released!](https://github.com/firehol/netdata/releases)** - - - netdata is now a fully featured **statsd** server - [read more here](https://github.com/firehol/netdata/wiki/statsd) - - netdata now monitors **ZFS** of Linux and FreeBSD - - netdata now monitors **ElasticSearch** - - netdata now monitors **RabbitMQ** - - netdata now monitors **Go applications** (via `expvar`) - [read more here](https://github.com/firehol/netdata/wiki/Monitoring-Go-Applications) - - netdata now monitors **squid logs** - - netdata now monitors **samba** - - several **backends** improvements and enhancements, including **metrics filtering** - - **prometheus** backend support has been rewritten - - alarm notifications now support **custom hooks** to execute shell commands in parallel with all other notification methods - - dozens of new and improved plugins - - dozens of new and improved alarms - - dozens more improvements and performance optimizations + - mainly a bug fix release - all users are advised to update this release + - better support for containers (`veth` interfaces are now visualized at their containers section, container sections now provide a summary view for each container) + - netdata can now listen on UNIX domain sockets + - dozens of more improvements, compatibility fixes and enhancements --- @@ -69,11 +55,11 @@ Netdata is featured at 0 + crit: $this > 10 + delay: up 0 down 5m multiplier 1.2 max 1h + info: the number of buried jobs aggregated across all tubes + to: sysadmin + +# get the number of buried jobs per queue + +#template: tube_buried_jobs +# on: beanstalk.jobs +# calc: $buried +# units: jobs +# every: 10s +# warn: $this > 0 +# crit: $this > 10 +# delay: up 0 down 5m multiplier 1.2 max 1h +# info: the number of jobs buried per tube +# to: sysadmin + +# get the current number of tubes + +#template: number_of_tubes +# on: beanstalk.current_tubes +# calc: $tubes +# every: 10s +# warn: $this < 5 +# delay: up 0 down 5m multiplier 1.2 max 1h +# info: the current number of tubes on the server +# to: sysadmin diff --git a/conf.d/health.d/bind_rndc.conf b/conf.d/health.d/bind_rndc.conf index 028bc9d08..4145e77cd 100644 --- a/conf.d/health.d/bind_rndc.conf +++ b/conf.d/health.d/bind_rndc.conf @@ -1,4 +1,4 @@ - alarm: bind_rndc_stats_file_size + template: bind_rndc_stats_file_size on: bind_rndc.stats_size units: megabytes every: 60 diff --git a/conf.d/health.d/couchdb.conf b/conf.d/health.d/couchdb.conf new file mode 100644 index 000000000..4a2895280 --- /dev/null +++ b/conf.d/health.d/couchdb.conf @@ -0,0 +1,13 @@ + +# make sure couchdb is running + +template: couchdb_last_collected_secs + on: couchdb.request_methods + calc: $now - $last_collected_t + units: seconds ago + every: 10s + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + delay: down 5m multiplier 1.5 max 1h + info: number of seconds since the last successful data collection + to: dba diff --git a/conf.d/health.d/redis.conf b/conf.d/health.d/redis.conf index 5f6d397ea..c08a884a6 100644 --- a/conf.d/health.d/redis.conf +++ b/conf.d/health.d/redis.conf @@ -12,3 +12,23 @@ template: redis_last_collected_secs info: number of seconds since the last successful data collection to: dba +template: redis_bgsave_broken +families: * + on: redis.bgsave_health + every: 10s + crit: $rdb_last_bgsave_status != 0 + units: ok/failed + info: states if redis bgsave is working + delay: down 5m multiplier 1.5 max 1h + to: dba + +template: redis_bgsave_slow +families: * + on: redis.bgsave_now + every: 10s + warn: $rdb_bgsave_in_progress > 600 + crit: $rdb_bgsave_in_progress > 1200 + units: seconds + info: the time redis needs to save its database + delay: down 5m multiplier 1.5 max 1h + to: dba diff --git a/conf.d/health.d/tcp_conn.conf b/conf.d/health.d/tcp_conn.conf new file mode 100644 index 000000000..7aa9a9800 --- /dev/null +++ b/conf.d/health.d/tcp_conn.conf @@ -0,0 +1,19 @@ + +# +# ${tcp_max_connections} may be nan or -1 if the system +# supports dynamic threshold for TCP connections. +# In this case, the alarm will always be zero. +# + + alarm: tcp_connections + on: ipv4.tcpsock + os: linux + hosts: * + calc: (${tcp_max_connections} > 0) ? ( ${connections} * 100 / ${tcp_max_connections} ) : 0 + units: % + every: 10s + warn: $this > (($status >= $WARNING ) ? ( 60 ) : ( 80 )) + crit: $this > (($status >= $CRITICAL) ? ( 80 ) : ( 90 )) + delay: up 0 down 5m multiplier 1.5 max 1h + info: the percentage of IPv4 TCP connections over the max allowed + to: sysadmin diff --git a/conf.d/health.d/tcp_listen.conf b/conf.d/health.d/tcp_listen.conf new file mode 100644 index 000000000..957964ae4 --- /dev/null +++ b/conf.d/health.d/tcp_listen.conf @@ -0,0 +1,27 @@ +# ----------------------------------------------------------------------------- +# tcp listen sockets issues + + alarm: 1m_ipv4_tcp_listen_overflows + on: ipv4.tcplistenissues + os: linux freebsd + hosts: * + lookup: sum -60s unaligned absolute of ListenOverflows + units: overflows + every: 10s + crit: $this > 0 + delay: up 0 down 5m multiplier 1.5 max 1h + info: the number of TCP listen socket overflows during the last minute + to: sysadmin + + alarm: 1m_ipv4_tcp_listen_drops + on: ipv4.tcplistenissues + os: linux + hosts: * + lookup: sum -60s unaligned absolute of ListenDrops + units: drops + every: 10s + crit: $this > 0 + delay: up 0 down 5m multiplier 1.5 max 1h + info: the number of TCP listen socket drops during the last minute + to: sysadmin + diff --git a/conf.d/health.d/tcp_mem.conf b/conf.d/health.d/tcp_mem.conf new file mode 100644 index 000000000..6927d5765 --- /dev/null +++ b/conf.d/health.d/tcp_mem.conf @@ -0,0 +1,20 @@ +# +# check +# http://blog.tsunanet.net/2011/03/out-of-socket-memory.html +# +# We give a warning when TCP is under memory pressure +# and a critical when TCP is 90% of its upper memory limit +# + + alarm: tcp_memory + on: ipv4.sockstat_tcp_mem + os: linux + hosts: * + calc: ${mem} * 100 / ${tcp_mem_high} + units: % + every: 10s + warn: ${mem} > (($status >= $WARNING ) ? ( ${tcp_mem_pressure} * 0.8 ) : ( ${tcp_mem_pressure} )) + crit: ${mem} > (($status >= $CRITICAL ) ? ( ${tcp_mem_pressure} ) : ( ${tcp_mem_high} * 0.9 )) + delay: up 0 down 5m multiplier 1.5 max 1h + info: the amount of TCP memory as a percentage of its max memory limit + to: sysadmin diff --git a/conf.d/health.d/tcp_orphans.conf b/conf.d/health.d/tcp_orphans.conf new file mode 100644 index 000000000..280d6590f --- /dev/null +++ b/conf.d/health.d/tcp_orphans.conf @@ -0,0 +1,21 @@ + +# +# check +# http://blog.tsunanet.net/2011/03/out-of-socket-memory.html +# +# The kernel may penalize orphans by 2x or even 4x +# so we alarm warning at 25% and critical at 50% +# + + alarm: tcp_orphans + on: ipv4.sockstat_tcp_sockets + os: linux + hosts: * + calc: ${orphan} * 100 / ${tcp_max_orphans} + units: % + every: 10s + warn: $this > (($status >= $WARNING ) ? ( 20 ) : ( 25 )) + crit: $this > (($status >= $CRITICAL) ? ( 25 ) : ( 50 )) + delay: up 0 down 5m multiplier 1.5 max 1h + info: the percentage of orphan IPv4 TCP sockets over the max allowed (this may lead to too-many-orphans errors) + to: sysadmin diff --git a/conf.d/health.d/tcp_resets.conf b/conf.d/health.d/tcp_resets.conf index fec124ac7..e6cfd39ab 100644 --- a/conf.d/health.d/tcp_resets.conf +++ b/conf.d/health.d/tcp_resets.conf @@ -37,7 +37,7 @@ every: 10s warn: $this > ((($1m_ipv4_tcp_resets_sent < 5)?(5):($1m_ipv4_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (20))) delay: up 0 down 60m multiplier 1.2 max 2h -options: no-clear-notification + options: no-clear-notification info: average TCP RESETS this host is sending, over the last 10 seconds (this can be an indication that a port scan is made, or that a service running on this host has crashed; clear notification for this alarm will not be sent) to: sysadmin @@ -62,6 +62,6 @@ options: no-clear-notification every: 10s warn: $this > ((($1m_ipv4_tcp_resets_received < 5)?(5):($1m_ipv4_tcp_resets_received)) * (($status >= $WARNING) ? (1) : (10))) delay: up 0 down 60m multiplier 1.2 max 2h -options: no-clear-notification + options: no-clear-notification info: average TCP RESETS this host is receiving, over the last 10 seconds (this can be an indication that a service this host needs, has crashed; clear notification for this alarm will not be sent) to: sysadmin diff --git a/conf.d/health_alarm_notify.conf b/conf.d/health_alarm_notify.conf index 641272ced..eb01e2bb9 100644 --- a/conf.d/health_alarm_notify.conf +++ b/conf.d/health_alarm_notify.conf @@ -7,6 +7,7 @@ # - e-mails (using the sendmail command), # - push notifications to your mobile phone (pushover.net), # - messages to your slack team (slack.com), +# - messages to your flock team (flock.com), # - messages to your discord guild (discordapp.com), # - messages to your telegram chat / group chat (telegram.org) # - sms messages to your cell phone or any sms enabled device (twilio.com) @@ -23,7 +24,7 @@ # proxy configuration # # If you need to send curl based notifications (pushover, pushbullet, slack, -# discord, telegram) via a proxy, set these to your proxy address: +# flock, discord, telegram) via a proxy, set these to your proxy address: #export http_proxy="http://10.0.0.1:3128/" #export https_proxy="http://10.0.0.1:3128/" @@ -63,6 +64,7 @@ curl="" # - pushover user tokens # - telegram chat ids # - slack channels +# - flock rooms # - discord channels # - hipchat rooms # - sms phone numbers @@ -77,9 +79,11 @@ curl="" # pushover : "2987343...9437837 8756278...2362736|critical" # telegram : "111827421 112746832|critical" # slack : "alarms disasters|critical" +# flock : "alarms disasters|critical" # discord : "alarms disasters|critical" # twilio : "+15555555555 +17777777777|critical" # messagebird: "+15555555555 +17777777777|critical" +# kavenegar : "09155555555 09177777777|critical" # pd : " |critical" # # If a recipient is set to empty string, the default recipient of the given @@ -160,6 +164,9 @@ SEND_PUSHBULLET="YES" PUSHBULLET_ACCESS_TOKEN="" DEFAULT_RECIPIENT_PUSHBULLET="" +# Device iden of the sending device. Optional. +PUSHBULLET_SOURCE_DEVICE="" + #------------------------------------------------------------------------------ # Twilio (twilio.com) SMS options @@ -204,6 +211,28 @@ MESSAGEBIRD_NUMBER="" DEFAULT_RECIPIENT_MESSAGEBIRD="" +#------------------------------------------------------------------------------ +# Kavenegar (Kavenegar.com) SMS options + +# multiple recipients can be given like this: +# "09155555555 09177777777" + +# enable/disable sending kavenegar SMS +SEND_KAVENEGAR="YES" + +# to get an access key, after selecting and purchasing your desired service +# at http://kavenegar.com/pricing.html +# login to your account, go to your dashboard and my account are +# https://panel.kavenegar.com/Client/setting/account from API Key +# copy your api key. You can generate new API Key too. +# You can find and select kevenegar sender number from this place. + +# Without an API key, netdata cannot send KAVENEGAR text messages. +KAVENEGAR_API_KEY="" +KAVENEGAR_SENDER="" +DEFAULT_RECIPIENT_KAVENEGAR="" + + #------------------------------------------------------------------------------ # telegram (telegram.org) global notification options @@ -246,6 +275,21 @@ SLACK_WEBHOOK_URL="" DEFAULT_RECIPIENT_SLACK="" +#------------------------------------------------------------------------------ +# flock (flock.com) global notification options + +# enable/disable sending flock notifications +SEND_FLOCK="YES" + +# Login to flock.com and create an incoming webhook. You need only one for all +# your netdata servers (or you can have one for each of your netdata). +# Without it, netdata cannot send flock notifications. +FLOCK_WEBHOOK_URL="" + +# if a role recipient is not configured, no notification will be sent +DEFAULT_RECIPIENT_FLOCK="" + + #------------------------------------------------------------------------------ # discord (discordapp.com) global notification options @@ -398,6 +442,8 @@ role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}" +role_recipients_flock[sysadmin]="${DEFAULT_RECIPIENT_FLOCK}" + role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" @@ -406,6 +452,8 @@ role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" +role_recipients_kavenegar[sysadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}" + role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -423,6 +471,8 @@ role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}" +role_recipients_flock[domainadmin]="${DEFAULT_RECIPIENT_FLOCK}" + role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" @@ -431,6 +481,8 @@ role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" +role_recipients_kavenegar[domainadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}" + role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -449,6 +501,8 @@ role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}" +role_recipients_flock[dba]="${DEFAULT_RECIPIENT_FLOCK}" + role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}" @@ -457,6 +511,8 @@ role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" +role_recipients_kavenegar[dba]="${DEFAULT_RECIPIENT_KAVENEGAR}" + role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -475,6 +531,8 @@ role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}" +role_recipients_flock[webmaster]="${DEFAULT_RECIPIENT_FLOCK}" + role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}" @@ -483,6 +541,8 @@ role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" +role_recipients_kavenegar[webmaster]="${DEFAULT_RECIPIENT_KAVENEGAR}" + role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -501,6 +561,8 @@ role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}" +role_recipients_flock[proxyadmin]="${DEFAULT_RECIPIENT_FLOCK}" + role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" @@ -509,6 +571,8 @@ role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" +role_recipients_kavenegar[proxyadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}" + role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}" diff --git a/conf.d/python.d.conf b/conf.d/python.d.conf index 741d49914..2c3d400ca 100644 --- a/conf.d/python.d.conf +++ b/conf.d/python.d.conf @@ -9,13 +9,6 @@ # Enable / disable the whole python.d.plugin (all its modules) enabled: yes -# Prevent log flood -# Define how many log messages can be written to log file in one log_interval -logs_per_interval: 200 - -# Define how long is one logging interval (in seconds) -log_interval: 3600 - # ---------------------------------------------------------------------- # Enable / Disable python.d.plugin modules #default_run: yes @@ -29,11 +22,14 @@ log_interval: 3600 # apache_cache has been replaced by web_log apache_cache: no # apache: yes +# beanstalk: yes # bind_rndc: yes -# chrony: yes +chrony: no +# couchdb: yes # cpufreq: yes # cpuidle: yes # dns_query_time: yes +# dnsdist: yes # dovecot: yes # elasticsearch: yes @@ -53,6 +49,7 @@ go_expvar: no # isc_dhcpd: yes # mdstat: yes # memcached: yes +# mongodb: yes # mysql: yes # nginx: yes # nsd: yes @@ -64,6 +61,7 @@ nginx_log: no # phpfpm: yes # postfix: yes # postgres: yes +# powerdns: yes # rabbitmq: yes # redis: yes # retroshare: yes diff --git a/conf.d/python.d/apache.conf b/conf.d/python.d/apache.conf index 5b151ef70..3bbc3f786 100644 --- a/conf.d/python.d/apache.conf +++ b/conf.d/python.d/apache.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, apache also supports the following: # diff --git a/conf.d/python.d/apache_cache.conf b/conf.d/python.d/apache_cache.conf deleted file mode 100644 index 98eecd0e8..000000000 --- a/conf.d/python.d/apache_cache.conf +++ /dev/null @@ -1,76 +0,0 @@ -# netdata python.d.plugin configuration for apache cache -# -# This file is in YaML format. Generally the format is: -# -# name: value -# -# There are 2 sections: -# - global variables -# - one or more JOBS -# -# JOBS allow you to collect values from multiple sources. -# Each source will have its own set of charts. -# -# JOB parameters have to be indented (using spaces only, example below). - -# ---------------------------------------------------------------------- -# Global Variables -# These variables set the defaults for all JOBs, however each JOB -# may define its own, overriding the defaults. - -# update_every sets the default data collection frequency. -# If unset, the python.d.plugin default is used. -# update_every: 1 - -# priority controls the order of charts at the netdata dashboard. -# Lower numbers move the charts towards the top of the page. -# If unset, the default for python.d.plugin is used. -# priority: 60000 - -# retries sets the number of retries to be made in case of failures. -# If unset, the default for python.d.plugin is used. -# Attempts to restore the service are made once every update_every -# and only if the module has collected values in the past. -# retries: 5 - -# ---------------------------------------------------------------------- -# JOBS (data collection sources) -# -# The default JOBS share the same *name*. JOBS with the same name -# are mutually exclusive. Only one of them will be allowed running at -# any time. This allows autodetection to try several alternatives and -# pick the one that works. -# -# Any number of jobs is supported. -# -# All python.d.plugin JOBS (for all its modules) support a set of -# predefined parameters. These are: -# -# job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts -# -# Additionally to the above, apache_cache also supports the following: -# -# path: 'PATH' # the path to apache's cache.log -# - -# ---------------------------------------------------------------------- -# AUTO-DETECTION JOBS -# only one of them will run (they have the same name) - -apache: - name: 'local' - path: '/var/log/apache/cache.log' - -apache2: - name: 'local' - path: '/var/log/apache2/cache.log' - -httpd: - name: 'local' - path: '/var/log/httpd/cache.log' diff --git a/conf.d/python.d/beanstalk.conf b/conf.d/python.d/beanstalk.conf new file mode 100644 index 000000000..940801877 --- /dev/null +++ b/conf.d/python.d/beanstalk.conf @@ -0,0 +1,80 @@ +# netdata python.d.plugin configuration for beanstalk +# +# This file is in YaML format. Generally the format is: +# +# name: value +# +# There are 2 sections: +# - global variables +# - one or more JOBS +# +# JOBS allow you to collect values from multiple sources. +# Each source will have its own set of charts. +# +# JOB parameters have to be indented (using spaces only, example below). + +# ---------------------------------------------------------------------- +# Global Variables +# These variables set the defaults for all JOBs, however each JOB +# may define its own, overriding the defaults. + +# update_every sets the default data collection frequency. +# If unset, the python.d.plugin default is used. +# update_every: 1 + +# priority controls the order of charts at the netdata dashboard. +# Lower numbers move the charts towards the top of the page. +# If unset, the default for python.d.plugin is used. +# priority: 60000 + +# retries sets the number of retries to be made in case of failures. +# If unset, the default for python.d.plugin is used. +# Attempts to restore the service are made once every update_every +# and only if the module has collected values in the past. +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 + +# chart_cleanup sets the default chart cleanup interval in iterations. +# A chart is marked as obsolete if it has not been updated +# 'chart_cleanup' iterations in a row. +# When a plugin sends the obsolete flag, the charts are not deleted +# from netdata immediately. +# They will be hidden immediately (not offered to dashboard viewer, +# streamed upstream and archived to backends) and deleted one hour +# later (configurable from netdata.conf). +# chart_cleanup: 10 + +# ---------------------------------------------------------------------- +# JOBS (data collection sources) +# +# The default JOBS share the same *name*. JOBS with the same name +# are mutually exclusive. Only one of them will be allowed running at +# any time. This allows autodetection to try several alternatives and +# pick the one that works. +# +# Any number of jobs is supported. +# +# All python.d.plugin JOBS (for all its modules) support a set of +# predefined parameters. These are: +# +# job_name: +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds +# chart_cleanup: 10 # the JOB's chart cleanup interval in iterations +# +# Additionally to the above, apache also supports the following: +# +# host: 'host' # Server ip address or hostname. Default: 127.0.0.1 +# port: port # Beanstalkd port. Default: +# +# ---------------------------------------------------------------------- diff --git a/conf.d/python.d/bind_rndc.conf b/conf.d/python.d/bind_rndc.conf index e4f7ac825..71958ff98 100644 --- a/conf.d/python.d/bind_rndc.conf +++ b/conf.d/python.d/bind_rndc.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, bind_rndc also supports the following: # @@ -75,7 +82,7 @@ # chown bind bind/ # # 4. RELOAD (NOT restart) BIND -# systemctl reload bind9.serice +# systemctl reload bind9.service # # 5. Run as a root 'rndc stats' to dump (BIND will create named.stats in new directory) # @@ -101,9 +108,5 @@ # To test your logrotate conf file run as root: # # logrotate /etc/logrotate.d/bind-rndc -d (debug dry-run mode) -# ------------------------------------------------------------------------------------------------------------------ -# AUTO-DETECTION JOBS -# only one of them will run (they have the same name) # -#local: -# named_stats_path: '/var/log/bind/named.stats' +# ---------------------------------------------------------------------- diff --git a/conf.d/python.d/chrony.conf b/conf.d/python.d/chrony.conf index 46229687b..9ac906b5f 100644 --- a/conf.d/python.d/chrony.conf +++ b/conf.d/python.d/chrony.conf @@ -31,7 +31,13 @@ update_every: 5 # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ update_every: 5 # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, chrony also supports the following: # diff --git a/conf.d/python.d/couchdb.conf b/conf.d/python.d/couchdb.conf new file mode 100644 index 000000000..5f6e75cff --- /dev/null +++ b/conf.d/python.d/couchdb.conf @@ -0,0 +1,91 @@ +# netdata python.d.plugin configuration for couchdb +# +# This file is in YaML format. Generally the format is: +# +# name: value +# +# There are 2 sections: +# - global variables +# - one or more JOBS +# +# JOBS allow you to collect values from multiple sources. +# Each source will have its own set of charts. +# +# JOB parameters have to be indented (using spaces only, example below). + +# ---------------------------------------------------------------------- +# Global Variables +# These variables set the defaults for all JOBs, however each JOB +# may define its own, overriding the defaults. + +# update_every sets the default data collection frequency. +# If unset, the python.d.plugin default is used. +# By default, CouchDB only updates its stats every 10 seconds. +update_every: 10 + +# priority controls the order of charts at the netdata dashboard. +# Lower numbers move the charts towards the top of the page. +# If unset, the default for python.d.plugin is used. +# priority: 60000 + +# retries sets the number of retries to be made in case of failures. +# If unset, the default for python.d.plugin is used. +# Attempts to restore the service are made once every update_every +# and only if the module has collected values in the past. +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 + +# ---------------------------------------------------------------------- +# JOBS (data collection sources) +# +# The default JOBS share the same *name*. JOBS with the same name +# are mutually exclusive. Only one of them will be allowed running at +# any time. This allows autodetection to try several alternatives and +# pick the one that works. +# +# Any number of jobs is supported. +# +# All python.d.plugin JOBS (for all its modules) support a set of +# predefined parameters. These are: +# +# job_name: +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds +# +# Additionally to the above, the couchdb plugin also supports the following: +# +# host: 'ipaddress' # Server ip address or hostname. Default: 127.0.0.1 +# port: 'port' # CouchDB port. Default: 15672 +# scheme: 'scheme' # http or https. Default: http +# node: 'couchdb@127.0.0.1' # CouchDB node name. Same as -name vm.args argument. +# +# if the URL is password protected, the following are supported: +# +# user: 'username' +# pass: 'password' +# +# if db-specific stats are desired, place their names in databases: +# databases: 'npm-registry animaldb' +# +# ---------------------------------------------------------------------- +# AUTO-DETECTION JOBS +# only one of them will run (they have the same name) +# +localhost: + name: 'local' + host: '127.0.0.1' + port: '5984' + node: 'couchdb@127.0.0.1' + scheme: 'http' +# user: 'admin' +# pass: 'password' diff --git a/conf.d/python.d/cpufreq.conf b/conf.d/python.d/cpufreq.conf index 10c96917f..0890245d9 100644 --- a/conf.d/python.d/cpufreq.conf +++ b/conf.d/python.d/cpufreq.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # The directory to search for the file scaling_cur_freq sys_dir: "/sys/devices" diff --git a/conf.d/python.d/dns_query_time.conf b/conf.d/python.d/dns_query_time.conf index f4d4dbf92..d32c6db83 100644 --- a/conf.d/python.d/dns_query_time.conf +++ b/conf.d/python.d/dns_query_time.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,26 +53,19 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, dns_query_time also supports the following: # -# dns_servers: 'dns servers' # list of dns servers to query -# domains: 'domains' # list of domains -# aggregate: yes/no # Default: yes. Aggregate all servers in one chart or not -# response_timeout: 4 # Defalt: 4. Dns query response timeout (query = -100 if response time > response_time) -# -# -# ---------------------------------------------------------------------- -# AUTO-DETECTION JOBS -# only one of them will run (they have the same name) -# +# dns_servers: 'dns servers' # List of dns servers to query +# domains: 'domains' # List of domains +# aggregate: yes/no # Aggregate all servers in one chart or not +# response_timeout: 4 # Dns query response timeout (query = -100 if response time > response_time) # -#aggregate: yes -#dns_servers: '8.8.8.8 8.8.4.4' -#domains: 'python.org distrowatch.com linuxmint.com linux.com rsyslog.com liblognorm.com archlinux.org cisco.com debian.org kernel.org gns3.com opera.com github.com youtube.com amazon.co.uk kde.org netdata.firehol.org ubuntu.com redhat.com opensuse.org wireshark.org vmware.com microsoft.com elastic.co' +# ---------------------------------------------------------------------- \ No newline at end of file diff --git a/conf.d/python.d/dnsdist.conf b/conf.d/python.d/dnsdist.conf new file mode 100644 index 000000000..aec58b8e1 --- /dev/null +++ b/conf.d/python.d/dnsdist.conf @@ -0,0 +1,85 @@ +# netdata python.d.plugin configuration for dnsdist +# +# This file is in YaML format. Generally the format is: +# +# name: value +# +# There are 2 sections: +# - global variables +# - one or more JOBS +# +# JOBS allow you to collect values from multiple sources. +# Each source will have its own set of charts. +# +# JOB parameters have to be indented (using spaces only, example below). + +# ---------------------------------------------------------------------- +# Global Variables +# These variables set the defaults for all JOBs, however each JOB +# may define its own, overriding the defaults. + +# update_every sets the default data collection frequency. +# If unset, the python.d.plugin default is used. +#update_every: 1 + +# priority controls the order of charts at the netdata dashboard. +# Lower numbers move the charts towards the top of the page. +# If unset, the default for python.d.plugin is used. +# priority: 60000 + +# retries sets the number of retries to be made in case of failures. +# If unset, the default for python.d.plugin is used. +# Attempts to restore the service are made once every update_every +# and only if the module has collected values in the past. +#retries: 600000 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +#autodetection_retry: 1 + +# ---------------------------------------------------------------------- +# JOBS (data collection sources) +# +# The default JOBS share the same *name*. JOBS with the same name +# are mutually exclusive. Only one of them will be allowed running at +# any time. This allows autodetection to try several alternatives and +# pick the one that works. +# +# Any number of jobs is supported. +# +# All python.d.plugin JOBS (for all its modules) support a set of +# predefined parameters. These are: +# +# job_name: +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds +# +# +# Additionally to the above, dnsdist also supports the following: +# +# url: 'URL' # the URL to fetch dnsdist performance statistics +# user: 'username' # username for basic auth +# pass: 'password' # password for basic auth +# header: +# X-API-Key: 'Key' # API key +# +# ---------------------------------------------------------------------- +# AUTO-DETECTION JOBS +# only one of them will run (they have the same name) + +# localhost: +# name : 'local' +# url : 'http://127.0.0.1:5053/jsonstat?command=stats' +# user : 'username' +# pass : 'password' +# header: +# X-API-Key: 'dnsdist-api-key' + + diff --git a/conf.d/python.d/dovecot.conf b/conf.d/python.d/dovecot.conf index 917c5272e..56c394991 100644 --- a/conf.d/python.d/dovecot.conf +++ b/conf.d/python.d/dovecot.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, dovecot also supports the following: # diff --git a/conf.d/python.d/elasticsearch.conf b/conf.d/python.d/elasticsearch.conf index 7c35df229..213843bf9 100644 --- a/conf.d/python.d/elasticsearch.conf +++ b/conf.d/python.d/elasticsearch.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, elasticsearch plugin also supports the following: # diff --git a/conf.d/python.d/example.conf b/conf.d/python.d/example.conf index 31f9a49a0..e7fed9b50 100644 --- a/conf.d/python.d/example.conf +++ b/conf.d/python.d/example.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, example also supports the following: # diff --git a/conf.d/python.d/exim.conf b/conf.d/python.d/exim.conf index 07d72c5a3..2add7b2cb 100644 --- a/conf.d/python.d/exim.conf +++ b/conf.d/python.d/exim.conf @@ -32,7 +32,13 @@ update_every: 10 # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -48,12 +54,13 @@ update_every: 10 # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, exim also supports the following: # diff --git a/conf.d/python.d/fail2ban.conf b/conf.d/python.d/fail2ban.conf index 76277108b..60ca87231 100644 --- a/conf.d/python.d/fail2ban.conf +++ b/conf.d/python.d/fail2ban.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,17 +53,18 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, fail2ban also supports the following: # -# log_path: 'path to fail2ban.log' # Default: '/var/log/fail2ban.log' -# conf_path: 'path to jail.local/jail.conf' # Default: '/etc/fail2ban/jail.local' -# conf_dir: 'path to jail.d/' # Default: '/etc/fail2ban/jail.d/' -# exclude: 'jails you want to exclude from autodetection' # Default: '[]' empty list +# log_path: 'path to fail2ban.log' # Default: '/var/log/fail2ban.log' +# conf_path: 'path to jail.local/jail.conf' # Default: '/etc/fail2ban/jail.local' +# conf_dir: 'path to jail.d/' # Default: '/etc/fail2ban/jail.d/' +# exclude: 'jails you want to exclude from autodetection' # Default: none #------------------------------------------------------------------------------------------------------------------ diff --git a/conf.d/python.d/freeradius.conf b/conf.d/python.d/freeradius.conf index b2c8abf6b..3336d4c49 100644 --- a/conf.d/python.d/freeradius.conf +++ b/conf.d/python.d/freeradius.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,21 +53,22 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, freeradius also supports the following: # -# host: 'host' # Default: 'localhost'. Server ip address or hostname. -# port: 'port' # Default: '18121'. Port on which freeradius server listen (type = status). +# host: 'host' # Default: 'localhost'. Server ip address or hostname. +# port: 'port' # Default: '18121'. Port on which freeradius server listen (type = status). # secret: 'secret' # Default: 'adminsecret'. -# acct: True/False # Defalt: False. Freeradius accounting statistics. -# proxy_auth: True/False # Default: False. Freeradius proxy authentication statistics. -# proxy_acct: True/False # Default: False. Freeradius proxy accounting statistics. +# acct: yes/no # Default: no. Freeradius accounting statistics. +# proxy_auth: yes/no # Default: no. Freeradius proxy authentication statistics. +# proxy_acct: yes/no # Default: no. Freeradius proxy accounting statistics. # # ------------------------------------------------------------------------------------------------------------------ # Freeradius server configuration: @@ -73,14 +80,3 @@ # ln -s ../sites-available/status status # and restart/reload your FREERADIUS server. # ------------------------------------------------------------------------------------------------------------------ -# -# AUTO-DETECTION JOBS -# only one of them will run (they have the same name) - -local: - host: 'localhost' - port: '18121' - secret: 'adminsecret' -#acct: False -#proxy_auth: False -#proxy_acct: False diff --git a/conf.d/python.d/go_expvar.conf b/conf.d/python.d/go_expvar.conf index 5be4890dc..c352b1674 100644 --- a/conf.d/python.d/go_expvar.conf +++ b/conf.d/python.d/go_expvar.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -42,19 +48,17 @@ # predefined parameters. These are: # # job_name: -# name: my_name # the JOB's name as it will appear at the -# # dashboard. If name: is not supplied the -# # job_name: will be used (use _ for spaces) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, this plugin also supports the following: # # url: 'http://127.0.0.1/debug/vars' # the URL of the expvar endpoint -# ss_cert: # ignore HTTPS self-signed certificate -# proxy: # use HTTP proxy # # As the plugin cannot possibly know the port your application listens on, there is no default value. Please include # the whole path of the endpoint, as the expvar handler can be installed in a non-standard location. diff --git a/conf.d/python.d/haproxy.conf b/conf.d/python.d/haproxy.conf index a9e048791..a40dd76a5 100644 --- a/conf.d/python.d/haproxy.conf +++ b/conf.d/python.d/haproxy.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, haproxy also supports the following: # diff --git a/conf.d/python.d/hddtemp.conf b/conf.d/python.d/hddtemp.conf index f74a09803..9165798a2 100644 --- a/conf.d/python.d/hddtemp.conf +++ b/conf.d/python.d/hddtemp.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,18 +53,18 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, hddtemp also supports the following: # # host: 'IP or HOSTNAME' # the host to connect to # port: PORT # the port to connect to -# fahrenheit: True/False # fahrenheit instead of celsius. Default is False # # By default this module will try to autodetect disks @@ -78,13 +84,11 @@ localhost: name: 'local' host: 'localhost' - fahrenheit: False port: 7634 localipv4: name: 'local' host: '127.0.0.1' - fahrenheit: False port: 7634 localipv6: diff --git a/conf.d/python.d/ipfs.conf b/conf.d/python.d/ipfs.conf index e039026cc..c247c1b7a 100644 --- a/conf.d/python.d/ipfs.conf +++ b/conf.d/python.d/ipfs.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, ipfs also supports the following: # diff --git a/conf.d/python.d/isc_dhcpd.conf b/conf.d/python.d/isc_dhcpd.conf index 938ca6e72..4a4c4a5e3 100644 --- a/conf.d/python.d/isc_dhcpd.conf +++ b/conf.d/python.d/isc_dhcpd.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, isc_dhcpd supports the following: # @@ -71,13 +78,4 @@ # This is the default, so it will work in most cases. # 3. Pools MUST BE in CIDR format. # -#----------------------------------------------------------------------- -# AUTO-DETECTION JOBS -# This is disabled by default. -# To enable it, uncomment the following. -# -#leases: -# leases_path : '/var/lib/dhcp/dhcpd.leases' -# pools: -# office: '192.168.2.0/24' -# wifi: '192.168.3.0/24' +# ---------------------------------------------------------------------- diff --git a/conf.d/python.d/mdstat.conf b/conf.d/python.d/mdstat.conf index c89d463be..66a2f153c 100644 --- a/conf.d/python.d/mdstat.conf +++ b/conf.d/python.d/mdstat.conf @@ -23,4 +23,10 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 diff --git a/conf.d/python.d/memcached.conf b/conf.d/python.d/memcached.conf index f1723dc81..85c3daf65 100644 --- a/conf.d/python.d/memcached.conf +++ b/conf.d/python.d/memcached.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, memcached also supports the following: # diff --git a/conf.d/python.d/mongodb.conf b/conf.d/python.d/mongodb.conf index a19b6570b..62faef68d 100644 --- a/conf.d/python.d/mongodb.conf +++ b/conf.d/python.d/mongodb.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, mongodb also supports the following: # diff --git a/conf.d/python.d/mysql.conf b/conf.d/python.d/mysql.conf index 63d635174..def9f7e96 100644 --- a/conf.d/python.d/mysql.conf +++ b/conf.d/python.d/mysql.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, mysql also supports the following: # diff --git a/conf.d/python.d/nginx.conf b/conf.d/python.d/nginx.conf index 645925a55..71c521066 100644 --- a/conf.d/python.d/nginx.conf +++ b/conf.d/python.d/nginx.conf @@ -43,7 +43,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -59,13 +65,13 @@ # predefined parameters. These are: # # job_name: -# name: my_name # the JOB's name as it will appear at the -# # dashboard. If name: is not supplied the -# # job_name: will be used (use _ for spaces) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, this plugin also supports the following: # @@ -99,5 +105,5 @@ localipv4: localipv6: name : 'local' - url : 'http://::1/stub_status' + url : 'http://[::1]/stub_status' diff --git a/conf.d/python.d/nsd.conf b/conf.d/python.d/nsd.conf index 7566fe85e..078e97216 100644 --- a/conf.d/python.d/nsd.conf +++ b/conf.d/python.d/nsd.conf @@ -32,7 +32,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -48,12 +54,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, nsd also supports the following: # diff --git a/conf.d/python.d/ovpn_status_log.conf b/conf.d/python.d/ovpn_status_log.conf index 39bc8e9d4..907f014f5 100644 --- a/conf.d/python.d/ovpn_status_log.conf +++ b/conf.d/python.d/ovpn_status_log.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, openvpn status log also supports the following: # @@ -84,3 +91,5 @@ # #default: # log_path: '/var/log/openvpn-status.log' +# +# ---------------------------------------------------------------------- \ No newline at end of file diff --git a/conf.d/python.d/phpfpm.conf b/conf.d/python.d/phpfpm.conf index f5d067cc7..08688e2fa 100644 --- a/conf.d/python.d/phpfpm.conf +++ b/conf.d/python.d/phpfpm.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, PHP-FPM also supports the following: # diff --git a/conf.d/python.d/postfix.conf b/conf.d/python.d/postfix.conf index ca9d8fada..e0d5a5f83 100644 --- a/conf.d/python.d/postfix.conf +++ b/conf.d/python.d/postfix.conf @@ -32,7 +32,13 @@ update_every: 10 # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -48,12 +54,13 @@ update_every: 10 # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, postfix also supports the following: # diff --git a/conf.d/python.d/postgres.conf b/conf.d/python.d/postgres.conf index 1dbb64f40..3a70a7184 100644 --- a/conf.d/python.d/postgres.conf +++ b/conf.d/python.d/postgres.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # A single connection is required in order to pull statistics. # @@ -70,7 +77,7 @@ # index_stats : false # database_poll : 'dbase_name1 dbase_name2' # poll only specified databases (all other will be excluded from charts) # -# Postfix permissions are configured at its pg_hba.conf file. You can +# Postgres permissions are configured at its pg_hba.conf file. You can # "trust" local clients to allow netdata to connect, or you can create # a postgres user for netdata and add its password below to allow # netdata connect. diff --git a/conf.d/python.d/powerdns.conf b/conf.d/python.d/powerdns.conf new file mode 100644 index 000000000..ca6200df1 --- /dev/null +++ b/conf.d/python.d/powerdns.conf @@ -0,0 +1,78 @@ +# netdata python.d.plugin configuration for powerdns +# +# This file is in YaML format. Generally the format is: +# +# name: value +# +# There are 2 sections: +# - global variables +# - one or more JOBS +# +# JOBS allow you to collect values from multiple sources. +# Each source will have its own set of charts. +# +# JOB parameters have to be indented (using spaces only, example below). + +# ---------------------------------------------------------------------- +# Global Variables +# These variables set the defaults for all JOBs, however each JOB +# may define its own, overriding the defaults. + +# update_every sets the default data collection frequency. +# If unset, the python.d.plugin default is used. +# update_every: 1 + +# priority controls the order of charts at the netdata dashboard. +# Lower numbers move the charts towards the top of the page. +# If unset, the default for python.d.plugin is used. +# priority: 60000 + +# retries sets the number of retries to be made in case of failures. +# If unset, the default for python.d.plugin is used. +# Attempts to restore the service are made once every update_every +# and only if the module has collected values in the past. +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 + +# ---------------------------------------------------------------------- +# JOBS (data collection sources) +# +# The default JOBS share the same *name*. JOBS with the same name +# are mutually exclusive. Only one of them will be allowed running at +# any time. This allows autodetection to try several alternatives and +# pick the one that works. +# +# Any number of jobs is supported. +# +# All python.d.plugin JOBS (for all its modules) support a set of +# predefined parameters. These are: +# +# job_name: +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds +# +# Additionally to the above, apache also supports the following: +# +# url: 'URL' # the URL to fetch powerdns performance statistics +# header: +# X-API-Key: 'Key' # API key +# +# ---------------------------------------------------------------------- +# AUTO-DETECTION JOBS +# only one of them will run (they have the same name) + +# localhost: +# name : 'local' +# url : 'http://127.0.0.1:8081/api/v1/servers/localhost/statistics' +# header: +# X-API-Key: 'change_me' diff --git a/conf.d/python.d/rabbitmq.conf b/conf.d/python.d/rabbitmq.conf index eccf65df9..3f90da8a2 100644 --- a/conf.d/python.d/rabbitmq.conf +++ b/conf.d/python.d/rabbitmq.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, rabbitmq plugin also supports the following: # diff --git a/conf.d/python.d/redis.conf b/conf.d/python.d/redis.conf index 983fbfbdb..6363f6da7 100644 --- a/conf.d/python.d/redis.conf +++ b/conf.d/python.d/redis.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, redis also supports the following: # diff --git a/conf.d/python.d/retroshare.conf b/conf.d/python.d/retroshare.conf index 79614373b..9c92583f7 100644 --- a/conf.d/python.d/retroshare.conf +++ b/conf.d/python.d/retroshare.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, RetroShare also supports the following: # diff --git a/conf.d/python.d/samba.conf b/conf.d/python.d/samba.conf index 865281cd6..ee513c60f 100644 --- a/conf.d/python.d/samba.conf +++ b/conf.d/python.d/samba.conf @@ -31,7 +31,13 @@ update_every: 5 # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,10 @@ update_every: 5 # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts -# -# - +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds \ No newline at end of file diff --git a/conf.d/python.d/sensors.conf b/conf.d/python.d/sensors.conf index 2e9a41338..83bbffd7d 100644 --- a/conf.d/python.d/sensors.conf +++ b/conf.d/python.d/sensors.conf @@ -23,7 +23,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # Limit the number of sensors types. @@ -54,5 +60,4 @@ types: # the prefix is matched (anything that starts like that) # #---------------------------------------------------------------------- -# To change celsius to fahrenheit uncomment line below -#fahrenheit: True + diff --git a/conf.d/python.d/smartd_log.conf b/conf.d/python.d/smartd_log.conf index 8764ffd3e..3fab3f1c0 100644 --- a/conf.d/python.d/smartd_log.conf +++ b/conf.d/python.d/smartd_log.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,18 +53,20 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, smartd_log also supports the following: # -# log_path: '/path/to/smartdlogs' # path to smartd log files. Default is /var/log/smartd -# raw_values: no # raw or normalized values on charts. Default is normalized. -# smart_attributes: '1 2 3 4 44' # smart attributes charts. Default are ['1', '4', '5', '7', '9', '12', '193', '194', '197', '198', '200']. +# log_path: '/path/to/smartdlogs' # path to smartd log files. Default is /var/log/smartd +# raw_values: yes # enable/disable raw values charts. Enabled by default. +# smart_attributes: '1 2 3 4 44' # smart attributes charts. Default are ['1', '4', '5', '7', '9', '12', '193', '194', '197', '198', '200']. +# exclude_disks: 'PATTERN1 PATTERN2' # space separated patterns. If the pattern is in the drive name, the module will not collect data for it. # # ---------------------------------------------------------------------- # Additional information @@ -79,7 +87,4 @@ # Some raw numbers provide valuable insight when properly interpreted. These cases will be discussed later on. # Raw values are typically listed in hexadecimal numbers. The raw value has different structure for different vendors and is often not meaningful as a decimal number. # -# -# JOB configuration -# -log_path: '/var/log/smartd' +# ---------------------------------------------------------------------- diff --git a/conf.d/python.d/squid.conf b/conf.d/python.d/squid.conf index 27800bde7..564187f00 100644 --- a/conf.d/python.d/squid.conf +++ b/conf.d/python.d/squid.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, squid also supports the following: # diff --git a/conf.d/python.d/tomcat.conf b/conf.d/python.d/tomcat.conf index ce89175f6..c63f06cfa 100644 --- a/conf.d/python.d/tomcat.conf +++ b/conf.d/python.d/tomcat.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,12 +53,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, tomcat also supports the following: # diff --git a/conf.d/python.d/varnish.conf b/conf.d/python.d/varnish.conf index c25f3010f..4b069d514 100644 --- a/conf.d/python.d/varnish.conf +++ b/conf.d/python.d/varnish.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -47,11 +53,12 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts -# +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # +# ---------------------------------------------------------------------- diff --git a/conf.d/python.d/web_log.conf b/conf.d/python.d/web_log.conf index cd1f1af00..dd1fff075 100644 --- a/conf.d/python.d/web_log.conf +++ b/conf.d/python.d/web_log.conf @@ -31,7 +31,13 @@ # If unset, the default for python.d.plugin is used. # Attempts to restore the service are made once every update_every # and only if the module has collected values in the past. -# retries: 5 +# retries: 60 + +# autodetection_retry sets the job re-check interval in seconds. +# The job is not deleted if check fails. +# Attempts to start the job are made once every autodetection_retry. +# This feature is disabled by default. +# autodetection_retry: 0 # ---------------------------------------------------------------------- # JOBS (data collection sources) @@ -50,12 +56,13 @@ # predefined parameters. These are: # # job_name: -# name: myname # the JOB's name as it will appear at the -# # dashboard (by default is the job_name) -# # JOBs sharing a name are mutually exclusive -# update_every: 1 # the JOB's data collection frequency -# priority: 60000 # the JOB's order on the dashboard -# retries: 5 # the JOB's number of restoration attempts +# name: myname # the JOB's name as it will appear at the +# # dashboard (by default is the job_name) +# # JOBs sharing a name are mutually exclusive +# update_every: 1 # the JOB's data collection frequency +# priority: 60000 # the JOB's order on the dashboard +# retries: 60 # the JOB's number of restoration attempts +# autodetection_retry: 0 # the JOB's re-check interval in seconds # # Additionally to the above, web_log also supports the following: # diff --git a/conf.d/stream.conf b/conf.d/stream.conf index 0ae5ba673..8945529ee 100644 --- a/conf.d/stream.conf +++ b/conf.d/stream.conf @@ -21,13 +21,13 @@ # # If many are given, the first available will get the metrics. # - # PROTOCOL = tcp or udp (only tcp is supported by masters) - # HOST = an IPv4, IPv6 IP, or a hostname. + # PROTOCOL = tcp, udp, or unix (only tcp and unix are supported by masters) + # HOST = an IPv4, IPv6 IP, or a hostname, or a unix domain socket path. # IPv6 IPs should be given with brackets [ip:address] - # INTERFACE = the network interface to use + # INTERFACE = the network interface to use (only for IPv6) # PORT = the port number or service name (/etc/services) # - # This communication is not HTTP (cannot be proxied by web proxies). + # This communication is not HTTP (it cannot be proxied by web proxies). destination = # The API_KEY to use (as the sender) @@ -56,26 +56,34 @@ # ----------------------------------------------------------------------------- # 2. ON MASTER NETDATA - THE ONE THAT WILL BE RECEIVING METRICS -# You can have one API key per slave, or the same API key for all slaves. +# You can have one API key per slave, +# or the same API key for all slaves. # # netdata searches for options in this order: # -# a) [MACHINE_GUID] section (settings for each machine) -# b) [API_KEY] section (settings for the API key) -# c) master netdata settings (netdata.conf) +# a) master netdata settings (netdata.conf) +# b) [API_KEY] section (below, settings for the API key) +# c) [MACHINE_GUID] section (below, settings for each machine) # # You can combine the above (the more specific setting will be used). # API key authentication -# If the key is not listed here, it will not be able to connect. +# If the key is not listed here, it will not be able to push metrics. +# [API_KEY] is [YOUR-API-KEY], i.e [11111111-2222-3333-4444-555555555555] [API_KEY] - # Default settings for the API key + # Default settings for this API key # You can disable the API key, by setting this to: no # The default (for unknown API keys) is: no enabled = no + # A list of simple patterns matching the IPs of the servers that + # will be pushing metrics using this API key. + # The metrics are received via the API port, so the same IPs + # should also be matched at netdata.conf [web].allow connections from + allow from = * + # The default history in entries, for all hosts using this API key. # You can also set it per host below. # If you don't set it here, the history size of the central netdata @@ -132,6 +140,13 @@ # Use only the API key for security. enabled = no + # A list of simple patterns matching the IPs of the servers that + # will be pushing metrics using this MACHINE GUID. + # The metrics are received via the API port, so the same IPs + # should also be matched at netdata.conf [web].allow connections from + # and at stream.conf [API_KEY].allow from + allow from = * + # The number of entries in the database history = 3600 diff --git a/configs.signatures b/configs.signatures index f22431c26..c23950c92 100644 --- a/configs.signatures +++ b/configs.signatures @@ -1,31 +1,40 @@ declare -A configs_signatures=( + ['00403e687213f3b7db9bf4563a5a92cc']='python.d/isc_dhcpd.conf' ['0056936ce99788ed9ae1c611c87aa6d8']='apps_groups.conf' + ['007fc019fb32e952b509d455c016a002']='health.d/tcp_resets.conf' ['0102351817595a85d01ebd54a5f2f36b']='python.d/ovpn_status_log.conf' ['01302e01162d465614276de43fad7546']='python.d.conf' + ['0147c7e8f8f57e37c5dade4e8aacacf9']='python.d/example.conf' + ['017036c1dc32c9312b2704b839bd078f']='python.d/haproxy.conf' ['01c54057e0ca55b5bb49df1662d6b8c3']='python.d/web_log.conf' ['02fa10fa85ab88e9723998de48d1aca0']='health.d/disks.conf' + ['0314f0f1f88773c0ed9e9a908335e7ca']='health.d/tcp_mem.conf' ['036dc300bd7b0e0ef229b9822686d63e']='python.d/isc_dhcpd.conf' ['0388b873d0d7e47c19005b7241db77d8']='python.d/tomcat.conf' ['04138a3d8e907c75329fe60ce2e27c1c']='health.d/tcp_resets.conf' ['0433d98a19d3b08e6f13884e46d39b47']='health.d/disks.conf' ['043f0a35dde85837fabeb85b990a41c1']='health.d/swap.conf' + ['044496086420b531487d3c57600ca673']='apps_groups.conf' ['0529b679d3c0e7e6332753c7f6484731']='health.d/net.conf' ['057d12aaff0467e64529e839a258806b']='health.d/entropy.conf' ['059d98d0c562e1c81653d1e64673deab']='python.d/web_log.conf' ['05a8f39f134850c1e8d6267dbe706273']='health.d/web_log.conf' ['061c45b0e34170d357e47883166ecf40']='python.d/nginx.conf' + ['074d618a7e9c72f9bdfda7611e01e0ca']='python.d/redis.conf' ['074df527cc70b5f38c0714f08f20e57c']='health.d/apache.conf' ['0787e67357804b934d2866f1b7c60b14']='health.d/ipc.conf' ['08042325ab27256b938575deafee8ecf']='python.d/nginx.conf' ['0847d54a7a0c7e0381c52e9d4d3fa7db']='health.d/mdstat.conf' ['084ee72d64760f2641b0720e79c922f3']='health.d/cpu.conf' ['0856124b1eecf01681b4fdf4e21efb3f']='health.d/net.conf' + ['0862e7cf3d32ef48795702c6aefd27e0']='python.d/fail2ban.conf' ['08ff5218f938fc48e09e718821169d14']='health.d/redis.conf' ['091572888425bc3b8b559c3c53367ec7']='apps_groups.conf' ['09225283977a6584f8063016091cc4f2']='health.d/tcp_resets.conf' ['09264cec953ae1c4c2985e6446abb386']='health.d/mysql.conf' ['093540fdc2a228e976ce5d48a3adf9fc']='health.d/disks.conf' ['09e030d26be08a13fa3560e47fa27825']='apps_groups.conf' + ['0a7039ecc7a86b480d9d499b12b02763']='python.d/freeradius.conf' ['0ad10fa896346202aee99384b0ec968d']='health.d/cpu.conf' ['0c5e0fa364d7bdf7c16e8459a0544572']='health.d/netfilter.conf' ['0cd4e1fb57497e4d4c2451a9e58f724d']='python.d/redis.conf' @@ -49,6 +58,7 @@ declare -A configs_signatures=( ['18710ef6523cef8630d644ab270bfe02']='health.d/varnish.conf' ['18ee1c6197a4381b1c1631ef6129824f']='apps_groups.conf' ['1972e48345e6c3f0d65f94a03317622b']='health_alarm_notify.conf' + ['1bc518219377499d1d05d6ee770f1a87']='python.d/go_expvar.conf' ['1c12b678ab65f271a96da1bbd0a1ab1c']='health.d/softnet.conf' ['1c3168c95b53e999df3d45162b3f50b8']='health.d/fping.conf' ['1c71a8792c5c0ed035dd97af93a04838']='health_alarm_notify.conf' @@ -57,6 +67,7 @@ declare -A configs_signatures=( ['1ea8e8ef1fa8a3a0fcdfba236f4cb195']='python.d/mysql.conf' ['1eb0bc80934a3166fcde4d153c476d14']='health.d/fping.conf' ['1ef0fd38e7969c023bc3fa6d89eaf6d6']='python.d/mdstat.conf' + ['1f43a9a820c02e0525de594299b55b15']='python.d.conf' ['1f5545b3ff52b3eb75ee05401f67a9bc']='fping.conf' ['1fa47f32ab52a22f8e7087cae85dd25e']='health.d/net.conf' ['203678a5be09d65993dcb3a9d475d187']='health.d/ipfs.conf' @@ -80,22 +91,31 @@ declare -A configs_signatures=( ['298504f331c55dff4055321ff3a7b5cc']='health.d/web_log.conf' ['29f97e10b92333790fbe0d2a3617b736']='health_alarm_notify.conf' ['2a0794fd43eadf30a51805bc9ba2c64d']='python.d/hddtemp.conf' + ['2acae80dbdbe536a160f5b216bac84bc']='python.d/samba.conf' ['2ad55a5d1e885cf142849a78d4b00401']='health.d/net.conf' ['2bbbebf52f84fd27fbefecd2a8a8076f']='health.d/memcached.conf' ['2c2b7e8df922b2bf121fb7db32bbc3bd']='health.d/udp_errors.conf' ['2d1d7498c72f4245cf32902c2b7e71e0']='health.d/entropy.conf' + ['2da0a2e7117292ece11d69723a294bd7']='python.d/mongodb.conf' + ['2ee5df033fe9c65a45566b6760b856e3']='python.d/web_log.conf' + ['2f05e09b69ea20cda56d8f8b6fd3e86d']='health.d/couchdb.conf' ['2f3a8e33df83f14e0af8ca2465697215']='python.d/exim.conf' ['2f4a85fedecce1bf425fa1039f6b021e']='apps_groups.conf' ['2fa8fb929fd597f2ab97b6efc540a043']='health_alarm_notify.conf' ['307ac41f6c67fcf007d6f7135fac314c']='stream.conf' ['312b4b8e2805e19cf9be554b319567d6']='health.d/softnet.conf' ['318bb45755726a25120bb33413d4b582']='health.d/net.conf' + ['318db50a701442890c269ab547041e97']='health.d/tcp_orphans.conf' ['325617412a628e3bc776e3fbb777a2a6']='health.d/redis.conf' ['326e1477131e0f73304711135f70a2a5']='health.d/memcached.conf' ['32fde0057c790964f2c743cb3c9aad29']='health.d/nginx.conf' + ['33486497112127badc4c47ed2008969c']='python.d/freeradius.conf' ['33b135e28aeaef2b8224ba69a0fde245']='health.d/cpu.conf' ['343bc919a2fbc93f687f9d1339ec5f79']='health.d/net.conf' + ['34f6cf10f8da18ddd7edd2a28a4fe5f9']='python.d/sensors.conf' ['35024ebd94542484c0866e6ee6b950cb']='health.d/net.conf' + ['35ac63a2f08b2c6dd901c542629ae5df']='python.d/postgres.conf' + ['35eb9785c844afd43fa7931915e2d566']='python.d/elasticsearch.conf' ['3634d5eddc46fb0d50cf47f370670c2c']='health.d/redis.conf' ['364b6e0081b116c9ec073b4d329a6dcc']='health_alarm_notify.conf' ['367d1463e520eb9dc89223bab161c6d1']='python.d/postgres.conf' @@ -108,11 +128,14 @@ declare -A configs_signatures=( ['3866efafd38e161136428d0f818cac43']='health.d/net.conf' ['38d1bf04fe9901481dd6febcc0404a86']='python.d.conf' ['39304b2570611c3acb35b72762b46778']='charts.d/sensors.conf' + ['394b7e91c97b7adb776460d309b335ff']='python.d/nginx.conf' ['39571e9fad9b759200c5d5b2ee13feb4']='python.d/redis.conf' + ['39b65042cafdd9b849a44ec81aa66cac']='health_alarm_notify.conf' ['39f9422b0f0c3eec11a31aff79d89514']='health.d/retroshare.conf' ['3a04a3bc66c49d0c24f65a44fd9caa80']='python.d/postgres.conf' ['3af522d65b50a5e447607ffb28c81ff5']='apps_groups.conf' ['3b1bfa40a4ff6a200bb2fc00bc51a664']='apps_groups.conf' + ['3b535da82cf2ff53e03d735d02fb2357']='python.d/squid.conf' ['3bc2776623889744a98178bad6fb3b79']='health.d/disks.conf' ['3bc65e997ab59b9de390fdf63d77f5e1']='python.d/postgres.conf' ['3c9c47163e9d4dbcb0079b6232398f2f']='apps_groups.conf' @@ -120,9 +143,11 @@ declare -A configs_signatures=( ['3cc6255457d4cba881ae0554ae5d9190']='health.d/squid.conf' ['3d974ac9fdaa44d4527d6503bec35e34']='stream.conf' ['3f170e3343cd784983b019163393f5af']='health.d/nginx.conf' + ['3fbe85671efd5d07e51584ab8262b48b']='health.d/tcp_listen.conf' ['3fc45cc18e884c22482524dff6d27833']='python.d/hddtemp.conf' ['3fcc3c449ce8e0388f9c23ca07cab608']='health.d/backend.conf' ['4063a01bffb43b0423425d1ba3004967']='health.d/tcp_resets.conf' + ['41fa6bb109763561be59d7bcd07bbe82']='python.d/dnsdist.conf' ['421d5dc6c2fce22d0816b6e6363bea57']='python.d/hddtemp.conf' ['42ad0e70b1365b6e7244cc305dbaa529']='health_alarm_notify.conf' ['42bf1c7c64ed77038a0aa094d792a9e2']='python.d/mysql.conf' @@ -140,6 +165,7 @@ declare -A configs_signatures=( ['48eef63bcf744bae114b502b6dacb4a1']='charts.d/phpfpm.conf' ['4960852f8951b54ca2fe10065752143e']='python.d.conf' ['4a448831776de8acf2e0bdc4cc994cb4']='apps_groups.conf' + ['4aba3b6a28ccd75faf5326aca997ee0d']='health_alarm_notify.conf' ['4b775fb31342f1478b3773d041a72911']='python.d.conf' ['4ccb06fff1ce06dc5bc80e0a9f568f6e']='charts.d.conf' ['4cd585f5dfdacaf287413ad037b4e60a']='apps_groups.conf' @@ -157,6 +183,7 @@ declare -A configs_signatures=( ['5120492fa26be3749192607f62dc05f8']='health.d/mdstat.conf' ['5271cf9fc0fd10915a9759add70f7d78']='health.d/swap.conf' ['5278ebbae19c60db600f0a119cb3664e']='python.d/apache.conf' + ['52d230aff57850a5aacc4e0420fcd8f5']='python.d.conf' ['52d4131cf9df84e2550b1a5d899ec61d']='health.d/swap.conf' ['53160707fdc6ce46c195b1b55bb0bcb1']='health.d/swap.conf' ['535e5113b07b0fc6f3abd59546c276f6']='charts.d.conf' @@ -164,12 +191,14 @@ declare -A configs_signatures=( ['5452eccad2f220d1191411737f6f4b2b']='python.d/isc_dhcpd.conf' ['54614490a14e1a4b7b3d9fecb6b4cfa5']='python.d/exim.conf' ['547779cdc460a926980de1590294b96b']='health.d/softnet.conf' + ['54c3934a03453453b8d7d0e8b84a7cd8']='health_alarm_notify.conf' ['55608bdd908a3806df1468f6ee318b2b']='health.d/qos.conf' ['5598b83e915e31f68027afe324a427cd']='apps_groups.conf' ['565f11c38ae6bd5cc9d3c2adb542bc1b']='health.d/softnet.conf' ['5664a814f9351b55da76edd472169a73']='health_alarm_notify.conf' ['56b689031cdcf138064825f31474b37d']='apps_groups.conf' ['573398335c0c71c075fa57f702bce287']='health.d/disks.conf' + ['57be306944cb09b7f024079728fd04b9']='apps_groups.conf' ['5829812db29598db5857c9f433e96fef']='python.d/apache.conf' ['5855dd70d71c8497e5591b0690162c9c']='health.d/tcp_resets.conf' ['58e835b7176865ec5a6f59f7aba832bf']='health.d/named.conf' @@ -179,6 +208,7 @@ declare -A configs_signatures=( ['5bbef0708f5eff4d4a53aaf35fc48a62']='health.d/disks.conf' ['5bf51bb24fb41db9b1e448bd060d3f8c']='apps_groups.conf' ['5da15d6e17a15213a720749045e5d419']='health.d/disks.conf' + ['5dddb6c9670f4aa605abe4b0d901acc4']='python.d/bind_rndc.conf' ['5e6fd588ef6934cf04ddb5e662aa02ea']='health.d/postgres.conf' ['5eb670b6fe39da5fec2523d910b0dd1e']='health.d/cpu.conf' ['5f05d4b248ab2637ada319b4e8c4e4c3']='python.d/varnish.conf' @@ -205,12 +235,12 @@ declare -A configs_signatures=( ['669ebef43ee341f6889d382e86d0e200']='health.d/named.conf' ['66dfe138058ca26a31a118007eb31f35']='health.d/nginx.conf' ['6814b9bc84483db428f6a479ba221855']='python.d/mysql.conf' + ['68607aef1802ed3dc0cd593bf6073beb']='python.d/postfix.conf' ['6a18f61a595c0d48c3363bcc0dbfa6b9']='health_alarm_notify.conf' ['6a47af861ad3dd112124c37fbf09672b']='apps_groups.conf' ['6aa4507f86657383917a0407f2a9cc0d']='python.d.conf' ['6acad8ce5c33e642742825db0eb9bb56']='python.d/web_log.conf' ['6b39de5d85db45115db236347a6896d4']='health.d/named.conf' - ['6b917300747e7e8314844237e2462261']='python.d/apache_cache.conf' ['6bb278bd9e171c4cb5c0fe639231288b']='python.d/web_log.conf' ['6bf0de6e3b251b765b10a71d8c5c319d']='python.d/apache.conf' ['6c9f2f0abe49a6f1a69db052ebcef1bf']='python.d/elasticsearch.conf' @@ -222,6 +252,7 @@ declare -A configs_signatures=( ['707a63f53f4b32e01d134ae90ba94aad']='health_alarm_notify.conf' ['707a63f53f4b32e01d134ae90ba94aad']='health_email_recipients.conf' ['70d82dabecb09a1da4684f293abef0c9']='health_alarm_notify.conf' + ['7117b7067ac2b712aa4c9e92a6cdbf5a']='python.d/couchdb.conf' ['7120cba2f55b1c0a97a0e10d4f6ef751']='health.d/ipmi.conf' ['729b3e24a72f7d566fd429617d51a21b']='health.d/web_log.conf' ['73125ae64d5c6e9361944cd9bd14844e']='python.d/exim.conf' @@ -229,10 +260,13 @@ declare -A configs_signatures=( ['73a8e10dfe4183aca751e9e2a80dabe3']='node.d.conf' ['7454ed74511d7b9819dfe173f9020786']='python.d/redis.conf' ['749fe31362969d75f1ea66d15231d98d']='python.d/retroshare.conf' + ['74e5e8d3a4b324f1770f61f78ee4b0e6']='health.d/beanstalkd.conf' ['7502c931aa9acbb92f54c67978d75983']='stream.conf' + ['751f15371d0987018abc4d4ad60819f5']='apps_groups.conf' ['7596ae54d46ce199ac599429ef753caf']='health.d/cpu.conf' ['75a9c4b0b1c73956df55585eb0619f6c']='charts.d/ap.conf' ['769aa4cdcdc3d78d0328d1f9e4edcdf9']='python.d/mysql.conf' + ['76a0c1b21e49850442a43efddb15a81e']='health.d/tcp_orphans.conf' ['777f4da70f461ef675bde07fb3644312']='python.d/redis.conf' ['777f55a95c5c25cf6176fece1ebbf4b8']='apps_groups.conf' ['7808ba2ca26bd0642270740cf6a8ee59']='charts.d/mem_apps.conf' @@ -256,6 +290,7 @@ declare -A configs_signatures=( ['80f109ff293ac94222bf3959432751bd']='health.d/qos.conf' ['81255035f6d53534938085df72cdef23']='health.d/nginx.conf' ['8170ba3ae507cf9322bd60350348552e']='health.d/net.conf' + ['81af92c7050873c7de2fb42e0c3f04f4']='python.d/tomcat.conf' ['81fd16f29d5f3d422fe1cee82dc8ed9d']='health.d/cpu.conf' ['8213d921b6a8382e27052fb42d81db3d']='python.d/freeradius.conf' ['8214bb8f4b005aa4691fcd38f7331e8f']='health.d/swap.conf' @@ -266,8 +301,10 @@ declare -A configs_signatures=( ['845023f9b4a526aa0e6493756dbe6034']='health.d/squid.conf' ['846ce94bfeeb90c0dc6a89e8d25f1a68']='health.d/named.conf' ['8490f690d97adacc4e2096df82e7e8a5']='charts.d/cpufreq.conf' + ['87155bea7383028b0c1846c802cfdd81']='python.d/mdstat.conf' ['871bbeea33b83ea9755600b6d574919c']='python.d/web_log.conf' ['87224d2f2b87646f3c0d38cc1eb30112']='python.d/nsd.conf' + ['87615ae5ac2412d853c717383fa53781']='python.d/chrony.conf' ['87642c568093daf3b2c30c5beffe2225']='python.d/elasticsearch.conf' ['8810140ce9c09af1d18b9602c4003904']='health_alarm_notify.conf' ['88f77865f75c9fb61c97d700bd4561ee']='python.d/mysql.conf' @@ -276,21 +313,30 @@ declare -A configs_signatures=( ['89fb3cbb223be4fa0cb676cfa3b07055']='health.d/backend.conf' ['8a1b95d375992d7b11330a0ac46f369c']='health.d/disks.conf' ['8a66a3085ad8892a002ff39b18b2cb07']='python.d/fail2ban.conf' + ['8abc7f66746b201b5b0af45c419d53bc']='health.d/bind_rndc.conf' ['8c1d41e2c88aeca78bc319ed74c8748c']='python.d/phpfpm.conf' ['8d0552371a7c9725a04196fa560813d1']='health.d/cpu.conf' ['8d24873bb25c195026918f15626310ea']='health.d/softnet.conf' ['8dc0bd0a70b5117454bd5f5b98f91c2c']='health.d/disks.conf' + ['8ec636a4f96158044d2cec0fd1ff8452']='python.d/rabbitmq.conf' ['8f4f925c1e97dd164007495ec5135ffc']='health.d/fping.conf' ['8f7b734ea0f89abf8acbb47c50234477']='health.d/web_log.conf' ['8fd472a854b0996327e8ed3562161182']='health_alarm_notify.conf' ['919911d13901d60a7580f5dfd7fc87bb']='health.d/ram.conf' + ['91c377e7d26a1120cfbbd488332f0398']='python.d/dns_query_time.conf' ['91c757ef6be3abdb86906d9dbb9c217a']='fping.conf' ['91cf3b3d42cac969b8b3fd4f531ecfb3']='python.d/squid.conf' ['91e1a9703debbdc64edf124419fdc14b']='python.d/elasticsearch.conf' + ['91f0a626c19f76241cadf9dbf28fb5a7']='health.d/beanstalkd.conf' + ['92024bbe088e55251665fb666305ff66']='python.d/mysql.conf' + ['920574fcfe56d5c9c11a583905e9db62']='health.d/tcp_conn.conf' ['9347bcce0b3574ac5193d43248d2e3cc']='python.d/chrony.conf' + ['94bb961f83ec724cf86239328f73a3db']='health.d/redis.conf' ['9542f80def48ba105190f6cdaa18248e']='health.d/mysql.conf' + ['978daf0777ffe774e5a9576d33972e97']='python.d/smartd_log.conf' ['97eee7a30e6419df4537242e9d4a719d']='health.d/mysql.conf' ['97f337eb96213f3ede05e522e3743a6c']='python.d/memcached.conf' + ['98e4dd6ba71bf76767bc59c63a51b617']='apps_groups.conf' ['99a3de85d1e7826ed64a5f8576712e5d']='python.d.conf' ['99b06e68f1da5917ae4cf60e901439f6']='health.d/ram.conf' ['99b6030ce25c8fee4598179c0f95fb0b']='health.d/redis.conf' @@ -300,6 +346,7 @@ declare -A configs_signatures=( ['9c0185ceff15415bc59b2ce2c1f04367']='apps_groups.conf' ['9c8ddfa810d83ae58c8614ee5229e66b']='health.d/disks.conf' ['9c981c75bdf4b1637f7113e7e45eb2bf']='health.d/memcached.conf' + ['9d304e41e32721224a743f25534263d9']='python.d/retroshare.conf' ['9e0553ebdc21b64295873fc104cfa79d']='python.d.conf' ['9eb3326ae2ee9badeaad31d8dd2eaa2b']='python.d/isc_dhcpd.conf' ['a02d14124b19c635c1426cee2e98bac5']='charts.d.conf' @@ -308,7 +355,10 @@ declare -A configs_signatures=( ['a0b3a12389c9c56dfe35964b20b59836']='health.d/bind_rndc.conf' ['a0c0ef7ca9671f4b5e797d4276e5c0dd']='health.d/disks.conf' ['a0ee8f351f213c0e8af9eb7a4a09cb95']='apps_groups.conf' + ['a1b53a225f225911cd8ac892bba8118b']='python.d/powerdns.conf' ['a1b6dfe312b896b0b1ba471e8ac07f95']='python.d/isc_dhcpd.conf' + ['a1bb5823c4926b65ef4b2dae467fc847']='python.d/couchdb.conf' + ['a250e12f1ab4c18796fdaff5b0ba8968']='python.d/varnish.conf' ['a2944a309f8ce1a3195451856478d6ae']='python.d.conf' ['a2a647dc492dc2d6ed1f5c0fdc97a96e']='python.d/mongodb.conf' ['a305b400378d6492efd15f9940c2779b']='health.d/softnet.conf' @@ -321,6 +371,7 @@ declare -A configs_signatures=( ['a5134d7cfbe27f5791e788c2add51abb']='apps_groups.conf' ['a55133f1b0be0a4255057849dd451b09']='health_alarm_notify.conf' ['a6d5ce2572bf7a1dce9e545fcd29273e']='health.d/apache.conf' + ['a70e14bda17b076d2486232355652ae6']='apps_groups.conf' ['a71d9082410200bf92e823675d78121c']='python.d/retroshare.conf' ['a731b7b164f42717c1c9a778ee637ff3']='health.d/memcached.conf' ['a7320c6f26191b9599ec3bc4be007a93']='health.d/swap.conf' @@ -331,6 +382,7 @@ declare -A configs_signatures=( ['a89c516a1144435a88decf25509318ac']='health_alarm_notify.conf' ['a8bb4e1d0525f59692778ad8f675a77a']='python.d/example.conf' ['a8feb36776005bf419c90278787a1be8']='health.d/entropy.conf' + ['a9150a0c61e1b360cf8c265ea2413d02']='python.d/couchdb.conf' ['a94af1c808aafdf00537d85ff2197ec8']='python.d/exim.conf' ['a9ab68845db2fb695b7060273a6ac68e']='health_alarm_notify.conf' ['a9cd91675467c5426f5b51c47602c889']='apps_groups.conf' @@ -345,6 +397,7 @@ declare -A configs_signatures=( ['ad15b251b93f8b16bb33ec508f44a598']='health.d/netfilter.conf' ['ade389c1b6efe0cff47c33e662731f0a']='python.d/squid.conf' ['ae5ac0a3521e50aa6f6eda2a330b4075']='python.d/example.conf' + ['aee501b7f9b122b962521c45893371bb']='python.d/smartd_log.conf' ['af12051cf57dd4e484ef8e64502b7549']='health.d/net.conf' ['af14667ee7993acea810f6d50923bdc9']='health.d/web_log.conf' ['af44cc53aa2bc5cc8935667119567522']='python.d.conf' @@ -354,11 +407,14 @@ declare -A configs_signatures=( ['b0f0a0ac415e4b1a82187b80d211e83b']='python.d/mysql.conf' ['b181dcca01a258d9792ad703583baed2']='statsd.d/example.conf' ['b185914d4f795e1732273dc4c7a35845']='health.d/memory.conf' + ['b210982cac9accfe43173cef5f46b361']='health.d/beanstalkd.conf' ['b27f10a38a95edbbec20f44a4728b7c4']='python.d.conf' + ['b28c77dceeb398ca4ceec44c646f5431']='stream.conf' ['b32164929eda7449a9677044e11151bf']='python.d.conf' ['b3d48935ab7f44a57d40ad349df0033d']='python.d/postgres.conf' ['b3fc4749b132e55ac0d3a0f92859237e']='health.d/tcp_resets.conf' ['b4825f731cc7eb03b374eade14a453c1']='health.d/net.conf' + ['b5246eed059e33e0903a819fa5460ce0']='python.d/ipfs.conf' ['b5b5a8d6d991fb1cef8d80afa23ba114']='python.d/cpufreq.conf' ['b636e5e603f9d93e52c7577ac8c6bf0c']='health.d/entropy.conf' ['b68706bb8101ef85192db92f865a5d80']='health_alarm_notify.conf' @@ -368,11 +424,13 @@ declare -A configs_signatures=( ['b81b8f331161b0d48e03f6fbf6b6d062']='health.d/memcached.conf' ['b846ca1f99fa6a65303b58186f47d7a4']='python.d/squid.conf' ['b8969be5b3ceb4a99477937119bd4323']='python.d.conf' + ['b8aff60806fb6829a4e72a824e655375']='health.d/beanstalkd.conf' ['b8b87574fd496a66ede884c5336493bd']='python.d/phpfpm.conf' ['b915126262d08aa9da81de539a58a3fb']='python.d/redis.conf' ['ba11ea2d2f632b2de4b1224bcdc54f07']='python.d/smartd_log.conf' ['bb51112d01ff20053196a57632df8962']='apps_groups.conf' ['bba2f3886587f137ea08a6e63dd3d376']='python.d.conf' + ['bcaba2347951b301127fd502a219b26a']='python.d/apache.conf' ['bcd94c4fa2f89c710ff807de061ab11c']='health.d/net.conf' ['bda5517ea01640cfdfa0a27549619d6a']='health.d/memcached.conf' ['bf66f113b2dd8d8fb444cbd5650f284c']='health_alarm_notify.conf' @@ -380,8 +438,11 @@ declare -A configs_signatures=( ['c080e006f544c949baca33cc24a9c126']='health_alarm_notify.conf' ['c132d2e257fc4df2925be7ad75100d5b']='health.d/entropy.conf' ['c1a7e634b5b8aad523a0d115a93379cd']='health.d/memcached.conf' + ['c1d014ffaebfa0952968aeaf330e5337']='python.d.conf' + ['c30ee008173ba9f77adfcacbf138143e']='python.d/ovpn_status_log.conf' ['c3296c08260bcd556e74711c820817be']='health.d/cpu.conf' ['c3661b68232e06de90bb5e63e725b8b6']='health_alarm_notify.conf' + ['c45ab106725e94615bccf8be4b136d0f']='python.d.conf' ['c61948101e0e6846679682794ee48c5b']='python.d/nginx.conf' ['c6403d8b1bcfa52d3abb941be155fc03']='python.d.conf' ['c6b9f31e14adca433f82054f62388c47']='python.d/web_log.conf' @@ -399,23 +460,31 @@ declare -A configs_signatures=( ['cb60badf376d246ad8ec9d3f524db430']='health.d/disks.conf' ['cb7f80cd2768c649d7448e01f8aa6579']='python.d.conf' ['cc4d31a0d1ff9c339892c1f8a0c5fcd3']='charts.d/load_average.conf' + ['cca26b4d2384043f1737e0ed4a995600']='python.d/bind_rndc.conf' ['ccde91d209aeb02c4a6be0e43a8d92b3']='health.d/apache.conf' ['cd08e5534c94bf1f2cd28396c76b8bbc']='health.d/ram.conf' + ['cd9a7de356d6424c4a71d87053726c86']='python.d/bind_rndc.conf' ['cdd504812ff93073c57d02209d4d0f69']='health.d/cpu.conf' ['ce2e8768964a936f58c4c2144aee8a01']='health_alarm_notify.conf' ['ce3b65eac6c472b21905f7f72104f4c9']='python.d/nginx.conf' ['cf48dfd828af70bea04db7a809f94358']='health.d/haproxy.conf' + ['cf8b87ede2d3233b6f55f4690af7fb08']='python.d/smartd_log.conf' ['cfecf298bdafaa7e0a3a263548e82132']='python.d/sensors.conf' ['d11711b3647bc2bdd0292dd7deebbeb1']='health.d/net.conf' ['d1596fe068c8674efade49a4a8e22b5d']='health.d/isc_dhcpd.conf' ['d162b7465a56151312e60151c1d74fba']='health.d/squid.conf' ['d1e79707cd9b51a14288e8dd40694fcc']='fping.conf' + ['d297104e43ce2b544003271181e26ff6']='python.d/cpufreq.conf' ['d29c5fa5faf74b86d01c2270a79388d8']='health.d/disks.conf' ['d2b2ad30e277a69d8713e620dabc18bc']='python.d/phpfpm.conf' + ['d3f397ead7f2ac8f88a99d7c5b8cff1d']='python.d/dovecot.conf' + ['d4adcebadc4c86332df247922b85aadc']='python.d/freeradius.conf' ['d55bdb83b9ff606852f6a97c1430258c']='health.d/ram.conf' ['d55be5bb5e108da1e7645da007c53cd4']='python.d.conf' + ['d56c28ece8354850011f213d94d02fe0']='python.d/hddtemp.conf' ['d5dab509d8792f795bece27de39dd476']='health.d/mysql.conf' ['d69eba15d3e968187a938a7b98e22dda']='python.d.conf' + ['d712df81b17971884443a4a9bc996c9e']='health_alarm_notify.conf' ['d74dc63fbe631dab9a2ff1b0f5d71719']='python.d/hddtemp.conf' ['d7e0bd12d4a60a761dcab3531a841711']='python.d/phpfpm.conf' ['d8dc489e32f7114c6298fce94e86a8ef']='health.d/entropy.conf' @@ -426,6 +495,7 @@ declare -A configs_signatures=( ['dad303c5cca7a69345811a01a74f5892']='health.d/net.conf' ['dc0d2b96378f290eec3fcf98b89ad824']='python.d/cpufreq.conf' ['dc9c2a66778623a759706c14c3d91983']='health.d/net.conf' + ['dd220677c42c487549952048ee1f7750']='python.d/postgres.conf' ['dd7764507804a2296bfd091a58ad4ad7']='health.d/memcached.conf' ['dd8254ef74509a3e38cb2838e30f7e63']='health.d/disks.conf' ['ddda2bb1c88be03b637d3285406f7910']='health.d/named.conf' @@ -438,9 +508,10 @@ declare -A configs_signatures=( ['dfd5431b11cf2f3852a40d390c1d5a92']='python.d/varnish.conf' ['e0242003fd2e3f9ac1b9314e802ada79']='python.d/hddtemp.conf' ['e0ba3bc216ffc9933b4741dbb6b1f8c8']='health.d/web_log.conf' - ['e0e96cc47ed61d6492416be5236cd4d3']='python.d/apache_cache.conf' ['e0ffc0c34424b35666fddf7f61e05def']='health.d/tcp_resets.conf' + ['e100d98f3ed1eff59678f035b3b8daf2']='python.d/beanstalk.conf' ['e1a8bf99d36683c10225100f207a2b59']='python.d/web_log.conf' + ['e22f30680148a29d9738bd4bfe8b252c']='health_alarm_notify.conf' ['e2f3388c06726154c10ec22bad5bc7ec']='fping.conf' ['e3023092e3b2bbb5351e0fe6682f4fe9']='health_alarm_notify.conf' ['e3112d8e06fa77888aab02e8fcd22e25']='apps_groups.conf' @@ -451,8 +522,10 @@ declare -A configs_signatures=( ['e449e5582279742496550df14b6fca95']='health.d/entropy.conf' ['e4ed13f996434ac17b40a2228c96283b']='python.d/tomcat.conf' ['e5f32f54d6d6728f21f9ac26f37d6573']='python.d/example.conf' + ['e70a7ee4999f30c6ceb75f31088a3a34']='python.d/powerdns.conf' ['e734c5951a8764d4d9de046dd7cf7407']='health.d/softnet.conf' ['e7bc22a1942cffbd2b1b0cfd119ee328']='health.d/ipfs.conf' + ['e8656d72dbd3b6fe603048ded751499a']='python.d/memcached.conf' ['e8ec8046c7007af6ca3e8c51e62c99f8']='health.d/disks.conf' ['ea031c1c0c36edee3bd08fae559c4203']='health_alarm_notify.conf' ['ea1a96c42ad464c354fb250e3408c3e8']='stream.conf' @@ -461,9 +534,11 @@ declare -A configs_signatures=( ['eb748d6fb69d11b0d29c5794657e206c']='health.d/qos.conf' ['eb9fedc3c1dface77312d9bf48f673a8']='stream.conf' ['ebd0612ccc5807524ebb2b647e3e56c9']='apps_groups.conf' + ['ecb6c01fae255d369748406945a50435']='apps_groups.conf' ['ecd3aa97e2581f88eb466d6612690ef2']='charts.d/nginx.conf' ['ed43efac299c31f8fd5e2abccff30071']='python.d/samba.conf' ['ed80e6b2cfc8b08adea7027fc03daa68']='python.d.conf' + ['edb48efc8f446624001e07d04f6cad1a']='apps_groups.conf' ['ee5343881744e6a97e6ee5cdd329cfb8']='health.d/retroshare.conf' ['ef1861bf5725d91e773cbdba05687597']='python.d.conf' ['ef9916ea144878a9f37cbb6b1b29da10']='health.d/squid.conf' @@ -471,26 +546,34 @@ declare -A configs_signatures=( ['f0a86c5bae3c4b32b266dacbf74ca4a3']='python.d/web_log.conf' ['f1446cb3f1a905ee06defa2aa15ee806']='python.d/web_log.conf' ['f1682835e3414f60284c13bf1662e50f']='health.d/web_log.conf' + ['f1f114647ed185c4812c361b1d870b44']='python.d/sensors.conf' ['f2f1b8656f5011e965ac45b818cf668d']='apps_groups.conf' + ['f42389e5497a28205ba6fef4f716db4f']='python.d/nginx.conf' ['f42df9f13abfae2426519c6728b34882']='charts.d/example.conf' ['f4c5d88c34d3fb853498124177cc77f1']='python.d.conf' ['f5736e0b2945182cb659cb0713eff923']='apps_groups.conf' ['f66e5236ba1245bb2e5fd99191f114c6']='charts.d/hddtemp.conf' + ['f68ac0fca6b4ffc96097779344cabac6']='health.d/tcp_listen.conf' ['f6c6656f900ff52d159dca12d624016a']='python.d/postgres.conf' ['f7401a6e7c7d4fe2e0e2be7f7f523275']='health.d/web_log.conf' ['f7a99e94231beda85c6254912d8d31c1']='python.d/tomcat.conf' ['f82924563e41d99cdae5431f0af69155']='python.d.conf' ['f8c30f22df92765e2c0fab3c8174e2fc']='health.d/memcached.conf' ['f8dade4484f1b6a48655388502df7d5a']='health_alarm_notify.conf' + ['f8e7d23a83fc8ee58f403da7bdbe7f8a']='python.d/phpfpm.conf' ['f96acba4b14b0c1b50d0187a04416151']='health_alarm_notify.conf' ['f9be549a849d023595d19d5d74263e0f']='health.d/tcp_resets.conf' ['fa4396513b358d6ec6a7f5bfb08439b8']='health.d/net.conf' + ['fb1c75159f855f4b4671835f5bca1ef6']='python.d.conf' ['fbdb6f5d3906d3d8ea4e28f6ba6965a6']='python.d/go_expvar.conf' + ['fc11bd9255ac382f442f31c1f1a32532']='health_alarm_notify.conf' ['fc40b83f173bc4676d686867a8369a62']='python.d/dns_query_time.conf' + ['fc987459f82e251e31c41d822e5e8202']='python.d/nsd.conf' ['fd3164e6e8cb6726706267eae49aa082']='health_alarm_notify.conf' ['fdd11640ba626cc2064c2fe3ea3eee4c']='health.d/cpu.conf' ['fde44f62c8d7e52f09705cd273fae6b1']='charts.d/tomcat.conf' ['fdea185e0e52b459b48852aa37f20e0f']='apps_groups.conf' + ['fe069e4d6579ecdda7f36ac2318ffefc']='python.d/exim.conf' ['fe478efe2e721724edb1fe2ef1addf93']='health_alarm_notify.conf' ['feb8bcf828aa2529a7ee4a140feeb12d']='health.d/net.conf' ['ff1b3d8ae8b2149c711d8da9b7a9c4bd']='health_alarm_notify.conf' diff --git a/configure b/configure index e1fae5540..adcd6eb25 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for netdata 1.8.0. +# Generated by GNU Autoconf 2.69 for netdata 1.9.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -577,8 +577,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='netdata' PACKAGE_TARNAME='netdata' -PACKAGE_VERSION='1.8.0' -PACKAGE_STRING='netdata 1.8.0' +PACKAGE_VERSION='1.9.0' +PACKAGE_STRING='netdata 1.9.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1374,7 +1374,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures netdata 1.8.0 to adapt to many kinds of systems. +\`configure' configures netdata 1.9.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1444,7 +1444,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of netdata 1.8.0:";; + short | recursive ) echo "Configuration of netdata 1.9.0:";; esac cat <<\_ACEOF @@ -1587,7 +1587,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -netdata configure 1.8.0 +netdata configure 1.9.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2365,7 +2365,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by netdata $as_me 1.8.0, which was +It was created by netdata $as_me 1.9.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2746,7 +2746,7 @@ $as_echo "$as_me: ***************** MAINTAINER MODE *****************" >&6;} PACKAGE_BUILT_DATE=$(date '+%d %b %Y') fi -PACKAGE_RPM_VERSION="1.8.0" +PACKAGE_RPM_VERSION="1.9.0" @@ -3273,7 +3273,7 @@ fi # Define the identity of the package. PACKAGE='netdata' - VERSION='1.8.0' + VERSION='1.9.0' cat >>confdefs.h <<_ACEOF @@ -8525,7 +8525,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by netdata $as_me 1.8.0, which was +This file was extended by netdata $as_me 1.9.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -8591,7 +8591,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -netdata config.status 1.8.0 +netdata config.status 1.9.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 16381f8dd..b8a0699b5 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ AC_PREREQ(2.60) define([VERSION_MAJOR], [1]) -define([VERSION_MINOR], [8]) +define([VERSION_MINOR], [9]) define([VERSION_FIX], [0]) define([VERSION_NUMBER], VERSION_MAJOR[.]VERSION_MINOR[.]VERSION_FIX) define([VERSION_SUFFIX], []) diff --git a/contrib/debian/changelog b/contrib/debian/changelog deleted file mode 100644 index 479a20208..000000000 --- a/contrib/debian/changelog +++ /dev/null @@ -1,3 +0,0 @@ -netdata (1.8.0) UNRELEASED; urgency=medium - * Latest release - -- Netdata Team <> Sun, 17 Sep 2017 17:00:08 +0000 diff --git a/contrib/debian/compat b/contrib/debian/compat deleted file mode 100644 index ec635144f..000000000 --- a/contrib/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/contrib/debian/control b/contrib/debian/control deleted file mode 100644 index 24c5f4c4b..000000000 --- a/contrib/debian/control +++ /dev/null @@ -1,25 +0,0 @@ -Source: netdata -Build-Depends: debhelper (>= 9), - dh-autoreconf, - dh-systemd (>= 1.5), - dpkg-dev (>= 1.13.19), - zlib1g-dev, - uuid-dev -Section: net -Priority: optional -Maintainer: Costa Tsaousis -Standards-Version: 3.9.6 -Homepage: https://github.com/firehol/netdata/wiki - -Package: netdata -Architecture: any -Depends: adduser, - libcap2-bin (>= 1:2.0), - lsb-base (>= 3.1-23.2), - ${misc:Depends}, - ${shlibs:Depends} -Description: real-time charts for system monitoring - Netdata is a daemon that collects data in realtime (per second) - and presents a web site to view and analyze them. The presentation - is also real-time and full of interactive charts that precisely - render all collected values. diff --git a/contrib/debian/control.wheezy b/contrib/debian/control.wheezy deleted file mode 100644 index 4103908a2..000000000 --- a/contrib/debian/control.wheezy +++ /dev/null @@ -1,25 +0,0 @@ -Source: netdata -Build-Depends: debhelper (>= 9), - dh-autoreconf, - pkg-config, - dpkg-dev (>= 1.13.19), - zlib1g-dev, - uuid-dev -Section: net -Priority: optional -Maintainer: Costa Tsaousis -Standards-Version: 3.9.6 -Homepage: https://github.com/firehol/netdata/wiki - -Package: netdata -Architecture: any -Depends: adduser, - libcap2-bin (>= 1:2.0), - lsb-base (>= 3.1-23.2), - ${misc:Depends}, - ${shlibs:Depends} -Description: real-time charts for system monitoring - Netdata is a daemon that collects data in realtime (per second) - and presents a web site to view and analyze them. The presentation - is also real-time and full of interactive charts that precisely - render all collected values. diff --git a/contrib/debian/copyright b/contrib/debian/copyright deleted file mode 100644 index 11a3d6390..000000000 --- a/contrib/debian/copyright +++ /dev/null @@ -1,10 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: Netdata -Upstream-Contact: Costa Tsaousis -Source: https://github.com/firehol/netdata - -Files: * -Copyright: 2014-2016, Costa Tsaousis -License: GPL-3+ - On Debian systems, the complete text of the GNU General Public - License version 3 can be found in /usr/share/common-licenses/GPL-3. diff --git a/contrib/debian/netdata.conf b/contrib/debian/netdata.conf deleted file mode 100644 index a963d80b7..000000000 --- a/contrib/debian/netdata.conf +++ /dev/null @@ -1,16 +0,0 @@ -# NetData Configuration - -# The current full configuration can be retrieved from the running -# server at the URL -# -# http://localhost:19999/netdata.conf -# -# for example: -# -# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf -# - -[global] - run as user = netdata - web files owner = root - web files group = netdata diff --git a/contrib/debian/netdata.default b/contrib/debian/netdata.default deleted file mode 100644 index 9e7f8ae6e..000000000 --- a/contrib/debian/netdata.default +++ /dev/null @@ -1,5 +0,0 @@ -# Extra arguments to pass to netdata -# -#EXTRA_OPTS="" -#uncomment following line if you are building a wheezy-package -#EXTRA_OPTS="-P /var/run/netdata.pid" diff --git a/contrib/debian/netdata.docs b/contrib/debian/netdata.docs deleted file mode 100644 index 56631abf1..000000000 --- a/contrib/debian/netdata.docs +++ /dev/null @@ -1 +0,0 @@ -ChangeLog diff --git a/contrib/debian/netdata.init b/contrib/debian/netdata.init deleted file mode 100755 index c1b2b74de..000000000 --- a/contrib/debian/netdata.init +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# Start/stop the netdata daemon. -# -### BEGIN INIT INFO -# Provides: netdata -# Required-Start: $remote_fs -# Required-Stop: $remote_fs -# Should-Start: $network -# Should-Stop: $network -# Default-Start: 2 3 4 5 -# Default-Stop: -# Short-Description: Real-time charts for system monitoring -# Description: Netdata is a daemon that collects data in realtime (per second) -# and presents a web site to view and analyze them. The presentation -# is also real-time and full of interactive charts that precisely -# render all collected values. -### END INIT INFO - -PATH=/bin:/usr/bin:/sbin:/usr/sbin -DESC="netdata daemon" -NAME=netdata -DAEMON=/usr/sbin/netdata -PIDFILE=/var/run/netdata/netdata.pid -SCRIPTNAME=/etc/init.d/"$NAME" - -test -f $DAEMON || exit 0 - -. /lib/lsb/init-functions - -[ -r /etc/default/netdata ] && . /etc/default/netdata - -case "$1" in -start) log_daemon_msg "Starting real-time system monitoring" "netdata" - start_daemon -p $PIDFILE $DAEMON $EXTRA_OPTS - log_end_msg $? - ;; -stop) log_daemon_msg "Stopping real-time system monitoring" "netdata" - killproc -p $PIDFILE $DAEMON - RETVAL=$? - [ $RETVAL -eq 0 ] && [ -e "$PIDFILE" ] && rm -f $PIDFILE - log_end_msg $RETVAL - # wait for plugins to exit - sleep 1 - ;; -restart|force-reload) log_daemon_msg "Restarting real-time system monitoring" "netdata" - $0 stop - $0 start - ;; -status) - status_of_proc -p $PIDFILE $DAEMON $NAME && exit 0 || exit $? - ;; -*) log_action_msg "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" - exit 2 - ;; -esac -exit 0 diff --git a/contrib/debian/netdata.install b/contrib/debian/netdata.install deleted file mode 100644 index 45d42b635..000000000 --- a/contrib/debian/netdata.install +++ /dev/null @@ -1 +0,0 @@ -debian/netdata.conf /etc/netdata/ diff --git a/contrib/debian/netdata.lintian-overrides b/contrib/debian/netdata.lintian-overrides deleted file mode 100644 index 45b2d868f..000000000 --- a/contrib/debian/netdata.lintian-overrides +++ /dev/null @@ -1,16 +0,0 @@ - -# See Debian policy 10.9. apps.plugin has extra capabilities, so don't let -# normal users run it. -netdata: non-standard-executable-perm usr/lib/*/netdata/plugins.d/apps.plugin 0754 != 0755 - - -# FontAwesome is at least in the fonts-font-awesome package, but this is -# not available in wheezy. glyphicons-halflings-regular isn't currently in -# a Debian package. Therefore don't complain about shipping them with netdata -# for the time being. -netdata: duplicate-font-file usr/share/netdata/fonts/* -netdata: font-in-non-font-package usr/share/netdata/fonts/* - -# Files here are marked as conffiles so that local updates to the html files -# isn't clobbered on upgrade. -netdata: non-etc-file-marked-as-conffile var/lib/netdata/www/* diff --git a/contrib/debian/netdata.postinst.in b/contrib/debian/netdata.postinst.in deleted file mode 100644 index 29615f541..000000000 --- a/contrib/debian/netdata.postinst.in +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/sh - -set -e - -case "$1" in - configure) - if [ -z "$2" ]; then - if ! getent group netdata >/dev/null; then - addgroup --quiet --system netdata - fi - - if ! getent passwd netdata >/dev/null; then - adduser --quiet --system --ingroup netdata --home /var/lib/netdata --no-create-home netdata - fi - - if ! dpkg-statoverride --list /var/lib/netdata >/dev/null 2>&1; then - dpkg-statoverride --update --add netdata netdata 0755 /var/lib/netdata - fi - - if ! dpkg-statoverride --list /var/lib/netdata/www >/dev/null 2>&1; then - dpkg-statoverride --update --add root netdata 0755 /var/lib/netdata/www - fi - - if ! dpkg-statoverride --list /var/cache/netdata >/dev/null 2>&1; then - dpkg-statoverride --update --add netdata netdata 0755 /var/cache/netdata - fi - - fi - - dpkg-statoverride --update --add --force root netdata 0775 /var/lib/netdata/registry - chown -R root:netdata /usr/share/netdata/* - chown -R root:netdata /usr/lib/@DEB_HOST_MULTIARCH@/netdata/plugins.d - setcap cap_dac_read_search,cap_sys_ptrace+ep /usr/lib/@DEB_HOST_MULTIARCH@/netdata/plugins.d/apps.plugin - -#PERMS# - ;; -esac - -#DEBHELPER# - -exit 0 diff --git a/contrib/debian/netdata.postrm b/contrib/debian/netdata.postrm deleted file mode 100644 index 4ab4eeadd..000000000 --- a/contrib/debian/netdata.postrm +++ /dev/null @@ -1,43 +0,0 @@ -#! /bin/sh - -set -e - -case "$1" in - remove) - ;; - - purge) - if dpkg-statoverride --list | grep -qw /var/cache/netdata; then - dpkg-statoverride --remove /var/cache/netdata - fi - - if dpkg-statoverride --list | grep -qw /var/lib/netdata/www; then - dpkg-statoverride --remove /var/lib/netdata/www - fi - - if dpkg-statoverride --list | grep -qw /var/lib/netdata; then - dpkg-statoverride --remove /var/lib/netdata - fi - - if getent passwd netdata >/dev/null; then - if [ -x /usr/sbin/deluser ]; then - deluser --quiet --system netdata || echo "Unable to remove netdata user" - fi - fi - - if getent group netdata >/dev/null; then - if [ -x /usr/sbin/delgroup ]; then - delgroup --quiet --system netdata || echo "Unable to remove netdata group" - fi - fi - - ;; - - *) - ;; -esac - -#DEBHELPER# - -exit 0 - diff --git a/contrib/debian/netdata.service b/contrib/debian/netdata.service deleted file mode 100644 index e5d3a3863..000000000 --- a/contrib/debian/netdata.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=netdata real-time system monitoring -After=network.target - -[Service] -Type=simple -EnvironmentFile=-/etc/default/netdata -ExecStart=/usr/sbin/netdata -D $EXTRA_OPTS -TimeoutStopSec=30 -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/contrib/debian/rules b/contrib/debian/rules deleted file mode 100755 index ec4ec4182..000000000 --- a/contrib/debian/rules +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/make -f - -# Find the arch we are building for, as this determines -# the location of plugins in /usr/lib -DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) -TOP = $(CURDIR)/debian/netdata - -%: - # For jessie and beyond - # - dh $@ --with autoreconf,systemd - - # For wheezy or other non-systemd distributions use the following. You - # should also see contrib/README.md which gives details of updates to - # make to debian/control. - # - #dh $@ --with autoreconf - -override_dh_auto_configure: - dh_auto_configure -- --with-math --with-webdir=/var/lib/netdata/www - -debian/%.postinst: debian/%.postinst.in - sed 's/@DEB_HOST_MULTIARCH@/$(DEB_HOST_MULTIARCH)/g' $< > $@ - -override_dh_install: debian/netdata.postinst - dh_install - - # Remove unneeded .keep files - # - find "$(TOP)" -name .keep -exec rm '{}' ';' - - # Move files that local user shouldn't be editing to /usr/share/netdata - # - mkdir -p "$(TOP)/usr/share/netdata" - for D in $$(find "$(TOP)/var/lib/netdata/www/" -maxdepth 1 -type d -printf '%f '); do \ - echo Relocating $$D; \ - mv "$(TOP)/var/lib/netdata/www/$$D" "$(TOP)/usr/share/netdata/$$D"; \ - ln -s "/usr/share/netdata/$$D" "$(TOP)/var/lib/netdata/www/$$D"; \ - done - - # Update postinst to set correct group for www files on installation. - # Should probably be dpkg-statoverride really, but that gets *really* - # messy. We also set all web files in /var as conffiles so an upgrade - # doesn't splat them. - # - for D in $$(find "$(TOP)/var/lib/netdata/www/" -maxdepth 1 -type f -printf '%f '); do \ - echo Updating postinst for $$D; \ - sed -i "s/^#PERMS#/chgrp netdata \/var\/lib\/netdata\/www\/$$D\n#PERMS#/g" \ - $(CURDIR)/debian/netdata.postinst; \ - echo "/var/lib/netdata/www/$$D" >> $(CURDIR)/debian/netdata.conffiles; \ - done - sed -i "/^#PERMS#/d" $(CURDIR)/debian/netdata.postinst - -override_dh_installdocs: - dh_installdocs - - # Docs should not be under /usr/lib - # - mv $(TOP)/usr/lib/$(DEB_HOST_MULTIARCH)/netdata/plugins.d/README.md \ - $(TOP)/usr/share/doc/netdata/README.plugins.md - mv $(TOP)/usr/lib/$(DEB_HOST_MULTIARCH)/netdata/charts.d/README.md \ - $(TOP)/usr/share/doc/netdata/README.charts.md - - # This doc is currently empty, so no point installing it. - # - rm $(TOP)/usr/lib/$(DEB_HOST_MULTIARCH)/netdata/node.d/README.md - -override_dh_fixperms: - dh_fixperms - - # apps.plugin should only be runnable by the netdata user. It will be - # given extra capabilities in the postinst script. - # - chmod 0754 $(TOP)/usr/lib/$(DEB_HOST_MULTIARCH)/netdata/plugins.d/apps.plugin - -override_dh_installlogrotate: - cp system/netdata.logrotate debian/netdata.logrotate - dh_installlogrotate - -override_dh_clean: - dh_clean - - # Tidy up copied/generated files - # - -[ -r $(CURDIR)/debian/netdata.logrotate ] && rm $(CURDIR)/debian/netdata.logrotate - -[ -r $(CURDIR)/debian/netdata.postinst ] && rm $(CURDIR)/debian/netdata.postinst - -[ -r $(CURDIR)/debian/netdata.conffiles ] && rm $(CURDIR)/debian/netdata.conffiles diff --git a/contrib/debian/source/format b/contrib/debian/source/format deleted file mode 100644 index 89ae9db8f..000000000 --- a/contrib/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/diagrams/netdata-overview.xml b/diagrams/netdata-overview.xml index d06950b47..fa7b64796 100644 --- a/diagrams/netdata-overview.xml +++ b/diagrams/netdata-overview.xml @@ -1 +1 @@ -7X1Zk9s2uvZvORddlVywC/ty6SWdpGo84xN7vvnmKgWCYDcTSVRIyXbn1x+AItVcIIlSQ0urbVfZEiVxwfvg3Zcb/G767edCzR8+5ImZ3CCQfLvB728QQoAD+5878rg6AiFAqyP3RZbUx54OfMr+NvXB+of3yywxZeeLizyfLLJ596DOZzOjF51jqijyr92vpfmke9W5ujeDA5+0mgyP/idLFg/1UQzA0we/mOz+YdE8cvNJrPSf90W+nNUXvEE4rf6sPp6q5mT198sHleRfW4fwTzf4XZHni9Wr6bd3ZuJWt1m31e/uNny6vvHCzBaeH/y7NMW/4j/cmiEwUbElXH2b7meTbPbn6v3DYuGW+o37Ibq7zxYPy/hW51P7Js0K85Dbi97NzCJRC2VfxZM8tv9NVbkwhX1h6ZLeJvbF/HHxkM+ql4VZFLl93MLcuo9XV2zum8r1s6+fqVw8NvSwP5u7l9m0Itzb6v835XxFe2CPqOZNmn0z9qRvv5hikVmK/sM95Me8zBZZPrOfx/liYZ/j6QtvJtm9+2CRz5sz23fdBdCTfJncrpZhaZfQPsDCLnC9IqoszaK0LxBjCANSvaJIMIrtS4GFkoSwiOBYRBAaHklq4gghLXDMqOBM385n9/biQ+LV9HT3ar61DtUr9bPJp3ZZH+1X6k/ZaiURaLYeRWx14GsLyLI+9tDCMKlBoOq9c78+9xN8HI1X5GnetgB1TIB9zf7Mng5EaV5Ev+afexhie2FoJ0Cqu3+73s/v8kleVKdqdvQmDA2wOIRrUJgB5mAGOOIi0W2YKZBGGCKUEKCQ5HE4mEHYMLAGZ0LCAc4EGsJMsMuH2dSUpaVPnBVJNMsXWWpp7NBR9gAnDIxjGccpUYTyOCLwtSKQrxDIMUWghUABIIio5Tea0zQ2MiACKYYdAOIGbC382W95+BwJAcCB5BX1pb6oydK0wdgCwNeHbGE+zZV2779a3ckuxcNi6rAL7cvSCsg/TYvK7/h7+JMjVppNJs3xWT5z4LkvVJLZ5Wt//d2d/eO+bqlWK1UQOzjU4ND2+1Y+47eJKh8cLqrLzk2R2RUwhbuzzNGnAkju3i3c6mL3tkJg9Rv3bq22uDf3EwuM+rUFSqabL1XP0yhQuL7uR7Ww15pVF0cAhkEDZh0wUDQEA4QcetDQHNwDDkdiPovCmLYS9aQ74fnDPJ1Pd3Afvh/3eSE6VPVSAJYkWJm2cAM6iYym0nAjcRKLgKylJ9oI4UM0Ie4BEz6ubGOTRb277ev7RfW0q2Nxc+DNxw/rY8X64Hw+qUWYPeUHe0uZLpuvOfW9fzqHUM9VkuxLc6hcqEWZuJUu8kWuHZTXP219zfPLHz6/+2h/Z5kWAv9+//HHTT8MIcibu9y6b/B2vdEesGanWTG9agdVSLbv92XnTAsTpz12bo8nyohU9/g28fHtbUw/APCF7CJfCjKUqcAjU9fa4HGRP8DpCqQtarG/lnnzQVRWS/nGSRk2//b0YXOWT5bjqqmTeNth79shLai+t5CLlUUIAr+Z1kY7Hqxbl4n+Zcn7JTNfuyDHKWQsJRhCSBXEkRBbMb4vkgWKMWMeJFMjEjIGySFskLXTpYYrY8ADV48NIvkF2yBb1IAvqphl5UOklX4w2wnO93OoXI8pAmGlL2iOgFBtfQFzEkGSImgESTEM6HOBsGeLQALpAInCwzebYy8NiKnKJihWsx0YfK3mMEQrDMYcJaSNQaRRFDMYawYFjw0LiUHIuxhEcOj4kx4McnEK0e3RCQ+X5sQnzdO5leS388nyPpsdINB36q3NoXLucH/oXdrt9TUv/qwWyUJJP7auvzrzWW5rms+yRV50daEx9xNCnakoF32sKbeNo6wMsUPVmLZSgobKi+UgSGufupOwmFEWZqMSCm4xxkQyYEUS7e5ZAIZOVNgExdqblp1E3z7BprUyJptPs4vft5XeOi+ySs1/cNj9vnHvni4e/frxw6877Gz0LBvkIjYvRPKiN2yb8J9zd7lZvnhw2ttWbLSAsabxqa1XnM+rEEsV27P3HJVmljhkzfJkh60DIdqKrHAeHMk4Vh6710Br+fKT2b207+32+CepB4RHd9OEgMe/S0f1f9aR3q/2V9HHwt3mg1mWO4AAtgPhpbrARWVOSG6AaUfXpFQwSkGsU21t3UTjcOaEBXoHYoR6PIHEY03QSw7vbrFozbdsuh1cfLvu+VKxJVfYEpgb0caWRZVVt0nMEsHTWNOApirom6q4cdK10MU98RV6ZFP1WOBK8i9GW7VtK77YdvXoheLL6UIWXzGDIm7npgiO0yjGSGEmJIhxGtIVAmgfX0N3HAS+HKgjB/CeBbCNSXZOQ1q9UFOTVNl1t9NkB9r242ZX5JurhKlz2UrS8Q/rGEWpITpJrD6ZmpD+YYC6Chu0kBwAsom1dfzDQXJVTs/w1Hx3hILRV4rAinYCYpoQ2ha5gkAZKbdOiaYKJSBgRgPqMUQ6jJTBhvl1nMNBAhSDZKmBX6/eH89IngqwSrxn3K9d6kcw7ncvyfPzyQIsCesvCTjnkpBLWBK7ly9pTfAlrAmB/TVpHGVnWRN0EWsC+msCz7km8BLWBMvB3jnnmoCLWBPRWxMgz7gm+CIkMYP9JfEYcCdbkouQxKhfcAHkML5/tDWBzxcz/YANdX99TnZW/QkEJNJdNUGHsppSj6sJ4v1XbVzi7u5In31mX6SvSdsFyTrhcGy27rhY49f6+d09zPJiqibDu9g77Et9z/JDXtyrmf1S4jDmbukhL52xZJcU6AdVuNdAzdzHSTY1s9KFhn7c8cAnjHx+MNO8eIx+M38t7ZfsDS56oQnUguWzYp2ieV//2reVUsM2xD65jAE4DgPyZVty7uE/8iRZ8R60Py87eJDxsEpRKJ2DY+/M+T4Gj3KD5psrrHG7FtT3al+9+fjrAff7nJv15ilYkLlfp46tLFb7PP/i3EZgns1NuefyhdjCP9WrVSce9UOLofavJ1p8GtE3RvIJz3Y9RPKN8fn4jPc9kSVGbYNs1mwDpqaOMrO4nHfBNNgG6wMrjACdTyb2UVYFM6ffQt79/oOLhTu6JEmUzyaO1EprU7pdvnDJHgvncl1rByOk5WBPXbg214M0Zz4NmHkwzfdP+A+lzI3k3f/IZstvYVD2vIQyzwa4hDy3H6YqW2HeRRqKXP+4532FEBm/zraJjO3xjBMYzb26RcE8fklfWD1MbtqA4fs8CUfIK/Xur7vCmLef3u9mgefKJ734bVY+lnoxOWSXnRT0AvPdoD+WV4SfEeEflP7Xp+/4vnZ8S3pGfEsPvnsr0M6WbfeEWHeYqDKT+lmw/UD8NEsSd8Z1gu70271rLXa7auaFVv+707plALfUXa/qNwblLeUVUooaZ65DRuGW5u0sX+iH+hb2Txf31FqnaSJD6bIYd5OUBB6mhPi8KQdYZ0Nv7mZbrEK7Z3c5+z2q6ei2V03KjYqtbzMF5oBr9Qw8WXxPjo9DGOOejGnEMk1MuvBwoWI5m1UlG46MznArjErK6knKzKW7g/rhtnZHaG3Dyr3S4T2Fseun4uoLbgPM88w5Ke236dsb+t7tyuUiL2svxZhNusqWKVdNYj67N+8jcjPMeQ8h1Uk370DiIddb74ROAl94r31IUe737DddKi5Vlv+wTOYu30vPz2F0+Xp4wGfZWpvab2xu2BEA0hDwAyX5+uClNtfwgmYVxbk9BNbhARxbbeRJOJwDxT+bmSnUJPp1luZRFD2tTidotL1gfk+X3Ekq4mg/Ui+OF6kPExUKjPMm+fpimXd1g3+UFwX/ZtE64H9eyPQc4Ef9tDcgfB7pK0b/U873xeJ/dYtngv/mHPkO9J8XbTwH9DEfQP94WWvP0WYKM80XLYvKLa0p7INubYDXQsBTtg0o2u293lVEWncQA0/5OVVCi/1gssim6ySdp258ddy7aoN39irrwmiTffHXWaPtjfFeQhCccXjLpT0lIAJy3lNWfNmnBHkwK44TIfE6kM/jYPtjOZ3/OnvTdrNBapfupvazIbbys1Xf/8Uod2VKPWT2OeI2dYnb3FfO43yTPHEmVBittce9JPf4F4DPFgvhVkXiwsmO1u5VjJ5D9kk2/2WDkBLa+IVULCihgVLWIOiFROVomxsEoDPTjFmgI0aoohrAqPFYnSVgpF1d3PxAz+h5gyC8l/ghPQ0vhK9efP/MwxFEHFFJ8T0qciilwfmiIkNKo++UPhqlRT+weVZKjyhv+E7pAykt+47vs1J6RA3pd0ofrG/JSyL1iLqtl0BqcUuwj9o6QcJL7abow0ttRTufvW9d2KhyEQYI/eQGTs6JgxEljd9xcBQcMHhJOLiSdCYIXh4QIOCoh4RhQ7mTIWFM4fdLQAK9RXIIhPO718gI99rpiH0l9rq4UGL3+6GMJfYhzUjPH/BU83l58c2qXX1MXRdWxR8vIfP6rI2q31iqjWowj7aPTXkJPapRv8fKiXtUj+DIV+JXu1Txi+H5xO+gecqV0PpSpW+/o9Ixpe9uYm8uiHqe8BgI8//NP+2Qszv63AdKLlpo+5l280/NQYlFZy0Q9YbRPFgJEUYb9N66nBj4VcoA2K+gOJ4MOH172elj+ddkuyJHr7Obtt2JdwIkMibdOdspTyNhiNZaqoTAgN20WeOyW/cuZmioTjZ9Qrt1Cy+0e3H519LCaju89huv+FLgtZq1bCjFoj1rWTACIvv0SuOEg5SGbNY+GAZAfG0yGxMmfG/i08OrMEm2Y5DJyhK7OnixCl6pBCjtNL52YxFZgklKklSrkGMRERBddGFfs17pa4pxyaMAto2Kz8vFfWF2AQztBbDrab7uAlIOgyxmuINBpliUiBTomDKQ6oDN19dzctYSFA1rpBD2jeaULxODxhpEltalUYVVkrcCkV8hp5NVEZx9mcSaW1gRbPnbCmXGqm0JBEkMRYxSHnDIBJE9TkfwMMK2Rt4zm2NdBMrc7EhnRO2aCsZfLa/jjtdBIFPKOrOdUmJfEYNTLBONQ6pzognrNiikntFOkPtG77xQbc6ip8i/Pe6A4PYCr+uFYNWuTUBEiZLdWSfMTWPnRCksaMqScBDsJ4tbRjgsylyrd9eg8mXzdBcLJK8Vf3CFPwFp12GieRxJmGodC80kDGhy9Jys7sJDDog89uwBbtaLgF/+ZT773XXpWJa/T/L7HUjcb+rTi9D3rKSVNZuDKew4TlKpI0mFNTENJLGRASd09osZsae2CWHhkbQvFGfOsrVE34Gv7TWrLxJf0j6cfZkgps3Kc9LYEzJOVAQSRmViJEoJDIgv1NPk1nkF7WiRR5E7dn33seBlianVjhmdfL+owhXJ0RWDk5qguCtHkzTSUiJoEg5RzMMBsJ/FYPW4YfIw5J6pieyFGrSz+2y2g72x/djbFQGwinxBrDGKUduWpdoaEhQrRU2aJNQE9Nv1BzNh7unu4qs7PaAL3ekAuHGM7FNPqrt87kauzMtqlux2RIordOBZuFWhCshRbDBr87tY00jF3MnHlFs4hoyE9Q1X6htb7PWdwOPi7Sj16I7rmFlySEF6K43mw7rryptCP1SdTI6ZSFofiOL1zW/dG8/KJt3cGGVTK5V+J85BbkwAmPLeLFnqa52BPUyRH5kpnqpb55uJKqbPb6Pwi+sP5JoSfmglRoeA6EN14mjaO+0mjMpn9Rzc3DJzU5PNU2CU9roInBijO5PfZL2FWslv71etqX4zybJul46eR4h1otmm1LRTEKI/M8xLCO6LSYWvDQksvn75/PnjTX820G5WEGKH//bTp8+RvXD0Be7Y28/qK3eZkOopSdSbwxqmM/T5daSbwQgrMO7QP/NFltbt3/aTVSEAqpyIjGbde9gK1O3xiRcphBjpOmgJ8cQBfGGoMPH43ULoctojeLuQ3TQtyFxN86D/2OYSqk1FV32at7uT+TK8Q0CgFwsi0pN85m0zF7Q28vSGj3ydbipZ9Qy8o0hhYKWQc9ST2m9ANYtiITCCisiEBXTUi6avQQ0y5JmKh7zxRnFciXgG2PG9YPdSfFFVVjYkaQKVars+IaIRNVRjzThXJmAaj2xKNGtMYeHR25lHb6eXnMVzGKZea8jHVW5a2Gkk00R3XKCpiVJtWVkCYcxRwNQdCGQPdw3NO7kTvk6P18fL9qswuR7coSpDGzIgU9PNpcAgAmmcJMh+h8YioAjtww57YNd8p2NUBqlw2qmqr3XE16qrh6BxL5yMPa4ob1eEdWZ+yPLXdUHTd4o+Q1r0vYueDCjhMbAh3J+ilyQaIHit6cR4FZY1UHDYyQIgsYwwNwrqlGmsAqZzDjqXITasq4DYY18RdiLhcDlTBEK3ND1/HT3tqQbMM4nwVL1UILiSpgng1jeG7+y0Fk2hwEXQ+oV2q6x7Rl70ppa9zC7Gh5mF3nrMcF7ZtQpwqvEPhbnPyupej9e17nmt9aaP0dOMpqmZLXffavD8nuljo4q5G4iyhZnu0MXg9vy3F5nj04+x+kwlb1MpfiKVB16O9bT4mv9HPb7psUjUsEgEb8PE0ts8tsyXoUJUshs8oGRIauahND2GyIPXO4Xh/DIPo24fH0pP14B5SOnrncJwfkpTQc9GaSoMjGMZxylRhPK4bqj68il9qf0fWZd/S0+JwtHaPw6I7Z1v1iN2Qx29LCaPbwvn8HLq2w49qfJLVVjwa01vJKTenvi1+8pDhbu7O3Qnx2hURxK9/dHZ0JMdwpv90/FIHzE58tM/P3zcZgu0NO5qeeZFVprRhoJVrr/mxZ9H0eA31rU0w6bvytl0XlW03E6TrlY/RPL2rGh7IJuXbWZTOTR34/gIqWo+aIM7eXcXyBbooRT5GmhRL4s5jTFwQbO7rnyaZr8+CXr8YxD7oBDCXBjuUTQg/INd00nV+b4HAPuIi55EMWX2t4qrL7gFmufZbFHdHn17Q987Ki0XeVkTZQw8ViGR0u79bHb/2b15H/ksuRCkGIxTYWKU3D+O2B/uwHKRF25i9jo1efWvRySkefFVFcnIJslXRkcJu+obk8NW/T7zO2iX3nYj7JomhZOyQFnl98HJ0rFNtXtOzWM46AKMNR/id3v2eThhLxnHyuP0MdAaEnyMsA/CO3qtoSHwZPdtEOnHqY8aUmREbro1IL6sLQIzS2qx+97qgfrPm41zOchKcy0e/79jGLeC8ObAf6sDqEpjqA58NEVmH88tfMVayoUqFs1lcntvzbG7bDJ5upHWu7V+8QSH/9Rr7LNWGh/vADjv+Hv4U6OwfFQLe0+z6qwI1BetH89la5hv2aJ+uPrdfzvv+o+1EVPW1im08WtdENYUsc9/bxab6Fh7Hkxyb7Zis4U8H/CaY4WZ2K3/xXRu0wfG+gofnTBoQR/1CgMZ7J1j9cj1z54wPTwT7k09YrJXtbFal8GZLHzUY+trtbjacsv9C9XvN95Zv+ZO8D1/YPlB5xf2xequn7b3mnKnT5UpJ0r/ua22aAjC/bLCryVbRmJnut2ZWMeaJlUxAquzZQBlUZIKaYCGwDAVsvlZF0trJbRT9ORpSnWAhXDyBN75snyIl5OJWeyHv1ebybvqPyogNmknWwtzGMkk1WkMU8TTgJm8pDcOB+OhQo2or2XLJTdFq/GXZKXOi2Qv8JHrbM8iKmRJqpAg7RxxKlBkJDEUpyaGOmQ7qmbsRcPZoIez+bJNXwRns7QpkuXicT9soWvEFl61rFXYYNwutxJSgYhiLRMrOFOUBMwxxbxnkTU9Vtpcq4mZnrLxTwhsLczE3Bdqaz34EFr7ZTi/CGhZhcx10bMGJoG8Eoia19ACmEVSKWqBZQADNCC0ZLeNo08h87Uwa3520cj6mk2yfD9c7dcI+UXgyrKsVZdjDVIZt7uVSSBExLllWDSlAJOQXY5ZN50EAU9WvM91SYJU6u12ITVi91LDQuuY0InbG4Sb6c54N13aa+t5uxvIY0QhmnB6i+D/zZcDmgeMIExMujgsfoBA876+MRiAHrAJiK011GH7QHKqmBAdkeHV57YtmvjXNIRpPIKlh+XNq+p9rZSBHd6s3UwOGjMZQyORimvePDq35d27u7sqyO9JAFh/1sfcUZp89RJZWKPNtnHn60O+vyQYgbsRkYPNvH5j8GaWz0yDrxbwBjJhTy2gLzGmpVbmdoU0Nc9+n6qZheK0ok2f/gBw+Z6diMaNy6RJKfQUSPsKJo4Qp6wiip4SA8fXo/rRXY1BUd3F7knCezeF9NYuDCOfAYorDn/KMpvOJ+bWfsGkqRP9X6o36qspLQH+56jXHqxwl40+VXncZvnJF6Yb7X72OOqiOfLDux/tt9/l88f6wgBVgy33Ggh9QU/4Li+rMpzPpcqXZVa+3Cf5YcetP+/sulonAhb1Ot3eF4eMAd9kCfc2y1bTlm7vuXU5em8AcSRZL2TJALlljHEiAGGCe+Yr+mxRKE6Uz0B9Jdp7yh1vVd+DfRiHH/PFuBtIjVosq8Qrq42655ol7l/XGXthb3f1UZ7eVIlZXS49lGA+juyTap6d38b94qG66NzMojqhAL1zm96oSbTIpvXbuSnSvLD6jvu8vu+HpkVyu5fx3nvr8rLHQu0CLPuB+wHsqdiSTRFW8R5Rm97K0qkV6hYpOrky/jyXzXkxntyaset9IbkpuB8CkL0mJaNTUwb5HINTbchNudmZ2jECBiPqfQ63vzaZQMe2y9TflnXervjR7y1udCLjC/YKuuTQ10p81tf+Cfi7CcxG+FVfh2NnNZFGSxqjTgwaiCRiFCogjTRaqxfr2GE9xw5hQwHja3zT1CyFxd2Iku/LdezkaZppc+vgVa7+HdIdAY7eipPxlA5puWfUTNO/6JkB5xGkRSeUGRS4v6cjuaX1F0f01f+/p6qYDm/q/RtiFfwTkR72jCfuqehqtJGOODnKth5RvfU6xMmq7WVCNO9OqqVcRABoaRIGNILyxYqTQb8+xod6jNeLHKAB4xB44wIF1w88UgEPcapcY99WgCpBMMKAqVgQkvL45QaoZDdVQYJhtyfmUWOOw+9G2EffzeTWFXrWba8cmdNbyiGBEGLk5i93TzfeZO6rQ1tPe0Tzmfusq811dsB55oHy+fPaPjNg2UXltxu69irf4qPLkwDZrFyo1e25tXEzqMDqvf28cBB9LBdmumI44zuDdUbTRO40y/k8LxwvGt6187ZFiVnU6Sp+Z2TnjCtX4eBEf5sit8ddtX92vyzU2NOleVE5HMvKfTpfFxuWa69kswqblmDwvJn3SVv+z01kMZbFuZ01rWYcNmeq6FJTxO3/OK/KZy357D9/LKsbh1Fp7KO7wz/ofDJplhMs3KJ8ycql5bN/Vw/240iq+Z8iddlEGx9gtZgOnqmr0gSLh3xZrlZxup7bODcVtur7fdfAb2aM64O7fqIbtwvBu4//vqk92O60pYX3xFR0LsZ3ofA9XnVX/TFKvYc1TqPPtu6kzq0/PfiHt81d//bmwxpJM0eMJCv/tP/9as/1L/fJYr3rvj5k1bNV5yyWs82QG/WAkzJvLW8V0tvxuNPYJEnlHPdunHc+QJhvCzc6duuPxqx1J16we9v+8PPHfzhkW8n+9sd91qkXxx3909vb25qIa168ItFq1359MIXZwiiPF5g4PDgHRRh1q+/JZp48NB4qHHf6Kd1fTfz7JO9N1USvtcaLVF5Quxu1/aBtPRhBI6hRarWbRGGGA3bkFl19Hq7jYJ1W/sgDsH6570VltW/BnFXm7vMk7mIO71fadT2Yw1X1l8VBIpKOxeo8JzhmiloBDGHIdHciUA9z1OM0Ax6PaQPWlwa5WZn04La9Jda1wk1aA9C+NBRQomJXEKZp7ZnTVEQ8YVAZppUOWWwoZDffEiI0lKHQ1xsuzJjTYdOtEd4yR8f5+IevjLOZvWSt5dzUceuNi8J7pUwQN86UTt8ST5ncAVPJTr/hslL/njzoeX/boVe57axmQWq/pN2BncR5iOOIGGZATBJqaLxx261Bu3vfNZDyNcLxAAofp4jJ28ouxMz697+8a3d17GZxXaIt8hQM21wgswd5G7Y61BMDNcI9PbsorZmdF/1JQa+z1YlVCWXFLJBGmoCOSkhphLggAsSKpyagjIZiULTtkdHUE1gg+4uj06mEG9uo6gdVLMpVR9Xlouqj2gXfjlYTx1YPMCV9eow0C5tq+yujB95vcunLqHWWVQ09jlMUa1CpBWnjcKAq4gYQZRUEofHmnd6oyCO2ejumdXLg/LPbLLnlP/z3x0+jHZWf83yy0XN8LmRu95ZdhAYC8W4NZA8kfesi5vGmzel2dPsMY9OdkT+puV7Oy8QnM/ZzKrwMHsVI1UIGYU0I0t3UUBhRHlMqjFQoBgEdVLznFPVqI77+REeaYducdp/8muvRR6vWewIRJiTsJgdDEakUwDhhXMcKhUMA6uujAgzZi2+IMd/fQzmk9gHZVFdEbVRRmzIUd4IggqY0wlDzlAMD7MYP2NdHsD61fVG2IbUbn9mzqM32p/ZLYd1Vz0KUxNTwDuvG3EQxSgxXSRwLHrJnYa+XO+Ry2MxdeHy9B0xbG5JyxPDYl0rKalgwkhomRLRJiTiNpCSaC6phnKYBuyL1pTCXQxc196S48gAprmREBd618mBZ9WO+i4kG9rVuN7vlFEYxY6lKCGfWPAzoAeK9KI3wUFt6VC4eoPNJ08T9NVLb7u2Vvy9lIGWdJo2JZdNKxQJb5UsKEjDtQDQVrmtqNx0Z29T27G22v/k2TBQ9oOrueqhNK3sKQxhL0qE2sZycAckMUDEHKGDfxEGHAR+1IfRI5QOCjUNyjyt2u1ZyV8YTRoamsJ3fIay9FEGpDUepxCgOyMrJQJv27G1PIPCAidBDYqPXTGyySuZJDNVxR+EmMI2U5Z6xEvYzKgPKbdKT2z4tzeeICyG36QEFb9dDbVoF9bGkypi2X0TGII4ETw1SmgoIdThqM0Z2U9unpR2QjTqk9qv2gtGqtAwrSUnSrWm0Onkaa2NZfSoTEtAC42xggQ0novoaHvMAfhFvm5/XQ222orZJqep4wRxjpwQbEHMsrUgPqJP397bEHi3N5wY7YN7DcZtZ1weqCWa/5p9XJ32qE2stw9WESWjlNsXU5U6CrjAgETCY0tQQFpOQaj3utlKxJvvQSQ6bZN8OYNDegPGESc6QOv5Y/jXpoold5UwQWrkEsE5VIjruPoVxBLCVMiDGUKuAqoXojW+EiHsSCH0tsOX+7GdM0G1kk5RrFUBihQBNNOvwE4v4yCQsoSnAMUxJyHlDPQEk5DAMw3zKZYCg28i+KddK7SpSg2MZG92pAqE6iSBPlcQJ5ipkFYhAXReg9ARqmmBOxwMYIFAzplGKZwpmp11Ar43AsHOAy3e5U9Ns4h7wFzP5YhyhNyTGdBqxuT+tvJt1+4Z9ehF05mvWnQ/a8zXdr7rzNZ9GXdKb1qjLbdRuD7VsNlB7hmVNqgtpdyBozw0JUG/O5NgeB7I3mU/y4wys7N+xrI2iA4dJDvfBCIt62EBjv21wYAOName0ZsvSm/Zk2RZcYRust5z2sd9s3v5w2VGgph5Mo4sCNe/71iE8ENQC9rcHG4fqQ4A3IsXlRANlVHuUjMVZhWS3JFDeUjfPuD3wZZYXjqZvZ/lCP9S30OvUk1AjEuIDvkAxZq4Ww9PDR/IEcB5GsmLUddswPFSkA5VqDAl7vdksrHK3EgpBNzFJShBHLNZAGmoUIwGd62u36dol4wul+LJZArQWZNeazSIxdIX2HMTGhbbb+Q2WqhHSwGjCtOYiYEZhP0wigafeEnp8qY2MfpYv9VVnFLLKNUYQ5xJ3R+ciFCUxMATSRCYgoCnbH+dlTVlPn7RjmbKvOHfJUrsyZYlQadrNVKNGRzC2HNRggggL6DnHjd9zG7V9SYcyBJt+1blLbJXNknIYi47jAhEQpYRgwoGSSgUUypQPqO0Ryh7PRYi97e1p93qoXSWzECufMWXdSgAYpQIxQLHUKUpCxkB7mWqceNzSvgSHEJ1W+av2QTO8Etwp17DNymWawkhxlkJt0gQjGDCmRegIcmPP7sY8ALnRqyY3WxlYMQdc9UMOjEqSxDIVMQyolbO+S5A3Nc2doJOn7jxEvQAf4X67YnLTmtwg1Z1kNUmp/QcQTBExImS3M057arnPCpM+IywEta/VNWJJ5VbBCjwkUuUYtYaNPR0nUQLsy8QkEomAchmCvm9EoJGc+oCWEbsGaY7tKAOFr6PMqgtuc65lczhpVZovPU01h51nnps389TnOFrdUxK5UQuZNr3eKHx7b5QXU3S+Pwfpx6hBo6K1Tb0wg6HHJCnwgyaFnSWCQxDsxHBuoYRjgzXVrz6aIrNr5tz8e4QlNWMiIYgRqqgGrgv/6kvtmE4TvruQmI7AXWehbIi6d0iH97RZ0B8nFC6kw1+xH+IsFXMc9ON1ZKjMUE8NFQmgzIgD/BAvQpk5S5xnoJcCPJQq1OMuPqB33pCUB/gYXgQpzxLnGViUoFnOHXlNISakNLm2L0AbGORzhEvaGCXyGwZ2KSK/N5ZSyp6+OD6Lo89KGlCEF/livzS6c8KNnxtvjXp0IXjr2zEH442LvkHUVy8C4u3lpKudHW+NVL8UvOG+WAQHpqlxgntnokdLUxM+l13H46PXQHly6mDDEpCYoZ/nabxJosy0M0Pp8rsI+1KVf2Lur7uffKFq/U5uNc33cPDBpgCjmdTMhkF1CMO4Wnb59zxtIe1izUa5/IjP5TeY0bOehOKdz9McW110z/E6Ye64NWML2Ju9qYZNpXkxVbPOWCHfLe6eCRTCe/n5wUSVFaL0xJRRnkYqsncarabFRnZ1o7SaAtX2ZIrtnsyvD9nCfLL7wr3/Wqh5dx/2cj/TNEVa+6RIwmJX0xkmAaHnHoLr4QWdgQaenXFA95QxTkgxwgl5omzeP5bT+a+zN+2cXkhvrfJVJ/Uitkrqrb7/i1HuypT2OB278Wf9XlymLxkkf/tiID4eSUN0X+gPXDykjdKFj4FNZvA2qwYfzhJT1DyoZjvuuPM6GHt0aqzyEqlZEsXLMpuZsozcx5b3WIlyB12t051zXBx77iskvm2UTrL5L7tUzD18HB3QkaGrCkoP5GDfxDjIw7EHxPSymDy+LSwyTCVht7PyCjxrO2E0gdYY8xDIUgfdyTEEanMbqwhX3CYAoURPV17nJLUHLXFPvPsAFWpAKnmAi/haAgKiSgC+ExTSODGddh2CqUgTljCgJINJQC+y7DdeIsOKSuiLTfbrfJ6hMFuDdpasZqi25qlu0+4m+SzJZ7fTx6hW6G6zvPqkraWtvBUbtbSrhZF0mt2djo0gxFT5raaGkRW9ERdSJpAgDVXT1qHPaUIoGV0zDAnf9CxPhhTsW/ihuzzsadWwUXbYGrW3f5QHGGMes2t0bshOs+vJf7DqB7/Rf7CnYfi8O1zNtp49PtmFDv/73Vpnr9PnWGRDwV398Qhu6v6OEc7h9xBpphC09tC6CWkny/A4/eTliFS0F1p+eRoTfD3GvvFNeeryjjwNSy9LJ8tacrbcLmjHulHeVSeO3vdOu8bOKx2LJd1AuTsj4wSKmLREsTRYRxxRTFAc60Sr44liyLv2/h6y+DhD9uTleH5eJiNhfVt62GDvWKXacoQp/Z1223ajpGcj3rom4Tv1DqRev0sCBcPw0pFF+Kf/97M9EKvk3uVXg7So5Lk7W6YDCfOfzcwUauEyut/W10EdGKFXKs1X1SdS4hhUqWHN6DuR0DjCxigIeRwndJNh3Y2V4kBmQq+jIqEe+c488h0FmYLp4TIjMny+c5mtbR+65aNnMBTCMBI1v62cDrflQ5+FvFLfHGNVk0wSgzSBnXpFA2HEiIk1xZwxHXBSlQC95Jc1UHaUn9Agkd8zdFw1U6204yM90LFXCjoOVqCDiqWd7PQYgwhLCIkEnCIcsM9v1Q22AzrPOETm8V4dMB5tD9DZRWtrSxYQ+Jtrr5g5f+RjviwcJHSRzReBdCkrXkz2xalSTX/p+uKR09wiK+0mk6hzxSe47tdz+orgiiq4JsgkqtONBQsaMZVwERuScK23q1n2LRooXaG8Khh2ZTQBnk6kYYqCRyld8FU3feCrpg8JSXuNhylEEU1iwLHhyMiAcw7WP3maczBMo2kyZjru+XCB0zAsykxVNoms1pullpIOA31OBLdHV15Yes6BEFt1HkjtFk46MjQxLGKWG1k5qpRal6s/pYaAvpFzP7FX6Rs8m7PL7Y1lujnPPpnmLdulTmvfJyFoU9ruMUJdvJka0fiofdzUN/0pSKjr9BpqoeI4W0z/6u+zVxomYXxVPplCwTttPSTC0r5VQCCqlEQhB46hXkLsOveok5Pm0VCbmsqLBF3N0+fL8iF3Ccxb2fprVTBBxcsBB9gk3RnDMokQ4ymD2EAacjw86wV/YTN0ruOl88zJOaAj2CiFER2QaXeVgp1UYECcqjjtdANMkKuSYioWhKQ8jo+dhou8Ps8A2JNd/w/3BIB9/uEDiotPL0nv89/Nt/kXVfR4W2dJXgtv44wK9+B3EhKUxKmKiBS6aSohUARFQplmiZAk5Px02asoQGgYFoO+VgT0qM6ewKib5YmpMJcW+Sxb9qUp2i8qdupuCA5CUfVRuRFUdtdQCZxX7M7VhAOOQES1MIhDETHedOgSwL7lVHLGjdScBRzwQlEPSpgM4+Ow6bfXaVDykkyBNZTKRWbsRczECpVZH1D4ygBFJaU4kjGkVCJiARUnjR/aqmGxs7FTBjTnIYMfsueHpmQYTIPY46nDQQDlXMB5vmh9/WcXofxgAeB+9H8= \ No newline at end of file +7X1Zk9pI2u5vORcV0X2hityXSy9d3R3x9YzP2HPmzFVHKpWqUjcgRlC2a379lykkSkolIKiEAqrtCBsEaMn3yXdfbvCH6fefKzV/+K3MzOQGgez7Df54gxAkhNn/3JGn1REh+OrAfVVkzZeeD3wu/muag6A5+lhkZtH74rIsJ8ti3j+oy9nM6GXvmKqq8lv/a3k56V91ru7N4MBnrSbDo/8qsuVDcxQD8PzBL6a4f2gujUD7Sar0n/dV+ThrLniDcF7/WX08Ve3Jmu8vHlRWfuscwj/d4A9VWS5Xr6bfP5iJW9x23Va/u9vw6frGKzNbBn7wz4Wp/p7+4dYMgYlKLd2a23Q/mxSzP1fvH5ZLt9Tv3A/R3X2xfHhMb3U5tW/yojIPpb3o3cwsM7VU9lU6KVP731QtlqayLyxd8tvMvXhQ1XJRvyznZrawBLx1H66u1941ziFjOcEQQqogTsTzSqyfcLF8aqlj12zuXhbTmozv6//fLeYrJAB7RLVv8uK7sRd5/9VUy8LS93/cI38qF8WyKGf287RcLu1TPX/h3aS4dx8sy3l7Zvuuvxx6Uj5mt6tFebQLah9oaZe7WR+1WJjlwr5AjCEMSP2KIsGQ3RR3AnKUGswSglORQGh4IlJNE5VyhCTKOcT6dj67txcfkrKhrrtX871zqFmpn005NcvqyX6l+VTIBmbNPoQUk9WBb8+wtrheHXvoIJo2P1TNTrpfn/sZTI7iK/K0bzvwOibclpUxXbjNn5YP5cyhDOvyUT9kaR9fEPG9ALUTLfWjvF9v9Q/lpKzqU7WbfROgBsAcYjeMOYezpP5osRF5z4DDEGAGOUw0Q4YBiBJFOV7BTaIUJhkWVELIKcvTBm71yd932desnJk4OCSA9XGIYQCHBAxxyNgZ43Aj23sG5F1l77a0AKtMgPFReY2MjmLH6LBQ0ioCHUYnqUkThLTAKaOCs4iMjjHSBxiRPMDo2BBghJ8xwL4VfxbPB5K8rJJfyy8ehtil87YDYQZqeQo44iLTXZgpkCcYIpQRoJDkaTyYQYg8gSokHOBMoCHMxDnzsQZmU7NYWPqkRZUls3JZ5JbGDh0LD3DCwDSVaZoTRShPEwLfKgL5CoEcUwS6Gh2AIKEISs1pnhoZEYEUwx4AcQu2Lp9DNMDnSAwADgwO0Vzqq5o8mi4YOwD49lAszee50u79N2sx2qV4WE4ddqF9ubAC8k/TofIH/hH+5IiVF5NJe7xRRu4rlRV2+bpf/3Bn/7ivW6o1tqRVaS0cGnBo+30rn/H7TC0eHC7qy85NVdgVMJW7s8LRpwZI6d4t3epi97ZGYP0b925trbk39xMLjOa1BUqh2y/Vz9Pajbi57ie1tNea1RdHAMZBA+6rVRQNwWCVPBiSenBvOJxemZ8/zPP5dAf34ftxnwvRoeqXArAsw8p0hRvQWWI0lYYbibNUxGMtxNPRCaZDNCEeABM+rmxjk2Wzu+3r+2X9tKtjaXvg3aff1seq9cH5fNKIMHvK3+wtFXrRfs2p7/7pHEIDV8mKr+2hxVItF5lb6apcltpBef3TztcCv/zhy4dP9neWaSHwz4+fftz0wxiCvL3LrfsGb9cb7YFivjArplfvoBrJ9v2+7JxpYdLcY+f2eKaMyLXHt0mIb29j+hGA73tJpAgYpyAgU9fa4HGRP8DpCqQdarH/PJbtB8miXsp3Tsqw+ffnD9uzfLYcV02dxNsO+9AO6UD1o4VcqixCEPiH6Wy048G6c5nk75a8XwvzrQ/ygSNRbMX4vkgWKMWMBZBMjcjIGCTHsEHWvuan1vYFAbgGbBB5zqbuFjXgq6pmxeIh0Uo/mO0E5/s5VK7HFIGw1hc0R0Corr6AOUkgyRE0guQYRvS5QOjZIpDAocYgAnyzPXZpQMxVMUGpmu3A4Fs1hyFaYTDlKCNdDCKNkpTBVDMoeGpYTAxC3sfg2snXwaAMYJCLU4jugE54uDQnIWmez60kv51PHu+L2QECfafe2h5azB3uD71Lu72+ldWf9SJZKOmnzvVXZ36V25qWs2JZVn1daMz9xFBnasolnxrKbeMoZHv4aoca01VK0FB5sRwEaR1SdzKWMsoiGZcU3GKMiWTAiiTa37MADJ2oEIaiQSfRt0+waa2MKebT4uz3ba23zquiVvMfHHb/2rh3zxdPfv3026877Gz0IhvkLDYvRPKsN2yX8F9Kd7lZuXxw2ttWbHSAsabxqa1XXM7rEEsd27P3nCzMLHPImpXZDlsHQrQVWfE8OJJxrAJ2r4HW8uUns3up7+0ehnghDYDw6G6aGPD458JR/W9NpPeb/VXyqXK3+WAeFzuAALYD4VJd4KI2JyQ3wHSja1IqmOQg1bm2tm6mcTxzwgK9BzFCQ2kqAWuCnnN4d4tFa74X0+3g4vulTl0KtuQKWwJzI7rYsqiy6jZJWSZ4nmoaEVttmGSdAxXw2/FAeIUe2VI9Fray8qvRVmvbCi+2XTu6UHg5VcjCK2VQpN3UFMFxnqQYKcyEBCnOY3pCgGdVYRCI34FQCtSR43cvAtjGHDunIK1eqKnJ6uS622m2A20Xnwd6KCBrWeo8tpL03MM6RUluiM4yq07mJqZ7GCAvoLxONO66h8kQj+2xs8TjFoan5rsDFIy+UQTWtBMQ04zQrsQVBMpEuXXKNFUoAxFzpZDHEOlQ4EISSH7nUeITg1ypgVuv2R8vyJ2KsErcs+3XHvUj2Pa7l+Tl6WQRloT5SwJec0nIOSyJ3cvntCb4HNaEQH9NWj/Zq6wJOos1Af6awNdcE3gOa4L9KiwIXnNNwFmsifDWBMhXXBN8FpKYQX9JAgbcyZbkLCQx8ustgByG94+2JvDlYsaP11D3N+RjZ/WfSEAi/VUTdCirKQ24miDef9XG5e3uDvTZZw4F+tqsXZCt8w3HJuuOCzV+a57f3cOsrKZqMryLvaO+NPQsP5TVvZrZL2UOY+6WHsqFM5bskoJVpbRb+pn7OCumrly6nC1+3PHAJwx8/mamZfWU/MP859F+yd7g0otMoA4sXxTqFO375tehrZQbtiH0yWUKwHEYUCjZkvMA/5EnSYoPoP1lycGDhIdVhkJdwr934ryPwaPcoPnu6mrcrgXNvdpX7z79esD9vuRmg2kKFmTu17ljK8vVPi+/OrcRmBdzs9hz+WJs4Z+a1WryjvzIYqz9GwgWn0b0jZF8IrBdD5F8Y3w+IeN9T2SJUdugmLXbgKmpo8wsXcz7YBpsg/WBFUaALicT+yirepnTb6Hgfv/BhcIdXbIsKWcTR2qltVm4Xb50uR5L53JdawcjpOVgT525NudBmrOQBswCmOb75/vHUuZG8u7/KWaP3+Og7GX5ZIENcA5pbj9MVbHCvIs0VKX+cc/7iiEyfp1tExnb4xknMJqpx/FZwC8ZCqvHSU0bMPyQJ+EIaaXB/XVXGfP+88fdLPC10knPfpstnhZ6OTlkl50U9ALz3aA/lleEvyLCf1P675//wve141vSV8S3DODbW4Fusmy3JcS6wUSdmeQnwfqB+GmRZe6M6/zc6fd710/xdtXCEK3+d6d1ywBuqbte3WQRylvKa6RUDc5cg4zKLc37WbnUD80t7J8tHii1zvNMxtJlMe4nKQk8TAkJeVMOsM6G3tzNtliN9sDucvZ70tDRba+GlBsV29BmiswB1+oZeLb4nh0fhzDGPRnTiGWamHwZ4ELV42xWV2w4MjrDrTIqW9RPsihctjtoHm5rc4TONqzdKz3eUxm7fiqtv+A2wLwsnJPSfpu+v6Ef3a58XJaLxksxZpOusmUWqx4xX9ybjwm5Gaa8x5DqpJ93IHGghxkO7A4S32sfU5SHPfttk4pzleU/PGZzl++l569hdIVaeMAX2Vqbum9s7tcRAdIQ8AMl+frgufbWCILmud/t3rCOD+DUaiPPwuE1UPyzmZlKTZJfZ3mZJMnz6vSCRtvr5fd0yZ2kII76kXpxvEh9nKhQZJy3yddny7zrG/xjcVbwbxetB/6XhUxfA/zIT3sDIuSRvmL0P+d8ny3+V7f4SvDfnCPfg/7Loo2vAX3MB9A/XtbaS7SZykzLZceicktrKvugW/vfdRDwnG0Dqm53rw81kdYNxMBzfk6d0GI/mCyL6TpJ57kZXxP3rrvgvXqRdWW0Kb6Gy6zR9r54lxAEZxzecmlPCYiAnHvKSij7lKAAZsVxIiRBB/LrONj+eJzOf52967rZILVLd9P42RBb+dnq7/9ilLsypQEyhxxxm5rEbW4rF3C+SZ45EyqO1upxL8kD/gUQssViuFWROHOyo7V7FaOXkH1SzH/ZIKSENmEhlQpKaKSUNQi8kKgcbXODCHRmmjELdMQIVVQDmLQeq1cJGGlXFzc/0DP6ukEQ7iV+yEC/CxGqF98/83AEEUdUUvwVFTmU0uD1oiJDSqO/KH00Sgs/sPmqlB5R3vAXpQ+ktPQd369K6RE1pH9R+mB9S54TqUfUbV0CqcUtwSFq6wyJILXboo8gtRXtffaxc2GjFss4QPCTGzh5TRyMKGn8CwdHwQGD54SDK0lnguDygAABRx4Shv3kToaEMYXfl4AEeovkEAiv714jI9xrpyP2ldjr4kyJ7fdDGUvsQ3qRvn7AU83ni7PvVe3qY5q6sDr+eA6Z16/ap/qdpdqo/vJo+9SUS2hRjfweKyduUT2CI1+JX+1cxS+Gryd+B81TroTW5yp9/Y5Kx5S+u4m9uSDqZcJjIMz/b/l5h5zd0eY+UnLRUtvPtBt/ag5KLHrVAtFgGC2AlRhhtEHvrfOJgV+lDIB+BcXxZMDp28tOnxb/mWxX5Oh1dtO2O/FOgEympD9mO+d5IgzRWkuVERixm7Yfe4frlO9eWTkKiJh2/tildS9e/OfRwmo7vPabrngp8FqNWjaUYtEdtSwYAYl9eqVxxkFOozZr93OoaaglZGvCxO9NfHp4VSYrdswxWVliVwcvVsMrlwDlvcbXbioiyzDJSZZrFXMqIgL+qIl2HGwXXUIM0XXsNOVjoWteLpb3ldkFMLQXwK6n+boLSDkMspThHgaZYkkmcqBTykCuIzZfX4/JWUvQ1vbqZpuGxjVdavt/Yw0iS+uFUZVVkrcCkV8hp5N1EZx9maWaW1gRbPnbCmXGqm0ZBFkKRYpyHnHIBPH1NBLo8Y/aWHj85PZXmEFsP3JG1K6hYPzN8jrueB0EMqesN9opJ/YVMTjHMtM4pjonW3/f8yDsQMa1DFgLrb16aSi06KnK7087ILi9wOt6IVi3axPWZiRK9medMDeMnROlsKA5y+JBkAlP3K45Y5cRwoDK105JuTQIFvN8FwskbxV/cIU/AWnfYaJ5mkiYa50KzSSMaHL4FfFEDuEHUcCePcDNehbwK7/OZ7+7Lh2Pi98n5f0OJO439eki9D0raWXD5mAOe46TXOpEUmFFnoEkNTLmUDFf4VsPE+8CrR3g80If71kAzZm2luo7ALa9aPUiASbtw9mXGWLarFwnrUEh00wlIGNUZkainMB4ABMY7saXCNgTl+o5scTUaseQTr5fWOGKBOmKw0lNUNoXpFmeaCkRNBmHKOUR/SZeMQEkKNBdg4QUuf3bPZ8FAmf3xWwHe2P7sbcrAmAd+oJYY5SirjFLtbUkKFaKmjzLqIkIQOhzQCYCEjYAQHzkyNdRCoQdCswsO6RCuJPX8Nu6Dca7Sj/UrSWOmdnXHEjS9c1v2z3iRel9mztVbOpt4bdGHCQrREAp94Z70lAvAxyojuZHltOnap/4bqKq6cvr2n9xDVtcl7jfOpmqMSD6UJ84mXqn3YRR+aImcJt7GG7qengKjFKvrPvEGN2ZjdQ6JDvZSB9XvYL+YbLHpn81ehkh1pk/m3KFTkEIf4hTkBCtnv9CQpxYfP3y5cunG39Yy25WEGOH/+Onz18Se+HkK9yxt1/U6Os8IeW5IWgwqTBOq97X15FuBjOFwLhDfyuXRd7049pPVsUAqHIiMpn172ErULc7jC9SCDHSV+dDcQHIQkPQo5iTu4XQ+dSrB9tC3bQ9oVyR6aAh1Oaalk1VMD7Nu+2iQim3MSDgZUATGYhOBvt+RS1WO73hI9+m20DWTdzuKFIYWCnkHKek8VtRzZJUCIygIjJjUR2n/dRrFBhThoIBIHFcifgKsON7we4inPGCoTpNFpI8g0p1XVEQ0YQaqrFmnCsTM6+irZlrMIVFQG9nAb2dnnOS7GGYeqsueJdaY2Gnkcwz3XXBp7lJcm1ZWQZhylHEXAoIpIe7lua9YHao9d718bL9Uv6vB3eoTpmFDMjc9IPbGCQgT7MM2e/QVEQUoT7scAB2wSyyKBmzO1X1tY74VnX1GDT2Cj9wwBUVLFNHeH+ZNoKi59Mm5FIpCoHvXQy02xUBAxvC/Sl6TqIBgrea34nrkh5ooOCwF5UlqUwwNwrqnGmsIubXDVpJITZsJQVxwL4iUaKyI1jJ+bR1j91j8vULm6mnGrDAaLhTNbeA4Eqq2MFtaC7aq9NatHkUZ0HrC20f2DTxO+tNLUVfcWB82CcSHzcquVYBTtWPvzL3xaK+1+O1EXtZr7PpU/I8NGdqZo+7bzV6fs/0qVXF3A0kxdJMd+hicHtF4UXm+Pgx1pCpFOzyw0+k8sDzsZ6W38p/qad3HotELYtE8DZOLL3LYxflY6wQlewHDygZkpoFKE2PIfLg9bbFf32Zh1G/NQGlp+uIO6T09bbFf31KU68i8ZSUpsLANJVpmhNFKE+bDpeXT+lzbcjH+vxb8mHw92j9+AbEDg6c8ojdUkc/VpOn95VzeDn1bYeeVPulaiyEtaZ3EtJgk/LGfRWgwt3dHbqTYzSqI4lev3IThrJDIA1U1B2hlfFa6f/8t98+bTMGOip3vT7zqliY0ZaC1a6/ldWfR1Hh00mZ3nQqXXQ5y12dy107/vduMZvOb+vD06yv1g+hvD0t2h4o5osut6k9mruBfIRctRC2wZ28u4tkDHgwRSAA01A7GYhOYw2c0TSlK59v6JcAw4CDDOIQFGLYC8M9igaEf7BrOql7kXsAsI+49ESKWRT/VWn9BbdA87KYLevbo+9v6EdHpcdluWiIMgYeq5jIwu79Ynb/xb35mIRMuRikGAy4CNSKhQT/ceT+cAculmXlZhivc5NX/wZEQl5W31SVjWxbe2V0lLCvvzE57HYZsr+j9k3ttiZuaFI5KQuU1X4fnCwd2+bY82oew0MXYdD0EL/b08/jCXvJOFYBr4+B1pLgY4R9FN7hNeuFIJDet0GkH6dAakiREcnp1oL4ujYJzCxrxO5HqwfqP282TkogK821evr/jmHcCsLbA/+uD6A6j6E+8MlUhX08t/A1a1ksVbVsL1Pae2uP3RWTyfONdN6t9YtnOPyrWeOQubJxSPkH/hH+1Cosn9TS3tOsPisCzUWbx3PpGuZ7sWwernn37947/7E2YsoaO5U2Ya0LwoYi9vnvzXITHRvXg8nuzVZsdpAXAl57rDITu/W/mt5thsDYXOGTEwYd6COvMpBB7xyrR25+9ozp4ZmwN4eGSa9sY7UugzNZ+KinztcacbXllv0LNe833plfdCf4nj+w/KD3C/tiddfP23tNudPnyiwmSv+5rbhoCML90sKvJV1GYme63ZlUp5pmdTUCa9JlAGVJlgtpgIbAMBWxHRXoY2mthO7qYXCAhXDyDN754+IhfZxMzHI//L3ZVN5VR0gBscl76VqYw0Rmuc5TmCOeR0zlJd6AEoyHCvW6YUsvW+uc26E1+MuKhS6rbC/wkSvseGuRJWpkSaqQIN0kcSpQYiQxFOcmhTpifyDM+hY3ggHOFko3vQjOZmlTZY/Lp/2wha4RW3jVRFRhg3G33kpIBRKKtcys4MxRFjHJFHPPImubrPR6iJIA19qiAp8LtpZmYu4rtbUgfAit/VKcLwJaViFzbc2sgUkgrwWi5g20AGaJVIpaYBnAAI0oED2LB6NhWz0ekIfiAuTh8lsxKcr9cLVfa9qLwJVlWau+sxrkMmVdRQsIkXBuGRbNKcAkZt9Z1s8nQSCQFh9yXZIopXq7XUh0RDD6mitsNvU3iDhunffzpYO2XrC9gTxGFKKNp3cI/u/ycUDziBGEicmXh8UPEGjfNzcGI9ADet3wERyyenKqmBAdkeLlc9sOTcJrGsM0HsHS4/LmVfm+VsrAHm/WbkoCTZlMoZFIpQ1vHp3c8uHD3V0d5A8kAKw/8zF3lC5fXiYLa7XZLu4CLp8DJMEI3I2IHGzm9RuDN7NyZlp8dYA3kAl7agG+xJgutDK3K6SpefH7VM0sFKc1bXz6A8DlR3YiGtN+c1waqJAOVUwcIU5ZRxQDNQaOryfNo7sig6q+i92zXffuChksXhhGPiNUVxz+lItiOp+YW/sFk+dO9H+t36hvZmEJ8H+Oeu3BCvfZ6HOZx21Rnnxh+tHuFw8IrtojP3z40X77Qzl/ai4M7G/5jsj5GT/hh3JR1+F8WajycVEsLvdJfthx6y87u67XiYBls06399Uhg5k3WcLeZtlq2tLtTbfOR++NII4k80KWDJBbxhgnAhAmAqMDgmk0MM4wshFayeZR5aPlTrCs78E+jMOP+WrcDeRGLR/rxCurjbrnmmXuX9cae2lvd/VRmd/UiVl9Lj2UYCGOHJJqG+ecr44tH+qLzs0saRIK0Ae36Y2aJMti2rydmyovK6vvuM+b+35oeyR3mxm/ZOj5mWSPxdoFWPqB+wHsaXDC+lEMvhHF6Z0snUah7pCilysTznPZnBcTyK0Zu95nkpuC/RCA9LqUjE5NGeRzDE61ITflZmdqxwgYjCj4Odz+2mQCHdsuU/+1rPN2xY9+73CjExlf0KvokkNfKwlZX/sn4O8mMBvhV30bjp3ViBAtaYp6MWggsoRRqIA00mitLtaxwzzHDmFDARPqfNMW/cXF3Yia7/N17JR5Xmhz6+C1WP07pDsCHL0XJ+MpPdJyEPAVB6ZtHRBwHkFadEKZQYH7ezqSW1p/dURf/f97rqrp8KY+viNWwT8R6aFnPPFARVerjfTEyVG29YjqrbchTlZ9LzOieX92KOUiAUBLkzGgEZQXK04GDfsYH+oxQS9yhA6MQ+CNCxRcP/BIDTzEqXKdfTsBqgzBBAOmUkFIztPLDVDJfqqCBMN2TyygxhyH342wj/4ykztX8KxbrxyZ01vKIYEQYuQm4vZPN95k9tWhrac9ovnMQ9bV5jo74DzzQIX8eV2fGbDsovbbDV17tW/xyeVJgGK2WKrV7bm1cUOowOq9/bxyEH1aLM10xXDGtwbrzaZJ3Gke5/OycrxoeNfO25ZkZtmkq4Sdkb0zrlyFgxP911SlPe6q/Yv7x0qNPV1eVrXDcVG7T+frYsPF2ivZrsKmJRg8bxF80o7/cxNZjGVxbmdN6yGH7ZlqujQUcfs/LevyWUs++88fj/WNw2Rh7KO7wz/ocjJplxMs3aJ8LRaPls/+t36wH0dSLfwUucsm2vgAq8V08MxdlSZYPpSPi9UqTteDG+emxlZzvx9a+M2McY1w109043Yh+PDpnzeNB9uddmHhPTE1navxXShCj1fflT9HyXtY4zT6YutO6t3684P/9r6963+8+22NpJkjRlYs/rT//WrP9Xf3yXK96749FPWz1eesHmebITfqASeLsrO8dUhvx+NOU5NltXM8uHE+hABhvludY1Fs/dGYte7FC3Zv2x9+/vQ/DtlWsr//cZ918uK4o396e3vbEHHNi1ckWu3abw+mMlsY5fECE4cH56CIo275nmwWyEPjscJxpx+b/M2kv09Kb6wmeqs1XqT2gtrdqO0HXevBCJpAjfLUDY3HDEdsyS28ljSkreLtdU5CAYD55b5nldW+BXNWmbsvs7SPObxfadf1YA7X1V8WB5nIehar85zglClqBTCEMdPdsWdDQsyGRmRwJFLbqOXSIDdbZB7ctrfEula4SSTqMmoKKFGpKwjTtPHMaSoSnjGoDNNKxyw2FLKfbwlRoGwHooCTNs6c02HTrRHeMkfH+fiHr42zmb1ko+XcNHHrzeEoP+kHt3ur17ckUCZ3wFiy02+4YqF/zx703N926E1uO6tZkMYvaXdgL3Ee4jQhhhmQkowamm7cdmvQ7t53LaRCjXACgMLHKWIKtrKLMbT+4y8ful0d+1lc52iLPAfDNhfI7EHelq0O49eROuGenl0srJldVv6ooLfZ6sSqhLJmFkgjTUBPJaQ0QVwQAVLFcxNRRkPhp1nhIbwgDwQW6DlbIRvbqOoHVS0Xq46qj8u6j2offDtaTRxbPcB+Q93QbPigWdhW218ZPfB+o0svo9ZZ1jX0OM1RqkGtFuStw4GqhBtAlFUQhMabd3qrIo/Y6t2Y1smB87d+s+SO//Cfnz6PdlR+KcvJRs/xayFzu7fsLDQQiHdrIHsg6XsfMU83XU63o9tnHJvuFfmTmuvH+SILyYz9nAqXwaMYqVvIIKwJQbqfGgoTylNKhZEKpSBinw860EaGyTXr9oc9dO2f5jDGlmln4+6TX3M9+mjdek8gwoSE/eRgKBKVA5hmjOtUoXgIsLKxjwABhuwlNMW4nX72kjwXckA21RVRG9XUpgylvSCIoDlNMNQ858AAu/EjOqQF86kdirINqc323+5DarP9qX0prLvuWYiylBreY92YmyRFmeEqS1PBY/Ys9Hq5Qy6HzdxFwNd7wLi1ISlHTI+9VFLW04KR1DAjoktKxGkiJdFcUA3TPI/YFYljn5TDXhg8FCWKkOJKRlTgXSsPlnU/5ruUaGBf626zW05hkjKWq4xwZs3DiB4g7kVpRIDabeCwR+0InU9ate0tUtvu7ZW/L2cgZ70mjZll00qlAlvlSwoSMe1AtBWua2rDoYYtA3ub7W++DRNFD6i6ux5q09qewhCmkvSoTSwnZ0AyA1TKAYrYN3HQYSBEbQgDUvmAYOOQ3OOK3a6V3LXxhJGhOezmdwhrLyVQasNRLjFKI7JyMtCmA3s7EAg8YCT0kNjoLRObrJJ5MkN12lO4CcwTZblnqoT9jMqIcpt4cjukpYUccTHkNj2g4O16qE3roD6WVBnT9YvIFKSJ4LlBSlMBoY5HbcbIbmqHtLQDslGH1H7TXjBal5ZhJSnJ+jWNVifPU20sq89lRiJaYJwNLLDhSFQRaHjMI/hFgm1+3g612YraJqeq5wVzjJ0SbEDKsbQiPaJO7u9tGfB6w5Ab7IB5D8dtZt0cqCeY/Vp+WZ30uU6sswxXEyahtdsUU5c7CfrCgCTAYEpzQ1hKYqr1uN9KxZrsgdxxEfCSE7Q3YAJhkldIHX9a/GfSRxO7ypkgtHYJYJ2rTPTcfQrjBGArZUCKoVYRVQvpd1pCbSuLXS2w5f7sZ0zQbWSTlGsVQGKFAE006/ETi/jEZCyjOcApzEnMeUOeABJyGIZhIeUyQtBtZN+Ua6V2HanBqUyN7lWBUJ0lkOdK4gxzFbMKRKC+C1AGAjVtMKfnAYwQqBnTKCUwBbPXLsBrIzDsHODyXe7UtJi4B/zFTL4aR+gNiTG9RmzuTyfvZt2+YZ9eBL35mk3ng+58Tfer/nzN51GX9KYz6nIbtbtDLdsN1J1h2ZDqTNodCD+tAyBvzuTYHgfSm8wn+XEGVvp3LBuj6MBhksN9MMKiHjbQ2G8bHNhAo94Zndmy9KY7WbYDV9gF6y2nPvbbzesPlx0FahrANDorUHPftw7hgaAW0N8ebByqDwHeiBSXEw2UUd1RMhZnNZLdkkB5S9084+7Al1lZOZq+n5VL/dDcgtepJ6NGZCQEfIFSzFwtRqCHj+QZ4DyOZMWo77ZheJi7HalUY0jY681mYbW7lVAI+olJUoI0YakG0lCjGInoXEd+WYQMhVJC2SwRWguya81mkRi6QnsOUuNC2938BkvVBGlgNGFacxExo9APk0gwdJyvWXdP3yUvJ+XIQS5Xatyw2jVGEOcS90fnIpRkKTAE0kxmIKIp64/zsqZsoE/asUzZN5y7ZKldm7JEqDzvZ6pRoxOYWg5qMEGERfSc49bvuY3aoaRDGYNNv+ncJbbKZsk5TEXPcYEISHJCMOFASaUiCmXKB9QOCOWA5yLG3g72tHs71K6TWYiVz5iyfiUATHKBGKBY6hxlMWOgXqYaJ4FKyFCCQ4xOq/xN+6AZXgnunGvYZeUyz2GiOMuhNnmGEYwYhSB0BLlxYHdjHoHc6E2Tm60MrJQDrvyQA6OSZKnMRQojauXMdwnytqa5F3QK1J3HqBfgI9xvV0xu2pAb5LqXrCYptf8AgikiRsTsdsapp5aHrDAZMsJiUPtaXSOWVG4VrMBDIleOUWvY2tNplmTAvsxMJpGIKJch8H0jItSiIMSpScwORvt1lIEi1FFm1QW3PddjezjrVJo/BppqDjvPvDRv5rnPcbK6pyxxoxYKbbzeKHx7b5SLKTrfn4P4MWrQqmhdUy/OYOgxSQr8oElhrxLBIQj2Yji3UMKxwZr6V59MVdg1c27+PcKSmjGREcQIVVQD14V/9aVuTKcN351JTEfgvrNQtkTdO6TDPW0W+OOE4oV0+Bv2Q7xKxRwHfryODJUZGmpSEEGZEQf4IS5CmXmVOM9ALwV4KFVowF18QO+8ISkP8DFcBClfJc4zsChBu5w78ppiTEhpc20vQBsY5HPES9oYJfJbBnYuIt8bSymlpy+Oz+LwWUkLivgiX+yXRveacOOvjbdWPToTvPl2zMF448I3iHz1IiLeLidd7dXx1kr1c8Eb9sUiODBNjRPsnYkeLU1NhFx2PY+PXgPl2amDDctAZoZ+nufxJpky094MpfPvIhxKVf6Jub/ufsqlavQ7udU038PBB9sCjHZSc2BKAIRxXC27/HuBtpB2sWajXH4k5PIbzOhZT0IJzudpj60uuud4nTh33JmxBezN3tTDpvKymqpZb6xQ6BZ3zwSK4b388mCS2gpRemIWSZknKrF3mqymxSZ2dZO8ngLV9WSK7Z7Mbw/F0ny2+8K9/1apeX8fermfeZ4jrUNSJGOpq+mMk4DguYfgenhBb6BBYGcc0D1ljBNSjHBCniib94/H6fzX2btuTi+kt1b5apJ6EVsl9dbf/8Uod2VKPU7HbsJZv2eX6UsGyd/BNs0hsR6j+4I/cPGQNkpnPgY2m8Hboh58OMtM1fCghu24487rYOzRqbHKS6JmWZI+LoqZWSwS97HlPVai3EFX63TnHBfHnvsKSWgb5ZNi/ssuFXMPH0cPdGToqoIyADnomxgHeTj2gJh+rCZP7yuLDFNL2O2svAbP2k4YTaA1xgIEstRBd3IMgbrcxirCNbeJQCjh6crrnKTukGgeiHcfoEINSCUPcBFfS0BA1AnAd4JCmmam165DMJVowjIGlGQwi+hFln7jJTKsqISh2KRf5/MChdkatLNsNUO1M091m3Y3KWdZObudPiWNQndblPUnXS1t5a3YqKVdLYyk0+zudGoEIabObzUNjKzoTbiQMoMEaajatg4+p4mhZPTNMCRC07MCGVLQt/Bjd3nY06pho+ywNWpv/1gcYIwFzK7RuSE7za5n/8GqH/xG/8GehuHL7nA123r29GwXOvzvd2u9vU5fYpENBXf9JyC4qfs7RjjH30OknULQ7Sffas+9LMP9i3/GGGxyRCrahZZfnsYEX4+xb31Tgbq8I0/D0o8LJ8s6cnaxXdCOdaN8qE+cfPROu8bOGx2LJd1AuTsj0wyKlHREsTRYJxxRTFCa6kyr44liyPv2/h6y+DhD9uT5eH4uk5Ew35YeNtg7Vqm2HGFK/0W7bbtR0lcj3rom4S/qHUg9v0sCBcPw0pFF+Of/97M9kKrs3uVXg7yq5bk7W6EjCfOfzcxUaukyut8310E9GKE3Ks1X1SdS4hTUqWHt6DuR0TTBxigIeZpmdJNh3Y+V4khmgtdRkdCAfGcB+Y72L2kYI94hGJHh8xeX2dr2oV8++gqGQhxGoua3tdPhdvHgs5A36ptjrG6SSVKQZ7BXr2ggTBgxqaaYM6YjTqqy/Nqf2w7QAFAhHy+NEvp9hZarZqqVdozEQx17o6jjYIU6qFjeS09PMUiwhJBIwCnCERv9Mkp80A39V22dfC+wEMXq3AQ6u2hddckCAsN6uZ1D8ql8rBwkdFXMl5GUKStfTPHV6VJtg+nm4olT3RIr7iaTpHfFZ7ju13T6iuCKarhmyGSq144FC5owlXGRGpJxrbfrWfYtGmhdsdwqGPaFNAGBVqRxqoJHaV3wTXd94KuuDxnJvc7DFKKEZing2HBkZMRBB+ufPA86GObR0ACD8/t+vrqaZqaqmCRW7S1yS0mHAZ8Twe3hlQvLzzkQYqvWA7ndwllPhmaGJcxyIytHlVLrevXn3BDgWzn3E3sV3+LZnF5ub6zQ7Xn2STXvGC9NXvs+GUGb8naPEeviuK8toBA3DY1/ihLrOr2GWqk0LZbT//j77I3GSRhf1U/mUPBeXw+JsLRvFRCIKiVRxFJYhr0mTajt+9BLSgsw8Lao8ixB1/D0+ePioXQZzFvZ+ltVMEHNywEH2GT9IcMySxDjOYPYQBpzPjzzor+QBtDWKocvbAk2SmFEB6TaXaVgJzUYEKcqzXvtADPkyqSYSgUhOU/TY+fhoqDTMwL2ZD9qwQMR4JCD+IDq4tNL0vvyd/N9/lVVHm/rLclb4W2cUeEe/E5CgrI0VwmRQrddJQRKoMgo0ywTksQcoO5VXcH1/NQecwv0ImDnLEp91M3KzNSYy6tyVjz60hTtFxY7dTsEB6Gk/mixEVR211AJnFfszhWFA45AQrUwiEORMN626BLAvuVUcsaN1JxFnPCCfV81EcMQK2wb7vU6lFySKbCG0mJZGHsRM7FCZeYDCl8ZoKikFCcyhZRKRCyg0qz1Q1s1LHU2ds6A5jxi9EP6EyoID8RHRZyWN2dhW87Lb6bKBso+2s+Hc4YCcQzgnqUhBoJLgUFCBINWHopEYSAbvLFcJwISyRW2DIZEbAbL2t7aaz2fDDN8EAzgDV8o3vSDZV1PPtouPtS2N9qsVEQywTkAeSaYQ5tqJ2kLbg2J3BCIudE4Incjng+D0mGQDaKA4nXWYGt8GLlreLHVgYEu3oGxJ8oQppRbVkYhyjLFSKIBbniagJAnqcZQAZ1TYCKiDON+xum6F1UXZSTkKUNnjLItLC01arZYqsmfPt7kW8MbgZRRkiiRS5VhnWhEaCNDAcoTA0ia5YAxaPpGwABcAQhuxBvlvs4WaG0XarmMo9gA9lBVlsvO1392WWW/WZ3d/eh/AQ== \ No newline at end of file diff --git a/docker-build.sh b/docker-build.sh index 1112ccb62..fd056c390 100644 --- a/docker-build.sh +++ b/docker-build.sh @@ -8,9 +8,6 @@ DEBIAN_FRONTEND=noninteractive # some mirrors have issues, i skipped httpredir in favor of an eu mirror -echo "deb http://ftp.nl.debian.org/debian/ jessie main" > /etc/apt/sources.list -echo "deb http://security.debian.org/debian-security jessie/updates main" >> /etc/apt/sources.list - # install dependencies for build apt-get -qq update diff --git a/installer/functions.sh b/installer/functions.sh index cfc2d7eaa..d403bfe52 100644 --- a/installer/functions.sh +++ b/installer/functions.sh @@ -120,6 +120,7 @@ netdata_banner() { # portable service command service_cmd="$(which_cmd service)" +rcservice_cmd="$(which_cmd rc-service)" systemctl_cmd="$(which_cmd systemctl)" service() { local cmd="${1}" action="${2}" @@ -132,6 +133,10 @@ service() { then run "${service_cmd}" "${cmd}" "${action}" return $? + elif [ ! -z "${rcservice_cmd}" ] + then + run "${rcservice_cmd}" "${cmd}" "${action}" + return $? fi return 1 } @@ -437,7 +442,7 @@ install_non_systemd_init() { run update-rc.d netdata defaults && \ run update-rc.d netdata enable && \ return 0 - elif [[ "${key}" =~ ^(amzn-201[567]|ol|CentOS release 6|Red Hat Enterprise Linux Server release 6).* ]] + elif [[ "${key}" =~ ^(amzn-201[567]|ol|CentOS release 6|Red Hat Enterprise Linux Server release 6|Scientific Linux CERN SLC release 6|CloudLinux Server release 6).* ]] then echo >&2 "Installing init.d file..." run cp system/netdata-init-d /etc/init.d/netdata && \ @@ -488,8 +493,15 @@ install_netdata_service() { if [ ${ret} -eq 0 ] then - NETDATA_START_CMD="service netdata start" - NETDATA_STOP_CMD="service netdata stop" + if [ ! -z "${service_cmd}" ] + then + NETDATA_START_CMD="service netdata start" + NETDATA_STOP_CMD="service netdata stop" + elif [ ! -z "${rcservice_cmd}" ] + then + NETDATA_START_CMD="rc-service netdata start" + NETDATA_STOP_CMD="rc-service netdata stop" + fi fi return ${ret} @@ -627,6 +639,84 @@ install_netdata_logrotate() { return 1 } +# ----------------------------------------------------------------------------- +# download netdata.conf + +fix_netdata_conf() { + local owner="${1}" + + if [ "${UID}" -eq 0 ] + then + run chown "${owner}" "${filename}" + fi + run chmod 0664 "${filename}" +} + +generate_netdata_conf() { + local owner="${1}" filename="${2}" url="${3}" + + if [ ! -s "${filename}" ] + then + cat >"${filename}" <&2 + echo >&2 "-------------------------------------------------------------------------------" + echo >&2 + echo >&2 "Downloading default configuration from netdata..." + sleep 5 + + # remove a possibly obsolete download + [ -f "${filename}.new" ] && rm "${filename}.new" + + # disable a proxy to get data from the local netdata + export http_proxy= + export https_proxy= + + # try curl + run curl -s -o "${filename}.new" "${url}" + ret=$? + + if [ ${ret} -ne 0 -o ! -s "${filename}.new" ] + then + # try wget + run wget -O "${filename}.new" "${url}" + ret=$? + fi + + if [ ${ret} -eq 0 -a -s "${filename}.new" ] + then + run mv "${filename}.new" "${filename}" + run_ok "New configuration saved for you to edit at ${filename}" + else + [ -f "${filename}.new" ] && rm "${filename}.new" + run_failed "Cannnot download configuration from netdata daemon using url '${url}'" + + generate_netdata_conf "${owner}" "${filename}" "${url}" + fi + + fix_netdata_conf "${owner}" + fi +} + + # ----------------------------------------------------------------------------- # add netdata user and group @@ -642,15 +732,15 @@ add_netdata_user_and_group() { if [ ${UID} -eq 0 ] then portable_add_group netdata || return 1 - portable_add_user netdata || return 1 + portable_add_user netdata || return 1 portable_add_user_to_group docker netdata && NETDATA_ADDED_TO_DOCKER=1 portable_add_user_to_group nginx netdata && NETDATA_ADDED_TO_NGINX=1 portable_add_user_to_group varnish netdata && NETDATA_ADDED_TO_VARNISH=1 portable_add_user_to_group haproxy netdata && NETDATA_ADDED_TO_HAPROXY=1 portable_add_user_to_group adm netdata && NETDATA_ADDED_TO_ADM=1 portable_add_user_to_group nsd netdata && NETDATA_ADDED_TO_NSD=1 - portable_add_user_to_group proxy netdata && NETDATA_ADDED_TO_PROXY=1 - portable_add_user_to_group squid netdata && NETDATA_ADDED_TO_SQUID=1 + portable_add_user_to_group proxy netdata && NETDATA_ADDED_TO_PROXY=1 + portable_add_user_to_group squid netdata && NETDATA_ADDED_TO_SQUID=1 return 0 fi diff --git a/makeself/install-or-update.sh b/makeself/install-or-update.sh index b64e7be33..34630cf16 100755 --- a/makeself/install-or-update.sh +++ b/makeself/install-or-update.sh @@ -162,7 +162,9 @@ progress "starting netdata" restart_netdata "/opt/netdata/bin/netdata" if [ $? -eq 0 ] then + download_netdata_conf "${NETDATA_USER}:${NETDATA_GROUP}" "/opt/netdata/etc/netdata/netdata.conf" "http://localhost:19999/netdata.conf" netdata_banner "is installed and running now!" else + generate_netdata_conf "${NETDATA_USER}:${NETDATA_GROUP}" "/opt/netdata/etc/netdata/netdata.conf" "http://localhost:19999/netdata.conf" netdata_banner "is installed now!" fi diff --git a/makeself/jobs/70-netdata-git.install.sh b/makeself/jobs/70-netdata-git.install.sh index 0486ce11a..b85481492 100755 --- a/makeself/jobs/70-netdata-git.install.sh +++ b/makeself/jobs/70-netdata-git.install.sh @@ -11,6 +11,13 @@ else export CFLAGS="-static -O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -D_FORTIFY_SOURCE=2 -DNETDATA_INTERNAL_CHECKS=1" fi +if [ ! -z "${NETDATA_INSTALL_PATH}" -a -d "${NETDATA_INSTALL_PATH}/etc" ] + then + # make sure we don't have an old etc path, so that the installer + # will install all files without examining changes + run mv "${NETDATA_INSTALL_PATH}/etc" "${NETDATA_INSTALL_PATH}/etc.new" +fi + run ./netdata-installer.sh --install "${NETDATA_INSTALL_PARENT}" \ --dont-wait \ --dont-start-it \ @@ -20,4 +27,5 @@ if [ ${NETDATA_BUILD_WITH_DEBUG} -eq 0 ] then run strip ${NETDATA_INSTALL_PATH}/bin/netdata run strip ${NETDATA_INSTALL_PATH}/usr/libexec/netdata/plugins.d/apps.plugin + run strip ${NETDATA_INSTALL_PATH}/usr/libexec/netdata/plugins.d/cgroup-network fi diff --git a/makeself/jobs/99-makeself.install.sh b/makeself/jobs/99-makeself.install.sh index 465a31952..698f2f92d 100755 --- a/makeself/jobs/99-makeself.install.sh +++ b/makeself/jobs/99-makeself.install.sh @@ -2,14 +2,46 @@ . $(dirname "${0}")/../functions.sh "${@}" || exit 1 +run cd "${NETDATA_SOURCE_PATH}" || exit 1 + +# ----------------------------------------------------------------------------- +# find the netdata version + +NOWNER="unknown" +ORIGIN="$(git config --get remote.origin.url || echo "unknown")" +if [[ "${ORIGIN}" =~ ^git@github.com:.*/netdata.*$ ]] + then + NOWNER="${ORIGIN/git@github.com:/}" + NOWNER="${NOWNER/\/netdata*/}" + +elif [[ "${ORIGIN}" =~ ^https://github.com/.*/netdata.*$ ]] + then + NOWNER="${ORIGIN/https:\/\/github.com\//}" + NOWNER="${NOWNER/\/netdata*/}" +fi + +# make sure it does not have any slashes in it +NOWNER="${NOWNER//\//_}" + +if [ "${NOWNER}" = "firehol" ] + then + NOWNER= +else + NOWNER="-${NOWNER}" +fi + +VERSION="$(git describe || echo "undefined")" +[ -z "${VERSION}" ] && VERSION="undefined" + +FILE_VERSION="${VERSION}-$(uname -m)-$(date +"%Y%m%d-%H%M%S")${NOWNER}" + # ----------------------------------------------------------------------------- # copy the files needed by makeself installation run mkdir -p "${NETDATA_INSTALL_PATH}/system" -run cd "${NETDATA_SOURCE_PATH}" || exit 1 -cp \ +run cp \ makeself/post-installer.sh \ makeself/install-or-update.sh \ installer/functions.sh \ @@ -25,9 +57,9 @@ cp \ # ----------------------------------------------------------------------------- # create a wrapper to start our netdata with a modified path -mkdir -p "${NETDATA_INSTALL_PATH}/bin/srv" +run mkdir -p "${NETDATA_INSTALL_PATH}/bin/srv" -mv "${NETDATA_INSTALL_PATH}/bin/netdata" \ +run mv "${NETDATA_INSTALL_PATH}/bin/netdata" \ "${NETDATA_INSTALL_PATH}/bin/srv/netdata" || exit 1 cat >"${NETDATA_INSTALL_PATH}/bin/netdata" <"${NETDATA_INSTALL_PATH}/bin/netdata" <"${NETDATA_MAKESELF_PATH}/makeself.lsm.tmp" + +run "${NETDATA_MAKESELF_PATH}/makeself.sh" \ --gzip \ --complevel 9 \ --notemp \ --needroot \ --target "${NETDATA_INSTALL_PATH}" \ --header "${NETDATA_MAKESELF_PATH}/makeself-header.sh" \ - --lsm "${NETDATA_MAKESELF_PATH}/makeself.lsm" \ + --lsm "${NETDATA_MAKESELF_PATH}/makeself.lsm.tmp" \ --license "${NETDATA_MAKESELF_PATH}/makeself-license.txt" \ --help-header "${NETDATA_MAKESELF_PATH}/makeself-help-header.txt" \ "${NETDATA_INSTALL_PATH}" \ @@ -81,41 +121,16 @@ rm "${NETDATA_INSTALL_PATH}/sbin" \ ./system/post-installer.sh \ ${NULL} +run rm "${NETDATA_MAKESELF_PATH}/makeself.lsm.tmp" # ----------------------------------------------------------------------------- # copy it to the netdata build dir -NOWNER="unknown" -ORIGIN="$(git config --get remote.origin.url || echo "unknown")" -if [[ "${ORIGIN}" =~ ^git@github.com:.*/netdata.*$ ]] - then - NOWNER="${ORIGIN/git@github.com:/}" - NOWNER="${NOWNER/\/netdata*/}" - -elif [[ "${ORIGIN}" =~ ^https://github.com/.*/netdata.*$ ]] - then - NOWNER="${ORIGIN/https:\/\/github.com\//}" - NOWNER="${NOWNER/\/netdata*/}" -fi - -# make sure it does not have any slashes in it -NOWNER="${NOWNER//\//_}" - -if [ "${NOWNER}" = "firehol" ] - then - NOWNER= -else - NOWNER="-${NOWNER}" -fi - -VERSION="$(git describe || echo "undefined")" -[ -z "${VERSION}" ] && VERSION="undefined" - -FILE="netdata-${VERSION}-$(uname -m)-$(date +"%Y%m%d-%H%M%S")${NOWNER}.gz.run" +FILE="netdata-${FILE_VERSION}.gz.run" -cp "${NETDATA_INSTALL_PATH}.gz.run" "${FILE}" +run cp "${NETDATA_INSTALL_PATH}.gz.run" "${FILE}" echo >&2 "Self-extracting installer copied to '${FILE}'" [ -f netdata-latest.gz.run ] && rm netdata-latest.gz.run -ln -s "${FILE}" netdata-latest.gz.run +run ln -s "${FILE}" netdata-latest.gz.run echo >&2 "Self-extracting installer linked to 'netdata-latest.gz.run'" diff --git a/makeself/makeself-header.sh b/makeself/makeself-header.sh index 93d937b37..83131a17c 100755 --- a/makeself/makeself-header.sh +++ b/makeself/makeself-header.sh @@ -218,10 +218,9 @@ MS_Check() UnTAR() { if test x"\$quiet" = xn; then - tar \$1 "$UNTAR_EXTRA" -vf - 2>&1 || { echo Extraction failed. > /dev/tty; kill -15 \$$; } + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." > /dev/tty; kill -15 \$$; } else - - tar \$1 "$UNTAR_EXTRA" -f - 2>&1 || { echo Extraction failed. > /dev/tty; kill -15 \$$; } + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. > /dev/tty; kill -15 \$$; } fi } diff --git a/makeself/makeself.lsm b/makeself/makeself.lsm index 026796294..6bd4703db 100644 --- a/makeself/makeself.lsm +++ b/makeself/makeself.lsm @@ -1,6 +1,6 @@ Begin3 Title: netdata -Version: 1.6.0 +Version: NETDATA_VERSION Description: netdata is a system for distributed real-time performance and health monitoring. It provides unparalleled insights, in real-time, of everything happening on the system it runs (including applications such as web and database servers), using diff --git a/makeself/makeself.sh b/makeself/makeself.sh index 709473aab..2ce37a24f 100755 --- a/makeself/makeself.sh +++ b/makeself/makeself.sh @@ -470,8 +470,8 @@ gpg-asymmetric) GUNZIP_CMD="gpg --yes -d" ;; openssl) - GZIP_CMD="openssl aes-256-cbc -a -salt" - GUNZIP_CMD="openssl aes-256-cbc -d -a" + GZIP_CMD="openssl aes-256-cbc -a -salt -md sha256" + GUNZIP_CMD="openssl aes-256-cbc -d -a -md sha256" ;; Unix) GZIP_CMD="compress -cf" @@ -529,7 +529,8 @@ if test "$QUIET" = "n";then echo Adding files to archive named \"$archname\"... fi exec 3<> "$tmpfile" -(cd "$archdir" && ( tar "$TAR_EXTRA" -$TAR_ARGS - . | eval "$GZIP_CMD" >&3 ) ) || { echo Aborting: Archive directory not found or temporary file: "$tmpfile" could not be created.; exec 3>&-; rm -f "$tmpfile"; exit 1; } +( cd "$archdir" && ( tar $TAR_EXTRA -$TAR_ARGS - . | eval "$GZIP_CMD" >&3 ) ) || \ + { echo Aborting: archive directory not found or temporary file: "$tmpfile" could not be created.; exec 3>&-; rm -f "$tmpfile"; exit 1; } exec 3>&- # try to close the archive fsize=`cat "$tmpfile" | wc -c | tr -d " "` @@ -617,4 +618,3 @@ else fi fi rm -f "$tmpfile" - diff --git a/netdata-installer.sh b/netdata-installer.sh index f3f444f66..356eb4e4f 100755 --- a/netdata-installer.sh +++ b/netdata-installer.sh @@ -817,16 +817,7 @@ install_netdata_service || run_failed "Cannot install netdata init service." started=0 if [ ${DONOTSTART} -eq 1 ] then - if [ ! -s "${NETDATA_PREFIX}/etc/netdata/netdata.conf" ] - then - echo "# Get config from http://127.0.0.1:${NETDATA_PORT}/netdata.conf" >"${NETDATA_PREFIX}/etc/netdata/netdata.conf" - - if [ "${UID}" -eq 0 ] - then - chown "${NETDATA_USER}" "${NETDATA_PREFIX}/etc/netdata/netdata.conf" - fi - chmod 0644 "${NETDATA_PREFIX}/etc/netdata/netdata.conf" - fi + generate_netdata_conf "${NETDATA_USER}" "${NETDATA_PREFIX}/etc/netdata/netdata.conf" "http://localhost:${NETDATA_PORT}/netdata.conf" else restart_netdata ${NETDATA_PREFIX}/usr/sbin/netdata "${@}" @@ -845,47 +836,7 @@ else # ----------------------------------------------------------------------------- # save a config file, if it is not already there - if [ ! -s "${NETDATA_PREFIX}/etc/netdata/netdata.conf" ] - then - echo >&2 - echo >&2 "-------------------------------------------------------------------------------" - echo >&2 - echo >&2 "Downloading default configuration from netdata..." - sleep 5 - - # remove a possibly obsolete download - [ -f "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" ] && rm "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" - - # disable a proxy to get data from the local netdata - export http_proxy= - export https_proxy= - - # try wget - wget 2>/dev/null -O "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" "http://localhost:${NETDATA_PORT}/netdata.conf" - ret=$? - - # try curl - if [ ${ret} -ne 0 -o ! -s "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" ] - then - curl -s -o "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" "http://localhost:${NETDATA_PORT}/netdata.conf" - ret=$? - fi - - if [ ${ret} -eq 0 -a -s "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" ] - then - mv "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" "${NETDATA_PREFIX}/etc/netdata/netdata.conf" - echo >&2 "New configuration saved for you to edit at ${NETDATA_PREFIX}/etc/netdata/netdata.conf" - - if [ "${UID}" -eq 0 ] - then - chown "${NETDATA_USER}" "${NETDATA_PREFIX}/etc/netdata/netdata.conf" - fi - chmod 0664 "${NETDATA_PREFIX}/etc/netdata/netdata.conf" - else - echo >&2 "Cannnot download configuration from netdata daemon using url 'http://localhost:${NETDATA_PORT}/netdata.conf'" - [ -f "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" ] && rm "${NETDATA_PREFIX}/etc/netdata/netdata.conf.new" - fi - fi + download_netdata_conf "${NETDATA_USER}" "${NETDATA_PREFIX}/etc/netdata/netdata.conf" "http://localhost:${NETDATA_PORT}/netdata.conf" fi # ----------------------------------------------------------------------------- diff --git a/netdata.spec b/netdata.spec index bb8171359..b614e2127 100644 --- a/netdata.spec +++ b/netdata.spec @@ -9,6 +9,7 @@ # Conditional build: %bcond_without systemd # systemd %bcond_with nfacct # build with nfacct plugin +%bcond_with freeipmi # build with freeipmi plugin %if 0%{?fedora} || 0%{?rhel} >= 7 || 0%{?suse_version} >= 1140 %else @@ -77,11 +78,11 @@ Recommends: python2-psycopg2 \ Summary: Real-time performance monitoring, done right Name: netdata -Version: 1.8.0 +Version: 1.9.0 Release: 1%{?dist} License: GPLv3+ Group: Applications/System -Source0: https://github.com/firehol/%{name}/releases/download/v1.8.0/%{name}-1.8.0.tar.xz +Source0: https://github.com/firehol/%{name}/releases/download/v1.9.0/%{name}-1.9.0.tar.xz URL: http://my-netdata.io BuildRequires: pkgconfig BuildRequires: xz @@ -98,6 +99,11 @@ Requires: libmnl Requires: libnetfilter_acct %endif +%if %{with freeipmi} +BuildRequires: freeipmi-devel +Requires: freeipmi +%endif + Requires(pre): /usr/sbin/groupadd Requires(pre): /usr/sbin/useradd Requires(post): libcap @@ -117,13 +123,14 @@ so that you can get insights of what is happening now and what just happened, on your systems and applications. %prep -%setup -q -n netdata-1.8.0 +%setup -q -n netdata-1.9.0 %build %configure \ --with-zlib \ --with-math \ %{?with_nfacct:--enable-plugin-nfacct} \ + %{?with_freeipmi:--enable-plugin-freeipmi} \ --with-user=netdata %{__make} %{?_smp_mflags} @@ -187,6 +194,16 @@ rm -rf "${RPM_BUILD_ROOT}" %caps(cap_dac_read_search,cap_sys_ptrace=ep) %attr(0555,root,root) %{_libexecdir}/%{name}/plugins.d/apps.plugin +# cgroup-network detects the network interfaces of CGROUPs +# it must be able to use setns() and run cgroup-network-helper.sh as root +# the helper script reads /proc/PID/fdinfo/* files, runs virsh, etc. +%caps(cap_setuid=ep) %attr(4555,root,root) %{_libexecdir}/%{name}/plugins.d/cgroup-network +%attr(0555,root,root) %{_libexecdir}/%{name}/plugins.d/cgroup-network-helper.sh + +%if %{with freeipmi} +%caps(cap_setuid=ep) %attr(4555,root,root) %{_libexecdir}/%{name}/plugins.d/freeipmi.plugin +%endif + %attr(0700,netdata,netdata) %dir %{_localstatedir}/cache/%{name} %attr(0700,netdata,netdata) %dir %{_localstatedir}/log/%{name} %attr(0700,netdata,netdata) %dir %{_localstatedir}/lib/%{name} @@ -207,6 +224,9 @@ rm -rf "${RPM_BUILD_ROOT}" %{_datadir}/%{name}/web %changelog +* Sun Dec 17 2017 Costa Tsaousis - 1.9.0-1 + Please check full changelog at github. + https://github.com/firehol/netdata/releases * Mon Sep 17 2017 Costa Tsaousis - 1.8.0-1 This is mainly a bugfix release. Please check full changelog at github. diff --git a/netdata.spec.in b/netdata.spec.in index eef2a0896..cd8ef6d33 100644 --- a/netdata.spec.in +++ b/netdata.spec.in @@ -9,6 +9,7 @@ # Conditional build: %bcond_without systemd # systemd %bcond_with nfacct # build with nfacct plugin +%bcond_with freeipmi # build with freeipmi plugin %if 0%{?fedora} || 0%{?rhel} >= 7 || 0%{?suse_version} >= 1140 %else @@ -98,6 +99,11 @@ Requires: libmnl Requires: libnetfilter_acct %endif +%if %{with freeipmi} +BuildRequires: freeipmi-devel +Requires: freeipmi +%endif + Requires(pre): /usr/sbin/groupadd Requires(pre): /usr/sbin/useradd Requires(post): libcap @@ -124,6 +130,7 @@ happened, on your systems and applications. --with-zlib \ --with-math \ %{?with_nfacct:--enable-plugin-nfacct} \ + %{?with_freeipmi:--enable-plugin-freeipmi} \ --with-user=netdata %{__make} %{?_smp_mflags} @@ -187,6 +194,16 @@ rm -rf "${RPM_BUILD_ROOT}" %caps(cap_dac_read_search,cap_sys_ptrace=ep) %attr(0555,root,root) %{_libexecdir}/%{name}/plugins.d/apps.plugin +# cgroup-network detects the network interfaces of CGROUPs +# it must be able to use setns() and run cgroup-network-helper.sh as root +# the helper script reads /proc/PID/fdinfo/* files, runs virsh, etc. +%caps(cap_setuid=ep) %attr(4555,root,root) %{_libexecdir}/%{name}/plugins.d/cgroup-network +%attr(0555,root,root) %{_libexecdir}/%{name}/plugins.d/cgroup-network-helper.sh + +%if %{with freeipmi} +%caps(cap_setuid=ep) %attr(4555,root,root) %{_libexecdir}/%{name}/plugins.d/freeipmi.plugin +%endif + %attr(0700,netdata,netdata) %dir %{_localstatedir}/cache/%{name} %attr(0700,netdata,netdata) %dir %{_localstatedir}/log/%{name} %attr(0700,netdata,netdata) %dir %{_localstatedir}/lib/%{name} @@ -207,6 +224,9 @@ rm -rf "${RPM_BUILD_ROOT}" %{_datadir}/%{name}/web %changelog +* Sun Dec 17 2017 Costa Tsaousis - 1.9.0-1 + Please check full changelog at github. + https://github.com/firehol/netdata/releases * Mon Sep 17 2017 Costa Tsaousis - 1.8.0-1 This is mainly a bugfix release. Please check full changelog at github. diff --git a/node.d/named.node.js b/node.d/named.node.js index 3d7946557..02c890c60 100644 --- a/node.d/named.node.js +++ b/node.d/named.node.js @@ -201,7 +201,7 @@ var named = { r = JSON.parse(data); if(typeof r === 'undefined' || r === null) { - netdata.serviceError(service, "Cannot parse these data: " + data); + service.error("Cannot parse these data: " + data.toString()); return; } diff --git a/node.d/node_modules/netdata.js b/node.d/node_modules/netdata.js index 143255d9e..4ab8308c1 100644 --- a/node.d/node_modules/netdata.js +++ b/node.d/node_modules/netdata.js @@ -68,7 +68,6 @@ var netdata = { modules_configuring: 0, charts: {}, - processors: { http: { name: 'http', @@ -245,7 +244,13 @@ var netdata = { if(__DEBUG === true) netdata.debug(service.module.name + ': ' + service.name + ': processing ' + service.processor.name + ' response (received in ' + (service.ended - service.started).toString() + ' ms)'); - responseProcessor(service, response); + try { + responseProcessor(service, response); + } + catch(e) { + netdata.error(e); + service.error("responseProcessor failed process response data."); + } service.running = false; service.module.active--; diff --git a/plugins.d/Makefile.am b/plugins.d/Makefile.am index 7d3bc44f7..41e6d5366 100644 --- a/plugins.d/Makefile.am +++ b/plugins.d/Makefile.am @@ -12,6 +12,7 @@ dist_plugins_SCRIPTS = \ alarm-notify.sh \ alarm-test.sh \ cgroup-name.sh \ + cgroup-network-helper.sh \ charts.d.dryrun-helper.sh \ charts.d.plugin \ fping.plugin \ diff --git a/plugins.d/Makefile.in b/plugins.d/Makefile.in index 256605f5d..059d68f6a 100644 --- a/plugins.d/Makefile.in +++ b/plugins.d/Makefile.in @@ -306,6 +306,7 @@ dist_plugins_SCRIPTS = \ alarm-notify.sh \ alarm-test.sh \ cgroup-name.sh \ + cgroup-network-helper.sh \ charts.d.dryrun-helper.sh \ charts.d.plugin \ fping.plugin \ diff --git a/plugins.d/alarm-notify.sh b/plugins.d/alarm-notify.sh index 9b7f6c8dd..0af98095d 100755 --- a/plugins.d/alarm-notify.sh +++ b/plugins.d/alarm-notify.sh @@ -219,11 +219,13 @@ sendmail= # enable / disable features SEND_SLACK="YES" +SEND_FLOCK="YES" SEND_DISCORD="YES" SEND_PUSHOVER="YES" SEND_TWILIO="YES" SEND_HIPCHAT="YES" SEND_MESSAGEBIRD="YES" +SEND_KAVENEGAR="YES" SEND_TELEGRAM="YES" SEND_EMAIL="YES" SEND_PUSHBULLET="YES" @@ -236,6 +238,11 @@ SLACK_WEBHOOK_URL= DEFAULT_RECIPIENT_SLACK= declare -A role_recipients_slack=() +# flock configs +FLOCK_WEBHOOK_URL= +DEFAULT_RECIPIENT_FLOCK= +declare -A role_recipients_flock=() + # discord configs DISCORD_WEBHOOK_URL= DEFAULT_RECIPIENT_DISCORD= @@ -248,6 +255,7 @@ declare -A role_recipients_pushover=() # pushbullet configs PUSHBULLET_ACCESS_TOKEN= +PUSHBULLET_SOURCE_DEVICE= DEFAULT_RECIPIENT_PUSHBULLET= declare -A role_recipients_pushbullet=() @@ -270,6 +278,12 @@ MESSAGEBIRD_NUMBER= DEFAULT_RECIPIENT_MESSAGEBIRD= declare -A role_recipients_messagebird=() +# kavenegar configs +KAVENEGAR_API_KEY="" +KAVENEGAR_SENDER="" +DEFAULT_RECIPIENT_KAVENEGAR=() +declare -A role_recipients_kavenegar="" + # telegram configs TELEGRAM_BOT_TOKEN= DEFAULT_RECIPIENT_TELEGRAM= @@ -372,6 +386,7 @@ filter_recipient_by_criticality() { # find the recipients' addresses per method declare -A arr_slack=() +declare -A arr_flock=() declare -A arr_discord=() declare -A arr_pushover=() declare -A arr_pushbullet=() @@ -382,6 +397,7 @@ declare -A arr_pd=() declare -A arr_email=() declare -A arr_custom=() declare -A arr_messagebird=() +declare -A arr_kavenegar=() # netdata may call us with multiple roles, and roles may have multiple but # overlapping recipients - so, here we find the unique recipients. @@ -439,6 +455,14 @@ do [ "${r}" != "disabled" ] && filter_recipient_by_criticality messagebird "${r}" && arr_messagebird[${r/|*/}]="1" done + # kavenegar + a="${role_recipients_kavenegar[${x}]}" + [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_KAVENEGAR}" + for r in ${a//,/ } + do + [ "${r}" != "disabled" ] && filter_recipient_by_criticality kavenegar "${r}" && arr_kavenegar[${r/|*/}]="1" + done + # telegram a="${role_recipients_telegram[${x}]}" [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}" @@ -455,6 +479,14 @@ do [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1" done + # flock + a="${role_recipients_flock[${x}]}" + [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLOCK}" + for r in ${a//,/ } + do + [ "${r}" != "disabled" ] && filter_recipient_by_criticality flock "${r}" && arr_flock[${r/|*/}]="1" + done + # discord a="${role_recipients_discord[${x}]}" [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_DISCORD}" @@ -485,6 +517,10 @@ done to_slack="${!arr_slack[*]}" [ -z "${to_slack}" ] && SEND_SLACK="NO" +# build the list of flock recipients (channels) +to_flock="${!arr_flock[*]}" +[ -z "${to_flock}" ] && SEND_FLOCK="NO" + # build the list of discord recipients (channels) to_discord="${!arr_discord[*]}" [ -z "${to_discord}" ] && SEND_DISCORD="NO" @@ -509,6 +545,10 @@ to_hipchat="${!arr_hipchat[*]}" to_messagebird="${!arr_messagebird[*]}" [ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO" +# build the list of kavenegar recipients (phone numbers) +to_kavenegar="${!arr_kavenegar[*]}" +[ -z "${to_kavenegar}" ] && SEND_KAVENEGAR="NO" + # check array of telegram recipients (chat ids) to_telegram="${!arr_telegram[*]}" [ -z "${to_telegram}" ] && SEND_TELEGRAM="NO" @@ -537,6 +577,9 @@ done # check slack [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO" +# check flock +[ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO" + # check discord [ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO" @@ -555,6 +598,9 @@ done # check messagebird [ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO" +# check kavenegar +[ -z "${KAVENEGAR_API_KEY}" -o -z "${KAVENEGAR_SENDER}" ] && SEND_KAVENEGAR="NO" + # check telegram [ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO" @@ -578,13 +624,16 @@ fi if [ \( \ "${SEND_PUSHOVER}" = "YES" \ -o "${SEND_SLACK}" = "YES" \ + -o "${SEND_FLOCK}" = "YES" \ -o "${SEND_DISCORD}" = "YES" \ -o "${SEND_HIPCHAT}" = "YES" \ -o "${SEND_TWILIO}" = "YES" \ -o "${SEND_MESSAGEBIRD}" = "YES" \ + -o "${SEND_KAVENEGAR}" = "YES" \ -o "${SEND_TELEGRAM}" = "YES" \ -o "${SEND_PUSHBULLET}" = "YES" \ -o "${SEND_KAFKA}" = "YES" \ + -o "${SEND_CUSTOM}" = "YES" \ \) -a -z "${curl}" ] then curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)" @@ -595,11 +644,14 @@ if [ \( \ SEND_PUSHBULLET="NO" SEND_TELEGRAM="NO" SEND_SLACK="NO" + SEND_FLOCK="NO" SEND_DISCORD="NO" SEND_TWILIO="NO" SEND_HIPCHAT="NO" SEND_MESSAGEBIRD="NO" + SEND_KAVENEGAR="NO" SEND_KAFKA="NO" + SEND_CUSTOM="NO" fi fi @@ -619,10 +671,12 @@ if [ "${SEND_EMAIL}" != "YES" \ -a "${SEND_PUSHOVER}" != "YES" \ -a "${SEND_TELEGRAM}" != "YES" \ -a "${SEND_SLACK}" != "YES" \ + -a "${SEND_FLOCK}" != "YES" \ -a "${SEND_DISCORD}" != "YES" \ -a "${SEND_TWILIO}" != "YES" \ -a "${SEND_HIPCHAT}" != "YES" \ -a "${SEND_MESSAGEBIRD}" != "YES" \ + -a "${SEND_KAVENEGAR}" != "YES" \ -a "${SEND_PUSHBULLET}" != "YES" \ -a "${SEND_KAFKA}" != "YES" \ -a "${SEND_PD}" != "YES" \ @@ -783,7 +837,7 @@ send_pushover() { priority=-2 case "${status}" in CLEAR) priority=-1;; # low priority: no sound or vibration - WARNING) priotity=0;; # normal priority: respect quiet hours + WARNING) priority=0;; # normal priority: respect quiet hours CRITICAL) priority=1;; # high priority: bypass quiet hours *) priority=-2;; # lowest priority: no notification at all esac @@ -802,7 +856,7 @@ send_pushover() { --form-string "priority=${priority}" \ https://api.pushover.net/1/messages.json) - if [ "${httpcode}" == "200" ] + if [ "${httpcode}" = "200" ] then info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'" sent=$((sent + 1)) @@ -821,7 +875,7 @@ send_pushover() { # pushbullet sender send_pushbullet() { - local userapikey="${1}" recipients="${2}" title="${3}" message="${4}" httpcode sent=0 user + local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ] then #https://docs.pushbullet.com/#create-push @@ -832,13 +886,15 @@ send_pushbullet() { --header 'Content-Type: application/json' \ --data-binary @<(cat < +--multipart-boundary-- EOF SENT_EMAIL=$? @@ -1570,10 +1733,12 @@ if [ ${SENT_EMAIL} -eq 0 \ -o ${SENT_PUSHOVER} -eq 0 \ -o ${SENT_TELEGRAM} -eq 0 \ -o ${SENT_SLACK} -eq 0 \ + -o ${SENT_FLOCK} -eq 0 \ -o ${SENT_DISCORD} -eq 0 \ -o ${SENT_TWILIO} -eq 0 \ -o ${SENT_HIPCHAT} -eq 0 \ -o ${SENT_MESSAGEBIRD} -eq 0 \ + -o ${SENT_KAVENEGAR} -eq 0 \ -o ${SENT_PUSHBULLET} -eq 0 \ -o ${SENT_KAFKA} -eq 0 \ -o ${SENT_PD} -eq 0 \ @@ -1586,4 +1751,3 @@ fi # we did not send anything exit 1 - diff --git a/plugins.d/cgroup-name.sh b/plugins.d/cgroup-name.sh index dc0bf7553..acdd6f4f9 100755 --- a/plugins.d/cgroup-name.sh +++ b/plugins.d/cgroup-name.sh @@ -126,6 +126,11 @@ if [ -z "${NAME}" ] # NAME="$(echo ${CGROUP} | sed 's/machine.slice_machine.*-qemu//; s/\/x2d//; s/\/x2d/\-/g; s/\.scope//g')" NAME="qemu_$(echo ${CGROUP} | sed 's/machine.slice_machine.*-qemu//; s/\/x2d[[:digit:]]*//; s/\/x2d//g; s/\.scope//g')" + elif [[ "${CGROUP}" =~ machine_.*\.libvirt-qemu ]] + then + # libvirtd / qemu virtual machines + NAME="qemu_$(echo ${CGROUP} | sed 's/^machine_//; s/\.libvirt-qemu$//; s/-/_/;')" + elif [[ "${CGROUP}" =~ qemu.slice_([0-9]+).scope && -d /etc/pve ]] then # Proxmox VMs diff --git a/plugins.d/cgroup-network-helper.sh b/plugins.d/cgroup-network-helper.sh new file mode 100755 index 000000000..d93fe356a --- /dev/null +++ b/plugins.d/cgroup-network-helper.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash + +# cgroup-network-helper.sh +# detect container and virtual machine interfaces +# +# (C) 2017 Costa Tsaousis +# GPL v3+ +# +# This script is called as root (by cgroup-network), with either a pid, or a cgroup path. +# It tries to find all the network interfaces that belong to the same cgroup. +# +# It supports several method for this detection: +# +# 1. cgroup-network (the binary father of this script) detects veth network interfaces, +# by examining iflink and ifindex IDs and switching namespaces +# (it also detects the interface name as it is used by the container). +# +# 2. this script, uses /proc/PID/fdinfo to find tun/tap network interfaces. +# +# 3. this script, calls virsh to find libvirt network interfaces. +# + +# ----------------------------------------------------------------------------- + +export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin" +export LC_ALL=C + +PROGRAM_NAME="$(basename "${0}")" + +logdate() { + date "+%Y-%m-%d %H:%M:%S" +} + +log() { + local status="${1}" + shift + + echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}" + +} + +warning() { + log WARNING "${@}" +} + +error() { + log ERROR "${@}" +} + +info() { + log INFO "${@}" +} + +fatal() { + log FATAL "${@}" + exit 1 +} + +debug=0 +debug() { + [ "${debug}" = "1" ] && log DEBUG "${@}" +} + +# ----------------------------------------------------------------------------- +# check for BASH v4+ (required for associative arrays) + +[ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \ + fatal "BASH version 4 or later is required (this is ${BASH_VERSION})." + +# ----------------------------------------------------------------------------- +# defaults to allow running this script by hand + +[ -z "${NETDATA_PLUGINS_DIR}" ] && NETDATA_PLUGINS_DIR="$(dirname "${0}")" +[ -z "${NETDATA_CONFIG_DIR}" ] && NETDATA_CONFIG_DIR="$(dirname "${0}")/../../../../etc/netdata" +[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="$(dirname "${0}")/../../../../var/cache/netdata" + +# ----------------------------------------------------------------------------- +# parse the arguments + +pid= +cgroup= +while [ ! -z "${1}" ] +do + case "${1}" in + --cgroup) cgroup="${2}"; shift 1;; + --pid|-p) pid="${2}"; shift 1;; + --debug|debug) debug=1;; + *) fatal "Cannot understand argument '${1}'";; + esac + + shift +done + +if [ -z "${pid}" -a -z "${cgroup}" ] +then + fatal "Either --pid or --cgroup is required" +fi + +# ----------------------------------------------------------------------------- + +set_source() { + [ ${debug} -eq 1 ] && echo "SRC ${*}" +} + + +# ----------------------------------------------------------------------------- +# veth interfaces via cgroup + +# cgroup-network can detect veth interfaces by itself (written in C). +# If you seek for a shell version of what it does, check this: +# https://github.com/firehol/netdata/issues/474#issuecomment-317866709 + + +# ----------------------------------------------------------------------------- +# tun/tap interfaces via /proc/PID/fdinfo + +# find any tun/tap devices linked to a pid +proc_pid_fdinfo_iff() { + local p="${1}" # the pid + + debug "Searching for tun/tap interfaces for pid ${p}..." + set_source "fdinfo" + grep ^iff:.* "${NETDATA_HOST_PREFIX}/proc/${p}/fdinfo"/* 2>/dev/null | cut -f 2 +} + +find_tun_tap_interfaces_for_cgroup() { + local c="${1}" # the cgroup path + + # for each pid of the cgroup + # find any tun/tap devices linked to the pid + if [ -f "${c}/emulator/cgroup.procs" ] + then + local p + for p in $(< "${c}/emulator/cgroup.procs" ) + do + proc_pid_fdinfo_iff ${p} + done + fi +} + + +# ----------------------------------------------------------------------------- +# virsh domain network interfaces + +virsh_cgroup_to_domain_name() { + local c="${1}" # the cgroup path + + debug "extracting a possible virsh domain from cgroup ${c}..." + + # extract for the cgroup path + sed -n -e "s|.*/machine-qemu\\\\x2d[0-9]\+\\\\x2d\(.*\)\.scope$|\1|p" \ + -e "s|.*/machine/\(.*\)\.libvirt-qemu$|\1|p" \ + <= (3, 1) - import importlib.machinery - PY_VERSION = 3 - # change this hack below if we want PY_VERSION to be used in modules - # import builtins - # builtins.PY_VERSION = 3 - msg.info('Using python v3') -except (AssertionError, ImportError): - try: - import imp - - # change this hack below if we want PY_VERSION to be used in modules - # import __builtin__ - # __builtin__.PY_VERSION = 2 - PY_VERSION = 2 - msg.info('Using python v2') - except ImportError: - msg.fatal('Cannot start. No importlib.machinery on python3 or lack of imp on python2') -# try: -# import yaml -# except ImportError: -# msg.fatal('Cannot find yaml library') -try: - if PY_VERSION == 3: - import pyyaml3 as yaml - else: - import pyyaml2 as yaml -except ImportError: - msg.fatal('Cannot find yaml library') +ALL_MODULES = [m for m in sorted(os.listdir(CHARTS_PY_DIR)) if module_ok(m)] -try: - from collections import OrderedDict - ORDERED = True - DICT = OrderedDict - msg.info('YAML output is ordered') -except ImportError: - try: - from ordereddict import OrderedDict - ORDERED = True - DICT = OrderedDict - msg.info('YAML output is ordered') - except ImportError: - ORDERED = False - DICT = dict - msg.info('YAML output is unordered') -if ORDERED: - def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict): - class OrderedLoader(Loader): - pass - - def construct_mapping(loader, node): - loader.flatten_mapping(node) - return object_pairs_hook(loader.construct_pairs(node)) - OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping) - return yaml.load(stream, OrderedLoader) - - -class PythonCharts(object): - """ - Main class used to control every python module. - """ - - def __init__(self, - modules=None, - modules_path='../python.d/', - modules_configs='../conf.d/', - modules_disabled=None, - modules_enabled=None, - default_run=None): + +def parse_cmd(): + debug = 'debug' in argv[1:] + trace = 'trace' in argv[1:] + override_update_every = next((arg for arg in argv[1:] if arg.isdigit() and int(arg) > 1), False) + modules = [''.join([m, MODULE_EXTENSION]) for m in argv[1:] if ''.join([m, MODULE_EXTENSION]) in ALL_MODULES] + return debug, trace, override_update_every, modules or ALL_MODULES + + +def multi_job_check(config): + return next((True for key in config if isinstance(config[key], dict)), False) + + +class Job(object): + def __init__(self, initialized_job, job_id): """ - :param modules: list - :param modules_path: str - :param modules_configs: str - :param modules_disabled: list - :param modules_enabled: list - :param default_run: bool + :param initialized_job: instance of + :param job_id: """ + self.job = initialized_job + self.id = job_id # key in Modules.jobs() + self.module_name = self.job.__module__ # used in Plugin.delete_job() + self.recheck_every = self.job.configuration.pop('autodetection_retry') + self.checked = False # used in Plugin.check_job() + self.created = False # used in Plugin.create_job_charts() + if OVERRIDE_UPDATE_EVERY: + self.job.update_every = int(OVERRIDE_UPDATE_EVERY) - if modules is None: - modules = [] - if modules_disabled is None: - modules_disabled = [] + def __getattr__(self, item): + return getattr(self.job, item) - self.first_run = True - # set configuration directory - self.configs = modules_configs + def __repr__(self): + return self.job.__repr__() - # load modules - loaded_modules = self._load_modules(modules_path, modules, modules_disabled, modules_enabled, default_run) + def is_dead(self): + return bool(self.ident) and not self.is_alive() - # load configuration files - configured_modules = self._load_configs(loaded_modules) + def not_launched(self): + return not bool(self.ident) - # good economy and prosperity: - self.jobs = self._create_jobs(configured_modules) # type + def is_autodetect(self): + return self.recheck_every - # enable timetable override like `python.d.plugin mysql debug 1` - if DEBUG_FLAG and OVERRIDE_UPDATE_EVERY: - for job in self.jobs: - job.create_timetable(BASE_CONFIG['update_every']) - @staticmethod - def _import_module(path, name=None): +class Module(object): + def __init__(self, service, config): """ - Try to import module using only its path. - :param path: str - :param name: str - :return: object + :param service: + :param config: """ + self.service = service + self.name = service.__name__ + self.config = self.jobs_configurations_builder(config) + self.jobs = OrderedDict() + self.counter = 1 - if name is None: - name = path.split('/')[-1] - if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION: - return None - name = name[:-len(MODULE_EXTENSION)] - try: - if PY_VERSION == 3: - return importlib.machinery.SourceFileLoader(name, path).load_module() - else: - return imp.load_source(name, path) - except Exception as e: - msg.error("Problem loading", name, str(e)) - return None + self.initialize_jobs() + + def __repr__(self): + return "".format(name=self.name) + + def __iter__(self): + return iter(OrderedDict(self.jobs).values()) + + def __getitem__(self, item): + return self.jobs[item] + + def __delitem__(self, key): + del self.jobs[key] + + def __len__(self): + return len(self.jobs) + + def __bool__(self): + return bool(self.jobs) - def _load_modules(self, path, modules, disabled, enabled, default_run): + def __nonzero__(self): + return self.__bool__() + + def jobs_configurations_builder(self, config): """ - Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files) - :param path: str - :param modules: list - :param disabled: list - :return: list + :param config: + :return: """ + counter = 0 + job_base_config = dict() - # check if plugin directory exists - if not os.path.isdir(path): - msg.fatal("cannot find charts directory ", path) - - # load modules - loaded = [] - if len(modules) > 0: - for m in modules: - if m in disabled: - continue - mod = self._import_module(path + m + MODULE_EXTENSION) - if mod is not None: - loaded.append(mod) - else: # exit if plugin is not found - msg.fatal('no modules found.') - else: - # scan directory specified in path and load all modules from there - if default_run is False: - names = [module for module in os.listdir(path) if module[:-9] in enabled] - else: - names = os.listdir(path) - for mod in names: - if mod.replace(MODULE_EXTENSION, "") in disabled: - msg.error(mod + ": disabled module ", mod.replace(MODULE_EXTENSION, "")) - continue - m = self._import_module(path + mod) - if m is not None: - msg.debug(mod + ": loading module '" + path + mod + "'") - loaded.append(m) - return loaded - - def _load_configs(self, modules): + for attr in BASE_CONFIG: + job_base_config[attr] = config.pop(attr, getattr(self.service, attr, BASE_CONFIG[attr])) + + if not config: + config = {str(): dict()} + elif not multi_job_check(config): + config = {str(): config} + + for job_name in config: + if not isinstance(config[job_name], dict): + continue + + job_config = setdefault_values(config[job_name], base_dict=job_base_config) + job_name = sub(r'\s+', '_', job_name) + config[job_name]['name'] = sub(r'\s+', '_', config[job_name]['name']) + counter += 1 + job_id = 'job' + str(counter).zfill(3) + + yield job_id, job_name, job_config + + def initialize_jobs(self): """ - Append configuration in list named `config` to every module. - For multi-job modules `config` list is created in _parse_config, - otherwise it is created here based on BASE_CONFIG prototype with None as identifier. - :param modules: list - :return: list + :return: """ - for mod in modules: - configfile = self.configs + mod.__name__ + ".conf" - if os.path.isfile(configfile): - msg.debug(mod.__name__ + ": loading module configuration: '" + configfile + "'") - try: - if not hasattr(mod, 'config'): - mod.config = {} - setattr(mod, - 'config', - self._parse_config(mod, read_config(configfile))) - except Exception as e: - msg.error(mod.__name__ + ": cannot parse configuration file '" + configfile + "':", str(e)) + for job_id, job_name, job_config in self.config: + job_config['job_name'] = job_name + job_config['override_name'] = job_config.pop('name') + + try: + initialized_job = self.service.Service(configuration=job_config) + except Exception as error: + Logger.error("job initialization: '{module_name} {job_name}' " + "=> ['FAILED'] ({error})".format(module_name=self.name, + job_name=job_name, + error=error)) + continue + else: + Logger.debug("job initialization: '{module_name} {job_name}' " + "=> ['OK']".format(module_name=self.name, + job_name=job_name or self.name)) + self.jobs[job_id] = Job(initialized_job=initialized_job, + job_id=job_id) + del self.config + del self.service + + +class Plugin(object): + def __init__(self): + self.loader = ModuleAndConfigLoader() + self.modules = OrderedDict() + self.sleep_time = 1 + self.runs_counter = 0 + self.config, error = self.loader.load_config_from_file(PLUGIN_CONFIG_DIR + 'python.d.conf') + if error: + run_and_exit(Logger.error)(error) + + if not self.config.get('enabled', True): + run_and_exit(Logger.info)('DISABLED in configuration file.') + + self.load_and_initialize_modules() + if not self.modules: + run_and_exit(Logger.info)('No modules to run. Exit...') + + def __iter__(self): + return iter(OrderedDict(self.modules).values()) + + @property + def jobs(self): + return (job for mod in self for job in mod) + + @property + def dead_jobs(self): + return (job for job in self.jobs if job.is_dead()) + + @property + def autodetect_jobs(self): + return [job for job in self.jobs if job.not_launched()] + + def enabled_modules(self): + for mod in MODULES_TO_RUN: + mod_name = mod[:-len(MODULE_EXTENSION)] + mod_path = CHARTS_PY_DIR + mod + conf_path = ''.join([CHARTS_PY_CONFIG_DIR, mod_name, '.conf']) + + if DEBUG: + yield mod, mod_name, mod_path, conf_path else: - msg.error(mod.__name__ + ": configuration file '" + configfile + "' not found. Using defaults.") - # set config if not found - if not hasattr(mod, 'config'): - msg.debug(mod.__name__ + ": setting configuration for only one job") - mod.config = {None: {}} - for var in BASE_CONFIG: - try: - mod.config[None][var] = getattr(mod, var) - except AttributeError: - mod.config[None][var] = BASE_CONFIG[var] - return modules + if all([self.config.get('default_run', True), + self.config.get(mod_name, True)]): + yield mod, mod_name, mod_path, conf_path + + elif all([not self.config.get('default_run'), + self.config.get(mod_name)]): + yield mod, mod_name, mod_path, conf_path + + def load_and_initialize_modules(self): + for mod, mod_name, mod_path, conf_path in self.enabled_modules(): + + # Load module from file ------------------------------------------------------------ + loaded_module, error = self.loader.load_module_from_file(mod_name, mod_path) + log = Logger.error if error else Logger.debug + log("module load source: '{module_name}' => [{status}]".format(status='FAILED' if error else 'OK', + module_name=mod_name)) + if error: + Logger.error("load source error : {0}".format(error)) + continue + + # Load module config from file ------------------------------------------------------ + loaded_config, error = self.loader.load_config_from_file(conf_path) + log = Logger.error if error else Logger.debug + log("module load config: '{module_name}' => [{status}]".format(status='FAILED' if error else 'OK', + module_name=mod_name)) + if error: + Logger.error('load config error : {0}'.format(error)) + + # Service instance initialization --------------------------------------------------- + initialized_module = Module(service=loaded_module, config=loaded_config) + Logger.debug("module status: '{module_name}' => [{status}] " + "(jobs: {jobs_number})".format(status='OK' if initialized_module else 'FAILED', + module_name=initialized_module.name, + jobs_number=len(initialized_module))) + + if initialized_module: + self.modules[initialized_module.name] = initialized_module @staticmethod - def _parse_config(module, config): + def check_job(job): """ - Parse configuration file or extract configuration from module file. - Example of returned dictionary: - config = {'name': { - 'update_every': 2, - 'retries': 3, - 'priority': 30000 - 'other_val': 123}} - :param module: object - :param config: dict - :return: dict + :param job: + :return: """ - if config is None: - config = {} - # get default values - defaults = {} - msg.debug(module.__name__ + ": reading configuration") - for key in BASE_CONFIG: - try: - # get defaults from module config - defaults[key] = int(config.pop(key)) - except (KeyError, ValueError): - try: - # get defaults from module source code - defaults[key] = getattr(module, key) - except (KeyError, ValueError, AttributeError): - # if above failed, get defaults from global dict - defaults[key] = BASE_CONFIG[key] - - # check if there are dict in config dict - many_jobs = False - for name in config: - if isinstance(config[name], DICT): - many_jobs = True - break - - # assign variables needed by supervisor to every job configuration - if many_jobs: - for name in config: - for key in defaults: - if key not in config[name]: - config[name][key] = defaults[key] - # if only one job is needed, values doesn't have to be in dict (in YAML) + try: + check_ok = bool(job.check()) + except Exception as error: + job.error('check() unhandled exception: {error}'.format(error=error)) + return None else: - config = {None: config.copy()} - config[None].update(defaults) - - # return dictionary of jobs where every job has BASE_CONFIG variables - return config + return check_ok @staticmethod - def _create_jobs(modules): - """ - Create jobs based on module.config dictionary and module.Service class definition. - :param modules: list - :return: list + def create_job_charts(job): """ - jobs = [] - for module in modules: - for name in module.config: - # register a new job - conf = module.config[name] - try: - job = module.Service(configuration=conf, name=name) - except Exception as e: - msg.error(module.__name__ + - ("/" + str(name) if name is not None else "") + - ": cannot start job: '" + - str(e)) - continue - else: - # set chart_name (needed to plot run time graphs) - job.chart_name = module.__name__ - if name is not None: - job.chart_name += "_" + name - jobs.append(job) - msg.debug(module.__name__ + ("/" + str(name) if name is not None else "") + ": job added") - - return [j for j in jobs if j is not None] - - def _stop(self, job, reason=None): + :param job: + :return: """ - Stop specified job and remove it from self.jobs list - Also notifies user about job failure if DEBUG_FLAG is set - :param job: object - :param reason: str - """ - prefix = job.__module__ - if job.name is not None and len(job.name) != 0: - prefix += "/" + job.name try: - msg.error("DISABLED:", prefix) - self.jobs.remove(job) - except Exception as e: - msg.debug("This shouldn't happen. NO " + prefix + " IN LIST:" + str(self.jobs) + " ERROR: " + str(e)) - - # TODO remove section below and remove `reason`. - prefix += ": " - if reason is None: - return - elif reason[:3] == "no ": - msg.error(prefix + - "does not seem to have " + - reason[3:] + - "() function. Disabling it.") - elif reason[:7] == "failed ": - msg.error(prefix + - reason[7:] + - "() function reports failure.") - elif reason[:13] == "configuration": - msg.error(prefix + - "configuration file '" + - self.configs + - job.__module__ + - ".conf' not found. Using defaults.") - elif reason[:11] == "misbehaving": - msg.error(prefix + "is " + reason) - - def check(self): - """ - Tries to execute check() on every job. - This cannot fail thus it is catching every exception - If job.check() fails job is stopped - """ - i = 0 - overridden = [] - msg.debug("all job objects", str(self.jobs)) - while i < len(self.jobs): - job = self.jobs[i] - try: - if not job.check(): - msg.error(job.chart_name, "check() failed - disabling job") - self._stop(job) - else: - msg.info("CHECKED OK:", job.chart_name) - i += 1 - try: - if job.override_name is not None: - new_name = job.__module__ + '_' + sub(r'\s+', '_', job.override_name) - if new_name in overridden: - msg.info("DROPPED:", job.name, ", job '" + job.override_name + - "' is already served by another job.") - self._stop(job) - i -= 1 - else: - job.name = job.override_name - msg.info("RENAMED:", new_name, ", from " + job.chart_name) - job.chart_name = new_name - overridden.append(job.chart_name) - except Exception: - pass - except AttributeError as e: - self._stop(job) - msg.error(job.chart_name, "cannot find check() function or it thrown unhandled exception.") - msg.debug(str(e)) - except (UnboundLocalError, Exception) as e: - msg.error(job.chart_name, str(e)) - self._stop(job) - msg.debug("overridden job names:", str(overridden)) - msg.debug("all remaining job objects:", str(self.jobs)) - - def create(self): - """ - Tries to execute create() on every job. - This cannot fail thus it is catching every exception. - If job.create() fails job is stopped. - This is also creating job run time chart. - """ - i = 0 - while i < len(self.jobs): - job = self.jobs[i] - try: - if not job.create(): - msg.error(job.chart_name, "create function failed.") - self._stop(job) - else: - chart = job.chart_name - sys.stdout.write( - "CHART netdata.plugin_pythond_" + - chart + - " '' 'Execution time for " + - chart + - " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " + - str(job.timetable['freq']) + - '\n') - sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n") - msg.debug("created charts for", job.chart_name) - # sys.stdout.flush() - i += 1 - except AttributeError: - msg.error(job.chart_name, "cannot find create() function or it thrown unhandled exception.") - self._stop(job) - except (UnboundLocalError, Exception) as e: - msg.error(job.chart_name, str(e)) - self._stop(job) - - def update(self): + create_ok = job.create() + except Exception as error: + job.error('create() unhandled exception: {error}'.format(error=error)) + return False + else: + return create_ok + + def delete_job(self, job): """ - Creates and supervises every job thread. - This will stay forever and ever and ever forever and ever it'll be the one... + :param job: + :return: """ - for job in self.jobs: - job.start() + del self.modules[job.module_name][job.id] - while True: - if threading.active_count() <= 1: - msg.fatal("no more jobs") - time.sleep(1) - - -def read_config(path): - """ - Read YAML configuration from specified file - :param path: str - :return: dict - """ - try: - with open(path, 'r') as stream: - if ORDERED: - config = ordered_load(stream, yaml.SafeLoader) + def run_check(self): + checked = list() + for job in self.jobs: + if job.name in checked: + job.info('check() => [DROPPED] (already served by another job)') + self.delete_job(job) + continue + ok = self.check_job(job) + if ok: + job.info('check() => [OK]') + checked.append(job.name) + job.checked = True + continue + if not job.is_autodetect() or ok is None: + job.error('check() => [FAILED]') + self.delete_job(job) else: - config = yaml.load(stream) - except (OSError, IOError) as error: - msg.error(str(path), 'reading error:', str(error)) - return None - except yaml.YAMLError as error: - msg.error(str(path), "is malformed:", str(error)) - return None - return config - - -def parse_cmdline(directory, *commands): - """ - Parse parameters from command line. - :param directory: str - :param commands: list of str - :return: dict - """ - global DEBUG_FLAG, TRACE_FLAG - global OVERRIDE_UPDATE_EVERY - global BASE_CONFIG - - changed_update = False - mods = [] - for cmd in commands[1:]: - if cmd == "check": - pass - elif cmd == "debug" or cmd == "all": - DEBUG_FLAG = True - # redirect stderr to stdout? - elif cmd == "trace" or cmd == "all": - TRACE_FLAG = True - elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd): - # DEBUG_FLAG = True - mods.append(cmd.replace(".chart.py", "")) - else: - try: - BASE_CONFIG['update_every'] = int(cmd) - changed_update = True - except ValueError: - pass - if changed_update and DEBUG_FLAG: - OVERRIDE_UPDATE_EVERY = True - msg.debug(PROGRAM, "overriding update interval to", str(BASE_CONFIG['update_every'])) - - msg.debug("started from", commands[0], "with options:", *commands[1:]) - - return mods - - -# if __name__ == '__main__': -def run(): - """ - Main program. - """ - global DEBUG_FLAG, TRACE_FLAG, BASE_CONFIG - - # read configuration file - disabled = ['nginx_log', 'gunicorn_log', 'apache_cache'] - enabled = list() - default_run = True - configfile = CONFIG_DIR + "python.d.conf" - msg.PROGRAM = PROGRAM - msg.info("reading configuration file:", configfile) - log_throttle = 200 - log_interval = 3600 - - conf = read_config(configfile) - if conf is not None: - try: - # exit the whole plugin when 'enabled: no' is set in 'python.d.conf' - if conf['enabled'] is False: - msg.fatal('disabled in configuration file.\n') - except (KeyError, TypeError): - pass - - try: - for param in BASE_CONFIG: - BASE_CONFIG[param] = conf[param] - except (KeyError, TypeError): - pass # use default update_every from NETDATA_UPDATE_EVERY - - try: - DEBUG_FLAG = conf['debug'] - except (KeyError, TypeError): - pass - - try: - TRACE_FLAG = conf['trace'] - except (KeyError, TypeError): - pass - - try: - log_throttle = conf['logs_per_interval'] - except (KeyError, TypeError): - pass + job.error('check() => [RECHECK] (autodetection_retry: {0})'.format(job.recheck_every)) - try: - log_interval = conf['log_interval'] - except (KeyError, TypeError): - pass + def run_create(self): + for job in self.jobs: + if not job.checked: + # skip autodetection_retry jobs + continue + ok = self.create_job_charts(job) + if ok: + job.debug('create() => [OK] (charts: {0})'.format(len(job.charts))) + job.created = True + continue + job.error('create() => [FAILED] (charts: {0})'.format(len(job.charts))) + self.delete_job(job) - default_run = True if ('default_run' not in conf or conf.get('default_run')) else False + def start(self): + self.run_check() + self.run_create() + for job in self.jobs: + if job.created: + job.start() - for k, v in conf.items(): - if k in ("update_every", "debug", "enabled", "default_run"): - continue - if default_run: - if v is False: - disabled.append(k) - else: - if v is True: - enabled.append(k) - # parse passed command line arguments - modules = parse_cmdline(MODULES_DIR, *sys.argv) - msg.DEBUG_FLAG = DEBUG_FLAG - msg.TRACE_FLAG = TRACE_FLAG - msg.LOG_THROTTLE = log_throttle - msg.LOG_INTERVAL = log_interval - msg.LOG_COUNTER = 0 - msg.LOG_NEXT_CHECK = 0 - msg.info("MODULES_DIR='" + MODULES_DIR + - "', CONFIG_DIR='" + CONFIG_DIR + - "', UPDATE_EVERY=" + str(BASE_CONFIG['update_every']) + - ", ONLY_MODULES=" + str(modules)) - - # run plugins - charts = PythonCharts(modules, MODULES_DIR, CONFIG_DIR + "python.d/", disabled, enabled, default_run) - charts.check() - charts.create() - charts.update() - msg.fatal("finished") + while True: + if threading.active_count() <= 1 and not self.autodetect_jobs: + run_and_exit(Logger.info)('FINISHED') + + sleep(self.sleep_time) + self.cleanup() + self.autodetect_retry() + + def cleanup(self): + for job in self.dead_jobs: + self.delete_job(job) + for mod in self: + if not mod: + del self.modules[mod.name] + + def autodetect_retry(self): + self.runs_counter += self.sleep_time + for job in self.autodetect_jobs: + if self.runs_counter % job.recheck_every == 0: + checked = self.check_job(job) + if checked: + created = self.create_job_charts(job) + if not created: + self.delete_job(job) + continue + job.start() if __name__ == '__main__': - run() + DEBUG, TRACE, OVERRIDE_UPDATE_EVERY, MODULES_TO_RUN = parse_cmd() + Logger = PythonDLogger() + if DEBUG: + Logger.logger.severity = 'DEBUG' + if TRACE: + Logger.log_traceback = True + Logger.info('Using python {version}'.format(version=PY_VERSION[0])) + + plugin = Plugin() + plugin.start() diff --git a/python.d/Makefile.am b/python.d/Makefile.am index 84c2aeadd..1c84cddb4 100644 --- a/python.d/Makefile.am +++ b/python.d/Makefile.am @@ -14,12 +14,14 @@ dist_python_SCRIPTS = \ dist_python_DATA = \ README.md \ apache.chart.py \ - apache_cache.chart.py \ + beanstalk.chart.py \ bind_rndc.chart.py \ chrony.chart.py \ + couchdb.chart.py \ cpufreq.chart.py \ cpuidle.chart.py \ dns_query_time.chart.py \ + dnsdist.chart.py \ dovecot.chart.py \ elasticsearch.chart.py \ example.chart.py \ @@ -41,6 +43,7 @@ dist_python_DATA = \ phpfpm.chart.py \ postfix.chart.py \ postgres.chart.py \ + powerdns.chart.py \ rabbitmq.chart.py \ redis.chart.py \ retroshare.chart.py \ @@ -57,8 +60,33 @@ pythonmodulesdir=$(pythondir)/python_modules dist_pythonmodules_DATA = \ python_modules/__init__.py \ python_modules/base.py \ - python_modules/msg.py \ - python_modules/lm_sensors.py \ + $(NULL) + +basesdir=$(pythonmodulesdir)/bases +dist_bases_DATA = \ + python_modules/bases/__init__.py \ + python_modules/bases/charts.py \ + python_modules/bases/collection.py \ + python_modules/bases/loaders.py \ + python_modules/bases/loggers.py \ + $(NULL) + +bases_framework_servicesdir=$(basesdir)/FrameworkServices +dist_bases_framework_services_DATA = \ + python_modules/bases/FrameworkServices/__init__.py \ + python_modules/bases/FrameworkServices/ExecutableService.py \ + python_modules/bases/FrameworkServices/LogService.py \ + python_modules/bases/FrameworkServices/MySQLService.py \ + python_modules/bases/FrameworkServices/SimpleService.py \ + python_modules/bases/FrameworkServices/SocketService.py \ + python_modules/bases/FrameworkServices/UrlService.py \ + $(NULL) + +third_partydir=$(pythonmodulesdir)/third_party +dist_third_party_DATA = \ + python_modules/third_party/__init__.py \ + python_modules/third_party/ordereddict.py \ + python_modules/third_party/lm_sensors.py \ $(NULL) pythonyaml2dir=$(pythonmodulesdir)/pyyaml2 diff --git a/python.d/Makefile.in b/python.d/Makefile.in index 104f4f1cf..dda54e1a5 100644 --- a/python.d/Makefile.in +++ b/python.d/Makefile.in @@ -81,6 +81,7 @@ build_triplet = @build@ host_triplet = @host@ DIST_COMMON = $(top_srcdir)/build/subst.inc $(srcdir)/Makefile.in \ $(srcdir)/Makefile.am $(dist_python_SCRIPTS) \ + $(dist_bases_DATA) $(dist_bases_framework_services_DATA) \ $(dist_python_DATA) $(dist_python_urllib3_DATA) \ $(dist_python_urllib3_backports_DATA) \ $(dist_python_urllib3_contrib_DATA) \ @@ -88,7 +89,8 @@ DIST_COMMON = $(top_srcdir)/build/subst.inc $(srcdir)/Makefile.in \ $(dist_python_urllib3_securetransport_DATA) \ $(dist_python_urllib3_ssl_match_hostname_DATA) \ $(dist_python_urllib3_util_DATA) $(dist_pythonmodules_DATA) \ - $(dist_pythonyaml2_DATA) $(dist_pythonyaml3_DATA) + $(dist_pythonyaml2_DATA) $(dist_pythonyaml3_DATA) \ + $(dist_third_party_DATA) subdir = python.d ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/ax_c___atomic.m4 \ @@ -132,8 +134,9 @@ am__uninstall_files_from_dir = { \ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ $(am__cd) "$$dir" && rm -f $$files; }; \ } -am__installdirs = "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(pythondir)" \ - "$(DESTDIR)$(python_urllib3dir)" \ +am__installdirs = "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(basesdir)" \ + "$(DESTDIR)$(bases_framework_servicesdir)" \ + "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(python_urllib3dir)" \ "$(DESTDIR)$(python_urllib3_backportsdir)" \ "$(DESTDIR)$(python_urllib3_contribdir)" \ "$(DESTDIR)$(python_urllib3_packagesdir)" \ @@ -141,7 +144,7 @@ am__installdirs = "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(pythondir)" \ "$(DESTDIR)$(python_urllib3_ssl_match_hostnamedir)" \ "$(DESTDIR)$(python_urllib3_utildir)" \ "$(DESTDIR)$(pythonmodulesdir)" "$(DESTDIR)$(pythonyaml2dir)" \ - "$(DESTDIR)$(pythonyaml3dir)" + "$(DESTDIR)$(pythonyaml3dir)" "$(DESTDIR)$(third_partydir)" SCRIPTS = $(dist_python_SCRIPTS) AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) @@ -162,14 +165,16 @@ am__can_run_installinfo = \ n|no|NO) false;; \ *) (install-info --version) >/dev/null 2>&1;; \ esac -DATA = $(dist_python_DATA) $(dist_python_urllib3_DATA) \ +DATA = $(dist_bases_DATA) $(dist_bases_framework_services_DATA) \ + $(dist_python_DATA) $(dist_python_urllib3_DATA) \ $(dist_python_urllib3_backports_DATA) \ $(dist_python_urllib3_contrib_DATA) \ $(dist_python_urllib3_packages_DATA) \ $(dist_python_urllib3_securetransport_DATA) \ $(dist_python_urllib3_ssl_match_hostname_DATA) \ $(dist_python_urllib3_util_DATA) $(dist_pythonmodules_DATA) \ - $(dist_pythonyaml2_DATA) $(dist_pythonyaml3_DATA) + $(dist_pythonyaml2_DATA) $(dist_pythonyaml3_DATA) \ + $(dist_third_party_DATA) am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) ACLOCAL = @ACLOCAL@ @@ -329,12 +334,14 @@ dist_python_SCRIPTS = \ dist_python_DATA = \ README.md \ apache.chart.py \ - apache_cache.chart.py \ + beanstalk.chart.py \ bind_rndc.chart.py \ chrony.chart.py \ + couchdb.chart.py \ cpufreq.chart.py \ cpuidle.chart.py \ dns_query_time.chart.py \ + dnsdist.chart.py \ dovecot.chart.py \ elasticsearch.chart.py \ example.chart.py \ @@ -356,6 +363,7 @@ dist_python_DATA = \ phpfpm.chart.py \ postfix.chart.py \ postgres.chart.py \ + powerdns.chart.py \ rabbitmq.chart.py \ redis.chart.py \ retroshare.chart.py \ @@ -372,8 +380,33 @@ pythonmodulesdir = $(pythondir)/python_modules dist_pythonmodules_DATA = \ python_modules/__init__.py \ python_modules/base.py \ - python_modules/msg.py \ - python_modules/lm_sensors.py \ + $(NULL) + +basesdir = $(pythonmodulesdir)/bases +dist_bases_DATA = \ + python_modules/bases/__init__.py \ + python_modules/bases/charts.py \ + python_modules/bases/collection.py \ + python_modules/bases/loaders.py \ + python_modules/bases/loggers.py \ + $(NULL) + +bases_framework_servicesdir = $(basesdir)/FrameworkServices +dist_bases_framework_services_DATA = \ + python_modules/bases/FrameworkServices/__init__.py \ + python_modules/bases/FrameworkServices/ExecutableService.py \ + python_modules/bases/FrameworkServices/LogService.py \ + python_modules/bases/FrameworkServices/MySQLService.py \ + python_modules/bases/FrameworkServices/SimpleService.py \ + python_modules/bases/FrameworkServices/SocketService.py \ + python_modules/bases/FrameworkServices/UrlService.py \ + $(NULL) + +third_partydir = $(pythonmodulesdir)/third_party +dist_third_party_DATA = \ + python_modules/third_party/__init__.py \ + python_modules/third_party/ordereddict.py \ + python_modules/third_party/lm_sensors.py \ $(NULL) pythonyaml2dir = $(pythonmodulesdir)/pyyaml2 @@ -552,6 +585,48 @@ uninstall-dist_pythonSCRIPTS: files=`for p in $$list; do echo "$$p"; done | \ sed -e 's,.*/,,;$(transform)'`; \ dir='$(DESTDIR)$(pythondir)'; $(am__uninstall_files_from_dir) +install-dist_basesDATA: $(dist_bases_DATA) + @$(NORMAL_INSTALL) + @list='$(dist_bases_DATA)'; test -n "$(basesdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(basesdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(basesdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(basesdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(basesdir)" || exit $$?; \ + done + +uninstall-dist_basesDATA: + @$(NORMAL_UNINSTALL) + @list='$(dist_bases_DATA)'; test -n "$(basesdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(basesdir)'; $(am__uninstall_files_from_dir) +install-dist_bases_framework_servicesDATA: $(dist_bases_framework_services_DATA) + @$(NORMAL_INSTALL) + @list='$(dist_bases_framework_services_DATA)'; test -n "$(bases_framework_servicesdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(bases_framework_servicesdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(bases_framework_servicesdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(bases_framework_servicesdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(bases_framework_servicesdir)" || exit $$?; \ + done + +uninstall-dist_bases_framework_servicesDATA: + @$(NORMAL_UNINSTALL) + @list='$(dist_bases_framework_services_DATA)'; test -n "$(bases_framework_servicesdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(bases_framework_servicesdir)'; $(am__uninstall_files_from_dir) install-dist_pythonDATA: $(dist_python_DATA) @$(NORMAL_INSTALL) @list='$(dist_python_DATA)'; test -n "$(pythondir)" || list=; \ @@ -783,6 +858,27 @@ uninstall-dist_pythonyaml3DATA: @list='$(dist_pythonyaml3_DATA)'; test -n "$(pythonyaml3dir)" || list=; \ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ dir='$(DESTDIR)$(pythonyaml3dir)'; $(am__uninstall_files_from_dir) +install-dist_third_partyDATA: $(dist_third_party_DATA) + @$(NORMAL_INSTALL) + @list='$(dist_third_party_DATA)'; test -n "$(third_partydir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(third_partydir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(third_partydir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(third_partydir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(third_partydir)" || exit $$?; \ + done + +uninstall-dist_third_partyDATA: + @$(NORMAL_UNINSTALL) + @list='$(dist_third_party_DATA)'; test -n "$(third_partydir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(third_partydir)'; $(am__uninstall_files_from_dir) tags TAGS: ctags CTAGS: @@ -824,7 +920,7 @@ check-am: all-am check: check-am all-am: Makefile $(SCRIPTS) $(DATA) installdirs: - for dir in "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(python_urllib3dir)" "$(DESTDIR)$(python_urllib3_backportsdir)" "$(DESTDIR)$(python_urllib3_contribdir)" "$(DESTDIR)$(python_urllib3_packagesdir)" "$(DESTDIR)$(python_urllib3_securetransportdir)" "$(DESTDIR)$(python_urllib3_ssl_match_hostnamedir)" "$(DESTDIR)$(python_urllib3_utildir)" "$(DESTDIR)$(pythonmodulesdir)" "$(DESTDIR)$(pythonyaml2dir)" "$(DESTDIR)$(pythonyaml3dir)"; do \ + for dir in "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(basesdir)" "$(DESTDIR)$(bases_framework_servicesdir)" "$(DESTDIR)$(pythondir)" "$(DESTDIR)$(python_urllib3dir)" "$(DESTDIR)$(python_urllib3_backportsdir)" "$(DESTDIR)$(python_urllib3_contribdir)" "$(DESTDIR)$(python_urllib3_packagesdir)" "$(DESTDIR)$(python_urllib3_securetransportdir)" "$(DESTDIR)$(python_urllib3_ssl_match_hostnamedir)" "$(DESTDIR)$(python_urllib3_utildir)" "$(DESTDIR)$(pythonmodulesdir)" "$(DESTDIR)$(pythonyaml2dir)" "$(DESTDIR)$(pythonyaml3dir)" "$(DESTDIR)$(third_partydir)"; do \ test -z "$$dir" || $(MKDIR_P) "$$dir"; \ done install: install-am @@ -879,7 +975,9 @@ info: info-am info-am: -install-data-am: install-dist_pythonDATA install-dist_pythonSCRIPTS \ +install-data-am: install-dist_basesDATA \ + install-dist_bases_framework_servicesDATA \ + install-dist_pythonDATA install-dist_pythonSCRIPTS \ install-dist_python_urllib3DATA \ install-dist_python_urllib3_backportsDATA \ install-dist_python_urllib3_contribDATA \ @@ -888,7 +986,7 @@ install-data-am: install-dist_pythonDATA install-dist_pythonSCRIPTS \ install-dist_python_urllib3_ssl_match_hostnameDATA \ install-dist_python_urllib3_utilDATA \ install-dist_pythonmodulesDATA install-dist_pythonyaml2DATA \ - install-dist_pythonyaml3DATA + install-dist_pythonyaml3DATA install-dist_third_partyDATA install-dvi: install-dvi-am @@ -932,7 +1030,9 @@ ps: ps-am ps-am: -uninstall-am: uninstall-dist_pythonDATA uninstall-dist_pythonSCRIPTS \ +uninstall-am: uninstall-dist_basesDATA \ + uninstall-dist_bases_framework_servicesDATA \ + uninstall-dist_pythonDATA uninstall-dist_pythonSCRIPTS \ uninstall-dist_python_urllib3DATA \ uninstall-dist_python_urllib3_backportsDATA \ uninstall-dist_python_urllib3_contribDATA \ @@ -941,15 +1041,18 @@ uninstall-am: uninstall-dist_pythonDATA uninstall-dist_pythonSCRIPTS \ uninstall-dist_python_urllib3_ssl_match_hostnameDATA \ uninstall-dist_python_urllib3_utilDATA \ uninstall-dist_pythonmodulesDATA \ - uninstall-dist_pythonyaml2DATA uninstall-dist_pythonyaml3DATA + uninstall-dist_pythonyaml2DATA uninstall-dist_pythonyaml3DATA \ + uninstall-dist_third_partyDATA .MAKE: install-am install-strip .PHONY: all all-am check check-am clean clean-generic cscopelist-am \ ctags-am distclean distclean-generic distdir dvi dvi-am html \ html-am info info-am install install-am install-data \ - install-data-am install-dist_pythonDATA \ - install-dist_pythonSCRIPTS install-dist_python_urllib3DATA \ + install-data-am install-dist_basesDATA \ + install-dist_bases_framework_servicesDATA \ + install-dist_pythonDATA install-dist_pythonSCRIPTS \ + install-dist_python_urllib3DATA \ install-dist_python_urllib3_backportsDATA \ install-dist_python_urllib3_contribDATA \ install-dist_python_urllib3_packagesDATA \ @@ -957,13 +1060,15 @@ uninstall-am: uninstall-dist_pythonDATA uninstall-dist_pythonSCRIPTS \ install-dist_python_urllib3_ssl_match_hostnameDATA \ install-dist_python_urllib3_utilDATA \ install-dist_pythonmodulesDATA install-dist_pythonyaml2DATA \ - install-dist_pythonyaml3DATA install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-pdf \ - install-pdf-am install-ps install-ps-am install-strip \ - installcheck installcheck-am installdirs maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-generic pdf \ - pdf-am ps ps-am tags-am uninstall uninstall-am \ + install-dist_pythonyaml3DATA install-dist_third_partyDATA \ + install-dvi install-dvi-am install-exec install-exec-am \ + install-html install-html-am install-info install-info-am \ + install-man install-pdf install-pdf-am install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-generic pdf pdf-am ps ps-am tags-am \ + uninstall uninstall-am uninstall-dist_basesDATA \ + uninstall-dist_bases_framework_servicesDATA \ uninstall-dist_pythonDATA uninstall-dist_pythonSCRIPTS \ uninstall-dist_python_urllib3DATA \ uninstall-dist_python_urllib3_backportsDATA \ @@ -973,7 +1078,8 @@ uninstall-am: uninstall-dist_pythonDATA uninstall-dist_pythonSCRIPTS \ uninstall-dist_python_urllib3_ssl_match_hostnameDATA \ uninstall-dist_python_urllib3_utilDATA \ uninstall-dist_pythonmodulesDATA \ - uninstall-dist_pythonyaml2DATA uninstall-dist_pythonyaml3DATA + uninstall-dist_pythonyaml2DATA uninstall-dist_pythonyaml3DATA \ + uninstall-dist_third_partyDATA .in: if sed \ diff --git a/python.d/README.md b/python.d/README.md index 1b04ccdf3..009265f72 100644 --- a/python.d/README.md +++ b/python.d/README.md @@ -125,13 +125,118 @@ If no configuration is given, module will attempt to read log file at `/var/log/ --- +# beanstalk + +Module provides server and tube level statistics: + +**Requirements:** + * `python-beanstalkc` + * `python-yaml` + +**Server statistics:** + +1. **Cpu usage** in cpu time + * user + * system + +2. **Jobs rate** in jobs/s + * total + * timeouts + +3. **Connections rate** in connections/s + * connections + +4. **Commands rate** in commands/s + * put + * peek + * peek-ready + * peek-delayed + * peek-buried + * reserve + * use + * watch + * ignore + * delete + * release + * bury + * kick + * stats + * stats-job + * stats-tube + * list-tubes + * list-tube-used + * list-tubes-watched + * pause-tube + +5. **Current tubes** in tubes + * tubes + +6. **Current jobs** in jobs + * urgent + * ready + * reserved + * delayed + * buried + +7. **Current connections** in connections + * written + * producers + * workers + * waiting + +8. **Binlog** in records/s + * written + * migrated + +9. **Uptime** in seconds + * uptime + +**Per tube statistics:** + +1. **Jobs rate** in jobs/s + * jobs + +2. **Jobs** in jobs + * using + * ready + * reserved + * delayed + * buried + +3. **Connections** in connections + * using + * waiting + * watching + +4. **Commands** in commands/s + * deletes + * pauses + +5. **Pause** in seconds + * since + * left + + +### configuration + +Sample: + +```yaml +host : '127.0.0.1' +port : 11300 +``` + +If no configuration is given, module will attempt to connect to beanstalkd on `127.0.0.1:11300` address + +--- + # bind_rndc Module parses bind dump file to collect real-time performance metrics **Requirements:** * Version of bind must be 9.6 + - * Netdata must have permissions to run `rndc status` + * Netdata must have permissions to run `rndc stats` It produces: @@ -218,6 +323,42 @@ local: --- +# couchdb + +This module monitors vital statistics of a local Apache CouchDB 2.x server, including: + +* Overall server reads/writes +* HTTP traffic breakdown + * Request methods (`GET`, `PUT`, `POST`, etc.) + * Response status codes (`200`, `201`, `4xx`, etc.) +* Active server tasks +* Replication status (CouchDB 2.1 and up only) +* Erlang VM stats +* Optional per-database statistics: sizes, # of docs, # of deleted docs + +### Configuration + +Sample for a local server running on port 5984: +```yaml +local: + user: 'admin' + pass: 'password' + node: 'couchdb@127.0.0.1' +``` + +Be sure to specify a correct admin-level username and password. + +You may also need to change the `node` name; this should match the value of `-name NODENAME` in your CouchDB's `etc/vm.args` file. Typically this is of the form `couchdb@fully.qualified.domain.name` in a cluster, or `couchdb@127.0.0.1` / `couchdb@localhost` for a single-node server. + +If you want per-database statistics, these need to be added to the configuration, separated by spaces: +```yaml +local: + ... + databases: 'db1 db2 db3 ...' +``` + +--- + # cpufreq This module shows the current CPU frequency as set by the cpufreq kernel @@ -271,6 +412,59 @@ It produces one aggregate chart or one chart per dns server, showing the query t --- +# dnsdist + +Module monitor dnsdist performance and health metrics. + +Following charts are drawn: + +1. **Response latency** + * latency-slow + * latency100-1000 + * latency50-100 + * latency10-50 + * latency1-10 + * latency0-1 + +2. **Cache performance** + * cache-hits + * cache-misses + +3. **ACL events** + * acl-drops + * rule-drop + * rule-nxdomain + * rule-refused + +4. **Noncompliant data** + * empty-queries + * no-policy + * noncompliant-queries + * noncompliant-responses + +5. **Queries** + * queries + * rdqueries + * rdqueries + +6. **Health** + * downstream-send-errors + * downstream-timeouts + * servfail-responses + * trunc-failures + +### configuration + +```yaml +localhost: + name : 'local' + url : 'http://127.0.0.1:5053/jsonstat?command=stats' + user : 'username' + pass : 'password' + header: + X-API-Key: 'dnsdist-api-key' +``` + # dovecot This module provides statistics information from dovecot server. @@ -1278,6 +1472,45 @@ When no configuration file is found, module tries to connect to TCP/IP socket: ` --- +# powerdns + +Module monitor powerdns performance and health metrics. + +Following charts are drawn: + +1. **Queries and Answers** + * udp-queries + * udp-answers + * tcp-queries + * tcp-answers + +2. **Cache Usage** + * query-cache-hit + * query-cache-miss + * packetcache-hit + * packetcache-miss + +3. **Cache Size** + * query-cache-size + * packetcache-size + * key-cache-size + * meta-cache-size + +4. **Latency** + * latency + +### configuration + +```yaml +local: + name : 'local' + url : 'http://127.0.0.1:8081/api/v1/servers/localhost/statistics' + header : + X-API-Key: 'change_me' +``` + +--- + # rabbitmq Module monitor rabbitmq performance and health metrics. @@ -1321,10 +1554,10 @@ Following charts are drawn: ```yaml socket: name : 'local' - host : '127.0.0.1' - port : 15672 - user : 'guest' - pass : 'guest' + host : '127.0.0.1' + port : 15672 + user : 'guest' + pass : 'guest' ``` @@ -1573,60 +1806,64 @@ Module uses the `varnishstat` command to provide varnish cache statistics. It produces: -1. **Client metrics** - * session accepted - * session dropped - * good client requests received - -2. **All history hit rate ratio** - * cache hits in percent - * cache miss in percent - * cache hits for pass percent - -3. **Curent poll hit rate ratio** - * cache hits in percent - * cache miss in percent - * cache hits for pass percent - -4. **Thread-related metrics** (only for varnish version 4+) - * total number of threads - * threads created - * threads creation failed - * threads hit max - * length os session queue - * sessions queued for thread - -5. **Backend health** - * backend conn. success - * backend conn. not attempted - * backend conn. too many - * backend conn. failures - * backend conn. reuses - * backend conn. recycles - * backend conn. retry - * backend requests made - -6. **Memory usage** - * memory available in megabytes - * memory allocated in megabytes - -7. **Problems summary** - * session dropped - * session accept failures - * session pipe overflow - * backend conn. not attempted - * fetch failed (all causes) - * backend conn. too many - * threads hit max - * threads destroyed - * length of session queue - * HTTP header overflows - * ESI parse errors - * ESI parse warnings - -8. **Uptime** - * varnish instance uptime in seconds +1. **Connections Statistics** in connections/s + * accepted + * dropped + +2. **Client Requests** in requests/s + * received + +3. **All History Hit Rate Ratio** in percent + * hit + * miss + * hitpass + +4. **Current Poll Hit Rate Ratio** in percent + * hit + * miss + * hitpass +5. **Expired Objects** in expired/s + * objects + +6. **Least Recently Used Nuked Objects** in nuked/s + * objects + + +7. **Number Of Threads In All Pools** in threads + * threads + +8. **Threads Statistics** in threads/s + * created + * failed + * limited + +9. **Current Queue Length** in requests + * in queue + +10. **Backend Connections Statistics** in connections/s + * successful + * unhealthy + * reused + * closed + * resycled + * failed + +10. **Requests To The Backend** in requests/s + * received + +11. **ESI Statistics** in problems/s + * errors + * warnings + +12. **Memory Usage** in MB + * free + * allocated + +13. **Uptime** in seconds + * uptime + + ### configuration No configuration is needed. diff --git a/python.d/apache.chart.py b/python.d/apache.chart.py index 71fe03001..789b3c099 100644 --- a/python.d/apache.chart.py +++ b/python.d/apache.chart.py @@ -2,7 +2,7 @@ # Description: apache netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import UrlService +from bases.FrameworkServices.UrlService import UrlService # default module values (can be overridden per job in `config`) # update_every = 2 @@ -31,9 +31,7 @@ CHARTS = { 'options': [None, 'apache Workers', 'workers', 'workers', 'apache.workers', 'stacked'], 'lines': [ ["idle"], - ["idle_servers", 'idle'], ["busy"], - ["busy_servers", 'busy'] ]}, 'reqpersec': { 'options': [None, 'apache Lifetime Avg. Requests/s', 'requests/s', 'statistics', @@ -42,10 +40,10 @@ CHARTS = { ["requests_sec"] ]}, 'bytespersec': { - 'options': [None, 'apache Lifetime Avg. Bandwidth/s', 'kilobytes/s', 'statistics', + 'options': [None, 'apache Lifetime Avg. Bandwidth/s', 'kilobits/s', 'statistics', 'apache.bytesperreq', 'area'], 'lines': [ - ["size_sec", None, 'absolute', 1, 1000] + ["size_sec", None, 'absolute', 8, 1000] ]}, 'requests': { 'options': [None, 'apache Requests', 'requests/s', 'requests', 'apache.requests', 'line'], @@ -53,9 +51,9 @@ CHARTS = { ["requests", None, 'incremental'] ]}, 'net': { - 'options': [None, 'apache Bandwidth', 'kilobytes/s', 'bandwidth', 'apache.net', 'area'], + 'options': [None, 'apache Bandwidth', 'kilobits/s', 'bandwidth', 'apache.net', 'area'], 'lines': [ - ["sent", None, 'incremental'] + ["sent", None, 'incremental', 8, 1] ]}, 'connections': { 'options': [None, 'apache Connections', 'connections', 'connections', 'apache.connections', 'line'], @@ -94,15 +92,22 @@ class Service(UrlService): self.url = self.configuration.get('url', 'http://localhost/server-status?auto') def check(self): - if UrlService.check(self): - if 'idle_servers' in self._data_from_check: - self.__module__ = 'lighttpd' - for chart in self.definitions: - opts = self.definitions[chart]['options'] - opts[1] = opts[1].replace('apache', 'lighttpd') - opts[4] = opts[4].replace('apache', 'lighttpd') - return True - return False + self._manager = self._build_manager() + data = self._get_data() + if not data: + return None + + if 'idle_servers' in data: + self.module_name = 'lighttpd' + for chart in self.definitions: + if chart == 'workers': + lines = self.definitions[chart]['lines'] + lines[0] = ["idle_servers", 'idle'] + lines[1] = ["busy_servers", 'busy'] + opts = self.definitions[chart]['options'] + opts[1] = opts[1].replace('apache', 'lighttpd') + opts[4] = opts[4].replace('apache', 'lighttpd') + return True def _get_data(self): """ diff --git a/python.d/apache_cache.chart.py b/python.d/apache_cache.chart.py deleted file mode 100644 index 3681a8511..000000000 --- a/python.d/apache_cache.chart.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# Description: apache cache netdata python.d module -# Author: Pawel Krupa (paulfantom) - -from base import LogService - -priority = 60000 -retries = 60 -# update_every = 3 - -ORDER = ['cache'] -CHARTS = { - 'cache': { - 'options': [None, 'apache cached responses', 'percent cached', 'cached', 'apache_cache.cache', 'stacked'], - 'lines': [ - ["hit", 'cache', "percentage-of-absolute-row"], - ["miss", None, "percentage-of-absolute-row"], - ["other", None, "percentage-of-absolute-row"] - ]} -} - - -class Service(LogService): - def __init__(self, configuration=None, name=None): - LogService.__init__(self, configuration=configuration, name=name) - if len(self.log_path) == 0: - self.log_path = "/var/log/apache2/cache.log" - self.order = ORDER - self.definitions = CHARTS - - def _get_data(self): - """ - Parse new log lines - :return: dict - """ - try: - raw = self._get_raw_data() - if raw is None: - return None - elif not raw: - return {'hit': 0, - 'miss': 0, - 'other': 0} - except (ValueError, AttributeError): - return None - - hit = 0 - miss = 0 - other = 0 - for line in raw: - if "cache hit" in line: - hit += 1 - elif "cache miss" in line: - miss += 1 - else: - other += 1 - - return {'hit': hit, - 'miss': miss, - 'other': other} diff --git a/python.d/beanstalk.chart.py b/python.d/beanstalk.chart.py new file mode 100644 index 000000000..8880afdd9 --- /dev/null +++ b/python.d/beanstalk.chart.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# Description: beanstalk netdata python.d module +# Author: l2isbad + +try: + import beanstalkc + BEANSTALKC = True +except ImportError: + BEANSTALKC = False + +try: + import yaml + YAML = True +except ImportError: + YAML = False + +from bases.FrameworkServices.SimpleService import SimpleService + +# default module values (can be overridden per job in `config`) +# update_every = 2 +priority = 60000 +retries = 60 + +ORDER = ['cpu_usage', 'jobs_rate', 'connections_rate', 'commands_rate', 'current_tubes', 'current_jobs', + 'current_connections', 'binlog', 'uptime'] + +CHARTS = { + 'cpu_usage': { + 'options': [None, 'Cpu Usage', 'cpu time', 'server statistics', 'beanstalk.cpu_usage', 'area'], + 'lines': [ + ['rusage-utime', 'user', 'incremental'], + ['rusage-stime', 'system', 'incremental'] + ] + }, + 'jobs_rate': { + 'options': [None, 'Jobs Rate', 'jobs/s', 'server statistics', 'beanstalk.jobs_rate', 'line'], + 'lines': [ + ['total-jobs', 'total', 'incremental'], + ['job-timeouts', 'timeouts', 'incremental'] + ] + }, + 'connections_rate': { + 'options': [None, 'Connections Rate', 'connections/s', 'server statistics', 'beanstalk.connections_rate', + 'area'], + 'lines': [ + ['total-connections', 'connections', 'incremental'] + ] + }, + 'commands_rate': { + 'options': [None, 'Commands Rate', 'commands/s', 'server statistics', 'beanstalk.commands_rate', 'stacked'], + 'lines': [ + ['cmd-put', 'put', 'incremental'], + ['cmd-peek', 'peek', 'incremental'], + ['cmd-peek-ready', 'peek-ready', 'incremental'], + ['cmd-peek-delayed', 'peek-delayed', 'incremental'], + ['cmd-peek-buried', 'peek-buried', 'incremental'], + ['cmd-reserve', 'reserve', 'incremental'], + ['cmd-use', 'use', 'incremental'], + ['cmd-watch', 'watch', 'incremental'], + ['cmd-ignore', 'ignore', 'incremental'], + ['cmd-delete', 'delete', 'incremental'], + ['cmd-release', 'release', 'incremental'], + ['cmd-bury', 'bury', 'incremental'], + ['cmd-kick', 'kick', 'incremental'], + ['cmd-stats', 'stats', 'incremental'], + ['cmd-stats-job', 'stats-job', 'incremental'], + ['cmd-stats-tube', 'stats-tube', 'incremental'], + ['cmd-list-tubes', 'list-tubes', 'incremental'], + ['cmd-list-tube-used', 'list-tube-used', 'incremental'], + ['cmd-list-tubes-watched', 'list-tubes-watched', 'incremental'], + ['cmd-pause-tube', 'pause-tube', 'incremental'] + ] + }, + 'current_tubes': { + 'options': [None, 'Current Tubes', 'tubes', 'server statistics', 'beanstalk.current_tubes', 'area'], + 'lines': [ + ['current-tubes', 'tubes'] + ] + }, + 'current_jobs': { + 'options': [None, 'Current Jobs', 'jobs', 'server statistics', 'beanstalk.current_jobs', 'stacked'], + 'lines': [ + ['current-jobs-urgent', 'urgent'], + ['current-jobs-ready', 'ready'], + ['current-jobs-reserved', 'reserved'], + ['current-jobs-delayed', 'delayed'], + ['current-jobs-buried', 'buried'] + ] + }, + 'current_connections': { + 'options': [None, 'Current Connections', 'connections', 'server statistics', + 'beanstalk.current_connections', 'line'], + 'lines': [ + ['current-connections', 'written'], + ['current-producers', 'producers'], + ['current-workers', 'workers'], + ['current-waiting', 'waiting'] + ] + }, + 'binlog': { + 'options': [None, 'Binlog', 'records/s', 'server statistics', 'beanstalk.binlog', 'line'], + 'lines': [ + ['binlog-records-written', 'written', 'incremental'], + ['binlog-records-migrated', 'migrated', 'incremental'] + ] + }, + 'uptime': { + 'options': [None, 'Uptime', 'seconds', 'server statistics', 'beanstalk.uptime', 'line'], + 'lines': [ + ['uptime'], + ] + } +} + + +def tube_chart_template(name): + order = ['{0}_jobs_rate'.format(name), + '{0}_jobs'.format(name), + '{0}_connections'.format(name), + '{0}_commands'.format(name), + '{0}_pause'.format(name) + ] + family = 'tube {0}'.format(name) + + charts = { + order[0]: { + 'options': [None, 'Job Rate', 'jobs/s', family, 'beanstalk.jobs_rate', 'area'], + 'lines': [ + ['_'.join([name, 'total-jobs']), 'jobs', 'incremental'] + ]}, + order[1]: { + 'options': [None, 'Jobs', 'jobs', family, 'beanstalk.jobs', 'stacked'], + 'lines': [ + ['_'.join([name, 'current-jobs-urgent']), 'urgent'], + ['_'.join([name, 'current-jobs-ready']), 'ready'], + ['_'.join([name, 'current-jobs-reserved']), 'reserved'], + ['_'.join([name, 'current-jobs-delayed']), 'delayed'], + ['_'.join([name, 'current-jobs-buried']), 'buried'] + ]}, + order[2]: { + 'options': [None, 'Connections', 'connections', family, 'beanstalk.connections', 'stacked'], + 'lines': [ + ['_'.join([name, 'current-using']), 'using'], + ['_'.join([name, 'current-waiting']), 'waiting'], + ['_'.join([name, 'current-watching']), 'watching'] + ]}, + order[3]: { + 'options': [None, 'Commands', 'commands/s', family, 'beanstalk.commands', 'stacked'], + 'lines': [ + ['_'.join([name, 'cmd-delete']), 'deletes', 'incremental'], + ['_'.join([name, 'cmd-pause-tube']), 'pauses', 'incremental'] + ]}, + order[4]: { + 'options': [None, 'Pause', 'seconds', family, 'beanstalk.pause', 'stacked'], + 'lines': [ + ['_'.join([name, 'pause']), 'since'], + ['_'.join([name, 'pause-time-left']), 'left'] + ]} + + } + + return order, charts + + +class Service(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.configuration = configuration + self.order = list(ORDER) + self.definitions = dict(CHARTS) + self.conn = None + self.alive = True + + def check(self): + if not BEANSTALKC: + self.error("'beanstalkc' module is needed to use beanstalk.chart.py") + return False + + if not YAML: + self.error("'yaml' module is needed to use beanstalk.chart.py") + return False + + self.conn = self.connect() + + return True if self.conn else False + + def get_data(self): + """ + :return: dict + """ + if not self.is_alive(): + return None + + active_charts = self.charts.active_charts() + data = dict() + + try: + data.update(self.conn.stats()) + + for tube in self.conn.tubes(): + stats = self.conn.stats_tube(tube) + + if tube + '_jobs_rate' not in active_charts: + self.create_new_tube_charts(tube) + + for stat in stats: + data['_'.join([tube, stat])] = stats[stat] + + except beanstalkc.SocketError: + self.alive = False + return None + + return data or None + + def create_new_tube_charts(self, tube): + order, charts = tube_chart_template(tube) + + for chart_name in order: + params = [chart_name] + charts[chart_name]['options'] + dimensions = charts[chart_name]['lines'] + + new_chart = self.charts.add_chart(params) + for dimension in dimensions: + new_chart.add_dimension(dimension) + + def connect(self): + host = self.configuration.get('host', '127.0.0.1') + port = self.configuration.get('port', 11300) + timeout = self.configuration.get('timeout', 1) + try: + return beanstalkc.Connection(host=host, + port=port, + connect_timeout=timeout, + parse_yaml=yaml.load) + except beanstalkc.SocketError as error: + self.error('Connection to {0}:{1} failed: {2}'.format(host, port, error)) + return None + + def reconnect(self): + try: + self.conn.reconnect() + self.alive = True + return True + except beanstalkc.SocketError: + return False + + def is_alive(self): + if not self.alive: + return self.reconnect() + return True diff --git a/python.d/bind_rndc.chart.py b/python.d/bind_rndc.chart.py index 5a9749287..cc96659b2 100644 --- a/python.d/bind_rndc.chart.py +++ b/python.d/bind_rndc.chart.py @@ -2,11 +2,13 @@ # Description: bind rndc netdata python.d module # Author: l2isbad -from os.path import getsize -from os import access, R_OK -from subprocess import Popen +import os + from collections import defaultdict -from base import SimpleService +from subprocess import Popen + +from bases.collection import find_binary +from bases.FrameworkServices.SimpleService import SimpleService priority = 60000 retries = 60 @@ -94,7 +96,7 @@ class Service(SimpleService): self.order = ORDER self.definitions = CHARTS self.named_stats_path = self.configuration.get('named_stats_path', '/var/log/bind/named.stats') - self.rndc = self.find_binary('rndc') + self.rndc = find_binary('rndc') self.data = dict(nms_requests=0, nms_responses=0, nms_failure=0, nms_auth=0, nms_non_auth=0, nms_nxrrset=0, nms_success=0, nms_nxdomain=0, nms_recursion=0, nms_duplicate=0, nms_rejected_queries=0, @@ -102,10 +104,10 @@ class Service(SimpleService): def check(self): if not self.rndc: - self.error('Can\'t locate \'rndc\' binary or binary is not executable by netdata') + self.error('Can\'t locate "rndc" binary or binary is not executable by netdata') return False - if not access(self.named_stats_path, R_OK): + if not (os.path.isfile(self.named_stats_path) and os.access(self.named_stats_path, os.R_OK)): self.error('Cannot access file %s' % self.named_stats_path) return False @@ -124,7 +126,7 @@ class Service(SimpleService): """ result = dict() try: - current_size = getsize(self.named_stats_path) + current_size = os.path.getsize(self.named_stats_path) run_rndc = Popen([self.rndc, 'stats'], shell=False) run_rndc.wait() @@ -159,12 +161,12 @@ class Service(SimpleService): parsed_key, chart_name = elem[0], elem[1] for dimension_id, value in queries_mapper(data=parsed[parsed_key], add=chart_name[:9]).items(): + if dimension_id not in self.data: dimension = dimension_id.replace(chart_name[:9], '') - self._add_new_dimension(dimension_id=dimension_id, - dimension=dimension, - chart_name=chart_name, - priority=self.priority + self.order.index(chart_name)) + if dimension_id not in self.charts[chart_name]: + self.charts[chart_name].add_dimension([dimension_id, dimension, 'incremental']) + self.data[dimension_id] = value self.data['stats_size'] = raw_data['size'] diff --git a/python.d/chrony.chart.py b/python.d/chrony.chart.py index 96d7e696e..8f331fa50 100644 --- a/python.d/chrony.chart.py +++ b/python.d/chrony.chart.py @@ -2,10 +2,10 @@ # Description: chrony netdata python.d module # Author: Dominik Schloesser (domschl) -from base import ExecutableService +from bases.FrameworkServices.ExecutableService import ExecutableService # default module values (can be overridden per job in `config`) -# update_every = 10 +update_every = 5 priority = 60000 retries = 10 diff --git a/python.d/couchdb.chart.py b/python.d/couchdb.chart.py new file mode 100644 index 000000000..558bac587 --- /dev/null +++ b/python.d/couchdb.chart.py @@ -0,0 +1,410 @@ +# -*- coding: utf-8 -*- +# Description: couchdb netdata python.d module +# Author: wohali +# Thanks to l2isbad for good examples :) + +from collections import namedtuple, defaultdict +from json import loads +from threading import Thread +from socket import gethostbyname, gaierror +try: + from queue import Queue +except ImportError: + from Queue import Queue + +from bases.FrameworkServices.UrlService import UrlService + +# default module values (can be overridden per job in `config`) +update_every = 1 +priority = 60000 +retries = 60 + +METHODS = namedtuple('METHODS', ['get_data', 'url', 'stats']) + +OVERVIEW_STATS = [ + 'couchdb.database_reads.value', + 'couchdb.database_writes.value', + 'couchdb.httpd.view_reads.value' + 'couchdb.httpd_request_methods.COPY.value', + 'couchdb.httpd_request_methods.DELETE.value', + 'couchdb.httpd_request_methods.GET.value', + 'couchdb.httpd_request_methods.HEAD.value', + 'couchdb.httpd_request_methods.OPTIONS.value', + 'couchdb.httpd_request_methods.POST.value', + 'couchdb.httpd_request_methods.PUT.value', + 'couchdb.httpd_status_codes.200.value', + 'couchdb.httpd_status_codes.201.value', + 'couchdb.httpd_status_codes.202.value', + 'couchdb.httpd_status_codes.204.value', + 'couchdb.httpd_status_codes.206.value', + 'couchdb.httpd_status_codes.301.value', + 'couchdb.httpd_status_codes.302.value', + 'couchdb.httpd_status_codes.304.value', + 'couchdb.httpd_status_codes.400.value', + 'couchdb.httpd_status_codes.401.value', + 'couchdb.httpd_status_codes.403.value', + 'couchdb.httpd_status_codes.404.value', + 'couchdb.httpd_status_codes.405.value', + 'couchdb.httpd_status_codes.406.value', + 'couchdb.httpd_status_codes.409.value', + 'couchdb.httpd_status_codes.412.value', + 'couchdb.httpd_status_codes.413.value', + 'couchdb.httpd_status_codes.414.value', + 'couchdb.httpd_status_codes.415.value', + 'couchdb.httpd_status_codes.416.value', + 'couchdb.httpd_status_codes.417.value', + 'couchdb.httpd_status_codes.500.value', + 'couchdb.httpd_status_codes.501.value', + 'couchdb.open_os_files.value', + 'couch_replicator.jobs.running.value', + 'couch_replicator.jobs.pending.value', + 'couch_replicator.jobs.crashed.value', +] + +SYSTEM_STATS = [ + 'context_switches', + 'run_queue', + 'ets_table_count', + 'reductions', + 'memory.atom', + 'memory.atom_used', + 'memory.binary', + 'memory.code', + 'memory.ets', + 'memory.other', + 'memory.processes', + 'io_input', + 'io_output', + 'os_proc_count', + 'process_count', + 'internal_replication_jobs' +] + +DB_STATS = [ + 'doc_count', + 'doc_del_count', + 'sizes.file', + 'sizes.external', + 'sizes.active' +] + +ORDER = [ + 'activity', + 'request_methods', + 'response_codes', + 'active_tasks', + 'replicator_jobs', + 'open_files', + 'db_sizes_file', + 'db_sizes_external', + 'db_sizes_active', + 'db_doc_counts', + 'db_doc_del_counts', + 'erlang_memory', + 'erlang_proc_counts', + 'erlang_peak_msg_queue', + 'erlang_reductions' +] + +CHARTS = { + 'activity': { + 'options': [None, 'Overall Activity', 'req/s', + 'dbactivity', 'couchdb.activity', 'stacked'], + 'lines': [ + ['couchdb_database_reads', 'DB reads', 'incremental'], + ['couchdb_database_writes', 'DB writes', 'incremental'], + ['couchdb_httpd_view_reads', 'View reads', 'incremental'] + ] + }, + 'request_methods': { + 'options': [None, 'HTTP request methods', 'req/s', + 'httptraffic', 'couchdb.request_methods', + 'stacked'], + 'lines': [ + ['couchdb_httpd_request_methods_COPY', 'COPY', 'incremental'], + ['couchdb_httpd_request_methods_DELETE', 'DELETE', 'incremental'], + ['couchdb_httpd_request_methods_GET', 'GET', 'incremental'], + ['couchdb_httpd_request_methods_HEAD', 'HEAD', 'incremental'], + ['couchdb_httpd_request_methods_OPTIONS', 'OPTIONS', + 'incremental'], + ['couchdb_httpd_request_methods_POST', 'POST', 'incremental'], + ['couchdb_httpd_request_methods_PUT', 'PUT', 'incremental'] + ] + }, + 'response_codes': { + 'options': [None, 'HTTP response status codes', 'resp/s', + 'httptraffic', 'couchdb.response_codes', + 'stacked'], + 'lines': [ + ['couchdb_httpd_status_codes_200', '200 OK', 'incremental'], + ['couchdb_httpd_status_codes_201', '201 Created', 'incremental'], + ['couchdb_httpd_status_codes_202', '202 Accepted', 'incremental'], + ['couchdb_httpd_status_codes_2xx', 'Other 2xx Success', + 'incremental'], + ['couchdb_httpd_status_codes_3xx', '3xx Redirection', + 'incremental'], + ['couchdb_httpd_status_codes_4xx', '4xx Client error', + 'incremental'], + ['couchdb_httpd_status_codes_5xx', '5xx Server error', + 'incremental'] + ] + }, + 'open_files': { + 'options': [None, 'Open files', 'files', + 'ops', 'couchdb.open_files', 'line'], + 'lines': [ + ['couchdb_open_os_files', '# files', 'absolute'] + ] + }, + 'active_tasks': { + 'options': [None, 'Active task breakdown', 'tasks', + 'ops', 'couchdb.active_tasks', 'stacked'], + 'lines': [ + ['activetasks_indexer', 'Indexer', 'absolute'], + ['activetasks_database_compaction', 'DB Compaction', 'absolute'], + ['activetasks_replication', 'Replication', 'absolute'], + ['activetasks_view_compaction', 'View Compaction', 'absolute'] + ] + }, + 'replicator_jobs': { + 'options': [None, 'Replicator job breakdown', 'jobs', + 'ops', 'couchdb.replicator_jobs', 'stacked'], + 'lines': [ + ['couch_replicator_jobs_running', 'Running', 'absolute'], + ['couch_replicator_jobs_pending', 'Pending', 'absolute'], + ['couch_replicator_jobs_crashed', 'Crashed', 'absolute'], + ['internal_replication_jobs', 'Internal replication jobs', + 'absolute'] + ] + }, + 'erlang_memory': { + 'options': [None, 'Erlang VM memory usage', 'bytes', + 'erlang', 'couchdb.erlang_vm_memory', 'stacked'], + 'lines': [ + ['memory_atom', 'atom', 'absolute'], + ['memory_binary', 'binaries', 'absolute'], + ['memory_code', 'code', 'absolute'], + ['memory_ets', 'ets', 'absolute'], + ['memory_processes', 'procs', 'absolute'], + ['memory_other', 'other', 'absolute'] + ] + }, + 'erlang_reductions': { + 'options': [None, 'Erlang reductions', 'count', + 'erlang', 'couchdb.reductions', 'line'], + 'lines': [ + ['reductions', 'reductions', 'incremental'] + ] + }, + 'erlang_proc_counts': { + 'options': [None, 'Process counts', 'count', + 'erlang', 'couchdb.proccounts', 'line'], + 'lines': [ + ['os_proc_count', 'OS procs', 'absolute'], + ['process_count', 'erl procs', 'absolute'] + ] + }, + 'erlang_peak_msg_queue': { + 'options': [None, 'Peak message queue size', 'count', + 'erlang', 'couchdb.peakmsgqueue', + 'line'], + 'lines': [ + ['peak_msg_queue', 'peak size', 'absolute'] + ] + }, + # Lines for the following are added as part of check() + 'db_sizes_file': { + 'options': [None, 'Database sizes (file)', 'KB', + 'perdbstats', 'couchdb.db_sizes_file', 'line'], + 'lines': [] + }, + 'db_sizes_external': { + 'options': [None, 'Database sizes (external)', 'KB', + 'perdbstats', 'couchdb.db_sizes_external', 'line'], + 'lines': [] + }, + 'db_sizes_active': { + 'options': [None, 'Database sizes (active)', 'KB', + 'perdbstats', 'couchdb.db_sizes_active', 'line'], + 'lines': [] + }, + 'db_doc_counts': { + 'options': [None, 'Database # of docs', 'docs', + 'perdbstats', 'couchdb_db_doc_count', 'line'], + 'lines': [] + }, + 'db_doc_del_counts': { + 'options': [None, 'Database # of deleted docs', 'docs', + 'perdbstats', 'couchdb_db_doc_del_count', 'line'], + 'lines': [] + } +} + + +class Service(UrlService): + def __init__(self, configuration=None, name=None): + UrlService.__init__(self, configuration=configuration, name=name) + self.order = ORDER + self.definitions = CHARTS + self.host = self.configuration.get('host', '127.0.0.1') + self.port = self.configuration.get('port', 5984) + self.node = self.configuration.get('node', 'couchdb@127.0.0.1') + self.scheme = self.configuration.get('scheme', 'http') + self.user = self.configuration.get('user') + self.password = self.configuration.get('pass') + try: + self.dbs = self.configuration.get('databases').split(' ') + except (KeyError, AttributeError): + self.dbs = [] + + def check(self): + if not (self.host and self.port): + self.error('Host is not defined in the module configuration file') + return False + try: + self.host = gethostbyname(self.host) + except gaierror as error: + self.error(str(error)) + return False + self.url = '{scheme}://{host}:{port}'.format(scheme=self.scheme, + host=self.host, + port=self.port) + stats = self.url + '/_node/{node}/_stats'.format(node=self.node) + active_tasks = self.url + '/_active_tasks' + system = self.url + '/_node/{node}/_system'.format(node=self.node) + self.methods = [METHODS(get_data=self._get_overview_stats, + url=stats, + stats=OVERVIEW_STATS), + METHODS(get_data=self._get_active_tasks_stats, + url=active_tasks, + stats=None), + METHODS(get_data=self._get_overview_stats, + url=system, + stats=SYSTEM_STATS), + METHODS(get_data=self._get_dbs_stats, + url=self.url, + stats=DB_STATS)] + # must initialise manager before using _get_raw_data + self._manager = self._build_manager() + self.dbs = [db for db in self.dbs + if self._get_raw_data(self.url + '/' + db)] + for db in self.dbs: + self.definitions['db_sizes_file']['lines'].append( + ['db_'+db+'_sizes_file', db, 'absolute', 1, 1000] + ) + self.definitions['db_sizes_external']['lines'].append( + ['db_'+db+'_sizes_external', db, 'absolute', 1, 1000] + ) + self.definitions['db_sizes_active']['lines'].append( + ['db_'+db+'_sizes_active', db, 'absolute', 1, 1000] + ) + self.definitions['db_doc_counts']['lines'].append( + ['db_'+db+'_doc_count', db, 'absolute'] + ) + self.definitions['db_doc_del_counts']['lines'].append( + ['db_'+db+'_doc_del_count', db, 'absolute'] + ) + return UrlService.check(self) + + def _get_data(self): + threads = list() + queue = Queue() + result = dict() + + for method in self.methods: + th = Thread(target=method.get_data, + args=(queue, method.url, method.stats)) + th.start() + threads.append(th) + + for thread in threads: + thread.join() + result.update(queue.get()) + + # self.info('couchdb result = ' + str(result)) + return result or None + + def _get_overview_stats(self, queue, url, stats): + raw_data = self._get_raw_data(url) + if not raw_data: + return queue.put(dict()) + data = loads(raw_data) + to_netdata = self._fetch_data(raw_data=data, metrics=stats) + if 'message_queues' in data: + to_netdata['peak_msg_queue'] = get_peak_msg_queue(data) + return queue.put(to_netdata) + + def _get_active_tasks_stats(self, queue, url, _): + taskdict = defaultdict(int) + taskdict["activetasks_indexer"] = 0 + taskdict["activetasks_database_compaction"] = 0 + taskdict["activetasks_replication"] = 0 + taskdict["activetasks_view_compaction"] = 0 + raw_data = self._get_raw_data(url) + if not raw_data: + return queue.put(dict()) + data = loads(raw_data) + for task in data: + taskdict["activetasks_" + task["type"]] += 1 + return queue.put(dict(taskdict)) + + def _get_dbs_stats(self, queue, url, stats): + to_netdata = {} + for db in self.dbs: + raw_data = self._get_raw_data(url + '/' + db) + if not raw_data: + continue + data = loads(raw_data) + for metric in stats: + value = data + metrics_list = metric.split('.') + try: + for m in metrics_list: + value = value[m] + except KeyError as e: + self.debug('cannot process ' + metric + ' for ' + db + + ": " + str(e)) + continue + metric_name = 'db_{0}_{1}'.format(db, '_'.join(metrics_list)) + to_netdata[metric_name] = value + return queue.put(to_netdata) + + def _fetch_data(self, raw_data, metrics): + data = dict() + for metric in metrics: + value = raw_data + metrics_list = metric.split('.') + try: + for m in metrics_list: + value = value[m] + except KeyError as e: + self.debug('cannot process ' + metric + ': ' + str(e)) + continue + # strip off .value from end of stat + if metrics_list[-1] == 'value': + metrics_list = metrics_list[:-1] + # sum up 3xx/4xx/5xx + if metrics_list[0:2] == ['couchdb', 'httpd_status_codes'] and \ + int(metrics_list[2]) > 202: + metrics_list[2] = '{0}xx'.format(int(metrics_list[2]) // 100) + if '_'.join(metrics_list) in data: + data['_'.join(metrics_list)] += value + else: + data['_'.join(metrics_list)] = value + else: + data['_'.join(metrics_list)] = value + return data + + +def get_peak_msg_queue(data): + maxsize = 0 + queues = data['message_queues'] + for queue in iter(queues.values()): + if isinstance(queue, dict) and 'count' in queue: + value = queue['count'] + elif isinstance(queue, int): + value = queue + else: + continue + maxsize = max(maxsize, value) + return maxsize diff --git a/python.d/cpufreq.chart.py b/python.d/cpufreq.chart.py index 01cc22b02..3abde736c 100644 --- a/python.d/cpufreq.chart.py +++ b/python.d/cpufreq.chart.py @@ -4,8 +4,8 @@ import glob import os -import time -from base import SimpleService + +from bases.FrameworkServices.SimpleService import SimpleService # default module values (can be overridden per job in `config`) # update_every = 2 @@ -14,12 +14,13 @@ ORDER = ['cpufreq'] CHARTS = { 'cpufreq': { - 'options': [None, 'CPU Clock', 'MHz', 'cpufreq', None, 'line'], + 'options': [None, 'CPU Clock', 'MHz', 'cpufreq', 'cpufreq.cpufreq', 'line'], 'lines': [ # lines are created dynamically in `check()` method ]} } + class Service(SimpleService): def __init__(self, configuration=None, name=None): prefix = os.getenv('NETDATA_HOST_PREFIX', "") @@ -29,7 +30,7 @@ class Service(SimpleService): SimpleService.__init__(self, configuration=configuration, name=name) self.order = ORDER self.definitions = CHARTS - self._orig_name = "" + self.fake_name = 'cpu' self.assignment = {} self.accurate_exists = True self.accurate_last = {} @@ -72,7 +73,6 @@ class Service(SimpleService): if accurate_ok: return data - for name, paths in self.assignment.items(): data[name] = open(paths['inaccurate'], 'r').read() @@ -84,8 +84,6 @@ class Service(SimpleService): except (KeyError, TypeError): self.error("No path specified. Using: '" + self.sys_dir + "'") - self._orig_name = self.chart_name - for path in glob.glob(self.sys_dir + '/system/cpu/cpu*/cpufreq/stats/time_in_state'): path_elem = path.split('/') cpu = path_elem[-4] @@ -113,14 +111,3 @@ class Service(SimpleService): return True - def create(self): - self.chart_name = "cpu" - status = SimpleService.create(self) - self.chart_name = self._orig_name - return status - - def update(self, interval): - self.chart_name = "cpu" - status = SimpleService.update(self, interval=interval) - self.chart_name = self._orig_name - return status diff --git a/python.d/cpuidle.chart.py b/python.d/cpuidle.chart.py index e5ed49bd2..d14c6aaf3 100644 --- a/python.d/cpuidle.chart.py +++ b/python.d/cpuidle.chart.py @@ -5,8 +5,8 @@ import glob import os import platform -import time -from base import SimpleService + +from bases.FrameworkServices.SimpleService import SimpleService import ctypes syscall = ctypes.CDLL('libc.so.6').syscall @@ -14,6 +14,7 @@ syscall = ctypes.CDLL('libc.so.6').syscall # default module values (can be overridden per job in `config`) # update_every = 2 + class Service(SimpleService): def __init__(self, configuration=None, name=None): prefix = os.getenv('NETDATA_HOST_PREFIX', "") @@ -24,11 +25,12 @@ class Service(SimpleService): SimpleService.__init__(self, configuration=configuration, name=name) self.order = [] self.definitions = {} - self._orig_name = "" + self.fake_name = 'cpu' self.assignment = {} self.last_schedstat = None - def __gettid(self): + @staticmethod + def __gettid(): # This is horrendous. We need the *thread id* (not the *process id*), # but there's no Python standard library way of doing that. If you need # to enable this module on a non-x86 machine type, you'll have to find @@ -108,8 +110,6 @@ class Service(SimpleService): self.error("Cannot get thread ID. Stats would be completely broken.") return False - self._orig_name = self.chart_name - for path in sorted(glob.glob(self.sys_dir + '/cpu*/cpuidle/state*/name')): # ['', 'sys', 'devices', 'system', 'cpu', 'cpu0', 'cpuidle', 'state3', 'name'] path_elem = path.split('/') @@ -122,7 +122,7 @@ class Service(SimpleService): self.order.append(orderid) active_name = '%s_active_time' % (cpu,) self.definitions[orderid] = { - 'options': [None, 'C-state residency', 'time%', 'cpuidle', None, 'stacked'], + 'options': [None, 'C-state residency', 'time%', 'cpuidle', 'cpuidle.cpuidle', 'stacked'], 'lines': [ [active_name, 'C0 (active)', 'percentage-of-incremental-row', 1, 1], ], @@ -146,16 +146,3 @@ class Service(SimpleService): return True - def create(self): - self.chart_name = "cpu" - status = SimpleService.create(self) - self.chart_name = self._orig_name - return status - - def update(self, interval): - self.chart_name = "cpu" - status = SimpleService.update(self, interval=interval) - self.chart_name = self._orig_name - return status - -# vim: set ts=4 sts=4 sw=4 et: diff --git a/python.d/dns_query_time.chart.py b/python.d/dns_query_time.chart.py index 9053d9a1b..9a794a9c9 100644 --- a/python.d/dns_query_time.chart.py +++ b/python.d/dns_query_time.chart.py @@ -2,6 +2,10 @@ # Description: dns_query_time netdata python.d module # Author: l2isbad +from random import choice +from threading import Thread +from socket import getaddrinfo, gaierror + try: from time import monotonic as time except ImportError: @@ -15,10 +19,8 @@ try: from queue import Queue except ImportError: from Queue import Queue -from random import choice -from threading import Thread -from socket import gethostbyname, gaierror -from base import SimpleService + +from bases.FrameworkServices.SimpleService import SimpleService # default module values (can be overridden per job in `config`) @@ -43,7 +45,6 @@ class Service(SimpleService): return False self.timeout = self.timeout if isinstance(self.timeout, int) else 4 - self.update_every = self.timeout + 1 if self.update_every <= self.timeout else self.update_every if not all([self.domains, self.server_list, isinstance(self.server_list, str), isinstance(self.domains, str)]): @@ -69,9 +70,7 @@ class Service(SimpleService): if not self.server_list: return False - self._data_from_check = data self.order, self.definitions = create_charts(aggregate=self.aggregate, server_list=self.server_list) - self.info(str({'domains': len(self.domains), 'servers': self.server_list})) return True def _get_data(self, timeout=None): @@ -110,7 +109,7 @@ def dns_request(server_list, timeout, domains): def check_ns(ns): try: - return gethostbyname(ns) + return getaddrinfo(ns, 'domain')[0][4][0] except gaierror: return False diff --git a/python.d/dnsdist.chart.py b/python.d/dnsdist.chart.py new file mode 100644 index 000000000..b40112cbc --- /dev/null +++ b/python.d/dnsdist.chart.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +from json import loads +from bases.FrameworkServices.UrlService import UrlService + +ORDER = ['queries', 'queries_dropped', 'packets_dropped', 'answers', 'backend_responses', 'backend_commerrors', 'backend_errors', 'cache', 'servercpu', 'servermem', 'query_latency', 'query_latency_avg'] +CHARTS = { + 'queries': { + 'options': [None, 'Client queries received', 'queries/s', 'queries', 'dnsdist.queries', 'line'], + 'lines': [ + ['queries', 'all', 'incremental'], + ['rdqueries', 'recursive', 'incremental'], + ['empty-queries', 'empty', 'incremental'] + ]}, + 'queries_dropped': { + 'options': [None, 'Client queries dropped', 'queries/s', 'queries', 'dnsdist.queries_dropped', 'line'], + 'lines': [ + ['rule-drop', 'rule drop', 'incremental'], + ['dyn-blocked', 'dynamic block', 'incremental'], + ['no-policy', 'no policy', 'incremental'], + ['noncompliant-queries', 'non compliant', 'incremental'] + ]}, + 'packets_dropped': { + 'options': [None, 'Packets dropped', 'packets/s', 'packets', 'dnsdist.packets_dropped', 'line'], + 'lines': [ + ['acl-drops', 'acl', 'incremental'] + ]}, + 'answers': { + 'options': [None, 'Answers statistics', 'answers/s', 'answers', 'dnsdist.answers', 'line'], + 'lines': [ + ['self-answered', 'self answered', 'incremental'], + ['rule-nxdomain', 'nxdomain', 'incremental', -1], + ['rule-refused', 'refused', 'incremental', -1], + ['trunc-failures', 'trunc failures', 'incremental', -1] + ]}, + 'backend_responses': { + 'options': [None, 'Backend responses', 'responses/s', 'backends', 'dnsdist.backend_responses', 'line'], + 'lines': [ + ['responses', 'responses', 'incremental'] + ]}, + 'backend_commerrors': { + 'options': [None, 'Backend Communication Errors', 'errors/s', 'backends', 'dnsdist.backend_commerrors', 'line'], + 'lines': [ + ['downstream-send-errors', 'send errors', 'incremental'] + ]}, + 'backend_errors': { + 'options': [None, 'Backend error responses', 'responses/s', 'backends', 'dnsdist.backend_errors', 'line'], + 'lines': [ + ['downstream-timeouts', 'timeout', 'incremental'], + ['servfail-responses', 'servfail', 'incremental'], + ['noncompliant-responses', 'non compliant', 'incremental'] + ]}, + 'cache': { + 'options': [None, 'Cache performance', 'answers/s', 'cache', 'dnsdist.cache', 'area'], + 'lines': [ + ['cache-hits', 'hits', 'incremental'], + ['cache-misses', 'misses', 'incremental', -1] + ]}, + 'servercpu': { + 'options': [None, 'DNSDIST server CPU utilization', 'ms/s', 'server', 'dnsdist.servercpu', 'stacked'], + 'lines': [ + ['cpu-sys-msec', 'system state', 'incremental'], + ['cpu-user-msec', 'user state', 'incremental'] + ]}, + 'servermem': { + 'options': [None, 'DNSDIST server memory utilization', 'MB', 'server', 'dnsdist.servermem', 'area'], + 'lines': [ + ['real-memory-usage', 'memory usage', 'absolute', 1, 1048576] + ]}, + 'query_latency': { + 'options': [None, 'Query latency', 'queries/s', 'latency', 'dnsdist.query_latency', 'stacked'], + 'lines': [ + ['latency0-1', '1ms', 'incremental'], + ['latency1-10', '10ms', 'incremental'], + ['latency10-50', '50ms', 'incremental'], + ['latency50-100', '100ms', 'incremental'], + ['latency100-1000', '1sec', 'incremental'], + ['latency-slow', 'slow', 'incremental'] + ]}, + 'query_latency_avg': { + 'options': [None, 'Average latency for the last N queries', 'ms/query', 'latency', 'dnsdist.query_latency_avg', 'line'], + 'lines': [ + ['latency-avg100', '100', 'absolute'], + ['latency-avg1000', '1k', 'absolute'], + ['latency-avg10000', '10k', 'absolute'], + ['latency-avg1000000', '1000k', 'absolute'] + ]} +} + +class Service(UrlService): + def __init__(self, configuration=None, name=None): + UrlService.__init__(self, configuration=configuration, name=name) + self.order = ORDER + self.definitions = CHARTS + + def _get_data(self): + data = self._get_raw_data() + if not data: + return None + + return loads(data) + diff --git a/python.d/dovecot.chart.py b/python.d/dovecot.chart.py index b2bef4956..5689f2ec9 100644 --- a/python.d/dovecot.chart.py +++ b/python.d/dovecot.chart.py @@ -2,7 +2,7 @@ # Description: dovecot netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import SocketService +from bases.FrameworkServices.SocketService import SocketService # default module values (can be overridden per job in `config`) # update_every = 2 @@ -117,19 +117,10 @@ class Service(SocketService): data = raw.split('\n')[:2] desc = data[0].split('\t') vals = data[1].split('\t') - # ret = dict(zip(desc, vals)) - ret = {} - for i in range(len(desc)): + ret = dict() + for i, _ in enumerate(desc): try: - #d = str(desc[i]) - #if d in ('user_cpu', 'sys_cpu', 'clock_time'): - # val = float(vals[i]) - #else: - # val = int(vals[i]) - #ret[d] = val ret[str(desc[i])] = int(vals[i]) except ValueError: - pass - if len(ret) == 0: - return None - return ret + continue + return ret or None diff --git a/python.d/elasticsearch.chart.py b/python.d/elasticsearch.chart.py index 2e0f18c0f..afdf0f1b4 100644 --- a/python.d/elasticsearch.chart.py +++ b/python.d/elasticsearch.chart.py @@ -11,10 +11,9 @@ try: except ImportError: from Queue import Queue -from base import UrlService +from bases.FrameworkServices.UrlService import UrlService # default module values (can be overridden per job in `config`) -# update_every = 2 update_every = 5 priority = 60000 retries = 60 diff --git a/python.d/example.chart.py b/python.d/example.chart.py index adf97a921..ee7ff62fc 100644 --- a/python.d/example.chart.py +++ b/python.d/example.chart.py @@ -2,35 +2,46 @@ # Description: example netdata python.d module # Author: Pawel Krupa (paulfantom) -import os -import random -from base import SimpleService +from random import SystemRandom -NAME = os.path.basename(__file__).replace(".chart.py", "") +from bases.FrameworkServices.SimpleService import SimpleService # default module values # update_every = 4 priority = 90000 retries = 60 +ORDER = ['random'] +CHARTS = { + 'random': { + 'options': [None, 'A random number', 'random number', 'random', 'random', 'line'], + 'lines': [ + ['random1'] + ] + } +} + class Service(SimpleService): def __init__(self, configuration=None, name=None): - super(self.__class__,self).__init__(configuration=configuration, name=name) + SimpleService.__init__(self, configuration=configuration, name=name) + self.order = ORDER + self.definitions = CHARTS + self.random = SystemRandom() - def check(self): - return True - - def create(self): - self.chart("example.python_random", '', 'A random number', 'random number', - 'random', 'random', 'line', self.priority, self.update_every) - self.dimension('random1') - self.commit() - return True - - def update(self, interval): - self.begin("example.python_random", interval) - self.set("random1", random.randint(0, 100)) - self.end() - self.commit() + @staticmethod + def check(): return True + + def get_data(self): + data = dict() + + for i in range(1, 4): + dimension_id = ''.join(['random', str(i)]) + + if dimension_id not in self.charts['random']: + self.charts['random'].add_dimension([dimension_id]) + + data[dimension_id] = self.random.randint(0, 100) + + return data diff --git a/python.d/exim.chart.py b/python.d/exim.chart.py index 1858cbc70..2e5b924ba 100644 --- a/python.d/exim.chart.py +++ b/python.d/exim.chart.py @@ -2,7 +2,7 @@ # Description: exim netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import ExecutableService +from bases.FrameworkServices.ExecutableService import ExecutableService # default module values (can be overridden per job in `config`) # update_every = 2 diff --git a/python.d/fail2ban.chart.py b/python.d/fail2ban.chart.py index 5238fa16e..895833f87 100644 --- a/python.d/fail2ban.chart.py +++ b/python.d/fail2ban.chart.py @@ -2,12 +2,15 @@ # Description: fail2ban log netdata python.d module # Author: l2isbad +import bisect + +from glob import glob from re import compile as r_compile from os import access as is_accessible, R_OK from os.path import isdir, getsize -from glob import glob -import bisect -from base import LogService + + +from bases.FrameworkServices.LogService import LogService priority = 60000 retries = 60 @@ -180,8 +183,8 @@ def find_jails_in_files(list_of_files, print_error): jails_list = list() for conf in list_of_files: if is_accessible(conf, R_OK): - with open(conf, 'rt') as conf: - raw_data = conf.readlines() + with open(conf, 'rt') as f: + raw_data = f.readlines() data = ' '.join(line for line in raw_data if line.startswith(('[', 'enabled'))) jails_list.extend(REGEX_JAILS.findall(data)) else: diff --git a/python.d/freeradius.chart.py b/python.d/freeradius.chart.py index f3de15735..3acc58d1a 100644 --- a/python.d/freeradius.chart.py +++ b/python.d/freeradius.chart.py @@ -2,15 +2,19 @@ # Description: freeradius netdata python.d module # Author: l2isbad -from base import SimpleService from re import findall from subprocess import Popen, PIPE +from bases.collection import find_binary +from bases.FrameworkServices.SimpleService import SimpleService + # default module values (can be overridden per job in `config`) priority = 60000 retries = 60 update_every = 15 +RADIUS_MSG = 'Message-Authenticator = 0x00, FreeRADIUS-Statistics-Type = 15, Response-Packet-Type = Access-Accept' + # charts order (can be overridden if you want less charts, or different order) ORDER = ['authentication', 'accounting', 'proxy-auth', 'proxy-acct'] @@ -18,34 +22,46 @@ CHARTS = { 'authentication': { 'options': [None, "Authentication", "packets/s", 'Authentication', 'freerad.auth', 'line'], 'lines': [ - ['access-accepts', None, 'incremental'], ['access-rejects', None, 'incremental'], - ['auth-dropped-requests', None, 'incremental'], ['auth-duplicate-requests', None, 'incremental'], - ['auth-invalid-requests', None, 'incremental'], ['auth-malformed-requests', None, 'incremental'], - ['auth-unknown-types', None, 'incremental'] - ]}, - 'accounting': { + ['access-accepts', None, 'incremental'], + ['access-rejects', None, 'incremental'], + ['auth-dropped-requests', 'dropped-requests', 'incremental'], + ['auth-duplicate-requests', 'duplicate-requests', 'incremental'], + ['auth-invalid-requests', 'invalid-requests', 'incremental'], + ['auth-malformed-requests', 'malformed-requests', 'incremental'], + ['auth-unknown-types', 'unknown-types', 'incremental'] + ]}, + 'accounting': { 'options': [None, "Accounting", "packets/s", 'Accounting', 'freerad.acct', 'line'], 'lines': [ - ['accounting-requests', None, 'incremental'], ['accounting-responses', None, 'incremental'], - ['acct-dropped-requests', None, 'incremental'], ['acct-duplicate-requests', None, 'incremental'], - ['acct-invalid-requests', None, 'incremental'], ['acct-malformed-requests', None, 'incremental'], - ['acct-unknown-types', None, 'incremental'] + ['accounting-requests', 'requests', 'incremental'], + ['accounting-responses', 'responses', 'incremental'], + ['acct-dropped-requests', 'dropped-requests', 'incremental'], + ['acct-duplicate-requests', 'duplicate-requests', 'incremental'], + ['acct-invalid-requests', 'invalid-requests', 'incremental'], + ['acct-malformed-requests', 'malformed-requests', 'incremental'], + ['acct-unknown-types', 'unknown-types', 'incremental'] ]}, 'proxy-auth': { 'options': [None, "Proxy Authentication", "packets/s", 'Authentication', 'freerad.proxy.auth', 'line'], 'lines': [ - ['proxy-access-accepts', None, 'incremental'], ['proxy-access-rejects', None, 'incremental'], - ['proxy-auth-dropped-requests', None, 'incremental'], ['proxy-auth-duplicate-requests', None, 'incremental'], - ['proxy-auth-invalid-requests', None, 'incremental'], ['proxy-auth-malformed-requests', None, 'incremental'], - ['proxy-auth-unknown-types', None, 'incremental'] + ['proxy-access-accepts', 'access-accepts', 'incremental'], + ['proxy-access-rejects', 'access-rejects', 'incremental'], + ['proxy-auth-dropped-requests', 'dropped-requests', 'incremental'], + ['proxy-auth-duplicate-requests', 'duplicate-requests', 'incremental'], + ['proxy-auth-invalid-requests', 'invalid-requests', 'incremental'], + ['proxy-auth-malformed-requests', 'malformed-requests', 'incremental'], + ['proxy-auth-unknown-types', 'unknown-types', 'incremental'] ]}, - 'proxy-acct': { + 'proxy-acct': { 'options': [None, "Proxy Accounting", "packets/s", 'Accounting', 'freerad.proxy.acct', 'line'], 'lines': [ - ['proxy-accounting-requests', None, 'incremental'], ['proxy-accounting-responses', None, 'incremental'], - ['proxy-acct-dropped-requests', None, 'incremental'], ['proxy-acct-duplicate-requests', None, 'incremental'], - ['proxy-acct-invalid-requests', None, 'incremental'], ['proxy-acct-malformed-requests', None, 'incremental'], - ['proxy-acct-unknown-types', None, 'incremental'] + ['proxy-accounting-requests', 'requests', 'incremental'], + ['proxy-accounting-responses', 'responses', 'incremental'], + ['proxy-acct-dropped-requests', 'dropped-requests', 'incremental'], + ['proxy-acct-duplicate-requests', 'duplicate-requests', 'incremental'], + ['proxy-acct-invalid-requests', 'invalid-requests', 'incremental'], + ['proxy-acct-malformed-requests', 'malformed-requests', 'incremental'], + ['proxy-acct-unknown-types', 'unknown-types', 'incremental'] ]} } @@ -54,31 +70,33 @@ CHARTS = { class Service(SimpleService): def __init__(self, configuration=None, name=None): SimpleService.__init__(self, configuration=configuration, name=name) + self.definitions = CHARTS self.host = self.configuration.get('host', 'localhost') self.port = self.configuration.get('port', '18121') - self.secret = self.configuration.get('secret', 'adminsecret') + self.secret = self.configuration.get('secret') self.acct = self.configuration.get('acct', False) self.proxy_auth = self.configuration.get('proxy_auth', False) self.proxy_acct = self.configuration.get('proxy_acct', False) - self.echo = self.find_binary('echo') - self.radclient = self.find_binary('radclient') - self.sub_echo = [self.echo, 'Message-Authenticator = 0x00, FreeRADIUS-Statistics-Type = 15, Response-Packet-Type = Access-Accept'] - self.sub_radclient = [self.radclient, '-r', '1', '-t', '1', ':'.join([self.host, self.port]), 'status', self.secret] - + chart_choice = [True, bool(self.acct), bool(self.proxy_auth), bool(self.proxy_acct)] + self.order = [chart for chart, choice in zip(ORDER, chart_choice) if choice] + self.echo = find_binary('echo') + self.radclient = find_binary('radclient') + self.sub_echo = [self.echo, RADIUS_MSG] + self.sub_radclient = [self.radclient, '-r', '1', '-t', '1', '-x', + ':'.join([self.host, self.port]), 'status', self.secret] + def check(self): if not all([self.echo, self.radclient]): - self.error('Can\'t locate \'radclient\' binary or binary is not executable by netdata') + self.error('Can\'t locate "radclient" binary or binary is not executable by netdata') return False + if not self.secret: + self.error('"secret" not set') + return None + if self._get_raw_data(): - chart_choice = [True, bool(self.acct), bool(self.proxy_auth), bool(self.proxy_acct)] - self.order = [chart for chart, choice in zip(ORDER, chart_choice) if choice] - self.definitions = dict([chart for chart in CHARTS.items() if chart[0] in self.order]) - self.info('Plugin was started succesfully') return True - else: - self.error('Request returned no data. Is server alive? Used options: host {0}, port {1}, secret {2}'.format(self.host, self.port, self.secret)) - return False - + self.error('Request returned no data. Is server alive?') + return False def _get_data(self): """ @@ -91,7 +109,8 @@ class Service(SimpleService): def _get_raw_data(self): """ The following code is equivalent to - 'echo "Message-Authenticator = 0x00, FreeRADIUS-Statistics-Type = 15, Response-Packet-Type = Access-Accept" | radclient -t 1 -r 1 host:port status secret' + 'echo "Message-Authenticator = 0x00, FreeRADIUS-Statistics-Type = 15, Response-Packet-Type = Access-Accept" + | radclient -t 1 -r 1 host:port status secret' :return: str """ try: @@ -99,10 +118,8 @@ class Service(SimpleService): process_rad = Popen(self.sub_radclient, stdin=process_echo.stdout, stdout=PIPE, stderr=PIPE, shell=False) process_echo.stdout.close() raw_result = process_rad.communicate()[0] - except Exception: + except OSError: return None - else: - if process_rad.returncode is 0: - return raw_result.decode() - else: - return None + if process_rad.returncode is 0: + return raw_result.decode() + return None diff --git a/python.d/go_expvar.chart.py b/python.d/go_expvar.chart.py index e1a334cc3..cbd462570 100644 --- a/python.d/go_expvar.chart.py +++ b/python.d/go_expvar.chart.py @@ -3,9 +3,10 @@ # Author: Jan Kral (kralewitz) from __future__ import division -from base import UrlService import json +from bases.FrameworkServices.UrlService import UrlService + # default module values (can be overridden per job in `config`) # update_every = 2 priority = 60000 @@ -14,44 +15,52 @@ retries = 60 MEMSTATS_CHARTS = { 'memstats_heap': { - 'options': ['heap', 'memory: size of heap memory structures', 'kB', 'memstats', 'expvar.memstats.heap', 'line'], + 'options': ['heap', 'memory: size of heap memory structures', 'kB', 'memstats', + 'expvar.memstats.heap', 'line'], 'lines': [ ['memstats_heap_alloc', 'alloc', 'absolute', 1, 1024], ['memstats_heap_inuse', 'inuse', 'absolute', 1, 1024] ]}, 'memstats_stack': { - 'options': ['stack', 'memory: size of stack memory structures', 'kB', 'memstats', 'expvar.memstats.stack', 'line'], + 'options': ['stack', 'memory: size of stack memory structures', 'kB', 'memstats', + 'expvar.memstats.stack', 'line'], 'lines': [ ['memstats_stack_inuse', 'inuse', 'absolute', 1, 1024] ]}, 'memstats_mspan': { - 'options': ['mspan', 'memory: size of mspan memory structures', 'kB', 'memstats', 'expvar.memstats.mspan', 'line'], + 'options': ['mspan', 'memory: size of mspan memory structures', 'kB', 'memstats', + 'expvar.memstats.mspan', 'line'], 'lines': [ ['memstats_mspan_inuse', 'inuse', 'absolute', 1, 1024] ]}, 'memstats_mcache': { - 'options': ['mcache', 'memory: size of mcache memory structures', 'kB', 'memstats', 'expvar.memstats.mcache', 'line'], + 'options': ['mcache', 'memory: size of mcache memory structures', 'kB', 'memstats', + 'expvar.memstats.mcache', 'line'], 'lines': [ ['memstats_mcache_inuse', 'inuse', 'absolute', 1, 1024] ]}, 'memstats_live_objects': { - 'options': ['live_objects', 'memory: number of live objects', 'objects', 'memstats', 'expvar.memstats.live_objects', 'line'], + 'options': ['live_objects', 'memory: number of live objects', 'objects', 'memstats', + 'expvar.memstats.live_objects', 'line'], 'lines': [ ['memstats_live_objects', 'live'] ]}, 'memstats_sys': { - 'options': ['sys', 'memory: size of reserved virtual address space', 'kB', 'memstats', 'expvar.memstats.sys', 'line'], + 'options': ['sys', 'memory: size of reserved virtual address space', 'kB', 'memstats', + 'expvar.memstats.sys', 'line'], 'lines': [ ['memstats_sys', 'sys', 'absolute', 1, 1024] ]}, 'memstats_gc_pauses': { - 'options': ['gc_pauses', 'memory: average duration of GC pauses', 'ns', 'memstats', 'expvar.memstats.gc_pauses', 'line'], + 'options': ['gc_pauses', 'memory: average duration of GC pauses', 'ns', 'memstats', + 'expvar.memstats.gc_pauses', 'line'], 'lines': [ ['memstats_gc_pauses', 'avg'] ]}, } -MEMSTATS_ORDER = ['memstats_heap', 'memstats_stack', 'memstats_mspan', 'memstats_mcache', 'memstats_sys', 'memstats_live_objects', 'memstats_gc_pauses'] +MEMSTATS_ORDER = ['memstats_heap', 'memstats_stack', 'memstats_mspan', 'memstats_mcache', + 'memstats_sys', 'memstats_live_objects', 'memstats_gc_pauses'] def flatten(d, top='', sep='.'): @@ -71,8 +80,8 @@ class Service(UrlService): # if memstats collection is enabled, add the charts and their order if self.configuration.get('collect_memstats'): - self.definitions = MEMSTATS_CHARTS - self.order = MEMSTATS_ORDER + self.definitions = dict(MEMSTATS_CHARTS) + self.order = list(MEMSTATS_ORDER) else: self.definitions = dict() self.order = list() diff --git a/python.d/haproxy.chart.py b/python.d/haproxy.chart.py index a9ee66650..e72698d10 100644 --- a/python.d/haproxy.chart.py +++ b/python.d/haproxy.chart.py @@ -10,7 +10,9 @@ try: except ImportError: from urllib.parse import urlparse -from base import UrlService, SocketService +from bases.FrameworkServices.SocketService import SocketService +from bases.FrameworkServices.UrlService import UrlService + # default module values (can be overridden per job in `config`) # update_every = 2 diff --git a/python.d/hddtemp.chart.py b/python.d/hddtemp.chart.py index 8a98995be..577cab09f 100644 --- a/python.d/hddtemp.chart.py +++ b/python.d/hddtemp.chart.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Description: hddtemp netdata python.d module # Author: Pawel Krupa (paulfantom) -# Modified by l2isbad + import os -from base import SocketService +from copy import deepcopy + +from bases.FrameworkServices.SocketService import SocketService # default module values (can be overridden per job in `config`) #update_every = 2 @@ -22,34 +24,39 @@ retries = 60 ORDER = ['temperatures'] +CHARTS = { + 'temperatures': { + 'options': ['disks_temp', 'Disks Temperatures', 'Celsius', 'temperatures', 'hddtemp.temperatures', 'line'], + 'lines': [ + # lines are created dynamically in `check()` method + ]}} + + class Service(SocketService): def __init__(self, configuration=None, name=None): SocketService.__init__(self, configuration=configuration, name=name) + self.order = ORDER + self.definitions = deepcopy(CHARTS) self._keep_alive = False self.request = "" self.host = "127.0.0.1" self.port = 7634 - self.order = ORDER - self.fahrenheit = ('Fahrenheit', lambda x: x * 9 / 5 + 32) if self.configuration.get('fahrenheit') else False - self.whatever = ('Whatever', lambda x: x * 33 / 22 + 11) if self.configuration.get('whatever') else False - self.choice = (choice for choice in [self.fahrenheit, self.whatever] if choice) - self.calc = lambda x: x - self.disks = [] + self.disks = list() - def _get_disks(self): + def get_disks(self): try: disks = self.configuration['devices'] - self.info("Using configured disks" + str(disks)) - except (KeyError, TypeError) as e: + self.info("Using configured disks {0}".format(disks)) + except (KeyError, TypeError): self.info("Autodetecting disks") return ["/dev/" + f for f in os.listdir("/dev") if len(f) == 3 and f.startswith("sd")] - ret = [] + ret = list() for disk in disks: if not disk.startswith('/dev/'): disk = "/dev/" + disk ret.append(disk) - if len(ret) == 0: + if not ret: self.error("Provided disks cannot be found in /dev directory.") return ret @@ -59,10 +66,9 @@ class Service(SocketService): if all(disk in data for disk in self.disks): return True - return False - def _get_data(self): + def get_data(self): """ Get data from TCP/IP socket :return: dict @@ -72,21 +78,20 @@ class Service(SocketService): except AttributeError: self.error("no data received") return None - data = {} + data = dict() for i in range(len(raw) // 5): if not raw[i*5+1] in self.disks: continue try: - val = self.calc(int(raw[i*5+3])) + val = int(raw[i*5+3]) except ValueError: val = 0 data[raw[i*5+1].replace("/dev/", "")] = val - if len(data) == 0: + if not data: self.error("received data doesn't have needed records") return None - else: - return data + return data def check(self): """ @@ -94,27 +99,12 @@ class Service(SocketService): :return: boolean """ self._parse_config() - self.disks = self._get_disks() + self.disks = self.get_disks() - data = self._get_data() + data = self.get_data() if data is None: return False - self.definitions = { - 'temperatures': { - 'options': ['disks_temp', 'Disks Temperatures', 'temperatures', 'hddtemp.temperatures', 'line'], - 'lines': [ - # lines are created dynamically in `check()` method - ]} - } - try: - self.choice = next(self.choice) - except StopIteration: - self.definitions[ORDER[0]]['options'].insert(2, 'Celsius') - else: - self.calc = self.choice[1] - self.definitions[ORDER[0]]['options'].insert(2, self.choice[0]) - for name in data: - self.definitions[ORDER[0]]['lines'].append([name]) + self.definitions['temperatures']['lines'].append([name]) return True diff --git a/python.d/ipfs.chart.py b/python.d/ipfs.chart.py index eaea3c5ee..43500dfb5 100644 --- a/python.d/ipfs.chart.py +++ b/python.d/ipfs.chart.py @@ -2,9 +2,10 @@ # Description: IPFS netdata python.d module # Authors: Pawel Krupa (paulfantom), davidak -from base import UrlService import json +from bases.FrameworkServices.UrlService import UrlService + # default module values (can be overridden per job in `config`) # update_every = 2 priority = 60000 @@ -49,75 +50,75 @@ CHARTS = { } SI_zeroes = {'k': 3, 'm': 6, 'g': 9, 't': 12, - 'p': 15, 'e': 18, 'z': 21, 'y': 24 } + 'p': 15, 'e': 18, 'z': 21, 'y': 24} class Service(UrlService): def __init__(self, configuration=None, name=None): UrlService.__init__(self, configuration=configuration, name=name) - try: - self.baseurl = str(self.configuration['url']) - except (KeyError, TypeError): - self.baseurl = "http://localhost:5001" + self.baseurl = self.configuration.get('url', 'http://localhost:5001') self.order = ORDER self.definitions = CHARTS - self.__storagemax = None + self.__storage_max = None - def _get_json(self, suburl): + def _get_json(self, sub_url): """ :return: json decoding of the specified url """ - self.url = self.baseurl + suburl + self.url = self.baseurl + sub_url try: return json.loads(self._get_raw_data()) - except: - return {} + except (TypeError, ValueError): + return dict() - def _recursive_pins(self, keys): + @staticmethod + def _recursive_pins(keys): return len([k for k in keys if keys[k]["Type"] == b"recursive"]) - def _dehumanize(self, storemax): + @staticmethod + def _dehumanize(store_max): # convert from '10Gb' to 10000000000 - if type(storemax) != int: - storemax = storemax.lower() - if storemax.endswith('b'): - val, units = storemax[:-2], storemax[-2] + if not isinstance(store_max, int): + store_max = store_max.lower() + if store_max.endswith('b'): + val, units = store_max[:-2], store_max[-2] if units in SI_zeroes: val += '0'*SI_zeroes[units] - storemax = val + store_max = val try: - storemax = int(storemax) - except: - storemax = None - return storemax + store_max = int(store_max) + except (TypeError, ValueError): + store_max = None + return store_max - def _storagemax(self, storecfg): - if self.__storagemax is None: - self.__storagemax = self._dehumanize(storecfg['StorageMax']) - return self.__storagemax + def _storagemax(self, store_cfg): + if self.__storage_max is None: + self.__storage_max = self._dehumanize(store_cfg['StorageMax']) + return self.__storage_max def _get_data(self): """ Get data from API :return: dict """ - cfg = { # suburl : List of (result-key, original-key, transform-func) - '/api/v0/stats/bw' :[('in', 'RateIn', int ), - ('out', 'RateOut', int )], - '/api/v0/swarm/peers':[('peers', 'Strings', len )], - '/api/v0/stats/repo' :[('size', 'RepoSize', int), - ('objects', 'NumObjects', int)], - '/api/v0/pin/ls': [('pinned', 'Keys', len), - ('recursive_pins', 'Keys', self._recursive_pins)], - '/api/v0/config/show': [('avail', 'Datastore', self._storagemax)] + # suburl : List of (result-key, original-key, transform-func) + cfg = { + '/api/v0/stats/bw': + [('in', 'RateIn', int), ('out', 'RateOut', int)], + '/api/v0/swarm/peers': + [('peers', 'Strings', len)], + '/api/v0/stats/repo': + [('size', 'RepoSize', int), ('objects', 'NumObjects', int)], + '/api/v0/pin/ls': + [('pinned', 'Keys', len), ('recursive_pins', 'Keys', self._recursive_pins)], + '/api/v0/config/show': [('avail', 'Datastore', self._storagemax)] } - r = {} + r = dict() for suburl in cfg: - json = self._get_json(suburl) - for newkey, origkey, xmute in cfg[suburl]: + in_json = self._get_json(suburl) + for new_key, orig_key, xmute in cfg[suburl]: try: - r[newkey] = xmute(json[origkey]) - except: pass + r[new_key] = xmute(in_json[orig_key]) + except Exception: + continue return r or None - - diff --git a/python.d/isc_dhcpd.chart.py b/python.d/isc_dhcpd.chart.py index a437f803b..60995342c 100644 --- a/python.d/isc_dhcpd.chart.py +++ b/python.d/isc_dhcpd.chart.py @@ -7,14 +7,15 @@ from os import stat, access, R_OK from os.path import isfile try: from ipaddress import ip_network, ip_address - have_ipaddress = True + HAVE_IPADDRESS = True except ImportError: - have_ipaddress = False + HAVE_IPADDRESS = False try: from itertools import filterfalse except ImportError: from itertools import ifilterfalse as filterfalse -from base import SimpleService + +from bases.FrameworkServices.SimpleService import SimpleService priority = 60000 retries = 60 @@ -55,11 +56,10 @@ class Service(SimpleService): # Will work only with 'default' db-time-format (weekday year/month/day hour:minute:second) # TODO: update algorithm to parse correctly 'local' db-time-format - # (epoch ; # :: ) # Also only ipv4 supported def check(self): - if not have_ipaddress: + if not HAVE_IPADDRESS: self.error('\'python-ipaddress\' module is needed') return False if not (isfile(self.leases_path) and access(self.leases_path, R_OK)): @@ -146,7 +146,16 @@ class Service(SimpleService): def binding_active(lease_end_time, current_time): - return mktime(strptime(lease_end_time, '%w %Y/%m/%d %H:%M:%S')) - current_time > 0 + # lease_end_time might be epoch + if lease_end_time.startswith('epoch'): + epoch = int(lease_end_time.split()[1].replace(';','')) + return epoch - current_time > 0 + # max. int for lease-time causes lease to expire in year 2038. + # dhcpd puts 'never' in the ends section of active lease + elif lease_end_time == 'never': + return True + else: + return mktime(strptime(lease_end_time, '%w %Y/%m/%d %H:%M:%S')) - current_time > 0 def find_lease(value): @@ -155,4 +164,3 @@ def find_lease(value): def find_ends(value): return value[2:6] != 'ends' - diff --git a/python.d/mdstat.chart.py b/python.d/mdstat.chart.py index 7ce7b1932..794d25641 100644 --- a/python.d/mdstat.chart.py +++ b/python.d/mdstat.chart.py @@ -2,9 +2,10 @@ # Description: mdstat netdata python.d module # Author: l2isbad -from re import compile as re_compile from collections import defaultdict -from base import SimpleService +from re import compile as re_compile + +from bases.FrameworkServices.SimpleService import SimpleService priority = 60000 retries = 60 @@ -18,7 +19,7 @@ class Service(SimpleService): SimpleService.__init__(self, configuration=configuration, name=name) self.regex = dict(disks=re_compile(r' (?P[a-zA-Z_0-9]+) : active .+\[' r'(?P[0-9]+)/' - r'(?P[0-9])\]'), + r'(?P[0-9]+)\]'), status=re_compile(r' (?P[a-zA-Z_0-9]+) : active .+ ' r'(?P[a-z]+) =[ ]{1,2}' r'(?P[0-9.]+).+finish=' diff --git a/python.d/memcached.chart.py b/python.d/memcached.chart.py index 0d6807ba7..4f7adfa23 100644 --- a/python.d/memcached.chart.py +++ b/python.d/memcached.chart.py @@ -2,7 +2,7 @@ # Description: memcached netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import SocketService +from bases.FrameworkServices.SocketService import SocketService # default module values (can be overridden per job in `config`) #update_every = 2 @@ -149,9 +149,8 @@ class Service(SocketService): data[t[0]] = t[1] except (IndexError, ValueError): self.debug("invalid line received: " + str(line)) - pass - if len(data) == 0: + if not data: self.error("received data doesn't have any records") return None @@ -159,7 +158,7 @@ class Service(SocketService): try: data['avail'] = int(data['limit_maxbytes']) - int(data['bytes']) data['used'] = int(data['bytes']) - except: + except (KeyError, ValueError, TypeError): pass return data @@ -178,11 +177,7 @@ class Service(SocketService): :return: boolean """ self._parse_config() - if self.name == "": - self.name = "local" - self.chart_name += "_" + self.name data = self._get_data() if data is None: return False - return True diff --git a/python.d/mongodb.chart.py b/python.d/mongodb.chart.py index bb4c44b09..909a419da 100644 --- a/python.d/mongodb.chart.py +++ b/python.d/mongodb.chart.py @@ -2,7 +2,6 @@ # Description: mongodb netdata python.d module # Author: l2isbad -from base import SimpleService from copy import deepcopy from datetime import datetime from sys import exc_info @@ -14,6 +13,8 @@ try: except ImportError: PYMONGO = False +from bases.FrameworkServices.SimpleService import SimpleService + # default module values (can be overridden per job in `config`) # update_every = 2 priority = 60000 @@ -36,6 +37,7 @@ REPL_SET_STATES = [ def multiply_by_100(value): return value * 100 + DEFAULT_METRICS = [ ('opcounters.delete', None, None), ('opcounters.update', None, None), diff --git a/python.d/mysql.chart.py b/python.d/mysql.chart.py index 6118f79f2..4c7058b26 100644 --- a/python.d/mysql.chart.py +++ b/python.d/mysql.chart.py @@ -2,11 +2,11 @@ # Description: MySQL netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import MySQLService +from bases.FrameworkServices.MySQLService import MySQLService # default module values (can be overridden per job in `config`) # update_every = 3 -priority = 90000 +priority = 60000 retries = 60 # query executed on MySQL server @@ -114,7 +114,16 @@ GLOBAL_STATS = [ 'Connection_errors_max_connections', 'Connection_errors_peer_address', 'Connection_errors_select', - 'Connection_errors_tcpwrap'] + 'Connection_errors_tcpwrap', + 'wsrep_local_recv_queue', + 'wsrep_local_send_queue', + 'wsrep_received', + 'wsrep_replicated', + 'wsrep_received_bytes', + 'wsrep_replicated_bytes', + 'wsrep_local_bf_aborts', + 'wsrep_local_cert_failures', + 'wsrep_flow_control_paused_ns'] def slave_seconds(value): try: @@ -122,6 +131,7 @@ def slave_seconds(value): except (TypeError, ValueError): return -1 + def slave_running(value): return 1 if value == 'Yes' else -1 @@ -146,7 +156,8 @@ ORDER = ['net', 'innodb_buffer_pool_read_ahead', 'innodb_buffer_pool_reqs', 'innodb_buffer_pool_ops', 'qcache_ops', 'qcache', 'qcache_freemem', 'qcache_memblocks', 'key_blocks', 'key_requests', 'key_disk_ops', - 'files', 'files_rate', 'slave_behind', 'slave_status'] + 'files', 'files_rate', 'slave_behind', 'slave_status', + 'galera_writesets', 'galera_bytes', 'galera_queue', 'galera_conflicts', 'galera_flow_control'] CHARTS = { 'net': { @@ -248,7 +259,8 @@ CHARTS = { ['Innodb_data_fsyncs', 'fsyncs', 'incremental'] ]}, 'innodb_io_pending_ops': { - 'options': [None, 'mysql InnoDB Pending I/O Operations', 'operations', 'innodb', 'mysql.innodb_io_pending_ops', 'line'], + 'options': [None, 'mysql InnoDB Pending I/O Operations', 'operations', 'innodb', + 'mysql.innodb_io_pending_ops', 'line'], 'lines': [ ['Innodb_data_pending_reads', 'reads', 'absolute'], ['Innodb_data_pending_writes', 'writes', 'absolute', -1, 1], @@ -274,7 +286,8 @@ CHARTS = { ['Innodb_os_log_written', 'write', 'incremental', -1, 1024], ]}, 'innodb_cur_row_lock': { - 'options': [None, 'mysql InnoDB Current Row Locks', 'operations', 'innodb', 'mysql.innodb_cur_row_lock', 'area'], + 'options': [None, 'mysql InnoDB Current Row Locks', 'operations', 'innodb', + 'mysql.innodb_cur_row_lock', 'area'], 'lines': [ ['Innodb_row_lock_current_waits', 'current_waits', 'absolute'] ]}, @@ -287,7 +300,8 @@ CHARTS = { ['Innodb_rows_deleted', 'deleted', 'incremental', -1, 1], ]}, 'innodb_buffer_pool_pages': { - 'options': [None, 'mysql InnoDB Buffer Pool Pages', 'pages', 'innodb', 'mysql.innodb_buffer_pool_pages', 'line'], + 'options': [None, 'mysql InnoDB Buffer Pool Pages', 'pages', 'innodb', + 'mysql.innodb_buffer_pool_pages', 'line'], 'lines': [ ['Innodb_buffer_pool_pages_data', 'data', 'absolute'], ['Innodb_buffer_pool_pages_dirty', 'dirty', 'absolute', -1, 1], @@ -303,20 +317,23 @@ CHARTS = { ['Innodb_buffer_pool_bytes_dirty', 'dirty', 'absolute', -1, 1024 * 1024] ]}, 'innodb_buffer_pool_read_ahead': { - 'options': [None, 'mysql InnoDB Buffer Pool Read Ahead', 'operations/s', 'innodb', 'mysql.innodb_buffer_pool_read_ahead', 'area'], + 'options': [None, 'mysql InnoDB Buffer Pool Read Ahead', 'operations/s', 'innodb', + 'mysql.innodb_buffer_pool_read_ahead', 'area'], 'lines': [ ['Innodb_buffer_pool_read_ahead', 'all', 'incremental'], ['Innodb_buffer_pool_read_ahead_evicted', 'evicted', 'incremental', -1, 1], ['Innodb_buffer_pool_read_ahead_rnd', 'random', 'incremental'] ]}, 'innodb_buffer_pool_reqs': { - 'options': [None, 'mysql InnoDB Buffer Pool Requests', 'requests/s', 'innodb', 'mysql.innodb_buffer_pool_reqs', 'area'], + 'options': [None, 'mysql InnoDB Buffer Pool Requests', 'requests/s', 'innodb', + 'mysql.innodb_buffer_pool_reqs', 'area'], 'lines': [ ['Innodb_buffer_pool_read_requests', 'reads', 'incremental'], ['Innodb_buffer_pool_write_requests', 'writes', 'incremental', -1, 1] ]}, 'innodb_buffer_pool_ops': { - 'options': [None, 'mysql InnoDB Buffer Pool Operations', 'operations/s', 'innodb', 'mysql.innodb_buffer_pool_ops', 'area'], + 'options': [None, 'mysql InnoDB Buffer Pool Operations', 'operations/s', 'innodb', + 'mysql.innodb_buffer_pool_ops', 'area'], 'lines': [ ['Innodb_buffer_pool_reads', 'disk reads', 'incremental'], ['Innodb_buffer_pool_wait_free', 'wait free', 'incremental', -1, 1] @@ -359,7 +376,8 @@ CHARTS = { ['Key_write_requests', 'writes', 'incremental', -1, 1] ]}, 'key_disk_ops': { - 'options': [None, 'mysql MyISAM Key Cache Disk Operations', 'operations/s', 'myisam', 'mysql.key_disk_ops', 'area'], + 'options': [None, 'mysql MyISAM Key Cache Disk Operations', 'operations/s', + 'myisam', 'mysql.key_disk_ops', 'area'], 'lines': [ ['Key_reads', 'reads', 'incremental'], ['Key_writes', 'writes', 'incremental', -1, 1] @@ -375,13 +393,15 @@ CHARTS = { ['Opened_files', 'files', 'incremental'] ]}, 'binlog_stmt_cache': { - 'options': [None, 'mysql Binlog Statement Cache', 'statements/s', 'binlog', 'mysql.binlog_stmt_cache', 'line'], + 'options': [None, 'mysql Binlog Statement Cache', 'statements/s', 'binlog', + 'mysql.binlog_stmt_cache', 'line'], 'lines': [ ['Binlog_stmt_cache_disk_use', 'disk', 'incremental'], ['Binlog_stmt_cache_use', 'all', 'incremental'] ]}, 'connection_errors': { - 'options': [None, 'mysql Connection Errors', 'connections/s', 'connections', 'mysql.connection_errors', 'line'], + 'options': [None, 'mysql Connection Errors', 'connections/s', 'connections', + 'mysql.connection_errors', 'line'], 'lines': [ ['Connection_errors_accept', 'accept', 'incremental'], ['Connection_errors_internal', 'internal', 'incremental'], @@ -400,6 +420,35 @@ CHARTS = { 'lines': [ ['Slave_SQL_Running', 'sql_running', 'absolute'], ['Slave_IO_Running', 'io_running', 'absolute'] + ]}, + 'galera_writesets': { + 'options': [None, 'Replicated writesets', 'writesets/s', 'galera', 'mysql.galera_writesets', 'line'], + 'lines': [ + ['wsrep_received', 'rx', 'incremental'], + ['wsrep_replicated', 'tx', 'incremental', -1, 1], + ]}, + 'galera_bytes': { + 'options': [None, 'Replicated bytes', 'KB/s', 'galera', 'mysql.galera_bytes', 'area'], + 'lines': [ + ['wsrep_received_bytes', 'rx', 'incremental', 1, 1024], + ['wsrep_replicated_bytes', 'tx', 'incremental', -1, 1024], + ]}, + 'galera_queue': { + 'options': [None, 'Galera queue', 'writesets', 'galera', 'mysql.galera_queue', 'line'], + 'lines': [ + ['wsrep_local_recv_queue', 'rx', 'absolute'], + ['wsrep_local_send_queue', 'tx', 'absolute', -1, 1], + ]}, + 'galera_conflicts': { + 'options': [None, 'Replication conflicts', 'transactions', 'galera', 'mysql.galera_conflicts', 'area'], + 'lines': [ + ['wsrep_local_bf_aborts', 'bf_aborts', 'incremental'], + ['wsrep_local_cert_failures', 'cert_fails', 'incremental', -1, 1], + ]}, + 'galera_flow_control': { + 'options': [None, 'Flow control', 'millisec', 'galera', 'mysql.galera_flow_control', 'area'], + 'lines': [ + ['wsrep_flow_control_paused_ns', 'paused', 'incremental', 1, 1000000], ]} } @@ -416,7 +465,7 @@ class Service(MySQLService): raw_data = self._get_raw_data(description=True) if not raw_data: - return None + return None to_netdata = dict() @@ -426,14 +475,15 @@ class Service(MySQLService): if key in global_status: to_netdata[key] = global_status[key] if 'Threads_created' in to_netdata and 'Connections' in to_netdata: - to_netdata['Thread_cache_misses'] = round(int(to_netdata['Threads_created']) / float(to_netdata['Connections']) * 10000) + to_netdata['Thread_cache_misses'] = round(int(to_netdata['Threads_created']) + / float(to_netdata['Connections']) * 10000) if 'slave_status' in raw_data: if raw_data['slave_status'][0]: slave_raw_data = dict(zip([e[0] for e in raw_data['slave_status'][1]], raw_data['slave_status'][0][0])) - for key, function in SLAVE_STATS: + for key, func in SLAVE_STATS: if key in slave_raw_data: - to_netdata[key] = function(slave_raw_data[key]) + to_netdata[key] = func(slave_raw_data[key]) else: self.queries.pop('slave_status') diff --git a/python.d/nginx.chart.py b/python.d/nginx.chart.py index 88849a921..2e4f0d1b5 100644 --- a/python.d/nginx.chart.py +++ b/python.d/nginx.chart.py @@ -2,7 +2,7 @@ # Description: nginx netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import UrlService +from bases.FrameworkServices.UrlService import UrlService # default module values (can be overridden per job in `config`) # update_every = 2 @@ -22,7 +22,8 @@ ORDER = ['connections', 'requests', 'connection_status', 'connect_rate'] CHARTS = { 'connections': { - 'options': [None, 'nginx Active Connections', 'connections', 'active connections', 'nginx.connections', 'line'], + 'options': [None, 'nginx Active Connections', 'connections', 'active connections', + 'nginx.connections', 'line'], 'lines': [ ["active"] ]}, @@ -32,14 +33,16 @@ CHARTS = { ["requests", None, 'incremental'] ]}, 'connection_status': { - 'options': [None, 'nginx Active Connections by Status', 'connections', 'status', 'nginx.connection_status', 'line'], + 'options': [None, 'nginx Active Connections by Status', 'connections', 'status', + 'nginx.connection_status', 'line'], 'lines': [ ["reading"], ["writing"], ["waiting", "idle"] ]}, 'connect_rate': { - 'options': [None, 'nginx Connections Rate', 'connections/s', 'connections rate', 'nginx.connect_rate', 'line'], + 'options': [None, 'nginx Connections Rate', 'connections/s', 'connections rate', + 'nginx.connect_rate', 'line'], 'lines': [ ["accepts", "accepted", "incremental"], ["handled", None, "incremental"] @@ -50,8 +53,7 @@ CHARTS = { class Service(UrlService): def __init__(self, configuration=None, name=None): UrlService.__init__(self, configuration=configuration, name=name) - if len(self.url) == 0: - self.url = "http://localhost/stub_status" + self.url = self.configuration.get('url', 'http://localhost/stub_status') self.order = ORDER self.definitions = CHARTS diff --git a/python.d/nsd.chart.py b/python.d/nsd.chart.py index 68bb4f237..499dfda2e 100644 --- a/python.d/nsd.chart.py +++ b/python.d/nsd.chart.py @@ -2,10 +2,10 @@ # Description: NSD `nsd-control stats_noreset` netdata python.d module # Author: <383c57 at gmail.com> - -from base import ExecutableService import re +from bases.FrameworkServices.ExecutableService import ExecutableService + # default module values (can be overridden per job in `config`) priority = 60000 retries = 5 diff --git a/python.d/ovpn_status_log.chart.py b/python.d/ovpn_status_log.chart.py index 3a7e8200d..519c77fa3 100644 --- a/python.d/ovpn_status_log.chart.py +++ b/python.d/ovpn_status_log.chart.py @@ -3,8 +3,8 @@ # Author: l2isbad from re import compile as r_compile -from collections import defaultdict -from base import SimpleService + +from bases.FrameworkServices.SimpleService import SimpleService priority = 60000 retries = 60 @@ -34,24 +34,30 @@ class Service(SimpleService): self.log_path = self.configuration.get('log_path') self.regex = dict(tls=r_compile(r'\d{1,3}(?:\.\d{1,3}){3}(?::\d+)? (?P\d+) (?P\d+)'), static_key=r_compile(r'TCP/[A-Z]+ (?P(?:read|write)) bytes,(?P\d+)')) - self.to_netdata = dict(bytes_in=0, bytes_out=0) def check(self): if not (self.log_path and isinstance(self.log_path, str)): - self.error('\'log_path\' is not defined') + self.error("'log_path' is not defined") return False - data = False - for method in (self._get_data_tls, self._get_data_static_key): - data = method() - if data: - self._get_data = method - self._data_from_check = data - break + data = self._get_raw_data() + if not data: + self.error('Make sure that the openvpn status log file exists and netdata has permission to read it') + return None - if data: + found = None + for row in data: + if 'ROUTING' in row: + self.get_data = self.get_data_tls + found = True + break + elif 'STATISTICS' in row: + self.get_data = self.get_data_static_key + found = True + break + if found: return True - self.error('Make sure that the openvpn status log file exists and netdata has permission to read it') + self.error("Failed to parse ovpenvpn log file") return False def _get_raw_data(self): @@ -68,7 +74,7 @@ class Service(SimpleService): else: return raw_data - def _get_data_static_key(self): + def get_data_static_key(self): """ Parse openvpn-status log file. """ @@ -77,7 +83,7 @@ class Service(SimpleService): if not raw_data: return None - data = defaultdict(lambda: 0) + data = dict(bytes_in=0, bytes_out=0) for row in raw_data: match = self.regex['static_key'].search(row) @@ -90,7 +96,7 @@ class Service(SimpleService): return data or None - def _get_data_tls(self): + def get_data_tls(self): """ Parse openvpn-status log file. """ @@ -99,7 +105,7 @@ class Service(SimpleService): if not raw_data: return None - data = defaultdict(lambda: 0) + data = dict(users=0, bytes_in=0, bytes_out=0) for row in raw_data: row = ' '.join(row.split(',')) if ',' in row else ' '.join(row.split()) match = self.regex['tls'].search(row) @@ -110,4 +116,3 @@ class Service(SimpleService): data['bytes_out'] += int(match['bytes_out']) return data or None - diff --git a/python.d/phpfpm.chart.py b/python.d/phpfpm.chart.py index 7a9835210..ea7a9a7e6 100644 --- a/python.d/phpfpm.chart.py +++ b/python.d/phpfpm.chart.py @@ -2,10 +2,11 @@ # Description: PHP-FPM netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import UrlService import json import re +from bases.FrameworkServices.UrlService import UrlService + # default module values (can be overridden per job in `config`) # update_every = 2 priority = 60000 @@ -40,6 +41,7 @@ PER_PROCESS_INFO = [ def average(collection): return sum(collection, 0.0) / max(len(collection), 1) + CALC = [ ('min', min), ('max', max), @@ -130,8 +132,8 @@ class Service(UrlService): if p_info: for new_name in PER_PROCESS_INFO: - for name, function in CALC: - to_netdata[name + new_name[1]] = function([p_info[k] for k in p_info if new_name[1] in k]) + for name, func in CALC: + to_netdata[name + new_name[1]] = func([p_info[k] for k in p_info if new_name[1] in k]) return to_netdata or None @@ -165,4 +167,3 @@ def parse_raw_data_(is_json, regex, raw_data): else: raw_data = ' '.join(raw_data.split()) return dict(regex.findall(raw_data)) - diff --git a/python.d/postfix.chart.py b/python.d/postfix.chart.py index ee4142aaf..a2129e4be 100644 --- a/python.d/postfix.chart.py +++ b/python.d/postfix.chart.py @@ -2,7 +2,7 @@ # Description: postfix netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import ExecutableService +from bases.FrameworkServices.ExecutableService import ExecutableService # default module values (can be overridden per job in `config`) # update_every = 2 diff --git a/python.d/postgres.chart.py b/python.d/postgres.chart.py index b17565e9d..ef69a9c77 100644 --- a/python.d/postgres.chart.py +++ b/python.d/postgres.chart.py @@ -13,11 +13,11 @@ try: except ImportError: PSYCOPG2 = False -from base import SimpleService +from bases.FrameworkServices.SimpleService import SimpleService # default module values update_every = 1 -priority = 90000 +priority = 60000 retries = 60 METRICS = dict( @@ -62,7 +62,7 @@ SELECT CAST(COALESCE(SUM(CAST(archive_file ~ $r$\.ready$$r$ as INT)), 0) AS INT) AS ready_count, CAST(COALESCE(SUM(CAST(archive_file ~ $r$\.done$$r$ AS INT)), 0) AS INT) AS done_count FROM - pg_catalog.pg_ls_dir('pg_xlog/archive_status') AS archive_files (archive_file); + pg_catalog.pg_ls_dir('{0}/archive_status') AS archive_files (archive_file); """, BACKENDS=""" SELECT @@ -125,7 +125,11 @@ AND NOT datname ~* '^template\d+'; """, IF_SUPERUSER=""" SELECT current_setting('is_superuser') = 'on' AS is_superuser; - """) + """, + DETECT_SERVER_VERSION=""" +SHOW server_version_num; + """ +) QUERY_STATS = { @@ -221,7 +225,7 @@ CHARTS = { class Service(SimpleService): def __init__(self, configuration=None, name=None): - super(self.__class__, self).__init__(configuration=configuration, name=name) + SimpleService.__init__(self, configuration=configuration, name=name) self.order = ORDER[:] self.definitions = deepcopy(CHARTS) self.table_stats = configuration.pop('table_stats', False) @@ -229,6 +233,7 @@ class Service(SimpleService): self.database_poll = configuration.pop('database_poll', None) self.configuration = configuration self.connection = False + self.server_version = None self.data = dict() self.locks_zeroed = dict() self.databases = list() @@ -257,17 +262,20 @@ class Service(SimpleService): return False result, error = self._connect() if not result: - conf = dict([(k, (lambda k, v: v if k != 'password' else '*****')(k, v)) for k, v in self.configuration.items()]) + conf = dict((k, (lambda k, v: v if k != 'password' else '*****')(k, v)) + for k, v in self.configuration.items()) self.error('Failed to connect to %s. Error: %s' % (str(conf), error)) return False try: cursor = self.connection.cursor() self.databases = discover_databases_(cursor, QUERIES['FIND_DATABASES']) is_superuser = check_if_superuser_(cursor, QUERIES['IF_SUPERUSER']) + self.server_version = detect_server_version(cursor, QUERIES['DETECT_SERVER_VERSION']) cursor.close() - if (self.database_poll and isinstance(self.database_poll, str)): - self.databases = [dbase for dbase in self.databases if dbase in self.database_poll.split()] or self.databases + if self.database_poll and isinstance(self.database_poll, str): + self.databases = [dbase for dbase in self.databases if dbase in self.database_poll.split()]\ + or self.databases self.locks_zeroed = populate_lock_types(self.databases) self.add_additional_queries_(is_superuser) @@ -284,7 +292,11 @@ class Service(SimpleService): self.queries[QUERIES['TABLE_STATS']] = METRICS['TABLE_STATS'] if is_superuser: self.queries[QUERIES['BGWRITER']] = METRICS['BGWRITER'] - self.queries[QUERIES['ARCHIVE']] = METRICS['ARCHIVE'] + if self.server_version >= 100000: + wal_dir_name = 'pg_wal' + else: + wal_dir_name = 'pg_xlog' + self.queries[QUERIES['ARCHIVE'].format(wal_dir_name)] = METRICS['ARCHIVE'] def create_dynamic_charts_(self): @@ -340,6 +352,9 @@ def check_if_superuser_(cursor, query): cursor.execute(query) return cursor.fetchone()[0] +def detect_server_version(cursor, query): + cursor.execute(query) + return int(cursor.fetchone()[0]) def populate_lock_types(databases): result = dict() diff --git a/python.d/powerdns.chart.py b/python.d/powerdns.chart.py new file mode 100644 index 000000000..a8d2f399c --- /dev/null +++ b/python.d/powerdns.chart.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Description: powerdns netdata python.d module +# Author: l2isbad + +from json import loads + +from bases.FrameworkServices.UrlService import UrlService + +priority = 60000 +retries = 60 +# update_every = 3 + +ORDER = ['questions', 'cache_usage', 'cache_size', 'latency'] +CHARTS = { + 'questions': { + 'options': [None, 'PowerDNS Queries and Answers', 'count', 'questions', 'powerdns.questions', 'line'], + 'lines': [ + ['udp-queries', None, 'incremental'], + ['udp-answers', None, 'incremental'], + ['tcp-queries', None, 'incremental'], + ['tcp-answers', None, 'incremental'] + ]}, + 'cache_usage': { + 'options': [None, 'PowerDNS Cache Usage', 'count', 'cache', 'powerdns.cache_usage', 'line'], + 'lines': [ + ['query-cache-hit', None, 'incremental'], + ['query-cache-miss', None, 'incremental'], + ['packetcache-hit', 'packet-cache-hit', 'incremental'], + ['packetcache-miss', 'packet-cache-miss', 'incremental'] + ]}, + 'cache_size': { + 'options': [None, 'PowerDNS Cache Size', 'count', 'cache', 'powerdns.cache_size', 'line'], + 'lines': [ + ['query-cache-size', None, 'absolute'], + ['packetcache-size', 'packet-cache-size', 'absolute'], + ['key-cache-size', None, 'absolute'], + ['meta-cache-size', None, 'absolute'] + ]}, + 'latency': { + 'options': [None, 'PowerDNS Latency', 'microseconds', 'latency', 'powerdns.latency', 'line'], + 'lines': [ + ['latency', None, 'absolute'] + ]} + +} + + +class Service(UrlService): + def __init__(self, configuration=None, name=None): + UrlService.__init__(self, configuration=configuration, name=name) + self.order = ORDER + self.definitions = CHARTS + + def _get_data(self): + data = self._get_raw_data() + if not data: + return None + return dict((d['name'], d['value']) for d in loads(data)) diff --git a/python.d/python_modules/base.py b/python.d/python_modules/base.py index 1d5417ec2..7c6e1d2f2 100644 --- a/python.d/python_modules/base.py +++ b/python.d/python_modules/base.py @@ -1,1119 +1,9 @@ # -*- coding: utf-8 -*- -# Description: netdata python modules framework -# Author: Pawel Krupa (paulfantom) - -# Remember: -# ALL CODE NEEDS TO BE COMPATIBLE WITH Python > 2.7 and Python > 3.1 -# Follow PEP8 as much as it is possible -# "check" and "create" CANNOT be blocking. -# "update" CAN be blocking -# "update" function needs to be fast, so follow: -# https://wiki.python.org/moin/PythonSpeed/PerformanceTips -# basically: -# - use local variables wherever it is possible -# - avoid dots in expressions that are executed many times -# - use "join()" instead of "+" -# - use "import" only at the beginning -# -# using ".encode()" in one thread can block other threads as well (only in python2) - -import os -import re -import socket -import time -import threading - -import urllib3 - -from glob import glob -from subprocess import Popen, PIPE -from sys import exc_info - -try: - import MySQLdb - PY_MYSQL = True -except ImportError: - try: - import pymysql as MySQLdb - PY_MYSQL = True - except ImportError: - PY_MYSQL = False - -import msg - - -PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':') -try: - urllib3.disable_warnings() -except AttributeError: - msg.error('urllib3: warnings were not disabled') - - -# class BaseService(threading.Thread): -class SimpleService(threading.Thread): - """ - Prototype of Service class. - Implemented basic functionality to run jobs by `python.d.plugin` - """ - def __init__(self, configuration=None, name=None): - """ - This needs to be initialized in child classes - :param configuration: dict - :param name: str - """ - threading.Thread.__init__(self) - self._data_stream = "" - self.daemon = True - self.retries = 0 - self.retries_left = 0 - self.priority = 140000 - self.update_every = 1 - self.name = name - self.override_name = None - self.chart_name = "" - self._dimensions = [] - self._charts = [] - self.__chart_set = False - self.__first_run = True - self.order = [] - self.definitions = {} - self._data_from_check = dict() - if configuration is None: - self.error("BaseService: no configuration parameters supplied. Cannot create Service.") - raise RuntimeError - else: - self._extract_base_config(configuration) - self.timetable = {} - self.create_timetable() - - # --- BASIC SERVICE CONFIGURATION --- - - def _extract_base_config(self, config): - """ - Get basic parameters to run service - Minimum config: - config = {'update_every':1, - 'priority':100000, - 'retries':0} - :param config: dict - """ - pop = config.pop - try: - self.override_name = pop('name') - except KeyError: - pass - self.update_every = int(pop('update_every')) - self.priority = int(pop('priority')) - self.retries = int(pop('retries')) - self.retries_left = self.retries - self.configuration = config - - def create_timetable(self, freq=None): - """ - Create service timetable. - `freq` is optional - Example: - timetable = {'last': 1466370091.3767564, - 'next': 1466370092, - 'freq': 1} - :param freq: int - """ - if freq is None: - freq = self.update_every - now = time.time() - self.timetable = {'last': now, - 'next': now - (now % freq) + freq, - 'freq': freq} - - # --- THREAD CONFIGURATION --- - - def _run_once(self): - """ - Executes self.update(interval) and draws run time chart. - Return value presents exit status of update() - :return: boolean - """ - t_start = float(time.time()) - chart_name = self.chart_name - - since_last = int((t_start - self.timetable['last']) * 1000000) - if self.__first_run: - since_last = 0 - - if not self.update(since_last): - self.error("update function failed.") - return False - - # draw performance graph - run_time = int((time.time() - t_start) * 1000) - print("BEGIN netdata.plugin_pythond_%s %s\nSET run_time = %s\nEND\n" % - (self.chart_name, str(since_last), str(run_time))) - - self.debug(chart_name, "updated in", str(run_time), "ms") - self.timetable['last'] = t_start - self.__first_run = False - return True - - def run(self): - """ - Runs job in thread. Handles retries. - Exits when job failed or timed out. - :return: None - """ - step = float(self.timetable['freq']) - penalty = 0 - self.timetable['last'] = float(time.time() - step) - self.debug("starting data collection - update frequency:", str(step), " retries allowed:", str(self.retries)) - while True: # run forever, unless something is wrong - now = float(time.time()) - next = self.timetable['next'] = now - (now % step) + step + penalty - - # it is important to do this in a loop - # sleep() is interruptable - while now < next: - self.debug("sleeping for", str(next - now), "secs to reach frequency of", - str(step), "secs, now:", str(now), " next:", str(next), " penalty:", str(penalty)) - time.sleep(next - now) - now = float(time.time()) - - # do the job - try: - status = self._run_once() - except Exception: - status = False - - if status: - # it is good - self.retries_left = self.retries - penalty = 0 - else: - # it failed - self.retries_left -= 1 - if self.retries_left <= 0: - if penalty == 0: - penalty = float(self.retries * step) / 2 - else: - penalty *= 1.5 - - if penalty > 600: - penalty = 600 - - self.retries_left = self.retries - self.alert("failed to collect data for " + str(self.retries) + - " times - increasing penalty to " + str(penalty) + " sec and trying again") - - else: - self.error("failed to collect data - " + str(self.retries_left) - + " retries left - penalty: " + str(penalty) + " sec") - - # --- CHART --- - - @staticmethod - def _format(*args): - """ - Escape and convert passed arguments. - :param args: anything - :return: list - """ - params = [] - append = params.append - for p in args: - if p is None: - append(p) - continue - if type(p) is not str: - p = str(p) - if ' ' in p: - p = "'" + p + "'" - append(p) - return params - - def _line(self, instruction, *params): - """ - Converts *params to string and joins them with one space between every one. - Result is appended to self._data_stream - :param params: str/int/float - """ - tmp = list(map((lambda x: "''" if x is None or len(x) == 0 else x), params)) - self._data_stream += "%s %s\n" % (instruction, str(" ".join(tmp))) - - def chart(self, type_id, name="", title="", units="", family="", - category="", chart_type="line", priority="", update_every=""): - """ - Defines a new chart. - :param type_id: str - :param name: str - :param title: str - :param units: str - :param family: str - :param category: str - :param chart_type: str - :param priority: int/str - :param update_every: int/str - """ - self._charts.append(type_id) - - p = self._format(type_id, name, title, units, family, category, chart_type, priority, update_every) - self._line("CHART", *p) - - def dimension(self, id, name=None, algorithm="absolute", multiplier=1, divisor=1, hidden=False): - """ - Defines a new dimension for the chart - :param id: str - :param name: str - :param algorithm: str - :param multiplier: int/str - :param divisor: int/str - :param hidden: boolean - :return: - """ - try: - int(multiplier) - except TypeError: - self.error("malformed dimension: multiplier is not a number:", multiplier) - multiplier = 1 - try: - int(divisor) - except TypeError: - self.error("malformed dimension: divisor is not a number:", divisor) - divisor = 1 - if name is None: - name = id - if algorithm not in ("absolute", "incremental", "percentage-of-absolute-row", "percentage-of-incremental-row"): - algorithm = "absolute" - - self._dimensions.append(str(id)) - if hidden: - p = self._format(id, name, algorithm, multiplier, divisor, "hidden") - else: - p = self._format(id, name, algorithm, multiplier, divisor) - - self._line("DIMENSION", *p) - - def begin(self, type_id, microseconds=0): - """ - Begin data set - :param type_id: str - :param microseconds: int - :return: boolean - """ - if type_id not in self._charts: - self.error("wrong chart type_id:", type_id) - return False - try: - int(microseconds) - except TypeError: - self.error("malformed begin statement: microseconds are not a number:", microseconds) - microseconds = "" - - self._line("BEGIN", type_id, str(microseconds)) - return True - - def set(self, id, value): - """ - Set value to dimension - :param id: str - :param value: int/float - :return: boolean - """ - if id not in self._dimensions: - self.error("wrong dimension id:", id, "Available dimensions are:", *self._dimensions) - return False - try: - value = str(int(value)) - except TypeError: - self.error("cannot set non-numeric value:", str(value)) - return False - self._line("SET", id, "=", str(value)) - self.__chart_set = True - return True - - def end(self): - if self.__chart_set: - self._line("END") - self.__chart_set = False - else: - pos = self._data_stream.rfind("BEGIN") - self._data_stream = self._data_stream[:pos] - - def commit(self): - """ - Upload new data to netdata. - """ - try: - print(self._data_stream) - except Exception as e: - msg.fatal('cannot send data to netdata:', str(e)) - self._data_stream = "" - - # --- ERROR HANDLING --- - - def error(self, *params): - """ - Show error message on stderr - """ - msg.error(self.chart_name, *params) - - def alert(self, *params): - """ - Show error message on stderr - """ - msg.alert(self.chart_name, *params) - - def debug(self, *params): - """ - Show debug message on stderr - """ - msg.debug(self.chart_name, *params) - - def info(self, *params): - """ - Show information message on stderr - """ - msg.info(self.chart_name, *params) - - # --- MAIN METHODS --- - - def _get_data(self): - """ - Get some data - :return: dict - """ - return {} - - def check(self): - """ - check() prototype - :return: boolean - """ - self.debug("Module", str(self.__module__), "doesn't implement check() function. Using default.") - data = self._get_data() - - if data is None: - self.debug("failed to receive data during check().") - return False - - if len(data) == 0: - self.debug("empty data during check().") - return False - - self.debug("successfully received data during check(): '" + str(data) + "'") - return True - - def create(self): - """ - Create charts - :return: boolean - """ - data = self._data_from_check or self._get_data() - if data is None: - self.debug("failed to receive data during create().") - return False - - idx = 0 - for name in self.order: - options = self.definitions[name]['options'] + [self.priority + idx, self.update_every] - self.chart(self.chart_name + "." + name, *options) - # check if server has this datapoint - for line in self.definitions[name]['lines']: - if line[0] in data: - self.dimension(*line) - idx += 1 - - self.commit() - return True - - def update(self, interval): - """ - Update charts - :param interval: int - :return: boolean - """ - data = self._get_data() - if data is None: - self.debug("failed to receive data during update().") - return False - - updated = False - for chart in self.order: - if self.begin(self.chart_name + "." + chart, interval): - updated = True - for dim in self.definitions[chart]['lines']: - try: - self.set(dim[0], data[dim[0]]) - except KeyError: - pass - self.end() - - self.commit() - if not updated: - self.error("no charts to update") - - return updated - - @staticmethod - def find_binary(binary): - try: - if isinstance(binary, str): - binary = os.path.basename(binary) - return next(('/'.join([p, binary]) for p in PATH - if os.path.isfile('/'.join([p, binary])) - and os.access('/'.join([p, binary]), os.X_OK))) - return None - except StopIteration: - return None - - def _add_new_dimension(self, dimension_id, chart_name, dimension=None, algorithm='incremental', - multiplier=1, divisor=1, priority=65000): - """ - :param dimension_id: - :param chart_name: - :param dimension: - :param algorithm: - :param multiplier: - :param divisor: - :param priority: - :return: - """ - if not all([dimension_id not in self._dimensions, - chart_name in self.order, - chart_name in self.definitions]): - return - self._dimensions.append(dimension_id) - dimension_list = list(map(str, [dimension_id, - dimension if dimension else dimension_id, - algorithm, - multiplier, - divisor])) - self.definitions[chart_name]['lines'].append(dimension_list) - add_to_name = self.override_name or self.name - job_name = ('_'.join([self.__module__, re.sub('\s+', '_', add_to_name)]) - if add_to_name != 'None' else self.__module__) - chart = 'CHART {0}.{1} '.format(job_name, chart_name) - options = '"" "{0}" {1} "{2}" {3} {4} '.format(*self.definitions[chart_name]['options'][1:6]) - other = '{0} {1}\n'.format(priority, self.update_every) - new_dimension = "DIMENSION {0}\n".format(' '.join(dimension_list)) - print(chart + options + other + new_dimension) - - -class UrlService(SimpleService): - def __init__(self, configuration=None, name=None): - SimpleService.__init__(self, configuration=configuration, name=name) - self.url = self.configuration.get('url') - self.user = self.configuration.get('user') - self.password = self.configuration.get('pass') - self.proxy_user = self.configuration.get('proxy_user') - self.proxy_password = self.configuration.get('proxy_pass') - self.proxy_url = self.configuration.get('proxy_url') - self._manager = None - - def __make_headers(self, **header_kw): - user = header_kw.get('user') or self.user - password = header_kw.get('pass') or self.password - proxy_user = header_kw.get('proxy_user') or self.proxy_user - proxy_password = header_kw.get('proxy_pass') or self.proxy_password - header_params = dict(keep_alive=True) - proxy_header_params = dict() - if user and password: - header_params['basic_auth'] = '{user}:{password}'.format(user=user, - password=password) - if proxy_user and proxy_password: - proxy_header_params['proxy_basic_auth'] = '{user}:{password}'.format(user=proxy_user, - password=proxy_password) - try: - return urllib3.make_headers(**header_params), urllib3.make_headers(**proxy_header_params) - except TypeError as error: - self.error('build_header() error: {error}'.format(error=error)) - return None, None - - def _build_manager(self, **header_kw): - header, proxy_header = self.__make_headers(**header_kw) - if header is None or proxy_header is None: - return None - proxy_url = header_kw.get('proxy_url') or self.proxy_url - if proxy_url: - manager = urllib3.ProxyManager - params = dict(proxy_url=proxy_url, headers=header, proxy_headers=proxy_header) - else: - manager = urllib3.PoolManager - params = dict(headers=header) - try: - return manager(**params) - except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error: - self.error('build_manager() error:', str(error)) - return None - - def _get_raw_data(self, url=None, manager=None): - """ - Get raw data from http request - :return: str - """ - try: - url = url or self.url - manager = manager or self._manager - # TODO: timeout, retries and method hardcoded.. - response = manager.request(method='GET', - url=url, - timeout=1, - retries=1, - headers=manager.headers) - except (urllib3.exceptions.HTTPError, TypeError, AttributeError) as error: - self.error('Url: {url}. Error: {error}'.format(url=url, error=error)) - return None - if response.status == 200: - return response.data.decode() - self.debug('Url: {url}. Http response status code: {code}'.format(url=url, code=response.status)) - return None - - def check(self): - """ - Format configuration data and try to connect to server - :return: boolean - """ - if not (self.url and isinstance(self.url, str)): - self.error('URL is not defined or type is not ') - return False - - self._manager = self._build_manager() - if not self._manager: - return False - - try: - data = self._get_data() - except Exception as error: - self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error)) - return False - - if isinstance(data, dict) and data: - self._data_from_check = data - return True - self.error('_get_data() returned no data or type is not ') - return False - - -class SocketService(SimpleService): - def __init__(self, configuration=None, name=None): - self._sock = None - self._keep_alive = False - self.host = "localhost" - self.port = None - self.unix_socket = None - self.request = "" - self.__socket_config = None - self.__empty_request = "".encode() - SimpleService.__init__(self, configuration=configuration, name=name) - - def _socketerror(self, message=None): - if self.unix_socket is not None: - self.error("unix socket '" + self.unix_socket + "':", message) - else: - if self.__socket_config is not None: - af, socktype, proto, canonname, sa = self.__socket_config - self.error("socket to '" + str(sa[0]) + "' port " + str(sa[1]) + ":", message) - else: - self.error("unknown socket:", message) - - def _connect2socket(self, res=None): - """ - Connect to a socket, passing the result of getaddrinfo() - :return: boolean - """ - if res is None: - res = self.__socket_config - if res is None: - self.error("Cannot create socket to 'None':") - return False - - af, socktype, proto, canonname, sa = res - try: - self.debug("creating socket to '" + str(sa[0]) + "', port " + str(sa[1])) - self._sock = socket.socket(af, socktype, proto) - except socket.error as e: - self.error("Failed to create socket to '" + str(sa[0]) + "', port " + str(sa[1]) + ":", str(e)) - self._sock = None - self.__socket_config = None - return False - - try: - self.debug("connecting socket to '" + str(sa[0]) + "', port " + str(sa[1])) - self._sock.connect(sa) - except socket.error as e: - self.error("Failed to connect to '" + str(sa[0]) + "', port " + str(sa[1]) + ":", str(e)) - self._disconnect() - self.__socket_config = None - return False - - self.debug("connected to '" + str(sa[0]) + "', port " + str(sa[1])) - self.__socket_config = res - return True - - def _connect2unixsocket(self): - """ - Connect to a unix socket, given its filename - :return: boolean - """ - if self.unix_socket is None: - self.error("cannot connect to unix socket 'None'") - return False - - try: - self.debug("attempting DGRAM unix socket '" + str(self.unix_socket) + "'") - self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - self._sock.connect(self.unix_socket) - self.debug("connected DGRAM unix socket '" + str(self.unix_socket) + "'") - return True - except socket.error as e: - self.debug("Failed to connect DGRAM unix socket '" + str(self.unix_socket) + "':", str(e)) - - try: - self.debug("attempting STREAM unix socket '" + str(self.unix_socket) + "'") - self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._sock.connect(self.unix_socket) - self.debug("connected STREAM unix socket '" + str(self.unix_socket) + "'") - return True - except socket.error as e: - self.debug("Failed to connect STREAM unix socket '" + str(self.unix_socket) + "':", str(e)) - self.error("Failed to connect to unix socket '" + str(self.unix_socket) + "':", str(e)) - self._sock = None - return False - - def _connect(self): - """ - Recreate socket and connect to it since sockets cannot be reused after closing - Available configurations are IPv6, IPv4 or UNIX socket - :return: - """ - try: - if self.unix_socket is not None: - self._connect2unixsocket() - - else: - if self.__socket_config is not None: - self._connect2socket() - else: - for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): - if self._connect2socket(res): break - - except Exception as e: - self._sock = None - self.__socket_config = None - - if self._sock is not None: - self._sock.setblocking(0) - self._sock.settimeout(5) - self.debug("set socket timeout to: " + str(self._sock.gettimeout())) - - def _disconnect(self): - """ - Close socket connection - :return: - """ - if self._sock is not None: - try: - self.debug("closing socket") - self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all - self._sock.close() - except Exception: - pass - self._sock = None - - def _send(self): - """ - Send request. - :return: boolean - """ - # Send request if it is needed - if self.request != self.__empty_request: - try: - self.debug("sending request:", str(self.request)) - self._sock.send(self.request) - except Exception as e: - self._socketerror("error sending request:" + str(e)) - self._disconnect() - return False - return True - - def _receive(self): - """ - Receive data from socket - :return: str - """ - data = "" - while True: - self.debug("receiving response") - try: - buf = self._sock.recv(4096) - except Exception as e: - self._socketerror("failed to receive response:" + str(e)) - self._disconnect() - break - - if buf is None or len(buf) == 0: # handle server disconnect - if data == "": - self._socketerror("unexpectedly disconnected") - else: - self.debug("server closed the connection") - self._disconnect() - break - - self.debug("received data:", str(buf)) - data += buf.decode('utf-8', 'ignore') - if self._check_raw_data(data): - break - - self.debug("final response:", str(data)) - return data - - def _get_raw_data(self): - """ - Get raw data with low-level "socket" module. - :return: str - """ - if self._sock is None: - self._connect() - if self._sock is None: - return None - - # Send request if it is needed - if not self._send(): - return None - - data = self._receive() - - if not self._keep_alive: - self._disconnect() - - return data - - def _check_raw_data(self, data): - """ - Check if all data has been gathered from socket - :param data: str - :return: boolean - """ - return True - - def _parse_config(self): - """ - Parse configuration data - :return: boolean - """ - if self.name is None or self.name == str(None): - self.name = "" - else: - self.name = str(self.name) - - try: - self.unix_socket = str(self.configuration['socket']) - except (KeyError, TypeError): - self.debug("No unix socket specified. Trying TCP/IP socket.") - self.unix_socket = None - try: - self.host = str(self.configuration['host']) - except (KeyError, TypeError): - self.debug("No host specified. Using: '" + self.host + "'") - try: - self.port = int(self.configuration['port']) - except (KeyError, TypeError): - self.debug("No port specified. Using: '" + str(self.port) + "'") - - try: - self.request = str(self.configuration['request']) - except (KeyError, TypeError): - self.debug("No request specified. Using: '" + str(self.request) + "'") - - self.request = self.request.encode() - - def check(self): - self._parse_config() - return SimpleService.check(self) - - -class LogService(SimpleService): - def __init__(self, configuration=None, name=None): - SimpleService.__init__(self, configuration=configuration, name=name) - self.log_path = self.configuration.get('path') - self.__glob_path = self.log_path - self._last_position = 0 - self.retries = 100000 # basically always retry - self.__re_find = dict(current=0, run=0, maximum=60) - - def _get_raw_data(self): - """ - Get log lines since last poll - :return: list - """ - lines = list() - try: - if self.__re_find['current'] == self.__re_find['run']: - self._find_recent_log_file() - size = os.path.getsize(self.log_path) - if size == self._last_position: - self.__re_find['current'] += 1 - return list() # return empty list if nothing has changed - elif size < self._last_position: - self._last_position = 0 # read from beginning if file has shrunk - - with open(self.log_path) as fp: - fp.seek(self._last_position) - for line in fp: - lines.append(line) - self._last_position = fp.tell() - self.__re_find['current'] = 0 - except (OSError, IOError) as error: - self.__re_find['current'] += 1 - self.error(str(error)) - - return lines or None - - def _find_recent_log_file(self): - """ - :return: - """ - self.__re_find['run'] = self.__re_find['maximum'] - self.__re_find['current'] = 0 - self.__glob_path = self.__glob_path or self.log_path # workaround for modules w/o config files - path_list = glob(self.__glob_path) - if path_list: - self.log_path = max(path_list) - return True - return False - - def check(self): - """ - Parse basic configuration and check if log file exists - :return: boolean - """ - if not self.log_path: - self.error("No path to log specified") - return None - - if all([self._find_recent_log_file(), - os.access(self.log_path, os.R_OK), - os.path.isfile(self.log_path)]): - return True - self.error("Cannot access %s" % self.log_path) - return False - - def create(self): - # set cursor at last byte of log file - self._last_position = os.path.getsize(self.log_path) - status = SimpleService.create(self) - # self._last_position = 0 - return status - - -class ExecutableService(SimpleService): - - def __init__(self, configuration=None, name=None): - SimpleService.__init__(self, configuration=configuration, name=name) - self.command = None - - def _get_raw_data(self, stderr=False): - """ - Get raw data from executed command - :return: - """ - try: - p = Popen(self.command, stdout=PIPE, stderr=PIPE) - except Exception as error: - self.error("Executing command", " ".join(self.command), "resulted in error:", str(error)) - return None - data = list() - std = p.stderr if stderr else p.stdout - for line in std.readlines(): - data.append(line.decode()) - - return data or None - - def check(self): - """ - Parse basic configuration, check if command is whitelisted and is returning values - :return: - """ - # Preference: 1. "command" from configuration file 2. "command" from plugin (if specified) - if 'command' in self.configuration: - self.command = self.configuration['command'] - - # "command" must be: 1.not None 2. type - if not (self.command and isinstance(self.command, str)): - self.error('Command is not defined or command type is not ') - return False - - # Split "command" into: 1. command 2. options - command, opts = self.command.split()[0], self.command.split()[1:] - - # Check for "bad" symbols in options. No pipes, redirects etc. TODO: what is missing? - bad_opts = set(''.join(opts)) & set(['&', '|', ';', '>', '<']) - if bad_opts: - self.error("Bad command argument(s): %s" % bad_opts) - return False - - # Find absolute path ('echo' => '/bin/echo') - if '/' not in command: - command = self.find_binary(command) - if not command: - self.error('Can\'t locate "%s" binary in PATH(%s)' % (self.command, PATH)) - return False - # Check if binary exist and executable - else: - if not (os.path.isfile(command) and os.access(command, os.X_OK)): - self.error('"%s" is not a file or not executable' % command) - return False - - self.command = [command] + opts if opts else [command] - - try: - data = self._get_data() - except Exception as error: - self.error('_get_data() failed. Command: %s. Error: %s' % (self.command, error)) - return False - - if isinstance(data, dict) and data: - # We need this for create() method. No reason to execute get_data() again if result is not empty dict() - self._data_from_check = data - return True - else: - self.error("Command", str(self.command), "returned no data") - return False - - -class MySQLService(SimpleService): - - def __init__(self, configuration=None, name=None): - SimpleService.__init__(self, configuration=configuration, name=name) - self.__connection = None - self.__conn_properties = dict() - self.extra_conn_properties = dict() - self.__queries = self.configuration.get('queries', dict()) - self.queries = dict() - - def __connect(self): - try: - connection = MySQLdb.connect(connect_timeout=self.update_every, **self.__conn_properties) - except (MySQLdb.MySQLError, TypeError, AttributeError) as error: - return None, str(error) - else: - return connection, None - - def check(self): - def get_connection_properties(conf, extra_conf): - properties = dict() - if conf.get('user'): - properties['user'] = conf['user'] - if conf.get('pass'): - properties['passwd'] = conf['pass'] - if conf.get('socket'): - properties['unix_socket'] = conf['socket'] - elif conf.get('host'): - properties['host'] = conf['host'] - properties['port'] = int(conf.get('port', 3306)) - elif conf.get('my.cnf'): - if MySQLdb.__name__ == 'pymysql': - self.error('"my.cnf" parsing is not working for pymysql') - else: - properties['read_default_file'] = conf['my.cnf'] - if isinstance(extra_conf, dict) and extra_conf: - properties.update(extra_conf) - - return properties or None - - def is_valid_queries_dict(raw_queries, log_error): - """ - :param raw_queries: dict: - :param log_error: function: - :return: dict or None - - raw_queries is valid when: type and not empty after is_valid_query(for all queries) - """ - def is_valid_query(query): - return all([isinstance(query, str), - query.startswith(('SELECT', 'select', 'SHOW', 'show'))]) - - if hasattr(raw_queries, 'keys') and raw_queries: - valid_queries = dict([(n, q) for n, q in raw_queries.items() if is_valid_query(q)]) - bad_queries = set(raw_queries) - set(valid_queries) - - if bad_queries: - log_error('Removed query(s): %s' % bad_queries) - return valid_queries - else: - log_error('Unsupported "queries" format. Must be not empty ') - return None - - if not PY_MYSQL: - self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin') - return False - - # Preference: 1. "queries" from the configuration file 2. "queries" from the module - self.queries = self.__queries or self.queries - # Check if "self.queries" exist, not empty and all queries are in valid format - self.queries = is_valid_queries_dict(self.queries, self.error) - if not self.queries: - return None - - # Get connection properties - self.__conn_properties = get_connection_properties(self.configuration, self.extra_conn_properties) - if not self.__conn_properties: - self.error('Connection properties are missing') - return False - - # Create connection to the database - self.__connection, error = self.__connect() - if error: - self.error('Can\'t establish connection to MySQL: %s' % error) - return False - - try: - data = self._get_data() - except Exception as error: - self.error('_get_data() failed. Error: %s' % error) - return False - - if isinstance(data, dict) and data: - # We need this for create() method - self._data_from_check = data - return True - else: - self.error("_get_data() returned no data or type is not ") - return False - - def _get_raw_data(self, description=None): - """ - Get raw data from MySQL server - :return: dict: fetchall() or (fetchall(), description) - """ - - if not self.__connection: - self.__connection, error = self.__connect() - if error: - return None - - raw_data = dict() - queries = dict(self.queries) - try: - with self.__connection as cursor: - for name, query in queries.items(): - try: - cursor.execute(query) - except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as error: - if self.__is_error_critical(err_class=exc_info()[0], err_text=str(error)): - raise RuntimeError - self.error('Removed query: %s[%s]. Error: %s' - % (name, query, error)) - self.queries.pop(name) - continue - else: - raw_data[name] = (cursor.fetchall(), cursor.description) if description else cursor.fetchall() - self.__connection.commit() - except (MySQLdb.MySQLError, RuntimeError, TypeError, AttributeError): - self.__connection.close() - self.__connection = None - return None - else: - return raw_data or None - - @staticmethod - def __is_error_critical(err_class, err_text): - return err_class == MySQLdb.OperationalError and all(['denied' not in err_text, - 'Unknown column' not in err_text]) +# Description: backward compatibility with old version + +from bases.FrameworkServices.SimpleService import SimpleService +from bases.FrameworkServices.UrlService import UrlService +from bases.FrameworkServices.SocketService import SocketService +from bases.FrameworkServices.LogService import LogService +from bases.FrameworkServices.ExecutableService import ExecutableService +from bases.FrameworkServices.MySQLService import MySQLService diff --git a/python.d/python_modules/bases/FrameworkServices/ExecutableService.py b/python.d/python_modules/bases/FrameworkServices/ExecutableService.py new file mode 100644 index 000000000..b6d7871fa --- /dev/null +++ b/python.d/python_modules/bases/FrameworkServices/ExecutableService.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Pawel Krupa (paulfantom) +# Author: Ilya Mashchenko (l2isbad) + +import os + +from subprocess import Popen, PIPE + +from bases.FrameworkServices.SimpleService import SimpleService +from bases.collection import find_binary + + +class ExecutableService(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.command = None + + def _get_raw_data(self, stderr=False): + """ + Get raw data from executed command + :return: + """ + try: + p = Popen(self.command, stdout=PIPE, stderr=PIPE) + except Exception as error: + self.error('Executing command {command} resulted in error: {error}'.format(command=self.command, + error=error)) + return None + data = list() + std = p.stderr if stderr else p.stdout + for line in std: + data.append(line.decode()) + + return data or None + + def check(self): + """ + Parse basic configuration, check if command is whitelisted and is returning values + :return: + """ + # Preference: 1. "command" from configuration file 2. "command" from plugin (if specified) + if 'command' in self.configuration: + self.command = self.configuration['command'] + + # "command" must be: 1.not None 2. type + if not (self.command and isinstance(self.command, str)): + self.error('Command is not defined or command type is not ') + return False + + # Split "command" into: 1. command 2. options + command, opts = self.command.split()[0], self.command.split()[1:] + + # Check for "bad" symbols in options. No pipes, redirects etc. + opts_list = ['&', '|', ';', '>', '<'] + bad_opts = set(''.join(opts)) & set(opts_list) + if bad_opts: + self.error("Bad command argument(s): {opts}".format(opts=bad_opts)) + return False + + # Find absolute path ('echo' => '/bin/echo') + if '/' not in command: + command = find_binary(command) + if not command: + self.error('Can\'t locate "{command}" binary'.format(command=self.command)) + return False + # Check if binary exist and executable + else: + if not os.access(command, os.X_OK): + self.error('"{binary}" is not executable'.format(binary=command)) + return False + + self.command = [command] + opts if opts else [command] + + try: + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Command: {command}. Error: {error}'.format(command=self.command, + error=error)) + return False + + if isinstance(data, dict) and data: + return True + self.error('Command "{command}" returned no data'.format(command=self.command)) + return False diff --git a/python.d/python_modules/bases/FrameworkServices/LogService.py b/python.d/python_modules/bases/FrameworkServices/LogService.py new file mode 100644 index 000000000..45daa2446 --- /dev/null +++ b/python.d/python_modules/bases/FrameworkServices/LogService.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Pawel Krupa (paulfantom) + +from glob import glob +import os + +from bases.FrameworkServices.SimpleService import SimpleService + + +class LogService(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.log_path = self.configuration.get('path') + self.__glob_path = self.log_path + self._last_position = 0 + self.__re_find = dict(current=0, run=0, maximum=60) + + def _get_raw_data(self): + """ + Get log lines since last poll + :return: list + """ + lines = list() + try: + if self.__re_find['current'] == self.__re_find['run']: + self._find_recent_log_file() + size = os.path.getsize(self.log_path) + if size == self._last_position: + self.__re_find['current'] += 1 + return list() # return empty list if nothing has changed + elif size < self._last_position: + self._last_position = 0 # read from beginning if file has shrunk + + with open(self.log_path) as fp: + fp.seek(self._last_position) + for line in fp: + lines.append(line) + self._last_position = fp.tell() + self.__re_find['current'] = 0 + except (OSError, IOError) as error: + self.__re_find['current'] += 1 + self.error(str(error)) + + return lines or None + + def _find_recent_log_file(self): + """ + :return: + """ + self.__re_find['run'] = self.__re_find['maximum'] + self.__re_find['current'] = 0 + self.__glob_path = self.__glob_path or self.log_path # workaround for modules w/o config files + path_list = glob(self.__glob_path) + if path_list: + self.log_path = max(path_list) + return True + return False + + def check(self): + """ + Parse basic configuration and check if log file exists + :return: boolean + """ + if not self.log_path: + self.error('No path to log specified') + return None + + if self._find_recent_log_file() and os.access(self.log_path, os.R_OK) and os.path.isfile(self.log_path): + return True + self.error('Cannot access {0}'.format(self.log_path)) + return False + + def create(self): + # set cursor at last byte of log file + self._last_position = os.path.getsize(self.log_path) + status = SimpleService.create(self) + return status diff --git a/python.d/python_modules/bases/FrameworkServices/MySQLService.py b/python.d/python_modules/bases/FrameworkServices/MySQLService.py new file mode 100644 index 000000000..3acc5b109 --- /dev/null +++ b/python.d/python_modules/bases/FrameworkServices/MySQLService.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Ilya Mashchenko (l2isbad) + +from sys import exc_info + +try: + import MySQLdb + + PY_MYSQL = True +except ImportError: + try: + import pymysql as MySQLdb + + PY_MYSQL = True + except ImportError: + PY_MYSQL = False + +from bases.FrameworkServices.SimpleService import SimpleService + + +class MySQLService(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.__connection = None + self.__conn_properties = dict() + self.extra_conn_properties = dict() + self.__queries = self.configuration.get('queries', dict()) + self.queries = dict() + + def __connect(self): + try: + connection = MySQLdb.connect(connect_timeout=self.update_every, **self.__conn_properties) + except (MySQLdb.MySQLError, TypeError, AttributeError) as error: + return None, str(error) + else: + return connection, None + + def check(self): + def get_connection_properties(conf, extra_conf): + properties = dict() + if conf.get('user'): + properties['user'] = conf['user'] + if conf.get('pass'): + properties['passwd'] = conf['pass'] + if conf.get('socket'): + properties['unix_socket'] = conf['socket'] + elif conf.get('host'): + properties['host'] = conf['host'] + properties['port'] = int(conf.get('port', 3306)) + elif conf.get('my.cnf'): + if MySQLdb.__name__ == 'pymysql': + self.error('"my.cnf" parsing is not working for pymysql') + else: + properties['read_default_file'] = conf['my.cnf'] + if isinstance(extra_conf, dict) and extra_conf: + properties.update(extra_conf) + + return properties or None + + def is_valid_queries_dict(raw_queries, log_error): + """ + :param raw_queries: dict: + :param log_error: function: + :return: dict or None + + raw_queries is valid when: type and not empty after is_valid_query(for all queries) + """ + + def is_valid_query(query): + return all([isinstance(query, str), + query.startswith(('SELECT', 'select', 'SHOW', 'show'))]) + + if hasattr(raw_queries, 'keys') and raw_queries: + valid_queries = dict([(n, q) for n, q in raw_queries.items() if is_valid_query(q)]) + bad_queries = set(raw_queries) - set(valid_queries) + + if bad_queries: + log_error('Removed query(s): {queries}'.format(queries=bad_queries)) + return valid_queries + else: + log_error('Unsupported "queries" format. Must be not empty ') + return None + + if not PY_MYSQL: + self.error('MySQLdb or PyMySQL module is needed to use mysql.chart.py plugin') + return False + + # Preference: 1. "queries" from the configuration file 2. "queries" from the module + self.queries = self.__queries or self.queries + # Check if "self.queries" exist, not empty and all queries are in valid format + self.queries = is_valid_queries_dict(self.queries, self.error) + if not self.queries: + return None + + # Get connection properties + self.__conn_properties = get_connection_properties(self.configuration, self.extra_conn_properties) + if not self.__conn_properties: + self.error('Connection properties are missing') + return False + + # Create connection to the database + self.__connection, error = self.__connect() + if error: + self.error('Can\'t establish connection to MySQL: {error}'.format(error=error)) + return False + + try: + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Error: {error}'.format(error=error)) + return False + + if isinstance(data, dict) and data: + return True + self.error("_get_data() returned no data or type is not ") + return False + + def _get_raw_data(self, description=None): + """ + Get raw data from MySQL server + :return: dict: fetchall() or (fetchall(), description) + """ + + if not self.__connection: + self.__connection, error = self.__connect() + if error: + return None + + raw_data = dict() + queries = dict(self.queries) + try: + with self.__connection as cursor: + for name, query in queries.items(): + try: + cursor.execute(query) + except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as error: + if self.__is_error_critical(err_class=exc_info()[0], err_text=str(error)): + raise RuntimeError + self.error('Removed query: {name}[{query}]. Error: error'.format(name=name, + query=query, + error=error)) + self.queries.pop(name) + continue + else: + raw_data[name] = (cursor.fetchall(), cursor.description) if description else cursor.fetchall() + self.__connection.commit() + except (MySQLdb.MySQLError, RuntimeError, TypeError, AttributeError): + self.__connection.close() + self.__connection = None + return None + else: + return raw_data or None + + @staticmethod + def __is_error_critical(err_class, err_text): + return err_class == MySQLdb.OperationalError and all(['denied' not in err_text, + 'Unknown column' not in err_text]) diff --git a/python.d/python_modules/bases/FrameworkServices/SimpleService.py b/python.d/python_modules/bases/FrameworkServices/SimpleService.py new file mode 100644 index 000000000..14c839101 --- /dev/null +++ b/python.d/python_modules/bases/FrameworkServices/SimpleService.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Pawel Krupa (paulfantom) +# Author: Ilya Mashchenko (l2isbad) + +from threading import Thread + +try: + from time import sleep, monotonic as time +except ImportError: + from time import sleep, time + +from bases.charts import Charts, ChartError, create_runtime_chart +from bases.collection import OldVersionCompatibility, safe_print +from bases.loggers import PythonDLimitedLogger + +RUNTIME_CHART_UPDATE = 'BEGIN netdata.runtime_{job_name} {since_last}\n' \ + 'SET run_time = {elapsed}\n' \ + 'END\n' + + +class RuntimeCounters: + def __init__(self, configuration): + """ + :param configuration: + """ + self.FREQ = int(configuration.pop('update_every')) + self.START_RUN = 0 + self.NEXT_RUN = 0 + self.PREV_UPDATE = 0 + self.SINCE_UPDATE = 0 + self.ELAPSED = 0 + self.RETRIES = 0 + self.RETRIES_MAX = configuration.pop('retries') + self.PENALTY = 0 + + def is_sleep_time(self): + return self.START_RUN < self.NEXT_RUN + + +class SimpleService(Thread, PythonDLimitedLogger, OldVersionCompatibility, object): + """ + Prototype of Service class. + Implemented basic functionality to run jobs by `python.d.plugin` + """ + def __init__(self, configuration, name=''): + """ + :param configuration: + :param name: + """ + Thread.__init__(self) + self.daemon = True + PythonDLimitedLogger.__init__(self) + OldVersionCompatibility.__init__(self) + self.configuration = configuration + self.order = list() + self.definitions = dict() + + self.module_name = self.__module__ + self.job_name = configuration.pop('job_name') + self.override_name = configuration.pop('override_name') + self.fake_name = None + + self._runtime_counters = RuntimeCounters(configuration=configuration) + self.charts = Charts(job_name=self.actual_name, + priority=configuration.pop('priority'), + cleanup=configuration.pop('chart_cleanup'), + get_update_every=self.get_update_every, + module_name=self.module_name) + + def __repr__(self): + return '<{cls_bases}: {name}>'.format(cls_bases=', '.join(c.__name__ for c in self.__class__.__bases__), + name=self.name) + + @property + def name(self): + if self.job_name: + return '_'.join([self.module_name, self.override_name or self.job_name]) + return self.module_name + + def actual_name(self): + return self.fake_name or self.name + + @property + def update_every(self): + return self._runtime_counters.FREQ + + @update_every.setter + def update_every(self, value): + """ + :param value: + :return: + """ + self._runtime_counters.FREQ = value + + def get_update_every(self): + return self.update_every + + def check(self): + """ + check() prototype + :return: boolean + """ + self.debug("job doesn't implement check() method. Using default which simply invokes get_data().") + data = self.get_data() + if data and isinstance(data, dict): + return True + self.debug('returned value is wrong: {0}'.format(data)) + return False + + @create_runtime_chart + def create(self): + for chart_name in self.order: + chart_config = self.definitions.get(chart_name) + + if not chart_config: + self.debug("create() => [NOT ADDED] chart '{chart_name}' not in definitions. " + "Skipping it.".format(chart_name=chart_name)) + continue + + # create chart + chart_params = [chart_name] + chart_config['options'] + try: + self.charts.add_chart(params=chart_params) + except ChartError as error: + self.error("create() => [NOT ADDED] (chart '{chart}': {error})".format(chart=chart_name, + error=error)) + continue + + # add dimensions to chart + for dimension in chart_config['lines']: + try: + self.charts[chart_name].add_dimension(dimension) + except ChartError as error: + self.error("create() => [NOT ADDED] (dimension '{dimension}': {error})".format(dimension=dimension, + error=error)) + continue + + # add variables to chart + if 'variables' in chart_config: + for variable in chart_config['variables']: + try: + self.charts[chart_name].add_variable(variable) + except ChartError as error: + self.error("create() => [NOT ADDED] (variable '{var}': {error})".format(var=variable, + error=error)) + continue + + del self.order + del self.definitions + + # True if job has at least 1 chart else False + return bool(self.charts) + + def run(self): + """ + Runs job in thread. Handles retries. + Exits when job failed or timed out. + :return: None + """ + job = self._runtime_counters + self.debug('started, update frequency: {freq}, ' + 'retries: {retries}'.format(freq=job.FREQ, retries=job.RETRIES_MAX - job.RETRIES)) + + while True: + job.START_RUN = time() + + job.NEXT_RUN = job.START_RUN - (job.START_RUN % job.FREQ) + job.FREQ + job.PENALTY + + self.sleep_until_next_run() + + if job.PREV_UPDATE: + job.SINCE_UPDATE = int((job.START_RUN - job.PREV_UPDATE) * 1e6) + + try: + updated = self.update(interval=job.SINCE_UPDATE) + except Exception as error: + self.error('update() unhandled exception: {error}'.format(error=error)) + updated = False + + if not updated: + if not self.manage_retries(): + return + else: + job.ELAPSED = int((time() - job.START_RUN) * 1e3) + job.PREV_UPDATE = job.START_RUN + job.RETRIES, job.PENALTY = 0, 0 + safe_print(RUNTIME_CHART_UPDATE.format(job_name=self.name, + since_last=job.SINCE_UPDATE, + elapsed=job.ELAPSED)) + self.debug('update => [{status}] (elapsed time: {elapsed}, ' + 'retries left: {retries})'.format(status='OK' if updated else 'FAILED', + elapsed=job.ELAPSED if updated else '-', + retries=job.RETRIES_MAX - job.RETRIES)) + + def update(self, interval): + """ + :return: + """ + data = self.get_data() + if not data: + self.debug('get_data() returned no data') + return False + elif not isinstance(data, dict): + self.debug('get_data() returned incorrect type data') + return False + + updated = False + + for chart in self.charts: + if chart.flags.obsoleted: + continue + elif self.charts.cleanup and chart.penalty >= self.charts.cleanup: + chart.obsolete() + self.error("chart '{0}' was suppressed due to non updating".format(chart.name)) + continue + + ok = chart.update(data, interval) + if ok: + updated = True + + if not updated: + self.debug('none of the charts has been updated') + + return updated + + def manage_retries(self): + rc = self._runtime_counters + rc.RETRIES += 1 + if rc.RETRIES % 5 == 0: + rc.PENALTY = int(rc.RETRIES * self.update_every / 2) + if rc.RETRIES >= rc.RETRIES_MAX: + self.error('stopped after {0} data collection failures in a row'.format(rc.RETRIES_MAX)) + return False + return True + + def sleep_until_next_run(self): + job = self._runtime_counters + + # sleep() is interruptable + while job.is_sleep_time(): + sleep_time = job.NEXT_RUN - job.START_RUN + self.debug('sleeping for {sleep_time} to reach frequency of {freq} sec'.format(sleep_time=sleep_time, + freq=job.FREQ + job.PENALTY)) + sleep(sleep_time) + job.START_RUN = time() + + def get_data(self): + return self._get_data() + + def _get_data(self): + raise NotImplementedError diff --git a/python.d/python_modules/bases/FrameworkServices/SocketService.py b/python.d/python_modules/bases/FrameworkServices/SocketService.py new file mode 100644 index 000000000..90631df16 --- /dev/null +++ b/python.d/python_modules/bases/FrameworkServices/SocketService.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Pawel Krupa (paulfantom) + +import socket + +from bases.FrameworkServices.SimpleService import SimpleService + + +class SocketService(SimpleService): + def __init__(self, configuration=None, name=None): + self._sock = None + self._keep_alive = False + self.host = 'localhost' + self.port = None + self.unix_socket = None + self.request = '' + self.__socket_config = None + self.__empty_request = "".encode() + SimpleService.__init__(self, configuration=configuration, name=name) + + def _socket_error(self, message=None): + if self.unix_socket is not None: + self.error('unix socket "{socket}": {message}'.format(socket=self.unix_socket, + message=message)) + else: + if self.__socket_config is not None: + af, sock_type, proto, canon_name, sa = self.__socket_config + self.error('socket to "{address}" port {port}: {message}'.format(address=sa[0], + port=sa[1], + message=message)) + else: + self.error('unknown socket: {0}'.format(message)) + + def _connect2socket(self, res=None): + """ + Connect to a socket, passing the result of getaddrinfo() + :return: boolean + """ + if res is None: + res = self.__socket_config + if res is None: + self.error("Cannot create socket to 'None':") + return False + + af, sock_type, proto, canon_name, sa = res + try: + self.debug('Creating socket to "{address}", port {port}'.format(address=sa[0], port=sa[1])) + self._sock = socket.socket(af, sock_type, proto) + except socket.error as error: + self.error('Failed to create socket "{address}", port {port}, error: {error}'.format(address=sa[0], + port=sa[1], + error=error)) + self._sock = None + self.__socket_config = None + return False + + try: + self.debug('connecting socket to "{address}", port {port}'.format(address=sa[0], port=sa[1])) + self._sock.connect(sa) + except socket.error as error: + self.error('Failed to connect to "{address}", port {port}, error: {error}'.format(address=sa[0], + port=sa[1], + error=error)) + self._disconnect() + self.__socket_config = None + return False + + self.debug('connected to "{address}", port {port}'.format(address=sa[0], port=sa[1])) + self.__socket_config = res + return True + + def _connect2unixsocket(self): + """ + Connect to a unix socket, given its filename + :return: boolean + """ + if self.unix_socket is None: + self.error("cannot connect to unix socket 'None'") + return False + + try: + self.debug('attempting DGRAM unix socket "{0}"'.format(self.unix_socket)) + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._sock.connect(self.unix_socket) + self.debug('connected DGRAM unix socket "{0}"'.format(self.unix_socket)) + return True + except socket.error as error: + self.debug('Failed to connect DGRAM unix socket "{socket}": {error}'.format(socket=self.unix_socket, + error=error)) + + try: + self.debug('attempting STREAM unix socket "{0}"'.format(self.unix_socket)) + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._sock.connect(self.unix_socket) + self.debug('connected STREAM unix socket "{0}"'.format(self.unix_socket)) + return True + except socket.error as error: + self.debug('Failed to connect STREAM unix socket "{socket}": {error}'.format(socket=self.unix_socket, + error=error)) + self._sock = None + return False + + def _connect(self): + """ + Recreate socket and connect to it since sockets cannot be reused after closing + Available configurations are IPv6, IPv4 or UNIX socket + :return: + """ + try: + if self.unix_socket is not None: + self._connect2unixsocket() + + else: + if self.__socket_config is not None: + self._connect2socket() + else: + for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): + if self._connect2socket(res): + break + + except Exception: + self._sock = None + self.__socket_config = None + + if self._sock is not None: + self._sock.setblocking(0) + self._sock.settimeout(5) + self.debug('set socket timeout to: {0}'.format(self._sock.gettimeout())) + + def _disconnect(self): + """ + Close socket connection + :return: + """ + if self._sock is not None: + try: + self.debug('closing socket') + self._sock.shutdown(2) # 0 - read, 1 - write, 2 - all + self._sock.close() + except Exception: + pass + self._sock = None + + def _send(self): + """ + Send request. + :return: boolean + """ + # Send request if it is needed + if self.request != self.__empty_request: + try: + self.debug('sending request: {0}'.format(self.request)) + self._sock.send(self.request) + except Exception as error: + self._socket_error('error sending request: {0}'.format(error)) + self._disconnect() + return False + return True + + def _receive(self): + """ + Receive data from socket + :return: str + """ + data = "" + while True: + self.debug('receiving response') + try: + buf = self._sock.recv(4096) + except Exception as error: + self._socket_error('failed to receive response: {0}'.format(error)) + self._disconnect() + break + + if buf is None or len(buf) == 0: # handle server disconnect + if data == "": + self._socket_error('unexpectedly disconnected') + else: + self.debug('server closed the connection') + self._disconnect() + break + + self.debug('received data') + data += buf.decode('utf-8', 'ignore') + if self._check_raw_data(data): + break + + self.debug('final response: {0}'.format(data)) + return data + + def _get_raw_data(self): + """ + Get raw data with low-level "socket" module. + :return: str + """ + if self._sock is None: + self._connect() + if self._sock is None: + return None + + # Send request if it is needed + if not self._send(): + return None + + data = self._receive() + + if not self._keep_alive: + self._disconnect() + + return data + + @staticmethod + def _check_raw_data(data): + """ + Check if all data has been gathered from socket + :param data: str + :return: boolean + """ + return bool(data) + + def _parse_config(self): + """ + Parse configuration data + :return: boolean + """ + try: + self.unix_socket = str(self.configuration['socket']) + except (KeyError, TypeError): + self.debug('No unix socket specified. Trying TCP/IP socket.') + self.unix_socket = None + try: + self.host = str(self.configuration['host']) + except (KeyError, TypeError): + self.debug('No host specified. Using: "{0}"'.format(self.host)) + try: + self.port = int(self.configuration['port']) + except (KeyError, TypeError): + self.debug('No port specified. Using: "{0}"'.format(self.port)) + + try: + self.request = str(self.configuration['request']) + except (KeyError, TypeError): + self.debug('No request specified. Using: "{0}"'.format(self.request)) + + self.request = self.request.encode() + + def check(self): + self._parse_config() + return SimpleService.check(self) diff --git a/python.d/python_modules/bases/FrameworkServices/UrlService.py b/python.d/python_modules/bases/FrameworkServices/UrlService.py new file mode 100644 index 000000000..0941ab168 --- /dev/null +++ b/python.d/python_modules/bases/FrameworkServices/UrlService.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Pawel Krupa (paulfantom) +# Author: Ilya Mashchenko (l2isbad) + +import urllib3 + +from bases.FrameworkServices.SimpleService import SimpleService + +try: + urllib3.disable_warnings() +except AttributeError: + pass + + +class UrlService(SimpleService): + def __init__(self, configuration=None, name=None): + SimpleService.__init__(self, configuration=configuration, name=name) + self.url = self.configuration.get('url') + self.user = self.configuration.get('user') + self.password = self.configuration.get('pass') + self.proxy_user = self.configuration.get('proxy_user') + self.proxy_password = self.configuration.get('proxy_pass') + self.proxy_url = self.configuration.get('proxy_url') + self.header = self.configuration.get('header') + self.request_timeout = self.configuration.get('timeout', 1) + self._manager = None + + def __make_headers(self, **header_kw): + user = header_kw.get('user') or self.user + password = header_kw.get('pass') or self.password + proxy_user = header_kw.get('proxy_user') or self.proxy_user + proxy_password = header_kw.get('proxy_pass') or self.proxy_password + custom_header = header_kw.get('header') or self.header + header_params = dict(keep_alive=True) + proxy_header_params = dict() + if user and password: + header_params['basic_auth'] = '{user}:{password}'.format(user=user, + password=password) + if proxy_user and proxy_password: + proxy_header_params['proxy_basic_auth'] = '{user}:{password}'.format(user=proxy_user, + password=proxy_password) + try: + header, proxy_header = urllib3.make_headers(**header_params), urllib3.make_headers(**proxy_header_params) + except TypeError as error: + self.error('build_header() error: {error}'.format(error=error)) + return None, None + else: + header.update(custom_header or dict()) + return header, proxy_header + + def _build_manager(self, **header_kw): + header, proxy_header = self.__make_headers(**header_kw) + if header is None or proxy_header is None: + return None + proxy_url = header_kw.get('proxy_url') or self.proxy_url + if proxy_url: + manager = urllib3.ProxyManager + params = dict(proxy_url=proxy_url, headers=header, proxy_headers=proxy_header) + else: + manager = urllib3.PoolManager + params = dict(headers=header) + try: + url = header_kw.get('url') or self.url + if url.startswith('https'): + return manager(assert_hostname=False, cert_reqs='CERT_NONE', **params) + return manager(**params) + except (urllib3.exceptions.ProxySchemeUnknown, TypeError) as error: + self.error('build_manager() error:', str(error)) + return None + + def _get_raw_data(self, url=None, manager=None): + """ + Get raw data from http request + :return: str + """ + try: + url = url or self.url + manager = manager or self._manager + response = manager.request(method='GET', + url=url, + timeout=self.request_timeout, + retries=1, + headers=manager.headers) + except (urllib3.exceptions.HTTPError, TypeError, AttributeError) as error: + self.error('Url: {url}. Error: {error}'.format(url=url, error=error)) + return None + if response.status == 200: + return response.data.decode() + self.debug('Url: {url}. Http response status code: {code}'.format(url=url, code=response.status)) + return None + + def check(self): + """ + Format configuration data and try to connect to server + :return: boolean + """ + if not (self.url and isinstance(self.url, str)): + self.error('URL is not defined or type is not ') + return False + + self._manager = self._build_manager() + if not self._manager: + return False + + try: + data = self._get_data() + except Exception as error: + self.error('_get_data() failed. Url: {url}. Error: {error}'.format(url=self.url, error=error)) + return False + + if isinstance(data, dict) and data: + return True + self.error('_get_data() returned no data or type is not ') + return False diff --git a/python.d/python_modules/bases/FrameworkServices/__init__.py b/python.d/python_modules/bases/FrameworkServices/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python.d/python_modules/bases/__init__.py b/python.d/python_modules/bases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python.d/python_modules/bases/charts.py b/python.d/python_modules/bases/charts.py new file mode 100644 index 000000000..1e9348e59 --- /dev/null +++ b/python.d/python_modules/bases/charts.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Ilya Mashchenko (l2isbad) + +from bases.collection import safe_print + +CHART_PARAMS = ['type', 'id', 'name', 'title', 'units', 'family', 'context', 'chart_type'] +DIMENSION_PARAMS = ['id', 'name', 'algorithm', 'multiplier', 'divisor', 'hidden'] +VARIABLE_PARAMS = ['id', 'value'] + +CHART_TYPES = ['line', 'area', 'stacked'] +DIMENSION_ALGORITHMS = ['absolute', 'incremental', 'percentage-of-absolute-row', 'percentage-of-incremental-row'] + +CHART_BEGIN = 'BEGIN {type}.{id} {since_last}\n' +CHART_CREATE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \ + "{chart_type} {priority} {update_every} '' 'python.d.plugin' '{module_name}'\n" +CHART_OBSOLETE = "CHART {type}.{id} '{name}' '{title}' '{units}' '{family}' '{context}' " \ + "{chart_type} {priority} {update_every} 'obsolete'\n" + + +DIMENSION_CREATE = "DIMENSION '{id}' '{name}' {algorithm} {multiplier} {divisor} '{hidden}'\n" +DIMENSION_SET = "SET '{id}' = {value}\n" + +CHART_VARIABLE_SET = "VARIABLE CHART '{id}' = {value}\n" + +RUNTIME_CHART_CREATE = "CHART netdata.runtime_{job_name} '' 'Execution time for {job_name}' 'ms' 'python.d' " \ + "netdata.pythond_runtime line 145000 {update_every}\n" \ + "DIMENSION run_time 'run time' absolute 1 1\n" + + +def create_runtime_chart(func): + """ + Calls a wrapped function, then prints runtime chart to stdout. + + Used as a decorator for SimpleService.create() method. + The whole point of making 'create runtime chart' functionality as a decorator was + to help users who re-implements create() in theirs classes. + + :param func: class method + :return: + """ + def wrapper(*args, **kwargs): + self = args[0] + ok = func(*args, **kwargs) + if ok: + safe_print(RUNTIME_CHART_CREATE.format(job_name=self.name, + update_every=self._runtime_counters.FREQ)) + return ok + return wrapper + + +class ChartError(Exception): + """Base-class for all exceptions raised by this module""" + + +class DuplicateItemError(ChartError): + """Occurs when user re-adds a chart or a dimension that has already been added""" + + +class ItemTypeError(ChartError): + """Occurs when user passes value of wrong type to Chart, Dimension or ChartVariable class""" + + +class ItemValueError(ChartError): + """Occurs when user passes inappropriate value to Chart, Dimension or ChartVariable class""" + + +class Charts: + """Represent a collection of charts + + All charts stored in a dict. + Chart is a instance of Chart class. + Charts adding must be done using Charts.add_chart() method only""" + def __init__(self, job_name, priority, cleanup, get_update_every, module_name): + """ + :param job_name: + :param priority: + :param get_update_every: + """ + self.job_name = job_name + self.priority = priority + self.cleanup = cleanup + self.get_update_every = get_update_every + self.module_name = module_name + self.charts = dict() + + def __len__(self): + return len(self.charts) + + def __iter__(self): + return iter(self.charts.values()) + + def __repr__(self): + return 'Charts({0})'.format(self) + + def __str__(self): + return str([chart for chart in self.charts]) + + def __contains__(self, item): + return item in self.charts + + def __getitem__(self, item): + return self.charts[item] + + def __delitem__(self, key): + del self.charts[key] + + def __bool__(self): + return bool(self.charts) + + def __nonzero__(self): + return self.__bool__() + + def add_chart(self, params): + """ + Create Chart instance and add it to the dict + + Manually adds job name, priority and update_every to params. + :param params: + :return: + """ + params = [self.job_name()] + params + new_chart = Chart(params) + + new_chart.params['update_every'] = self.get_update_every() + new_chart.params['priority'] = self.priority + new_chart.params['module_name'] = self.module_name + + self.priority += 1 + self.charts[new_chart.id] = new_chart + + return new_chart + + def active_charts(self): + return [chart.id for chart in self if not chart.flags.obsoleted] + + +class Chart: + """Represent a chart""" + def __init__(self, params): + """ + :param params: + """ + if not isinstance(params, list): + raise ItemTypeError("'chart' must be a list type") + if not len(params) >= 8: + raise ItemValueError("invalid value for 'chart', must be {0}".format(CHART_PARAMS)) + + self.params = dict(zip(CHART_PARAMS, (p or str() for p in params))) + self.name = '{type}.{id}'.format(type=self.params['type'], + id=self.params['id']) + if self.params.get('chart_type') not in CHART_TYPES: + self.params['chart_type'] = 'absolute' + + self.dimensions = list() + self.variables = set() + self.flags = ChartFlags() + self.penalty = 0 + + def __getattr__(self, item): + try: + return self.params[item] + except KeyError: + raise AttributeError("'{instance}' has no attribute '{attr}'".format(instance=repr(self), + attr=item)) + + def __repr__(self): + return 'Chart({0})'.format(self.id) + + def __str__(self): + return self.id + + def __iter__(self): + return iter(self.dimensions) + + def __contains__(self, item): + return item in [dimension.id for dimension in self.dimensions] + + def add_variable(self, variable): + """ + :param variable: + :return: + """ + self.variables.add(ChartVariable(variable)) + + def add_dimension(self, dimension): + """ + :param dimension: + :return: + """ + dim = Dimension(dimension) + + if dim.id in self: + raise DuplicateItemError("'{dimension}' already in '{chart}' dimensions".format(dimension=dim.id, + chart=self.name)) + self.refresh() + self.dimensions.append(dim) + return dim + + def hide_dimension(self, dimension_id, reverse=False): + if dimension_id in self: + idx = self.dimensions.index(dimension_id) + dimension = self.dimensions[idx] + dimension.params['hidden'] = 'hidden' if not reverse else str() + self.refresh() + + def create(self): + """ + :return: + """ + chart = CHART_CREATE.format(**self.params) + dimensions = ''.join([dimension.create() for dimension in self.dimensions]) + variables = ''.join([var.set(var.value) for var in self.variables if var]) + + self.flags.push = False + self.flags.created = True + + safe_print(chart + dimensions + variables) + + def update(self, data, interval): + updated_dimensions, updated_variables = str(), str() + + for dim in self.dimensions: + value = dim.get_value(data) + if value is not None: + updated_dimensions += dim.set(value) + + for var in self.variables: + value = var.get_value(data) + if value is not None: + updated_variables += var.set(value) + + if updated_dimensions: + since_last = interval if self.flags.updated else 0 + + if self.flags.push: + self.create() + + chart_begin = CHART_BEGIN.format(type=self.type, id=self.id, since_last=since_last) + safe_print(chart_begin, updated_dimensions, updated_variables, 'END\n') + + self.flags.updated = True + self.penalty = 0 + else: + self.penalty += 1 + self.flags.updated = False + + return bool(updated_dimensions) + + def obsolete(self): + self.flags.obsoleted = True + if self.flags.created: + safe_print(CHART_OBSOLETE.format(**self.params)) + + def refresh(self): + self.penalty = 0 + self.flags.push = True + self.flags.obsoleted = False + + +class Dimension: + """Represent a dimension""" + def __init__(self, params): + """ + :param params: + """ + if not isinstance(params, list): + raise ItemTypeError("'dimension' must be a list type") + if not params: + raise ItemValueError("invalid value for 'dimension', must be {0}".format(DIMENSION_PARAMS)) + + self.params = dict(zip(DIMENSION_PARAMS, (p or str() for p in params))) + self.params['name'] = self.params.get('name') or self.params['id'] + + if self.params.get('algorithm') not in DIMENSION_ALGORITHMS: + self.params['algorithm'] = 'absolute' + if not isinstance(self.params.get('multiplier'), int): + self.params['multiplier'] = 1 + if not isinstance(self.params.get('divisor'), int): + self.params['divisor'] = 1 + self.params.setdefault('hidden', '') + + def __getattr__(self, item): + try: + return self.params[item] + except KeyError: + raise AttributeError("'{instance}' has no attribute '{attr}'".format(instance=repr(self), + attr=item)) + + def __repr__(self): + return 'Dimension({0})'.format(self.id) + + def __str__(self): + return self.id + + def __eq__(self, other): + if not isinstance(other, Dimension): + return self.id == other + return self.id == other.id + + def create(self): + return DIMENSION_CREATE.format(**self.params) + + def set(self, value): + """ + :param value: : must be a digit + :return: + """ + return DIMENSION_SET.format(id=self.id, + value=value) + + def get_value(self, data): + try: + return int(data[self.id]) + except (KeyError, TypeError): + return None + + +class ChartVariable: + """Represent a chart variable""" + def __init__(self, params): + """ + :param params: + """ + if not isinstance(params, list): + raise ItemTypeError("'variable' must be a list type") + if not params: + raise ItemValueError("invalid value for 'variable' must be: {0}".format(VARIABLE_PARAMS)) + + self.params = dict(zip(VARIABLE_PARAMS, params)) + self.params.setdefault('value', None) + + def __getattr__(self, item): + try: + return self.params[item] + except KeyError: + raise AttributeError("'{instance}' has no attribute '{attr}'".format(instance=repr(self), + attr=item)) + + def __bool__(self): + return self.value is not None + + def __nonzero__(self): + return self.__bool__() + + def __repr__(self): + return 'ChartVariable({0})'.format(self.id) + + def __str__(self): + return self.id + + def __eq__(self, other): + if isinstance(other, ChartVariable): + return self.id == other.id + return False + + def __hash__(self): + return hash(repr(self)) + + def set(self, value): + return CHART_VARIABLE_SET.format(id=self.id, + value=value) + + def get_value(self, data): + try: + return int(data[self.id]) + except (KeyError, TypeError): + return None + + +class ChartFlags: + def __init__(self): + self.push = True + self.created = False + self.updated = False + self.obsoleted = False diff --git a/python.d/python_modules/bases/collection.py b/python.d/python_modules/bases/collection.py new file mode 100644 index 000000000..e03b4f58e --- /dev/null +++ b/python.d/python_modules/bases/collection.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Ilya Mashchenko (l2isbad) + +import os + +PATH = os.getenv('PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin').split(':') + +CHART_BEGIN = 'BEGIN {0} {1}\n' +CHART_CREATE = "CHART {0} '{1}' '{2}' '{3}' '{4}' '{5}' {6} {7} {8}\n" +DIMENSION_CREATE = "DIMENSION '{0}' '{1}' {2} {3} {4} '{5}'\n" +DIMENSION_SET = "SET '{0}' = {1}\n" + + +def setdefault_values(config, base_dict): + for key, value in base_dict.items(): + config.setdefault(key, value) + return config + + +def run_and_exit(func): + def wrapper(*args, **kwargs): + func(*args, **kwargs) + exit(1) + return wrapper + + +def on_try_except_finally(on_except=(None, ), on_finally=(None, )): + except_func = on_except[0] + finally_func = on_finally[0] + + def decorator(func): + def wrapper(*args, **kwargs): + try: + func(*args, **kwargs) + except Exception: + if except_func: + except_func(*on_except[1:]) + finally: + if finally_func: + finally_func(*on_finally[1:]) + return wrapper + return decorator + + +def static_vars(**kwargs): + def decorate(func): + for k in kwargs: + setattr(func, k, kwargs[k]) + return func + return decorate + + +@on_try_except_finally(on_except=(exit, 1)) +def safe_print(*msg): + """ + :param msg: + :return: + """ + print(''.join(msg)) + + +def find_binary(binary): + """ + :param binary: + :return: + """ + for directory in PATH: + binary_name = '/'.join([directory, binary]) + if os.path.isfile(binary_name) and os.access(binary_name, os.X_OK): + return binary_name + return None + + +def read_last_line(f): + with open(f, 'rb') as opened: + opened.seek(-2, 2) + while opened.read(1) != b'\n': + opened.seek(-2, 1) + if opened.tell() == 0: + break + result = opened.readline() + return result.decode() + + +class OldVersionCompatibility: + + def __init__(self): + self._data_stream = str() + + def begin(self, type_id, microseconds=0): + """ + :param type_id: + :param microseconds: or : must be a digit + :return: + """ + self._data_stream += CHART_BEGIN.format(type_id, microseconds) + + def set(self, dim_id, value): + """ + :param dim_id: + :param value: or : must be a digit + :return: + """ + self._data_stream += DIMENSION_SET.format(dim_id, value) + + def end(self): + self._data_stream += 'END\n' + + def chart(self, type_id, name='', title='', units='', family='', category='', chart_type='line', + priority='', update_every=''): + """ + :param type_id: + :param name: + :param title: + :param units: + :param family: + :param category: + :param chart_type: + :param priority: or + :param update_every: or + :return: + """ + self._data_stream += CHART_CREATE.format(type_id, name, title, units, + family, category, chart_type, + priority, update_every) + + def dimension(self, dim_id, name=None, algorithm="absolute", multiplier=1, divisor=1, hidden=False): + """ + :param dim_id: + :param name: or None + :param algorithm: + :param multiplier: or : must be a digit + :param divisor: or : must be a digit + :param hidden: : literally "hidden" or "" + :return: + """ + self._data_stream += DIMENSION_CREATE.format(dim_id, name or dim_id, algorithm, + multiplier, divisor, hidden or str()) + + @on_try_except_finally(on_except=(exit, 1)) + def commit(self): + print(self._data_stream) + self._data_stream = str() diff --git a/python.d/python_modules/bases/loaders.py b/python.d/python_modules/bases/loaders.py new file mode 100644 index 000000000..d18b9dcd0 --- /dev/null +++ b/python.d/python_modules/bases/loaders.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Ilya Mashchenko (l2isbad) + +import types +from sys import version_info + +PY_VERSION = version_info[:2] + +if PY_VERSION > (3, 1): + from pyyaml3 import SafeLoader as YamlSafeLoader + from importlib.machinery import SourceFileLoader + DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' +else: + from pyyaml2 import SafeLoader as YamlSafeLoader + from imp import load_source as SourceFileLoader + DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map' + +try: + from collections import OrderedDict +except ImportError: + from third_party.ordereddict import OrderedDict + + +def dict_constructor(loader, node): + return OrderedDict(loader.construct_pairs(node)) + + +YamlSafeLoader.add_constructor(DEFAULT_MAPPING_TAG, dict_constructor) + + +class YamlOrderedLoader: + @staticmethod + def load_config_from_file(file_name): + opened, loaded = False, False + try: + stream = open(file_name, 'r') + opened = True + loader = YamlSafeLoader(stream) + loaded = True + parsed = loader.get_single_data() or dict() + except Exception as error: + return dict(), error + else: + return parsed, None + finally: + if opened: + stream.close() + if loaded: + loader.dispose() + + +class SourceLoader: + @staticmethod + def load_module_from_file(name, path): + try: + loaded = SourceFileLoader(name, path) + if isinstance(loaded, types.ModuleType): + return loaded, None + return loaded.load_module(), None + except Exception as error: + return None, error + + +class ModuleAndConfigLoader(YamlOrderedLoader, SourceLoader): + pass diff --git a/python.d/python_modules/bases/loggers.py b/python.d/python_modules/bases/loggers.py new file mode 100644 index 000000000..fc40b83d3 --- /dev/null +++ b/python.d/python_modules/bases/loggers.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- +# Description: +# Author: Ilya Mashchenko (l2isbad) + +import logging +import traceback + +from sys import exc_info + +try: + from time import monotonic as time +except ImportError: + from time import time + +from bases.collection import on_try_except_finally + + +LOGGING_LEVELS = {'CRITICAL': 50, + 'ERROR': 40, + 'WARNING': 30, + 'INFO': 20, + 'DEBUG': 10, + 'NOTSET': 0} + +DEFAULT_LOG_LINE_FORMAT = '%(asctime)s: %(name)s %(levelname)s : %(message)s' +DEFAULT_LOG_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' + +PYTHON_D_LOG_LINE_FORMAT = '%(asctime)s: %(name)s %(levelname)s: %(module_name)s: %(job_name)s: %(message)s' +PYTHON_D_LOG_NAME = 'python.d' + + +def limiter(log_max_count=30, allowed_in_seconds=60): + def on_decorator(func): + + def on_call(*args): + current_time = args[0]._runtime_counters.START_RUN + lc = args[0]._logger_counters + + if lc.logged and lc.logged % log_max_count == 0: + if current_time - lc.time_to_compare <= allowed_in_seconds: + lc.dropped += 1 + return + lc.time_to_compare = current_time + + lc.logged += 1 + func(*args) + + return on_call + return on_decorator + + +def add_traceback(func): + def on_call(*args): + self = args[0] + + if not self.log_traceback: + func(*args) + else: + if exc_info()[0]: + func(*args) + func(self, traceback.format_exc()) + else: + func(*args) + + return on_call + + +class LoggerCounters: + def __init__(self): + self.logged = 0 + self.dropped = 0 + self.time_to_compare = time() + + def __repr__(self): + return 'LoggerCounter(logged: {logged}, dropped: {dropped})'.format(logged=self.logged, + dropped=self.dropped) + + +class BaseLogger(object): + def __init__(self, logger_name, log_fmt=DEFAULT_LOG_LINE_FORMAT, date_fmt=DEFAULT_LOG_TIME_FORMAT, + handler=logging.StreamHandler): + """ + :param logger_name: + :param log_fmt: + :param date_fmt: + :param handler: + """ + self.logger = logging.getLogger(logger_name) + if not self.has_handlers(): + self.severity = 'INFO' + self.logger.addHandler(handler()) + self.set_formatter(fmt=log_fmt, date_fmt=date_fmt) + + def __repr__(self): + return ''.format(name=self.logger.name) + + def set_formatter(self, fmt, date_fmt=DEFAULT_LOG_TIME_FORMAT): + """ + :param fmt: + :param date_fmt: + :return: + """ + if self.has_handlers(): + self.logger.handlers[0].setFormatter(logging.Formatter(fmt=fmt, datefmt=date_fmt)) + + def has_handlers(self): + return self.logger.handlers + + @property + def severity(self): + return self.logger.getEffectiveLevel() + + @severity.setter + def severity(self, level): + """ + :param level: or + :return: + """ + if level in LOGGING_LEVELS: + self.logger.setLevel(LOGGING_LEVELS[level]) + + def debug(self, *msg, **kwargs): + self.logger.debug(' '.join(map(str, msg)), **kwargs) + + def info(self, *msg, **kwargs): + self.logger.info(' '.join(map(str, msg)), **kwargs) + + def warning(self, *msg, **kwargs): + self.logger.warning(' '.join(map(str, msg)), **kwargs) + + def error(self, *msg, **kwargs): + self.logger.error(' '.join(map(str, msg)), **kwargs) + + def alert(self, *msg, **kwargs): + self.logger.critical(' '.join(map(str, msg)), **kwargs) + + @on_try_except_finally(on_finally=(exit, 1)) + def fatal(self, *msg, **kwargs): + self.logger.critical(' '.join(map(str, msg)), **kwargs) + + +class PythonDLogger(object): + def __init__(self, logger_name=PYTHON_D_LOG_NAME, log_fmt=PYTHON_D_LOG_LINE_FORMAT): + """ + :param logger_name: + :param log_fmt: + """ + self.logger = BaseLogger(logger_name, log_fmt=log_fmt) + self.module_name = 'plugin' + self.job_name = 'main' + self._logger_counters = LoggerCounters() + + _LOG_TRACEBACK = False + + @property + def log_traceback(self): + return PythonDLogger._LOG_TRACEBACK + + @log_traceback.setter + def log_traceback(self, value): + PythonDLogger._LOG_TRACEBACK = value + + def debug(self, *msg): + self.logger.debug(*msg, extra={'module_name': self.module_name, + 'job_name': self.job_name or self.module_name}) + + def info(self, *msg): + self.logger.info(*msg, extra={'module_name': self.module_name, + 'job_name': self.job_name or self.module_name}) + + def warning(self, *msg): + self.logger.warning(*msg, extra={'module_name': self.module_name, + 'job_name': self.job_name or self.module_name}) + + @add_traceback + def error(self, *msg): + self.logger.error(*msg, extra={'module_name': self.module_name, + 'job_name': self.job_name or self.module_name}) + + @add_traceback + def alert(self, *msg): + self.logger.alert(*msg, extra={'module_name': self.module_name, + 'job_name': self.job_name or self.module_name}) + + def fatal(self, *msg): + self.logger.fatal(*msg, extra={'module_name': self.module_name, + 'job_name': self.job_name or self.module_name}) + + +class PythonDLimitedLogger(PythonDLogger): + @limiter() + def info(self, *msg): + PythonDLogger.info(self, *msg) + + @limiter() + def warning(self, *msg): + PythonDLogger.warning(self, *msg) + + @limiter() + def error(self, *msg): + PythonDLogger.error(self, *msg) + + @limiter() + def alert(self, *msg): + PythonDLogger.alert(self, *msg) diff --git a/python.d/python_modules/lm_sensors.py b/python.d/python_modules/lm_sensors.py deleted file mode 100644 index 1d868f0e2..000000000 --- a/python.d/python_modules/lm_sensors.py +++ /dev/null @@ -1,257 +0,0 @@ -""" -@package sensors.py -Python Bindings for libsensors3 - -use the documentation of libsensors for the low level API. -see example.py for high level API usage. - -@author: Pavel Rojtberg (http://www.rojtberg.net) -@see: https://github.com/paroj/sensors.py -@copyright: LGPLv2 (same as libsensors) -""" - -from ctypes import * -import ctypes.util - -_libc = cdll.LoadLibrary(ctypes.util.find_library("c")) -# see https://github.com/paroj/sensors.py/issues/1 -_libc.free.argtypes = [c_void_p] -_hdl = cdll.LoadLibrary(ctypes.util.find_library("sensors")) - -version = c_char_p.in_dll(_hdl, "libsensors_version").value.decode("ascii") - - -class bus_id(Structure): - _fields_ = [("type", c_short), - ("nr", c_short)] - - -class chip_name(Structure): - _fields_ = [("prefix", c_char_p), - ("bus", bus_id), - ("addr", c_int), - ("path", c_char_p)] - - -class feature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int)] - - # sensors_feature_type - IN = 0x00 - FAN = 0x01 - TEMP = 0x02 - POWER = 0x03 - ENERGY = 0x04 - CURR = 0x05 - HUMIDITY = 0x06 - MAX_MAIN = 0x7 - VID = 0x10 - INTRUSION = 0x11 - MAX_OTHER = 0x12 - BEEP_ENABLE = 0x18 - - -class subfeature(Structure): - _fields_ = [("name", c_char_p), - ("number", c_int), - ("type", c_int), - ("mapping", c_int), - ("flags", c_uint)] - - -_hdl.sensors_get_detected_chips.restype = POINTER(chip_name) -_hdl.sensors_get_features.restype = POINTER(feature) -_hdl.sensors_get_all_subfeatures.restype = POINTER(subfeature) -_hdl.sensors_get_label.restype = c_void_p # return pointer instead of str so we can free it -_hdl.sensors_get_adapter_name.restype = c_char_p # docs do not say whether to free this or not -_hdl.sensors_strerror.restype = c_char_p - -### RAW API ### -MODE_R = 1 -MODE_W = 2 -COMPUTE_MAPPING = 4 - - -def init(cfg_file=None): - file = _libc.fopen(cfg_file.encode("utf-8"), "r") if cfg_file is not None else None - - if _hdl.sensors_init(file) != 0: - raise Exception("sensors_init failed") - - if file is not None: - _libc.fclose(file) - - -def cleanup(): - _hdl.sensors_cleanup() - - -def parse_chip_name(orig_name): - ret = chip_name() - err = _hdl.sensors_parse_chip_name(orig_name.encode("utf-8"), byref(ret)) - - if err < 0: - raise Exception(strerror(err)) - - return ret - - -def strerror(errnum): - return _hdl.sensors_strerror(errnum).decode("utf-8") - - -def free_chip_name(chip): - _hdl.sensors_free_chip_name(byref(chip)) - - -def get_detected_chips(match, nr): - """ - @return: (chip, next nr to query) - """ - _nr = c_int(nr) - - if match is not None: - match = byref(match) - - chip = _hdl.sensors_get_detected_chips(match, byref(_nr)) - chip = chip.contents if bool(chip) else None - return chip, _nr.value - - -def chip_snprintf_name(chip, buffer_size=200): - """ - @param buffer_size defaults to the size used in the sensors utility - """ - ret = create_string_buffer(buffer_size) - err = _hdl.sensors_snprintf_chip_name(ret, buffer_size, byref(chip)) - - if err < 0: - raise Exception(strerror(err)) - - return ret.value.decode("utf-8") - - -def do_chip_sets(chip): - """ - @attention this function was not tested - """ - err = _hdl.sensors_do_chip_sets(byref(chip)) - if err < 0: - raise Exception(strerror(err)) - - -def get_adapter_name(bus): - return _hdl.sensors_get_adapter_name(byref(bus)).decode("utf-8") - - -def get_features(chip, nr): - """ - @return: (feature, next nr to query) - """ - _nr = c_int(nr) - feature = _hdl.sensors_get_features(byref(chip), byref(_nr)) - feature = feature.contents if bool(feature) else None - return feature, _nr.value - - -def get_label(chip, feature): - ptr = _hdl.sensors_get_label(byref(chip), byref(feature)) - val = cast(ptr, c_char_p).value.decode("utf-8") - _libc.free(ptr) - return val - - -def get_all_subfeatures(chip, feature, nr): - """ - @return: (subfeature, next nr to query) - """ - _nr = c_int(nr) - subfeature = _hdl.sensors_get_all_subfeatures(byref(chip), byref(feature), byref(_nr)) - subfeature = subfeature.contents if bool(subfeature) else None - return subfeature, _nr.value - - -def get_value(chip, subfeature_nr): - val = c_double() - err = _hdl.sensors_get_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - return val.value - - -def set_value(chip, subfeature_nr, value): - """ - @attention this function was not tested - """ - val = c_double(value) - err = _hdl.sensors_set_value(byref(chip), subfeature_nr, byref(val)) - if err < 0: - raise Exception(strerror(err)) - - -### Convenience API ### -class ChipIterator: - def __init__(self, match=None): - self.match = parse_chip_name(match) if match is not None else None - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - chip, self.nr = get_detected_chips(self.match, self.nr) - - if chip is None: - raise StopIteration - - return chip - - def __del__(self): - if self.match is not None: - free_chip_name(self.match) - - def next(self): # python2 compability - return self.__next__() - - -class FeatureIterator: - def __init__(self, chip): - self.chip = chip - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - feature, self.nr = get_features(self.chip, self.nr) - - if feature is None: - raise StopIteration - - return feature - - def next(self): # python2 compability - return self.__next__() - - -class SubFeatureIterator: - def __init__(self, chip, feature): - self.chip = chip - self.feature = feature - self.nr = 0 - - def __iter__(self): - return self - - def __next__(self): - subfeature, self.nr = get_all_subfeatures(self.chip, self.feature, self.nr) - - if subfeature is None: - raise StopIteration - - return subfeature - - def next(self): # python2 compability - return self.__next__() \ No newline at end of file diff --git a/python.d/python_modules/msg.py b/python.d/python_modules/msg.py deleted file mode 100644 index 74716770c..000000000 --- a/python.d/python_modules/msg.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -# Description: logging for netdata python.d modules - -import traceback -import sys -from time import time, strftime - -DEBUG_FLAG = False -TRACE_FLAG = False -PROGRAM = "" -LOG_COUNTER = 0 -LOG_THROTTLE = 10000 # has to be too big during init -LOG_INTERVAL = 1 # has to be too low during init -LOG_NEXT_CHECK = 0 - -WRITE = sys.stderr.write -FLUSH = sys.stderr.flush - - -def log_msg(msg_type, *args): - """ - Print message on stderr. - :param msg_type: str - """ - global LOG_COUNTER - global LOG_THROTTLE - global LOG_INTERVAL - global LOG_NEXT_CHECK - now = time() - - if not DEBUG_FLAG: - LOG_COUNTER += 1 - - # WRITE("COUNTER " + str(LOG_COUNTER) + " THROTTLE " + str(LOG_THROTTLE) + " INTERVAL " + str(LOG_INTERVAL) + " NOW " + str(now) + " NEXT " + str(LOG_NEXT_CHECK) + "\n") - - if LOG_COUNTER <= LOG_THROTTLE or msg_type == "FATAL" or msg_type == "ALERT": - timestamp = strftime('%Y-%m-%d %X') - msg = "%s: %s %s: %s" % (timestamp, PROGRAM, str(msg_type), " ".join(args)) - WRITE(msg + "\n") - FLUSH() - elif LOG_COUNTER == LOG_THROTTLE + 1: - timestamp = strftime('%Y-%m-%d %X') - msg = "%s: python.d.plugin: throttling further log messages for %s seconds" % (timestamp, str(int(LOG_NEXT_CHECK + 0.5) - int(now))) - WRITE(msg + "\n") - FLUSH() - - if LOG_NEXT_CHECK <= now: - if LOG_COUNTER >= LOG_THROTTLE: - timestamp = strftime('%Y-%m-%d %X') - msg = "%s: python.d.plugin: Prevented %s log messages from displaying" % (timestamp, str(LOG_COUNTER - LOG_THROTTLE)) - WRITE(msg + "\n") - FLUSH() - LOG_NEXT_CHECK = now - (now % LOG_INTERVAL) + LOG_INTERVAL - LOG_COUNTER = 0 - - if TRACE_FLAG: - if msg_type == "FATAL" or msg_type == "ERROR" or msg_type == "ALERT": - traceback.print_exc() - - -def debug(*args): - """ - Print debug message on stderr. - """ - if not DEBUG_FLAG: - return - - log_msg("DEBUG", *args) - - -def error(*args): - """ - Print message on stderr. - """ - log_msg("ERROR", *args) - - -def alert(*args): - """ - Print message on stderr. - """ - log_msg("ALERT", *args) - - -def info(*args): - """ - Print message on stderr. - """ - log_msg("INFO", *args) - - -def fatal(*args): - """ - Print message on stderr and exit. - """ - try: - log_msg("FATAL", *args) - print('DISABLE') - except: - pass - sys.exit(1) diff --git a/python.d/python_modules/third_party/__init__.py b/python.d/python_modules/third_party/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python.d/python_modules/third_party/lm_sensors.py b/python.d/python_modules/third_party/lm_sensors.py new file mode 100644 index 000000000..1d868f0e2 --- /dev/null +++ b/python.d/python_modules/third_party/lm_sensors.py @@ -0,0 +1,257 @@ +""" +@package sensors.py +Python Bindings for libsensors3 + +use the documentation of libsensors for the low level API. +see example.py for high level API usage. + +@author: Pavel Rojtberg (http://www.rojtberg.net) +@see: https://github.com/paroj/sensors.py +@copyright: LGPLv2 (same as libsensors) +""" + +from ctypes import * +import ctypes.util + +_libc = cdll.LoadLibrary(ctypes.util.find_library("c")) +# see https://github.com/paroj/sensors.py/issues/1 +_libc.free.argtypes = [c_void_p] +_hdl = cdll.LoadLibrary(ctypes.util.find_library("sensors")) + +version = c_char_p.in_dll(_hdl, "libsensors_version").value.decode("ascii") + + +class bus_id(Structure): + _fields_ = [("type", c_short), + ("nr", c_short)] + + +class chip_name(Structure): + _fields_ = [("prefix", c_char_p), + ("bus", bus_id), + ("addr", c_int), + ("path", c_char_p)] + + +class feature(Structure): + _fields_ = [("name", c_char_p), + ("number", c_int), + ("type", c_int)] + + # sensors_feature_type + IN = 0x00 + FAN = 0x01 + TEMP = 0x02 + POWER = 0x03 + ENERGY = 0x04 + CURR = 0x05 + HUMIDITY = 0x06 + MAX_MAIN = 0x7 + VID = 0x10 + INTRUSION = 0x11 + MAX_OTHER = 0x12 + BEEP_ENABLE = 0x18 + + +class subfeature(Structure): + _fields_ = [("name", c_char_p), + ("number", c_int), + ("type", c_int), + ("mapping", c_int), + ("flags", c_uint)] + + +_hdl.sensors_get_detected_chips.restype = POINTER(chip_name) +_hdl.sensors_get_features.restype = POINTER(feature) +_hdl.sensors_get_all_subfeatures.restype = POINTER(subfeature) +_hdl.sensors_get_label.restype = c_void_p # return pointer instead of str so we can free it +_hdl.sensors_get_adapter_name.restype = c_char_p # docs do not say whether to free this or not +_hdl.sensors_strerror.restype = c_char_p + +### RAW API ### +MODE_R = 1 +MODE_W = 2 +COMPUTE_MAPPING = 4 + + +def init(cfg_file=None): + file = _libc.fopen(cfg_file.encode("utf-8"), "r") if cfg_file is not None else None + + if _hdl.sensors_init(file) != 0: + raise Exception("sensors_init failed") + + if file is not None: + _libc.fclose(file) + + +def cleanup(): + _hdl.sensors_cleanup() + + +def parse_chip_name(orig_name): + ret = chip_name() + err = _hdl.sensors_parse_chip_name(orig_name.encode("utf-8"), byref(ret)) + + if err < 0: + raise Exception(strerror(err)) + + return ret + + +def strerror(errnum): + return _hdl.sensors_strerror(errnum).decode("utf-8") + + +def free_chip_name(chip): + _hdl.sensors_free_chip_name(byref(chip)) + + +def get_detected_chips(match, nr): + """ + @return: (chip, next nr to query) + """ + _nr = c_int(nr) + + if match is not None: + match = byref(match) + + chip = _hdl.sensors_get_detected_chips(match, byref(_nr)) + chip = chip.contents if bool(chip) else None + return chip, _nr.value + + +def chip_snprintf_name(chip, buffer_size=200): + """ + @param buffer_size defaults to the size used in the sensors utility + """ + ret = create_string_buffer(buffer_size) + err = _hdl.sensors_snprintf_chip_name(ret, buffer_size, byref(chip)) + + if err < 0: + raise Exception(strerror(err)) + + return ret.value.decode("utf-8") + + +def do_chip_sets(chip): + """ + @attention this function was not tested + """ + err = _hdl.sensors_do_chip_sets(byref(chip)) + if err < 0: + raise Exception(strerror(err)) + + +def get_adapter_name(bus): + return _hdl.sensors_get_adapter_name(byref(bus)).decode("utf-8") + + +def get_features(chip, nr): + """ + @return: (feature, next nr to query) + """ + _nr = c_int(nr) + feature = _hdl.sensors_get_features(byref(chip), byref(_nr)) + feature = feature.contents if bool(feature) else None + return feature, _nr.value + + +def get_label(chip, feature): + ptr = _hdl.sensors_get_label(byref(chip), byref(feature)) + val = cast(ptr, c_char_p).value.decode("utf-8") + _libc.free(ptr) + return val + + +def get_all_subfeatures(chip, feature, nr): + """ + @return: (subfeature, next nr to query) + """ + _nr = c_int(nr) + subfeature = _hdl.sensors_get_all_subfeatures(byref(chip), byref(feature), byref(_nr)) + subfeature = subfeature.contents if bool(subfeature) else None + return subfeature, _nr.value + + +def get_value(chip, subfeature_nr): + val = c_double() + err = _hdl.sensors_get_value(byref(chip), subfeature_nr, byref(val)) + if err < 0: + raise Exception(strerror(err)) + return val.value + + +def set_value(chip, subfeature_nr, value): + """ + @attention this function was not tested + """ + val = c_double(value) + err = _hdl.sensors_set_value(byref(chip), subfeature_nr, byref(val)) + if err < 0: + raise Exception(strerror(err)) + + +### Convenience API ### +class ChipIterator: + def __init__(self, match=None): + self.match = parse_chip_name(match) if match is not None else None + self.nr = 0 + + def __iter__(self): + return self + + def __next__(self): + chip, self.nr = get_detected_chips(self.match, self.nr) + + if chip is None: + raise StopIteration + + return chip + + def __del__(self): + if self.match is not None: + free_chip_name(self.match) + + def next(self): # python2 compability + return self.__next__() + + +class FeatureIterator: + def __init__(self, chip): + self.chip = chip + self.nr = 0 + + def __iter__(self): + return self + + def __next__(self): + feature, self.nr = get_features(self.chip, self.nr) + + if feature is None: + raise StopIteration + + return feature + + def next(self): # python2 compability + return self.__next__() + + +class SubFeatureIterator: + def __init__(self, chip, feature): + self.chip = chip + self.feature = feature + self.nr = 0 + + def __iter__(self): + return self + + def __next__(self): + subfeature, self.nr = get_all_subfeatures(self.chip, self.feature, self.nr) + + if subfeature is None: + raise StopIteration + + return subfeature + + def next(self): # python2 compability + return self.__next__() \ No newline at end of file diff --git a/python.d/python_modules/third_party/ordereddict.py b/python.d/python_modules/third_party/ordereddict.py new file mode 100644 index 000000000..d0b97d47c --- /dev/null +++ b/python.d/python_modules/third_party/ordereddict.py @@ -0,0 +1,128 @@ +# Copyright (c) 2009 Raymond Hettinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from UserDict import DictMixin + + +class OrderedDict(dict, DictMixin): + + def __init__(self, *args, **kwds): + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__end + except AttributeError: + self.clear() + self.update(*args, **kwds) + + def clear(self): + self.__end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.__map = {} # key --> [key, prev, next] + dict.clear(self) + + def __setitem__(self, key, value): + if key not in self: + end = self.__end + curr = end[1] + curr[2] = end[1] = self.__map[key] = [key, curr, end] + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + dict.__delitem__(self, key) + key, prev, next = self.__map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.__end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.__end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def popitem(self, last=True): + if not self: + raise KeyError('dictionary is empty') + if last: + key = reversed(self).next() + else: + key = iter(self).next() + value = self.pop(key) + return key, value + + def __reduce__(self): + items = [[k, self[k]] for k in self] + tmp = self.__map, self.__end + del self.__map, self.__end + inst_dict = vars(self).copy() + self.__map, self.__end = tmp + if inst_dict: + return self.__class__, (items,), inst_dict + return self.__class__, (items,) + + def keys(self): + return list(self) + + setdefault = DictMixin.setdefault + update = DictMixin.update + pop = DictMixin.pop + values = DictMixin.values + items = DictMixin.items + iterkeys = DictMixin.iterkeys + itervalues = DictMixin.itervalues + iteritems = DictMixin.iteritems + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + + def copy(self): + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + if isinstance(other, OrderedDict): + if len(self) != len(other): + return False + for p, q in zip(self.items(), other.items()): + if p != q: + return False + return True + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other diff --git a/python.d/rabbitmq.chart.py b/python.d/rabbitmq.chart.py index 763912039..eef472bf2 100644 --- a/python.d/rabbitmq.chart.py +++ b/python.d/rabbitmq.chart.py @@ -11,7 +11,7 @@ try: except ImportError: from Queue import Queue -from base import UrlService +from bases.FrameworkServices.UrlService import UrlService # default module values (can be overridden per job in `config`) update_every = 1 diff --git a/python.d/redis.chart.py b/python.d/redis.chart.py index 7c3c43f5a..bcfcf16a6 100644 --- a/python.d/redis.chart.py +++ b/python.d/redis.chart.py @@ -2,10 +2,9 @@ # Description: redis netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import SocketService +from bases.FrameworkServices.SocketService import SocketService # default module values (can be overridden per job in `config`) -#update_every = 2 priority = 60000 retries = 60 @@ -19,7 +18,8 @@ retries = 60 # 'unix_socket': None # }} -ORDER = ['operations', 'hit_rate', 'memory', 'keys', 'net', 'connections', 'clients', 'slaves', 'persistence'] +ORDER = ['operations', 'hit_rate', 'memory', 'keys', 'net', 'connections', 'clients', 'slaves', 'persistence', + 'bgsave_now', 'bgsave_health'] CHARTS = { 'operations': { @@ -72,6 +72,18 @@ CHARTS = { 'redis.rdb_changes', 'line'], 'lines': [ ['rdb_changes_since_last_save', 'changes', 'absolute'] + ]}, + 'bgsave_now': { + 'options': [None, 'Duration of the RDB Save Operation', 'seconds', 'persistence', + 'redis.bgsave_now', 'absolute'], + 'lines': [ + ['rdb_bgsave_in_progress', 'rdb save', 'absolute'] + ]}, + 'bgsave_health': { + 'options': [None, 'Status of the Last RDB Save Operation', 'status', 'persistence', + 'redis.bgsave_health', 'line'], + 'lines': [ + ['rdb_last_bgsave_status', 'rdb save', 'absolute'] ]} } @@ -87,6 +99,7 @@ class Service(SocketService): self.port = self.configuration.get('port', 6379) self.unix_socket = self.configuration.get('socket') password = self.configuration.get('pass', str()) + self.bgsave_time = 0 self.requests = dict(request='INFO\r\n'.encode(), password=' '.join(['AUTH', password, '\r\n']).encode() if password else None) self.request = self.requests['request'] @@ -130,9 +143,8 @@ class Service(SocketService): data[t[0]] = t[1] except (IndexError, ValueError): self.debug("invalid line received: " + str(line)) - pass - if len(data) == 0: + if not data: self.error("received data doesn't have any records") return None @@ -142,6 +154,14 @@ class Service(SocketService): except (KeyError, ZeroDivisionError, TypeError): data['hit_rate'] = 0 + if data['rdb_bgsave_in_progress'] != '0\r': + self.bgsave_time += self.update_every + else: + self.bgsave_time = 0 + + data['rdb_last_bgsave_status'] = 0 if data['rdb_last_bgsave_status'] == 'ok\r' else 1 + data['rdb_bgsave_in_progress'] = self.bgsave_time + return data def _check_raw_data(self, data): @@ -170,9 +190,6 @@ class Service(SocketService): Parse configuration, check if redis is available, and dynamically create chart lines data :return: boolean """ - if self.name == "": - self.name = "local" - self.chart_name += "_" + self.name data = self._get_data() if data is None: return False diff --git a/python.d/retroshare.chart.py b/python.d/retroshare.chart.py index 0c97973f6..8c0330ec6 100644 --- a/python.d/retroshare.chart.py +++ b/python.d/retroshare.chart.py @@ -2,9 +2,10 @@ # Description: RetroShare netdata python.d module # Authors: sehraf -from base import UrlService import json +from bases.FrameworkServices.UrlService import UrlService + # default module values (can be overridden per job in `config`) # update_every = 2 priority = 60000 @@ -38,10 +39,7 @@ CHARTS = { class Service(UrlService): def __init__(self, configuration=None, name=None): UrlService.__init__(self, configuration=configuration, name=name) - try: - self.baseurl = str(self.configuration['url']) - except (KeyError, TypeError): - self.baseurl = 'http://localhost:9090' + self.baseurl = self.configuration.get('url', 'http://localhost:9090') self.order = ORDER self.definitions = CHARTS @@ -55,7 +53,7 @@ class Service(UrlService): parsed = json.loads(raw) if str(parsed['returncode']) != 'ok': return None - except: + except (TypeError, ValueError): return None return parsed['data'][0] diff --git a/python.d/samba.chart.py b/python.d/samba.chart.py index 767c97469..3f4fd5a12 100644 --- a/python.d/samba.chart.py +++ b/python.d/samba.chart.py @@ -15,9 +15,11 @@ # a lot there, bring one or more out into its own chart and make it incremental # (like find and notify... good examples). -from base import ExecutableService import re +from bases.collection import find_binary +from bases.FrameworkServices.ExecutableService import ExecutableService + # default module values (can be overridden per job in `config`) update_every = 5 priority = 60000 @@ -94,10 +96,10 @@ class Service(ExecutableService): self.rgx_smb2 = re.compile(r'(smb2_[^:]+|syscall_.*file_bytes):\s+(\d+)') def check(self): - sudo_binary, smbstatus_binary = self.find_binary('sudo'), self.find_binary('smbstatus') + sudo_binary, smbstatus_binary = find_binary('sudo'), find_binary('smbstatus') if not (sudo_binary and smbstatus_binary): - self.error('Can\'t locate \'sudo\' or \'smbstatus\' binary') + self.error("Can\'t locate 'sudo' or 'smbstatus' binary") return False self.command = [sudo_binary, '-v'] diff --git a/python.d/sensors.chart.py b/python.d/sensors.chart.py index e83aacfd8..06e420b68 100644 --- a/python.d/sensors.chart.py +++ b/python.d/sensors.chart.py @@ -2,8 +2,8 @@ # Description: sensors netdata python.d plugin # Author: Pawel Krupa (paulfantom) -from base import SimpleService -import lm_sensors as sensors +from bases.FrameworkServices.SimpleService import SimpleService +from third_party import lm_sensors as sensors # default module values (can be overridden per job in `config`) # update_every = 2 @@ -75,15 +75,12 @@ TYPE_MAP = { class Service(SimpleService): def __init__(self, configuration=None, name=None): SimpleService.__init__(self, configuration=configuration, name=name) - self.order = [] - self.definitions = {} - self.celsius = ('Celsius', lambda x: x) - self.fahrenheit = ('Fahrenheit', lambda x: x * 9 / 5 + 32) if self.configuration.get('fahrenheit') else False - self.choice = (choice for choice in [self.fahrenheit, self.celsius] if choice) - self.chips = [] + self.order = list() + self.definitions = dict() + self.chips = list() - def _get_data(self): - data = {} + def get_data(self): + data = dict() try: for chip in sensors.ChipIterator(): prefix = sensors.chip_snprintf_name(chip) @@ -92,46 +89,39 @@ class Service(SimpleService): for sf in sfi: val = sensors.get_value(chip, sf.number) break - typeName = TYPE_MAP[feature.type] - if typeName in LIMITS: - limit = LIMITS[typeName]; + type_name = TYPE_MAP[feature.type] + if type_name in LIMITS: + limit = LIMITS[type_name] if val < limit[0] or val > limit[1]: continue - if 'temp' in str(feature.name.decode()): - data[prefix + "_" + str(feature.name.decode())] = int(self.calc(val) * 1000) - else: data[prefix + "_" + str(feature.name.decode())] = int(val * 1000) - except Exception as e: - self.error(e) + except Exception as error: + self.error(error) return None - if len(data) == 0: - return None - return data + return data or None - def _create_definitions(self): - for type in ORDER: + def create_definitions(self): + for sensor in ORDER: for chip in sensors.ChipIterator(): chip_name = sensors.chip_snprintf_name(chip) - if len(self.chips) != 0 and not any([chip_name.startswith(ex) for ex in self.chips]): + if self.chips and not any([chip_name.startswith(ex) for ex in self.chips]): continue for feature in sensors.FeatureIterator(chip): sfi = sensors.SubFeatureIterator(chip, feature) vals = [sensors.get_value(chip, sf.number) for sf in sfi] if vals[0] == 0: continue - if TYPE_MAP[feature.type] == type: + if TYPE_MAP[feature.type] == sensor: # create chart name = chip_name + "_" + TYPE_MAP[feature.type] if name not in self.order: self.order.append(name) - chart_def = list(CHARTS[type]['options']) + chart_def = list(CHARTS[sensor]['options']) chart_def[1] = chip_name + chart_def[1] - if chart_def[2] == 'Celsius': - chart_def[2] = self.choice[0] self.definitions[name] = {'options': chart_def} self.definitions[name]['lines'] = [] - line = list(CHARTS[type]['lines'][0]) + line = list(CHARTS[sensor]['lines'][0]) line[0] = chip_name + "_" + str(feature.name.decode()) line[1] = sensors.get_label(chip, feature) self.definitions[name]['lines'].append(line) @@ -139,23 +129,11 @@ class Service(SimpleService): def check(self): try: sensors.init() - except Exception as e: - self.error(e) + except Exception as error: + self.error(error) return False - - try: - self.choice = next(self.choice) - except StopIteration: - # That can not happen but.. - self.choice = ('Celsius', lambda x: x) - self.calc = self.choice[1] - else: - self.calc = self.choice[1] - try: - self._create_definitions() - except Exception as e: - self.error(e) - return False + self.create_definitions() return True + diff --git a/python.d/smartd_log.chart.py b/python.d/smartd_log.chart.py index 4039c1536..07ad88cd4 100644 --- a/python.d/smartd_log.chart.py +++ b/python.d/smartd_log.chart.py @@ -2,221 +2,345 @@ # Description: smart netdata python.d module # Author: l2isbad, vorph1 -from re import compile as r_compile -from os import listdir, access, R_OK -from os.path import isfile, join, getsize, basename, isdir -try: - from queue import Queue -except ImportError: - from Queue import Queue -from threading import Thread -from base import SimpleService +import os +import re + from collections import namedtuple +from time import time -# default module values (can be overridden per job in `config`) -update_every = 5 -priority = 60000 +from bases.collection import read_last_line +from bases.FrameworkServices.SimpleService import SimpleService # charts order (can be overridden if you want less charts, or different order) ORDER = ['1', '4', '5', '7', '9', '12', '193', '194', '197', '198', '200'] SMART_ATTR = { - '1': 'Read Error Rate', - '2': 'Throughput Performance', - '3': 'Spin-Up Time', - '4': 'Start/Stop Count', - '5': 'Reallocated Sectors Count', - '6': 'Read Channel Margin', - '7': 'Seek Error Rate', - '8': 'Seek Time Performance', - '9': 'Power-On Hours Count', - '10': 'Spin-up Retries', - '11': 'Calibration Retries', - '12': 'Power Cycle Count', - '13': 'Soft Read Error Rate', - '100': 'Erase/Program Cycles', - '103': 'Translation Table Rebuild', - '108': 'Unknown (108)', - '170': 'Reserved Block Count', - '171': 'Program Fail Count', - '172': 'Erase Fail Count', - '173': 'Wear Leveller Worst Case Erase Count', - '174': 'Unexpected Power Loss', - '175': 'Program Fail Count', - '176': 'Erase Fail Count', - '177': 'Wear Leveling Count', - '178': 'Used Reserved Block Count', - '179': 'Used Reserved Block Count', - '180': 'Unused Reserved Block Count', - '181': 'Program Fail Count', - '182': 'Erase Fail Count', - '183': 'SATA Downshifts', - '184': 'End-to-End error', - '185': 'Head Stability', - '186': 'Induced Op-Vibration Detection', - '187': 'Reported Uncorrectable Errors', - '188': 'Command Timeout', - '189': 'High Fly Writes', - '190': 'Temperature', - '191': 'G-Sense Errors', - '192': 'Power-Off Retract Cycles', - '193': 'Load/Unload Cycles', - '194': 'Temperature', - '195': 'Hardware ECC Recovered', - '196': 'Reallocation Events', - '197': 'Current Pending Sectors', - '198': 'Off-line Uncorrectable', - '199': 'UDMA CRC Error Rate', - '200': 'Write Error Rate', - '201': 'Soft Read Errors', - '202': 'Data Address Mark Errors', - '203': 'Run Out Cancel', - '204': 'Soft ECC Corrections', - '205': 'Thermal Asperity Rate', - '206': 'Flying Height', - '207': 'Spin High Current', - '209': 'Offline Seek Performance', - '220': 'Disk Shift', - '221': 'G-Sense Error Rate', - '222': 'Loaded Hours', - '223': 'Load/Unload Retries', - '224': 'Load Friction', - '225': 'Load/Unload Cycles', - '226': 'Load-in Time', - '227': 'Torque Amplification Count', - '228': 'Power-Off Retracts', - '230': 'GMR Head Amplitude', - '231': 'Temperature', - '232': 'Available Reserved Space', - '233': 'Media Wearout Indicator', - '240': 'Head Flying Hours', - '241': 'Total LBAs Written', - '242': 'Total LBAs Read', - '250': 'Read Error Retry Rate' + '1': 'Read Error Rate', + '2': 'Throughput Performance', + '3': 'Spin-Up Time', + '4': 'Start/Stop Count', + '5': 'Reallocated Sectors Count', + '6': 'Read Channel Margin', + '7': 'Seek Error Rate', + '8': 'Seek Time Performance', + '9': 'Power-On Hours Count', + '10': 'Spin-up Retries', + '11': 'Calibration Retries', + '12': 'Power Cycle Count', + '13': 'Soft Read Error Rate', + '100': 'Erase/Program Cycles', + '103': 'Translation Table Rebuild', + '108': 'Unknown (108)', + '170': 'Reserved Block Count', + '171': 'Program Fail Count', + '172': 'Erase Fail Count', + '173': 'Wear Leveller Worst Case Erase Count', + '174': 'Unexpected Power Loss', + '175': 'Program Fail Count', + '176': 'Erase Fail Count', + '177': 'Wear Leveling Count', + '178': 'Used Reserved Block Count', + '179': 'Used Reserved Block Count', + '180': 'Unused Reserved Block Count', + '181': 'Program Fail Count', + '182': 'Erase Fail Count', + '183': 'SATA Downshifts', + '184': 'End-to-End error', + '185': 'Head Stability', + '186': 'Induced Op-Vibration Detection', + '187': 'Reported Uncorrectable Errors', + '188': 'Command Timeout', + '189': 'High Fly Writes', + '190': 'Temperature', + '191': 'G-Sense Errors', + '192': 'Power-Off Retract Cycles', + '193': 'Load/Unload Cycles', + '194': 'Temperature', + '195': 'Hardware ECC Recovered', + '196': 'Reallocation Events', + '197': 'Current Pending Sectors', + '198': 'Off-line Uncorrectable', + '199': 'UDMA CRC Error Rate', + '200': 'Write Error Rate', + '201': 'Soft Read Errors', + '202': 'Data Address Mark Errors', + '203': 'Run Out Cancel', + '204': 'Soft ECC Corrections', + '205': 'Thermal Asperity Rate', + '206': 'Flying Height', + '207': 'Spin High Current', + '209': 'Offline Seek Performance', + '220': 'Disk Shift', + '221': 'G-Sense Error Rate', + '222': 'Loaded Hours', + '223': 'Load/Unload Retries', + '224': 'Load Friction', + '225': 'Load/Unload Cycles', + '226': 'Load-in Time', + '227': 'Torque Amplification Count', + '228': 'Power-Off Retracts', + '230': 'GMR Head Amplitude', + '231': 'Temperature', + '232': 'Available Reserved Space', + '233': 'Media Wearout Indicator', + '240': 'Head Flying Hours', + '241': 'Total LBAs Written', + '242': 'Total LBAs Read', + '250': 'Read Error Retry Rate' +} + +LIMIT = namedtuple('LIMIT', ['min', 'max']) + +LIMITS = { + '194': LIMIT(0, 200) } -NAMED_DISKS = namedtuple('disks', ['name', 'size', 'number']) +RESCAN_INTERVAL = 60 + +REGEX = re.compile( + '(\d+);' # attribute + '(\d+);' # normalized value + '(\d+)', # raw value + re.X +) + + +def chart_template(chart_name): + units, attr_id = chart_name.split('_')[-2:] + title = '{value_type} {description}'.format(value_type=units.capitalize(), + description=SMART_ATTR[attr_id]) + family = SMART_ATTR[attr_id].lower() + + return { + chart_name: { + 'options': [None, title, units, family, 'smartd_log.' + chart_name, 'line'], + 'lines': [] + } + } + + +def handle_os_error(method): + def on_call(*args): + try: + return method(*args) + except OSError: + return None + return on_call + + +class SmartAttribute(object): + def __init__(self, idx, normalized, raw): + self.id = idx + self.normalized = normalized + self._raw = raw + + @property + def raw(self): + if self.id in LIMITS: + limit = LIMITS[self.id] + if limit.min <= int(self._raw) <= limit.max: + return self._raw + return None + return self._raw + + @raw.setter + def raw(self, value): + self._raw = value + + +class DiskLogFile: + def __init__(self, path): + self.path = path + self.size = os.path.getsize(path) + + @handle_os_error + def is_changed(self): + new_size = os.path.getsize(self.path) + old_size, self.size = self.size, new_size + + return new_size != old_size and new_size + + @staticmethod + @handle_os_error + def is_valid(log_file, exclude): + return all([log_file.endswith('.csv'), + not [p for p in exclude if p in log_file], + os.access(log_file, os.R_OK), + os.path.getsize(log_file)]) + + +class Disk: + def __init__(self, full_path, age): + self.log_file = DiskLogFile(full_path) + self.name = os.path.basename(full_path).split('.')[-3] + self.age = int(age) + self.status = True + self.attributes = dict() + + self.get_attributes() + + def __eq__(self, other): + if isinstance(other, Disk): + return self.name == other.name + return self.name == other + + @handle_os_error + def is_active(self): + return (time() - os.path.getmtime(self.log_file.path)) / 60 < self.age + + @handle_os_error + def get_attributes(self): + last_line = read_last_line(self.log_file.path) + self.attributes = dict((attr, SmartAttribute(attr, normalized, raw)) for attr, normalized, raw + in REGEX.findall(last_line)) + return True + + def data(self): + data = dict() + for attr in self.attributes.values(): + data['_'.join([self.name, 'normalized', attr.id])] = attr.normalized + if attr.raw is not None: + data['_'.join([self.name, 'raw', attr.id])] = attr.raw + return data class Service(SimpleService): def __init__(self, configuration=None, name=None): SimpleService.__init__(self, configuration=configuration, name=name) - self.regex = r_compile(r'(\d+);(\d+);(\d+)') self.log_path = self.configuration.get('log_path', '/var/log/smartd') - self.raw_values = self.configuration.get('raw_values') - self.attr = self.configuration.get('smart_attributes', []) - self.previous_data = dict() + self.raw = self.configuration.get('raw_values', True) + self.exclude = self.configuration.get('exclude_disks', str()).split() + self.age = self.configuration.get('age', 30) + + self.runs = 0 + self.disks = list() + self.order = list() + self.definitions = dict() def check(self): - # Can\'t start without smartd readable diks log files - disks = find_disks_in_log_path(self.log_path) - if not disks: - self.error('Can\'t locate any smartd log files in %s' % self.log_path) - return False - - # List of namedtuples to track smartd log file size - self.disks = [NAMED_DISKS(name=disks[i], size=0, number=i) for i in range(len(disks))] - - if self._get_data(): - self.create_charts() - return True - else: - self.error('Can\'t collect any data. Sorry.') - return False - - def _get_raw_data(self, queue, disk): - # The idea is to open a file. - # Jump to the end. - # Seek backward until '\n' symbol appears - # If '\n' is found or it's the beginning of the file - # readline()! (last or first line) - with open(disk, 'rb') as f: - f.seek(-2, 2) - while f.read(1) != b'\n': - f.seek(-2, 1) - if f.tell() == 0: - break - result = f.readline() - - result = result.decode() - result = self.regex.findall(result) - - queue.put([basename(disk), result]) - - def _get_data(self): - threads, result = list(), list() - queue = Queue() - to_netdata = dict() - - # If the size has not changed there is no reason to poll log files. - disks = [disk for disk in self.disks if self.size_changed(disk)] - if disks: - for disk in disks: - th = Thread(target=self._get_raw_data, args=(queue, disk.name)) - th.start() - threads.append(th) - - for thread in threads: - thread.join() - result.append(queue.get()) + self.disks = self.scan() + + if not self.disks: + return None + + user_defined_sa = self.configuration.get('smart_attributes') + + if user_defined_sa: + order = user_defined_sa.split() or ORDER else: - # Data from last real poll - return self.previous_data or None + order = ORDER - for elem in result: - for a, n, r in elem[1]: - to_netdata.update({'_'.join([elem[0], a]): r if self.raw_values else n}) + self.create_charts(order) - self.previous_data.update(to_netdata) + return True - return to_netdata or None + def get_data(self): + self.runs += 1 - def size_changed(self, disk): - # We are not interested in log files: - # 1. zero size - # 2. size is not changed since last poll - try: - size = getsize(disk.name) - if size != disk.size and size: - self.disks[disk.number] = disk._replace(size=size) - return True - else: - return False - except OSError: - # Remove unreadable/nonexisting log files from list of disks and previous_data - self.disks.remove(disk) - self.previous_data = dict([(k, v) for k, v in self.previous_data.items() if basename(disk.name) not in k]) - return False + if self.runs % RESCAN_INTERVAL == 0: + self.cleanup_and_rescan() + + data = dict() + + for disk in self.disks: + + if not disk.status: + continue + + changed = disk.log_file.is_changed() + + # True = changed, False = unchanged, None = Exception + if changed is None: + disk.status = False + continue + + if changed: + success = disk.get_attributes() + if not success: + disk.status = False + continue + + data.update(disk.data()) + + return data or None - def create_charts(self): + def create_charts(self, order): + for attr in order: + raw_name, normalized_name = 'attr_id_raw_' + attr, 'attr_id_normalized_' + attr + raw, normalized = chart_template(raw_name), chart_template(normalized_name) + self.order.extend([normalized_name, raw_name]) + self.definitions.update(raw) + self.definitions.update(normalized) - def create_lines(attrid): - result = list() for disk in self.disks: - name = basename(disk.name) - result.append(['_'.join([name, attrid]), name[:name.index('.')], 'absolute']) - return result + if attr not in disk.attributes: + self.debug("'{disk}' has no attribute '{attr_id}'".format(disk=disk.name, + attr_id=attr)) + continue + normalized[normalized_name]['lines'].append(['_'.join([disk.name, 'normalized', attr]), disk.name]) - # Use configured attributes, if present. If something goes wrong we don't care. - order = ORDER - try: - order = [attr for attr in self.attr.split() if attr in SMART_ATTR.keys()] or ORDER - except Exception: - pass - self.order = [''.join(['attrid', i]) for i in order] - self.definitions = dict() - units = 'raw' if self.raw_values else 'normalized' - - for k, v in dict([(k, v) for k, v in SMART_ATTR.items() if k in ORDER]).items(): - self.definitions.update({''.join(['attrid', k]): { - 'options': [None, v, units, v.lower(), 'smartd.attrid' + k, 'line'], - 'lines': create_lines(k)}}) - -def find_disks_in_log_path(log_path): - # smartd log file is OK if: - # 1. it is a file - # 2. file name endswith with 'csv' - # 3. file is readable - if not isdir(log_path): return None - return [join(log_path, f) for f in listdir(log_path) - if all([isfile(join(log_path, f)), f.endswith('.csv'), access(join(log_path, f), R_OK)])] + if not self.raw: + continue + + if disk.attributes[attr].raw is not None: + raw[raw_name]['lines'].append(['_'.join([disk.name, 'raw', attr]), disk.name]) + continue + self.debug("'{disk}' attribute '{attr_id}' value not in {limits}".format(disk=disk.name, + attr_id=attr, + limits=LIMITS[attr])) + + def cleanup_and_rescan(self): + self.cleanup() + new_disks = self.scan(only_new=True) + + for disk in new_disks: + valid = False + + for chart in self.charts: + value_type, idx = chart.id.split('_')[2:] + + if idx in disk.attributes: + valid = True + dimension_id = '_'.join([disk.name, value_type, idx]) + + if dimension_id in chart: + chart.hide_dimension(dimension_id=dimension_id, reverse=True) + else: + chart.add_dimension([dimension_id, disk.name]) + if valid: + self.disks.append(disk) + + def cleanup(self): + for disk in self.disks: + + if not disk.is_active(): + disk.status = False + if not disk.status: + for chart in self.charts: + dimension_id = '_'.join([disk.name, chart.id[8:]]) + chart.hide_dimension(dimension_id=dimension_id) + + self.disks = [disk for disk in self.disks if disk.status] + + def scan(self, only_new=None): + new_disks = list() + for f in os.listdir(self.log_path): + full_path = os.path.join(self.log_path, f) + + if DiskLogFile.is_valid(full_path, self.exclude): + disk = Disk(full_path, self.age) + + active = disk.is_active() + if active is None: + continue + if active: + if not only_new: + new_disks.append(disk) + else: + if disk not in self.disks: + new_disks.append(disk) + else: + if not only_new: + self.debug("'{disk}' not updated in the last {age} minutes, " + "skipping it.".format(disk=disk.name, age=self.age)) + return new_disks diff --git a/python.d/squid.chart.py b/python.d/squid.chart.py index e9e8f1d08..ba8f982ff 100644 --- a/python.d/squid.chart.py +++ b/python.d/squid.chart.py @@ -2,8 +2,8 @@ # Description: squid netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import SocketService -import select +from bases.FrameworkServices.SocketService import SocketService + # default module values (can be overridden per job in `config`) # update_every = 2 @@ -60,7 +60,7 @@ class Service(SocketService): """ response = self._get_raw_data() - data = {} + data = dict() try: raw = "" for tmp in response.split('\r\n'): @@ -81,11 +81,10 @@ class Service(SocketService): self.error("invalid data received") return None - if len(data) == 0: + if not data: self.error("no data received") return None - else: - return data + return data def _check_raw_data(self, data): header = data[:1024].lower() diff --git a/python.d/tomcat.chart.py b/python.d/tomcat.chart.py index 05547236a..a570d5643 100644 --- a/python.d/tomcat.chart.py +++ b/python.d/tomcat.chart.py @@ -2,9 +2,10 @@ # Description: tomcat netdata python.d module # Author: Pawel Krupa (paulfantom) -from base import UrlService import xml.etree.ElementTree as ET +from bases.FrameworkServices.UrlService import UrlService + # default module values (can be overridden per job in `config`) # update_every = 2 priority = 60000 @@ -71,6 +72,7 @@ CHARTS = { ]}, } + class Service(UrlService): def __init__(self, configuration=None, name=None): UrlService.__init__(self, configuration=configuration, name=name) @@ -87,7 +89,6 @@ class Service(UrlService): data = None raw_data = self._get_raw_data() if raw_data: - xml = None try: xml = ET.fromstring(raw_data) except ET.ParseError: @@ -100,27 +101,27 @@ class Service(UrlService): connector = None if self.connector_name: for conn in xml.findall('connector'): - if conn.get('name') == self.connector_name: + if self.connector_name in conn.get('name'): connector = conn break else: connector = xml.find('connector') memory = jvm.find('memory') - data['free'] = memory.get('free') + data['free'] = memory.get('free') data['total'] = memory.get('total') for pool in jvm.findall('memorypool'): name = pool.get('name') - if name == 'Eden Space': + if 'Eden Space' in name: data['eden_used'] = pool.get('usageUsed') data['eden_commited'] = pool.get('usageCommitted') data['eden_max'] = pool.get('usageMax') - elif name == 'Survivor Space': + elif 'Survivor Space' in name: data['survivor_used'] = pool.get('usageUsed') data['survivor_commited'] = pool.get('usageCommitted') data['survivor_max'] = pool.get('usageMax') - elif name == 'Tenured Gen': + elif 'Tenured Gen' in name or 'Old Gen' in name: data['tenured_used'] = pool.get('usageUsed') data['tenured_commited'] = pool.get('usageCommitted') data['tenured_max'] = pool.get('usageMax') diff --git a/python.d/varnish.chart.py b/python.d/varnish.chart.py index 2665bb383..d8145c0b6 100644 --- a/python.d/varnish.chart.py +++ b/python.d/varnish.chart.py @@ -2,230 +2,240 @@ # Description: varnish netdata python.d module # Author: l2isbad -from base import SimpleService -from re import compile -from subprocess import Popen, PIPE +import re + +from bases.collection import find_binary +from bases.FrameworkServices.ExecutableService import ExecutableService # default module values (can be overridden per job in `config`) # update_every = 2 priority = 60000 retries = 60 -ORDER = ['session', 'hit_rate', 'chit_rate', 'expunge', 'threads', 'backend_health', 'memory_usage', 'bad', 'uptime'] - -CHARTS = {'backend_health': - {'lines': [['backend_conn', 'conn', 'incremental', 1, 1], - ['backend_unhealthy', 'unhealthy', 'incremental', 1, 1], - ['backend_busy', 'busy', 'incremental', 1, 1], - ['backend_fail', 'fail', 'incremental', 1, 1], - ['backend_reuse', 'reuse', 'incremental', 1, 1], - ['backend_recycle', 'resycle', 'incremental', 1, 1], - ['backend_toolate', 'toolate', 'incremental', 1, 1], - ['backend_retry', 'retry', 'incremental', 1, 1], - ['backend_req', 'req', 'incremental', 1, 1]], - 'options': [None, 'Backend health', 'connections', 'Backend health', 'varnish.backend_traf', 'line']}, - 'bad': - {'lines': [['sess_drop_b', None, 'incremental', 1, 1], - ['backend_unhealthy_b', None, 'incremental', 1, 1], - ['fetch_failed', None, 'incremental', 1, 1], - ['backend_busy_b', None, 'incremental', 1, 1], - ['threads_failed_b', None, 'incremental', 1, 1], - ['threads_limited_b', None, 'incremental', 1, 1], - ['threads_destroyed_b', None, 'incremental', 1, 1], - ['thread_queue_len_b', 'queue_len', 'absolute', 1, 1], - ['losthdr_b', None, 'incremental', 1, 1], - ['esi_errors_b', None, 'incremental', 1, 1], - ['esi_warnings_b', None, 'incremental', 1, 1], - ['sess_fail_b', None, 'incremental', 1, 1], - ['sc_pipe_overflow_b', None, 'incremental', 1, 1], - ['sess_pipe_overflow_b', None, 'incremental', 1, 1]], - 'options': [None, 'Misbehavior', 'problems', 'Problems summary', 'varnish.bad', 'line']}, - 'expunge': - {'lines': [['n_expired', 'expired', 'incremental', 1, 1], - ['n_lru_nuked', 'lru_nuked', 'incremental', 1, 1]], - 'options': [None, 'Object expunging', 'objects', 'Cache performance', 'varnish.expunge', 'line']}, - 'hit_rate': - {'lines': [['cache_hit_perc', 'hit', 'absolute', 1, 100], - ['cache_miss_perc', 'miss', 'absolute', 1, 100], - ['cache_hitpass_perc', 'hitpass', 'absolute', 1, 100]], - 'options': [None, 'All history hit rate ratio','percent', 'Cache performance', 'varnish.hit_rate', 'stacked']}, - 'chit_rate': - {'lines': [['cache_hit_cperc', 'hit', 'absolute', 1, 100], - ['cache_miss_cperc', 'miss', 'absolute', 1, 100], - ['cache_hitpass_cperc', 'hitpass', 'absolute', 1, 100]], - 'options': [None, 'Current poll hit rate ratio','percent', 'Cache performance', 'varnish.chit_rate', 'stacked']}, - 'memory_usage': - {'lines': [['s0.g_space', 'available', 'absolute', 1, 1048576], - ['s0.g_bytes', 'allocated', 'absolute', -1, 1048576]], - 'options': [None, 'Memory usage', 'megabytes', 'Memory usage', 'varnish.memory_usage', 'stacked']}, - 'session': - {'lines': [['sess_conn', 'sess_conn', 'incremental', 1, 1], - ['client_req', 'client_requests', 'incremental', 1, 1], - ['client_conn', 'client_conn', 'incremental', 1, 1], - ['client_drop', 'client_drop', 'incremental', 1, 1], - ['sess_dropped', 'sess_dropped', 'incremental', 1, 1]], - 'options': [None, 'Sessions', 'units', 'Client metrics', 'varnish.session', 'line']}, - 'threads': - {'lines': [['threads', None, 'absolute', 1, 1], - ['threads_created', 'created', 'incremental', 1, 1], - ['threads_failed', 'failed', 'incremental', 1, 1], - ['threads_limited', 'limited', 'incremental', 1, 1], - ['thread_queue_len', 'queue_len', 'incremental', 1, 1], - ['sess_queued', 'sess_queued', 'incremental', 1, 1]], - 'options': [None, 'Thread status', 'threads', 'Thread-related metrics', 'varnish.threads', 'line']}, - 'uptime': - {'lines': [['uptime', None, 'absolute', 1, 1]], - 'options': [None, 'Varnish uptime', 'seconds', 'Uptime', 'varnish.uptime', 'line']} +ORDER = ['session_connections', 'client_requests', + 'all_time_hit_rate', 'current_poll_hit_rate', 'cached_objects_expired', 'cached_objects_nuked', + 'threads_total', 'threads_statistics', 'threads_queue_len', + 'backend_connections', 'backend_requests', + 'esi_statistics', + 'memory_usage', + 'uptime'] + +CHARTS = { + 'session_connections': { + 'options': [None, 'Connections Statistics', 'connections/s', + 'client metrics', 'varnish.session_connection', 'line'], + 'lines': [ + ['sess_conn', 'accepted', 'incremental'], + ['sess_dropped', 'dropped', 'incremental'] + ] + }, + 'client_requests': { + 'options': [None, 'Client Requests', 'requests/s', + 'client metrics', 'varnish.client_requests', 'line'], + 'lines': [ + ['client_req', 'received', 'incremental'] + ] + }, + 'all_time_hit_rate': { + 'options': [None, 'All History Hit Rate Ratio', 'percent', 'cache performance', + 'varnish.all_time_hit_rate', 'stacked'], + 'lines': [ + ['cache_hit', 'hit', 'percentage-of-absolute-row'], + ['cache_miss', 'miss', 'percentage-of-absolute-row'], + ['cache_hitpass', 'hitpass', 'percentage-of-absolute-row']] + }, + 'current_poll_hit_rate': { + 'options': [None, 'Current Poll Hit Rate Ratio', 'percent', 'cache performance', + 'varnish.current_poll_hit_rate', 'stacked'], + 'lines': [ + ['cache_hit', 'hit', 'percentage-of-incremental-row'], + ['cache_miss', 'miss', 'percentage-of-incremental-row'], + ['cache_hitpass', 'hitpass', 'percentage-of-incremental-row'] + ] + }, + 'cached_objects_expired': { + 'options': [None, 'Expired Objects', 'expired/s', 'cache performance', + 'varnish.cached_objects_expired', 'line'], + 'lines': [ + ['n_expired', 'objects', 'incremental'] + ] + }, + 'cached_objects_nuked': { + 'options': [None, 'Least Recently Used Nuked Objects', 'nuked/s', 'cache performance', + 'varnish.cached_objects_nuked', 'line'], + 'lines': [ + ['n_lru_nuked', 'objects', 'incremental'] + ] + }, + 'threads_total': { + 'options': [None, 'Number Of Threads In All Pools', 'number', 'thread related metrics', + 'varnish.threads_total', 'line'], + 'lines': [ + ['threads', None, 'absolute'] + ] + }, + 'threads_statistics': { + 'options': [None, 'Threads Statistics', 'threads/s', 'thread related metrics', + 'varnish.threads_statistics', 'line'], + 'lines': [ + ['threads_created', 'created', 'incremental'], + ['threads_failed', 'failed', 'incremental'], + ['threads_limited', 'limited', 'incremental'] + ] + }, + 'threads_queue_len': { + 'options': [None, 'Current Queue Length', 'requests', 'thread related metrics', + 'varnish.threads_queue_len', 'line'], + 'lines': [ + ['thread_queue_len', 'in queue'] + ] + }, + 'backend_connections': { + 'options': [None, 'Backend Connections Statistics', 'connections/s', 'backend metrics', + 'varnish.backend_connections', 'line'], + 'lines': [ + ['backend_conn', 'successful', 'incremental'], + ['backend_unhealthy', 'unhealthy', 'incremental'], + ['backend_reuse', 'reused', 'incremental'], + ['backend_toolate', 'closed', 'incremental'], + ['backend_recycle', 'resycled', 'incremental'], + ['backend_fail', 'failed', 'incremental'] + ] + }, + 'backend_requests': { + 'options': [None, 'Requests To The Backend', 'requests/s', 'backend metrics', + 'varnish.backend_requests', 'line'], + 'lines': [ + ['backend_req', 'sent', 'incremental'] + ] + }, + 'esi_statistics': { + 'options': [None, 'ESI Statistics', 'problems/s', 'esi related metrics', 'varnish.esi_statistics', 'line'], + 'lines': [ + ['esi_errors', 'errors', 'incremental'], + ['esi_warnings', 'warnings', 'incremental'] + ] + }, + 'memory_usage': { + 'options': [None, 'Memory Usage', 'MB', 'memory usage', 'varnish.memory_usage', 'stacked'], + 'lines': [ + ['memory_free', 'free', 'absolute', 1, 1 << 20], + ['memory_allocated', 'allocated', 'absolute', 1, 1 << 20]] + }, + 'uptime': { + 'lines': [ + ['uptime', None, 'absolute'] + ], + 'options': [None, 'Uptime', 'seconds', 'uptime', 'varnish.uptime', 'line'] + } } -class Service(SimpleService): +class Parser: + _backend_new = re.compile(r'VBE.([\d\w_.]+)\(.*?\).(beresp[\w_]+)\s+(\d+)') + _backend_old = re.compile(r'VBE\.[\d\w-]+\.([\w\d_]+).(beresp[\w_]+)\s+(\d+)') + _default = re.compile(r'([A-Z]+\.)?([\d\w_.]+)\s+(\d+)') + + def __init__(self): + self.re_default = None + self.re_backend = None + + def init(self, data): + data = ''.join(data) + parsed_main = Parser._default.findall(data) + if parsed_main: + self.re_default = Parser._default + + parsed_backend = Parser._backend_new.findall(data) + if parsed_backend: + self.re_backend = Parser._backend_new + else: + parsed_backend = Parser._backend_old.findall(data) + if parsed_backend: + self.re_backend = Parser._backend_old + + def server_stats(self, data): + return self.re_default.findall(''.join(data)) + + def backend_stats(self, data): + return self.re_backend.findall(''.join(data)) + + +class Service(ExecutableService): def __init__(self, configuration=None, name=None): - SimpleService.__init__(self, configuration=configuration, name=name) - self.varnish = self.find_binary('varnishstat') - self.rgx_all = compile(r'([A-Z]+\.)?([\d\w_.]+)\s+(\d+)') - # Could be - # VBE.boot.super_backend.pipe_hdrbyte (new) - # or - # VBE.default2(127.0.0.2,,81).bereq_bodybytes (old) - # Regex result: [('super_backend', 'beresp_hdrbytes', '0'), ('super_backend', 'beresp_bodybytes', '0')] - self.rgx_bck = (compile(r'VBE.([\d\w_.]+)\(.*?\).(beresp[\w_]+)\s+(\d+)'), - compile(r'VBE\.[\d\w-]+\.([\w\d_]+).(beresp[\w_]+)\s+(\d+)')) - self.cache_prev = list() + ExecutableService.__init__(self, configuration=configuration, name=name) + self.order = ORDER + self.definitions = CHARTS + varnishstat = find_binary('varnishstat') + self.command = [varnishstat, '-1'] if varnishstat else None + self.parser = Parser() def check(self): - # Cant start without 'varnishstat' command - if not self.varnish: - self.error('Can\'t locate \'varnishstat\' binary or binary is not executable by netdata') + if not self.command: + self.error("Can't locate 'varnishstat' binary or binary is not executable by user netdata") return False - # If command is present and we can execute it we need to make sure.. - # 1. STDOUT is not empty + # STDOUT is not empty reply = self._get_raw_data() if not reply: - self.error('No output from \'varnishstat\' (not enough privileges?)') - return False - - # 2. Output is parsable (list is not empty after regex findall) - is_parsable = self.rgx_all.findall(reply) - if not is_parsable: - self.error('Cant parse output...') + self.error("No output from 'varnishstat'. Not enough privileges?") return False - # We need to find the right regex for backend parse - self.backend_list = self.rgx_bck[0].findall(reply)[::2] - if self.backend_list: - self.rgx_bck = self.rgx_bck[0] - else: - self.backend_list = self.rgx_bck[1].findall(reply)[::2] - self.rgx_bck = self.rgx_bck[1] + self.parser.init(reply) - # We are about to start! - self.create_charts() + # Output is parsable + if not self.parser.re_default: + self.error('Cant parse the output...') + return False - self.info('Plugin was started successfully') + if self.parser.re_backend: + backends = [b[0] for b in self.parser.backend_stats(reply)[::2]] + self.create_backends_charts(backends) return True - - def _get_raw_data(self): - try: - reply = Popen([self.varnish, '-1'], stdout=PIPE, stderr=PIPE, shell=False) - except OSError: - return None - - raw_data = reply.communicate()[0] - if not raw_data: - return None - - return raw_data.decode() - - def _get_data(self): + def get_data(self): """ Format data received from shell command :return: dict """ - raw_data = self._get_raw_data() - data_all = self.rgx_all.findall(raw_data) - data_backend = self.rgx_bck.findall(raw_data) + raw = self._get_raw_data() + if not raw: + return None - if not data_all: + data = dict() + server_stats = self.parser.server_stats(raw) + if not server_stats: return None - # 1. ALL data from 'varnishstat -1'. t - type(MAIN, MEMPOOL etc) - to_netdata = dict([(k, int(v)) for t, k, v in data_all]) - - # 2. ADD backend statistics - to_netdata.update(dict([('_'.join([n, k]), int(v)) for n, k, v in data_backend])) - - # 3. ADD additional keys to dict - # 3.1 Cache hit/miss/hitpass OVERALL in percent - cache_summary = sum([to_netdata.get('cache_hit', 0), to_netdata.get('cache_miss', 0), - to_netdata.get('cache_hitpass', 0)]) - to_netdata['cache_hit_perc'] = find_percent(to_netdata.get('cache_hit', 0), cache_summary, 10000) - to_netdata['cache_miss_perc'] = find_percent(to_netdata.get('cache_miss', 0), cache_summary, 10000) - to_netdata['cache_hitpass_perc'] = find_percent(to_netdata.get('cache_hitpass', 0), cache_summary, 10000) - - # 3.2 Cache hit/miss/hitpass CURRENT in percent - if self.cache_prev: - cache_summary = sum([to_netdata.get('cache_hit', 0), to_netdata.get('cache_miss', 0), - to_netdata.get('cache_hitpass', 0)]) - sum(self.cache_prev) - to_netdata['cache_hit_cperc'] = find_percent(to_netdata.get('cache_hit', 0) - self.cache_prev[0], cache_summary, 10000) - to_netdata['cache_miss_cperc'] = find_percent(to_netdata.get('cache_miss', 0) - self.cache_prev[1], cache_summary, 10000) - to_netdata['cache_hitpass_cperc'] = find_percent(to_netdata.get('cache_hitpass', 0) - self.cache_prev[2], cache_summary, 10000) - else: - to_netdata['cache_hit_cperc'] = 0 - to_netdata['cache_miss_cperc'] = 0 - to_netdata['cache_hitpass_cperc'] = 0 - - self.cache_prev = [to_netdata.get('cache_hit', 0), to_netdata.get('cache_miss', 0), to_netdata.get('cache_hitpass', 0)] - - # 3.3 Problems summary chart - for elem in ['backend_busy', 'backend_unhealthy', 'esi_errors', 'esi_warnings', 'losthdr', 'sess_drop', 'sc_pipe_overflow', - 'sess_fail', 'sess_pipe_overflow', 'threads_destroyed', 'threads_failed', 'threads_limited', 'thread_queue_len']: - if to_netdata.get(elem) is not None: - to_netdata[''.join([elem, '_b'])] = to_netdata.get(elem) - - # Ready steady go! - return to_netdata - - def create_charts(self): - # If 'all_charts' is true...ALL charts are displayed. If no only default + 'extra_charts' - #if self.configuration.get('all_charts'): - # self.order = EXTRA_ORDER - #else: - # try: - # extra_charts = list(filter(lambda chart: chart in EXTRA_ORDER, self.extra_charts.split())) - # except (AttributeError, NameError, ValueError): - # self.error('Extra charts disabled.') - # extra_charts = [] - - self.order = ORDER[:] - #self.order.extend(extra_charts) - - # Create static charts - #self.definitions = {chart: values for chart, values in CHARTS.items() if chart in self.order} - self.definitions = CHARTS - - # Create dynamic backend charts - if self.backend_list: - for backend in self.backend_list: - self.order.insert(0, ''.join([backend[0], '_resp_stats'])) - self.definitions.update({''.join([backend[0], '_resp_stats']): { - 'options': [None, - '%s response statistics' % backend[0].capitalize(), - "kilobit/s", - 'Backend response', - 'varnish.backend', - 'area'], - 'lines': [[''.join([backend[0], '_beresp_hdrbytes']), - 'header', 'incremental', 8, 1000], - [''.join([backend[0], '_beresp_bodybytes']), - 'body', 'incremental', -8, 1000]]}}) - - -def find_percent(value1, value2, multiply): - # If value2 is 0 return 0 - if not value2: - return 0 - else: - return round(float(value1) / float(value2) * multiply) + if self.parser.re_backend: + backend_stats = self.parser.backend_stats(raw) + data.update(dict(('_'.join([name, param]), value) for name, param, value in backend_stats)) + + data.update(dict((param, value) for _, param, value in server_stats)) + + data['memory_allocated'] = data['s0.g_bytes'] + data['memory_free'] = data['s0.g_space'] + + return data + + def create_backends_charts(self, backends): + for backend in backends: + chart_name = ''.join([backend, '_response_statistics']) + title = 'Backend "{0}"'.format(backend.capitalize()) + hdr_bytes = ''.join([backend, '_beresp_hdrbytes']) + body_bytes = ''.join([backend, '_beresp_bodybytes']) + + chart = { + chart_name: + { + 'options': [None, title, 'kilobits/s', 'backend response statistics', + 'varnish.backend', 'area'], + 'lines': [ + [hdr_bytes, 'header', 'incremental', 8, 1000], + [body_bytes, 'body', 'incremental', -8, 1000] + ] + } + } + + self.order.insert(0, chart_name) + self.definitions.update(chart) diff --git a/python.d/web_log.chart.py b/python.d/web_log.chart.py index a5359bc4d..954ecd41d 100644 --- a/python.d/web_log.chart.py +++ b/python.d/web_log.chart.py @@ -4,22 +4,20 @@ import bisect import re +import os from collections import namedtuple, defaultdict from copy import deepcopy -from os import access, R_OK -from os.path import getsize try: from itertools import filterfalse except ImportError: + from itertools import ifilter as filter from itertools import ifilterfalse as filterfalse -from base import LogService -import msg +from bases.collection import read_last_line +from bases.FrameworkServices.LogService import LogService -priority = 60000 -retries = 60 ORDER_APACHE_CACHE = ['apache_cache'] @@ -246,6 +244,8 @@ SQUID_CODES = dict(TCP='squid_transport_methods', UDP='squid_transport_methods', DENIED='squid_cache_events', NOFETCH='squid_cache_events', TUNNEL='squid_cache_events', ABORTED='squid_transport_errors', TIMEOUT='squid_transport_errors') +REQUEST_REGEX = re.compile(r'(?P[A-Z]+) (?P[^ ]+) [A-Z]+/(?P\d(?:.\d)?)') + class Service(LogService): def __init__(self, configuration=None, name=None): @@ -254,8 +254,9 @@ class Service(LogService): :param name: """ LogService.__init__(self, configuration=configuration, name=name) - self.log_type = self.configuration.get('type', 'web') + self.configuration = configuration self.log_path = self.configuration.get('path') + self.job = None def check(self): """ @@ -267,123 +268,43 @@ class Service(LogService): 4. other checks depends on log "type" """ + log_type = self.configuration.get('type', 'web') log_types = dict(web=Web, apache_cache=ApacheCache, squid=Squid) - if self.log_type not in log_types: - self.error('bad log type (%s). Supported types: %s' % (self.log_type, log_types.keys())) + if log_type not in log_types: + self.error("bad log type {log_type}. Supported types: {types}".format(log_type=log_type, + types=log_types.keys())) return False if not self.log_path: self.error('log path is not specified') return False - if not (self._find_recent_log_file() and access(self.log_path, R_OK)): - self.error('%s not readable or not exist' % self.log_path) + if not (self._find_recent_log_file() and os.access(self.log_path, os.R_OK)): + self.error('{log_file} not readable or not exist'.format(log_file=self.log_path)) return False - if not getsize(self.log_path): - self.error('%s is empty' % self.log_path) + if not os.path.getsize(self.log_path): + self.error('{log_file} is empty'.format(log_file=self.log_path)) return False - self.configuration['update_every'] = self.update_every - self.configuration['name'] = self.name - self.configuration['override_name'] = self.override_name - self.configuration['_dimensions'] = self._dimensions - self.configuration['path'] = self.log_path - - cls = log_types[self.log_type] - self.Job = cls(configuration=self.configuration) - if self.Job.check(): - self.order = self.Job.order - self.definitions = self.Job.definitions - self.info('Current log file: %s' % self.log_path) + self.job = log_types[log_type](self) + if self.job.check(): + self.order = self.job.order + self.definitions = self.job.definitions return True return False def _get_data(self): - return self.Job.get_data(self._get_raw_data()) - - -class Mixin: - def filter_data(self, raw_data): - """ - :param raw_data: list - :return: - """ - if not self.pre_filter: - return raw_data - filtered = raw_data - for elem in self.pre_filter: - if elem.description == 'filter_include': - filtered = filter(elem.func, filtered) - elif elem.description == 'filter_exclude': - filtered = filterfalse(elem.func, filtered) - return filtered - - def add_new_dimension(self, dimension_id, chart_key, dimension=None, - algorithm='incremental', multiplier=1, divisor=1): - """ - :param dimension: - :param chart_key: - :param dimension_id: - :param algorithm: - :param multiplier: - :param divisor: - :return: - """ - - self.data[dimension_id] = 0 - # SET method check if dim in _dimensions - self.conf['_dimensions'].append(dimension_id) - # UPDATE method do SET only if dim in definitions - dimension_list = list(map(str, [dimension_id, - dimension if dimension else dimension_id, - algorithm, - multiplier, - divisor])) - self.definitions[chart_key]['lines'].append(dimension_list) - job_name = find_job_name(self.conf['override_name'], self.conf['name']) - opts = self.definitions[chart_key]['options'] - chart = 'CHART %s.%s "" "%s" %s "%s" %s %s 60000 %s\n' % (job_name, chart_key, - opts[1], opts[2], opts[3], - opts[4], opts[5], self.conf['update_every']) - print(chart + "DIMENSION %s\n" % ' '.join(dimension_list)) - - def get_last_line(self): - """ - Reads last line from the log file - :return: str: - """ - # Read last line (or first if there is only one line) - with open(self.conf['path'], 'rb') as logs: - logs.seek(-2, 2) - while logs.read(1) != b'\n': - logs.seek(-2, 1) - if logs.tell() == 0: - break - last_line = logs.readline() - try: - return last_line.decode() - except UnicodeDecodeError: - try: - return last_line.decode(encoding='utf-8') - except (TypeError, UnicodeDecodeError) as error: - msg.error('web_log', str(error)) - return False - - @staticmethod - def error(*params): - msg.error('web_log', ' '.join(map(str, params))) - - @staticmethod - def info(*params): - msg.info('web_log', ' '.join(map(str, params))) + return self.job.get_data(self._get_raw_data()) -class Web(Mixin): - def __init__(self, configuration): - self.conf = configuration - self.pre_filter = check_patterns('filter', self.conf.get('filter')) +class Web: + def __init__(self, service): + self.service = service + self.order = ORDER_WEB[:] + self.definitions = deepcopy(CHARTS_WEB) + self.pre_filter = check_patterns('filter', self.configuration.get('filter')) self.storage = dict() self.data = {'bytes_sent': 0, 'resp_length': 0, 'resp_time_min': 0, 'resp_time_max': 0, 'resp_time_avg': 0, 'resp_time_upstream_min': 0, 'resp_time_upstream_max': 0, @@ -392,23 +313,27 @@ class Web(Mixin): 'req_ipv6': 0, 'unique_tot_ipv4': 0, 'unique_tot_ipv6': 0, 'successful_requests': 0, 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0, 'GET': 0} + def __getattr__(self, item): + return getattr(self.service, item) + def check(self): - last_line = self.get_last_line() + last_line = read_last_line(self.log_path) if not last_line: return False # Custom_log_format or predefined log format. - if self.conf.get('custom_log_format'): + if self.configuration.get('custom_log_format'): match_dict, error = self.find_regex_custom(last_line) else: match_dict, error = self.find_regex(last_line) # "match_dict" is None if there are any problems if match_dict is None: - self.error(str(error)) + self.error(error) return False + self.storage['unique_all_time'] = list() - self.storage['url_pattern'] = check_patterns('url_pattern', self.conf.get('categories')) - self.storage['user_pattern'] = check_patterns('user_pattern', self.conf.get('user_defined')) + self.storage['url_pattern'] = check_patterns('url_pattern', self.configuration.get('categories')) + self.storage['user_pattern'] = check_patterns('user_pattern', self.configuration.get('user_defined')) self.create_web_charts(match_dict) # Create charts self.info('Collected data: %s' % list(match_dict.keys())) @@ -420,20 +345,21 @@ class Web(Mixin): :return: Create/remove additional charts depending on the 'match_dict' keys and configuration file options """ - self.order = ORDER_WEB[:] - self.definitions = deepcopy(CHARTS_WEB) - if 'resp_time' not in match_dict: self.order.remove('response_time') if 'resp_time_upstream' not in match_dict: self.order.remove('response_time_upstream') - if not self.conf.get('all_time', True): + if not self.configuration.get('all_time', True): self.order.remove('clients_all') # Add 'detailed_response_codes' chart if specified in the configuration - if self.conf.get('detailed_response_codes', True): - codes = DET_RESP_AGGR[:1] if self.conf.get('detailed_response_aggregate', True) else DET_RESP_AGGR[1:] + if self.configuration.get('detailed_response_codes', True): + if self.configuration.get('detailed_response_aggregate', True): + codes = DET_RESP_AGGR[:1] + else: + codes = DET_RESP_AGGR[1:] + for code in codes: self.order.append('detailed_response_codes%s' % code) self.definitions['detailed_response_codes%s' % code] \ @@ -444,9 +370,8 @@ class Web(Mixin): # Add 'requests_per_url' chart if specified in the configuration if self.storage['url_pattern']: for elem in self.storage['url_pattern']: - self.definitions['requests_per_url']['lines'].append([elem.description, - elem.description[12:], - 'incremental']) + dim = [elem.description, elem.description[12:], 'incremental'] + self.definitions['requests_per_url']['lines'].append(dim) self.data[elem.description] = 0 self.data['url_pattern_other'] = 0 else: @@ -455,9 +380,8 @@ class Web(Mixin): # Add 'requests_per_user_defined' chart if specified in the configuration if self.storage['user_pattern'] and 'user_defined' in match_dict: for elem in self.storage['user_pattern']: - self.definitions['requests_per_user_defined']['lines'].append([elem.description, - elem.description[13:], - 'incremental']) + dim = [elem.description, elem.description[13:], 'incremental'] + self.definitions['requests_per_user_defined']['lines'].append(dim) self.data[elem.description] = 0 self.data['user_pattern_other'] = 0 else: @@ -473,7 +397,7 @@ class Web(Mixin): if not raw_data: return None if raw_data is None else self.data - filtered_data = self.filter_data(raw_data=raw_data) + filtered_data = filter_data(raw_data=raw_data, pre_filter=self.pre_filter) unique_current = set() timings = defaultdict(lambda: dict(minimum=None, maximum=0, summary=0, count=0)) @@ -488,39 +412,24 @@ class Web(Mixin): except KeyError: self.data['0xx'] += 1 # detailed response code - if self.conf.get('detailed_response_codes', True): + if self.configuration.get('detailed_response_codes', True): self.get_data_per_response_codes_detailed(code=match_dict['code']) # response statuses self.get_data_per_statuses(code=match_dict['code']) - # requests per url - if self.storage['url_pattern']: - self.get_data_per_pattern(row=match_dict['url'], - other='url_pattern_other', - pattern=self.storage['url_pattern']) # requests per user defined pattern if self.storage['user_pattern'] and 'user_defined' in match_dict: self.get_data_per_pattern(row=match_dict['user_defined'], other='user_pattern_other', pattern=self.storage['user_pattern']) - # requests per http method - if match_dict['method'] not in self.data: - self.add_new_dimension(dimension_id=match_dict['method'], - chart_key='http_method') - self.data[match_dict['method']] += 1 - # requests per http version - if 'http_version' in match_dict: - dim_id = match_dict['http_version'].replace('.', '_') - if dim_id not in self.data: - self.add_new_dimension(dimension_id=dim_id, - chart_key='http_version', - dimension=match_dict['http_version']) - self.data[dim_id] += 1 + # method, url, http version + self.get_data_from_request_field(match_dict=match_dict) # bandwidth sent bytes_sent = match_dict['bytes_sent'] if '-' not in match_dict['bytes_sent'] else 0 self.data['bytes_sent'] += int(bytes_sent) # request processing time and bandwidth received if 'resp_length' in match_dict: - self.data['resp_length'] += int(match_dict['resp_length']) + resp_length = match_dict['resp_length'] if '-' not in match_dict['resp_length'] else 0 + self.data['resp_length'] += int(resp_length) if 'resp_time' in match_dict: get_timings(timings=timings['resp_time'], time=self.storage['func_resp_time'](float(match_dict['resp_time']))) @@ -531,7 +440,7 @@ class Web(Mixin): proto = 'ipv6' if ':' in match_dict['address'] else 'ipv4' self.data['req_' + proto] += 1 # unique clients ips - if self.conf.get('all_time', True): + if self.configuration.get('all_time', True): if address_not_in_pool(pool=self.storage['unique_all_time'], address=match_dict['address'], pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']): @@ -562,45 +471,35 @@ class Web(Mixin): # REGEX: 1.IPv4 address 2.HTTP method 3. URL 4. Response code # 5. Bytes sent 6. Response length 7. Response process time default = re.compile(r'(?P
[\da-f.:]+|localhost)' - r' -.*?"(?P[A-Z]+)' - r' (?P[^ ]+)' - r' [A-Z]+/(?P\d\.\d)"' + r' -.*?"(?P[^"]*)"' r' (?P[1-9]\d{2})' r' (?P\d+|-)') apache_ext_insert = re.compile(r'(?P
[\da-f.:]+|localhost)' - r' -.*?"(?P[A-Z]+)' - r' (?P[^ ]+)' - r' [A-Z]+/(?P\d\.\d)"' + r' -.*?"(?P[^"]*)"' r' (?P[1-9]\d{2})' r' (?P\d+|-)' - r' (?P\d+)' + r' (?P\d+|-)' r' (?P\d+) ') apache_ext_append = re.compile(r'(?P
[\da-f.:]+|localhost)' - r' -.*?"(?P[A-Z]+)' - r' (?P[^ ]+)' - r' [A-Z]+/(?P\d\.\d)"' + r' -.*?"(?P[^"]*)"' r' (?P[1-9]\d{2})' r' (?P\d+|-)' r' .*?' - r' (?P\d+)' + r' (?P\d+|-)' r' (?P\d+)' r'(?: |$)') nginx_ext_insert = re.compile(r'(?P
[\da-f.:]+)' - r' -.*?"(?P[A-Z]+)' - r' (?P[^ ]+)' - r' [A-Z]+/(?P\d\.\d)"' + r' -.*?"(?P[^"]*)"' r' (?P[1-9]\d{2})' r' (?P\d+)' r' (?P\d+)' r' (?P\d+\.\d+) ') nginx_ext2_insert = re.compile(r'(?P
[\da-f.:]+)' - r' -.*?"(?P[A-Z]+)' - r' (?P[^ ]+)' - r' [A-Z]+/(?P\d\.\d)"' + r' -.*?"(?P[^"]*)"' r' (?P[1-9]\d{2})' r' (?P\d+)' r' (?P\d+)' @@ -608,9 +507,7 @@ class Web(Mixin): r' (?P[\d.-]+) ') nginx_ext_append = re.compile(r'(?P
[\da-f.:]+)' - r' -.*?"(?P[A-Z]+)' - r' (?P[^ ]+)' - r' [A-Z]+/(?P\d\.\d)"' + r' -.*?"(?P[^"]*)"' r' (?P[1-9]\d{2})' r' (?P\d+)' r' .*?' @@ -669,14 +566,14 @@ class Web(Mixin): ("resp_length" is integer or "-", "resp_time" is integer or float) """ - if not hasattr(self.conf.get('custom_log_format'), 'keys'): + if not hasattr(self.configuration.get('custom_log_format'), 'keys'): return find_regex_return(msg='Custom log: "custom_log_format" is not a ') - pattern = self.conf.get('custom_log_format', dict()).get('pattern') + pattern = self.configuration.get('custom_log_format', dict()).get('pattern') if not (pattern and isinstance(pattern, str)): return find_regex_return(msg='Custom log: "pattern" option is not specified or type is not ') - resp_time_func = self.conf.get('custom_log_format', dict()).get('time_multiplier') or 0 + resp_time_func = self.configuration.get('custom_log_format', dict()).get('time_multiplier') or 0 if not isinstance(resp_time_func, int): return find_regex_return(msg='Custom log: "time_multiplier" is not an integer') @@ -685,6 +582,7 @@ class Web(Mixin): regex = re.compile(pattern) except re.error as error: return find_regex_return(msg='Pattern compile error: %s' % str(error)) + match = regex.search(last_line) if not match: return find_regex_return(msg='Custom log: pattern search FAILED') @@ -693,14 +591,14 @@ class Web(Mixin): if match_dict is None: return find_regex_return(msg='Custom log: search OK but contains no named subgroups' ' (you need to use ?P)') - mandatory_dict = {'address': r'[\da-f.:]+|localhost', + mandatory_dict = {'address': r'[\w.:-]+', 'code': r'[1-9]\d{2}', - 'method': r'[A-Z]+', 'bytes_sent': r'\d+|-'} - optional_dict = {'resp_length': r'\d+', + optional_dict = {'resp_length': r'\d+|-', 'resp_time': r'[\d.]+', 'resp_time_upstream': r'[\d.-]+', - 'http_version': r'\d(\.\d)?'} + 'method': r'[A-Z]+', + 'http_version': r'\d(?:.\d)?'} mandatory_values = set(mandatory_dict) - set(match_dict) if mandatory_values: @@ -726,6 +624,36 @@ class Web(Mixin): self.storage['regex'] = regex return find_regex_return(match_dict=match_dict) + def get_data_from_request_field(self, match_dict): + if match_dict.get('request'): + match_dict = REQUEST_REGEX.search(match_dict['request']) + if match_dict: + match_dict = match_dict.groupdict() + else: + return + # requests per url + if match_dict.get('url') and self.storage['url_pattern']: + self.get_data_per_pattern(row=match_dict['url'], + other='url_pattern_other', + pattern=self.storage['url_pattern']) + # requests per http method + if match_dict.get('method'): + if match_dict['method'] not in self.data: + self.charts['http_method'].add_dimension([match_dict['method'], + match_dict['method'], + 'incremental']) + self.data[match_dict['method']] = 0 + self.data[match_dict['method']] += 1 + # requests per http version + if match_dict.get('http_version'): + dim_id = match_dict['http_version'].replace('.', '_') + if dim_id not in self.data: + self.charts['http_version'].add_dimension([dim_id, + match_dict['http_version'], + 'incremental']) + self.data[dim_id] = 0 + self.data[dim_id] += 1 + def get_data_per_response_codes_detailed(self, code): """ :param code: str: CODE from parsed line. Ex.: '202, '499' @@ -733,14 +661,14 @@ class Web(Mixin): Calls add_new_dimension method If the value is found for the first time """ if code not in self.data: - if self.conf.get('detailed_response_aggregate', True): - self.add_new_dimension(dimension_id=code, - chart_key='detailed_response_codes') + if self.configuration.get('detailed_response_aggregate', True): + self.charts['detailed_response_codes'].add_dimension([code, code, 'incremental']) + self.data[code] = 0 else: code_index = int(code[0]) if int(code[0]) < 6 else 6 chart_key = 'detailed_response_codes' + DET_RESP_AGGR[code_index] - self.add_new_dimension(dimension_id=code, - chart_key=chart_key) + self.charts[chart_key].add_dimension([code, code, 'incremental']) + self.data[code] = 0 self.data[code] += 1 def get_data_per_pattern(self, row, other, pattern): @@ -780,8 +708,8 @@ class Web(Mixin): class ApacheCache: - def __init__(self, configuration): - self.conf = configuration + def __init__(self, service): + self.service = service self.order = ORDER_APACHE_CACHE self.definitions = CHARTS_APACHE_CACHE @@ -805,12 +733,12 @@ class ApacheCache: return data -class Squid(Mixin): - def __init__(self, configuration): - self.conf = configuration +class Squid: + def __init__(self, service): + self.service = service self.order = ORDER_SQUID self.definitions = CHARTS_SQUID - self.pre_filter = check_patterns('filter', self.conf.get('filter')) + self.pre_filter = check_patterns('filter', self.configuration.get('filter')) self.storage = dict() self.data = {'duration_max': 0, 'duration_avg': 0, 'duration_min': 0, 'bytes': 0, '0xx': 0, '1xx': 0, '2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0, @@ -819,8 +747,11 @@ class Squid(Mixin): 'redirects': 0, 'bad_requests': 0, 'server_errors': 0, 'other_requests': 0 } + def __getattr__(self, item): + return getattr(self.service, item) + def check(self): - last_line = self.get_last_line() + last_line = read_last_line(self.log_path) if not last_line: return False self.storage['unique_all_time'] = list() @@ -856,7 +787,7 @@ class Squid(Mixin): 'chart': 'squid_mime_type', 'func_dim_id': lambda v: v.split('/')[0], 'func_dim': None}} - if not self.conf.get('all_time', True): + if not self.configuration.get('all_time', True): self.order.remove('squid_clients_all') return True @@ -864,7 +795,7 @@ class Squid(Mixin): if not raw_data: return None if raw_data is None else self.data - filtered_data = self.filter_data(raw_data=raw_data) + filtered_data = filter_data(raw_data=raw_data, pre_filter=self.pre_filter) unique_ip = set() timings = defaultdict(lambda: dict(minimum=None, maximum=0, summary=0, count=0)) @@ -888,7 +819,7 @@ class Squid(Mixin): proto = 'ipv4' if '.' in match['client_address'] else 'ipv6' # unique clients ips - if self.conf.get('all_time', True): + if self.configuration.get('all_time', True): if address_not_in_pool(pool=self.storage['unique_all_time'], address=match['client_address'], pool_size=self.data['unique_tot_ipv4'] + self.data['unique_tot_ipv6']): @@ -904,9 +835,10 @@ class Squid(Mixin): dimension_id = values['func_dim_id'](match[key]) if values['func_dim_id'] else match[key] if dimension_id not in self.data: dimension = values['func_dim'](match[key]) if values['func_dim'] else dimension_id - self.add_new_dimension(dimension_id=dimension_id, - chart_key=values['chart'], - dimension=dimension) + self.charts[values['chart']].add_dimension([dimension_id, + dimension, + 'incremental']) + self.data[dimension_id] = 0 self.data[dimension_id] += 1 else: self.data['unmatched'] += 1 @@ -940,8 +872,8 @@ class Squid(Mixin): :return: """ if code not in self.data: - self.add_new_dimension(dimension_id=code, - chart_key='squid_code') + self.charts['squid_code'].add_dimension([code, code, 'incremental']) + self.data[code] = 0 self.data[code] += 1 for tag in code.split('_'): @@ -951,9 +883,8 @@ class Squid(Mixin): continue dimension_id = '_'.join(['code_detailed', tag]) if dimension_id not in self.data: - self.add_new_dimension(dimension_id=dimension_id, - dimension=tag, - chart_key=chart_key) + self.charts[chart_key].add_dimension([dimension_id, tag, 'incremental']) + self.data[dimension_id] = 0 self.data[dimension_id] += 1 @@ -1038,14 +969,19 @@ def check_patterns(string, dimension_regex_dict): return result or None -def find_job_name(override_name, name): +def filter_data(raw_data, pre_filter): """ - :param override_name: str: 'name' var from configuration file - :param name: str: 'job_name' from configuration file - :return: str: new job name - We need this for dynamic charts. Actually the same logic as in python.d.plugin. + :param raw_data: + :param pre_filter: + :return: """ - add_to_name = override_name or name - if add_to_name: - return '_'.join(['web_log', re.sub('\s+', '_', add_to_name)]) - return 'web_log' + + if not pre_filter: + return raw_data + filtered = raw_data + for elem in pre_filter: + if elem.description == 'filter_include': + filtered = filter(elem.func, filtered) + elif elem.description == 'filter_exclude': + filtered = filterfalse(elem.func, filtered) + return filtered diff --git a/src/Makefile.am b/src/Makefile.am index feddc326d..1a1d37483 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -137,7 +137,8 @@ netdata_SOURCES = \ sys_devices_system_node.c \ unit_test.c \ unit_test.h \ - url.c url.h \ + url.c \ + url.h \ web_api_old.c \ web_api_old.h \ web_api_v1.c \ @@ -176,7 +177,8 @@ netdata_SOURCES += \ $(NULL) else netdata_SOURCES += \ - ipc.c ipc.h \ + ipc.c \ + ipc.h \ plugin_proc.c \ plugin_proc.h \ plugin_proc_diskspace.c \ @@ -193,6 +195,8 @@ netdata_SOURCES += \ proc_net_rpc_nfsd.c \ proc_net_snmp.c \ proc_net_snmp6.c \ + proc_net_sockstat.c \ + proc_net_sockstat6.c \ proc_net_softnet_stat.c \ proc_net_stat_conntrack.c \ proc_net_stat_synproxy.c \ @@ -258,6 +262,8 @@ cgroup_network_SOURCES = \ inlined.h \ log.c log.h \ procfile.c procfile.h \ + popen.c popen.h \ + signals.c signals.h \ $(NULL) cgroup_network_LDADD = \ diff --git a/src/Makefile.in b/src/Makefile.in index bc73c7125..bf902c21c 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -106,7 +106,8 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) $(am__EXEEXT_3) @FREEBSD_FALSE@@MACOS_TRUE@ $(NULL) @FREEBSD_FALSE@@MACOS_FALSE@am__append_6 = \ -@FREEBSD_FALSE@@MACOS_FALSE@ ipc.c ipc.h \ +@FREEBSD_FALSE@@MACOS_FALSE@ ipc.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ ipc.h \ @FREEBSD_FALSE@@MACOS_FALSE@ plugin_proc.c \ @FREEBSD_FALSE@@MACOS_FALSE@ plugin_proc.h \ @FREEBSD_FALSE@@MACOS_FALSE@ plugin_proc_diskspace.c \ @@ -123,6 +124,8 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) $(am__EXEEXT_3) @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_rpc_nfsd.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp6.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_sockstat.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_sockstat6.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.c \ @@ -179,7 +182,8 @@ apps_plugin_OBJECTS = $(am_apps_plugin_OBJECTS) am__DEPENDENCIES_1 = apps_plugin_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) am_cgroup_network_OBJECTS = cgroup-network.$(OBJEXT) clocks.$(OBJEXT) \ - common.$(OBJEXT) log.$(OBJEXT) procfile.$(OBJEXT) + common.$(OBJEXT) log.$(OBJEXT) procfile.$(OBJEXT) \ + popen.$(OBJEXT) signals.$(OBJEXT) cgroup_network_OBJECTS = $(am_cgroup_network_OBJECTS) cgroup_network_DEPENDENCIES = $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) @@ -224,6 +228,7 @@ am__netdata_SOURCES_DIST = adaptive_resortable_list.c \ proc_loadavg.c proc_meminfo.c proc_net_dev.c \ proc_net_ip_vs_stats.c proc_net_netstat.c proc_net_rpc_nfs.c \ proc_net_rpc_nfsd.c proc_net_snmp.c proc_net_snmp6.c \ + proc_net_sockstat.c proc_net_sockstat6.c \ proc_net_softnet_stat.c proc_net_stat_conntrack.c \ proc_net_stat_synproxy.c proc_spl_kstat_zfs.c proc_stat.c \ proc_sys_kernel_random_entropy_avail.c proc_vmstat.c \ @@ -254,6 +259,8 @@ am__netdata_SOURCES_DIST = adaptive_resortable_list.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_rpc_nfsd.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp6.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_sockstat.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_sockstat6.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.$(OBJEXT) \ @@ -616,6 +623,8 @@ cgroup_network_SOURCES = \ inlined.h \ log.c log.h \ procfile.c procfile.h \ + popen.c popen.h \ + signals.c signals.h \ $(NULL) cgroup_network_LDADD = \ @@ -815,6 +824,8 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_rpc_nfsd.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_snmp.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_snmp6.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_sockstat.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_sockstat6.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_softnet_stat.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_stat_conntrack.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_stat_synproxy.Po@am__quote@ diff --git a/src/adaptive_resortable_list.c b/src/adaptive_resortable_list.c index f74c53eae..add1d8c96 100644 --- a/src/adaptive_resortable_list.c +++ b/src/adaptive_resortable_list.c @@ -2,7 +2,7 @@ // the default processor() of the ARL // can be overwritten at arl_create() -static inline void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst) { +inline void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst) { (void)name; (void)hash; @@ -11,6 +11,15 @@ static inline void arl_callback_str2ull(const char *name, uint32_t hash, const c // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, *d); } +inline void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, const char *value, void *dst) { + (void)name; + (void)hash; + + register kernel_uint_t *d = dst; + *d = str2kernel_uint_t(value); + // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, (unsigned long long)*d); +} + // create a new ARL ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks) { ARL_BASE *base = callocz(1, sizeof(ARL_BASE)); diff --git a/src/adaptive_resortable_list.h b/src/adaptive_resortable_list.h index 609cd0c43..c007fa31e 100644 --- a/src/adaptive_resortable_list.h +++ b/src/adaptive_resortable_list.h @@ -110,6 +110,9 @@ extern int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const ch // begin an ARL iteration extern void arl_begin(ARL_BASE *base); +extern void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst); +extern void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, const char *value, void *dst); + // check a keyword against the ARL // this is to be called for each keyword read from source data // s = the keyword, as collected diff --git a/src/appconfig.c b/src/appconfig.c index 2c7721b8c..40cade818 100644 --- a/src/appconfig.c +++ b/src/appconfig.c @@ -567,6 +567,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) else pri = 2; if(i == pri) { + int loaded = 0; int used = 0; int changed = 0; int count = 0; @@ -574,13 +575,14 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) config_section_wrlock(co); for(cv = co->values; cv ; cv = cv->next) { used += (cv->flags & CONFIG_VALUE_USED)?1:0; + loaded += (cv->flags & CONFIG_VALUE_LOADED)?1:0; changed += (cv->flags & CONFIG_VALUE_CHANGED)?1:0; count++; } config_section_unlock(co); if(!count) continue; - if(only_changed && !changed) continue; + if(only_changed && !changed && !loaded) continue; if(!used) { buffer_sprintf(wb, "\n# section '%s' is not used.", co->name); @@ -594,7 +596,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) if(used && !(cv->flags & CONFIG_VALUE_USED)) { buffer_sprintf(wb, "\n\t# option '%s' is not used.\n", cv->name); } - buffer_sprintf(wb, "\t%s%s = %s\n", ((!(cv->flags & CONFIG_VALUE_CHANGED)) && (cv->flags & CONFIG_VALUE_USED))?"# ":"", cv->name, cv->value); + buffer_sprintf(wb, "\t%s%s = %s\n", ((!(cv->flags & CONFIG_VALUE_LOADED)) && (!(cv->flags & CONFIG_VALUE_CHANGED)) && (cv->flags & CONFIG_VALUE_USED))?"# ":"", cv->name, cv->value); } config_section_unlock(co); } diff --git a/src/apps_plugin.c b/src/apps_plugin.c index c0eb5c083..3ac79777b 100644 --- a/src/apps_plugin.c +++ b/src/apps_plugin.c @@ -1572,14 +1572,22 @@ static inline int read_pid_file_descriptors(struct pid_stat *p, void *ptr) { break; default: /* print protocol number and socket address */ +#if __FreeBSD_version < 1200031 sprintf(fdsname, "socket: other: %d %s %s", fds->kf_sock_protocol, fds->kf_sa_local.__ss_pad1, fds->kf_sa_local.__ss_pad2); +#else + sprintf(fdsname, "socket: other: %d %s %s", fds->kf_sock_protocol, fds->kf_un.kf_sock.kf_sa_local.__ss_pad1, fds->kf_un.kf_sock.kf_sa_local.__ss_pad2); +#endif } break; case KF_TYPE_PIPE: sprintf(fdsname, "pipe: %lu %lu", fds->kf_un.kf_pipe.kf_pipe_addr, fds->kf_un.kf_pipe.kf_pipe_peer); break; case KF_TYPE_PTS: +#if __FreeBSD_version < 1200031 sprintf(fdsname, "other: pts: %u", fds->kf_un.kf_pts.kf_pts_dev); +#else + sprintf(fdsname, "other: pts: %lu", fds->kf_un.kf_pts.kf_pts_dev); +#endif break; case KF_TYPE_SHM: sprintf(fdsname, "other: shm: %s size: %lu", fds->kf_path, fds->kf_un.kf_file.kf_file_size); diff --git a/src/avl.c b/src/avl.c index 0ea119a7c..a2c6911e7 100644 --- a/src/avl.c +++ b/src/avl.c @@ -306,7 +306,10 @@ int avl_walker(avl *node, int (*callback)(void *entry, void *data), void *data) } int avl_traverse(avl_tree *t, int (*callback)(void *entry, void *data), void *data) { - return avl_walker(t->root, callback, data); + if(t->root) + return avl_walker(t->root, callback, data); + else + return 0; } // --------------------------- diff --git a/src/backends.c b/src/backends.c index eed4ab409..df9a1ccbc 100644 --- a/src/backends.c +++ b/src/backends.c @@ -615,18 +615,18 @@ void *backends_main(void *ptr) { chart_backend_reconnects = 0, chart_backend_latency = 0; - RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, backend_update_every, RRDSET_TYPE_LINE); + RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, backend_update_every, RRDSET_TYPE_LINE); rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, backend_update_every, RRDSET_TYPE_AREA); + RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", "backends", NULL, 130610, backend_update_every, RRDSET_TYPE_AREA); rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, backend_update_every, RRDSET_TYPE_LINE); + RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", "backends", NULL, 130630, backend_update_every, RRDSET_TYPE_LINE); rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -640,11 +640,11 @@ void *backends_main(void *ptr) { * * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html * - RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, backend_update_every, RRDSET_TYPE_AREA); + RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, backend_update_every, RRDSET_TYPE_AREA); rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); */ - RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, backend_update_every, RRDSET_TYPE_STACKED); + RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", "backends", NULL, 130630, backend_update_every, RRDSET_TYPE_STACKED); rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); diff --git a/src/cgroup-network.c b/src/cgroup-network.c index 8894d60ee..7b1a02342 100644 --- a/src/cgroup-network.c +++ b/src/cgroup-network.c @@ -1,4 +1,5 @@ #include "common.h" +#include #ifdef HAVE_SETNS #ifndef _GNU_SOURCE @@ -14,6 +15,10 @@ void netdata_cleanup_and_exit(int ret) { exit(ret); } +void health_reload(void) {}; +void rrdhost_save_all(void) {}; + + struct iface { const char *device; uint32_t hash; @@ -269,38 +274,62 @@ pid_t read_pid_from_cgroup(const char *path) { return pid; } -void usage(void) { - fprintf(stderr, "%s [ -p PID | --pid PID | --cgroup /path/to/cgroup ]\n", program_name); - exit(1); -} -int main(int argc, char **argv) { - pid_t pid = 0; +// ---------------------------------------------------------------------------- +// send the result to netdata - program_name = argv[0]; - program_version = VERSION; - error_log_syslog = 0; +struct found_device { + const char *host_device; + const char *guest_device; - if(argc == 2 && (!strcmp(argv[1], "version") || !strcmp(argv[1], "-version") || !strcmp(argv[1], "--version") || !strcmp(argv[1], "-v") || !strcmp(argv[1], "-V"))) { - fprintf(stderr, "cgroup-network %s\n", VERSION); - exit(0); - } + uint32_t host_device_hash; - if(argc != 3) - usage(); + struct found_device *next; +} *detected_devices = NULL; - if(!strcmp(argv[1], "-p") || !strcmp(argv[1], "--pid")) { - pid = atoi(argv[2]); +void add_device(const char *host, const char *guest) { + uint32_t hash = simple_hash(host); + + if(guest && (!*guest || strcmp(host, guest) == 0)) + guest = NULL; + + struct found_device *f; + for(f = detected_devices; f ; f = f->next) { + if(f->host_device_hash == hash && strcmp(host, f->host_device) == 0) { + + if(guest && !f->guest_device) + f->guest_device = strdup(guest); + + return; + } } - else if(!strcmp(argv[1], "--cgroup")) { - pid = read_pid_from_cgroup(argv[2]); + + f = mallocz(sizeof(struct found_device)); + f->host_device = strdupz(host); + f->host_device_hash = hash; + f->guest_device = (guest)?strdupz(guest):NULL; + f->next = detected_devices; + detected_devices = f; +} + +int send_devices(void) { + int found = 0; + + struct found_device *f; + for(f = detected_devices; f ; f = f->next) { + found++; + printf("%s %s\n", f->host_device, (f->guest_device)?f->guest_device:f->host_device); } - else - usage(); - if(pid <= 0) - fatal("Invalid pid %d", (int)pid); + return found; +} + +// ---------------------------------------------------------------------------- +// this function should be called only **ONCE** +// also it has to be the **LAST** to be called +// since it switches namespaces, so after this call, everything is different! +void detect_veth_interfaces(pid_t pid) { struct iface *host, *cgroup, *h, *c; const char *prefix = getenv("NETDATA_HOST_PREFIX"); @@ -321,20 +350,117 @@ int main(int argc, char **argv) { if(!eligible_ifaces(cgroup)) fatal("there are not double-linked cgroup interfaces available."); - int found = 0; for(h = host; h ; h = h->next) { if(iface_is_eligible(h)) { for (c = cgroup; c; c = c->next) { if(iface_is_eligible(c) && h->ifindex == c->iflink && h->iflink == c->ifindex) { - printf("%s %s\n", h->device, c->device); - found++; + add_device(h->device, c->device); } } } } +} - if(!found) - return 1; +// ---------------------------------------------------------------------------- +// call the external helper - return 0; +#define CGROUP_NETWORK_INTERFACE_MAX_LINE 2048 +void call_the_helper(const char *me, pid_t pid, const char *cgroup) { + const char *pluginsdir = getenv("NETDATA_PLUGINS_DIR"); + char *m = NULL; + + if(!pluginsdir || !*pluginsdir) { + m = strdupz(me); + pluginsdir = dirname(m); + } + + if(setresuid(0, 0, 0) == -1) + error("setresuid(0, 0, 0) failed."); + + char buffer[CGROUP_NETWORK_INTERFACE_MAX_LINE + 1]; + if(cgroup) + snprintfz(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, "exec %s/cgroup-network-helper.sh --cgroup '%s'", pluginsdir, cgroup); + else + snprintfz(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, "exec %s/cgroup-network-helper.sh --pid %d", pluginsdir, pid); + + info("running: %s", buffer); + + pid_t cgroup_pid; + FILE *fp = mypopen(buffer, &cgroup_pid); + if(fp) { + char *s; + while((s = fgets(buffer, CGROUP_NETWORK_INTERFACE_MAX_LINE, fp))) { + trim(s); + + if(*s && *s != '\n') { + char *t = s; + while(*t && *t != ' ') t++; + if(*t == ' ') { + *t = '\0'; + t++; + } + + if(!*s || !*t) continue; + add_device(s, t); + } + } + + mypclose(fp, cgroup_pid); + } + else + error("cannot execute cgroup-network helper script: %s", buffer); + + freez(m); +} + + +// ---------------------------------------------------------------------------- +// main + +void usage(void) { + fprintf(stderr, "%s [ -p PID | --pid PID | --cgroup /path/to/cgroup ]\n", program_name); + exit(1); +} + +int main(int argc, char **argv) { + pid_t pid = 0; + + program_name = argv[0]; + program_version = VERSION; + error_log_syslog = 0; + + if(argc == 2 && (!strcmp(argv[1], "version") || !strcmp(argv[1], "-version") || !strcmp(argv[1], "--version") || !strcmp(argv[1], "-v") || !strcmp(argv[1], "-V"))) { + fprintf(stderr, "cgroup-network %s\n", VERSION); + exit(0); + } + + if(argc != 3) + usage(); + + if(!strcmp(argv[1], "-p") || !strcmp(argv[1], "--pid")) { + pid = atoi(argv[2]); + + if(pid <= 0) { + errno = 0; + fatal("Invalid pid %d given", (int) pid); + } + + call_the_helper(argv[0], pid, NULL); + } + else if(!strcmp(argv[1], "--cgroup")) { + pid = read_pid_from_cgroup(argv[2]); + call_the_helper(argv[0], pid, argv[2]); + + if(pid <= 0 && !detected_devices) { + errno = 0; + fatal("Invalid pid %d read from cgroup '%s'", (int) pid, argv[2]); + } + } + else + usage(); + + if(pid > 0) + detect_veth_interfaces(pid); + + return send_devices(); } diff --git a/src/clocks.c b/src/clocks.c index 8f2aa740e..2b1c36e3f 100644 --- a/src/clocks.c +++ b/src/clocks.c @@ -115,8 +115,12 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) if(likely(*hb != 0ULL)) { usec_t dt = now - *hb; *hb = now; - if(unlikely(dt >= tick + tick / 2)) + + if(unlikely(dt >= tick + tick / 2)) { + errno = 0; error("heartbeat missed %llu microseconds", dt - tick); + } + return dt; } else { diff --git a/src/common.c b/src/common.c index 5a953672d..a976e96eb 100644 --- a/src/common.c +++ b/src/common.c @@ -17,6 +17,7 @@ char *netdata_configured_cache_dir = NULL; char *netdata_configured_varlib_dir = NULL; char *netdata_configured_home_dir = NULL; char *netdata_configured_host_prefix = NULL; +char *netdata_configured_timezone = NULL; int enable_ksm = 1; @@ -1119,10 +1120,14 @@ pid_t gettid(void) { #ifdef __FreeBSD__ return (pid_t)pthread_getthreadid_np(); #elif defined(__APPLE__) +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060) uint64_t curthreadid; pthread_threadid_np(NULL, &curthreadid); return (pid_t)curthreadid; -#else +#else /* __MAC_OS_X_VERSION_MIN_REQUIRED */ + return (pid_t)pthread_self; +#endif /* __MAC_OS_X_VERSION_MIN_REQUIRED */ +#else /* __APPLE__*/ return (pid_t)syscall(SYS_gettid); #endif /* __FreeBSD__, __APPLE__*/ } diff --git a/src/common.h b/src/common.h index 51d2bba55..667fe9d76 100644 --- a/src/common.h +++ b/src/common.h @@ -203,10 +203,21 @@ #define NETDATA_OS_TYPE "linux" #endif /* __FreeBSD__, __APPLE__*/ -#include "statistical.h" -#include "socket.h" +typedef enum rrdcalc_status { + RRDCALC_STATUS_REMOVED = -2, + RRDCALC_STATUS_UNDEFINED = -1, + RRDCALC_STATUS_UNINITIALIZED = 0, + RRDCALC_STATUS_CLEAR = 1, + RRDCALC_STATUS_RAISED = 2, + RRDCALC_STATUS_WARNING = 3, + RRDCALC_STATUS_CRITICAL = 4 +} RRDCALC_STATUS; + #include "eval.h" #include "health.h" + +#include "statistical.h" +#include "socket.h" #include "rrd.h" #include "plugin_tc.h" #include "plugins_d.h" @@ -232,12 +243,14 @@ extern char *netdata_configured_hostname; extern char *netdata_configured_config_dir; extern char *netdata_configured_log_dir; +extern char *netdata_configured_plugins_dir_base; extern char *netdata_configured_plugins_dir; extern char *netdata_configured_web_dir; extern char *netdata_configured_cache_dir; extern char *netdata_configured_varlib_dir; extern char *netdata_configured_home_dir; extern char *netdata_configured_host_prefix; +extern char *netdata_configured_timezone; extern void netdata_fix_chart_id(char *s); extern void netdata_fix_chart_name(char *s); diff --git a/src/eval.c b/src/eval.c index 9248109b0..84369f6d4 100644 --- a/src/eval.c +++ b/src/eval.c @@ -166,7 +166,7 @@ static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABL } if(exp->rrdcalc && health_variable_lookup(v->name, v->hash, exp->rrdcalc, &n)) { - buffer_sprintf(exp->error_msg, "[ $%s = ", v->name); + buffer_sprintf(exp->error_msg, "[ ${%s} = ", v->name); print_parsed_as_constant(exp->error_msg, n); buffer_strcat(exp->error_msg, " ] "); return n; @@ -232,7 +232,7 @@ calculated_number eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { if(isinf(n1) && isinf(n2)) return 1; if(isnan(n1) || isnan(n2)) return 0; if(isinf(n1) || isinf(n2)) return 0; - return n1 == n2; + return calculated_number_equal(n1, n2); } calculated_number eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { return !eval_equal(exp, op, error); @@ -355,7 +355,7 @@ static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, i static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *error) { (void)error; - buffer_sprintf(out, "$%s", v->name); + buffer_sprintf(out, "${%s}", v->name); } static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) { @@ -703,17 +703,31 @@ static inline int parse_variable(const char **string, char *buffer, size_t len) const char *s = *string; // $ - if(s[0] == '$') { + if(*s == '$') { size_t i = 0; s++; - while(*s && !isvariableterm(*s) && i < len) - buffer[i++] = *s++; + if(*s == '{') { + // ${variable_name} + + s++; + while (*s && *s != '}' && i < len) + buffer[i++] = *s++; + + if(*s == '}') + s++; + } + else { + // $variable_name + + while (*s && !isvariableterm(*s) && i < len) + buffer[i++] = *s++; + } buffer[i] = '\0'; - if(buffer[0]) { - *string = &s[0]; + if (buffer[0]) { + *string = s; return 1; } } diff --git a/src/eval.h b/src/eval.h index d68b9af47..cd271148c 100644 --- a/src/eval.h +++ b/src/eval.h @@ -14,7 +14,7 @@ typedef struct eval_expression { const char *source; const char *parsed_as; - int *status; + RRDCALC_STATUS *status; calculated_number *this; time_t *after; time_t *before; diff --git a/src/freebsd_devstat.c b/src/freebsd_devstat.c index 5b0687d5b..2ed64ad49 100644 --- a/src/freebsd_devstat.c +++ b/src/freebsd_devstat.c @@ -27,10 +27,15 @@ struct disk { struct prev_dstat { collected_number bytes_read; collected_number bytes_write; + collected_number bytes_free; collected_number operations_read; collected_number operations_write; + collected_number operations_other; + collected_number operations_free; collected_number duration_read_ms; collected_number duration_write_ms; + collected_number duration_other_ms; + collected_number duration_free_ms; collected_number busy_time_ms; } prev_dstat; @@ -39,10 +44,13 @@ struct disk { RRDSET *st_io; RRDDIM *rd_io_in; RRDDIM *rd_io_out; + RRDDIM *rd_io_free; RRDSET *st_ops; RRDDIM *rd_ops_in; RRDDIM *rd_ops_out; + RRDDIM *rd_ops_other; + RRDDIM *rd_ops_free; RRDSET *st_qops; RRDDIM *rd_qops; @@ -53,14 +61,19 @@ struct disk { RRDSET *st_iotime; RRDDIM *rd_iotime_in; RRDDIM *rd_iotime_out; + RRDDIM *rd_iotime_other; + RRDDIM *rd_iotime_free; RRDSET *st_await; RRDDIM *rd_await_in; RRDDIM *rd_await_out; + RRDDIM *rd_await_other; + RRDDIM *rd_await_free; RRDSET *st_avagsz; RRDDIM *rd_avagsz_in; RRDDIM *rd_avagsz_out; + RRDDIM *rd_avagsz_free; RRDSET *st_svctm; RRDDIM *rd_svctm; @@ -244,7 +257,8 @@ int do_kern_devstat(int update_every, usec_t dt) { for (i = 0; i < numdevs; i++) { if (likely(do_system_io)) { - if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { + if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || + ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { total_disk_kbytes_read += dstat[i].bytes[DEVSTAT_READ] / KILO_FACTOR; total_disk_kbytes_write += dstat[i].bytes[DEVSTAT_WRITE] / KILO_FACTOR; } @@ -254,11 +268,14 @@ int do_kern_devstat(int update_every, usec_t dt) { if ((dstat[i].device_type & DEVSTAT_TYPE_PASS) == DEVSTAT_TYPE_PASS) continue; - if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { + if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || + ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { char disk[DEVSTAT_NAME_LEN + MAX_INT_DIGITS + 1]; struct cur_dstat { collected_number duration_read_ms; collected_number duration_write_ms; + collected_number duration_other_ms; + collected_number duration_free_ms; collected_number busy_time_ms; } cur_dstat; @@ -299,12 +316,19 @@ int do_kern_devstat(int update_every, usec_t dt) { dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ]; dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE]; + dm->prev_dstat.bytes_free = dstat[i].bytes[DEVSTAT_FREE]; dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ]; dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE]; + dm->prev_dstat.operations_other = dstat[i].operations[DEVSTAT_NO_DATA]; + dm->prev_dstat.operations_free = dstat[i].operations[DEVSTAT_FREE]; dm->prev_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000 + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; dm->prev_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000 - + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + + dstat[i].duration[DEVSTAT_WRITE].frac * BINTIME_SCALE; + dm->prev_dstat.duration_other_ms = dstat[i].duration[DEVSTAT_NO_DATA].sec * 1000 + + dstat[i].duration[DEVSTAT_NO_DATA].frac * BINTIME_SCALE; + dm->prev_dstat.duration_free_ms = dstat[i].duration[DEVSTAT_FREE].sec * 1000 + + dstat[i].duration[DEVSTAT_FREE].frac * BINTIME_SCALE; dm->prev_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE; } @@ -312,13 +336,20 @@ int do_kern_devstat(int update_every, usec_t dt) { cur_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000 + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; cur_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000 - + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + + dstat[i].duration[DEVSTAT_WRITE].frac * BINTIME_SCALE; + cur_dstat.duration_other_ms = dstat[i].duration[DEVSTAT_NO_DATA].sec * 1000 + + dstat[i].duration[DEVSTAT_NO_DATA].frac * BINTIME_SCALE; + cur_dstat.duration_free_ms = dstat[i].duration[DEVSTAT_FREE].sec * 1000 + + dstat[i].duration[DEVSTAT_FREE].frac * BINTIME_SCALE; + cur_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE; // -------------------------------------------------------------------- if(dm->do_io == CONFIG_BOOLEAN_YES || (dm->do_io == CONFIG_BOOLEAN_AUTO && - (dstat[i].bytes[DEVSTAT_READ] || dstat[i].bytes[DEVSTAT_WRITE]))) { + (dstat[i].bytes[DEVSTAT_READ] || + dstat[i].bytes[DEVSTAT_WRITE] || + dstat[i].bytes[DEVSTAT_FREE]))) { if (unlikely(!dm->st_io)) { dm->st_io = rrdset_create_localhost("disk", disk, @@ -327,6 +358,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.io", "Disk I/O Bandwidth", "kilobytes/s", + "freebsd", + "devstat", 2000, update_every, RRDSET_TYPE_AREA @@ -336,18 +369,24 @@ int do_kern_devstat(int update_every, usec_t dt) { RRD_ALGORITHM_INCREMENTAL); dm->rd_io_out = rrddim_add(dm->st_io, "writes", NULL, -1, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); + dm->rd_io_free = rrddim_add(dm->st_io, "frees", NULL, -1, KILO_FACTOR, + RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(dm->st_io); - rrddim_set_by_pointer(dm->st_io, dm->rd_io_in, dstat[i].bytes[DEVSTAT_READ]); - rrddim_set_by_pointer(dm->st_io, dm->rd_io_out, dstat[i].bytes[DEVSTAT_WRITE]); + rrddim_set_by_pointer(dm->st_io, dm->rd_io_in, dstat[i].bytes[DEVSTAT_READ]); + rrddim_set_by_pointer(dm->st_io, dm->rd_io_out, dstat[i].bytes[DEVSTAT_WRITE]); + rrddim_set_by_pointer(dm->st_io, dm->rd_io_free, dstat[i].bytes[DEVSTAT_FREE]); rrdset_done(dm->st_io); } // -------------------------------------------------------------------- if(dm->do_ops == CONFIG_BOOLEAN_YES || (dm->do_ops == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + (dstat[i].operations[DEVSTAT_READ] || + dstat[i].operations[DEVSTAT_WRITE] || + dstat[i].operations[DEVSTAT_NO_DATA] || + dstat[i].operations[DEVSTAT_FREE]))) { if (unlikely(!dm->st_ops)) { dm->st_ops = rrdset_create_localhost("disk_ops", disk, @@ -356,6 +395,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.ops", "Disk Completed I/O Operations", "operations/s", + "freebsd", + "devstat", 2001, update_every, RRDSET_TYPE_LINE @@ -363,14 +404,21 @@ int do_kern_devstat(int update_every, usec_t dt) { rrdset_flag_set(dm->st_ops, RRDSET_FLAG_DETAIL); - dm->rd_ops_in = rrddim_add(dm->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - dm->rd_ops_out = rrddim_add(dm->st_ops, "writes", NULL, -1, 1, - RRD_ALGORITHM_INCREMENTAL); + dm->rd_ops_in = rrddim_add(dm->st_ops, "reads", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_ops_out = rrddim_add(dm->st_ops, "writes", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_ops_other = rrddim_add(dm->st_ops, "other", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_ops_free = rrddim_add(dm->st_ops, "frees", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(dm->st_ops); - rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_in, dstat[i].operations[DEVSTAT_READ]); - rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_out, dstat[i].operations[DEVSTAT_WRITE]); + rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_in, dstat[i].operations[DEVSTAT_READ]); + rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_out, dstat[i].operations[DEVSTAT_WRITE]); + rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_other, dstat[i].operations[DEVSTAT_NO_DATA]); + rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_free, dstat[i].operations[DEVSTAT_FREE]); rrdset_done(dm->st_ops); } @@ -386,6 +434,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.qops", "Disk Current I/O Operations", "operations", + "freebsd", + "devstat", 2002, update_every, RRDSET_TYPE_LINE @@ -413,6 +463,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.util", "Disk Utilization Time", "% of time working", + "freebsd", + "devstat", 2004, update_every, RRDSET_TYPE_AREA @@ -432,7 +484,10 @@ int do_kern_devstat(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(dm->do_iotime == CONFIG_BOOLEAN_YES || (dm->do_iotime == CONFIG_BOOLEAN_AUTO && - (cur_dstat.duration_read_ms || cur_dstat.duration_write_ms))) { + (cur_dstat.duration_read_ms || + cur_dstat.duration_write_ms || + cur_dstat.duration_other_ms || + cur_dstat.duration_free_ms))) { if (unlikely(!dm->st_iotime)) { dm->st_iotime = rrdset_create_localhost("disk_iotime", disk, @@ -441,6 +496,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.iotime", "Disk Total I/O Time", "milliseconds/s", + "freebsd", + "devstat", 2022, update_every, RRDSET_TYPE_LINE @@ -448,15 +505,21 @@ int do_kern_devstat(int update_every, usec_t dt) { rrdset_flag_set(dm->st_iotime, RRDSET_FLAG_DETAIL); - dm->rd_iotime_in = rrddim_add(dm->st_iotime, "reads", NULL, 1, 1, - RRD_ALGORITHM_INCREMENTAL); - dm->rd_iotime_out = rrddim_add(dm->st_iotime, "writes", NULL, -1, 1, - RRD_ALGORITHM_INCREMENTAL); + dm->rd_iotime_in = rrddim_add(dm->st_iotime, "reads", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_iotime_out = rrddim_add(dm->st_iotime, "writes", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_iotime_other = rrddim_add(dm->st_iotime, "other", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_iotime_free = rrddim_add(dm->st_iotime, "frees", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(dm->st_iotime); - rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_in, cur_dstat.duration_read_ms); - rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_out, cur_dstat.duration_write_ms); + rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_in, cur_dstat.duration_read_ms); + rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_out, cur_dstat.duration_write_ms); + rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_other, cur_dstat.duration_other_ms); + rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_free, cur_dstat.duration_free_ms); rrdset_done(dm->st_iotime); } @@ -469,7 +532,10 @@ int do_kern_devstat(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(dm->do_await == CONFIG_BOOLEAN_YES || (dm->do_await == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + (dstat[i].operations[DEVSTAT_READ] || + dstat[i].operations[DEVSTAT_WRITE] || + dstat[i].operations[DEVSTAT_NO_DATA] || + dstat[i].operations[DEVSTAT_FREE]))) { if (unlikely(!dm->st_await)) { dm->st_await = rrdset_create_localhost("disk_await", disk, @@ -478,6 +544,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.await", "Average Completed I/O Operation Time", "ms per operation", + "freebsd", + "devstat", 2005, update_every, RRDSET_TYPE_LINE @@ -485,9 +553,13 @@ int do_kern_devstat(int update_every, usec_t dt) { rrdset_flag_set(dm->st_await, RRDSET_FLAG_DETAIL); - dm->rd_await_in = rrddim_add(dm->st_await, "reads", NULL, 1, 1, + dm->rd_await_in = rrddim_add(dm->st_await, "reads", NULL, 1, 1, + RRD_ALGORITHM_ABSOLUTE); + dm->rd_await_out = rrddim_add(dm->st_await, "writes", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); - dm->rd_await_out = rrddim_add(dm->st_await, "writes", NULL, -1, 1, + dm->rd_await_other = rrddim_add(dm->st_await, "other", NULL, 1, 1, + RRD_ALGORITHM_ABSOLUTE); + dm->rd_await_free = rrddim_add(dm->st_await, "frees", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(dm->st_await); @@ -506,13 +578,29 @@ int do_kern_devstat(int update_every, usec_t dt) { (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write) : 0); + rrddim_set_by_pointer(dm->st_await, dm->rd_await_other, + (dstat[i].operations[DEVSTAT_NO_DATA] - + dm->prev_dstat.operations_other) ? + (cur_dstat.duration_other_ms - dm->prev_dstat.duration_other_ms) / + (dstat[i].operations[DEVSTAT_NO_DATA] - + dm->prev_dstat.operations_other) : + 0); + rrddim_set_by_pointer(dm->st_await, dm->rd_await_free, + (dstat[i].operations[DEVSTAT_FREE] - + dm->prev_dstat.operations_free) ? + (cur_dstat.duration_free_ms - dm->prev_dstat.duration_free_ms) / + (dstat[i].operations[DEVSTAT_FREE] - + dm->prev_dstat.operations_free) : + 0); rrdset_done(dm->st_await); } // -------------------------------------------------------------------- if(dm->do_avagsz == CONFIG_BOOLEAN_YES || (dm->do_avagsz == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + (dstat[i].operations[DEVSTAT_READ] || + dstat[i].operations[DEVSTAT_WRITE] || + dstat[i].operations[DEVSTAT_FREE]))) { if (unlikely(!dm->st_avagsz)) { dm->st_avagsz = rrdset_create_localhost("disk_avgsz", disk, @@ -521,6 +609,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.avgsz", "Average Completed I/O Operation Bandwidth", "kilobytes per operation", + "freebsd", + "devstat", 2006, update_every, RRDSET_TYPE_AREA @@ -528,10 +618,12 @@ int do_kern_devstat(int update_every, usec_t dt) { rrdset_flag_set(dm->st_avagsz, RRDSET_FLAG_DETAIL); - dm->rd_avagsz_in = rrddim_add(dm->st_avagsz, "reads", NULL, 1, KILO_FACTOR, - RRD_ALGORITHM_ABSOLUTE); - dm->rd_avagsz_out = rrddim_add(dm->st_avagsz, "writes", NULL, -1, KILO_FACTOR, - RRD_ALGORITHM_ABSOLUTE); + dm->rd_avagsz_in = rrddim_add(dm->st_avagsz, "reads", NULL, 1, KILO_FACTOR, + RRD_ALGORITHM_ABSOLUTE); + dm->rd_avagsz_out = rrddim_add(dm->st_avagsz, "writes", NULL, -1, KILO_FACTOR, + RRD_ALGORITHM_ABSOLUTE); + dm->rd_avagsz_free = rrddim_add(dm->st_avagsz, "frees", NULL, -1, KILO_FACTOR, + RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(dm->st_avagsz); @@ -549,13 +641,23 @@ int do_kern_devstat(int update_every, usec_t dt) { (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write) : 0); + rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_free, + (dstat[i].operations[DEVSTAT_FREE] - + dm->prev_dstat.operations_free) ? + (dstat[i].bytes[DEVSTAT_FREE] - dm->prev_dstat.bytes_free) / + (dstat[i].operations[DEVSTAT_FREE] - + dm->prev_dstat.operations_free) : + 0); rrdset_done(dm->st_avagsz); } // -------------------------------------------------------------------- if(dm->do_svctm == CONFIG_BOOLEAN_YES || (dm->do_svctm == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + (dstat[i].operations[DEVSTAT_READ] || + dstat[i].operations[DEVSTAT_WRITE] || + dstat[i].operations[DEVSTAT_NO_DATA] || + dstat[i].operations[DEVSTAT_FREE]))) { if (unlikely(!dm->st_svctm)) { dm->st_svctm = rrdset_create_localhost("disk_svctm", disk, @@ -564,6 +666,8 @@ int do_kern_devstat(int update_every, usec_t dt) { "disk.svctm", "Average Service Time", "ms per operation", + "freebsd", + "devstat", 2007, update_every, RRDSET_TYPE_LINE @@ -578,10 +682,14 @@ int do_kern_devstat(int update_every, usec_t dt) { rrddim_set_by_pointer(dm->st_svctm, dm->rd_svctm, ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) + - (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) ? + (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write) + + (dstat[i].operations[DEVSTAT_NO_DATA] - dm->prev_dstat.operations_other) + + (dstat[i].operations[DEVSTAT_FREE] - dm->prev_dstat.operations_free)) ? (cur_dstat.busy_time_ms - dm->prev_dstat.busy_time_ms) / ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) + - (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) : + (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write) + + (dstat[i].operations[DEVSTAT_NO_DATA] - dm->prev_dstat.operations_other) + + (dstat[i].operations[DEVSTAT_FREE] - dm->prev_dstat.operations_free)) : 0); rrdset_done(dm->st_svctm); } @@ -590,10 +698,15 @@ int do_kern_devstat(int update_every, usec_t dt) { dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ]; dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE]; + dm->prev_dstat.bytes_free = dstat[i].bytes[DEVSTAT_FREE]; dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ]; dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE]; + dm->prev_dstat.operations_other = dstat[i].operations[DEVSTAT_NO_DATA]; + dm->prev_dstat.operations_free = dstat[i].operations[DEVSTAT_FREE]; dm->prev_dstat.duration_read_ms = cur_dstat.duration_read_ms; dm->prev_dstat.duration_write_ms = cur_dstat.duration_write_ms; + dm->prev_dstat.duration_other_ms = cur_dstat.duration_other_ms; + dm->prev_dstat.duration_free_ms = cur_dstat.duration_free_ms; dm->prev_dstat.busy_time_ms = cur_dstat.busy_time_ms; } } @@ -613,6 +726,8 @@ int do_kern_devstat(int update_every, usec_t dt) { NULL, "Disk I/O", "kilobytes/s", + "freebsd", + "devstat", 150, update_every, RRDSET_TYPE_AREA diff --git a/src/freebsd_getifaddrs.c b/src/freebsd_getifaddrs.c index 1a4448bd3..94c0a6a4b 100644 --- a/src/freebsd_getifaddrs.c +++ b/src/freebsd_getifaddrs.c @@ -228,6 +228,8 @@ int do_getifaddrs(int update_every, usec_t dt) { NULL, "IPv4 Bandwidth", "kilobits/s", + "freebsd", + "getifaddrs", 500, update_every, RRDSET_TYPE_AREA @@ -265,6 +267,8 @@ int do_getifaddrs(int update_every, usec_t dt) { NULL, "IPv6 Bandwidth", "kilobits/s", + "freebsd", + "getifaddrs", 500, update_every, RRDSET_TYPE_AREA @@ -333,6 +337,8 @@ int do_getifaddrs(int update_every, usec_t dt) { "net.net", "Bandwidth", "kilobits/s", + "freebsd", + "getifaddrs", 7000, update_every, RRDSET_TYPE_AREA @@ -360,6 +366,8 @@ int do_getifaddrs(int update_every, usec_t dt) { "net.packets", "Packets", "packets/s", + "freebsd", + "getifaddrs", 7001, update_every, RRDSET_TYPE_LINE @@ -397,6 +405,8 @@ int do_getifaddrs(int update_every, usec_t dt) { "net.errors", "Interface Errors", "errors/s", + "freebsd", + "getifaddrs", 7002, update_every, RRDSET_TYPE_LINE @@ -429,6 +439,8 @@ int do_getifaddrs(int update_every, usec_t dt) { "net.drops", "Interface Drops", "drops/s", + "freebsd", + "getifaddrs", 7003, update_every, RRDSET_TYPE_LINE @@ -462,6 +474,8 @@ int do_getifaddrs(int update_every, usec_t dt) { "net.events", "Network Interface Events", "events/s", + "freebsd", + "getifaddrs", 7006, update_every, RRDSET_TYPE_LINE diff --git a/src/freebsd_getmntinfo.c b/src/freebsd_getmntinfo.c index e7ca56b57..66be53315 100644 --- a/src/freebsd_getmntinfo.c +++ b/src/freebsd_getmntinfo.c @@ -223,6 +223,8 @@ int do_getmntinfo(int update_every, usec_t dt) { "disk.space", title, "GB", + "freebsd", + "getmntinfo", 2023, update_every, RRDSET_TYPE_STACKED @@ -260,6 +262,8 @@ int do_getmntinfo(int update_every, usec_t dt) { "disk.inodes", title, "Inodes", + "freebsd", + "getmntinfo", 2024, update_every, RRDSET_TYPE_STACKED diff --git a/src/freebsd_ipfw.c b/src/freebsd_ipfw.c index b89650a04..81264b3f3 100644 --- a/src/freebsd_ipfw.c +++ b/src/freebsd_ipfw.c @@ -162,6 +162,8 @@ int do_ipfw(int update_every, usec_t dt) { NULL, "Memory allocated by rules", "bytes", + "freebsd", + "ipfw", 3005, update_every, RRDSET_TYPE_STACKED @@ -193,6 +195,8 @@ int do_ipfw(int update_every, usec_t dt) { NULL, "Packets", "packets/s", + "freebsd", + "ipfw", 3001, update_every, RRDSET_TYPE_STACKED @@ -208,6 +212,8 @@ int do_ipfw(int update_every, usec_t dt) { NULL, "Bytes", "bytes/s", + "freebsd", + "ipfw", 3002, update_every, RRDSET_TYPE_STACKED @@ -310,6 +316,8 @@ int do_ipfw(int update_every, usec_t dt) { NULL, "Active rules", "rules", + "freebsd", + "ipfw", 3003, update_every, RRDSET_TYPE_STACKED @@ -325,6 +333,8 @@ int do_ipfw(int update_every, usec_t dt) { NULL, "Expired rules", "rules", + "freebsd", + "ipfw", 3004, update_every, RRDSET_TYPE_STACKED diff --git a/src/freebsd_kstat_zfs.c b/src/freebsd_kstat_zfs.c index 17642994e..1bd48d4bf 100644 --- a/src/freebsd_kstat_zfs.c +++ b/src/freebsd_kstat_zfs.c @@ -1,9 +1,11 @@ #include "common.h" #include "zfs_common.h" -struct arcstats arcstats = { 0 }; +extern struct arcstats arcstats; +// -------------------------------------------------------------------------------------------------------------------- // kstat.zfs.misc.arcstats + int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt) { (void)dt; @@ -103,15 +105,15 @@ int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt) { int arc_sys_free[5]; } mibs; - l2exist = -1; + arcstats.l2exist = -1; if(unlikely(sysctlbyname("kstat.zfs.misc.arcstats.l2_size", &l2_size, &uint64_t_size, NULL, 0))) return 0; if(likely(l2_size)) - l2exist = 1; + arcstats.l2exist = 1; else - l2exist = 0; + arcstats.l2exist = 0; GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hits", mibs.hits, arcstats.hits); GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.misses", mibs.misses, arcstats.misses); @@ -205,8 +207,92 @@ int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt) { // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_need_free", mibs.arc_need_free, arcstats.arc_need_free); // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_sys_free", mibs.arc_sys_free, arcstats.arc_sys_free); - generate_charts_arcstats(update_every); - generate_charts_arc_summary(update_every); + generate_charts_arcstats("freebsd", update_every); + generate_charts_arc_summary("freebsd", update_every); + + return 0; +} + +// -------------------------------------------------------------------------------------------------------------------- +// kstat.zfs.misc.zio_trim + +int do_kstat_zfs_misc_zio_trim(int update_every, usec_t dt) { + (void)dt; + static int mib_bytes[5] = {0, 0, 0, 0, 0}, mib_success[5] = {0, 0, 0, 0, 0}, + mib_failed[5] = {0, 0, 0, 0, 0}, mib_unsupported[5] = {0, 0, 0, 0, 0}; + uint64_t bytes, success, failed, unsupported; + + if (unlikely(GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.bytes", mib_bytes, bytes) || + GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.success", mib_success, success) || + GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.failed", mib_failed, failed) || + GETSYSCTL_SIMPLE("kstat.zfs.misc.zio_trim.unsupported", mib_unsupported, unsupported))) { + error("DISABLED: zfs.trim_bytes chart"); + error("DISABLED: zfs.trim_success chart"); + error("DISABLED: kstat.zfs.misc.zio_trim module"); + return 1; + } else { + + // -------------------------------------------------------------------- + + static RRDSET *st_bytes = NULL; + static RRDDIM *rd_bytes = NULL; + + if (unlikely(!st_bytes)) { + st_bytes = rrdset_create_localhost( + "zfs", + "trim_bytes", + NULL, + "trim", + NULL, + "Successfully TRIMmed bytes", + "bytes", + "freebsd", + "zfs", + 2320, + update_every, + RRDSET_TYPE_LINE + ); + + rd_bytes = rrddim_add(st_bytes, "TRIMmed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st_bytes); + + rrddim_set_by_pointer(st_bytes, rd_bytes, bytes); + rrdset_done(st_bytes); + + // -------------------------------------------------------------------- + + static RRDSET *st_requests = NULL; + static RRDDIM *rd_successful = NULL, *rd_failed = NULL, *rd_unsupported = NULL; + + if (unlikely(!st_requests)) { + st_requests = rrdset_create_localhost( + "zfs", + "trim_requests", + NULL, + "trim", + NULL, + "TRIM requests", + "requests", + "freebsd", + "zfs", + 2321, + update_every, + RRDSET_TYPE_STACKED + ); + + rd_successful = rrddim_add(st_requests, "successful", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_failed = rrddim_add(st_requests, "failed", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_unsupported = rrddim_add(st_requests, "unsupported", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st_requests); + + rrddim_set_by_pointer(st_requests, rd_successful, success); + rrddim_set_by_pointer(st_requests, rd_failed, failed); + rrddim_set_by_pointer(st_requests, rd_unsupported, unsupported); + rrdset_done(st_requests); + + } return 0; } \ No newline at end of file diff --git a/src/freebsd_sysctl.c b/src/freebsd_sysctl.c index a87b872d9..9f5615df8 100644 --- a/src/freebsd_sysctl.c +++ b/src/freebsd_sysctl.c @@ -27,6 +27,62 @@ int system_pagesize = PAGE_SIZE; int number_of_cpus = 1; +#if __FreeBSD_version >= 1200029 +struct __vmmeter { + uint64_t v_swtch; + uint64_t v_trap; + uint64_t v_syscall; + uint64_t v_intr; + uint64_t v_soft; + uint64_t v_vm_faults; + uint64_t v_io_faults; + uint64_t v_cow_faults; + uint64_t v_cow_optim; + uint64_t v_zfod; + uint64_t v_ozfod; + uint64_t v_swapin; + uint64_t v_swapout; + uint64_t v_swappgsin; + uint64_t v_swappgsout; + uint64_t v_vnodein; + uint64_t v_vnodeout; + uint64_t v_vnodepgsin; + uint64_t v_vnodepgsout; + uint64_t v_intrans; + uint64_t v_reactivated; + uint64_t v_pdwakeups; + uint64_t v_pdpages; + uint64_t v_pdshortfalls; + uint64_t v_dfree; + uint64_t v_pfree; + uint64_t v_tfree; + uint64_t v_forks; + uint64_t v_vforks; + uint64_t v_rforks; + uint64_t v_kthreads; + uint64_t v_forkpages; + uint64_t v_vforkpages; + uint64_t v_rforkpages; + uint64_t v_kthreadpages; + u_int v_page_size; + u_int v_page_count; + u_int v_free_reserved; + u_int v_free_target; + u_int v_free_min; + u_int v_free_count; + u_int v_wire_count; + u_int v_active_count; + u_int v_inactive_target; + u_int v_inactive_count; + u_int v_laundry_count; + u_int v_pageout_free_min; + u_int v_interrupt_free_min; + u_int v_free_severe; +}; +typedef struct __vmmeter vmmeter_t; +#else +typedef struct vmmeter vmmeter_t; +#endif // -------------------------------------------------------------------------------------------------------------------- // FreeBSD plugin initialization @@ -77,16 +133,19 @@ int do_vm_loadavg(int update_every, usec_t dt){ static RRDDIM *rd_load1 = NULL, *rd_load2 = NULL, *rd_load3 = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "load", - NULL, - "load", - NULL, - "System Load Average", - "load", - 100, - (update_every < MIN_LOADAVG_UPDATE_EVERY) ? - MIN_LOADAVG_UPDATE_EVERY : update_every, RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "load", + NULL, + "load", + NULL, + "System Load Average", + "load", + "freebsd", + "vm.loadavg", + 100, + (update_every < MIN_LOADAVG_UPDATE_EVERY) ? + MIN_LOADAVG_UPDATE_EVERY : update_every, RRDSET_TYPE_LINE ); rd_load1 = rrddim_add(st, "load1", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); rd_load2 = rrddim_add(st, "load5", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); @@ -143,16 +202,19 @@ int do_vm_vmtotal(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "active_processes", - NULL, - "processes", - NULL, - "System Active Processes", - "processes", - 750, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "active_processes", + NULL, + "processes", + NULL, + "System Active Processes", + "processes", + "freebsd", + "vm.vmtotal", + 750, + update_every, + RRDSET_TYPE_LINE ); rd = rrddim_add(st, "active", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -169,16 +231,19 @@ int do_vm_vmtotal(int update_every, usec_t dt) { static RRDDIM *rd_running = NULL, *rd_blocked = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "processes", - NULL, - "processes", - NULL, - "System Processes", - "processes", - 600, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "processes", + NULL, + "processes", + NULL, + "System Processes", + "processes", + "freebsd", + "vm.vmtotal", + 600, + update_every, + RRDSET_TYPE_LINE ); rd_running = rrddim_add(st, "running", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -198,16 +263,19 @@ int do_vm_vmtotal(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("mem", - "committed", - NULL, - "system", - NULL, - "Committed (Allocated) Memory", - "MB", - 5000, - update_every, - RRDSET_TYPE_AREA + st = rrdset_create_localhost( + "mem", + "committed", + NULL, + "system", + NULL, + "Committed (Allocated) Memory", + "MB", + "freebsd", + "vm.vmtotal", + 5000, + update_every, + RRDSET_TYPE_AREA ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -254,15 +322,18 @@ int do_kern_cp_time(int update_every, usec_t dt) { static RRDDIM *rd_nice = NULL, *rd_system = NULL, *rd_user = NULL, *rd_interrupt = NULL, *rd_idle = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "cpu", - NULL, - "cpu", - "system.cpu", - "Total CPU utilization", - "percentage", - 100, update_every, - RRDSET_TYPE_STACKED + st = rrdset_create_localhost( + "system", + "cpu", + NULL, + "cpu", + "system.cpu", + "Total CPU utilization", + "percentage", + "freebsd", + "kern.cp_time", + 100, update_every, + RRDSET_TYPE_STACKED ); rd_nice = rrddim_add(st, "nice", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); @@ -332,16 +403,19 @@ int do_kern_cp_times(int update_every, usec_t dt) { for (i = 0; i < number_of_cpus; i++) { if (unlikely(!all_cpu_charts[i].st)) { snprintfz(all_cpu_charts[i].cpuid, MAX_INT_DIGITS, "cpu%d", i); - all_cpu_charts[i].st = rrdset_create_localhost("cpu", - all_cpu_charts[i].cpuid, - NULL, - "utilization", - "cpu.cpu", - "Core utilization", - "percentage", - 1000, - update_every, - RRDSET_TYPE_STACKED + all_cpu_charts[i].st = rrdset_create_localhost( + "cpu", + all_cpu_charts[i].cpuid, + NULL, + "utilization", + "cpu.cpu", + "Core utilization", + "percentage", + "freebsd", + "kern.cp_times", + 1000, + update_every, + RRDSET_TYPE_STACKED ); all_cpu_charts[i].rd_nice = rrddim_add(all_cpu_charts[i].st, "nice", NULL, 1, 1, @@ -413,18 +487,22 @@ int do_dev_cpu_temperature(int update_every, usec_t dt) { } if (unlikely(!st)) { - st = rrdset_create_localhost("cpu", - "temperature", - NULL, - "temperature", - "cpu.temperatute", - "Core temperature", - "degree", - 1050, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "cpu", + "temperature", + NULL, + "temperature", + "cpu.temperatute", + "Core temperature", + "Celsius", + "freebsd", + "dev.cpu.temperature", + 1050, + update_every, + RRDSET_TYPE_LINE ); - } else rrdset_next(st); + } + else rrdset_next(st); for (i = 0; i < number_of_cpus; i++) { if (unlikely(!rd_pcpu_temperature[i])) { @@ -442,6 +520,52 @@ int do_dev_cpu_temperature(int update_every, usec_t dt) { return 0; } +// -------------------------------------------------------------------------------------------------------------------- +// dev.cpu.0.freq + +int do_dev_cpu_0_freq(int update_every, usec_t dt) { + (void)dt; + static int mib[4] = {0, 0, 0, 0}; + int cpufreq; + + if (unlikely(GETSYSCTL_SIMPLE("dev.cpu.0.freq", mib, cpufreq))) { + error("DISABLED: cpu.scaling_cur_freq chart"); + error("DISABLED: dev.cpu.0.freq module"); + return 1; + } else { + + // -------------------------------------------------------------------- + + static RRDSET *st = NULL; + static RRDDIM *rd = NULL; + + if (unlikely(!st)) { + st = rrdset_create_localhost( + "cpu", + "scaling_cur_freq", + NULL, + "cpufreq", + NULL, + "Current CPU Scaling Frequency", + "MHz", + "freebsd", + "dev.cpu.0.freq", + 5003, + update_every, + RRDSET_TYPE_LINE + ); + + rd = rrddim_add(st, "frequency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd, cpufreq); + rrdset_done(st); + } + + return 0; +} + // -------------------------------------------------------------------------------------------------------------------- // hw.intrcnt @@ -480,16 +604,19 @@ int do_hw_intcnt(int update_every, usec_t dt) { static RRDDIM *rd_intr = NULL; if (unlikely(!st_intr)) { - st_intr = rrdset_create_localhost("system", - "intr", - NULL, - "interrupts", - NULL, - "Total Hardware Interrupts", - "interrupts/s", - 900, - update_every, - RRDSET_TYPE_LINE + st_intr = rrdset_create_localhost( + "system", + "intr", + NULL, + "interrupts", + NULL, + "Total Hardware Interrupts", + "interrupts/s", + "freebsd", + "hw.intrcnt", + 900, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st_intr, RRDSET_FLAG_DETAIL); @@ -519,20 +646,22 @@ int do_hw_intcnt(int update_every, usec_t dt) { // -------------------------------------------------------------------- static RRDSET *st_interrupts = NULL; - RRDDIM *rd_interrupts = NULL; void *p; if (unlikely(!st_interrupts)) - st_interrupts = rrdset_create_localhost("system", - "interrupts", - NULL, - "interrupts", - NULL, - "System interrupts", - "interrupts/s", - 1000, - update_every, - RRDSET_TYPE_STACKED + st_interrupts = rrdset_create_localhost( + "system", + "interrupts", + NULL, + "interrupts", + NULL, + "System interrupts", + "interrupts/s", + "freebsd", + "hw.intrcnt", + 1000, + update_every, + RRDSET_TYPE_STACKED ); else rrdset_next(st_interrupts); @@ -540,9 +669,11 @@ int do_hw_intcnt(int update_every, usec_t dt) { for (i = 0; i < nintr; i++) { p = intrnames + i * (MAXCOMLEN + 1); if (unlikely((intrcnt[i] != 0) && (*(char *) p != 0))) { - rd_interrupts = rrddim_find(st_interrupts, p); + RRDDIM *rd_interrupts = rrddim_find(st_interrupts, p); + if (unlikely(!rd_interrupts)) rd_interrupts = rrddim_add(st_interrupts, p, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st_interrupts, rd_interrupts, intrcnt[i]); } } @@ -576,16 +707,19 @@ int do_vm_stats_sys_v_intr(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "dev_intr", - NULL, - "interrupts", - NULL, - "Device Interrupts", - "interrupts/s", - 1000, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "dev_intr", + NULL, + "interrupts", + NULL, + "Device Interrupts", + "interrupts/s", + "freebsd", + "vm.stats.sys.v_intr", + 1000, + update_every, + RRDSET_TYPE_LINE ); rd = rrddim_add(st, "interrupts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -619,16 +753,19 @@ int do_vm_stats_sys_v_soft(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "soft_intr", - NULL, - "interrupts", - NULL, - "Software Interrupts", - "interrupts/s", - 1100, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "soft_intr", + NULL, + "interrupts", + NULL, + "Software Interrupts", + "interrupts/s", + "freebsd", + "vm.stats.sys.v_soft", + 1100, + update_every, + RRDSET_TYPE_LINE ); rd = rrddim_add(st, "interrupts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -662,16 +799,19 @@ int do_vm_stats_sys_v_swtch(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "ctxt", - NULL, - "processes", - NULL, - "CPU Context Switches", - "context switches/s", - 800, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "ctxt", + NULL, + "processes", + NULL, + "CPU Context Switches", + "context switches/s", + "freebsd", + "vm.stats.sys.v_swtch", + 800, + update_every, + RRDSET_TYPE_LINE ); rd = rrddim_add(st, "switches", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -705,16 +845,19 @@ int do_vm_stats_sys_v_forks(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "forks", - NULL, - "processes", - NULL, - "Started Processes", - "processes/s", - 700, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "forks", + NULL, + "processes", + NULL, + "Started Processes", + "processes/s", + "freebsd", + "vm.stats.sys.v_swtch", + 700, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -779,16 +922,19 @@ int do_vm_swap_info(int update_every, usec_t dt) { static RRDDIM *rd_free = NULL, *rd_used = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "swap", - NULL, - "swap", - NULL, - "System Swap", - "MB", - 201, - update_every, - RRDSET_TYPE_STACKED + st = rrdset_create_localhost( + "system", + "swap", + NULL, + "swap", + NULL, + "System Swap", + "MB", + "freebsd", + "vm.swap_info", + 201, + update_every, + RRDSET_TYPE_STACKED ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -813,7 +959,7 @@ int do_system_ram(int update_every, usec_t dt) { (void)dt; static int mib_active_count[4] = {0, 0, 0, 0}, mib_inactive_count[4] = {0, 0, 0, 0}, mib_wire_count[4] = {0, 0, 0, 0}, mib_cache_count[4] = {0, 0, 0, 0}, mib_vfs_bufspace[2] = {0, 0}, mib_free_count[4] = {0, 0, 0, 0}; - struct vmmeter vmmeter_data; + vmmeter_t vmmeter_data; int vfs_bufspace_count; if (unlikely(GETSYSCTL_SIMPLE("vm.stats.vm.v_active_count", mib_active_count, vmmeter_data.v_active_count) || @@ -825,7 +971,7 @@ int do_system_ram(int update_every, usec_t dt) { GETSYSCTL_SIMPLE("vfs.bufspace", mib_vfs_bufspace, vfs_bufspace_count) || GETSYSCTL_SIMPLE("vm.stats.vm.v_free_count", mib_free_count, vmmeter_data.v_free_count))) { error("DISABLED: system.ram chart"); - error("DISABLED: System.ram module"); + error("DISABLED: system.ram module"); return 1; } else { @@ -835,18 +981,20 @@ int do_system_ram(int update_every, usec_t dt) { static RRDDIM *rd_free = NULL, *rd_active = NULL, *rd_inactive = NULL, *rd_wired = NULL, *rd_cache = NULL, *rd_buffers = NULL; - st = rrdset_find_localhost("system.ram"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "ram", - NULL, - "ram", - NULL, - "System RAM", - "MB", - 200, - update_every, - RRDSET_TYPE_STACKED + st = rrdset_create_localhost( + "system", + "ram", + NULL, + "ram", + NULL, + "System RAM", + "MB", + "freebsd", + "system.ram", + 200, + update_every, + RRDSET_TYPE_STACKED ); rd_free = rrddim_add(st, "free", NULL, system_pagesize, MEGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); @@ -880,7 +1028,7 @@ int do_system_ram(int update_every, usec_t dt) { int do_vm_stats_sys_v_swappgs(int update_every, usec_t dt) { (void)dt; static int mib_swappgsin[4] = {0, 0, 0, 0}, mib_swappgsout[4] = {0, 0, 0, 0}; - struct vmmeter vmmeter_data; + vmmeter_t vmmeter_data; if (unlikely(GETSYSCTL_SIMPLE("vm.stats.vm.v_swappgsin", mib_swappgsin, vmmeter_data.v_swappgsin) || GETSYSCTL_SIMPLE("vm.stats.vm.v_swappgsout", mib_swappgsout, vmmeter_data.v_swappgsout))) { @@ -895,16 +1043,19 @@ int do_vm_stats_sys_v_swappgs(int update_every, usec_t dt) { static RRDDIM *rd_in = NULL, *rd_out = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "swapio", - NULL, - "swap", - NULL, - "Swap I/O", - "kilobytes/s", - 250, - update_every, - RRDSET_TYPE_AREA + st = rrdset_create_localhost( + "system", + "swapio", + NULL, + "swap", + NULL, + "Swap I/O", + "kilobytes/s", + "freebsd", + "vm.stats.vm.v_swappgs", + 250, + update_every, + RRDSET_TYPE_AREA ); rd_in = rrddim_add(st, "in", NULL, system_pagesize, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); @@ -927,7 +1078,7 @@ int do_vm_stats_sys_v_pgfaults(int update_every, usec_t dt) { (void)dt; static int mib_vm_faults[4] = {0, 0, 0, 0}, mib_io_faults[4] = {0, 0, 0, 0}, mib_cow_faults[4] = {0, 0, 0, 0}, mib_cow_optim[4] = {0, 0, 0, 0}, mib_intrans[4] = {0, 0, 0, 0}; - struct vmmeter vmmeter_data; + vmmeter_t vmmeter_data; if (unlikely(GETSYSCTL_SIMPLE("vm.stats.vm.v_vm_faults", mib_vm_faults, vmmeter_data.v_vm_faults) || GETSYSCTL_SIMPLE("vm.stats.vm.v_io_faults", mib_io_faults, vmmeter_data.v_io_faults) || @@ -946,16 +1097,19 @@ int do_vm_stats_sys_v_pgfaults(int update_every, usec_t dt) { *rd_cow_optimized = NULL, *rd_in_transit = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("mem", - "pgfaults", - NULL, - "system", - NULL, - "Memory Page Faults", - "page faults/s", - 500, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "mem", + "pgfaults", + NULL, + "system", + NULL, + "Memory Page Faults", + "page faults/s", + "freebsd", + "vm.stats.vm.v_pgfaults", + 500, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -1025,16 +1179,19 @@ int do_kern_ipc_sem(int update_every, usec_t dt) { static RRDDIM *rd_semaphores = NULL, *rd_semaphore_arrays = NULL; if (unlikely(!st_semaphores)) { - st_semaphores = rrdset_create_localhost("system", - "ipc_semaphores", - NULL, - "ipc semaphores", - NULL, - "IPC Semaphores", - "semaphores", - 1000, - update_every, - RRDSET_TYPE_AREA + st_semaphores = rrdset_create_localhost( + "system", + "ipc_semaphores", + NULL, + "ipc semaphores", + NULL, + "IPC Semaphores", + "semaphores", + "freebsd", + "kern.ipc.sem", + 1000, + update_every, + RRDSET_TYPE_AREA ); rd_semaphores = rrddim_add(st_semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1047,16 +1204,19 @@ int do_kern_ipc_sem(int update_every, usec_t dt) { // -------------------------------------------------------------------- if (unlikely(!st_semaphore_arrays)) { - st_semaphore_arrays = rrdset_create_localhost("system", - "ipc_semaphore_arrays", - NULL, - "ipc semaphores", - NULL, - "IPC Semaphore Arrays", - "arrays", - 1000, - update_every, - RRDSET_TYPE_AREA + st_semaphore_arrays = rrdset_create_localhost( + "system", + "ipc_semaphore_arrays", + NULL, + "ipc semaphores", + NULL, + "IPC Semaphore Arrays", + "arrays", + "freebsd", + "kern.ipc.sem", + 1000, + update_every, + RRDSET_TYPE_AREA ); rd_semaphore_arrays = rrddim_add(st_semaphore_arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1118,16 +1278,19 @@ int do_kern_ipc_shm(int update_every, usec_t dt) { static RRDDIM *rd_segments = NULL, *rd_allocated = NULL; if (unlikely(!st_segs)) { - st_segs = rrdset_create_localhost("system", - "ipc_shared_mem_segs", - NULL, - "ipc shared memory", - NULL, - "IPC Shared Memory Segments", - "segments", - 1000, - update_every, - RRDSET_TYPE_AREA + st_segs = rrdset_create_localhost( + "system", + "ipc_shared_mem_segs", + NULL, + "ipc shared memory", + NULL, + "IPC Shared Memory Segments", + "segments", + "freebsd", + "kern.ipc.shm", + 1000, + update_every, + RRDSET_TYPE_AREA ); rd_segments = rrddim_add(st_segs, "segments", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1140,16 +1303,19 @@ int do_kern_ipc_shm(int update_every, usec_t dt) { // -------------------------------------------------------------------- if (unlikely(!st_size)) { - st_size = rrdset_create_localhost("system", - "ipc_shared_mem_size", - NULL, - "ipc shared memory", - NULL, - "IPC Shared Memory Segments Size", - "kilobytes", - 1000, - update_every, - RRDSET_TYPE_AREA + st_size = rrdset_create_localhost( + "system", + "ipc_shared_mem_size", + NULL, + "ipc shared memory", + NULL, + "IPC Shared Memory Segments Size", + "kilobytes", + "freebsd", + "kern.ipc.shm", + 1000, + update_every, + RRDSET_TYPE_AREA ); rd_allocated = rrddim_add(st_size, "allocated", NULL, 1, KILO_FACTOR, RRD_ALGORITHM_ABSOLUTE); @@ -1217,16 +1383,19 @@ int do_kern_ipc_msq(int update_every, usec_t dt) { static RRDDIM *rd_queues = NULL, *rd_messages = NULL, *rd_allocated = NULL, *rd_used = NULL; if (unlikely(!st_queues)) { - st_queues = rrdset_create_localhost("system", - "ipc_msq_queues", - NULL, - "ipc message queues", - NULL, - "Number of IPC Message Queues", - "queues", - 990, - update_every, - RRDSET_TYPE_AREA + st_queues = rrdset_create_localhost( + "system", + "ipc_msq_queues", + NULL, + "ipc message queues", + NULL, + "Number of IPC Message Queues", + "queues", + "freebsd", + "kern.ipc.msq", + 990, + update_every, + RRDSET_TYPE_AREA ); rd_queues = rrddim_add(st_queues, "queues", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1239,16 +1408,19 @@ int do_kern_ipc_msq(int update_every, usec_t dt) { // -------------------------------------------------------------------- if (unlikely(!st_messages)) { - st_messages = rrdset_create_localhost("system", - "ipc_msq_messages", - NULL, - "ipc message queues", - NULL, - "Number of Messages in IPC Message Queues", - "messages", - 1000, - update_every, - RRDSET_TYPE_AREA + st_messages = rrdset_create_localhost( + "system", + "ipc_msq_messages", + NULL, + "ipc message queues", + NULL, + "Number of Messages in IPC Message Queues", + "messages", + "freebsd", + "kern.ipc.msq", + 1000, + update_every, + RRDSET_TYPE_AREA ); rd_messages = rrddim_add(st_messages, "messages", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1261,16 +1433,19 @@ int do_kern_ipc_msq(int update_every, usec_t dt) { // -------------------------------------------------------------------- if (unlikely(!st_size)) { - st_size = rrdset_create_localhost("system", - "ipc_msq_size", - NULL, - "ipc message queues", - NULL, - "Size of IPC Message Queues", - "bytes", - 1100, - update_every, - RRDSET_TYPE_LINE + st_size = rrdset_create_localhost( + "system", + "ipc_msq_size", + NULL, + "ipc message queues", + NULL, + "Size of IPC Message Queues", + "bytes", + "freebsd", + "kern.ipc.msq", + 1100, + update_every, + RRDSET_TYPE_LINE ); rd_allocated = rrddim_add(st_size, "allocated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1302,16 +1477,19 @@ int do_uptime(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if(unlikely(!st)) { - st = rrdset_create_localhost("system", - "uptime", - NULL, - "uptime", - NULL, - "System Uptime", - "seconds", - 1000, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "uptime", + NULL, + "uptime", + NULL, + "System Uptime", + "seconds", + "freebsd", + "uptime", + 1000, + update_every, + RRDSET_TYPE_LINE ); rd = rrddim_add(st, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1427,16 +1605,19 @@ int do_net_isr(int update_every, usec_t dt) { static RRDDIM *rd_dispatched = NULL, *rd_hybrid_dispatched = NULL, *rd_qdrops = NULL, *rd_queued = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "softnet_stat", - NULL, - "softnet_stat", - NULL, - "System softnet_stat", - "events/s", - 955, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "system", + "softnet_stat", + NULL, + "softnet_stat", + NULL, + "System softnet_stat", + "events/s", + "freebsd", + "net.isr", + 955, + update_every, + RRDSET_TYPE_LINE ); rd_dispatched = rrddim_add(st, "dispatched", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1477,16 +1658,19 @@ int do_net_isr(int update_every, usec_t dt) { snprintfz(all_softnet_charts[i].netisr_cpuid, MAX_INT_DIGITS + 17, "cpu%d_softnet_stat", i); if (unlikely(!all_softnet_charts[i].st)) { - all_softnet_charts[i].st = rrdset_create_localhost("cpu", - all_softnet_charts[i].netisr_cpuid, - NULL, - "softnet_stat", - NULL, - "Per CPU netisr statistics", - "events/s", - 1101 + i, - update_every, - RRDSET_TYPE_LINE + all_softnet_charts[i].st = rrdset_create_localhost( + "cpu", + all_softnet_charts[i].netisr_cpuid, + NULL, + "softnet_stat", + NULL, + "Per CPU netisr statistics", + "events/s", + "freebsd", + "net.isr", + 1101 + i, + update_every, + RRDSET_TYPE_LINE ); all_softnet_charts[i].rd_dispatched = rrddim_add(all_softnet_charts[i].st, "dispatched", @@ -1536,16 +1720,19 @@ int do_net_inet_tcp_states(int update_every, usec_t dt) { static RRDDIM *rd = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "tcpsock", - NULL, - "tcp", - NULL, - "IPv4 TCP Connections", - "active connections", - 2500, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "tcpsock", + NULL, + "tcp", + NULL, + "IPv4 TCP Connections", + "active connections", + "freebsd", + "net.inet.tcp.states", + 2500, + update_every, + RRDSET_TYPE_LINE ); rd = rrddim_add(st, "CurrEstab", "connections", 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -1564,7 +1751,8 @@ int do_net_inet_tcp_states(int update_every, usec_t dt) { int do_net_inet_tcp_stats(int update_every, usec_t dt) { (void)dt; - static int do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, do_tcpext_connaborts = -1, do_tcpext_ofo = -1, do_tcpext_syncookies = -1, do_ecn = -1; + static int do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, do_tcpext_connaborts = -1, do_tcpext_ofo = -1, + do_tcpext_syncookies = -1, do_tcpext_listen = -1, do_ecn = -1; if (unlikely(do_tcp_packets == -1)) { do_tcp_packets = config_get_boolean("plugin:freebsd:net.inet.tcp.stats", "ipv4 TCP packets", 1); @@ -1576,12 +1764,15 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { CONFIG_BOOLEAN_AUTO); do_tcpext_syncookies = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "TCP SYN cookies", CONFIG_BOOLEAN_AUTO); + do_tcpext_listen = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "TCP listen issues", + CONFIG_BOOLEAN_AUTO); do_ecn = config_get_boolean_ondemand("plugin:freebsd:net.inet.tcp.stats", "ECN packets", CONFIG_BOOLEAN_AUTO); } // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html - if (likely(do_tcp_packets || do_tcp_errors || do_tcp_handshake || do_tcpext_connaborts || do_tcpext_ofo || do_tcpext_syncookies || do_ecn)) { + if (likely(do_tcp_packets || do_tcp_errors || do_tcp_handshake || do_tcpext_connaborts || do_tcpext_ofo || + do_tcpext_syncookies || do_tcpext_listen || do_ecn)) { static int mib[4] = {0, 0, 0, 0}; struct tcpstat tcpstat; @@ -1598,6 +1789,8 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { error("DISABLED: ipv4.tcpofo chart"); do_tcpext_syncookies = 0; error("DISABLED: ipv4.tcpsyncookies chart"); + do_tcpext_listen = 0; + error("DISABLED: ipv4.tcplistenissues chart"); do_ecn = 0; error("DISABLED: ipv4.ecnpkts chart"); error("DISABLED: net.inet.tcp.stats module"); @@ -1611,16 +1804,19 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { static RRDDIM *rd_in_segs = NULL, *rd_out_segs = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "tcppackets", - NULL, - "tcp", - NULL, - "IPv4 TCP Packets", - "packets/s", - 2600, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "tcppackets", + NULL, + "tcp", + NULL, + "IPv4 TCP Packets", + "packets/s", + "freebsd", + "net.inet.tcp.stats", + 2600, + update_every, + RRDSET_TYPE_LINE ); rd_in_segs = rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1640,16 +1836,19 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { static RRDDIM *rd_in_errs = NULL, *rd_in_csum_errs = NULL, *rd_retrans_segs = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "tcperrors", - NULL, - "tcp", - NULL, - "IPv4 TCP Errors", - "packets/s", - 2700, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "tcperrors", + NULL, + "tcp", + NULL, + "IPv4 TCP Errors", + "packets/s", + "freebsd", + "net.inet.tcp.stats", + 2700, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -1679,16 +1878,19 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { *rd_attempt_fails = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "tcphandshake", - NULL, - "tcp", - NULL, - "IPv4 TCP Handshake Issues", - "events/s", - 2900, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "tcphandshake", + NULL, + "tcp", + NULL, + "IPv4 TCP Handshake Issues", + "events/s", + "freebsd", + "net.inet.tcp.stats", + 2900, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -1717,16 +1919,19 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { *rd_on_timeout = NULL, *rd_on_linger = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "tcpconnaborts", - NULL, - "tcp", - NULL, - "TCP Connection Aborts", - "connections/s", - 3010, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "tcpconnaborts", + NULL, + "tcp", + NULL, + "TCP Connection Aborts", + "connections/s", + "freebsd", + "net.inet.tcp.stats", + 3010, + update_every, + RRDSET_TYPE_LINE ); rd_on_data = rrddim_add(st, "TCPAbortOnData", "baddata", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1754,16 +1959,19 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { static RRDDIM *rd_ofo_queue = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "tcpofo", - NULL, - "tcp", - NULL, - "TCP Out-Of-Order Queue", - "packets/s", - 3050, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "tcpofo", + NULL, + "tcp", + NULL, + "TCP Out-Of-Order Queue", + "packets/s", + "freebsd", + "net.inet.tcp.stats", + 3050, + update_every, + RRDSET_TYPE_LINE ); rd_ofo_queue = rrddim_add(st, "TCPOFOQueue", "inqueue", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1783,16 +1991,19 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { static RRDDIM *rd_recv = NULL, *rd_send = NULL, *rd_failed = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "tcpsyncookies", - NULL, - "tcp", - NULL, - "TCP SYN Cookies", - "packets/s", - 3100, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "tcpsyncookies", + NULL, + "tcp", + NULL, + "TCP SYN Cookies", + "packets/s", + "freebsd", + "net.inet.tcp.stats", + 3100, + update_every, + RRDSET_TYPE_LINE ); rd_recv = rrddim_add(st, "SyncookiesRecv", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1806,6 +2017,41 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { rrddim_set_by_pointer(st, rd_failed, tcpstat.tcps_sc_zonefail); rrdset_done(st); } + + // -------------------------------------------------------------------- + + if(do_tcpext_listen == CONFIG_BOOLEAN_YES || (do_tcpext_listen == CONFIG_BOOLEAN_AUTO && tcpstat.tcps_listendrop)) { + do_tcpext_listen = CONFIG_BOOLEAN_YES; + + static RRDSET *st_listen = NULL; + static RRDDIM *rd_overflows = NULL; + + if(unlikely(!st_listen)) { + + st_listen = rrdset_create_localhost( + "ipv4", + "tcplistenissues", + NULL, + "tcp", + NULL, + "TCP Listen Socket Issues", + "packets/s", + "freebsd", + "net.inet.tcp.stats", + 3015, + update_every, + RRDSET_TYPE_LINE + ); + + rd_overflows = rrddim_add(st_listen, "ListenOverflows", "overflows", 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_listen); + + rrddim_set_by_pointer(st_listen, rd_overflows, tcpstat.tcps_listendrop); + + rrdset_done(st_listen); + } // -------------------------------------------------------------------- @@ -1816,16 +2062,19 @@ int do_net_inet_tcp_stats(int update_every, usec_t dt) { static RRDDIM *rd_ce = NULL, *rd_no_ect = NULL, *rd_ect0 = NULL, *rd_ect1 = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "ecnpkts", - NULL, - "ecn", - NULL, - "IPv4 ECN Statistics", - "packets/s", - 8700, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "ecnpkts", + NULL, + "ecn", + NULL, + "IPv4 ECN Statistics", + "packets/s", + "freebsd", + "net.inet.tcp.stats", + 8700, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -1887,16 +2136,19 @@ int do_net_inet_udp_stats(int update_every, usec_t dt) { static RRDDIM *rd_in = NULL, *rd_out = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "udppackets", - NULL, - "udp", - NULL, - "IPv4 UDP Packets", - "packets/s", - 2601, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "udppackets", + NULL, + "udp", + NULL, + "IPv4 UDP Packets", + "packets/s", + "freebsd", + "net.inet.udp.stats", + 2601, + update_every, + RRDSET_TYPE_LINE ); rd_in = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1917,16 +2169,19 @@ int do_net_inet_udp_stats(int update_every, usec_t dt) { *rd_in_csum_errors = NULL, *rd_ignored_multi = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "udperrors", - NULL, - "udp", - NULL, - "IPv4 UDP Errors", - "events/s", - 2701, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "udperrors", + NULL, + "udp", + NULL, + "IPv4 UDP Errors", + "events/s", + "freebsd", + "net.inet.udp.stats", + 2701, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -2000,16 +2255,19 @@ int do_net_inet_icmp_stats(int update_every, usec_t dt) { static RRDDIM *rd_in = NULL, *rd_out = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "icmp", - NULL, - "icmp", - NULL, - "IPv4 ICMP Packets", - "packets/s", - 2602, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4" + , "icmp" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Packets" + , "packets/s" + , "freebsd" + , "net.inet.icmp.stats" + , 2602 + , update_every + , RRDSET_TYPE_LINE ); rd_in = rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2030,9 +2288,20 @@ int do_net_inet_icmp_stats(int update_every, usec_t dt) { static RRDDIM *rd_in = NULL, *rd_out = NULL, *rd_in_csum = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "icmp_errors", NULL, "icmp", NULL, "IPv4 ICMP Errors", - "packets/s", - 2603, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "icmp_errors" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Errors" + , "packets/s" + , "freebsd" + , "net.inet.icmp.stats" + , 2603 + , update_every + , RRDSET_TYPE_LINE + ); rd_in = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rd_out = rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2055,8 +2324,20 @@ int do_net_inet_icmp_stats(int update_every, usec_t dt) { static RRDDIM *rd_in_reps = NULL, *rd_out_reps = NULL, *rd_in = NULL, *rd_out = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "icmpmsg", NULL, "icmp", NULL, "IPv4 ICMP Messages", - "packets/s", 2604, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "icmpmsg" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Messages" + , "packets/s" + , "freebsd" + , "net.inet.icmp.stats" + , 2604 + , update_every + , RRDSET_TYPE_LINE + ); rd_in_reps = rrddim_add(st, "InEchoReps", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rd_out_reps = rrddim_add(st, "OutEchoReps", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2121,16 +2402,19 @@ int do_net_inet_ip_stats(int update_every, usec_t dt) { *rd_in_delivers = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "packets", - NULL, - "packets", - NULL, - "IPv4 Packets", - "packets/s", - 3000, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "packets", + NULL, + "packets", + NULL, + "IPv4 Packets", + "packets/s", + "freebsd", + "net.inet.ip.stats", + 3000, + update_every, + RRDSET_TYPE_LINE ); rd_in_receives = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2154,16 +2438,19 @@ int do_net_inet_ip_stats(int update_every, usec_t dt) { static RRDDIM *rd_ok = NULL, *rd_fails = NULL, *rd_created = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "fragsout", - NULL, - "fragments", - NULL, - "IPv4 Fragments Sent", - "packets/s", - 3010, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "fragsout", + NULL, + "fragments", + NULL, + "IPv4 Fragments Sent", + "packets/s", + "freebsd", + "net.inet.ip.stats", + 3010, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -2187,16 +2474,19 @@ int do_net_inet_ip_stats(int update_every, usec_t dt) { static RRDDIM *rd_ok = NULL, *rd_failed = NULL, *rd_all = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "fragsin", - NULL, - "fragments", - NULL, - "IPv4 Fragments Reassembly", - "packets/s", - 3011, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "fragsin", + NULL, + "fragments", + NULL, + "IPv4 Fragments Reassembly", + "packets/s", + "freebsd", + "net.inet.ip.stats", + 3011, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -2222,16 +2512,19 @@ int do_net_inet_ip_stats(int update_every, usec_t dt) { *rd_in_addr_errors = NULL, *rd_in_unknown_protos = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", - "errors", - NULL, - "errors", - NULL, - "IPv4 Errors", - "packets/s", - 3002, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv4", + "errors", + NULL, + "errors", + NULL, + "IPv4 Errors", + "packets/s", + "freebsd", + "net.inet.ip.stats", + 3002, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -2310,16 +2603,19 @@ int do_net_inet6_ip6_stats(int update_every, usec_t dt) { static RRDDIM *rd_received = NULL, *rd_sent = NULL, *rd_forwarded = NULL, *rd_delivers = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "packets", - NULL, - "packets", - NULL, - "IPv6 Packets", - "packets/s", - 3000, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "packets", + NULL, + "packets", + NULL, + "IPv6 Packets", + "packets/s", + "freebsd", + "net.inet6.ip6.stats", + 3000, + update_every, + RRDSET_TYPE_LINE ); rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2347,16 +2643,19 @@ int do_net_inet6_ip6_stats(int update_every, usec_t dt) { static RRDDIM *rd_ok = NULL, *rd_failed = NULL, *rd_all = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "fragsout", - NULL, - "fragments", - NULL, - "IPv6 Fragments Sent", - "packets/s", - 3010, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "fragsout", + NULL, + "fragments", + NULL, + "IPv6 Fragments Sent", + "packets/s", + "freebsd", + "net.inet6.ip6.stats", + 3010, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -2384,16 +2683,19 @@ int do_net_inet6_ip6_stats(int update_every, usec_t dt) { static RRDDIM *rd_ok = NULL, *rd_failed = NULL, *rd_timeout = NULL, *rd_all = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "fragsin", - NULL, - "fragments", - NULL, - "IPv6 Fragments Reassembly", - "packets/s", - 3011, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "fragsin", + NULL, + "fragments", + NULL, + "IPv6 Fragments Reassembly", + "packets/s", + "freebsd", + "net.inet6.ip6.stats", + 3011, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -2432,16 +2734,19 @@ int do_net_inet6_ip6_stats(int update_every, usec_t dt) { *rd_in_no_routes = NULL, *rd_out_no_routes = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "errors", - NULL, - "errors", - NULL, - "IPv6 Errors", - "packets/s", - 3002, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "errors", + NULL, + "errors", + NULL, + "IPv6 Errors", + "packets/s", + "freebsd", + "net.inet6.ip6.stats", + 3002, + update_every, + RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); @@ -2543,16 +2848,19 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { static RRDDIM *rd_received = NULL, *rd_sent = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "icmp", - NULL, - "icmp", - NULL, - "IPv6 ICMP Messages", - "messages/s", - 10000, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "icmp", + NULL, + "icmp", + NULL, + "IPv6 ICMP Messages", + "messages/s", + "freebsd", + "net.inet6.icmp6.stats", + 10000, + update_every, + RRDSET_TYPE_LINE ); rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2575,16 +2883,19 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { static RRDDIM *rd_received = NULL, *rd_sent = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "icmpredir", - NULL, - "icmp", - NULL, - "IPv6 ICMP Redirects", - "redirects/s", - 10050, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "icmpredir", + NULL, + "icmp", + NULL, + "IPv6 ICMP Redirects", + "redirects/s", + "freebsd", + "net.inet6.icmp6.stats", + 10050, + update_every, + RRDSET_TYPE_LINE ); rd_received = rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2620,15 +2931,18 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { *rd_out_parm_problems = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "icmperrors", - NULL, "icmp", - NULL, - "IPv6 ICMP Errors", - "errors/s", - 10100, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "icmperrors", + NULL, "icmp", + NULL, + "IPv6 ICMP Errors", + "errors/s", + "freebsd", + "net.inet6.icmp6.stats", + 10100, + update_every, + RRDSET_TYPE_LINE ); rd_in_errors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2671,16 +2985,19 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { static RRDDIM *rd_in = NULL, *rd_out = NULL, *rd_in_replies = NULL, *rd_out_replies = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "icmpechos", - NULL, - "icmp", - NULL, - "IPv6 ICMP Echo", - "messages/s", - 10200, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "icmpechos", + NULL, + "icmp", + NULL, + "IPv6 ICMP Echo", + "messages/s", + "freebsd", + "net.inet6.icmp6.stats", + 10200, + update_every, + RRDSET_TYPE_LINE ); rd_in = rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2711,16 +3028,19 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { *rd_in_advertisements = NULL, *rd_out_advertisements = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "icmprouter", - NULL, - "icmp", - NULL, - "IPv6 Router Messages", - "messages/s", - 10400, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "icmprouter", + NULL, + "icmp", + NULL, + "IPv6 Router Messages", + "messages/s", + "freebsd", + "net.inet6.icmp6.stats", + 10400, + update_every, + RRDSET_TYPE_LINE ); rd_in_solicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2751,16 +3071,19 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { *rd_in_advertisements = NULL, *rd_out_advertisements = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "icmpneighbor", - NULL, - "icmp", - NULL, - "IPv6 Neighbor Messages", - "messages/s", - 10500, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "icmpneighbor", + NULL, + "icmp", + NULL, + "IPv6 Neighbor Messages", + "messages/s", + "freebsd", + "net.inet6.icmp6.stats", + 10500, + update_every, + RRDSET_TYPE_LINE ); rd_in_solicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -2798,16 +3121,19 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { *rd_out_135 = NULL, *rd_out_143 = NULL; if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", - "icmptypes", - NULL, - "icmp", - NULL, - "IPv6 ICMP Types", - "messages/s", - 10700, - update_every, - RRDSET_TYPE_LINE + st = rrdset_create_localhost( + "ipv6", + "icmptypes", + NULL, + "icmp", + NULL, + "IPv6 ICMP Types", + "messages/s", + "freebsd", + "net.inet6.icmp6.stats", + 10700, + update_every, + RRDSET_TYPE_LINE ); rd_in_1 = rrddim_add(st, "InType1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); diff --git a/src/freeipmi_plugin.c b/src/freeipmi_plugin.c index 42a1ac01d..9cd736bba 100644 --- a/src/freeipmi_plugin.c +++ b/src/freeipmi_plugin.c @@ -522,6 +522,49 @@ static void send_metrics_to_netdata() { } +static int *excluded_record_ids = NULL; +size_t excluded_record_ids_length = 0; + +static void excluded_record_ids_parse(const char *s) { + if(!s) return; + + while(*s) { + while(*s && !isdigit(*s)) s++; + + if(isdigit(*s)) { + char *e; + unsigned long n = strtoul(s, &e, 10); + s = e; + + if(n != 0) { + excluded_record_ids = realloc(excluded_record_ids, (excluded_record_ids_length + 1) * sizeof(int)); + excluded_record_ids[excluded_record_ids_length++] = (int)n; + } + } + } + + if(debug) { + fprintf(stderr, "freeipmi.plugin: excluded record ids:"); + size_t i; + for(i = 0; i < excluded_record_ids_length; i++) { + fprintf(stderr, " %d", excluded_record_ids[i]); + } + fprintf(stderr, "\n"); + } +} + + +static int excluded_record_ids_check(int record_id) { + size_t i; + + for(i = 0; i < excluded_record_ids_length; i++) { + if(excluded_record_ids[i] == record_id) + return 1; + } + + return 0; +} + static void netdata_get_sensor( int record_id , int sensor_number @@ -546,6 +589,10 @@ static void netdata_get_sensor( if(!sn) { // not found, create it + // check if it is excluded + if(excluded_record_ids_check(record_id)) + return; + sn = calloc(1, sizeof(struct sensor)); if(!sn) { fatal("cannot allocate %zu bytes of memory.", sizeof(struct sensor)); @@ -1482,6 +1529,9 @@ int main (int argc, char **argv) { " sensor-config-file FILE filename to read sensor configuration\n" " default: %s\n" "\n" + " ignore N1,N2,N3,... sensor IDs to ignore\n" + " default: none\n" + "\n" " -v\n" " -V\n" " version print version and exit\n" @@ -1538,10 +1588,16 @@ int main (int argc, char **argv) { if(debug) fprintf(stderr, "freeipmi.plugin: sensor config file set to '%s'\n", sensor_config_file); continue; } + else if(i < argc && strcmp("ignore", argv[i]) == 0) { + excluded_record_ids_parse(argv[++i]); + continue; + } error("freeipmi.plugin: ignoring parameter '%s'", argv[i]); } + errno = 0; + if(freq > netdata_update_every) netdata_update_every = freq; diff --git a/src/global_statistics.c b/src/global_statistics.c index 886889086..c184b6d68 100644 --- a/src/global_statistics.c +++ b/src/global_statistics.c @@ -114,13 +114,13 @@ inline void global_statistics_copy(struct global_statistics *gs, uint8_t options } void global_statistics_charts(void) { - static unsigned long long old_web_requests = 0, old_web_usec = 0, - old_content_size = 0, old_compressed_content_size = 0; + static unsigned long long old_web_requests = 0, + old_web_usec = 0, + old_content_size = 0, + old_compressed_content_size = 0; - static collected_number compression_ratio = -1, average_response_time = -1; - - static RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL, - *stcompression = NULL; + static collected_number compression_ratio = -1, + average_response_time = -1; struct global_statistics gs; struct rusage me, thread; @@ -129,143 +129,280 @@ void global_statistics_charts(void) { getrusage(RUSAGE_THREAD, &thread); getrusage(RUSAGE_SELF, &me); + { + static RRDSET *st_cpu_thread = NULL; + static RRDDIM *rd_cpu_thread_user = NULL, + *rd_cpu_thread_system = NULL; + #ifdef __FreeBSD__ - if (!stcpu_thread) stcpu_thread = rrdset_find_localhost("netdata.plugin_freebsd_cpu"); - if (!stcpu_thread) { - stcpu_thread = rrdset_create_localhost("netdata", "plugin_freebsd_cpu", NULL, "freebsd", NULL - , "NetData FreeBSD Plugin CPU usage", "milliseconds/s", 132000 - , localhost->rrd_update_every, RRDSET_TYPE_STACKED); + if (unlikely(!st_cpu_thread)) { + st_cpu_thread = rrdset_create_localhost( + "netdata" + , "plugin_freebsd_cpu" + , NULL + , "freebsd" + , NULL + , "NetData FreeBSD Plugin CPU usage" + , "milliseconds/s" + , "netdata" + , "stats" + , 132000 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); #else - if (!stcpu_thread) stcpu_thread = rrdset_find_localhost("netdata.plugin_proc_cpu"); - if (!stcpu_thread) { - stcpu_thread = rrdset_create_localhost("netdata", "plugin_proc_cpu", NULL, "proc", NULL - , "NetData Proc Plugin CPU usage", "milliseconds/s", 132000 - , localhost->rrd_update_every, RRDSET_TYPE_STACKED); + if (unlikely(!st_cpu_thread)) { + st_cpu_thread = rrdset_create_localhost( + "netdata" + , "plugin_proc_cpu" + , NULL + , "proc" + , NULL + , "NetData Proc Plugin CPU usage" + , "milliseconds/s" + , "netdata" + , "stats" + , 132000 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); #endif - rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - } else rrdset_next(stcpu_thread); + rd_cpu_thread_user = rrddim_add(st_cpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + rd_cpu_thread_system = rrddim_add(st_cpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_cpu_thread); - rrddim_set(stcpu_thread, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); - rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); - rrdset_done(stcpu_thread); + rrddim_set_by_pointer(st_cpu_thread, rd_cpu_thread_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set_by_pointer(st_cpu_thread, rd_cpu_thread_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrdset_done(st_cpu_thread); + } // ---------------------------------------------------------------- - if (!stcpu) stcpu = rrdset_find_localhost("netdata.server_cpu"); - if (!stcpu) { - stcpu = rrdset_create_localhost("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage" - , "milliseconds/s", 130000, localhost->rrd_update_every, RRDSET_TYPE_STACKED); - - rrddim_add(stcpu, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(stcpu, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - } else rrdset_next(stcpu); - - rrddim_set(stcpu, "user", me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec); - rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec); - rrdset_done(stcpu); + { + static RRDSET *st_cpu = NULL; + static RRDDIM *rd_cpu_user = NULL, + *rd_cpu_system = NULL; + + if (unlikely(!st_cpu)) { + st_cpu = rrdset_create_localhost( + "netdata" + , "server_cpu" + , NULL + , "netdata" + , NULL + , "NetData CPU usage" + , "milliseconds/s" + , "netdata" + , "stats" + , 130000 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); + + rd_cpu_user = rrddim_add(st_cpu, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + rd_cpu_system = rrddim_add(st_cpu, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_cpu); + + rrddim_set_by_pointer(st_cpu, rd_cpu_user, me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec); + rrddim_set_by_pointer(st_cpu, rd_cpu_system, me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec); + rrdset_done(st_cpu); + } // ---------------------------------------------------------------- - if (!stclients) stclients = rrdset_find_localhost("netdata.clients"); - if (!stclients) { - stclients = rrdset_create_localhost("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients" - , "connected clients", 130200, localhost->rrd_update_every, RRDSET_TYPE_LINE); - - rrddim_add(stclients, "clients", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - } else rrdset_next(stclients); - - rrddim_set(stclients, "clients", gs.connected_clients); - rrdset_done(stclients); + { + static RRDSET *st_clients = NULL; + static RRDDIM *rd_clients = NULL; + + if (unlikely(!st_clients)) { + st_clients = rrdset_create_localhost( + "netdata" + , "clients" + , NULL + , "netdata" + , NULL + , "NetData Web Clients" + , "connected clients" + , "netdata" + , "stats" + , 130200 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_clients = rrddim_add(st_clients, "clients", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_clients); + + rrddim_set_by_pointer(st_clients, rd_clients, gs.connected_clients); + rrdset_done(st_clients); + } // ---------------------------------------------------------------- - if (!streqs) streqs = rrdset_find_localhost("netdata.requests"); - if (!streqs) { - streqs = rrdset_create_localhost("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests" - , "requests/s", 130300, localhost->rrd_update_every, RRDSET_TYPE_LINE); - - rrddim_add(streqs, "requests", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - } else rrdset_next(streqs); - - rrddim_set(streqs, "requests", (collected_number) gs.web_requests); - rrdset_done(streqs); + { + static RRDSET *st_reqs = NULL; + static RRDDIM *rd_requests = NULL; + + if (unlikely(!st_reqs)) { + st_reqs = rrdset_create_localhost( + "netdata" + , "requests" + , NULL + , "netdata" + , NULL + , "NetData Web Requests" + , "requests/s" + , "netdata" + , "stats" + , 130300 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_requests = rrddim_add(st_reqs, "requests", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_reqs); + + rrddim_set_by_pointer(st_reqs, rd_requests, (collected_number) gs.web_requests); + rrdset_done(st_reqs); + } // ---------------------------------------------------------------- - if (!stbytes) stbytes = rrdset_find_localhost("netdata.net"); - if (!stbytes) { - stbytes = rrdset_create_localhost("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic" - , "kilobits/s", 130000, localhost->rrd_update_every, RRDSET_TYPE_AREA); - - rrddim_add(stbytes, "in", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(stbytes, "out", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - } else rrdset_next(stbytes); - - rrddim_set(stbytes, "in", (collected_number) gs.bytes_received); - rrddim_set(stbytes, "out", (collected_number) gs.bytes_sent); - rrdset_done(stbytes); + { + static RRDSET *st_bytes = NULL; + static RRDDIM *rd_in = NULL, + *rd_out = NULL; + + if (unlikely(!st_bytes)) { + st_bytes = rrdset_create_localhost( + "netdata" + , "net" + , NULL + , "netdata" + , NULL + , "NetData Network Traffic" + , "kilobits/s" + , "netdata" + , "stats" + , 130000 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA + ); + + rd_in = rrddim_add(st_bytes, "in", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_bytes, "out", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_bytes); + + rrddim_set_by_pointer(st_bytes, rd_in, (collected_number) gs.bytes_received); + rrddim_set_by_pointer(st_bytes, rd_out, (collected_number) gs.bytes_sent); + rrdset_done(st_bytes); + } // ---------------------------------------------------------------- - if (!stduration) stduration = rrdset_find_localhost("netdata.response_time"); - if (!stduration) { - stduration = rrdset_create_localhost("netdata", "response_time", NULL, "netdata", NULL - , "NetData API Response Time", "ms/request", 130400, localhost->rrd_update_every - , RRDSET_TYPE_LINE); - - rrddim_add(stduration, "average", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(stduration, "max", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - } else rrdset_next(stduration); - - uint64_t gweb_usec = gs.web_usec; - uint64_t gweb_requests = gs.web_requests; - - uint64_t web_usec = (gweb_usec >= old_web_usec) ? gweb_usec - old_web_usec : 0; - uint64_t web_requests = (gweb_requests >= old_web_requests) ? gweb_requests - old_web_requests : 0; - - old_web_usec = gweb_usec; - old_web_requests = gweb_requests; - - if (web_requests) - average_response_time = (collected_number) (web_usec / web_requests); - - if (unlikely(average_response_time != -1)) - rrddim_set(stduration, "average", average_response_time); - else - rrddim_set(stduration, "average", 0); - - rrddim_set(stduration, "max", ((gs.web_usec_max)?(collected_number)gs.web_usec_max:average_response_time)); - rrdset_done(stduration); + { + static RRDSET *st_duration = NULL; + static RRDDIM *rd_average = NULL, + *rd_max = NULL; + + if (unlikely(!st_duration)) { + st_duration = rrdset_create_localhost( + "netdata" + , "response_time" + , NULL + , "netdata" + , NULL + , "NetData API Response Time" + , "ms/request" + , "netdata" + , "stats" + , 130400 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_average = rrddim_add(st_duration, "average", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_max = rrddim_add(st_duration, "max", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_duration); + + uint64_t gweb_usec = gs.web_usec; + uint64_t gweb_requests = gs.web_requests; + + uint64_t web_usec = (gweb_usec >= old_web_usec) ? gweb_usec - old_web_usec : 0; + uint64_t web_requests = (gweb_requests >= old_web_requests) ? gweb_requests - old_web_requests : 0; + + old_web_usec = gweb_usec; + old_web_requests = gweb_requests; + + if (web_requests) + average_response_time = (collected_number) (web_usec / web_requests); + + if (unlikely(average_response_time != -1)) + rrddim_set_by_pointer(st_duration, rd_average, average_response_time); + else + rrddim_set_by_pointer(st_duration, rd_average, 0); + + rrddim_set_by_pointer(st_duration, rd_max, ((gs.web_usec_max)?(collected_number)gs.web_usec_max:average_response_time)); + rrdset_done(st_duration); + } // ---------------------------------------------------------------- - if (!stcompression) stcompression = rrdset_find_localhost("netdata.compression_ratio"); - if (!stcompression) { - stcompression = rrdset_create_localhost("netdata", "compression_ratio", NULL, "netdata", NULL - , "NetData API Responses Compression Savings Ratio", "percentage" - , 130500, localhost->rrd_update_every, RRDSET_TYPE_LINE); - - rrddim_add(stcompression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - } else rrdset_next(stcompression); - - // since we don't lock here to read the global statistics - // read the smaller value first - unsigned long long gcompressed_content_size = gs.compressed_content_size; - unsigned long long gcontent_size = gs.content_size; - - unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size; - unsigned long long content_size = gcontent_size - old_content_size; - - old_compressed_content_size = gcompressed_content_size; - old_content_size = gcontent_size; - - if (content_size && content_size >= compressed_content_size) - compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size; - - if (compression_ratio != -1) - rrddim_set(stcompression, "savings", compression_ratio); - - rrdset_done(stcompression); + { + static RRDSET *st_compression = NULL; + static RRDDIM *rd_savings = NULL; + + if (unlikely(!st_compression)) { + st_compression = rrdset_create_localhost( + "netdata" + , "compression_ratio" + , NULL + , "netdata" + , NULL + , "NetData API Responses Compression Savings Ratio" + , "percentage" + , "netdata" + , "stats" + , 130500 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_compression); + + // since we don't lock here to read the global statistics + // read the smaller value first + unsigned long long gcompressed_content_size = gs.compressed_content_size; + unsigned long long gcontent_size = gs.content_size; + + unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size; + unsigned long long content_size = gcontent_size - old_content_size; + + old_compressed_content_size = gcompressed_content_size; + old_content_size = gcontent_size; + + if (content_size && content_size >= compressed_content_size) + compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size; + + if (compression_ratio != -1) + rrddim_set_by_pointer(st_compression, rd_savings, compression_ratio); + + rrdset_done(st_compression); + } } diff --git a/src/health.c b/src/health.c index 136a1ecd7..dfa7007b9 100644 --- a/src/health.c +++ b/src/health.c @@ -34,10 +34,10 @@ void health_reload_host(RRDHOST *host) { rrdhost_wrlock(host); while(host->templates) - rrdcalctemplate_free(host, host->templates); + rrdcalctemplate_unlink_and_free(host, host->templates); while(host->alarms) - rrdcalc_free(host, host->alarms); + rrdcalc_unlink_and_free(host, host->alarms); rrdhost_unlock(host); @@ -84,7 +84,7 @@ void health_reload(void) { // ---------------------------------------------------------------------------- // health main thread and friends -static inline int rrdcalc_value2status(calculated_number n) { +static inline RRDCALC_STATUS rrdcalc_value2status(calculated_number n) { if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED; if(n) return RRDCALC_STATUS_RAISED; return RRDCALC_STATUS_CLEAR; @@ -189,7 +189,6 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { done: health_alarm_log_save(host, ae); - return; } static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) { @@ -537,8 +536,8 @@ void *health_main(void *ptr) { if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))) continue; - int warning_status = RRDCALC_STATUS_UNDEFINED; - int critical_status = RRDCALC_STATUS_UNDEFINED; + RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED; + RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED; // -------------------------------------------------------- // check the warning expression @@ -605,7 +604,7 @@ void *health_main(void *ptr) { // -------------------------------------------------------- // decide the final alarm status - int status = RRDCALC_STATUS_UNDEFINED; + RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED; switch(warning_status) { case RRDCALC_STATUS_CLEAR: diff --git a/src/health.h b/src/health.h index 7028a914b..f41571803 100644 --- a/src/health.h +++ b/src/health.h @@ -5,26 +5,28 @@ extern int default_health_enabled; extern int rrdvar_compare(void *a, void *b); -#define RRDVAR_TYPE_CALCULATED 1 -#define RRDVAR_TYPE_TIME_T 2 -#define RRDVAR_TYPE_COLLECTED 3 -#define RRDVAR_TYPE_TOTAL 4 -#define RRDVAR_TYPE_INT 5 -#define RRDVAR_TYPE_CALCULATED_ALLOCATED 6 - +typedef enum rrdvar_type { + RRDVAR_TYPE_CALCULATED = 1, + RRDVAR_TYPE_TIME_T = 2, + RRDVAR_TYPE_COLLECTED = 3, + RRDVAR_TYPE_TOTAL = 4, + RRDVAR_TYPE_INT = 5, + RRDVAR_TYPE_CALCULATED_ALLOCATED = 6 // a custom variable, allocated on purpose (ie. not inherited from charts) + // used only for custom host global variables +} RRDVAR_TYPE; // the variables as stored in the variables indexes // there are 3 indexes: -// 1. at each chart (RRDSET.variables_root_index) -// 2. at each context (RRDFAMILY.variables_root_index) -// 3. at each host (RRDHOST.variables_root_index) +// 1. at each chart (RRDSET.rrdvar_root_index) +// 2. at each context (RRDFAMILY.rrdvar_root_index) +// 3. at each host (RRDHOST.rrdvar_root_index) typedef struct rrdvar { avl avl; char *name; uint32_t hash; - int type; + RRDVAR_TYPE type; void *value; time_t last_updated; @@ -35,15 +37,24 @@ typedef struct rrdvar { // calculated / processed by the normal data collection process // This means, there will be no speed penalty for using // these variables + +typedef enum rrdvar_options { + RRDVAR_OPTION_DEFAULT = (0 << 0), + RRDVAR_OPTION_ALLOCATED = (1 << 0) // the value ptr is allocated (not a reference) + // future use +} RRDVAR_OPTIONS; + typedef struct rrdsetvar { + char *variable; // variable name + uint32_t hash; // variable name hash + char *key_fullid; // chart type.chart id.variable char *key_fullname; // chart type.chart name.variable - char *variable; // variable - int type; + RRDVAR_TYPE type; void *value; - uint32_t options; + RRDVAR_OPTIONS options; RRDVAR *var_local; RRDVAR *var_family; @@ -75,10 +86,10 @@ typedef struct rrddimvar { char *key_fullnameid; // chart type.chart name + dimension id char *key_fullnamename; // chart type.chart name + dimension name - int type; + RRDVAR_TYPE type; void *value; - uint32_t options; + RRDVAR_OPTIONS options; RRDVAR *var_local_id; RRDVAR *var_local_name; @@ -101,7 +112,7 @@ typedef struct rrddimvar { // calculated variables (defined in health configuration) // These aggregate time-series data at fixed intervals // (defined in their update_every member below) -// These increase the overhead of netdata. +// They increase the overhead of netdata. // // These calculations are allocated and linked (->next) // under RRDHOST. @@ -111,14 +122,6 @@ typedef struct rrddimvar { // having as RRDSET.calculations the RRDCALC to be processed // next. -#define RRDCALC_STATUS_REMOVED -2 -#define RRDCALC_STATUS_UNDEFINED -1 -#define RRDCALC_STATUS_UNINITIALIZED 0 -#define RRDCALC_STATUS_CLEAR 1 -#define RRDCALC_STATUS_RAISED 2 -#define RRDCALC_STATUS_WARNING 3 -#define RRDCALC_STATUS_CRITICAL 4 - #define RRDCALC_FLAG_DB_ERROR 0x00000001 #define RRDCALC_FLAG_DB_NAN 0x00000002 /* #define RRDCALC_FLAG_DB_STALE 0x00000004 */ @@ -179,7 +182,7 @@ typedef struct rrdcalc { // ------------------------------------------------------------------------ // runtime information - int status; // the current status of the alarm + RRDCALC_STATUS status; // the current status of the alarm calculated_number value; // the current value of the alarm calculated_number old_value; // the previous value of the alarm @@ -314,8 +317,8 @@ typedef struct alarm_entry { char *old_value_string; char *new_value_string; - int old_status; - int new_status; + RRDCALC_STATUS old_status; + RRDCALC_STATUS new_status; uint32_t flags; @@ -340,11 +343,11 @@ typedef struct alarm_log { #include "rrd.h" extern void rrdsetvar_rename_all(RRDSET *st); -extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options); +extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options); extern void rrdsetvar_free(RRDSETVAR *rs); extern void rrddimvar_rename_all(RRDDIM *rd); -extern RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options); +extern RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options); extern void rrddimvar_free(RRDDIMVAR *rs); extern void rrdsetcalc_link_matching(RRDSET *st); @@ -364,10 +367,14 @@ extern void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after); void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf); extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name); -extern void rrdvar_custom_host_variable_destroy(RRDHOST *host, const char *name); -extern void rrdvar_custom_host_variable_set(RRDVAR *rv, calculated_number value); +extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value); -extern const char *rrdcalc_status2string(int status); +extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name); +extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, calculated_number value); + +extern void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock); + +extern const char *rrdcalc_status2string(RRDCALC_STATUS status); extern int health_alarm_log_open(RRDHOST *host); @@ -389,8 +396,8 @@ extern void health_alarm_log( time_t duration, calculated_number old_value, calculated_number new_value, - int old_status, - int new_status, + RRDCALC_STATUS old_status, + RRDCALC_STATUS new_status, const char *source, const char *units, const char *info, @@ -403,8 +410,13 @@ extern char *health_config_dir(void); extern void health_reload_host(RRDHOST *host); extern void health_alarm_log_free(RRDHOST *host); -extern void rrdcalc_free(RRDHOST *host, RRDCALC *rc); -extern void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt); +extern void rrdcalc_free(RRDCALC *rc); +extern void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc); + +extern void rrdcalctemplate_free(RRDCALCTEMPLATE *rt); +extern void rrdcalctemplate_unlink_and_free(RRDHOST *host, RRDCALCTEMPLATE *rt); + +extern int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data); #ifdef NETDATA_HEALTH_INTERNALS #define RRDVAR_MAX_LENGTH 1024 @@ -416,7 +428,7 @@ extern int rrdvar_fix_name(char *variable); extern RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart); extern void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc); -extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, void *value); +extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, void *value); extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv); extern void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae); diff --git a/src/health_config.c b/src/health_config.c index 2ead82ef5..108eecc4a 100644 --- a/src/health_config.c +++ b/src/health_config.c @@ -505,11 +505,12 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) { if (rc && (ignore_this || !rrdcalc_add_alarm_from_config(host, rc))) - rrdcalc_free(host, rc); + rrdcalc_free(rc); if(rt) { if (ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) - rrdcalctemplate_free(host, rt); + rrdcalctemplate_free(rt); + rt = NULL; } @@ -532,12 +533,13 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) { if(rc) { if(ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) - rrdcalc_free(host, rc); + rrdcalc_free(rc); + rc = NULL; } if(rt && (ignore_this || !rrdcalctemplate_add_template_from_config(host, rt))) - rrdcalctemplate_free(host, rt); + rrdcalctemplate_free(rt); rt = callocz(1, sizeof(RRDCALCTEMPLATE)); rt->name = strdupz(value); @@ -833,10 +835,10 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { } if(rc && (ignore_this || !rrdcalc_add_alarm_from_config(host, rc))) - rrdcalc_free(host, rc); + rrdcalc_free(rc); if(rt && (ignore_this || !rrdcalctemplate_add_template_from_config(host, rt))) - rrdcalctemplate_free(host, rt); + rrdcalctemplate_free(rt); fclose(fp); return 1; diff --git a/src/health_log.c b/src/health_log.c index 9881d35d4..0314b086c 100644 --- a/src/health_log.c +++ b/src/health_log.c @@ -351,8 +351,8 @@ inline void health_alarm_log( time_t duration, calculated_number old_value, calculated_number new_value, - int old_status, - int new_status, + RRDCALC_STATUS old_status, + RRDCALC_STATUS new_status, const char *source, const char *units, const char *info, diff --git a/src/ipc.c b/src/ipc.c index 1dabf5e19..a9076fca4 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -163,11 +163,12 @@ static inline int ipc_sem_get_status(struct ipc_status *st) { int do_ipc(int update_every, usec_t dt) { (void)dt; - static int initialized = 0, read_limits_next = 0; + static int initialized = 0, read_limits_next = -1; static struct ipc_limits limits; static struct ipc_status status; - static RRDVAR *arrays_max = NULL, *semaphores_max = NULL; - static RRDSET *semaphores = NULL, *arrays = NULL; + static RRDSETVAR *arrays_max = NULL, *semaphores_max = NULL; + static RRDSET *st_semaphores = NULL, *st_arrays = NULL; + static RRDDIM *rd_semaphores = NULL, *rd_arrays = NULL; if(unlikely(!initialized)) { initialized = 1; @@ -184,28 +185,46 @@ int do_ipc(int update_every, usec_t dt) { return 1; } - arrays_max = rrdvar_custom_host_variable_create(localhost, "ipc.semaphores.arrays.max"); - semaphores_max = rrdvar_custom_host_variable_create(localhost, "ipc.semaphores.max"); - - if(arrays_max) rrdvar_custom_host_variable_set(arrays_max, limits.semmni); - if(semaphores_max) rrdvar_custom_host_variable_set(semaphores_max, limits.semmns); - // create the charts - semaphores = rrdset_find_localhost("system.ipc_semaphores"); - if(!semaphores) { - semaphores = rrdset_create_localhost("system", "ipc_semaphores", NULL, "ipc semaphores", NULL - , "IPC Semaphores", "semaphores", 1000, localhost->rrd_update_every - , RRDSET_TYPE_AREA); - rrddim_add(semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + if(unlikely(!st_semaphores)) { + st_semaphores = rrdset_create_localhost( + "system" + , "ipc_semaphores" + , NULL + , "ipc semaphores" + , NULL + , "IPC Semaphores" + , "semaphores" + , "linux" + , "ipc" + , 1000 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA + ); + rd_semaphores = rrddim_add(st_semaphores, "semaphores", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } - arrays = rrdset_find_localhost("system.ipc_semaphore_arrays"); - if(!arrays) { - arrays = rrdset_create_localhost("system", "ipc_semaphore_arrays", NULL, "ipc semaphores", NULL - , "IPC Semaphore Arrays", "arrays", 1000, localhost->rrd_update_every - , RRDSET_TYPE_AREA); - rrddim_add(arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + if(unlikely(!st_arrays)) { + st_arrays = rrdset_create_localhost( + "system" + , "ipc_semaphore_arrays" + , NULL + , "ipc semaphores" + , NULL + , "IPC Semaphore Arrays" + , "arrays" + , "linux" + , "ipc" + , 1000 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA + ); + rd_arrays = rrddim_add(st_arrays, "arrays", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } + + // variables + semaphores_max = rrdsetvar_custom_chart_variable_create(st_semaphores, "ipc.semaphores.max"); + arrays_max = rrdsetvar_custom_chart_variable_create(st_arrays, "ipc.semaphores.arrays.max"); } if(unlikely(read_limits_next < 0)) { @@ -213,11 +232,11 @@ int do_ipc(int update_every, usec_t dt) { error("Unable to fetch semaphore limits."); } else { - if(arrays_max) rrdvar_custom_host_variable_set(arrays_max, limits.semmni); - if(semaphores_max) rrdvar_custom_host_variable_set(semaphores_max, limits.semmns); + if(semaphores_max) rrdsetvar_custom_chart_variable_set(semaphores_max, limits.semmns); + if(arrays_max) rrdsetvar_custom_chart_variable_set(arrays_max, limits.semmni); - arrays->red = limits.semmni; - semaphores->red = limits.semmns; + st_arrays->red = limits.semmni; + st_semaphores->red = limits.semmns; read_limits_next = 60 / update_every; } @@ -230,13 +249,13 @@ int do_ipc(int update_every, usec_t dt) { return 0; } - if(semaphores->counter_done) rrdset_next(semaphores); - rrddim_set(semaphores, "semaphores", status.semaem); - rrdset_done(semaphores); + if(st_semaphores->counter_done) rrdset_next(st_semaphores); + rrddim_set_by_pointer(st_semaphores, rd_semaphores, status.semaem); + rrdset_done(st_semaphores); - if(arrays->counter_done) rrdset_next(arrays); - rrddim_set(arrays, "arrays", status.semusz); - rrdset_done(arrays); + if(st_arrays->counter_done) rrdset_next(st_arrays); + rrddim_set_by_pointer(st_arrays, rd_arrays, status.semusz); + rrdset_done(st_arrays); return 0; } diff --git a/src/macos_fw.c b/src/macos_fw.c index fa103e110..5e8ce0ee4 100644 --- a/src/macos_fw.c +++ b/src/macos_fw.c @@ -112,8 +112,14 @@ int do_macos_iokit(int update_every, usec_t dt) { CFRelease(properties); IOObjectRelease(drive_media); + if(unlikely(!diskstat.name || !*diskstat.name)) { + IOObjectRelease(drive); + continue; + } + /* Obtain the properties for this drive object. */ if (unlikely(IORegistryEntryCreateCFProperties(drive, (CFMutableDictionaryRef *)&properties, kCFAllocatorDefault, 0))) { + IOObjectRelease(drive); error("MACOS: IORegistryEntryCreateCFProperties() failed"); do_io = 0; error("DISABLED: system.io"); @@ -138,9 +144,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk", diskstat.name); if (unlikely(!st)) { - st = rrdset_create_localhost("disk", diskstat.name, NULL, diskstat.name, "disk.io" - , "Disk I/O Bandwidth", "kilobytes/s", 2000, update_every - , RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + "disk" + , diskstat.name + , NULL + , diskstat.name + , "disk.io" + , "Disk I/O Bandwidth" + , "kilobytes/s" + , "macos" + , "iokit" + , 2000 + , update_every + , RRDSET_TYPE_AREA + ); rrddim_add(st, "reads", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "writes", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); @@ -165,9 +182,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_ops", diskstat.name); if (unlikely(!st)) { - st = rrdset_create_localhost("disk_ops", diskstat.name, NULL, diskstat.name, "disk.ops" - , "Disk Completed I/O Operations", "operations/s", 2001 - , update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "disk_ops" + , diskstat.name + , NULL + , diskstat.name + , "disk.ops" + , "Disk Completed I/O Operations" + , "operations/s" + , "macos" + , "iokit" + , 2001 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -193,9 +221,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_util", diskstat.name); if (unlikely(!st)) { - st = rrdset_create_localhost("disk_util", diskstat.name, NULL, diskstat.name, "disk.util" - , "Disk Utilization Time", "% of time working", 2004, update_every - , RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + "disk_util" + , diskstat.name + , NULL + , diskstat.name + , "disk.util" + , "Disk Utilization Time" + , "% of time working" + , "macos" + , "iokit" + , 2004 + , update_every + , RRDSET_TYPE_AREA + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "utilization", NULL, 1, 10000000, RRD_ALGORITHM_INCREMENTAL); @@ -220,9 +259,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name); if (unlikely(!st)) { - st = rrdset_create_localhost("disk_iotime", diskstat.name, NULL, diskstat.name, "disk.iotime" - , "Disk Total I/O Time", "milliseconds/s", 2022, update_every - , RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "disk_iotime" + , diskstat.name + , NULL + , diskstat.name + , "disk.iotime" + , "Disk Total I/O Time" + , "milliseconds/s" + , "macos" + , "iokit" + , 2022 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "reads", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL); @@ -246,9 +296,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_await", diskstat.name); if (unlikely(!st)) { - st = rrdset_create_localhost("disk_await", diskstat.name, NULL, diskstat.name, "disk.await" - , "Average Completed I/O Operation Time", "ms per operation" - , 2005, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "disk_await" + , diskstat.name + , NULL + , diskstat.name + , "disk.await" + , "Average Completed I/O Operation Time" + , "ms per operation" + , "macos" + , "iokit" + , 2005 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "reads", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE); @@ -266,10 +327,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name); if (unlikely(!st)) { - st = rrdset_create_localhost("disk_avgsz", diskstat.name, NULL, diskstat.name, "disk.avgsz" - , "Average Completed I/O Operation Bandwidth" - , "kilobytes per operation", 2006, update_every - , RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + "disk_avgsz" + , diskstat.name + , NULL + , diskstat.name + , "disk.avgsz" + , "Average Completed I/O Operation Bandwidth" + , "kilobytes per operation" + , "macos" + , "iokit" + , 2006 + , update_every + , RRDSET_TYPE_AREA + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "reads", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); @@ -287,9 +358,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name); if (unlikely(!st)) { - st = rrdset_create_localhost("disk_svctm", diskstat.name, NULL, diskstat.name, "disk.svctm" - , "Average Service Time", "ms per operation", 2007 - , update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "disk_svctm" + , diskstat.name + , NULL + , diskstat.name + , "disk.svctm" + , "Average Service Time" + , "ms per operation" + , "macos" + , "iokit" + , 2007 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "svctm", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE); @@ -318,8 +400,20 @@ int do_macos_iokit(int update_every, usec_t dt) { if (likely(do_io)) { st = rrdset_find_bytype_localhost("system", "io"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", "io", NULL, "disk", NULL, "Disk I/O", "kilobytes/s", 150 - , update_every, RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + "system" + , "io" + , NULL + , "disk" + , NULL + , "Disk I/O" + , "kilobytes/s" + , "macos" + , "iokit" + , 150 + , update_every + , RRDSET_TYPE_AREA + ); rrddim_add(st, "in", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "out", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); } @@ -359,14 +453,24 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname); if (unlikely(!st)) { snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); - st = rrdset_create_localhost("disk_space", mntbuf[i].f_mntonname, NULL, mntbuf[i].f_mntonname - , "disk.space", title, "GB", 2023, update_every - , RRDSET_TYPE_STACKED); + st = rrdset_create_localhost( + "disk_space" + , mntbuf[i].f_mntonname + , NULL + , mntbuf[i].f_mntonname + , "disk.space" + , title + , "GB" + , "macos" + , "iokit" + , 2023 + , update_every + , RRDSET_TYPE_STACKED + ); rrddim_add(st, "avail", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "used", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "reserved_for_root", "reserved for root", mntbuf[i].f_bsize, GIGA_FACTOR, - RRD_ALGORITHM_ABSOLUTE); + rrddim_add(st, "reserved_for_root", "reserved for root", mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(st); @@ -382,9 +486,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname); if (unlikely(!st)) { snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); - st = rrdset_create_localhost("disk_inodes", mntbuf[i].f_mntonname, NULL, mntbuf[i].f_mntonname - , "disk.inodes", title, "Inodes", 2024, update_every - , RRDSET_TYPE_STACKED); + st = rrdset_create_localhost( + "disk_inodes" + , mntbuf[i].f_mntonname + , NULL + , mntbuf[i].f_mntonname + , "disk.inodes" + , title + , "Inodes" + , "macos" + , "iokit" + , 2024 + , update_every + , RRDSET_TYPE_STACKED + ); rrddim_add(st, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -417,8 +532,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("net", ifa->ifa_name); if (unlikely(!st)) { - st = rrdset_create_localhost("net", ifa->ifa_name, NULL, ifa->ifa_name, "net.net", "Bandwidth" - , "kilobits/s", 7000, update_every, RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + "net" + , ifa->ifa_name + , NULL + , ifa->ifa_name + , "net.net" + , "Bandwidth" + , "kilobits/s" + , "macos" + , "iokit" + , 7000 + , update_every + , RRDSET_TYPE_AREA + ); rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); @@ -433,8 +560,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name); if (unlikely(!st)) { - st = rrdset_create_localhost("net_packets", ifa->ifa_name, NULL, ifa->ifa_name, "net.packets" - , "Packets", "packets/s", 7001, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "net_packets" + , ifa->ifa_name + , NULL + , ifa->ifa_name + , "net.packets" + , "Packets" + , "packets/s" + , "macos" + , "iokit" + , 7001 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -454,8 +593,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name); if (unlikely(!st)) { - st = rrdset_create_localhost("net_errors", ifa->ifa_name, NULL, ifa->ifa_name, "net.errors" - , "Interface Errors", "errors/s", 7002, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "net_errors" + , ifa->ifa_name + , NULL + , ifa->ifa_name + , "net.errors" + , "Interface Errors" + , "errors/s" + , "macos" + , "iokit" + , 7002 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -471,8 +622,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name); if (unlikely(!st)) { - st = rrdset_create_localhost("net_drops", ifa->ifa_name, NULL, ifa->ifa_name, "net.drops" - , "Interface Drops", "drops/s", 7003, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "net_drops" + , ifa->ifa_name + , NULL + , ifa->ifa_name + , "net.drops" + , "Interface Drops" + , "drops/s" + , "macos" + , "iokit" + , 7003 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -486,9 +649,20 @@ int do_macos_iokit(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name); if (unlikely(!st)) { - st = rrdset_create_localhost("net_events", ifa->ifa_name, NULL, ifa->ifa_name, "net.events" - , "Network Interface Events", "events/s", 7006, update_every - , RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "net_events" + , ifa->ifa_name + , NULL + , ifa->ifa_name + , "net.events" + , "Network Interface Events" + , "events/s" + , "macos" + , "iokit" + , 7006 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "frames", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); diff --git a/src/macos_mach_smi.c b/src/macos_mach_smi.c index da2825513..bcde589f0 100644 --- a/src/macos_mach_smi.c +++ b/src/macos_mach_smi.c @@ -25,7 +25,11 @@ int do_macos_mach_smi(int update_every, usec_t dt) { natural_t cp_time[CPU_STATE_MAX]; // NEEDED BY: do_ram, do_swapio, do_pgfaults +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060) vm_statistics64_data_t vm_statistics; +#else + vm_statistics_data_t vm_statistics; +#endif host = mach_host_self(); kr = host_page_size(host, &system_pagesize); @@ -50,8 +54,20 @@ int do_macos_mach_smi(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("system", "cpu"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", "cpu", NULL, "cpu", "system.cpu", "Total CPU utilization" - , "percentage", 100, update_every, RRDSET_TYPE_STACKED); + st = rrdset_create_localhost( + "system" + , "cpu" + , NULL + , "cpu" + , "system.cpu" + , "Total CPU utilization" + , "percentage" + , "macos" + , "mach_smi" + , 100 + , update_every + , RRDSET_TYPE_STACKED + ); rrddim_add(st, "user", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); rrddim_add(st, "nice", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); @@ -73,8 +89,13 @@ int do_macos_mach_smi(int update_every, usec_t dt) { // -------------------------------------------------------------------- if (likely(do_ram || do_swapio || do_pgfaults)) { +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060) count = sizeof(vm_statistics64_data_t); kr = host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&vm_statistics, &count); +#else + count = sizeof(vm_statistics_data_t); + kr = host_statistics(host, HOST_VM_INFO, (host_info_t)&vm_statistics, &count); +#endif if (unlikely(kr != KERN_SUCCESS)) { error("MACOS: host_statistics64() failed: %s", mach_error_string(kr)); do_ram = 0; @@ -87,13 +108,27 @@ int do_macos_mach_smi(int update_every, usec_t dt) { if (likely(do_ram)) { st = rrdset_find_localhost("system.ram"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", "ram", NULL, "ram", NULL, "System RAM", "MB", 200 - , update_every, RRDSET_TYPE_STACKED); + st = rrdset_create_localhost( + "system" + , "ram" + , NULL + , "ram" + , NULL + , "System RAM" + , "MB" + , "macos" + , "mach_smi" + , 200 + , update_every + , RRDSET_TYPE_STACKED + ); rrddim_add(st, "active", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "wired", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE); +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) rrddim_add(st, "throttled", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "compressor", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE); +#endif rrddim_add(st, "inactive", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "purgeable", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "speculative", NULL, system_pagesize, 1048576, RRD_ALGORITHM_ABSOLUTE); @@ -103,8 +138,10 @@ int do_macos_mach_smi(int update_every, usec_t dt) { rrddim_set(st, "active", vm_statistics.active_count); rrddim_set(st, "wired", vm_statistics.wire_count); +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) rrddim_set(st, "throttled", vm_statistics.throttled_count); rrddim_set(st, "compressor", vm_statistics.compressor_page_count); +#endif rrddim_set(st, "inactive", vm_statistics.inactive_count); rrddim_set(st, "purgeable", vm_statistics.purgeable_count); rrddim_set(st, "speculative", vm_statistics.speculative_count); @@ -112,13 +149,26 @@ int do_macos_mach_smi(int update_every, usec_t dt) { rrdset_done(st); } +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) // -------------------------------------------------------------------- if (likely(do_swapio)) { st = rrdset_find_localhost("system.swapio"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", "swapio", NULL, "swap", NULL, "Swap I/O", "kilobytes/s", 250 - , update_every, RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + "system" + , "swapio" + , NULL + , "swap" + , NULL + , "Swap I/O" + , "kilobytes/s" + , "macos" + , "mach_smi" + , 250 + , update_every + , RRDSET_TYPE_AREA + ); rrddim_add(st, "in", NULL, system_pagesize, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "out", NULL, -system_pagesize, 1024, RRD_ALGORITHM_INCREMENTAL); @@ -129,22 +179,37 @@ int do_macos_mach_smi(int update_every, usec_t dt) { rrddim_set(st, "out", vm_statistics.swapouts); rrdset_done(st); } +#endif // -------------------------------------------------------------------- if (likely(do_pgfaults)) { st = rrdset_find_localhost("mem.pgfaults"); if (unlikely(!st)) { - st = rrdset_create_localhost("mem", "pgfaults", NULL, "system", NULL, "Memory Page Faults" - , "page faults/s", 500, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "mem" + , "pgfaults" + , NULL + , "system" + , NULL + , "Memory Page Faults" + , "page faults/s" + , "macos" + , "mach_smi" + , 500 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "memory", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "cow", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "pagein", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "pageout", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) rrddim_add(st, "compress", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "decompress", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#endif rrddim_add(st, "zero_fill", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "reactivate", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "purge", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -155,8 +220,10 @@ int do_macos_mach_smi(int update_every, usec_t dt) { rrddim_set(st, "cow", vm_statistics.cow_faults); rrddim_set(st, "pagein", vm_statistics.pageins); rrddim_set(st, "pageout", vm_statistics.pageouts); +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) rrddim_set(st, "compress", vm_statistics.compressions); rrddim_set(st, "decompress", vm_statistics.decompressions); +#endif rrddim_set(st, "zero_fill", vm_statistics.zero_fill_count); rrddim_set(st, "reactivate", vm_statistics.reactivations); rrddim_set(st, "purge", vm_statistics.purges); diff --git a/src/macos_sysctl.c b/src/macos_sysctl.c index 843aceae0..cb6fa8af9 100644 --- a/src/macos_sysctl.c +++ b/src/macos_sysctl.c @@ -1,4 +1,5 @@ #include "common.h" +#include #include // NEEDED BY: do_bandwidth #include @@ -112,9 +113,21 @@ int do_macos_sysctl(int update_every, usec_t dt) { /* * Dirty workaround for /usr/include/netinet6/ip6_var.h absence. * Struct ip6stat was copied from bsd/netinet6/ip6_var.h from xnu sources. + * Do the same for previously missing scope6_var.h on OS X < 10.11. */ #define IP6S_SRCRULE_COUNT 16 + +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED < 101100) +#ifndef _NETINET6_SCOPE6_VAR_H_ +#define _NETINET6_SCOPE6_VAR_H_ +#include + +#define SCOPE6_ID_MAX 16 +#endif +#else #include +#endif + struct ip6stat { u_quad_t ip6s_total; /* total packets received */ u_quad_t ip6s_tooshort; /* packet too short */ @@ -217,9 +230,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("system", "load"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", "load", NULL, "load", NULL, "System Load Average", "load" - , 100, (update_every < MIN_LOADAVG_UPDATE_EVERY) - ? MIN_LOADAVG_UPDATE_EVERY : update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "system" + , "load" + , NULL + , "load" + , NULL + , "System Load Average" + , "load" + , "macos" + , "sysctl" + , 100 + , (update_every < MIN_LOADAVG_UPDATE_EVERY) ? MIN_LOADAVG_UPDATE_EVERY : update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "load1", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "load5", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); rrddim_add(st, "load15", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); @@ -246,8 +270,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { } else { st = rrdset_find_localhost("system.swap"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", "swap", NULL, "swap", NULL, "System Swap", "MB", 201 - , update_every, RRDSET_TYPE_STACKED); + st = rrdset_create_localhost( + "system" + , "swap" + , NULL + , "swap" + , NULL + , "System Swap" + , "MB" + , "macos" + , "sysctl" + , 201 + , update_every + , RRDSET_TYPE_STACKED + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "free", NULL, 1, 1048576, RRD_ALGORITHM_ABSOLUTE); @@ -296,8 +332,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { } st = rrdset_find_localhost("system.ipv4"); if (unlikely(!st)) { - st = rrdset_create_localhost("system", "ipv4", NULL, "network", NULL, "IPv4 Bandwidth", "kilobits/s" - , 500, update_every, RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + "system" + , "ipv4" + , NULL + , "network" + , NULL + , "IPv4 Bandwidth" + , "kilobits/s" + , "macos" + , "sysctl" + , 500 + , update_every + , RRDSET_TYPE_AREA + ); rrddim_add(st, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); @@ -334,8 +382,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_tcp_packets)) { st = rrdset_find_localhost("ipv4.tcppackets"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "tcppackets", NULL, "tcp", NULL, "IPv4 TCP Packets" - , "packets/s", 2600, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "tcppackets" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Packets" + , "packets/s" + , "macos" + , "sysctl" + , 2600 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutSegs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -352,8 +412,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_tcp_errors)) { st = rrdset_find_localhost("ipv4.tcperrors"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "tcperrors", NULL, "tcp", NULL, "IPv4 TCP Errors", "packets/s" - , 2700, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "tcperrors" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Errors" + , "packets/s" + , "macos" + , "sysctl" + , 2700 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "InErrs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -373,8 +445,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_tcp_handshake)) { st = rrdset_find_localhost("ipv4.tcphandshake"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "tcphandshake", NULL, "tcp", NULL, "IPv4 TCP Handshake Issues" - , "events/s", 2900, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "tcphandshake" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Handshake Issues" + , "events/s" + , "macos" + , "sysctl" + , 2900 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -397,8 +481,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_tcpext_connaborts = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv4.tcpconnaborts"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "tcpconnaborts", NULL, "tcp", NULL, "TCP Connection Aborts" - , "connections/s", 3010, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "tcpconnaborts" + , NULL + , "tcp" + , NULL + , "TCP Connection Aborts" + , "connections/s" + , "macos" + , "sysctl" + , 3010 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "TCPAbortOnData", "baddata", 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "TCPAbortOnClose", "userclosed", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -420,8 +516,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_tcpext_ofo = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv4.tcpofo"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "tcpofo", NULL, "tcp", NULL, "TCP Out-Of-Order Queue" - , "packets/s", 3050, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "tcpofo" + , NULL + , "tcp" + , NULL + , "TCP Out-Of-Order Queue" + , "packets/s" + , "macos" + , "sysctl" + , 3050 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "TCPOFOQueue", "inqueue", 1, 1, RRD_ALGORITHM_INCREMENTAL); } @@ -438,8 +546,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { st = rrdset_find_localhost("ipv4.tcpsyncookies"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "tcpsyncookies", NULL, "tcp", NULL, "TCP SYN Cookies" - , "packets/s", 3100, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "tcpsyncookies" + , NULL + , "tcp" + , NULL + , "TCP SYN Cookies" + , "packets/s" + , "macos" + , "sysctl" + , 3100 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "SyncookiesRecv", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "SyncookiesSent", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -455,12 +575,26 @@ int do_macos_sysctl(int update_every, usec_t dt) { // -------------------------------------------------------------------- + +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101100) if (do_ecn == CONFIG_BOOLEAN_YES || (do_ecn == CONFIG_BOOLEAN_AUTO && (tcpstat.tcps_ecn_recv_ce || tcpstat.tcps_ecn_not_supported))) { do_ecn = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv4.ecnpkts"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "ecnpkts", NULL, "ecn", NULL, "IPv4 ECN Statistics" - , "packets/s", 8700, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "ecnpkts" + , NULL + , "ecn" + , NULL + , "IPv4 ECN Statistics" + , "packets/s" + , "macos" + , "sysctl" + , 8700 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "InCEPkts", "CEP", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -472,6 +606,7 @@ int do_macos_sysctl(int update_every, usec_t dt) { rrddim_set(st, "InNoECTPkts", tcpstat.tcps_ecn_not_supported); rrdset_done(st); } +#endif } } @@ -489,8 +624,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_udp_packets)) { st = rrdset_find_localhost("ipv4.udppackets"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "udppackets", NULL, "udp", NULL, "IPv4 UDP Packets" - , "packets/s", 2601, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "udppackets" + , NULL + , "udp" + , NULL + , "IPv4 UDP Packets" + , "packets/s" + , "macos" + , "sysctl" + , 2601 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -507,23 +654,41 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_udp_errors)) { st = rrdset_find_localhost("ipv4.udperrors"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "udperrors", NULL, "udp", NULL, "IPv4 UDP Errors", "events/s" - , 2701, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "udperrors" + , NULL + , "udp" + , NULL + , "IPv4 UDP Errors" + , "events/s" + , "macos" + , "sysctl" + , 2701 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#endif } else rrdset_next(st); rrddim_set(st, "InErrors", udpstat.udps_hdrops + udpstat.udps_badlen); rrddim_set(st, "NoPorts", udpstat.udps_noport); rrddim_set(st, "RcvbufErrors", udpstat.udps_fullsock); +#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) rrddim_set(st, "InCsumErrors", udpstat.udps_badsum + udpstat.udps_nosum); rrddim_set(st, "IgnoredMulti", udpstat.udps_filtermcast); +#else + rrddim_set(st, "InCsumErrors", udpstat.udps_badsum); +#endif rrdset_done(st); } } @@ -550,8 +715,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_icmp_packets)) { st = rrdset_find_localhost("ipv4.icmp"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "icmp", NULL, "icmp", NULL, "IPv4 ICMP Packets", "packets/s" - , 2602, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "icmp" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Packets" + , "packets/s" + , "macos" + , "sysctl" + , 2602 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -567,8 +744,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { st = rrdset_find_localhost("ipv4.icmp_errors"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "icmp_errors", NULL, "icmp", NULL, "IPv4 ICMP Errors" - , "packets/s", 2603, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "icmp_errors" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Errors" + , "packets/s" + , "macos" + , "sysctl" + , 2603 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -588,8 +777,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_icmpmsg)) { st = rrdset_find_localhost("ipv4.icmpmsg"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "icmpmsg", NULL, "icmp", NULL, "IPv4 ICMP Messages" - , "packets/s", 2604, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "icmpmsg" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Messages" + , "packets/s" + , "macos" + , "sysctl" + , 2604 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InEchoReps", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutEchoReps", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -625,8 +826,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_ip_packets)) { st = rrdset_find_localhost("ipv4.packets"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "packets", NULL, "packets", NULL, "IPv4 Packets", "packets/s" - , 3000, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "packets" + , NULL + , "packets" + , NULL + , "IPv4 Packets" + , "packets/s" + , "macos" + , "sysctl" + , 3000 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -647,8 +860,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_ip_fragsout)) { st = rrdset_find_localhost("ipv4.fragsout"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "fragsout", NULL, "fragments", NULL, "IPv4 Fragments Sent" - , "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "fragsout" + , NULL + , "fragments" + , NULL + , "IPv4 Fragments Sent" + , "packets/s" + , "macos" + , "sysctl" + , 3010 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -668,8 +893,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_ip_fragsin)) { st = rrdset_find_localhost("ipv4.fragsin"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "fragsin", NULL, "fragments", NULL, "IPv4 Fragments Reassembly" - , "packets/s", 3011, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "fragsin" + , NULL + , "fragments" + , NULL + , "IPv4 Fragments Reassembly" + , "packets/s" + , "macos" + , "sysctl" + , 3011 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -689,8 +926,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { if (likely(do_ip_errors)) { st = rrdset_find_localhost("ipv4.errors"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv4", "errors", NULL, "errors", NULL, "IPv4 Errors", "packets/s" - , 3002, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv4" + , "errors" + , NULL + , "errors" + , NULL + , "IPv4 Errors" + , "packets/s" + , "macos" + , "sysctl" + , 3002 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -734,8 +983,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_ip6_packets = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.packets"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "packets", NULL, "packets", NULL, "IPv6 Packets", "packets/s" - , 3000, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "packets" + , NULL + , "packets" + , NULL + , "IPv6 Packets" + , "packets/s" + , "macos" + , "sysctl" + , 3000 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -759,8 +1020,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_ip6_fragsout = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.fragsout"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "fragsout", NULL, "fragments", NULL, "IPv6 Fragments Sent" - , "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "fragsout" + , NULL + , "fragments" + , NULL + , "IPv6 Fragments Sent" + , "packets/s" + , "macos" + , "sysctl" + , 3010 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -783,8 +1056,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_ip6_fragsin = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.fragsin"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "fragsin", NULL, "fragments", NULL, "IPv6 Fragments Reassembly" - , "packets/s", 3011, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "fragsin" + , NULL + , "fragments" + , NULL + , "IPv6 Fragments Reassembly" + , "packets/s" + , "macos" + , "sysctl" + , 3011 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -816,8 +1101,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_ip6_errors = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.errors"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "errors", NULL, "errors", NULL, "IPv6 Errors", "packets/s" - , 3002, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "errors" + , NULL + , "errors" + , NULL + , "IPv6 Errors" + , "packets/s" + , "macos" + , "sysctl" + , 3002 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -863,8 +1160,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_icmp6 = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.icmp"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "icmp", NULL, "icmp", NULL, "IPv6 ICMP Messages", "messages/s" - , 10000, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "icmp" + , NULL + , "icmp" + , NULL + , "IPv6 ICMP Messages" + , "messages/s" + , "macos" + , "sysctl" + , 10000 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -882,8 +1191,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_icmp6_redir = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.icmpredir"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "icmpredir", NULL, "icmp", NULL, "IPv6 ICMP Redirects" - , "redirects/s", 10050, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "icmpredir" + , NULL + , "icmp" + , NULL + , "IPv6 ICMP Redirects" + , "redirects/s" + , "macos" + , "sysctl" + , 10050 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -912,8 +1233,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_icmp6_errors = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.icmperrors"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "icmperrors", NULL, "icmp", NULL, "IPv6 ICMP Errors" - , "errors/s", 10100, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "icmperrors" + , NULL + , "icmp" + , NULL + , "IPv6 ICMP Errors" + , "errors/s" + , "macos" + , "sysctl" + , 10100 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -952,8 +1285,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_icmp6_echos = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.icmpechos"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "icmpechos", NULL, "icmp", NULL, "IPv6 ICMP Echo", "messages/s" - , 10200, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "icmpechos" + , NULL + , "icmp" + , NULL + , "IPv6 ICMP Echo" + , "messages/s" + , "macos" + , "sysctl" + , 10200 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -979,8 +1324,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_icmp6_router = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.icmprouter"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "icmprouter", NULL, "icmp", NULL, "IPv6 Router Messages" - , "messages/s", 10400, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "icmprouter" + , NULL + , "icmp" + , NULL + , "IPv6 Router Messages" + , "messages/s" + , "macos" + , "sysctl" + , 10400 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1006,8 +1363,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_icmp6_neighbor = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.icmpneighbor"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "icmpneighbor", NULL, "icmp", NULL, "IPv6 Neighbor Messages" - , "messages/s", 10500, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "icmpneighbor" + , NULL + , "icmp" + , NULL + , "IPv6 Neighbor Messages" + , "messages/s" + , "macos" + , "sysctl" + , 10500 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1039,8 +1408,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { do_icmp6_types = CONFIG_BOOLEAN_YES; st = rrdset_find_localhost("ipv6.icmptypes"); if (unlikely(!st)) { - st = rrdset_create_localhost("ipv6", "icmptypes", NULL, "icmp", NULL, "IPv6 ICMP Types" - , "messages/s", 10700, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "ipv6" + , "icmptypes" + , NULL + , "icmp" + , NULL + , "IPv6 ICMP Types" + , "messages/s" + , "macos" + , "sysctl" + , 10700 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "InType1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "InType128", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -1081,8 +1462,20 @@ int do_macos_sysctl(int update_every, usec_t dt) { st = rrdset_find_localhost("system.uptime"); if(unlikely(!st)) { - st = rrdset_create_localhost("system", "uptime", NULL, "uptime", NULL, "System Uptime", "seconds", 1000 - , update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "system" + , "uptime" + , NULL + , "uptime" + , NULL + , "System Uptime" + , "seconds" + , "macos" + , "sysctl" + , 1000 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "uptime", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(st); diff --git a/src/main.c b/src/main.c index 89ca828a1..b14ebc0c1 100644 --- a/src/main.c +++ b/src/main.c @@ -32,24 +32,34 @@ void netdata_cleanup_and_exit(int ret) { } struct netdata_static_thread static_threads[] = { + #ifdef INTERNAL_PLUGIN_NFACCT -// nfacct requires root access + // nfacct requires root access // so, we build it as an external plugin with setuid to root {"nfacct", CONFIG_SECTION_PLUGINS, "nfacct", 1, NULL, NULL, nfacct_main}, #endif - {"tc", CONFIG_SECTION_PLUGINS, "tc", 1, NULL, NULL, tc_main}, - {"idlejitter", CONFIG_SECTION_PLUGINS, "idlejitter", 1, NULL, NULL, cpuidlejitter_main}, +#ifdef NETDATA_INTERNAL_CHECKS + // debugging plugin + {"check", CONFIG_SECTION_PLUGINS, "checks", 0, NULL, NULL, checks_main}, +#endif + #if defined(__FreeBSD__) + // FreeBSD internal plugins {"freebsd", CONFIG_SECTION_PLUGINS, "freebsd", 1, NULL, NULL, freebsd_main}, #elif defined(__APPLE__) + // macOS internal plugins {"macos", CONFIG_SECTION_PLUGINS, "macos", 1, NULL, NULL, macos_main}, #else + // linux internal plugins {"proc", CONFIG_SECTION_PLUGINS, "proc", 1, NULL, NULL, proc_main}, {"diskspace", CONFIG_SECTION_PLUGINS, "diskspace", 1, NULL, NULL, proc_diskspace_main}, {"cgroups", CONFIG_SECTION_PLUGINS, "cgroups", 1, NULL, NULL, cgroups_main}, + {"tc", CONFIG_SECTION_PLUGINS, "tc", 1, NULL, NULL, tc_main}, #endif /* __FreeBSD__, __APPLE__*/ - {"check", CONFIG_SECTION_PLUGINS, "checks", 0, NULL, NULL, checks_main}, + + // common plugins for all systems + {"idlejitter", CONFIG_SECTION_PLUGINS, "idlejitter", 1, NULL, NULL, cpuidlejitter_main}, {"backends", NULL, NULL, 1, NULL, NULL, backends_main}, {"health", NULL, NULL, 1, NULL, NULL, health_main}, {"plugins.d", NULL, NULL, 1, NULL, NULL, pluginsd_main}, @@ -57,6 +67,7 @@ struct netdata_static_thread static_threads[] = { {"web-single-threaded", NULL, NULL, 0, NULL, NULL, socket_listen_main_single_threaded}, {"push-metrics", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread}, {"statsd", NULL, NULL, 1, NULL, NULL, statsd_main}, + {NULL, NULL, NULL, 0, NULL, NULL, NULL} }; @@ -83,6 +94,13 @@ void web_server_config_options(void) { web_x_frame_options = config_get(CONFIG_SECTION_WEB, "x-frame-options response header", ""); if(!*web_x_frame_options) web_x_frame_options = NULL; + web_allow_connections_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"), SIMPLE_PATTERN_EXACT); + web_allow_dashboard_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"), SIMPLE_PATTERN_EXACT); + web_allow_badges_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), SIMPLE_PATTERN_EXACT); + web_allow_registry_from = simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), SIMPLE_PATTERN_EXACT); + web_allow_streaming_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow streaming from", "*"), SIMPLE_PATTERN_EXACT); + web_allow_netdataconf_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow netdata.conf from", "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.* 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.* 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.* 172.31.*"), SIMPLE_PATTERN_EXACT); + #ifdef NETDATA_WITH_ZLIB web_enable_gzip = config_get_boolean(CONFIG_SECTION_WEB, "enable gzip compression", web_enable_gzip); @@ -195,23 +213,8 @@ void kill_childs() tc_child_pid = 0; } - struct plugind *cd; - for(cd = pluginsd_root ; cd ; cd = cd->next) { - if(cd->enabled && !cd->obsolete) { - info("Stopping %s plugin thread", cd->id); - pthread_cancel(cd->thread); - - if(cd->pid) { - info("killing %s plugin child process pid %d", cd->id, cd->pid); - if(killpid(cd->pid, SIGTERM) != -1) - waitid(P_PID, (id_t) cd->pid, &info, WEXITED); - - cd->pid = 0; - } - - cd->obsolete = 1; - } - } + // stop all running plugins + pluginsd_stop_all_external_plugins(); // if, for any reason there is any child exited // catch it here @@ -221,7 +224,7 @@ void kill_childs() info("All threads/childs stopped."); } -struct option_def options[] = { +struct option_def option_definitions[] = { // opt description arg name default value { 'c', "Configuration file to load.", "filename", CONFIG_DIR "/" CONFIG_FILENAME}, { 'D', "Do not fork. Run in the foreground.", NULL, "run in the background"}, @@ -237,21 +240,21 @@ struct option_def options[] = { { 'W', "See Advanced options below.", "options", NULL}, }; -void help(int exitcode) { +int help(int exitcode) { FILE *stream; if(exitcode == 0) stream = stdout; else stream = stderr; - int num_opts = sizeof(options) / sizeof(struct option_def); + int num_opts = sizeof(option_definitions) / sizeof(struct option_def); int i; int max_len_arg = 0; // Compute maximum argument length for( i = 0; i < num_opts; i++ ) { - if(options[i].arg_name) { - int len_arg = (int)strlen(options[i].arg_name); + if(option_definitions[i].arg_name) { + int len_arg = (int)strlen(option_definitions[i].arg_name); if(len_arg > max_len_arg) max_len_arg = len_arg; } } @@ -289,9 +292,9 @@ void help(int exitcode) { // Output options description. for( i = 0; i < num_opts; i++ ) { - fprintf(stream, " -%c %-*s %s", options[i].val, max_len_arg, options[i].arg_name ? options[i].arg_name : "", options[i].description); - if(options[i].default_value) { - fprintf(stream, "\n %c %-*s Default: %s\n", ' ', max_len_arg, "", options[i].default_value); + fprintf(stream, " -%c %-*s %s", option_definitions[i].val, max_len_arg, option_definitions[i].arg_name ? option_definitions[i].arg_name : "", option_definitions[i].description); + if(option_definitions[i].default_value) { + fprintf(stream, "\n %c %-*s Default: %s\n", ' ', max_len_arg, "", option_definitions[i].default_value); } else { fprintf(stream, "\n"); } @@ -316,7 +319,7 @@ void help(int exitcode) { ); fflush(stream); - exit(exitcode); + return exitcode; } // TODO: Remove this function with the nix major release. @@ -454,17 +457,23 @@ static void get_netdata_configured_variables() { } // ------------------------------------------------------------------------ - // let the plugins know the min update_every - // get system paths + netdata_configured_config_dir = config_get(CONFIG_SECTION_GLOBAL, "config directory", CONFIG_DIR); netdata_configured_log_dir = config_get(CONFIG_SECTION_GLOBAL, "log directory", LOG_DIR); - netdata_configured_plugins_dir = config_get(CONFIG_SECTION_GLOBAL, "plugins directory", PLUGINS_DIR); netdata_configured_web_dir = config_get(CONFIG_SECTION_GLOBAL, "web files directory", WEB_DIR); netdata_configured_cache_dir = config_get(CONFIG_SECTION_GLOBAL, "cache directory", CACHE_DIR); netdata_configured_varlib_dir = config_get(CONFIG_SECTION_GLOBAL, "lib directory", VARLIB_DIR); netdata_configured_home_dir = config_get(CONFIG_SECTION_GLOBAL, "home directory", CACHE_DIR); + { + char plugins_dirs[(FILENAME_MAX * 2) + 1]; + snprintfz(plugins_dirs, FILENAME_MAX * 2, "\"%s\" \"%s/custom-plugins.d\"", PLUGINS_DIR, CONFIG_DIR); + netdata_configured_plugins_dir_base = strdupz(config_get(CONFIG_SECTION_GLOBAL, "plugins directory", plugins_dirs)); + quoted_strings_splitter(netdata_configured_plugins_dir_base, plugin_directories, PLUGINSD_MAX_DIRECTORIES, config_isspace); + netdata_configured_plugins_dir = plugin_directories[0]; + } + // ------------------------------------------------------------------------ // get default memory mode for the database @@ -489,6 +498,95 @@ static void get_netdata_configured_variables() { get_system_pid_max(); } +static void get_system_timezone(void) { + // avoid flood calls to stat(/etc/localtime) + // http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux + const char *tz = getenv("TZ"); + if(!tz || !*tz) + setenv("TZ", config_get(CONFIG_SECTION_GLOBAL, "TZ environment variable", ":/etc/localtime"), 0); + + char buffer[FILENAME_MAX + 1] = ""; + const char *timezone = NULL; + ssize_t ret; + + // use the TZ variable + if(tz && *tz && *tz != ':') { + timezone = tz; + // info("TIMEZONE: using TZ variable '%s'", timezone); + } + + // use the contents of /etc/timezone + if(!timezone && !read_file("/etc/timezone", buffer, FILENAME_MAX)) { + timezone = buffer; + // info("TIMEZONE: using the contents of /etc/timezone: '%s'", timezone); + } + + // read the link /etc/localtime + if(!timezone) { + ret = readlink("/etc/localtime", buffer, FILENAME_MAX); + + if(ret > 0) { + buffer[ret] = '\0'; + + char *cmp = "/usr/share/zoneinfo/"; + size_t cmp_len = strlen(cmp); + + char *s = strstr(buffer, cmp); + if (s && s[cmp_len]) { + timezone = &s[cmp_len]; + // info("TIMEZONE: using the link of /etc/localtime: '%s'", timezone); + } + } + else + buffer[0] = '\0'; + } + + // find the timezone from strftime() + if(!timezone) { + time_t t; + struct tm *tmp, tmbuf; + + t = now_realtime_sec(); + tmp = localtime_r(&t, &tmbuf); + + if (tmp != NULL) { + if(strftime(buffer, FILENAME_MAX, "%Z", tmp) == 0) + buffer[0] = '\0'; + else { + buffer[FILENAME_MAX] = '\0'; + timezone = buffer; + // info("TIMEZONE: using strftime(): '%s'", timezone); + } + } + } + + if(timezone && *timezone) { + // make sure it does not have illegal characters + // info("TIMEZONE: fixing '%s'", timezone); + + size_t len = strlen(timezone); + char tmp[len + 1]; + char *d = tmp; + *d = '\0'; + + while(*timezone) { + if(isalnum(*timezone) || *timezone == '_' || *timezone == '/') + *d++ = *timezone++; + else + timezone++; + } + *d = '\0'; + strncpyz(buffer, tmp, len); + timezone = buffer; + // info("TIMEZONE: fixed as '%s'", timezone); + } + + if(!timezone || !*timezone) + timezone = "unknown"; + + netdata_configured_timezone = config_get(CONFIG_SECTION_GLOBAL, "timezone", timezone); +} + void set_global_environment() { { char b[16]; @@ -506,11 +604,7 @@ void set_global_environment() { setenv("HOME" , verify_required_directory(netdata_configured_home_dir), 1); setenv("NETDATA_HOST_PREFIX", netdata_configured_host_prefix, 1); - // avoid flood calls to stat(/etc/localtime) - // http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux - const char *tz = getenv("TZ"); - if(!tz || !*tz) - setenv("TZ", config_get(CONFIG_SECTION_GLOBAL, "TZ environment variable", ":/etc/localtime"), 0); + get_system_timezone(); // set the path we need char path[1024 + 1], *p = getenv("PATH"); @@ -571,14 +665,14 @@ int main(int argc, char **argv) { // parse options { - int num_opts = sizeof(options) / sizeof(struct option_def); + int num_opts = sizeof(option_definitions) / sizeof(struct option_def); char optstring[(num_opts * 2) + 1]; int string_i = 0; for( i = 0; i < num_opts; i++ ) { - optstring[string_i] = options[i].val; + optstring[string_i] = option_definitions[i].val; string_i++; - if(options[i].arg_name) { + if(option_definitions[i].arg_name) { optstring[string_i] = ':'; string_i++; } @@ -593,7 +687,7 @@ int main(int argc, char **argv) { case 'c': if(config_load(optarg, 1) != 1) { error("Cannot load configuration file %s.", optarg); - exit(1); + return 1; } else { debug(D_OPTIONS, "Configuration loaded from %s.", optarg); @@ -604,8 +698,7 @@ int main(int argc, char **argv) { dont_fork = 1; break; case 'h': - help(0); - break; + return help(0); case 'i': config_set(CONFIG_SECTION_WEB, "bind to", optarg); break; @@ -635,8 +728,8 @@ int main(int argc, char **argv) { char* debug_flags_string = "debug_flags="; if(strcmp(optarg, "unittest") == 0) { - if(unit_test_buffer()) exit(1); - if(unit_test_str2ld()) exit(1); + if(unit_test_buffer()) return 1; + if(unit_test_str2ld()) return 1; //default_rrd_update_every = 1; //default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; //if(!config_loaded) config_load(NULL, 0); @@ -646,10 +739,10 @@ int main(int argc, char **argv) { default_health_enabled = 0; rrd_init("unittest"); default_rrdpush_enabled = 0; - if(run_all_mockup_tests()) exit(1); - if(unit_test_storage()) exit(1); + if(run_all_mockup_tests()) return 1; + if(unit_test_storage()) return 1; fprintf(stderr, "\n\nALL TESTS PASSED\n\n"); - exit(0); + return 0; } else if(strcmp(optarg, "simple-pattern") == 0) { if(optind + 2 > argc) { @@ -673,24 +766,25 @@ int main(int argc, char **argv) { " -W simple-pattern '!/path/*/*.ext /path/*.ext' '/path/test.ext'\n" "\n" ); - exit(1); + return 1; } const char *heystack = argv[optind]; const char *needle = argv[optind + 1]; + size_t len = strlen(needle) + 1; + char wildcarded[len]; - SIMPLE_PATTERN *p = simple_pattern_create(heystack - , SIMPLE_PATTERN_EXACT); - int ret = simple_pattern_matches(p, needle); + SIMPLE_PATTERN *p = simple_pattern_create(heystack, SIMPLE_PATTERN_EXACT); + int ret = simple_pattern_matches_extract(p, needle, wildcarded, len); simple_pattern_free(p); if(ret) { - fprintf(stdout, "RESULT: MATCHED - pattern '%s' matches '%s'\n", heystack, needle); - exit(0); + fprintf(stdout, "RESULT: MATCHED - pattern '%s' matches '%s', wildcarded '%s'\n", heystack, needle, wildcarded); + return 0; } else { - fprintf(stdout, "RESULT: NOT MATCHED - pattern '%s' does not match '%s'\n", heystack, needle); - exit(1); + fprintf(stdout, "RESULT: NOT MATCHED - pattern '%s' does not match '%s', wildcarded '%s'\n", heystack, needle, wildcarded); + return 1; } } else if(strncmp(optarg, stacksize_string, strlen(stacksize_string)) == 0) { @@ -716,7 +810,7 @@ int main(int argc, char **argv) { " parameters." "\n" ); - exit(1); + return 1; } const char *section = argv[optind]; const char *key = argv[optind + 1]; @@ -741,7 +835,7 @@ int main(int argc, char **argv) { " -c netdata.conf has to be given before -W get.\n" "\n" ); - exit(1); + return 1; } if(!config_loaded) { @@ -757,18 +851,18 @@ int main(int argc, char **argv) { const char *def = argv[optind + 2]; const char *value = config_get(section, key, def); printf("%s\n", value); - exit(0); + return 0; } else { fprintf(stderr, "Unknown -W parameter '%s'\n", optarg); - help(1); + return help(1); } } break; + default: /* ? */ fprintf(stderr, "Unknown parameter '%c'\n", opt); - help(1); - break; + return help(1); } } } @@ -906,6 +1000,7 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // create the listening sockets + web_client_api_v1_init(); web_server_threading_selection(); if(web_server_mode != WEB_SERVER_MODE_NONE) @@ -997,4 +1092,8 @@ int main(int argc, char **argv) { // Handle signals signals_handle(); + + // should never reach this point + // but we need it for rpmlint #2752 + return 1; } diff --git a/src/main.h b/src/main.h index 38df0fea4..09567bc7c 100644 --- a/src/main.h +++ b/src/main.h @@ -16,12 +16,6 @@ struct option_def { const char *default_value; }; -/** - * List of command line options. - * This can be used to compute manpage, help messages, ect. - */ -extern struct option_def options[]; - struct netdata_static_thread { char *name; diff --git a/src/plugin_checks.c b/src/plugin_checks.c index 3a0a83bda..9c1e42cc6 100644 --- a/src/plugin_checks.c +++ b/src/plugin_checks.c @@ -1,5 +1,7 @@ #include "common.h" +#ifdef NETDATA_INTERNAL_CHECKS + void *checks_main(void *ptr) { struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; @@ -16,18 +18,55 @@ void *checks_main(void *ptr) { RRDSET *check1, *check2, *check3, *apps_cpu = NULL; - check1 = rrdset_create_localhost("netdata", "check1", NULL, "netdata", NULL, "Caller gives microseconds" - , "a million !", 99999, localhost->rrd_update_every, RRDSET_TYPE_LINE); + check1 = rrdset_create_localhost( + "netdata" + , "check1" + , NULL + , "netdata" + , NULL + , "Caller gives microseconds" + , "a million !" + , "netdata" + , "checks" + , 99999 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(check1, "absolute", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(check1, "incremental", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - check2 = rrdset_create_localhost("netdata", "check2", NULL, "netdata", NULL, "Netdata calcs microseconds" - , "a million !", 99999, localhost->rrd_update_every, RRDSET_TYPE_LINE); + check2 = rrdset_create_localhost( + "netdata" + , "check2" + , NULL + , "netdata" + , NULL + , "Netdata calcs microseconds" + , "a million !" + , "netdata" + , "checks" + , 99999 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); rrddim_add(check2, "absolute", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(check2, "incremental", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - check3 = rrdset_create_localhost("netdata", "checkdt", NULL, "netdata", NULL, "Clock difference" - , "microseconds diff", 99999, localhost->rrd_update_every, RRDSET_TYPE_LINE); + check3 = rrdset_create_localhost( + "netdata" + , "checkdt" + , NULL + , "netdata" + , NULL + , "Clock difference" + , "microseconds diff" + , "netdata" + , "checks" + , 99999 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); rrddim_add(check3, "caller", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(check3, "netdata", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(check3, "apps.plugin", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -87,3 +126,4 @@ void *checks_main(void *ptr) { return NULL; } +#endif // NETDATA_INTERNAL_CHECKS diff --git a/src/plugin_checks.h b/src/plugin_checks.h index c27685b84..05a40bea0 100644 --- a/src/plugin_checks.h +++ b/src/plugin_checks.h @@ -1,6 +1,8 @@ #ifndef NETDATA_PLUGIN_CHECKS_H #define NETDATA_PLUGIN_CHECKS_H 1 +#ifdef NETDATA_INTERNAL_CHECKS void *checks_main(void *ptr); +#endif // NETDATA_INTERNAL_CHECKS #endif /* NETDATA_PLUGIN_PROC_H */ diff --git a/src/plugin_freebsd.c b/src/plugin_freebsd.c index c09b28a5f..a7825d850 100644 --- a/src/plugin_freebsd.c +++ b/src/plugin_freebsd.c @@ -37,6 +37,7 @@ static struct freebsd_module { // CPU metrics { .name = "kern.cp_times", .dim = "cp_times", .enabled = 1, .func = do_kern_cp_times }, { .name = "dev.cpu.temperature", .dim = "cpu_temperature", .enabled = 1, .func = do_dev_cpu_temperature }, + { .name = "dev.cpu.0.freq", .dim = "cpu_frequency", .enabled = 1, .func = do_dev_cpu_0_freq }, // disk metrics { .name = "kern.devstat", .dim = "kern_devstat", .enabled = 1, .func = do_kern_devstat }, @@ -56,6 +57,7 @@ static struct freebsd_module { // ZFS metrics { .name = "kstat.zfs.misc.arcstats", .dim = "arcstats", .enabled = 1, .func = do_kstat_zfs_misc_arcstats }, + { .name = "kstat.zfs.misc.zio_trim", .dim = "trim", .enabled = 1, .func = do_kstat_zfs_misc_zio_trim }, // ipfw metrics { .name = "ipfw", .dim = "ipfw", .enabled = 1, .func = do_ipfw }, @@ -127,9 +129,20 @@ void *freebsd_main(void *ptr) { st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules"); if(!st) { - st = rrdset_create_localhost("netdata", "plugin_freebsd_modules", NULL, "freebsd", NULL - , "NetData FreeBSD Plugin Modules Durations", "milliseconds/run", 132001 - , localhost->rrd_update_every, RRDSET_TYPE_STACKED); + st = rrdset_create_localhost( + "netdata" + , "plugin_freebsd_modules" + , NULL + , "freebsd" + , NULL + , "NetData FreeBSD Plugin Modules Durations" + , "milliseconds/run" + , "netdata" + , "stats" + , 132001 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); for(i = 0 ; freebsd_modules[i].name ;i++) { struct freebsd_module *pm = &freebsd_modules[i]; diff --git a/src/plugin_freebsd.h b/src/plugin_freebsd.h index 78fe33d7e..0a6f40c15 100644 --- a/src/plugin_freebsd.h +++ b/src/plugin_freebsd.h @@ -18,6 +18,7 @@ extern int do_vm_vmtotal(int update_every, usec_t dt); extern int do_kern_cp_time(int update_every, usec_t dt); extern int do_kern_cp_times(int update_every, usec_t dt); extern int do_dev_cpu_temperature(int update_every, usec_t dt); +extern int do_dev_cpu_0_freq(int update_every, usec_t dt); extern int do_hw_intcnt(int update_every, usec_t dt); extern int do_vm_stats_sys_v_intr(int update_every, usec_t dt); extern int do_vm_stats_sys_v_soft(int update_every, usec_t dt); @@ -43,6 +44,7 @@ extern int do_getifaddrs(int update_every, usec_t dt); extern int do_getmntinfo(int update_every, usec_t dt); extern int do_kern_devstat(int update_every, usec_t dt); extern int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt); +extern int do_kstat_zfs_misc_zio_trim(int update_every, usec_t dt); extern int do_ipfw(int update_every, usec_t dt); #define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int)) diff --git a/src/plugin_idlejitter.c b/src/plugin_idlejitter.c index 89f490233..8d9336835 100644 --- a/src/plugin_idlejitter.c +++ b/src/plugin_idlejitter.c @@ -27,6 +27,8 @@ void *cpuidlejitter_main(void *ptr) { , NULL , "CPU Idle Jitter" , "microseconds lost/s" + , "idlejitter" + , NULL , 9999 , localhost->rrd_update_every , RRDSET_TYPE_AREA diff --git a/src/plugin_nfacct.c b/src/plugin_nfacct.c index 4c691be05..8319c6726 100644 --- a/src/plugin_nfacct.c +++ b/src/plugin_nfacct.c @@ -302,6 +302,8 @@ static void nfstat_send_metrics() { , NULL , "Connection Tracker New Connections" , "connections/s" + , "nfacct" + , NULL , 3001 , nfstat_root.update_every , RRDSET_TYPE_LINE @@ -336,6 +338,8 @@ static void nfstat_send_metrics() { , NULL , "Connection Tracker Changes" , "changes/s" + , "nfacct" + , NULL , 3002 , nfstat_root.update_every , RRDSET_TYPE_LINE @@ -371,6 +375,8 @@ static void nfstat_send_metrics() { , NULL , "Connection Tracker Searches" , "searches/s" + , "nfacct" + , NULL , 3010 , nfstat_root.update_every , RRDSET_TYPE_LINE @@ -406,6 +412,8 @@ static void nfstat_send_metrics() { , NULL , "Connection Tracker Errors" , "events/s" + , "nfacct" + , NULL , 3005 , nfstat_root.update_every , RRDSET_TYPE_LINE @@ -443,6 +451,8 @@ static void nfstat_send_metrics() { , NULL , "Connection Tracker Expectations" , "expectations/s" + , "nfacct" + , NULL , 3003 , nfstat_root.update_every , RRDSET_TYPE_LINE @@ -661,6 +671,8 @@ static void nfacct_send_metrics() { , NULL , "Netfilter Accounting Packets" , "packets/s" + , "nfacct" + , NULL , 3206 , nfacct_root.update_every , RRDSET_TYPE_STACKED @@ -702,6 +714,8 @@ static void nfacct_send_metrics() { , NULL , "Netfilter Accounting Bandwidth" , "kilobytes/s" + , "nfacct" + , NULL , 3207 , nfacct_root.update_every , RRDSET_TYPE_STACKED diff --git a/src/plugin_proc.c b/src/plugin_proc.c index e64f57398..c4249c847 100644 --- a/src/plugin_proc.c +++ b/src/plugin_proc.c @@ -32,6 +32,8 @@ static struct proc_module { // network metrics { .name = "/proc/net/dev", .dim = "netdev", .func = do_proc_net_dev }, + { .name = "/proc/net/sockstat", .dim = "sockstat", .func = do_proc_net_sockstat }, + { .name = "/proc/net/sockstat6", .dim = "sockstat6", .func = do_proc_net_sockstat6 }, { .name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat }, // this has to be before /proc/net/snmp, because there is a shared metric { .name = "/proc/net/snmp", .dim = "snmp", .func = do_proc_net_snmp }, { .name = "/proc/net/snmp6", .dim = "snmp6", .func = do_proc_net_snmp6 }, @@ -118,9 +120,20 @@ void *proc_main(void *ptr) { st = rrdset_find_bytype_localhost("netdata", "plugin_proc_modules"); if(!st) { - st = rrdset_create_localhost("netdata", "plugin_proc_modules", NULL, "proc", NULL - , "NetData Proc Plugin Modules Durations", "milliseconds/run", 132001 - , localhost->rrd_update_every, RRDSET_TYPE_STACKED); + st = rrdset_create_localhost( + "netdata" + , "plugin_proc_modules" + , NULL + , "proc" + , NULL + , "NetData Proc Plugin Modules Durations" + , "milliseconds/run" + , "netdata" + , "stats" + , 132001 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); for(i = 0 ; proc_modules[i].name ;i++) { struct proc_module *pm = &proc_modules[i]; diff --git a/src/plugin_proc.h b/src/plugin_proc.h index 72cfc6aa9..fa5675440 100644 --- a/src/plugin_proc.h +++ b/src/plugin_proc.h @@ -26,6 +26,8 @@ extern int do_proc_uptime(int update_every, usec_t dt); extern int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt); extern int do_proc_sys_devices_system_node(int update_every, usec_t dt); extern int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt); +extern int do_proc_net_sockstat(int update_every, usec_t dt); +extern int do_proc_net_sockstat6(int update_every, usec_t dt); extern int get_numa_node_count(void); diff --git a/src/plugin_proc_diskspace.c b/src/plugin_proc_diskspace.c index 52c1f5ae6..e41e76182 100644 --- a/src/plugin_proc_diskspace.c +++ b/src/plugin_proc_diskspace.c @@ -227,6 +227,12 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { fsfilcnt_t freserved_root = favail_root - favail; fsfilcnt_t fused = ftotal - favail_root; + if(m->do_inodes == CONFIG_BOOLEAN_AUTO && favail == (fsfilcnt_t)-1) { + // this file system does not support inodes reporting + // eg. cephfs + m->do_inodes = CONFIG_BOOLEAN_NO; + } + #ifdef NETDATA_INTERNAL_CHECKS if(unlikely(btotal != bavail + breserved_root + bused)) error("DISKSPACE: disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused); @@ -251,6 +257,8 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { , "disk.space" , title , "GB" + , "diskspace" + , NULL , 2023 , update_every , RRDSET_TYPE_STACKED @@ -289,6 +297,8 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { , "disk.inodes" , title , "Inodes" + , "diskspace" + , NULL , 2024 , update_every , RRDSET_TYPE_STACKED @@ -384,21 +394,21 @@ void *proc_diskspace_main(void *ptr) { getrusage(RUSAGE_THREAD, &thread); - if(!stcpu_thread) { - stcpu_thread = rrdset_find_localhost("netdata.plugin_diskspace"); - if(!stcpu_thread) - stcpu_thread = rrdset_create_localhost( - "netdata" - , "plugin_diskspace" - , NULL - , "diskspace" - , NULL - , "NetData Disk Space Plugin CPU usage" - , "milliseconds/s" - , 132020 - , update_every - , RRDSET_TYPE_STACKED - ); + if(unlikely(!stcpu_thread)) { + stcpu_thread = rrdset_create_localhost( + "netdata" + , "plugin_diskspace" + , NULL + , "diskspace" + , NULL + , "NetData Disk Space Plugin CPU usage" + , "milliseconds/s" + , "diskspace" + , NULL + , 132020 + , update_every + , RRDSET_TYPE_STACKED + ); rd_user = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); @@ -412,21 +422,21 @@ void *proc_diskspace_main(void *ptr) { // ---------------------------------------------------------------- - if(!st_duration) { - st_duration = rrdset_find_localhost("netdata.plugin_diskspace_dt"); - if(!st_duration) - st_duration = rrdset_create_localhost( - "netdata" - , "plugin_diskspace_dt" - , NULL - , "diskspace" - , NULL - , "NetData Disk Space Plugin Duration" - , "milliseconds/run" - , 132021 - , update_every - , RRDSET_TYPE_AREA - ); + if(unlikely(!st_duration)) { + st_duration = rrdset_create_localhost( + "netdata" + , "plugin_diskspace_dt" + , NULL + , "diskspace" + , NULL + , "NetData Disk Space Plugin Duration" + , "milliseconds/run" + , "diskspace" + , NULL + , 132021 + , update_every + , RRDSET_TYPE_AREA + ); rd_duration = rrddim_add(st_duration, "duration", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); } diff --git a/src/plugin_tc.c b/src/plugin_tc.c index c928e61ba..0d8fd3d6b 100644 --- a/src/plugin_tc.c +++ b/src/plugin_tc.c @@ -81,6 +81,7 @@ struct tc_device { avl_tree classes_index; struct tc_class *classes; + struct tc_class *last_class; struct tc_device *next; struct tc_device *prev; @@ -139,11 +140,19 @@ static inline struct tc_class *tc_class_index_find(struct tc_device *st, const c static inline void tc_class_free(struct tc_device *n, struct tc_class *c) { if(c == n->classes) { - if(c->next) + if(likely(c->next)) n->classes = c->next; else n->classes = c->prev; } + + if(c == n->last_class) { + if(unlikely(c->next)) + n->last_class = c->next; + else + n->last_class = c->prev; + } + if(c->next) c->next->prev = c->prev; if(c->prev) c->prev->next = c->next; @@ -374,11 +383,20 @@ static inline void tc_device_commit(struct tc_device *d) { d->enabled_bytes = CONFIG_BOOLEAN_YES; if(unlikely(!d->st_bytes)) - d->st_bytes = rrdset_create_localhost(RRD_TYPE_TC, d->id, d->name ? d->name : d->id - , d->family ? d->family : d->id, RRD_TYPE_TC ".qos", "Class Usage" - , "kilobits/s", 7000, localhost->rrd_update_every - , d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE - : RRDSET_TYPE_STACKED); + d->st_bytes = rrdset_create_localhost( + RRD_TYPE_TC + , d->id + , d->name ? d->name : d->id + , d->family ? d->family : d->id + , RRD_TYPE_TC ".qos" + , "Class Usage" + , "kilobits/s" + , "tc" + , NULL + , 7000 + , localhost->rrd_update_every + , d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED + ); else { rrdset_next(d->st_bytes); @@ -413,10 +431,20 @@ static inline void tc_device_commit(struct tc_device *d) { snprintfz(id, RRD_ID_LENGTH_MAX, "%s_packets", d->id); snprintfz(name, RRD_ID_LENGTH_MAX, "%s_packets", d->name?d->name:d->id); - d->st_packets = rrdset_create_localhost(RRD_TYPE_TC, id, name, d->family ? d->family : d->id - , RRD_TYPE_TC ".qos_packets", "Class Packets", "packets/s", 7010 - , localhost->rrd_update_every, d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE - : RRDSET_TYPE_STACKED); + d->st_packets = rrdset_create_localhost( + RRD_TYPE_TC + , id + , name + , d->family ? d->family : d->id + , RRD_TYPE_TC ".qos_packets" + , "Class Packets" + , "packets/s" + , "tc" + , NULL + , 7010 + , localhost->rrd_update_every + , d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED + ); } else { rrdset_next(d->st_packets); @@ -456,11 +484,20 @@ static inline void tc_device_commit(struct tc_device *d) { snprintfz(id, RRD_ID_LENGTH_MAX, "%s_dropped", d->id); snprintfz(name, RRD_ID_LENGTH_MAX, "%s_dropped", d->name?d->name:d->id); - d->st_dropped = rrdset_create_localhost(RRD_TYPE_TC, id, name, d->family ? d->family : d->id - , RRD_TYPE_TC ".qos_dropped", "Class Dropped Packets", "packets/s" - , 7020, localhost->rrd_update_every - , d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE - : RRDSET_TYPE_STACKED); + d->st_dropped = rrdset_create_localhost( + RRD_TYPE_TC + , id + , name + , d->family ? d->family : d->id + , RRD_TYPE_TC ".qos_dropped" + , "Class Dropped Packets" + , "packets/s" + , "tc" + , NULL + , 7020 + , localhost->rrd_update_every + , d->enabled_all_classes_qdiscs ? RRDSET_TYPE_LINE : RRDSET_TYPE_STACKED + ); } else { rrdset_next(d->st_dropped); @@ -500,9 +537,20 @@ static inline void tc_device_commit(struct tc_device *d) { snprintfz(id, RRD_ID_LENGTH_MAX, "%s_tokens", d->id); snprintfz(name, RRD_ID_LENGTH_MAX, "%s_tokens", d->name?d->name:d->id); - d->st_tokens = rrdset_create_localhost(RRD_TYPE_TC, id, name, d->family ? d->family : d->id - , RRD_TYPE_TC ".qos_tokens", "Class Tokens", "tokens", 7030 - , localhost->rrd_update_every, RRDSET_TYPE_LINE); + d->st_tokens = rrdset_create_localhost( + RRD_TYPE_TC + , id + , name + , d->family ? d->family : d->id + , RRD_TYPE_TC ".qos_tokens" + , "Class Tokens" + , "tokens" + , "tc" + , NULL + , 7030 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); } else { rrdset_next(d->st_tokens); @@ -543,9 +591,20 @@ static inline void tc_device_commit(struct tc_device *d) { snprintfz(id, RRD_ID_LENGTH_MAX, "%s_ctokens", d->id); snprintfz(name, RRD_ID_LENGTH_MAX, "%s_ctokens", d->name?d->name:d->id); - d->st_ctokens = rrdset_create_localhost(RRD_TYPE_TC, id, name, d->family ? d->family : d->id - , RRD_TYPE_TC ".qos_ctokens", "Class cTokens", "ctokens", 7040 - , localhost->rrd_update_every, RRDSET_TYPE_LINE); + d->st_ctokens = rrdset_create_localhost( + RRD_TYPE_TC + , id + , name + , d->family ? d->family : d->id + , RRD_TYPE_TC ".qos_ctokens" + , "Class cTokens" + , "ctokens" + , "tc" + , NULL + , 7040 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); } else { debug(D_TC_LOOP, "TC: Updating _ctokens chart for device '%s'", d->name?d->name:d->id); @@ -605,7 +664,7 @@ static inline void tc_device_set_device_name(struct tc_device *d, char *name) { d->name = NULL; } - if(likely(name && *name && strcmp(d->id, name))) { + if(likely(name && *name && strcmp(d->id, name) != 0)) { debug(D_TC_LOOP, "TC: Setting device '%s' name to '%s'", d->id, name); d->name = strdupz(name); d->name_updated = 1; @@ -663,9 +722,15 @@ static inline struct tc_class *tc_class_add(struct tc_device *n, char *id, char c = callocz(1, sizeof(struct tc_class)); - if(n->classes) n->classes->prev = c; - c->next = n->classes; - n->classes = c; + if(unlikely(!n->classes)) + n->classes = c; + + else if(likely(n->last_class)) { + n->last_class->next = c; + c->prev = n->last_class; + } + + n->last_class = c; c->id = strdupz(id); c->hash = simple_hash(c->id); @@ -776,7 +841,6 @@ void *tc_main(void *ptr) { error("Cannot set pthread cancel state to ENABLE."); struct rusage thread; - RRDSET *stcpu = NULL, *sttime = NULL; char buffer[TC_LINE_MAX+1] = ""; char *words[PLUGINSD_MAX_WORDS] = { NULL }; @@ -989,30 +1053,56 @@ void *tc_main(void *ptr) { // debug(D_TC_LOOP, "WORKTIME line '%s' '%s'", words[1], words[2]); getrusage(RUSAGE_THREAD, &thread); - if(unlikely(!stcpu)) stcpu = rrdset_find_localhost("netdata.plugin_tc_cpu"); + static RRDSET *stcpu = NULL; + static RRDDIM *rd_user = NULL, *rd_system = NULL; + if(unlikely(!stcpu)) { - stcpu = rrdset_create_localhost("netdata", "plugin_tc_cpu", NULL, "tc.helper", NULL - , "NetData TC CPU usage", "milliseconds/s", 135000, localhost->rrd_update_every - , RRDSET_TYPE_STACKED); - rrddim_add(stcpu, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(stcpu, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + stcpu = rrdset_create_localhost( + "netdata" + , "plugin_tc_cpu" + , NULL + , "tc.helper" + , NULL + , "NetData TC CPU usage" + , "milliseconds/s" + , "tc" + , NULL + , 135000 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); + rd_user = rrddim_add(stcpu, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + rd_system = rrddim_add(stcpu, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(stcpu); - rrddim_set(stcpu, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); - rrddim_set(stcpu, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrddim_set_by_pointer(stcpu, rd_user , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set_by_pointer(stcpu, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); rrdset_done(stcpu); - if(unlikely(!sttime)) sttime = rrdset_find_localhost("netdata.plugin_tc_time"); + static RRDSET *sttime = NULL; + static RRDDIM *rd_run_time = NULL; + if(unlikely(!sttime)) { - sttime = rrdset_create_localhost("netdata", "plugin_tc_time", NULL, "tc.helper", NULL - , "NetData TC script execution", "milliseconds/run", 135001 - , localhost->rrd_update_every, RRDSET_TYPE_AREA); - rrddim_add(sttime, "run_time", "run time", 1, 1, RRD_ALGORITHM_ABSOLUTE); + sttime = rrdset_create_localhost( + "netdata" + , "plugin_tc_time" + , NULL + , "tc.helper" + , NULL + , "NetData TC script execution" + , "milliseconds/run" + , "tc" + , NULL + , 135001 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA + ); + rd_run_time = rrddim_add(sttime, "run_time", "run time", 1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(sttime); - rrddim_set(sttime, "run_time", atoll(words[1])); + rrddim_set_by_pointer(sttime, rd_run_time, str2ll(words[1], NULL)); rrdset_done(sttime); } diff --git a/src/plugins_d.c b/src/plugins_d.c index 42433b552..d0f29f4d4 100644 --- a/src/plugins_d.c +++ b/src/plugins_d.c @@ -1,5 +1,8 @@ #include "common.h" +char *plugin_directories[PLUGINSD_MAX_DIRECTORIES] = { NULL }; +char *netdata_configured_plugins_dir_base; + struct plugind *pluginsd_root = NULL; static inline int pluginsd_space(char c) { @@ -16,13 +19,27 @@ static inline int pluginsd_space(char c) { } } +inline int config_isspace(char c) { + switch(c) { + case ' ': + case '\t': + case '\r': + case '\n': + case ',': + return 1; + + default: + return 0; + } +} + // split a text into words, respecting quotes -inline int pluginsd_split_words(char *str, char **words, int max_words) { +inline int quoted_strings_splitter(char *str, char **words, int max_words, int (*custom_isspace)(char)) { char *s = str, quote = 0; int i = 0, j; // skip all white space - while(unlikely(pluginsd_space(*s))) s++; + while(unlikely(custom_isspace(*s))) s++; // check for quote if(unlikely(*s == '\'' || *s == '"')) { @@ -49,13 +66,13 @@ inline int pluginsd_split_words(char *str, char **words, int max_words) { } // if it is a space - else if(unlikely(quote == 0 && pluginsd_space(*s))) { + else if(unlikely(quote == 0 && custom_isspace(*s))) { // terminate the word *s++ = '\0'; // skip all white space - while(likely(pluginsd_space(*s))) s++; + while(likely(custom_isspace(*s))) s++; // check for quote if(unlikely(*s == '\'' || *s == '"')) { @@ -82,6 +99,10 @@ inline int pluginsd_split_words(char *str, char **words, int max_words) { return i; } +inline int pluginsd_split_words(char *str, char **words, int max_words) { + return quoted_strings_splitter(str, words, max_words, pluginsd_space); +} + inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations) { int enabled = cd->enabled; @@ -101,6 +122,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int uint32_t CHART_HASH = simple_hash(PLUGINSD_KEYWORD_CHART); uint32_t DIMENSION_HASH = simple_hash(PLUGINSD_KEYWORD_DIMENSION); uint32_t DISABLE_HASH = simple_hash(PLUGINSD_KEYWORD_DISABLE); + uint32_t VARIABLE_HASH = simple_hash(PLUGINSD_KEYWORD_VARIABLE); RRDSET *st = NULL; uint32_t hash; @@ -212,10 +234,6 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int count++; } - else if(likely(hash == FLUSH_HASH && !strcmp(s, PLUGINSD_KEYWORD_FLUSH))) { - debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename); - st = NULL; - } else if(likely(hash == CHART_HASH && !strcmp(s, PLUGINSD_KEYWORD_CHART))) { st = NULL; @@ -229,6 +247,8 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int char *priority_s = words[8]; char *update_every_s = words[9]; char *options = words[10]; + char *plugin = words[11]; + char *module = words[12]; // parse the id from type char *id = NULL; @@ -275,22 +295,31 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int if(unlikely(!title)) title = ""; if(unlikely(!units)) units = "unknown"; - st = rrdset_find_bytype(host, type, id); - if(unlikely(!st)) { - debug(D_PLUGINSD, "PLUGINSD: Creating chart type='%s', id='%s', name='%s', family='%s', context='%s', chart='%s', priority=%d, update_every=%d" - , type, id - , name?name:"" - , family?family:"" - , context?context:"" - , rrdset_type_name(chart_type) - , priority - , update_every - ); - - st = rrdset_create(host, type, id, name, family, context, title, units, priority, update_every, chart_type); - cd->update_every = update_every; - } - else debug(D_PLUGINSD, "PLUGINSD: Chart '%s' already exists. Not adding it again.", st->id); + debug(D_PLUGINSD, "PLUGINSD: Creating chart type='%s', id='%s', name='%s', family='%s', context='%s', chart='%s', priority=%d, update_every=%d" + , type, id + , name?name:"" + , family?family:"" + , context?context:"" + , rrdset_type_name(chart_type) + , priority + , update_every + ); + + st = rrdset_create( + host + , type + , id + , name + , family + , context + , title + , units + , (plugin && *plugin)?plugin:cd->filename + , module + , priority + , update_every + , chart_type + ); if(options && *options) { if(strstr(options, "obsolete")) @@ -308,6 +337,11 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int else rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST); } + else { + rrdset_isnot_obsolete(st); + rrdset_flag_clear(st, RRDSET_FLAG_DETAIL); + rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST); + } } else if(likely(hash == DIMENSION_HASH && !strcmp(s, PLUGINSD_KEYWORD_DIMENSION))) { char *id = words[1]; @@ -359,6 +393,64 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int if(strstr(options, "nooverflow") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); } } + else if(likely(hash == VARIABLE_HASH && !strcmp(s, PLUGINSD_KEYWORD_VARIABLE))) { + char *name = words[1]; + char *value = words[2]; + int global = (st)?0:1; + + if(name && *name) { + if((strcmp(name, "GLOBAL") == 0 || strcmp(name, "HOST") == 0)) { + global = 1; + name = words[2]; + value = words[3]; + } + else if((strcmp(name, "LOCAL") == 0 || strcmp(name, "CHART") == 0)) { + global = 0; + name = words[2]; + value = words[3]; + } + } + + if(unlikely(!name || !*name)) { + error("PLUGINSD: '%s' is requesting a VARIABLE on host '%s', without a variable name. Disabling it.", cd->fullfilename, host->hostname); + enabled = 0; + break; + } + + if(unlikely(!value || !*value)) + value = NULL; + + if(value) { + char *endptr = NULL; + calculated_number v = (calculated_number)str2ld(value, &endptr); + + if(unlikely(endptr && *endptr)) { + if(endptr == value) + error("PLUGINSD: '%s': the value '%s' of VARIABLE '%s' on host '%s' cannot be parsed as a number", cd->fullfilename, value, name, host->hostname); + else + error("PLUGINSD: '%s': the value '%s' of VARIABLE '%s' on host '%s' has leftovers: '%s'", cd->fullfilename, value, name, host->hostname, endptr); + } + + if(global) { + RRDVAR *rv = rrdvar_custom_host_variable_create(host, name); + if (rv) rrdvar_custom_host_variable_set(host, rv, v); + else error("PLUGINSD: '%s': cannot find/create HOST VARIABLE '%s' on host '%s'", cd->fullfilename, name, host->hostname); + } + else if(st) { + RRDSETVAR *rs = rrdsetvar_custom_chart_variable_create(st, name); + if (rs) rrdsetvar_custom_chart_variable_set(rs, v); + else error("PLUGINSD: '%s': cannot find/create CHART VARIABLE '%s' on host '%s', chart '%s'", cd->fullfilename, name, host->hostname, st->id); + } + else + error("PLUGINSD: '%s': cannot find/create CHART VARIABLE '%s' on host '%s' without a chart", cd->fullfilename, name, host->hostname); + } + else + error("PLUGINSD: '%s': cannot set %s VARIABLE '%s' on host '%s' to an empty value", cd->fullfilename, (global)?"HOST":"CHART", name, host->hostname); + } + else if(likely(hash == FLUSH_HASH && !strcmp(s, PLUGINSD_KEYWORD_FLUSH))) { + debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename); + st = NULL; + } else if(unlikely(hash == DISABLE_HASH && !strcmp(s, PLUGINSD_KEYWORD_DISABLE))) { info("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename); enabled = 0; @@ -476,99 +568,131 @@ void *pluginsd_main(void *ptr) { int automatic_run = config_get_boolean(CONFIG_SECTION_PLUGINS, "enable running new plugins", 1); int scan_frequency = (int) config_get_number(CONFIG_SECTION_PLUGINS, "check for new plugins every", 60); - DIR *dir = NULL; - struct dirent *file = NULL; - struct plugind *cd; - - // enable the apps plugin by default - // config_get_boolean(CONFIG_SECTION_PLUGINS, "apps", 1); - if(scan_frequency < 1) scan_frequency = 1; + // store the errno for each plugins directory + // so that we don't log broken directories on each loop + int directory_errors[PLUGINSD_MAX_DIRECTORIES] = { 0 }; + for(;;) { if(unlikely(netdata_exit)) break; - dir = opendir(netdata_configured_plugins_dir); - if(unlikely(!dir)) { - error("Cannot open directory '%s'.", netdata_configured_plugins_dir); - goto cleanup; - } + int idx; + const char *directory_name; - while(likely((file = readdir(dir)))) { + for( idx = 0; idx < PLUGINSD_MAX_DIRECTORIES && (directory_name = plugin_directories[idx]) ; idx++ ) { if(unlikely(netdata_exit)) break; - debug(D_PLUGINSD, "PLUGINSD: Examining file '%s'", file->d_name); - - if(unlikely(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0)) continue; - - int len = (int) strlen(file->d_name); - if(unlikely(len <= (int)PLUGINSD_FILE_SUFFIX_LEN)) continue; - if(unlikely(strcmp(PLUGINSD_FILE_SUFFIX, &file->d_name[len - (int)PLUGINSD_FILE_SUFFIX_LEN]) != 0)) { - debug(D_PLUGINSD, "PLUGINSD: File '%s' does not end in '%s'.", file->d_name, PLUGINSD_FILE_SUFFIX); + errno = 0; + DIR *dir = opendir(directory_name); + if(unlikely(!dir)) { + if(directory_errors[idx] != errno) { + directory_errors[idx] = errno; + error("PLUGINSD: Cannot open plugins directory '%s'.", directory_name); + } continue; } - char pluginname[CONFIG_MAX_NAME + 1]; - snprintfz(pluginname, CONFIG_MAX_NAME, "%.*s", (int)(len - PLUGINSD_FILE_SUFFIX_LEN), file->d_name); - int enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, pluginname, automatic_run); + struct dirent *file = NULL; + while(likely((file = readdir(dir)))) { + if(unlikely(netdata_exit)) break; - if(unlikely(!enabled)) { - debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is not enabled", file->d_name); - continue; - } + debug(D_PLUGINSD, "PLUGINSD: Examining file '%s'", file->d_name); - // check if it runs already - for(cd = pluginsd_root ; cd ; cd = cd->next) - if(unlikely(strcmp(cd->filename, file->d_name) == 0)) break; + if(unlikely(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0)) continue; - if(likely(cd && !cd->obsolete)) { - debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is already running", cd->filename); - continue; - } + int len = (int) strlen(file->d_name); + if(unlikely(len <= (int)PLUGINSD_FILE_SUFFIX_LEN)) continue; + if(unlikely(strcmp(PLUGINSD_FILE_SUFFIX, &file->d_name[len - (int)PLUGINSD_FILE_SUFFIX_LEN]) != 0)) { + debug(D_PLUGINSD, "PLUGINSD: File '%s' does not end in '%s'.", file->d_name, PLUGINSD_FILE_SUFFIX); + continue; + } + + char pluginname[CONFIG_MAX_NAME + 1]; + snprintfz(pluginname, CONFIG_MAX_NAME, "%.*s", (int)(len - PLUGINSD_FILE_SUFFIX_LEN), file->d_name); + int enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, pluginname, automatic_run); + + if(unlikely(!enabled)) { + debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is not enabled", file->d_name); + continue; + } + + // check if it runs already + struct plugind *cd; + for(cd = pluginsd_root ; cd ; cd = cd->next) + if(unlikely(strcmp(cd->filename, file->d_name) == 0)) break; - // it is not running - // allocate a new one, or use the obsolete one - if(unlikely(!cd)) { - cd = callocz(sizeof(struct plugind), 1); + if(likely(cd && !cd->obsolete)) { + debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is already running", cd->filename); + continue; + } + + // it is not running + // allocate a new one, or use the obsolete one + if(unlikely(!cd)) { + cd = callocz(sizeof(struct plugind), 1); - snprintfz(cd->id, CONFIG_MAX_NAME, "plugin:%s", pluginname); + snprintfz(cd->id, CONFIG_MAX_NAME, "plugin:%s", pluginname); - strncpyz(cd->filename, file->d_name, FILENAME_MAX); - snprintfz(cd->fullfilename, FILENAME_MAX, "%s/%s", netdata_configured_plugins_dir, cd->filename); + strncpyz(cd->filename, file->d_name, FILENAME_MAX); + snprintfz(cd->fullfilename, FILENAME_MAX, "%s/%s", directory_name, cd->filename); - cd->enabled = enabled; - cd->update_every = (int) config_get_number(cd->id, "update every", localhost->rrd_update_every); - cd->started_t = now_realtime_sec(); + cd->enabled = enabled; + cd->update_every = (int) config_get_number(cd->id, "update every", localhost->rrd_update_every); + cd->started_t = now_realtime_sec(); - char *def = ""; - snprintfz(cd->cmd, PLUGINSD_CMD_MAX, "exec %s %d %s", cd->fullfilename, cd->update_every, config_get(cd->id, "command options", def)); + char *def = ""; + snprintfz(cd->cmd, PLUGINSD_CMD_MAX, "exec %s %d %s", cd->fullfilename, cd->update_every, config_get(cd->id, "command options", def)); - // link it - if(likely(pluginsd_root)) cd->next = pluginsd_root; - pluginsd_root = cd; + // link it + if(likely(pluginsd_root)) cd->next = pluginsd_root; + pluginsd_root = cd; - // it is not currently running - cd->obsolete = 1; + // it is not currently running + cd->obsolete = 1; - if(cd->enabled) { - // spawn a new thread for it - if(unlikely(pthread_create(&cd->thread, NULL, pluginsd_worker_thread, cd) != 0)) - error("PLUGINSD: failed to create new thread for plugin '%s'.", cd->filename); + if(cd->enabled) { + // spawn a new thread for it + if(unlikely(pthread_create(&cd->thread, NULL, pluginsd_worker_thread, cd) != 0)) + error("PLUGINSD: failed to create new thread for plugin '%s'.", cd->filename); - else if(unlikely(pthread_detach(cd->thread) != 0)) - error("PLUGINSD: Cannot request detach of newly created thread for plugin '%s'.", cd->filename); + else if(unlikely(pthread_detach(cd->thread) != 0)) + error("PLUGINSD: Cannot request detach of newly created thread for plugin '%s'.", cd->filename); + } } } + + closedir(dir); } - closedir(dir); sleep((unsigned int) scan_frequency); } -cleanup: info("PLUGINS.D thread exiting"); static_thread->enabled = 0; pthread_exit(NULL); return NULL; } + + +void pluginsd_stop_all_external_plugins() { + siginfo_t info; + struct plugind *cd; + for(cd = pluginsd_root ; cd ; cd = cd->next) { + if(cd->enabled && !cd->obsolete) { + info("Stopping %s plugin thread", cd->id); + pthread_cancel(cd->thread); + + if(cd->pid) { + info("killing %s plugin child process pid %d", cd->id, cd->pid); + if(killpid(cd->pid, SIGTERM) != -1) + waitid(P_PID, (id_t) cd->pid, &info, WEXITED); + + cd->pid = 0; + } + + cd->obsolete = 1; + } + } +} diff --git a/src/plugins_d.h b/src/plugins_d.h index 595a515c4..4d708386f 100644 --- a/src/plugins_d.h +++ b/src/plugins_d.h @@ -11,10 +11,14 @@ #define PLUGINSD_KEYWORD_END "END" #define PLUGINSD_KEYWORD_FLUSH "FLUSH" #define PLUGINSD_KEYWORD_DISABLE "DISABLE" +#define PLUGINSD_KEYWORD_VARIABLE "VARIABLE" #define PLUGINSD_LINE_MAX 1024 #define PLUGINSD_MAX_WORDS 20 +#define PLUGINSD_MAX_DIRECTORIES 20 +extern char *plugin_directories[PLUGINSD_MAX_DIRECTORIES]; + struct plugind { char id[CONFIG_MAX_NAME+1]; // config node id @@ -43,7 +47,12 @@ struct plugind { extern struct plugind *pluginsd_root; extern void *pluginsd_main(void *ptr); +extern void pluginsd_stop_all_external_plugins(void); + extern size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations); extern int pluginsd_split_words(char *str, char **words, int max_words); +extern int quoted_strings_splitter(char *str, char **words, int max_words, int (*custom_isspace)(char)); +extern int config_isspace(char c); + #endif /* NETDATA_PLUGINS_D_H */ diff --git a/src/proc_diskstats.c b/src/proc_diskstats.c index 4a32ec949..866e49c77 100644 --- a/src/proc_diskstats.c +++ b/src/proc_diskstats.c @@ -2,12 +2,13 @@ #define RRD_TYPE_DISK "disk" +#define DISK_TYPE_UNKNOWN 0 #define DISK_TYPE_PHYSICAL 1 #define DISK_TYPE_PARTITION 2 -#define DISK_TYPE_CONTAINER 3 +#define DISK_TYPE_VIRTUAL 3 #define CONFIG_SECTION_DISKSTATS "plugin:proc:/proc/diskstats" -#define DELAULT_EXLUDED_DISKS "loop* ram*" +#define DEFAULT_EXCLUDED_DISKS "loop* ram*" static struct disk { char *disk; // the name of the disk (sda, sdb, etc, after being looked up) @@ -31,16 +32,41 @@ static struct disk { int updated; - RRDSET *st_avgsz; - RRDSET *st_await; - RRDSET *st_backlog; RRDSET *st_io; - RRDSET *st_iotime; - RRDSET *st_mops; + RRDDIM *rd_io_reads; + RRDDIM *rd_io_writes; + RRDSET *st_ops; + RRDDIM *rd_ops_reads; + RRDDIM *rd_ops_writes; + RRDSET *st_qops; - RRDSET *st_svctm; + RRDDIM *rd_qops_operations; + + RRDSET *st_backlog; + RRDDIM *rd_backlog_backlog; + RRDSET *st_util; + RRDDIM *rd_util_utilization; + + RRDSET *st_mops; + RRDDIM *rd_mops_reads; + RRDDIM *rd_mops_writes; + + RRDSET *st_iotime; + RRDDIM *rd_iotime_reads; + RRDDIM *rd_iotime_writes; + + RRDSET *st_await; + RRDDIM *rd_await_reads; + RRDDIM *rd_await_writes; + + RRDSET *st_avgsz; + RRDDIM *rd_avgsz_reads; + RRDDIM *rd_avgsz_writes; + + RRDSET *st_svctm; + RRDDIM *rd_svctm_svctm; struct disk *next; } *disk_root = NULL; @@ -49,21 +75,23 @@ static struct disk { static char *path_to_get_hw_sector_size = NULL; static char *path_to_get_hw_sector_size_partitions = NULL; -static char *path_to_find_block_device = NULL; +static char *path_to_sys_dev_block_major_minor_string = NULL; +static char *path_to_sys_block_device = NULL; +static char *path_to_sys_devices_virtual_block_device = NULL; static char *path_to_device_mapper = NULL; +static char *path_to_device_label = NULL; +static char *path_to_device_id = NULL; +static int name_disks_by_id = CONFIG_BOOLEAN_NO; -static inline char *get_disk_name(unsigned long major, unsigned long minor, char *disk) { - static int enabled = 1; - - if(!enabled) goto cleanup; - +static inline int get_disk_name_from_path(const char *path, char *result, size_t result_size, unsigned long major, unsigned long minor, char *disk) { char filename[FILENAME_MAX + 1]; - char link[FILENAME_MAX + 1]; + int found = 0; - DIR *dir = opendir(path_to_device_mapper); + result_size--; + + DIR *dir = opendir(path); if (!dir) { - error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot open directory '%s'. Disabling device-mapper support.", disk, major, minor, path_to_device_mapper); - enabled = 0; + error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot open directory '%s'. Disabling device-mapper support.", disk, major, minor, path); goto cleanup; } @@ -71,18 +99,18 @@ static inline char *get_disk_name(unsigned long major, unsigned long minor, char while ((de = readdir(dir))) { if(de->d_type != DT_LNK) continue; - snprintfz(filename, FILENAME_MAX, "%s/%s", path_to_device_mapper, de->d_name); - ssize_t len = readlink(filename, link, FILENAME_MAX); + snprintfz(filename, FILENAME_MAX, "%s/%s", path, de->d_name); + ssize_t len = readlink(filename, result, result_size); if(len <= 0) { error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot read link '%s'.", disk, major, minor, filename); continue; } - link[len] = '\0'; - if(link[0] != '/') - snprintfz(filename, FILENAME_MAX, "%s/%s", path_to_device_mapper, link); + result[len] = '\0'; + if(result[0] != '/') + snprintfz(filename, FILENAME_MAX, "%s/%s", path, result); else - strncpyz(filename, link, FILENAME_MAX); + strncpyz(filename, result, FILENAME_MAX); struct stat sb; if(stat(filename, &sb) == -1) { @@ -102,17 +130,37 @@ static inline char *get_disk_name(unsigned long major, unsigned long minor, char // info("DEVICE-MAPPER ('%s', %lu:%lu): filename '%s' matches.", disk, major, minor, filename); - strncpy(link, de->d_name, FILENAME_MAX); - netdata_fix_chart_name(link); - disk = link; + strncpy(result, de->d_name, result_size); + found = 1; break; } closedir(dir); + cleanup: - return strdupz(disk); + + if(!found) + result[0] = '\0'; + + return found; } +static inline char *get_disk_name(unsigned long major, unsigned long minor, char *disk) { + char result[FILENAME_MAX + 1] = ""; + + if(!path_to_device_mapper || !*path_to_device_mapper || !get_disk_name_from_path(path_to_device_mapper, result, FILENAME_MAX + 1, major, minor, disk)) + if(!path_to_device_label || !*path_to_device_label || !get_disk_name_from_path(path_to_device_label, result, FILENAME_MAX + 1, major, minor, disk)) + if(name_disks_by_id != CONFIG_BOOLEAN_YES || !path_to_device_id || !*path_to_device_id || !get_disk_name_from_path(path_to_device_id, result, FILENAME_MAX + 1, major, minor, disk)) + strncpy(result, disk, FILENAME_MAX); + + if(!result[0]) + strncpy(result, disk, FILENAME_MAX); + + netdata_fix_chart_name(result); + return strdup(result); +} + + static struct disk *get_disk(unsigned long major, unsigned long minor, char *disk) { static struct mountinfo *disk_mountinfo_root = NULL; @@ -134,7 +182,7 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis d->device = strdupz(disk); d->major = major; d->minor = minor; - d->type = DISK_TYPE_PHYSICAL; // Default type. Changed later if not correct. + d->type = DISK_TYPE_UNKNOWN; // Default type. Changed later if not correct. d->configured = 0; d->sector_size = 512; // the default, will be changed below d->next = NULL; @@ -148,33 +196,50 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis last->next = d; } + char buffer[FILENAME_MAX + 1]; + + // find if it is a physical disk + // by checking if /sys/block/DISK is readable. + snprintfz(buffer, FILENAME_MAX, path_to_sys_block_device, disk); + if(likely(access(buffer, R_OK) == 0)) { + // assign it here, but it will be overwritten if it is not a physical disk + d->type = DISK_TYPE_PHYSICAL; + } + // find if it is a partition // by checking if /sys/dev/block/MAJOR:MINOR/partition is readable. - char buffer[FILENAME_MAX + 1]; - snprintfz(buffer, FILENAME_MAX, path_to_find_block_device, major, minor, "partition"); + snprintfz(buffer, FILENAME_MAX, path_to_sys_dev_block_major_minor_string, major, minor, "partition"); if(likely(access(buffer, R_OK) == 0)) { d->type = DISK_TYPE_PARTITION; } else { - // find if it is a container - // by checking if /sys/dev/block/MAJOR:MINOR/slaves has entries - snprintfz(buffer, FILENAME_MAX, path_to_find_block_device, major, minor, "slaves/"); - DIR *dirp = opendir(buffer); - if(likely(dirp != NULL)) { - struct dirent *dp; - while( (dp = readdir(dirp)) ) { - // . and .. are also files in empty folders. - if(unlikely(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0)) { - continue; + // find if it is a virtual disk + // by checking if /sys/devices/virtual/block/DISK is readable. + snprintfz(buffer, FILENAME_MAX, path_to_sys_devices_virtual_block_device, disk); + if(likely(access(buffer, R_OK) == 0)) { + d->type = DISK_TYPE_VIRTUAL; + } + else { + // find if it is a virtual device + // by checking if /sys/dev/block/MAJOR:MINOR/slaves has entries + snprintfz(buffer, FILENAME_MAX, path_to_sys_dev_block_major_minor_string, major, minor, "slaves/"); + DIR *dirp = opendir(buffer); + if (likely(dirp != NULL)) { + struct dirent *dp; + while ((dp = readdir(dirp))) { + // . and .. are also files in empty folders. + if (unlikely(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0)) { + continue; + } + + d->type = DISK_TYPE_VIRTUAL; + + // Stop the loop after we found one file. + break; } - - d->type = DISK_TYPE_CONTAINER; - - // Stop the loop after we found one file. - break; + if (unlikely(closedir(dirp) == -1)) + error("Unable to close dir %s", buffer); } - if(unlikely(closedir(dirp) == -1)) - error("Unable to close dir %s", buffer); } } @@ -296,8 +361,14 @@ int do_proc_diskstats(int update_every, usec_t dt) { char buffer[FILENAME_MAX + 1]; + snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s"); + path_to_sys_block_device = config_get(CONFIG_SECTION_DISKSTATS, "path to get block device", buffer); + + snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/virtual/block/%s"); + path_to_sys_devices_virtual_block_device = config_get(CONFIG_SECTION_DISKSTATS, "path to get virtual block device", buffer); + snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/%s"); - path_to_find_block_device = config_get(CONFIG_SECTION_DISKSTATS, "path to get block device infos", buffer); + path_to_sys_dev_block_major_minor_string = config_get(CONFIG_SECTION_DISKSTATS, "path to get block device infos", buffer); snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/queue/hw_sector_size"); path_to_get_hw_sector_size = config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size", buffer); @@ -307,6 +378,14 @@ int do_proc_diskstats(int update_every, usec_t dt) { snprintfz(buffer, FILENAME_MAX, "%s/dev/mapper", netdata_configured_host_prefix); path_to_device_mapper = config_get(CONFIG_SECTION_DISKSTATS, "path to device mapper", buffer); + + snprintfz(buffer, FILENAME_MAX, "%s/dev/disk/by-label", netdata_configured_host_prefix); + path_to_device_label = config_get(CONFIG_SECTION_DISKSTATS, "path to /dev/disk/by-label", buffer); + + snprintfz(buffer, FILENAME_MAX, "%s/dev/disk/by-id", netdata_configured_host_prefix); + path_to_device_id = config_get(CONFIG_SECTION_DISKSTATS, "path to /dev/disk/by-id", buffer); + + name_disks_by_id = config_get_boolean(CONFIG_SECTION_DISKSTATS, "name disks by id", name_disks_by_id); } // -------------------------------------------------------------------------- @@ -323,6 +402,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { size_t lines = procfile_lines(ff), l; + collected_number system_read_kb = 0, system_write_kb = 0; + for(l = 0; l < lines ;l++) { // -------------------------------------------------------------------------- // Read parameters @@ -398,6 +479,11 @@ int do_proc_diskstats(int update_every, usec_t dt) { struct disk *d = get_disk(major, minor, disk); d->updated = 1; + if(unlikely(d->type == DISK_TYPE_PHYSICAL)) { + system_read_kb += readsectors * d->sector_size / 1024; + system_write_kb += writesectors * d->sector_size / 1024; + } + // -------------------------------------------------------------------------- // Set its family based on mount point @@ -415,7 +501,7 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!excluded_disks)) { excluded_disks = simple_pattern_create( - config_get(CONFIG_SECTION_DISKSTATS, "exclude disks", DELAULT_EXLUDED_DISKS), + config_get(CONFIG_SECTION_DISKSTATS, "exclude disks", DEFAULT_EXCLUDED_DISKS), SIMPLE_PATTERN_EXACT ); } @@ -449,6 +535,10 @@ int do_proc_diskstats(int update_every, usec_t dt) { // based on the type of disk switch(d->type) { + default: + case DISK_TYPE_UNKNOWN: + break; + case DISK_TYPE_PHYSICAL: def_performance = global_enable_performance_for_physical_disks; break; @@ -457,7 +547,7 @@ int do_proc_diskstats(int update_every, usec_t dt) { def_performance = global_enable_performance_for_partitions; break; - case DISK_TYPE_CONTAINER: + case DISK_TYPE_VIRTUAL: def_performance = global_enable_performance_for_virtual_disks; break; } @@ -518,18 +608,20 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.io" , "Disk I/O Bandwidth" , "kilobytes/s" + , "proc" + , "diskstats" , 2000 , update_every , RRDSET_TYPE_AREA ); - rrddim_add(d->st_io, "reads", NULL, d->sector_size, 1024, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(d->st_io, "writes", NULL, d->sector_size * -1, 1024, RRD_ALGORITHM_INCREMENTAL); + d->rd_io_reads = rrddim_add(d->st_io, "reads", NULL, d->sector_size, 1024, RRD_ALGORITHM_INCREMENTAL); + d->rd_io_writes = rrddim_add(d->st_io, "writes", NULL, d->sector_size * -1, 1024, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(d->st_io); - last_readsectors = rrddim_set(d->st_io, "reads", readsectors); - last_writesectors = rrddim_set(d->st_io, "writes", writesectors); + last_readsectors = rrddim_set_by_pointer(d->st_io, d->rd_io_reads, readsectors); + last_writesectors = rrddim_set_by_pointer(d->st_io, d->rd_io_writes, writesectors); rrdset_done(d->st_io); } @@ -547,6 +639,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.ops" , "Disk Completed I/O Operations" , "operations/s" + , "proc" + , "diskstats" , 2001 , update_every , RRDSET_TYPE_LINE @@ -554,13 +648,13 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_ops, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(d->st_ops, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + d->rd_ops_reads = rrddim_add(d->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + d->rd_ops_writes = rrddim_add(d->st_ops, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(d->st_ops); - last_reads = rrddim_set(d->st_ops, "reads", reads); - last_writes = rrddim_set(d->st_ops, "writes", writes); + last_reads = rrddim_set_by_pointer(d->st_ops, d->rd_ops_reads, reads); + last_writes = rrddim_set_by_pointer(d->st_ops, d->rd_ops_writes, writes); rrdset_done(d->st_ops); } @@ -578,6 +672,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.qops" , "Disk Current I/O Operations" , "operations" + , "proc" + , "diskstats" , 2002 , update_every , RRDSET_TYPE_LINE @@ -585,11 +681,11 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_qops, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + d->rd_qops_operations = rrddim_add(d->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(d->st_qops); - rrddim_set(d->st_qops, "operations", queued_ios); + rrddim_set_by_pointer(d->st_qops, d->rd_qops_operations, queued_ios); rrdset_done(d->st_qops); } @@ -607,6 +703,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.backlog" , "Disk Backlog" , "backlog (ms)" + , "proc" + , "diskstats" , 2003 , update_every , RRDSET_TYPE_AREA @@ -614,11 +712,11 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_backlog, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_backlog, "backlog", NULL, 1, 10, RRD_ALGORITHM_INCREMENTAL); + d->rd_backlog_backlog = rrddim_add(d->st_backlog, "backlog", NULL, 1, 10, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(d->st_backlog); - rrddim_set(d->st_backlog, "backlog", backlog_ms); + rrddim_set_by_pointer(d->st_backlog, d->rd_backlog_backlog, backlog_ms); rrdset_done(d->st_backlog); } @@ -636,6 +734,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.util" , "Disk Utilization Time" , "% of time working" + , "proc" + , "diskstats" , 2004 , update_every , RRDSET_TYPE_AREA @@ -643,11 +743,11 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_util, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_util, "utilization", NULL, 1, 10, RRD_ALGORITHM_INCREMENTAL); + d->rd_util_utilization = rrddim_add(d->st_util, "utilization", NULL, 1, 10, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(d->st_util); - last_busy_ms = rrddim_set(d->st_util, "utilization", busy_ms); + last_busy_ms = rrddim_set_by_pointer(d->st_util, d->rd_util_utilization, busy_ms); rrdset_done(d->st_util); } @@ -665,6 +765,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.mops" , "Disk Merged Operations" , "merged operations/s" + , "proc" + , "diskstats" , 2021 , update_every , RRDSET_TYPE_LINE @@ -672,13 +774,13 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_mops, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_mops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(d->st_mops, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + d->rd_mops_reads = rrddim_add(d->st_mops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + d->rd_mops_writes = rrddim_add(d->st_mops, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(d->st_mops); - rrddim_set(d->st_mops, "reads", mreads); - rrddim_set(d->st_mops, "writes", mwrites); + rrddim_set_by_pointer(d->st_mops, d->rd_mops_reads, mreads); + rrddim_set_by_pointer(d->st_mops, d->rd_mops_writes, mwrites); rrdset_done(d->st_mops); } @@ -696,6 +798,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.iotime" , "Disk Total I/O Time" , "milliseconds/s" + , "proc" + , "diskstats" , 2022 , update_every , RRDSET_TYPE_LINE @@ -703,13 +807,13 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_iotime, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_iotime, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(d->st_iotime, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + d->rd_iotime_reads = rrddim_add(d->st_iotime, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + d->rd_iotime_writes = rrddim_add(d->st_iotime, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(d->st_iotime); - last_readms = rrddim_set(d->st_iotime, "reads", readms); - last_writems = rrddim_set(d->st_iotime, "writes", writems); + last_readms = rrddim_set_by_pointer(d->st_iotime, d->rd_iotime_reads, readms); + last_writems = rrddim_set_by_pointer(d->st_iotime, d->rd_iotime_writes, writems); rrdset_done(d->st_iotime); } @@ -730,6 +834,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.await" , "Average Completed I/O Operation Time" , "ms per operation" + , "proc" + , "diskstats" , 2005 , update_every , RRDSET_TYPE_LINE @@ -737,13 +843,13 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_await, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_await, "reads", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(d->st_await, "writes", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); + d->rd_await_reads = rrddim_add(d->st_await, "reads", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + d->rd_await_writes = rrddim_add(d->st_await, "writes", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(d->st_await); - rrddim_set(d->st_await, "reads", (reads - last_reads) ? (readms - last_readms) / (reads - last_reads) : 0); - rrddim_set(d->st_await, "writes", (writes - last_writes) ? (writems - last_writems) / (writes - last_writes) : 0); + rrddim_set_by_pointer(d->st_await, d->rd_await_reads, (reads - last_reads) ? (readms - last_readms) / (reads - last_reads) : 0); + rrddim_set_by_pointer(d->st_await, d->rd_await_writes, (writes - last_writes) ? (writems - last_writems) / (writes - last_writes) : 0); rrdset_done(d->st_await); } @@ -759,6 +865,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.avgsz" , "Average Completed I/O Operation Bandwidth" , "kilobytes per operation" + , "proc" + , "diskstats" , 2006 , update_every , RRDSET_TYPE_AREA @@ -766,13 +874,13 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_avgsz, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_avgsz, "reads", NULL, d->sector_size, 1024, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(d->st_avgsz, "writes", NULL, d->sector_size * -1, 1024, RRD_ALGORITHM_ABSOLUTE); + d->rd_avgsz_reads = rrddim_add(d->st_avgsz, "reads", NULL, d->sector_size, 1024, RRD_ALGORITHM_ABSOLUTE); + d->rd_avgsz_writes = rrddim_add(d->st_avgsz, "writes", NULL, d->sector_size * -1, 1024, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(d->st_avgsz); - rrddim_set(d->st_avgsz, "reads", (reads - last_reads) ? (readsectors - last_readsectors) / (reads - last_reads) : 0); - rrddim_set(d->st_avgsz, "writes", (writes - last_writes) ? (writesectors - last_writesectors) / (writes - last_writes) : 0); + rrddim_set_by_pointer(d->st_avgsz, d->rd_avgsz_reads, (reads - last_reads) ? (readsectors - last_readsectors) / (reads - last_reads) : 0); + rrddim_set_by_pointer(d->st_avgsz, d->rd_avgsz_writes, (writes - last_writes) ? (writesectors - last_writesectors) / (writes - last_writes) : 0); rrdset_done(d->st_avgsz); } @@ -788,6 +896,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { , "disk.svctm" , "Average Service Time" , "ms per operation" + , "proc" + , "diskstats" , 2007 , update_every , RRDSET_TYPE_LINE @@ -795,16 +905,52 @@ int do_proc_diskstats(int update_every, usec_t dt) { rrdset_flag_set(d->st_svctm, RRDSET_FLAG_DETAIL); - rrddim_add(d->st_svctm, "svctm", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + d->rd_svctm_svctm = rrddim_add(d->st_svctm, "svctm", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(d->st_svctm); - rrddim_set(d->st_svctm, "svctm", ((reads - last_reads) + (writes - last_writes)) ? (busy_ms - last_busy_ms) / ((reads - last_reads) + (writes - last_writes)) : 0); + rrddim_set_by_pointer(d->st_svctm, d->rd_svctm_svctm, ((reads - last_reads) + (writes - last_writes)) ? (busy_ms - last_busy_ms) / ((reads - last_reads) + (writes - last_writes)) : 0); rrdset_done(d->st_svctm); } } } + + // ------------------------------------------------------------------------ + // update the system total I/O + + if(global_do_io == CONFIG_BOOLEAN_YES || (global_do_io == CONFIG_BOOLEAN_AUTO && (system_read_kb || system_write_kb))) { + static RRDSET *st_io = NULL; + static RRDDIM *rd_in = NULL, *rd_out = NULL; + + if(unlikely(!st_io)) { + st_io = rrdset_create_localhost( + "system" + , "io" + , NULL + , "disk" + , NULL + , "Disk I/O" + , "kilobytes/s" + , "proc" + , "diskstats" + , 150 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_in = rrddim_add(st_io, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_io, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st_io); + + rrddim_set_by_pointer(st_io, rd_in, system_read_kb); + rrddim_set_by_pointer(st_io, rd_out, system_write_kb); + rrdset_done(st_io); + } + + + // ------------------------------------------------------------------------ // cleanup removed disks struct disk *d = disk_root, *last = NULL; diff --git a/src/proc_interrupts.c b/src/proc_interrupts.c index b9f3941d2..867f39eb2 100644 --- a/src/proc_interrupts.c +++ b/src/proc_interrupts.c @@ -154,6 +154,8 @@ int do_proc_interrupts(int update_every, usec_t dt) { , NULL , "System interrupts" , "interrupts/s" + , "proc" + , "interrupts" , 1000 , update_every , RRDSET_TYPE_STACKED @@ -217,6 +219,8 @@ int do_proc_interrupts(int update_every, usec_t dt) { , "cpu.interrupts" , title , "interrupts/s" + , "proc" + , "interrupts" , 1100 + c , update_every , RRDSET_TYPE_STACKED diff --git a/src/proc_loadavg.c b/src/proc_loadavg.c index 54fc545a5..868f7d50a 100644 --- a/src/proc_loadavg.c +++ b/src/proc_loadavg.c @@ -60,6 +60,8 @@ int do_proc_loadavg(int update_every, usec_t dt) { , NULL , "System Load Average" , "load" + , "proc" + , "loadavg" , 100 , (update_every < MIN_LOADAVG_UPDATE_EVERY) ? MIN_LOADAVG_UPDATE_EVERY : update_every , RRDSET_TYPE_LINE @@ -98,6 +100,8 @@ int do_proc_loadavg(int update_every, usec_t dt) { , NULL , "System Active Processes" , "processes" + , "proc" + , "loadavg" , 750 , update_every , RRDSET_TYPE_LINE diff --git a/src/proc_meminfo.c b/src/proc_meminfo.c index 152a6366e..085850c2c 100644 --- a/src/proc_meminfo.c +++ b/src/proc_meminfo.c @@ -150,6 +150,8 @@ int do_proc_meminfo(int update_every, usec_t dt) { , NULL , "System RAM" , "MB" + , "proc" + , "meminfo" , 200 , update_every , RRDSET_TYPE_STACKED @@ -189,6 +191,8 @@ int do_proc_meminfo(int update_every, usec_t dt) { , NULL , "System Swap" , "MB" + , "proc" + , "meminfo" , 201 , update_every , RRDSET_TYPE_STACKED @@ -224,6 +228,8 @@ int do_proc_meminfo(int update_every, usec_t dt) { , NULL , "Corrupted Memory, detected by ECC" , "MB" + , "proc" + , "meminfo" , 9000 , update_every , RRDSET_TYPE_LINE @@ -255,6 +261,8 @@ int do_proc_meminfo(int update_every, usec_t dt) { , NULL , "Committed (Allocated) Memory" , "MB" + , "proc" + , "meminfo" , 5000 , update_every , RRDSET_TYPE_AREA @@ -286,6 +294,8 @@ int do_proc_meminfo(int update_every, usec_t dt) { , NULL , "Writeback Memory" , "MB" + , "proc" + , "meminfo" , 4000 , update_every , RRDSET_TYPE_LINE @@ -324,6 +334,8 @@ int do_proc_meminfo(int update_every, usec_t dt) { , NULL , "Memory Used by Kernel" , "MB" + , "proc" + , "meminfo" , 6000 , update_every , RRDSET_TYPE_STACKED @@ -361,6 +373,8 @@ int do_proc_meminfo(int update_every, usec_t dt) { , NULL , "Reclaimable Kernel Memory" , "MB" + , "proc" + , "meminfo" , 6500 , update_every , RRDSET_TYPE_STACKED diff --git a/src/proc_net_dev.c b/src/proc_net_dev.c index 6e12fc5dd..32bb5bab1 100644 --- a/src/proc_net_dev.c +++ b/src/proc_net_dev.c @@ -9,6 +9,7 @@ static struct netdev { size_t len; // flags + int virtual; int configured; int enabled; int updated; @@ -428,8 +429,14 @@ int do_proc_net_dev(int update_every, usec_t dt) { static procfile *ff = NULL; static int enable_new_interfaces = -1; static int do_bandwidth = -1, do_packets = -1, do_errors = -1, do_drops = -1, do_fifo = -1, do_compressed = -1, do_events = -1; + static char *path_to_sys_devices_virtual_net = NULL; if(unlikely(enable_new_interfaces == -1)) { + char filename[FILENAME_MAX + 1]; + + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/devices/virtual/net/%s"); + path_to_sys_devices_virtual_net = config_get("plugin:proc:/proc/net/dev", "path to get virtual interfaces", filename); + enable_new_interfaces = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "enable new interfaces detected at runtime", CONFIG_BOOLEAN_AUTO); do_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "bandwidth for all interfaces", CONFIG_BOOLEAN_AUTO); @@ -459,6 +466,9 @@ int do_proc_net_dev(int update_every, usec_t dt) { netdev_found = 0; + kernel_uint_t system_rbytes = 0; + kernel_uint_t system_tbytes = 0; + size_t lines = procfile_lines(ff), l; for(l = 2; l < lines ;l++) { // require 17 words on each line @@ -479,28 +489,42 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->enabled) d->enabled = !simple_pattern_matches(disabled_list, d->name); - char var_name[512 + 1]; - snprintfz(var_name, 512, "plugin:proc:/proc/net/dev:%s", d->name); - d->enabled = config_get_boolean_ondemand(var_name, "enabled", d->enabled); + char buffer[FILENAME_MAX + 1]; + + snprintfz(buffer, FILENAME_MAX, path_to_sys_devices_virtual_net, d->name); + if(likely(access(buffer, R_OK) == 0)) { + d->virtual = 1; + } + else + d->virtual = 0; + + snprintfz(buffer, FILENAME_MAX, "plugin:proc:/proc/net/dev:%s", d->name); + d->enabled = config_get_boolean_ondemand(buffer, "enabled", d->enabled); + d->virtual = config_get_boolean(buffer, "virtual", d->virtual); if(d->enabled == CONFIG_BOOLEAN_NO) continue; - d->do_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", do_bandwidth); - d->do_packets = config_get_boolean_ondemand(var_name, "packets", do_packets); - d->do_errors = config_get_boolean_ondemand(var_name, "errors", do_errors); - d->do_drops = config_get_boolean_ondemand(var_name, "drops", do_drops); - d->do_fifo = config_get_boolean_ondemand(var_name, "fifo", do_fifo); - d->do_compressed = config_get_boolean_ondemand(var_name, "compressed", do_compressed); - d->do_events = config_get_boolean_ondemand(var_name, "events", do_events); + d->do_bandwidth = config_get_boolean_ondemand(buffer, "bandwidth", do_bandwidth); + d->do_packets = config_get_boolean_ondemand(buffer, "packets", do_packets); + d->do_errors = config_get_boolean_ondemand(buffer, "errors", do_errors); + d->do_drops = config_get_boolean_ondemand(buffer, "drops", do_drops); + d->do_fifo = config_get_boolean_ondemand(buffer, "fifo", do_fifo); + d->do_compressed = config_get_boolean_ondemand(buffer, "compressed", do_compressed); + d->do_events = config_get_boolean_ondemand(buffer, "events", do_events); } if(unlikely(!d->enabled)) continue; - if(likely(d->do_bandwidth != CONFIG_BOOLEAN_NO)) { + if(likely(d->do_bandwidth != CONFIG_BOOLEAN_NO || !d->virtual)) { d->rbytes = str2kernel_uint_t(procfile_lineword(ff, l, 1)); d->tbytes = str2kernel_uint_t(procfile_lineword(ff, l, 9)); + + if(likely(!d->virtual)) { + system_rbytes += d->rbytes; + system_tbytes += d->tbytes; + } } if(likely(d->do_packets != CONFIG_BOOLEAN_NO)) { @@ -551,6 +575,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { , "net.net" , "Bandwidth" , "kilobits/s" + , "proc" + , "net/dev" , d->priority , update_every , RRDSET_TYPE_AREA @@ -590,6 +616,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { , "net.packets" , "Packets" , "packets/s" + , "proc" + , "net/dev" , d->priority + 1 , update_every , RRDSET_TYPE_LINE @@ -633,6 +661,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { , "net.errors" , "Interface Errors" , "errors/s" + , "proc" + , "net/dev" , d->priority + 2 , update_every , RRDSET_TYPE_LINE @@ -674,6 +704,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { , "net.drops" , "Interface Drops" , "drops/s" + , "proc" + , "net/dev" , d->priority + 3 , update_every , RRDSET_TYPE_LINE @@ -715,6 +747,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { , "net.fifo" , "Interface FIFO Buffer Errors" , "errors" + , "proc" + , "net/dev" , d->priority + 4 , update_every , RRDSET_TYPE_LINE @@ -756,6 +790,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { , "net.compressed" , "Compressed Packets" , "packets/s" + , "proc" + , "net/dev" , d->priority + 5 , update_every , RRDSET_TYPE_LINE @@ -797,6 +833,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { , "net.events" , "Network Interface Events" , "events/s" + , "proc" + , "net/dev" , d->priority + 6 , update_every , RRDSET_TYPE_LINE @@ -817,6 +855,39 @@ int do_proc_net_dev(int update_every, usec_t dt) { } } + if(do_bandwidth == CONFIG_BOOLEAN_YES || (do_bandwidth == CONFIG_BOOLEAN_AUTO && (system_rbytes || system_tbytes))) { + do_bandwidth = CONFIG_BOOLEAN_YES; + static RRDSET *st_system_net = NULL; + static RRDDIM *rd_in = NULL, *rd_out = NULL; + + if(unlikely(!st_system_net)) { + st_system_net = rrdset_create_localhost( + "system" + , "net" + , NULL + , "network" + , NULL + , "Physical Network Interfaces Aggregated Bandwidth" + , "kilobits/s" + , "proc" + , "net/dev" + , 500 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_in = rrddim_add(st_system_net, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st_system_net, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_system_net); + + rrddim_set_by_pointer(st_system_net, rd_in, (collected_number)system_rbytes); + rrddim_set_by_pointer(st_system_net, rd_out, (collected_number)system_tbytes); + + rrdset_done(st_system_net); + } + netdev_cleanup(); return 0; diff --git a/src/proc_net_ip_vs_stats.c b/src/proc_net_ip_vs_stats.c index aa806b460..d76972f3c 100644 --- a/src/proc_net_ip_vs_stats.c +++ b/src/proc_net_ip_vs_stats.c @@ -35,16 +35,27 @@ int do_proc_net_ip_vs_stats(int update_every, usec_t dt) { InBytes = strtoull(procfile_lineword(ff, 2, 3), NULL, 16); OutBytes = strtoull(procfile_lineword(ff, 2, 4), NULL, 16); - RRDSET *st; // -------------------------------------------------------------------- if(do_sockets) { - st = rrdset_find_localhost(RRD_TYPE_NET_IPVS ".sockets"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_IPVS, "sockets", NULL, RRD_TYPE_NET_IPVS, NULL - , "IPVS New Connections", "connections/s", 3101, update_every - , RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_IPVS + , "sockets" + , NULL + , RRD_TYPE_NET_IPVS + , NULL + , "IPVS New Connections" + , "connections/s" + , "proc" + , "net/ip_vs_stats" + , 3101 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "connections", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } @@ -57,10 +68,22 @@ int do_proc_net_ip_vs_stats(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_packets) { - st = rrdset_find_localhost(RRD_TYPE_NET_IPVS ".packets"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_IPVS, "packets", NULL, RRD_TYPE_NET_IPVS, NULL, "IPVS Packets" - , "packets/s", 3102, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_IPVS + , "packets" + , NULL + , RRD_TYPE_NET_IPVS + , NULL + , "IPVS Packets" + , "packets/s" + , "proc" + , "net/ip_vs_stats" + , 3102 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -75,10 +98,22 @@ int do_proc_net_ip_vs_stats(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_bandwidth) { - st = rrdset_find_localhost(RRD_TYPE_NET_IPVS ".net"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_IPVS, "net", NULL, RRD_TYPE_NET_IPVS, NULL, "IPVS Bandwidth" - , "kilobits/s", 3100, update_every, RRDSET_TYPE_AREA); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_IPVS + , "net" + , NULL + , RRD_TYPE_NET_IPVS + , NULL + , "IPVS Bandwidth" + , "kilobits/s" + , "proc" + , "net/ip_vs_stats" + , 3100 + , update_every + , RRDSET_TYPE_AREA + ); rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); diff --git a/src/proc_net_netstat.c b/src/proc_net_netstat.c index e01b81d28..dd070e4c3 100644 --- a/src/proc_net_netstat.c +++ b/src/proc_net_netstat.c @@ -22,7 +22,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) { (void)dt; static int do_bandwidth = -1, do_inerrors = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, do_bcast_p = -1, do_ecn = -1, \ - do_tcpext_reorder = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, do_tcpext_memory = -1; + do_tcpext_reorder = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, do_tcpext_memory = -1, + do_tcpext_listen = -1; + static uint32_t hash_ipext = 0, hash_tcpext = 0; static procfile *ff = NULL; @@ -93,6 +95,10 @@ int do_proc_net_netstat(int update_every, usec_t dt) { static unsigned long long tcpext_TCPAbortOnLinger = 0; // connections aborted after user close in linger timeout static unsigned long long tcpext_TCPAbortFailed = 0; // times unable to send RST due to no memory + // https://perfchron.com/2015/12/26/investigating-linux-network-issues-with-netstat-and-nstat/ + static unsigned long long tcpext_ListenOverflows = 0; // times the listen queue of a socket overflowed + static unsigned long long tcpext_ListenDrops = 0; // SYNs to LISTEN sockets ignored + // IPv4 TCP memory pressures static unsigned long long tcpext_TCPMemoryPressures = 0; @@ -116,6 +122,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) { do_tcpext_ofo = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP out-of-order queue", CONFIG_BOOLEAN_AUTO); do_tcpext_connaborts = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP connection aborts", CONFIG_BOOLEAN_AUTO); do_tcpext_memory = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP memory pressures", CONFIG_BOOLEAN_AUTO); + do_tcpext_listen = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP listen issues", CONFIG_BOOLEAN_AUTO); arl_ipext = arl_create("netstat/ipext", NULL, 60); arl_tcpext = arl_create("netstat/tcpext", NULL, 60); @@ -197,6 +204,11 @@ int do_proc_net_netstat(int update_every, usec_t dt) { arl_expect(arl_tcpext, "TCPMemoryPressures", &tcpext_TCPMemoryPressures); } + if(do_tcpext_listen != CONFIG_BOOLEAN_NO) { + arl_expect(arl_tcpext, "ListenOverflows", &tcpext_ListenOverflows); + arl_expect(arl_tcpext, "ListenDrops", &tcpext_ListenDrops); + } + // shared metrics arl_expect(arl_tcpext, "TCPSynRetrans", &tcpext_TCPSynRetrans); } @@ -248,7 +260,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "IPv4 Bandwidth" , "kilobits/s" - , 500 + , "proc" + , "net/netstat" + , 501 , update_every , RRDSET_TYPE_AREA ); @@ -281,6 +295,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "IPv4 Input Errors" , "packets/s" + , "proc" + , "net/netstat" , 4000 , update_every , RRDSET_TYPE_LINE @@ -318,6 +334,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "IPv4 Multicast Bandwidth" , "kilobits/s" + , "proc" + , "net/netstat" , 9000 , update_every , RRDSET_TYPE_AREA @@ -354,6 +372,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "IPv4 Broadcast Bandwidth" , "kilobits/s" + , "proc" + , "net/netstat" , 8000 , update_every , RRDSET_TYPE_AREA @@ -390,6 +410,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "IPv4 Multicast Packets" , "packets/s" + , "proc" + , "net/netstat" , 8600 , update_every , RRDSET_TYPE_LINE @@ -425,6 +447,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "IPv4 Broadcast Packets" , "packets/s" + , "proc" + , "net/netstat" , 8500 , update_every , RRDSET_TYPE_LINE @@ -461,6 +485,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "IPv4 ECN Statistics" , "packets/s" + , "proc" + , "net/netstat" , 8700 , update_every , RRDSET_TYPE_LINE @@ -511,6 +537,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "TCP Memory Pressures" , "events/s" + , "proc" + , "net/netstat" , 3000 , update_every , RRDSET_TYPE_LINE @@ -543,6 +571,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "TCP Connection Aborts" , "connections/s" + , "proc" + , "net/netstat" , 3010 , update_every , RRDSET_TYPE_LINE @@ -585,6 +615,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "TCP Reordered Packets by Detection Method" , "packets/s" + , "proc" + , "net/netstat" , 3020 , update_every , RRDSET_TYPE_LINE @@ -624,6 +656,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "TCP Out-Of-Order Queue" , "packets/s" + , "proc" + , "net/netstat" , 3050 , update_every , RRDSET_TYPE_LINE @@ -663,6 +697,8 @@ int do_proc_net_netstat(int update_every, usec_t dt) { , NULL , "TCP SYN Cookies" , "packets/s" + , "proc" + , "net/netstat" , 3100 , update_every , RRDSET_TYPE_LINE @@ -682,6 +718,42 @@ int do_proc_net_netstat(int update_every, usec_t dt) { rrdset_done(st_syncookies); } + // -------------------------------------------------------------------- + + if(do_tcpext_listen == CONFIG_BOOLEAN_YES || (do_tcpext_listen == CONFIG_BOOLEAN_AUTO && (tcpext_ListenOverflows || tcpext_ListenDrops))) { + do_tcpext_listen = CONFIG_BOOLEAN_YES; + + static RRDSET *st_listen = NULL; + static RRDDIM *rd_overflows = NULL, *rd_drops = NULL; + + if(unlikely(!st_listen)) { + + st_listen = rrdset_create_localhost( + "ipv4" + , "tcplistenissues" + , NULL + , "tcp" + , NULL + , "TCP Listen Socket Issues" + , "packets/s" + , "proc" + , "net/netstat" + , 3015 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_overflows = rrddim_add(st_listen, "ListenOverflows", "overflows", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_drops = rrddim_add(st_listen, "ListenDrops", "drops", 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_listen); + + rrddim_set_by_pointer(st_listen, rd_overflows, tcpext_ListenOverflows); + rrddim_set_by_pointer(st_listen, rd_drops, tcpext_ListenDrops); + + rrdset_done(st_listen); + } } } diff --git a/src/proc_net_rpc_nfs.c b/src/proc_net_rpc_nfs.c index 0df919635..126216e0b 100644 --- a/src/proc_net_rpc_nfs.c +++ b/src/proc_net_rpc_nfs.c @@ -4,127 +4,133 @@ struct nfs_procs { char name[30]; unsigned long long value; int present; + RRDDIM *rd; }; struct nfs_procs nfs_proc2_values[] = { - { "null", 0ULL, 0 }, - { "getattr", 0ULL, 0 }, - { "setattr", 0ULL, 0 }, - { "root", 0ULL, 0 }, - { "lookup", 0ULL, 0 }, - { "readlink", 0ULL, 0 }, - { "read", 0ULL, 0 }, - { "wrcache", 0ULL, 0 }, - { "write", 0ULL, 0 }, - { "create", 0ULL, 0 }, - { "remove", 0ULL, 0 }, - { "rename", 0ULL, 0 }, - { "link", 0ULL, 0 }, - { "symlink", 0ULL, 0 }, - { "mkdir", 0ULL, 0 }, - { "rmdir", 0ULL, 0 }, - { "readdir", 0ULL, 0 }, - { "fsstat", 0ULL, 0 }, - - /* termination */ - { "", 0ULL, 0 } + { "null" , 0ULL, 0, NULL} + , {"getattr" , 0ULL, 0, NULL} + , {"setattr" , 0ULL, 0, NULL} + , {"root" , 0ULL, 0, NULL} + , {"lookup" , 0ULL, 0, NULL} + , {"readlink", 0ULL, 0, NULL} + , {"read" , 0ULL, 0, NULL} + , {"wrcache" , 0ULL, 0, NULL} + , {"write" , 0ULL, 0, NULL} + , {"create" , 0ULL, 0, NULL} + , {"remove" , 0ULL, 0, NULL} + , {"rename" , 0ULL, 0, NULL} + , {"link" , 0ULL, 0, NULL} + , {"symlink" , 0ULL, 0, NULL} + , {"mkdir" , 0ULL, 0, NULL} + , {"rmdir" , 0ULL, 0, NULL} + , {"readdir" , 0ULL, 0, NULL} + , {"fsstat" , 0ULL, 0, NULL} + , + + /* termination */ + { "" , 0ULL, 0, NULL} }; struct nfs_procs nfs_proc3_values[] = { - { "null", 0ULL, 0 }, - { "getattr", 0ULL, 0 }, - { "setattr", 0ULL, 0 }, - { "lookup", 0ULL, 0 }, - { "access", 0ULL, 0 }, - { "readlink", 0ULL, 0 }, - { "read", 0ULL, 0 }, - { "write", 0ULL, 0 }, - { "create", 0ULL, 0 }, - { "mkdir", 0ULL, 0 }, - { "symlink", 0ULL, 0 }, - { "mknod", 0ULL, 0 }, - { "remove", 0ULL, 0 }, - { "rmdir", 0ULL, 0 }, - { "rename", 0ULL, 0 }, - { "link", 0ULL, 0 }, - { "readdir", 0ULL, 0 }, - { "readdirplus", 0ULL, 0 }, - { "fsstat", 0ULL, 0 }, - { "fsinfo", 0ULL, 0 }, - { "pathconf", 0ULL, 0 }, - { "commit", 0ULL, 0 }, - - /* termination */ - { "", 0ULL, 0 } + { "null" , 0ULL, 0, NULL} + , {"getattr" , 0ULL, 0, NULL} + , {"setattr" , 0ULL, 0, NULL} + , {"lookup" , 0ULL, 0, NULL} + , {"access" , 0ULL, 0, NULL} + , {"readlink" , 0ULL, 0, NULL} + , {"read" , 0ULL, 0, NULL} + , {"write" , 0ULL, 0, NULL} + , {"create" , 0ULL, 0, NULL} + , {"mkdir" , 0ULL, 0, NULL} + , {"symlink" , 0ULL, 0, NULL} + , {"mknod" , 0ULL, 0, NULL} + , {"remove" , 0ULL, 0, NULL} + , {"rmdir" , 0ULL, 0, NULL} + , {"rename" , 0ULL, 0, NULL} + , {"link" , 0ULL, 0, NULL} + , {"readdir" , 0ULL, 0, NULL} + , {"readdirplus", 0ULL, 0, NULL} + , {"fsstat" , 0ULL, 0, NULL} + , {"fsinfo" , 0ULL, 0, NULL} + , {"pathconf" , 0ULL, 0, NULL} + , {"commit" , 0ULL, 0, NULL} + , + + /* termination */ + { "" , 0ULL, 0, NULL} }; struct nfs_procs nfs_proc4_values[] = { - { "null", 0ULL, 0 }, - { "read", 0ULL, 0 }, - { "write", 0ULL, 0 }, - { "commit", 0ULL, 0 }, - { "open", 0ULL, 0 }, - { "open_conf", 0ULL, 0 }, - { "open_noat", 0ULL, 0 }, - { "open_dgrd", 0ULL, 0 }, - { "close", 0ULL, 0 }, - { "setattr", 0ULL, 0 }, - { "fsinfo", 0ULL, 0 }, - { "renew", 0ULL, 0 }, - { "setclntid", 0ULL, 0 }, - { "confirm", 0ULL, 0 }, - { "lock", 0ULL, 0 }, - { "lockt", 0ULL, 0 }, - { "locku", 0ULL, 0 }, - { "access", 0ULL, 0 }, - { "getattr", 0ULL, 0 }, - { "lookup", 0ULL, 0 }, - { "lookup_root", 0ULL, 0 }, - { "remove", 0ULL, 0 }, - { "rename", 0ULL, 0 }, - { "link", 0ULL, 0 }, - { "symlink", 0ULL, 0 }, - { "create", 0ULL, 0 }, - { "pathconf", 0ULL, 0 }, - { "statfs", 0ULL, 0 }, - { "readlink", 0ULL, 0 }, - { "readdir", 0ULL, 0 }, - { "server_caps", 0ULL, 0 }, - { "delegreturn", 0ULL, 0 }, - { "getacl", 0ULL, 0 }, - { "setacl", 0ULL, 0 }, - { "fs_locations", 0ULL, 0 }, - { "rel_lkowner", 0ULL, 0 }, - { "secinfo", 0ULL, 0 }, - { "fsid_present", 0ULL, 0 }, - - /* nfsv4.1 client ops */ - { "exchange_id", 0ULL, 0 }, - { "create_session", 0ULL, 0 }, - { "destroy_session", 0ULL, 0 }, - { "sequence", 0ULL, 0 }, - { "get_lease_time", 0ULL, 0 }, - { "reclaim_comp", 0ULL, 0 }, - { "layoutget", 0ULL, 0 }, - { "getdevinfo", 0ULL, 0 }, - { "layoutcommit", 0ULL, 0 }, - { "layoutreturn", 0ULL, 0 }, - { "secinfo_no", 0ULL, 0 }, - { "test_stateid", 0ULL, 0 }, - { "free_stateid", 0ULL, 0 }, - { "getdevicelist", 0ULL, 0 }, - { "bind_conn_to_ses", 0ULL, 0 }, - { "destroy_clientid", 0ULL, 0 }, - - /* nfsv4.2 client ops */ - { "seek", 0ULL, 0 }, - { "allocate", 0ULL, 0 }, - { "deallocate", 0ULL, 0 }, - { "layoutstats", 0ULL, 0 }, - { "clone", 0ULL, 0 }, - - /* termination */ - { "", 0ULL, 0 } + { "null" , 0ULL, 0, NULL} + , {"read" , 0ULL, 0, NULL} + , {"write" , 0ULL, 0, NULL} + , {"commit" , 0ULL, 0, NULL} + , {"open" , 0ULL, 0, NULL} + , {"open_conf" , 0ULL, 0, NULL} + , {"open_noat" , 0ULL, 0, NULL} + , {"open_dgrd" , 0ULL, 0, NULL} + , {"close" , 0ULL, 0, NULL} + , {"setattr" , 0ULL, 0, NULL} + , {"fsinfo" , 0ULL, 0, NULL} + , {"renew" , 0ULL, 0, NULL} + , {"setclntid" , 0ULL, 0, NULL} + , {"confirm" , 0ULL, 0, NULL} + , {"lock" , 0ULL, 0, NULL} + , {"lockt" , 0ULL, 0, NULL} + , {"locku" , 0ULL, 0, NULL} + , {"access" , 0ULL, 0, NULL} + , {"getattr" , 0ULL, 0, NULL} + , {"lookup" , 0ULL, 0, NULL} + , {"lookup_root" , 0ULL, 0, NULL} + , {"remove" , 0ULL, 0, NULL} + , {"rename" , 0ULL, 0, NULL} + , {"link" , 0ULL, 0, NULL} + , {"symlink" , 0ULL, 0, NULL} + , {"create" , 0ULL, 0, NULL} + , {"pathconf" , 0ULL, 0, NULL} + , {"statfs" , 0ULL, 0, NULL} + , {"readlink" , 0ULL, 0, NULL} + , {"readdir" , 0ULL, 0, NULL} + , {"server_caps" , 0ULL, 0, NULL} + , {"delegreturn" , 0ULL, 0, NULL} + , {"getacl" , 0ULL, 0, NULL} + , {"setacl" , 0ULL, 0, NULL} + , {"fs_locations" , 0ULL, 0, NULL} + , {"rel_lkowner" , 0ULL, 0, NULL} + , {"secinfo" , 0ULL, 0, NULL} + , {"fsid_present" , 0ULL, 0, NULL} + , + + /* nfsv4.1 client ops */ + { "exchange_id" , 0ULL, 0, NULL} + , {"create_session" , 0ULL, 0, NULL} + , {"destroy_session" , 0ULL, 0, NULL} + , {"sequence" , 0ULL, 0, NULL} + , {"get_lease_time" , 0ULL, 0, NULL} + , {"reclaim_comp" , 0ULL, 0, NULL} + , {"layoutget" , 0ULL, 0, NULL} + , {"getdevinfo" , 0ULL, 0, NULL} + , {"layoutcommit" , 0ULL, 0, NULL} + , {"layoutreturn" , 0ULL, 0, NULL} + , {"secinfo_no" , 0ULL, 0, NULL} + , {"test_stateid" , 0ULL, 0, NULL} + , {"free_stateid" , 0ULL, 0, NULL} + , {"getdevicelist" , 0ULL, 0, NULL} + , {"bind_conn_to_ses", 0ULL, 0, NULL} + , {"destroy_clientid", 0ULL, 0, NULL} + , + + /* nfsv4.2 client ops */ + { "seek" , 0ULL, 0, NULL} + , {"allocate" , 0ULL, 0, NULL} + , {"deallocate" , 0ULL, 0, NULL} + , {"layoutstats" , 0ULL, 0, NULL} + , {"clone" , 0ULL, 0, NULL} + , + + /* termination */ + { "" , 0ULL, 0, NULL} }; int do_proc_net_rpc_nfs(int update_every, usec_t dt) { @@ -264,68 +270,111 @@ int do_proc_net_rpc_nfs(int update_every, usec_t dt) { } } - RRDSET *st; - // -------------------------------------------------------------------- if(do_net == 2) { - st = rrdset_find_bytype_localhost("nfs", "net"); - if(!st) { - st = rrdset_create_localhost("nfs", "net", NULL, "network", NULL, "NFS Client Network", "operations/s", 5007 - , update_every, RRDSET_TYPE_STACKED); + static RRDSET *st = NULL; + static RRDDIM *rd_udp = NULL, + *rd_tcp = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfs" + , "net" + , NULL + , "network" + , NULL + , "NFS Client Network" + , "operations/s" + , "proc" + , "net/rpc/nfs" + , 5007 + , update_every + , RRDSET_TYPE_STACKED + ); + rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_udp = rrddim_add(st, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_tcp = rrddim_add(st, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); // ignore net_count, net_tcp_connections - if(net_count) {}; - if(net_tcp_connections) {}; + (void)net_count; + (void)net_tcp_connections; - rrddim_set(st, "udp", net_udp_count); - rrddim_set(st, "tcp", net_tcp_count); + rrddim_set_by_pointer(st, rd_udp, net_udp_count); + rrddim_set_by_pointer(st, rd_tcp, net_tcp_count); rrdset_done(st); } // -------------------------------------------------------------------- if(do_rpc == 2) { - st = rrdset_find_bytype_localhost("nfs", "rpc"); - if(!st) { - st = rrdset_create_localhost("nfs", "rpc", NULL, "rpc", NULL, "NFS Client Remote Procedure Calls Statistics" - , "calls/s", 5008, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_calls = NULL, + *rd_retransmits = NULL, + *rd_auth_refresh = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfs" + , "rpc" + , NULL + , "rpc" + , NULL + , "NFS Client Remote Procedure Calls Statistics" + , "calls/s" + , "proc" + , "net/rpc/nfs" + , 5008 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "calls", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "retransmits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "auth_refresh", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_calls = rrddim_add(st, "calls", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_retransmits = rrddim_add(st, "retransmits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_auth_refresh = rrddim_add(st, "auth_refresh", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "calls", rpc_calls); - rrddim_set(st, "retransmits", rpc_retransmits); - rrddim_set(st, "auth_refresh", rpc_auth_refresh); + rrddim_set_by_pointer(st, rd_calls, rpc_calls); + rrddim_set_by_pointer(st, rd_retransmits, rpc_retransmits); + rrddim_set_by_pointer(st, rd_auth_refresh, rpc_auth_refresh); rrdset_done(st); } // -------------------------------------------------------------------- if(do_proc2 == 2) { - unsigned int i; - st = rrdset_find_bytype_localhost("nfs", "proc2"); - if(!st) { - st = rrdset_create_localhost("nfs", "proc2", NULL, "nfsv2rpc", NULL, "NFS v2 Client Remote Procedure Calls" - , "calls/s", 5009, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; nfs_proc2_values[i].present ; i++) - rrddim_add(st, nfs_proc2_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfs" + , "proc2" + , NULL + , "nfsv2rpc" + , NULL + , "NFS v2 Client Remote Procedure Calls" + , "calls/s" + , "proc" + , "net/rpc/nfs" + , 5009 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st); - for(i = 0; nfs_proc2_values[i].present ; i++) - rrddim_set(st, nfs_proc2_values[i].name, nfs_proc2_values[i].value); + size_t i; + for(i = 0; nfs_proc2_values[i].present ; i++) { + if(unlikely(!nfs_proc2_values[i].rd)) + nfs_proc2_values[i].rd = rrddim_add(st, nfs_proc2_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrddim_set_by_pointer(st, nfs_proc2_values[i].rd, nfs_proc2_values[i].value); + } rrdset_done(st); } @@ -333,19 +382,32 @@ int do_proc_net_rpc_nfs(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_proc3 == 2) { - unsigned int i; - st = rrdset_find_bytype_localhost("nfs", "proc3"); - if(!st) { - st = rrdset_create_localhost("nfs", "proc3", NULL, "nfsv3rpc", NULL, "NFS v3 Client Remote Procedure Calls" - , "calls/s", 5010, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; nfs_proc3_values[i].present ; i++) - rrddim_add(st, nfs_proc3_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfs" + , "proc3" + , NULL + , "nfsv3rpc" + , NULL + , "NFS v3 Client Remote Procedure Calls" + , "calls/s" + , "proc" + , "net/rpc/nfs" + , 5010 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st); - for(i = 0; nfs_proc3_values[i].present ; i++) - rrddim_set(st, nfs_proc3_values[i].name, nfs_proc3_values[i].value); + size_t i; + for(i = 0; nfs_proc3_values[i].present ; i++) { + if(unlikely(!nfs_proc3_values[i].rd)) + nfs_proc3_values[i].rd = rrddim_add(st, nfs_proc3_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrddim_set_by_pointer(st, nfs_proc3_values[i].rd, nfs_proc3_values[i].value); + } rrdset_done(st); } @@ -353,19 +415,32 @@ int do_proc_net_rpc_nfs(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_proc4 == 2) { - unsigned int i; - st = rrdset_find_bytype_localhost("nfs", "proc4"); - if(!st) { - st = rrdset_create_localhost("nfs", "proc4", NULL, "nfsv4rpc", NULL, "NFS v4 Client Remote Procedure Calls" - , "calls/s", 5011, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; nfs_proc4_values[i].present ; i++) - rrddim_add(st, nfs_proc4_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfs" + , "proc4" + , NULL + , "nfsv4rpc" + , NULL + , "NFS v4 Client Remote Procedure Calls" + , "calls/s" + , "proc" + , "net/rpc/nfs" + , 5011 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st); - for(i = 0; nfs_proc4_values[i].present ; i++) - rrddim_set(st, nfs_proc4_values[i].name, nfs_proc4_values[i].value); + size_t i; + for(i = 0; nfs_proc4_values[i].present ; i++) { + if(unlikely(!nfs_proc4_values[i].rd)) + nfs_proc4_values[i].rd = rrddim_add(st, nfs_proc4_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrddim_set_by_pointer(st, nfs_proc4_values[i].rd, nfs_proc4_values[i].value); + } rrdset_done(st); } diff --git a/src/proc_net_rpc_nfsd.c b/src/proc_net_rpc_nfsd.c index b0ed58d13..f0c9a20ce 100644 --- a/src/proc_net_rpc_nfsd.c +++ b/src/proc_net_rpc_nfsd.c @@ -4,208 +4,217 @@ struct nfsd_procs { char name[30]; unsigned long long value; int present; + RRDDIM *rd; }; struct nfsd_procs nfsd_proc2_values[] = { - { "null", 0ULL, 0 }, - { "getattr", 0ULL, 0 }, - { "setattr", 0ULL, 0 }, - { "root", 0ULL, 0 }, - { "lookup", 0ULL, 0 }, - { "readlink", 0ULL, 0 }, - { "read", 0ULL, 0 }, - { "wrcache", 0ULL, 0 }, - { "write", 0ULL, 0 }, - { "create", 0ULL, 0 }, - { "remove", 0ULL, 0 }, - { "rename", 0ULL, 0 }, - { "link", 0ULL, 0 }, - { "symlink", 0ULL, 0 }, - { "mkdir", 0ULL, 0 }, - { "rmdir", 0ULL, 0 }, - { "readdir", 0ULL, 0 }, - { "fsstat", 0ULL, 0 }, - - /* termination */ - { "", 0ULL, 0 } + { "null" , 0ULL, 0, NULL} + , {"getattr" , 0ULL, 0, NULL} + , {"setattr" , 0ULL, 0, NULL} + , {"root" , 0ULL, 0, NULL} + , {"lookup" , 0ULL, 0, NULL} + , {"readlink", 0ULL, 0, NULL} + , {"read" , 0ULL, 0, NULL} + , {"wrcache" , 0ULL, 0, NULL} + , {"write" , 0ULL, 0, NULL} + , {"create" , 0ULL, 0, NULL} + , {"remove" , 0ULL, 0, NULL} + , {"rename" , 0ULL, 0, NULL} + , {"link" , 0ULL, 0, NULL} + , {"symlink" , 0ULL, 0, NULL} + , {"mkdir" , 0ULL, 0, NULL} + , {"rmdir" , 0ULL, 0, NULL} + , {"readdir" , 0ULL, 0, NULL} + , {"fsstat" , 0ULL, 0, NULL} + , + + /* termination */ + { "" , 0ULL, 0, NULL} }; struct nfsd_procs nfsd_proc3_values[] = { - { "null", 0ULL, 0 }, - { "getattr", 0ULL, 0 }, - { "setattr", 0ULL, 0 }, - { "lookup", 0ULL, 0 }, - { "access", 0ULL, 0 }, - { "readlink", 0ULL, 0 }, - { "read", 0ULL, 0 }, - { "write", 0ULL, 0 }, - { "create", 0ULL, 0 }, - { "mkdir", 0ULL, 0 }, - { "symlink", 0ULL, 0 }, - { "mknod", 0ULL, 0 }, - { "remove", 0ULL, 0 }, - { "rmdir", 0ULL, 0 }, - { "rename", 0ULL, 0 }, - { "link", 0ULL, 0 }, - { "readdir", 0ULL, 0 }, - { "readdirplus", 0ULL, 0 }, - { "fsstat", 0ULL, 0 }, - { "fsinfo", 0ULL, 0 }, - { "pathconf", 0ULL, 0 }, - { "commit", 0ULL, 0 }, - - /* termination */ - { "", 0ULL, 0 } + { "null" , 0ULL, 0, NULL} + , {"getattr" , 0ULL, 0, NULL} + , {"setattr" , 0ULL, 0, NULL} + , {"lookup" , 0ULL, 0, NULL} + , {"access" , 0ULL, 0, NULL} + , {"readlink" , 0ULL, 0, NULL} + , {"read" , 0ULL, 0, NULL} + , {"write" , 0ULL, 0, NULL} + , {"create" , 0ULL, 0, NULL} + , {"mkdir" , 0ULL, 0, NULL} + , {"symlink" , 0ULL, 0, NULL} + , {"mknod" , 0ULL, 0, NULL} + , {"remove" , 0ULL, 0, NULL} + , {"rmdir" , 0ULL, 0, NULL} + , {"rename" , 0ULL, 0, NULL} + , {"link" , 0ULL, 0, NULL} + , {"readdir" , 0ULL, 0, NULL} + , {"readdirplus", 0ULL, 0, NULL} + , {"fsstat" , 0ULL, 0, NULL} + , {"fsinfo" , 0ULL, 0, NULL} + , {"pathconf" , 0ULL, 0, NULL} + , {"commit" , 0ULL, 0, NULL} + , + + /* termination */ + { "" , 0ULL, 0, NULL} }; struct nfsd_procs nfsd_proc4_values[] = { - { "null", 0ULL, 0 }, - { "read", 0ULL, 0 }, - { "write", 0ULL, 0 }, - { "commit", 0ULL, 0 }, - { "open", 0ULL, 0 }, - { "open_conf", 0ULL, 0 }, - { "open_noat", 0ULL, 0 }, - { "open_dgrd", 0ULL, 0 }, - { "close", 0ULL, 0 }, - { "setattr", 0ULL, 0 }, - { "fsinfo", 0ULL, 0 }, - { "renew", 0ULL, 0 }, - { "setclntid", 0ULL, 0 }, - { "confirm", 0ULL, 0 }, - { "lock", 0ULL, 0 }, - { "lockt", 0ULL, 0 }, - { "locku", 0ULL, 0 }, - { "access", 0ULL, 0 }, - { "getattr", 0ULL, 0 }, - { "lookup", 0ULL, 0 }, - { "lookup_root", 0ULL, 0 }, - { "remove", 0ULL, 0 }, - { "rename", 0ULL, 0 }, - { "link", 0ULL, 0 }, - { "symlink", 0ULL, 0 }, - { "create", 0ULL, 0 }, - { "pathconf", 0ULL, 0 }, - { "statfs", 0ULL, 0 }, - { "readlink", 0ULL, 0 }, - { "readdir", 0ULL, 0 }, - { "server_caps", 0ULL, 0 }, - { "delegreturn", 0ULL, 0 }, - { "getacl", 0ULL, 0 }, - { "setacl", 0ULL, 0 }, - { "fs_locations", 0ULL, 0 }, - { "rel_lkowner", 0ULL, 0 }, - { "secinfo", 0ULL, 0 }, - { "fsid_present", 0ULL, 0 }, - - /* nfsv4.1 client ops */ - { "exchange_id", 0ULL, 0 }, - { "create_session", 0ULL, 0 }, - { "destroy_session", 0ULL, 0 }, - { "sequence", 0ULL, 0 }, - { "get_lease_time", 0ULL, 0 }, - { "reclaim_comp", 0ULL, 0 }, - { "layoutget", 0ULL, 0 }, - { "getdevinfo", 0ULL, 0 }, - { "layoutcommit", 0ULL, 0 }, - { "layoutreturn", 0ULL, 0 }, - { "secinfo_no", 0ULL, 0 }, - { "test_stateid", 0ULL, 0 }, - { "free_stateid", 0ULL, 0 }, - { "getdevicelist", 0ULL, 0 }, - { "bind_conn_to_ses", 0ULL, 0 }, - { "destroy_clientid", 0ULL, 0 }, - - /* nfsv4.2 client ops */ - { "seek", 0ULL, 0 }, - { "allocate", 0ULL, 0 }, - { "deallocate", 0ULL, 0 }, - { "layoutstats", 0ULL, 0 }, - { "clone", 0ULL, 0 }, - - /* termination */ - { "", 0ULL, 0 } + { "null" , 0ULL, 0, NULL} + , {"read" , 0ULL, 0, NULL} + , {"write" , 0ULL, 0, NULL} + , {"commit" , 0ULL, 0, NULL} + , {"open" , 0ULL, 0, NULL} + , {"open_conf" , 0ULL, 0, NULL} + , {"open_noat" , 0ULL, 0, NULL} + , {"open_dgrd" , 0ULL, 0, NULL} + , {"close" , 0ULL, 0, NULL} + , {"setattr" , 0ULL, 0, NULL} + , {"fsinfo" , 0ULL, 0, NULL} + , {"renew" , 0ULL, 0, NULL} + , {"setclntid" , 0ULL, 0, NULL} + , {"confirm" , 0ULL, 0, NULL} + , {"lock" , 0ULL, 0, NULL} + , {"lockt" , 0ULL, 0, NULL} + , {"locku" , 0ULL, 0, NULL} + , {"access" , 0ULL, 0, NULL} + , {"getattr" , 0ULL, 0, NULL} + , {"lookup" , 0ULL, 0, NULL} + , {"lookup_root" , 0ULL, 0, NULL} + , {"remove" , 0ULL, 0, NULL} + , {"rename" , 0ULL, 0, NULL} + , {"link" , 0ULL, 0, NULL} + , {"symlink" , 0ULL, 0, NULL} + , {"create" , 0ULL, 0, NULL} + , {"pathconf" , 0ULL, 0, NULL} + , {"statfs" , 0ULL, 0, NULL} + , {"readlink" , 0ULL, 0, NULL} + , {"readdir" , 0ULL, 0, NULL} + , {"server_caps" , 0ULL, 0, NULL} + , {"delegreturn" , 0ULL, 0, NULL} + , {"getacl" , 0ULL, 0, NULL} + , {"setacl" , 0ULL, 0, NULL} + , {"fs_locations" , 0ULL, 0, NULL} + , {"rel_lkowner" , 0ULL, 0, NULL} + , {"secinfo" , 0ULL, 0, NULL} + , {"fsid_present" , 0ULL, 0, NULL} + , + + /* nfsv4.1 client ops */ + { "exchange_id" , 0ULL, 0, NULL} + , {"create_session" , 0ULL, 0, NULL} + , {"destroy_session" , 0ULL, 0, NULL} + , {"sequence" , 0ULL, 0, NULL} + , {"get_lease_time" , 0ULL, 0, NULL} + , {"reclaim_comp" , 0ULL, 0, NULL} + , {"layoutget" , 0ULL, 0, NULL} + , {"getdevinfo" , 0ULL, 0, NULL} + , {"layoutcommit" , 0ULL, 0, NULL} + , {"layoutreturn" , 0ULL, 0, NULL} + , {"secinfo_no" , 0ULL, 0, NULL} + , {"test_stateid" , 0ULL, 0, NULL} + , {"free_stateid" , 0ULL, 0, NULL} + , {"getdevicelist" , 0ULL, 0, NULL} + , {"bind_conn_to_ses", 0ULL, 0, NULL} + , {"destroy_clientid", 0ULL, 0, NULL} + , + + /* nfsv4.2 client ops */ + { "seek" , 0ULL, 0, NULL} + , {"allocate" , 0ULL, 0, NULL} + , {"deallocate" , 0ULL, 0, NULL} + , {"layoutstats" , 0ULL, 0, NULL} + , {"clone" , 0ULL, 0, NULL} + , + + /* termination */ + { "" , 0ULL, 0, NULL} }; struct nfsd_procs nfsd4_ops_values[] = { - { "unused_op0", 0ULL, 0}, - { "unused_op1", 0ULL, 0}, - { "future_op2", 0ULL, 0}, - { "access", 0ULL, 0}, - { "close", 0ULL, 0}, - { "commit", 0ULL, 0}, - { "create", 0ULL, 0}, - { "delegpurge", 0ULL, 0}, - { "delegreturn", 0ULL, 0}, - { "getattr", 0ULL, 0}, - { "getfh", 0ULL, 0}, - { "link", 0ULL, 0}, - { "lock", 0ULL, 0}, - { "lockt", 0ULL, 0}, - { "locku", 0ULL, 0}, - { "lookup", 0ULL, 0}, - { "lookup_root", 0ULL, 0}, - { "nverify", 0ULL, 0}, - { "open", 0ULL, 0}, - { "openattr", 0ULL, 0}, - { "open_confirm", 0ULL, 0}, - { "open_downgrade", 0ULL, 0}, - { "putfh", 0ULL, 0}, - { "putpubfh", 0ULL, 0}, - { "putrootfh", 0ULL, 0}, - { "read", 0ULL, 0}, - { "readdir", 0ULL, 0}, - { "readlink", 0ULL, 0}, - { "remove", 0ULL, 0}, - { "rename", 0ULL, 0}, - { "renew", 0ULL, 0}, - { "restorefh", 0ULL, 0}, - { "savefh", 0ULL, 0}, - { "secinfo", 0ULL, 0}, - { "setattr", 0ULL, 0}, - { "setclientid", 0ULL, 0}, - { "setclientid_confirm", 0ULL, 0}, - { "verify", 0ULL, 0}, - { "write", 0ULL, 0}, - { "release_lockowner", 0ULL, 0}, - - /* nfs41 */ - { "backchannel_ctl", 0ULL, 0}, - { "bind_conn_to_session", 0ULL, 0}, - { "exchange_id", 0ULL, 0}, - { "create_session", 0ULL, 0}, - { "destroy_session", 0ULL, 0}, - { "free_stateid", 0ULL, 0}, - { "get_dir_delegation", 0ULL, 0}, - { "getdeviceinfo", 0ULL, 0}, - { "getdevicelist", 0ULL, 0}, - { "layoutcommit", 0ULL, 0}, - { "layoutget", 0ULL, 0}, - { "layoutreturn", 0ULL, 0}, - { "secinfo_no_name", 0ULL, 0}, - { "sequence", 0ULL, 0}, - { "set_ssv", 0ULL, 0}, - { "test_stateid", 0ULL, 0}, - { "want_delegation", 0ULL, 0}, - { "destroy_clientid", 0ULL, 0}, - { "reclaim_complete", 0ULL, 0}, - - /* nfs42 */ - { "allocate", 0ULL, 0}, - { "copy", 0ULL, 0}, - { "copy_notify", 0ULL, 0}, - { "deallocate", 0ULL, 0}, - { "ioadvise", 0ULL, 0}, - { "layouterror", 0ULL, 0}, - { "layoutstats", 0ULL, 0}, - { "offload_cancel", 0ULL, 0}, - { "offload_status", 0ULL, 0}, - { "read_plus", 0ULL, 0}, - { "seek", 0ULL, 0}, - { "write_same", 0ULL, 0}, - - /* termination */ - { "", 0ULL, 0 } + { "unused_op0" , 0ULL, 0, NULL} + , {"unused_op1" , 0ULL, 0, NULL} + , {"future_op2" , 0ULL, 0, NULL} + , {"access" , 0ULL, 0, NULL} + , {"close" , 0ULL, 0, NULL} + , {"commit" , 0ULL, 0, NULL} + , {"create" , 0ULL, 0, NULL} + , {"delegpurge" , 0ULL, 0, NULL} + , {"delegreturn" , 0ULL, 0, NULL} + , {"getattr" , 0ULL, 0, NULL} + , {"getfh" , 0ULL, 0, NULL} + , {"link" , 0ULL, 0, NULL} + , {"lock" , 0ULL, 0, NULL} + , {"lockt" , 0ULL, 0, NULL} + , {"locku" , 0ULL, 0, NULL} + , {"lookup" , 0ULL, 0, NULL} + , {"lookup_root" , 0ULL, 0, NULL} + , {"nverify" , 0ULL, 0, NULL} + , {"open" , 0ULL, 0, NULL} + , {"openattr" , 0ULL, 0, NULL} + , {"open_confirm" , 0ULL, 0, NULL} + , {"open_downgrade" , 0ULL, 0, NULL} + , {"putfh" , 0ULL, 0, NULL} + , {"putpubfh" , 0ULL, 0, NULL} + , {"putrootfh" , 0ULL, 0, NULL} + , {"read" , 0ULL, 0, NULL} + , {"readdir" , 0ULL, 0, NULL} + , {"readlink" , 0ULL, 0, NULL} + , {"remove" , 0ULL, 0, NULL} + , {"rename" , 0ULL, 0, NULL} + , {"renew" , 0ULL, 0, NULL} + , {"restorefh" , 0ULL, 0, NULL} + , {"savefh" , 0ULL, 0, NULL} + , {"secinfo" , 0ULL, 0, NULL} + , {"setattr" , 0ULL, 0, NULL} + , {"setclientid" , 0ULL, 0, NULL} + , {"setclientid_confirm" , 0ULL, 0, NULL} + , {"verify" , 0ULL, 0, NULL} + , {"write" , 0ULL, 0, NULL} + , {"release_lockowner" , 0ULL, 0, NULL} + , + + /* nfs41 */ + { "backchannel_ctl" , 0ULL, 0, NULL} + , {"bind_conn_to_session", 0ULL, 0, NULL} + , {"exchange_id" , 0ULL, 0, NULL} + , {"create_session" , 0ULL, 0, NULL} + , {"destroy_session" , 0ULL, 0, NULL} + , {"free_stateid" , 0ULL, 0, NULL} + , {"get_dir_delegation" , 0ULL, 0, NULL} + , {"getdeviceinfo" , 0ULL, 0, NULL} + , {"getdevicelist" , 0ULL, 0, NULL} + , {"layoutcommit" , 0ULL, 0, NULL} + , {"layoutget" , 0ULL, 0, NULL} + , {"layoutreturn" , 0ULL, 0, NULL} + , {"secinfo_no_name" , 0ULL, 0, NULL} + , {"sequence" , 0ULL, 0, NULL} + , {"set_ssv" , 0ULL, 0, NULL} + , {"test_stateid" , 0ULL, 0, NULL} + , {"want_delegation" , 0ULL, 0, NULL} + , {"destroy_clientid" , 0ULL, 0, NULL} + , {"reclaim_complete" , 0ULL, 0, NULL} + , + + /* nfs42 */ + { "allocate" , 0ULL, 0, NULL} + , {"copy" , 0ULL, 0, NULL} + , {"copy_notify" , 0ULL, 0, NULL} + , {"deallocate" , 0ULL, 0, NULL} + , {"ioadvise" , 0ULL, 0, NULL} + , {"layouterror" , 0ULL, 0, NULL} + , {"layoutstats" , 0ULL, 0, NULL} + , {"offload_cancel" , 0ULL, 0, NULL} + , {"offload_status" , 0ULL, 0, NULL} + , {"read_plus" , 0ULL, 0, NULL} + , {"seek" , 0ULL, 0, NULL} + , {"write_same" , 0ULL, 0, NULL} + , + + /* termination */ + { "" , 0ULL, 0, NULL} }; @@ -487,234 +496,403 @@ int do_proc_net_rpc_nfsd(int update_every, usec_t dt) { } } - RRDSET *st; - // -------------------------------------------------------------------- if(do_rc == 2) { - st = rrdset_find_bytype_localhost("nfsd", "readcache"); - if(!st) { - st = rrdset_create_localhost("nfsd", "readcache", NULL, "cache", NULL, "NFS Server Read Cache", "reads/s" - , 5000, update_every, RRDSET_TYPE_STACKED); - - rrddim_add(st, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "nocache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + static RRDDIM *rd_hits = NULL, + *rd_misses = NULL, + *rd_nocache = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "readcache" + , NULL + , "cache" + , NULL + , "NFS Server Read Cache" + , "reads/s" + , "proc" + , "net/rpc/nfsd" + , 5000 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_hits = rrddim_add(st, "hits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_misses = rrddim_add(st, "misses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_nocache = rrddim_add(st, "nocache", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "hits", rc_hits); - rrddim_set(st, "misses", rc_misses); - rrddim_set(st, "nocache", rc_nocache); + rrddim_set_by_pointer(st, rd_hits, rc_hits); + rrddim_set_by_pointer(st, rd_misses, rc_misses); + rrddim_set_by_pointer(st, rd_nocache, rc_nocache); rrdset_done(st); } // -------------------------------------------------------------------- if(do_fh == 2) { - st = rrdset_find_bytype_localhost("nfsd", "filehandles"); - if(!st) { - st = rrdset_create_localhost("nfsd", "filehandles", NULL, "filehandles", NULL, "NFS Server File Handles" - , "handles/s", 5001, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_stale = NULL, + *rd_total_lookups = NULL, + *rd_anonymous_lookups = NULL, + *rd_dir_not_in_dcache = NULL, + *rd_non_dir_not_in_dcache = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "filehandles" + , NULL + , "filehandles" + , NULL + , "NFS Server File Handles" + , "handles/s" + , "proc" + , "net/rpc/nfsd" + , 5001 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "stale", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "total_lookups", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "anonymous_lookups", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "dir_not_in_dcache", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "non_dir_not_in_dcache", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_stale = rrddim_add(st, "stale", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_total_lookups = rrddim_add(st, "total_lookups", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_anonymous_lookups = rrddim_add(st, "anonymous_lookups", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_dir_not_in_dcache = rrddim_add(st, "dir_not_in_dcache", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_non_dir_not_in_dcache = rrddim_add(st, "non_dir_not_in_dcache", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "stale", fh_stale); - rrddim_set(st, "total_lookups", fh_total_lookups); - rrddim_set(st, "anonymous_lookups", fh_anonymous_lookups); - rrddim_set(st, "dir_not_in_dcache", fh_dir_not_in_dcache); - rrddim_set(st, "non_dir_not_in_dcache", fh_non_dir_not_in_dcache); + rrddim_set_by_pointer(st, rd_stale, fh_stale); + rrddim_set_by_pointer(st, rd_total_lookups, fh_total_lookups); + rrddim_set_by_pointer(st, rd_anonymous_lookups, fh_anonymous_lookups); + rrddim_set_by_pointer(st, rd_dir_not_in_dcache, fh_dir_not_in_dcache); + rrddim_set_by_pointer(st, rd_non_dir_not_in_dcache, fh_non_dir_not_in_dcache); rrdset_done(st); } // -------------------------------------------------------------------- if(do_io == 2) { - st = rrdset_find_bytype_localhost("nfsd", "io"); - if(!st) { - st = rrdset_create_localhost("nfsd", "io", NULL, "io", NULL, "NFS Server I/O", "kilobytes/s", 5002 - , update_every, RRDSET_TYPE_AREA); - - rrddim_add(st, "read", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "write", NULL, -1, 1000, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + static RRDDIM *rd_read = NULL, + *rd_write = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "io" + , NULL + , "io" + , NULL + , "NFS Server I/O" + , "kilobytes/s" + , "proc" + , "net/rpc/nfsd" + , 5002 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_read = rrddim_add(st, "read", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + rd_write = rrddim_add(st, "write", NULL, -1, 1000, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "read", io_read); - rrddim_set(st, "write", io_write); + rrddim_set_by_pointer(st, rd_read, io_read); + rrddim_set_by_pointer(st, rd_write, io_write); rrdset_done(st); } // -------------------------------------------------------------------- if(do_th == 2) { - st = rrdset_find_bytype_localhost("nfsd", "threads"); - if(!st) { - st = rrdset_create_localhost("nfsd", "threads", NULL, "threads", NULL, "NFS Server Threads", "threads", 5003 - , update_every, RRDSET_TYPE_LINE); + { + static RRDSET *st = NULL; + static RRDDIM *rd_threads = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "threads" + , NULL + , "threads" + , NULL + , "NFS Server Threads" + , "threads" + , "proc" + , "net/rpc/nfsd" + , 5003 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_threads = rrddim_add(st, "threads", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); - rrddim_add(st, "threads", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_set_by_pointer(st, rd_threads, th_threads); + rrdset_done(st); } - else rrdset_next(st); - - rrddim_set(st, "threads", th_threads); - rrdset_done(st); - st = rrdset_find_bytype_localhost("nfsd", "threads_fullcnt"); - if(!st) { - st = rrdset_create_localhost("nfsd", "threads_fullcnt", NULL, "threads", NULL - , "NFS Server Threads Full Count", "ops/s", 5004, update_every - , RRDSET_TYPE_LINE); + { + static RRDSET *st = NULL; + static RRDDIM *rd_full_count = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "threads_fullcnt" + , NULL + , "threads" + , NULL + , "NFS Server Threads Full Count" + , "ops/s" + , "proc" + , "net/rpc/nfsd" + , 5004 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_full_count = rrddim_add(st, "full_count", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st); - rrddim_add(st, "full_count", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st, rd_full_count, th_fullcnt); + rrdset_done(st); } - else rrdset_next(st); - rrddim_set(st, "full_count", th_fullcnt); - rrdset_done(st); - - st = rrdset_find_bytype_localhost("nfsd", "threads_histogram"); - if(!st) { - st = rrdset_create_localhost("nfsd", "threads_histogram", NULL, "threads", NULL - , "NFS Server Threads Usage Histogram", "percentage", 5005, update_every - , RRDSET_TYPE_LINE); - - rrddim_add(st, "0%-10%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "10%-20%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "20%-30%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "30%-40%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "40%-50%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "50%-60%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "60%-70%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "70%-80%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "80%-90%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); - rrddim_add(st, "90%-100%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + { + static RRDSET *st = NULL; + static RRDDIM *rd_th_hist10 = NULL, + *rd_th_hist20 = NULL, + *rd_th_hist30 = NULL, + *rd_th_hist40 = NULL, + *rd_th_hist50 = NULL, + *rd_th_hist60 = NULL, + *rd_th_hist70 = NULL, + *rd_th_hist80 = NULL, + *rd_th_hist90 = NULL, + *rd_th_hist100 = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "threads_histogram" + , NULL + , "threads" + , NULL + , "NFS Server Threads Usage Histogram" + , "percentage" + , "proc" + , "net/rpc/nfsd" + , 5005 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_th_hist10 = rrddim_add(st, "0%-10%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist20 = rrddim_add(st, "10%-20%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist30 = rrddim_add(st, "20%-30%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist40 = rrddim_add(st, "30%-40%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist50 = rrddim_add(st, "40%-50%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist60 = rrddim_add(st, "50%-60%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist70 = rrddim_add(st, "60%-70%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist80 = rrddim_add(st, "70%-80%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist90 = rrddim_add(st, "80%-90%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + rd_th_hist100 = rrddim_add(st, "90%-100%", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_th_hist10, th_hist10); + rrddim_set_by_pointer(st, rd_th_hist20, th_hist20); + rrddim_set_by_pointer(st, rd_th_hist30, th_hist30); + rrddim_set_by_pointer(st, rd_th_hist40, th_hist40); + rrddim_set_by_pointer(st, rd_th_hist50, th_hist50); + rrddim_set_by_pointer(st, rd_th_hist60, th_hist60); + rrddim_set_by_pointer(st, rd_th_hist70, th_hist70); + rrddim_set_by_pointer(st, rd_th_hist80, th_hist80); + rrddim_set_by_pointer(st, rd_th_hist90, th_hist90); + rrddim_set_by_pointer(st, rd_th_hist100, th_hist100); + rrdset_done(st); } - else rrdset_next(st); - - rrddim_set(st, "0%-10%", th_hist10); - rrddim_set(st, "10%-20%", th_hist20); - rrddim_set(st, "20%-30%", th_hist30); - rrddim_set(st, "30%-40%", th_hist40); - rrddim_set(st, "40%-50%", th_hist50); - rrddim_set(st, "50%-60%", th_hist60); - rrddim_set(st, "60%-70%", th_hist70); - rrddim_set(st, "70%-80%", th_hist80); - rrddim_set(st, "80%-90%", th_hist90); - rrddim_set(st, "90%-100%", th_hist100); - rrdset_done(st); } // -------------------------------------------------------------------- if(do_ra == 2) { - st = rrdset_find_bytype_localhost("nfsd", "readahead"); - if(!st) { - st = rrdset_create_localhost("nfsd", "readahead", NULL, "readahead", NULL, "NFS Server Read Ahead Depth" - , "percentage", 5005, update_every, RRDSET_TYPE_STACKED); - - rrddim_add(st, "10%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "20%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "30%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "40%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "50%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "60%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "70%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "80%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "90%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "100%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); - rrddim_add(st, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + static RRDSET *st = NULL; + static RRDDIM *rd_ra_hist10 = NULL, + *rd_ra_hist20 = NULL, + *rd_ra_hist30 = NULL, + *rd_ra_hist40 = NULL, + *rd_ra_hist50 = NULL, + *rd_ra_hist60 = NULL, + *rd_ra_hist70 = NULL, + *rd_ra_hist80 = NULL, + *rd_ra_hist90 = NULL, + *rd_ra_hist100 = NULL, + *rd_ra_none = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "readahead" + , NULL + , "readahead" + , NULL + , "NFS Server Read Ahead Depth" + , "percentage" + , "proc" + , "net/rpc/nfsd" + , 5005 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_ra_hist10 = rrddim_add(st, "10%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist20 = rrddim_add(st, "20%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist30 = rrddim_add(st, "30%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist40 = rrddim_add(st, "40%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist50 = rrddim_add(st, "50%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist60 = rrddim_add(st, "60%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist70 = rrddim_add(st, "70%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist80 = rrddim_add(st, "80%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist90 = rrddim_add(st, "90%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_hist100 = rrddim_add(st, "100%", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_ra_none = rrddim_add(st, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); } else rrdset_next(st); // ignore ra_size - if(ra_size) {}; - - rrddim_set(st, "10%", ra_hist10); - rrddim_set(st, "20%", ra_hist20); - rrddim_set(st, "30%", ra_hist30); - rrddim_set(st, "40%", ra_hist40); - rrddim_set(st, "50%", ra_hist50); - rrddim_set(st, "60%", ra_hist60); - rrddim_set(st, "70%", ra_hist70); - rrddim_set(st, "80%", ra_hist80); - rrddim_set(st, "90%", ra_hist90); - rrddim_set(st, "100%", ra_hist100); - rrddim_set(st, "misses", ra_none); + (void)ra_size; + + rrddim_set_by_pointer(st, rd_ra_hist10, ra_hist10); + rrddim_set_by_pointer(st, rd_ra_hist20, ra_hist20); + rrddim_set_by_pointer(st, rd_ra_hist30, ra_hist30); + rrddim_set_by_pointer(st, rd_ra_hist40, ra_hist40); + rrddim_set_by_pointer(st, rd_ra_hist50, ra_hist50); + rrddim_set_by_pointer(st, rd_ra_hist60, ra_hist60); + rrddim_set_by_pointer(st, rd_ra_hist70, ra_hist70); + rrddim_set_by_pointer(st, rd_ra_hist80, ra_hist80); + rrddim_set_by_pointer(st, rd_ra_hist90, ra_hist90); + rrddim_set_by_pointer(st, rd_ra_hist100,ra_hist100); + rrddim_set_by_pointer(st, rd_ra_none, ra_none); rrdset_done(st); } // -------------------------------------------------------------------- if(do_net == 2) { - st = rrdset_find_bytype_localhost("nfsd", "net"); - if(!st) { - st = rrdset_create_localhost("nfsd", "net", NULL, "network", NULL, "NFS Server Network Statistics" - , "packets/s", 5007, update_every, RRDSET_TYPE_STACKED); + static RRDSET *st = NULL; + static RRDDIM *rd_udp = NULL, + *rd_tcp = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "net" + , NULL + , "network" + , NULL + , "NFS Server Network Statistics" + , "packets/s" + , "proc" + , "net/rpc/nfsd" + , 5007 + , update_every + , RRDSET_TYPE_STACKED + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_udp = rrddim_add(st, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_tcp = rrddim_add(st, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); // ignore net_count, net_tcp_connections - if(net_count) {}; - if(net_tcp_connections) {}; + (void)net_count; + (void)net_tcp_connections; - rrddim_set(st, "udp", net_udp_count); - rrddim_set(st, "tcp", net_tcp_count); + rrddim_set_by_pointer(st, rd_udp, net_udp_count); + rrddim_set_by_pointer(st, rd_tcp, net_tcp_count); rrdset_done(st); } // -------------------------------------------------------------------- if(do_rpc == 2) { - st = rrdset_find_bytype_localhost("nfsd", "rpc"); - if(!st) { - st = rrdset_create_localhost("nfsd", "rpc", NULL, "rpc", NULL - , "NFS Server Remote Procedure Calls Statistics", "calls/s", 5008, update_every - , RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_calls = NULL, + *rd_bad_format = NULL, + *rd_bad_auth = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "rpc" + , NULL + , "rpc" + , NULL + , "NFS Server Remote Procedure Calls Statistics" + , "calls/s" + , "proc" + , "net/rpc/nfsd" + , 5008 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "calls", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "bad_format", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "bad_auth", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_calls = rrddim_add(st, "calls", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_bad_format = rrddim_add(st, "bad_format", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_bad_auth = rrddim_add(st, "bad_auth", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); // ignore rpc_bad_client - if(rpc_bad_client) {}; + (void)rpc_bad_client; - rrddim_set(st, "calls", rpc_calls); - rrddim_set(st, "bad_format", rpc_bad_format); - rrddim_set(st, "bad_auth", rpc_bad_auth); + rrddim_set_by_pointer(st, rd_calls, rpc_calls); + rrddim_set_by_pointer(st, rd_bad_format, rpc_bad_format); + rrddim_set_by_pointer(st, rd_bad_auth, rpc_bad_auth); rrdset_done(st); } // -------------------------------------------------------------------- if(do_proc2 == 2) { - unsigned int i; - st = rrdset_find_bytype_localhost("nfsd", "proc2"); - if(!st) { - st = rrdset_create_localhost("nfsd", "proc2", NULL, "nfsv2rpc", NULL, "NFS v2 Server Remote Procedure Calls" - , "calls/s", 5009, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; nfsd_proc2_values[i].present ; i++) - rrddim_add(st, nfsd_proc2_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "proc2" + , NULL + , "nfsv2rpc" + , NULL + , "NFS v2 Server Remote Procedure Calls" + , "calls/s" + , "proc" + , "net/rpc/nfsd" + , 5009 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st); - for(i = 0; nfsd_proc2_values[i].present ; i++) - rrddim_set(st, nfsd_proc2_values[i].name, nfsd_proc2_values[i].value); + size_t i; + for(i = 0; nfsd_proc2_values[i].present ; i++) { + if(unlikely(!nfsd_proc2_values[i].rd)) + nfsd_proc2_values[i].rd = rrddim_add(st, nfsd_proc2_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrddim_set_by_pointer(st, nfsd_proc2_values[i].rd, nfsd_proc2_values[i].value); + } rrdset_done(st); } @@ -722,19 +900,32 @@ int do_proc_net_rpc_nfsd(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_proc3 == 2) { - unsigned int i; - st = rrdset_find_bytype_localhost("nfsd", "proc3"); - if(!st) { - st = rrdset_create_localhost("nfsd", "proc3", NULL, "nfsv3rpc", NULL, "NFS v3 Server Remote Procedure Calls" - , "calls/s", 5010, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; nfsd_proc3_values[i].present ; i++) - rrddim_add(st, nfsd_proc3_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "proc3" + , NULL + , "nfsv3rpc" + , NULL + , "NFS v3 Server Remote Procedure Calls" + , "calls/s" + , "proc" + , "net/rpc/nfsd" + , 5010 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st); - for(i = 0; nfsd_proc3_values[i].present ; i++) - rrddim_set(st, nfsd_proc3_values[i].name, nfsd_proc3_values[i].value); + size_t i; + for(i = 0; nfsd_proc3_values[i].present ; i++) { + if(unlikely(!nfsd_proc3_values[i].rd)) + nfsd_proc3_values[i].rd = rrddim_add(st, nfsd_proc3_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrddim_set_by_pointer(st, nfsd_proc3_values[i].rd, nfsd_proc3_values[i].value); + } rrdset_done(st); } @@ -742,19 +933,32 @@ int do_proc_net_rpc_nfsd(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_proc4 == 2) { - unsigned int i; - st = rrdset_find_bytype_localhost("nfsd", "proc4"); - if(!st) { - st = rrdset_create_localhost("nfsd", "proc4", NULL, "nfsv4rpc", NULL, "NFS v4 Server Remote Procedure Calls" - , "calls/s", 5011, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; nfsd_proc4_values[i].present ; i++) - rrddim_add(st, nfsd_proc4_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "proc4" + , NULL + , "nfsv4rpc" + , NULL + , "NFS v4 Server Remote Procedure Calls" + , "calls/s" + , "proc" + , "net/rpc/nfsd" + , 5011 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st); - for(i = 0; nfsd_proc4_values[i].present ; i++) - rrddim_set(st, nfsd_proc4_values[i].name, nfsd_proc4_values[i].value); + size_t i; + for(i = 0; nfsd_proc4_values[i].present ; i++) { + if(unlikely(!nfsd_proc4_values[i].rd)) + nfsd_proc4_values[i].rd = rrddim_add(st, nfsd_proc4_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrddim_set_by_pointer(st, nfsd_proc4_values[i].rd, nfsd_proc4_values[i].value); + } rrdset_done(st); } @@ -762,19 +966,32 @@ int do_proc_net_rpc_nfsd(int update_every, usec_t dt) { // -------------------------------------------------------------------- if(do_proc4ops == 2) { - unsigned int i; - st = rrdset_find_bytype_localhost("nfsd", "proc4ops"); - if(!st) { - st = rrdset_create_localhost("nfsd", "proc4ops", NULL, "nfsv2ops", NULL, "NFS v4 Server Operations" - , "operations/s", 5012, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; nfsd4_ops_values[i].present ; i++) - rrddim_add(st, nfsd4_ops_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + if(unlikely(!st)) { + st = rrdset_create_localhost( + "nfsd" + , "proc4ops" + , NULL + , "nfsv2ops" + , NULL + , "NFS v4 Server Operations" + , "operations/s" + , "proc" + , "net/rpc/nfsd" + , 5012 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st); - for(i = 0; nfsd4_ops_values[i].present ; i++) - rrddim_set(st, nfsd4_ops_values[i].name, nfsd4_ops_values[i].value); + size_t i; + for(i = 0; nfsd4_ops_values[i].present ; i++) { + if(unlikely(!nfsd4_ops_values[i].rd)) + nfsd4_ops_values[i].rd = rrddim_add(st, nfsd4_ops_values[i].name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + rrddim_set_by_pointer(st, nfsd4_ops_values[i].rd, nfsd4_ops_values[i].value); + } rrdset_done(st); } diff --git a/src/proc_net_snmp.c b/src/proc_net_snmp.c index 817c964b4..fabfdf8c6 100644 --- a/src/proc_net_snmp.c +++ b/src/proc_net_snmp.c @@ -2,197 +2,88 @@ #define RRD_TYPE_NET_SNMP "ipv4" -struct netstat_columns { - char *name; - uint32_t hash; - unsigned long long value; - int multiplier; // not needed everywhere - char *label; // not needed everywhere -}; - -static struct netstat_columns ip_data[] = { -// { "Forwarding", 0, 0, 1, NULL }, -// { "DefaultTTL", 0, 0, 1, NULL }, - { "InReceives", 0, 0, 1, NULL }, - { "InHdrErrors", 0, 0, 1, NULL }, - { "InAddrErrors", 0, 0, 1, NULL }, - { "ForwDatagrams", 0, 0, 1, NULL }, - { "InUnknownProtos", 0, 0, 1, NULL }, - { "InDiscards", 0, 0, 1, NULL }, - { "InDelivers", 0, 0, 1, NULL }, - { "OutRequests", 0, 0, 1, NULL }, - { "OutDiscards", 0, 0, 1, NULL }, - { "OutNoRoutes", 0, 0, 1, NULL }, -// { "ReasmTimeout", 0, 0, 1, NULL }, - { "ReasmReqds", 0, 0, 1, NULL }, - { "ReasmOKs", 0, 0, 1, NULL }, - { "ReasmFails", 0, 0, 1, NULL }, - { "FragOKs", 0, 0, 1, NULL }, - { "FragFails", 0, 0, 1, NULL }, - { "FragCreates", 0, 0, 1, NULL }, - { NULL, 0, 0, 0, NULL } -}; - -static struct netstat_columns icmp_data[] = { - { "InMsgs", 0, 0, 1, NULL }, - { "OutMsgs", 0, 0, -1, NULL }, - { "InErrors", 0, 0, 1, NULL }, - { "OutErrors", 0, 0, -1, NULL }, - { "InCsumErrors", 0, 0, 1, NULL }, - - // all these are available in icmpmsg -// { "InDestUnreachs", 0, 0, 1, NULL }, -// { "OutDestUnreachs", 0, 0, -1, NULL }, -// { "InTimeExcds", 0, 0, 1, NULL }, -// { "OutTimeExcds", 0, 0, -1, NULL }, -// { "InParmProbs", 0, 0, 1, NULL }, -// { "OutParmProbs", 0, 0, -1, NULL }, -// { "InSrcQuenchs", 0, 0, 1, NULL }, -// { "OutSrcQuenchs", 0, 0, -1, NULL }, -// { "InRedirects", 0, 0, 1, NULL }, -// { "OutRedirects", 0, 0, -1, NULL }, -// { "InEchos", 0, 0, 1, NULL }, -// { "OutEchos", 0, 0, -1, NULL }, -// { "InEchoReps", 0, 0, 1, NULL }, -// { "OutEchoReps", 0, 0, -1, NULL }, -// { "InTimestamps", 0, 0, 1, NULL }, -// { "OutTimestamps", 0, 0, -1, NULL }, -// { "InTimestampReps", 0, 0, 1, NULL }, -// { "OutTimestampReps", 0, 0, -1, NULL }, -// { "InAddrMasks", 0, 0, 1, NULL }, -// { "OutAddrMasks", 0, 0, -1, NULL }, -// { "InAddrMaskReps", 0, 0, 1, NULL }, -// { "OutAddrMaskReps", 0, 0, -1, NULL }, - - { NULL, 0, 0, 0, NULL } -}; - -static struct netstat_columns icmpmsg_data[] = { - { "InType0", 0, 0, 1, "InEchoReps" }, - { "OutType0", 0, 0, -1, "OutEchoReps" }, -// { "InType1", 0, 0, 1, NULL }, // unassigned -// { "OutType1", 0, 0, -1, NULL }, // unassigned -// { "InType2", 0, 0, 1, NULL }, // unassigned -// { "OutType2", 0, 0, -1, NULL }, // unassigned - { "InType3", 0, 0, 1, "InDestUnreachs" }, - { "OutType3", 0, 0, -1, "OutDestUnreachs" }, -// { "InType4", 0, 0, 1, "InSrcQuenchs" }, // deprecated -// { "OutType4", 0, 0, -1, "OutSrcQuenchs" }, // deprecated - { "InType5", 0, 0, 1, "InRedirects" }, - { "OutType5", 0, 0, -1, "OutRedirects" }, -// { "InType6", 0, 0, 1, "InAlterHostAddr" }, // deprecated -// { "OutType6", 0, 0, -1, "OutAlterHostAddr" }, // deprecated -// { "InType7", 0, 0, 1, NULL }, // unassigned -// { "OutType7", 0, 0, -1, NULL }, // unassigned - { "InType8", 0, 0, 1, "InEchos" }, - { "OutType8", 0, 0, -1, "OutEchos" }, - { "InType9", 0, 0, 1, "InRouterAdvert" }, - { "OutType9", 0, 0, -1, "OutRouterAdvert" }, - { "InType10", 0, 0, 1, "InRouterSelect" }, - { "OutType10", 0, 0, -1, "OutRouterSelect" }, - { "InType11", 0, 0, 1, "InTimeExcds" }, - { "OutType11", 0, 0, -1, "OutTimeExcds" }, - { "InType12", 0, 0, 1, "InParmProbs" }, - { "OutType12", 0, 0, -1, "OutParmProbs" }, - { "InType13", 0, 0, 1, "InTimestamps" }, - { "OutType13", 0, 0, -1, "OutTimestamps" }, - { "InType14", 0, 0, 1, "InTimestampReps" }, - { "OutType14", 0, 0, -1, "OutTimestampReps" }, -// { "InType15", 0, 0, 1, "InInfos" }, // deprecated -// { "OutType15", 0, 0, -1, "OutInfos" }, // deprecated -// { "InType16", 0, 0, 1, "InInfoReps" }, // deprecated -// { "OutType16", 0, 0, -1, "OutInfoReps" }, // deprecated -// { "InType17", 0, 0, 1, "InAddrMasks" }, // deprecated -// { "OutType17", 0, 0, -1, "OutAddrMasks" }, // deprecated -// { "InType18", 0, 0, 1, "InAddrMaskReps" }, // deprecated -// { "OutType18", 0, 0, -1, "OutAddrMaskReps" }, // deprecated -// { "InType30", 0, 0, 1, "InTraceroute" }, // deprecated -// { "OutType30", 0, 0, -1, "OutTraceroute" }, // deprecated - { NULL, 0, 0, 0, NULL } -}; - -static struct netstat_columns tcp_data[] = { -// { "RtoAlgorithm", 0, 0, 1, NULL }, -// { "RtoMin", 0, 0, 1, NULL }, -// { "RtoMax", 0, 0, 1, NULL }, -// { "MaxConn", 0, 0, 1, NULL }, - { "ActiveOpens", 0, 0, 1, NULL }, - { "PassiveOpens", 0, 0, 1, NULL }, - { "AttemptFails", 0, 0, 1, NULL }, - { "EstabResets", 0, 0, 1, NULL }, - { "CurrEstab", 0, 0, 1, NULL }, - { "InSegs", 0, 0, 1, NULL }, - { "OutSegs", 0, 0, 1, NULL }, - { "RetransSegs", 0, 0, 1, NULL }, - { "InErrs", 0, 0, 1, NULL }, - { "OutRsts", 0, 0, 1, NULL }, - { "InCsumErrors", 0, 0, 1, NULL }, - { NULL, 0, 0, 0, NULL } -}; - -static struct netstat_columns udp_data[] = { - { "InDatagrams", 0, 0, 1, NULL }, - { "NoPorts", 0, 0, 1, NULL }, - { "InErrors", 0, 0, 1, NULL }, - { "OutDatagrams", 0, 0, 1, NULL }, - { "RcvbufErrors", 0, 0, 1, NULL }, - { "SndbufErrors", 0, 0, 1, NULL }, - { "InCsumErrors", 0, 0, 1, NULL }, - { "IgnoredMulti", 0, 0, 1, NULL }, - { NULL, 0, 0, 0, NULL } -}; - -static struct netstat_columns udplite_data[] = { - { "InDatagrams", 0, 0, 1, NULL }, - { "NoPorts", 0, 0, 1, NULL }, - { "InErrors", 0, 0, 1, NULL }, - { "OutDatagrams", 0, 0, 1, NULL }, - { "RcvbufErrors", 0, 0, 1, NULL }, - { "SndbufErrors", 0, 0, 1, NULL }, - { "InCsumErrors", 0, 0, 1, NULL }, - { "IgnoredMulti", 0, 0, 1, NULL }, - { NULL, 0, 0, 0, NULL } -}; - -static void hash_array(struct netstat_columns *nc) { - int i; - - for(i = 0; nc[i].name ;i++) - nc[i].hash = simple_hash(nc[i].name); -} - -static unsigned long long *netstat_columns_find(struct netstat_columns *nc, const char *name) { - uint32_t i, hash = simple_hash(name); - - for(i = 0; nc[i].name ;i++) - if(unlikely(nc[i].hash == hash && !strcmp(nc[i].name, name))) - return &nc[i].value; - - fatal("Cannot find key '%s' in /proc/net/snmp internal array.", name); -} - -static void parse_line_pair(procfile *ff, struct netstat_columns *nc, size_t header_line, size_t values_line) { - size_t hwords = procfile_linewords(ff, header_line); - size_t vwords = procfile_linewords(ff, values_line); - size_t w, i; - - if(unlikely(vwords > hwords)) { - error("File /proc/net/snmp on header line %zu has %zu words, but on value line %zu has %zu words.", header_line, hwords, values_line, vwords); - vwords = hwords; - } - - for(w = 1; w < vwords ;w++) { - char *key = procfile_lineword(ff, header_line, w); - uint32_t hash = simple_hash(key); - - for(i = 0 ; nc[i].name ;i++) { - if(unlikely(hash == nc[i].hash && !strcmp(key, nc[i].name))) { - nc[i].value = str2ull(procfile_lineword(ff, values_line, w)); - break; - } - } - } -} +static struct proc_net_snmp { + // kernel_uint_t ip_Forwarding; + kernel_uint_t ip_DefaultTTL; + kernel_uint_t ip_InReceives; + kernel_uint_t ip_InHdrErrors; + kernel_uint_t ip_InAddrErrors; + kernel_uint_t ip_ForwDatagrams; + kernel_uint_t ip_InUnknownProtos; + kernel_uint_t ip_InDiscards; + kernel_uint_t ip_InDelivers; + kernel_uint_t ip_OutRequests; + kernel_uint_t ip_OutDiscards; + kernel_uint_t ip_OutNoRoutes; + kernel_uint_t ip_ReasmTimeout; + kernel_uint_t ip_ReasmReqds; + kernel_uint_t ip_ReasmOKs; + kernel_uint_t ip_ReasmFails; + kernel_uint_t ip_FragOKs; + kernel_uint_t ip_FragFails; + kernel_uint_t ip_FragCreates; + + kernel_uint_t icmp_InMsgs; + kernel_uint_t icmp_OutMsgs; + kernel_uint_t icmp_InErrors; + kernel_uint_t icmp_OutErrors; + kernel_uint_t icmp_InCsumErrors; + + kernel_uint_t icmpmsg_InEchoReps; + kernel_uint_t icmpmsg_OutEchoReps; + kernel_uint_t icmpmsg_InDestUnreachs; + kernel_uint_t icmpmsg_OutDestUnreachs; + kernel_uint_t icmpmsg_InRedirects; + kernel_uint_t icmpmsg_OutRedirects; + kernel_uint_t icmpmsg_InEchos; + kernel_uint_t icmpmsg_OutEchos; + kernel_uint_t icmpmsg_InRouterAdvert; + kernel_uint_t icmpmsg_OutRouterAdvert; + kernel_uint_t icmpmsg_InRouterSelect; + kernel_uint_t icmpmsg_OutRouterSelect; + kernel_uint_t icmpmsg_InTimeExcds; + kernel_uint_t icmpmsg_OutTimeExcds; + kernel_uint_t icmpmsg_InParmProbs; + kernel_uint_t icmpmsg_OutParmProbs; + kernel_uint_t icmpmsg_InTimestamps; + kernel_uint_t icmpmsg_OutTimestamps; + kernel_uint_t icmpmsg_InTimestampReps; + kernel_uint_t icmpmsg_OutTimestampReps; + + //kernel_uint_t tcp_RtoAlgorithm; + //kernel_uint_t tcp_RtoMin; + //kernel_uint_t tcp_RtoMax; + ssize_t tcp_MaxConn; + kernel_uint_t tcp_ActiveOpens; + kernel_uint_t tcp_PassiveOpens; + kernel_uint_t tcp_AttemptFails; + kernel_uint_t tcp_EstabResets; + kernel_uint_t tcp_CurrEstab; + kernel_uint_t tcp_InSegs; + kernel_uint_t tcp_OutSegs; + kernel_uint_t tcp_RetransSegs; + kernel_uint_t tcp_InErrs; + kernel_uint_t tcp_OutRsts; + kernel_uint_t tcp_InCsumErrors; + + kernel_uint_t udp_InDatagrams; + kernel_uint_t udp_NoPorts; + kernel_uint_t udp_InErrors; + kernel_uint_t udp_OutDatagrams; + kernel_uint_t udp_RcvbufErrors; + kernel_uint_t udp_SndbufErrors; + kernel_uint_t udp_InCsumErrors; + kernel_uint_t udp_IgnoredMulti; + + kernel_uint_t udplite_InDatagrams; + kernel_uint_t udplite_NoPorts; + kernel_uint_t udplite_InErrors; + kernel_uint_t udplite_OutDatagrams; + kernel_uint_t udplite_RcvbufErrors; + kernel_uint_t udplite_SndbufErrors; + kernel_uint_t udplite_InCsumErrors; + kernel_uint_t udplite_IgnoredMulti; +} snmp_root = { 0 }; int do_proc_net_snmp(int update_every, usec_t dt) { (void)dt; @@ -203,67 +94,17 @@ int do_proc_net_snmp(int update_every, usec_t dt) { do_udp_packets = -1, do_udp_errors = -1, do_icmp_packets = -1, do_icmpmsg = -1, do_udplite_packets = -1; static uint32_t hash_ip = 0, hash_icmp = 0, hash_tcp = 0, hash_udp = 0, hash_icmpmsg = 0, hash_udplite = 0; - //static unsigned long long *ip_Forwarding = NULL; - //static unsigned long long *ip_DefaultTTL = NULL; - static unsigned long long *ip_InReceives = NULL; - static unsigned long long *ip_InHdrErrors = NULL; - static unsigned long long *ip_InAddrErrors = NULL; - static unsigned long long *ip_ForwDatagrams = NULL; - static unsigned long long *ip_InUnknownProtos = NULL; - static unsigned long long *ip_InDiscards = NULL; - static unsigned long long *ip_InDelivers = NULL; - static unsigned long long *ip_OutRequests = NULL; - static unsigned long long *ip_OutDiscards = NULL; - static unsigned long long *ip_OutNoRoutes = NULL; - //static unsigned long long *ip_ReasmTimeout = NULL; - static unsigned long long *ip_ReasmReqds = NULL; - static unsigned long long *ip_ReasmOKs = NULL; - static unsigned long long *ip_ReasmFails = NULL; - static unsigned long long *ip_FragOKs = NULL; - static unsigned long long *ip_FragFails = NULL; - static unsigned long long *ip_FragCreates = NULL; - - static unsigned long long *icmp_InMsgs = NULL; - static unsigned long long *icmp_OutMsgs = NULL; - static unsigned long long *icmp_InErrors = NULL; - static unsigned long long *icmp_OutErrors = NULL; - static unsigned long long *icmp_InCsumErrors = NULL; - - //static unsigned long long *tcp_RtoAlgorithm = NULL; - //static unsigned long long *tcp_RtoMin = NULL; - //static unsigned long long *tcp_RtoMax = NULL; - //static unsigned long long *tcp_MaxConn = NULL; - static unsigned long long *tcp_ActiveOpens = NULL; - static unsigned long long *tcp_PassiveOpens = NULL; - static unsigned long long *tcp_AttemptFails = NULL; - static unsigned long long *tcp_EstabResets = NULL; - static unsigned long long *tcp_CurrEstab = NULL; - static unsigned long long *tcp_InSegs = NULL; - static unsigned long long *tcp_OutSegs = NULL; - static unsigned long long *tcp_RetransSegs = NULL; - static unsigned long long *tcp_InErrs = NULL; - static unsigned long long *tcp_OutRsts = NULL; - static unsigned long long *tcp_InCsumErrors = NULL; - - static unsigned long long *udp_InDatagrams = NULL; - static unsigned long long *udp_NoPorts = NULL; - static unsigned long long *udp_InErrors = NULL; - static unsigned long long *udp_OutDatagrams = NULL; - static unsigned long long *udp_RcvbufErrors = NULL; - static unsigned long long *udp_SndbufErrors = NULL; - static unsigned long long *udp_InCsumErrors = NULL; - static unsigned long long *udp_IgnoredMulti = NULL; - - static unsigned long long *udplite_InDatagrams = NULL; - static unsigned long long *udplite_NoPorts = NULL; - static unsigned long long *udplite_InErrors = NULL; - static unsigned long long *udplite_OutDatagrams = NULL; - static unsigned long long *udplite_RcvbufErrors = NULL; - static unsigned long long *udplite_SndbufErrors = NULL; - static unsigned long long *udplite_InCsumErrors = NULL; - static unsigned long long *udplite_IgnoredMulti = NULL; - - if(unlikely(do_ip_packets == -1)) { + static ARL_BASE *arl_ip = NULL, + *arl_icmp = NULL, + *arl_icmpmsg = NULL, + *arl_tcp = NULL, + *arl_udp = NULL, + *arl_udplite = NULL; + + static RRDVAR *tcp_max_connections_var = NULL; + static ssize_t last_max_connections = 0; + + if(unlikely(!arl_ip)) { do_ip_packets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 packets", 1); do_ip_fragsout = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 fragments sent", 1); do_ip_fragsin = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 fragments assembly", 1); @@ -285,72 +126,94 @@ int do_proc_net_snmp(int update_every, usec_t dt) { hash_icmpmsg = simple_hash("IcmpMsg"); hash_udplite = simple_hash("UdpLite"); - hash_array(ip_data); - hash_array(tcp_data); - hash_array(udp_data); - hash_array(icmp_data); - hash_array(icmpmsg_data); - hash_array(udplite_data); - - //ip_Forwarding = netstat_columns_find(ip_data, "Forwarding"); - //ip_DefaultTTL = netstat_columns_find(ip_data, "DefaultTTL"); - ip_InReceives = netstat_columns_find(ip_data, "InReceives"); - ip_InHdrErrors = netstat_columns_find(ip_data, "InHdrErrors"); - ip_InAddrErrors = netstat_columns_find(ip_data, "InAddrErrors"); - ip_ForwDatagrams = netstat_columns_find(ip_data, "ForwDatagrams"); - ip_InUnknownProtos = netstat_columns_find(ip_data, "InUnknownProtos"); - ip_InDiscards = netstat_columns_find(ip_data, "InDiscards"); - ip_InDelivers = netstat_columns_find(ip_data, "InDelivers"); - ip_OutRequests = netstat_columns_find(ip_data, "OutRequests"); - ip_OutDiscards = netstat_columns_find(ip_data, "OutDiscards"); - ip_OutNoRoutes = netstat_columns_find(ip_data, "OutNoRoutes"); - //ip_ReasmTimeout = netstat_columns_find(ip_data, "ReasmTimeout"); - ip_ReasmReqds = netstat_columns_find(ip_data, "ReasmReqds"); - ip_ReasmOKs = netstat_columns_find(ip_data, "ReasmOKs"); - ip_ReasmFails = netstat_columns_find(ip_data, "ReasmFails"); - ip_FragOKs = netstat_columns_find(ip_data, "FragOKs"); - ip_FragFails = netstat_columns_find(ip_data, "FragFails"); - ip_FragCreates = netstat_columns_find(ip_data, "FragCreates"); - - icmp_InMsgs = netstat_columns_find(icmp_data, "InMsgs"); - icmp_OutMsgs = netstat_columns_find(icmp_data, "OutMsgs"); - icmp_InErrors = netstat_columns_find(icmp_data, "InErrors"); - icmp_OutErrors = netstat_columns_find(icmp_data, "OutErrors"); - icmp_InCsumErrors = netstat_columns_find(icmp_data, "InCsumErrors"); - - //tcp_RtoAlgorithm = netstat_columns_find(tcp_data, "RtoAlgorithm"); - //tcp_RtoMin = netstat_columns_find(tcp_data, "RtoMin"); - //tcp_RtoMax = netstat_columns_find(tcp_data, "RtoMax"); - //tcp_MaxConn = netstat_columns_find(tcp_data, "MaxConn"); - tcp_ActiveOpens = netstat_columns_find(tcp_data, "ActiveOpens"); - tcp_PassiveOpens = netstat_columns_find(tcp_data, "PassiveOpens"); - tcp_AttemptFails = netstat_columns_find(tcp_data, "AttemptFails"); - tcp_EstabResets = netstat_columns_find(tcp_data, "EstabResets"); - tcp_CurrEstab = netstat_columns_find(tcp_data, "CurrEstab"); - tcp_InSegs = netstat_columns_find(tcp_data, "InSegs"); - tcp_OutSegs = netstat_columns_find(tcp_data, "OutSegs"); - tcp_RetransSegs = netstat_columns_find(tcp_data, "RetransSegs"); - tcp_InErrs = netstat_columns_find(tcp_data, "InErrs"); - tcp_OutRsts = netstat_columns_find(tcp_data, "OutRsts"); - tcp_InCsumErrors = netstat_columns_find(tcp_data, "InCsumErrors"); - - udp_InDatagrams = netstat_columns_find(udp_data, "InDatagrams"); - udp_NoPorts = netstat_columns_find(udp_data, "NoPorts"); - udp_InErrors = netstat_columns_find(udp_data, "InErrors"); - udp_OutDatagrams = netstat_columns_find(udp_data, "OutDatagrams"); - udp_RcvbufErrors = netstat_columns_find(udp_data, "RcvbufErrors"); - udp_SndbufErrors = netstat_columns_find(udp_data, "SndbufErrors"); - udp_InCsumErrors = netstat_columns_find(udp_data, "InCsumErrors"); - udp_IgnoredMulti = netstat_columns_find(udp_data, "IgnoredMulti"); - - udplite_InDatagrams = netstat_columns_find(udplite_data, "InDatagrams"); - udplite_NoPorts = netstat_columns_find(udplite_data, "NoPorts"); - udplite_InErrors = netstat_columns_find(udplite_data, "InErrors"); - udplite_OutDatagrams = netstat_columns_find(udplite_data, "OutDatagrams"); - udplite_RcvbufErrors = netstat_columns_find(udplite_data, "RcvbufErrors"); - udplite_SndbufErrors = netstat_columns_find(udplite_data, "SndbufErrors"); - udplite_InCsumErrors = netstat_columns_find(udplite_data, "InCsumErrors"); - udplite_IgnoredMulti = netstat_columns_find(udplite_data, "IgnoredMulti"); + arl_ip = arl_create("snmp/Ip", arl_callback_str2kernel_uint_t, 60); + // arl_expect(arl_ip, "Forwarding", &snmp_root.ip_Forwarding); + arl_expect(arl_ip, "DefaultTTL", &snmp_root.ip_DefaultTTL); + arl_expect(arl_ip, "InReceives", &snmp_root.ip_InReceives); + arl_expect(arl_ip, "InHdrErrors", &snmp_root.ip_InHdrErrors); + arl_expect(arl_ip, "InAddrErrors", &snmp_root.ip_InAddrErrors); + arl_expect(arl_ip, "ForwDatagrams", &snmp_root.ip_ForwDatagrams); + arl_expect(arl_ip, "InUnknownProtos", &snmp_root.ip_InUnknownProtos); + arl_expect(arl_ip, "InDiscards", &snmp_root.ip_InDiscards); + arl_expect(arl_ip, "InDelivers", &snmp_root.ip_InDelivers); + arl_expect(arl_ip, "OutRequests", &snmp_root.ip_OutRequests); + arl_expect(arl_ip, "OutDiscards", &snmp_root.ip_OutDiscards); + arl_expect(arl_ip, "OutNoRoutes", &snmp_root.ip_OutNoRoutes); + arl_expect(arl_ip, "ReasmTimeout", &snmp_root.ip_ReasmTimeout); + arl_expect(arl_ip, "ReasmReqds", &snmp_root.ip_ReasmReqds); + arl_expect(arl_ip, "ReasmOKs", &snmp_root.ip_ReasmOKs); + arl_expect(arl_ip, "ReasmFails", &snmp_root.ip_ReasmFails); + arl_expect(arl_ip, "FragOKs", &snmp_root.ip_FragOKs); + arl_expect(arl_ip, "FragFails", &snmp_root.ip_FragFails); + arl_expect(arl_ip, "FragCreates", &snmp_root.ip_FragCreates); + + arl_icmp = arl_create("snmp/Icmp", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_icmp, "InMsgs", &snmp_root.icmp_InMsgs); + arl_expect(arl_icmp, "OutMsgs", &snmp_root.icmp_OutMsgs); + arl_expect(arl_icmp, "InErrors", &snmp_root.icmp_InErrors); + arl_expect(arl_icmp, "OutErrors", &snmp_root.icmp_OutErrors); + arl_expect(arl_icmp, "InCsumErrors", &snmp_root.icmp_InCsumErrors); + + arl_icmpmsg = arl_create("snmp/Icmpmsg", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_icmpmsg, "InType0", &snmp_root.icmpmsg_InEchoReps); + arl_expect(arl_icmpmsg, "OutType0", &snmp_root.icmpmsg_OutEchoReps); + arl_expect(arl_icmpmsg, "InType3", &snmp_root.icmpmsg_InDestUnreachs); + arl_expect(arl_icmpmsg, "OutType3", &snmp_root.icmpmsg_OutDestUnreachs); + arl_expect(arl_icmpmsg, "InType5", &snmp_root.icmpmsg_InRedirects); + arl_expect(arl_icmpmsg, "OutType5", &snmp_root.icmpmsg_OutRedirects); + arl_expect(arl_icmpmsg, "InType8", &snmp_root.icmpmsg_InEchos); + arl_expect(arl_icmpmsg, "OutType8", &snmp_root.icmpmsg_OutEchos); + arl_expect(arl_icmpmsg, "InType9", &snmp_root.icmpmsg_InRouterAdvert); + arl_expect(arl_icmpmsg, "OutType9", &snmp_root.icmpmsg_OutRouterAdvert); + arl_expect(arl_icmpmsg, "InType10", &snmp_root.icmpmsg_InRouterSelect); + arl_expect(arl_icmpmsg, "OutType10", &snmp_root.icmpmsg_OutRouterSelect); + arl_expect(arl_icmpmsg, "InType11", &snmp_root.icmpmsg_InTimeExcds); + arl_expect(arl_icmpmsg, "OutType11", &snmp_root.icmpmsg_OutTimeExcds); + arl_expect(arl_icmpmsg, "InType12", &snmp_root.icmpmsg_InParmProbs); + arl_expect(arl_icmpmsg, "OutType12", &snmp_root.icmpmsg_OutParmProbs); + arl_expect(arl_icmpmsg, "InType13", &snmp_root.icmpmsg_InTimestamps); + arl_expect(arl_icmpmsg, "OutType13", &snmp_root.icmpmsg_OutTimestamps); + arl_expect(arl_icmpmsg, "InType14", &snmp_root.icmpmsg_InTimestampReps); + arl_expect(arl_icmpmsg, "OutType14", &snmp_root.icmpmsg_OutTimestampReps); + + arl_tcp = arl_create("snmp/Tcp", arl_callback_str2kernel_uint_t, 60); + // arl_expect(arl_tcp, "RtoAlgorithm", &snmp_root.tcp_RtoAlgorithm); + // arl_expect(arl_tcp, "RtoMin", &snmp_root.tcp_RtoMin); + // arl_expect(arl_tcp, "RtoMax", &snmp_root.tcp_RtoMax); + arl_expect(arl_tcp, "MaxConn", &snmp_root.tcp_MaxConn); + arl_expect(arl_tcp, "ActiveOpens", &snmp_root.tcp_ActiveOpens); + arl_expect(arl_tcp, "PassiveOpens", &snmp_root.tcp_PassiveOpens); + arl_expect(arl_tcp, "AttemptFails", &snmp_root.tcp_AttemptFails); + arl_expect(arl_tcp, "EstabResets", &snmp_root.tcp_EstabResets); + arl_expect(arl_tcp, "CurrEstab", &snmp_root.tcp_CurrEstab); + arl_expect(arl_tcp, "InSegs", &snmp_root.tcp_InSegs); + arl_expect(arl_tcp, "OutSegs", &snmp_root.tcp_OutSegs); + arl_expect(arl_tcp, "RetransSegs", &snmp_root.tcp_RetransSegs); + arl_expect(arl_tcp, "InErrs", &snmp_root.tcp_InErrs); + arl_expect(arl_tcp, "OutRsts", &snmp_root.tcp_OutRsts); + arl_expect(arl_tcp, "InCsumErrors", &snmp_root.tcp_InCsumErrors); + + arl_udp = arl_create("snmp/Udp", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_udp, "InDatagrams", &snmp_root.udp_InDatagrams); + arl_expect(arl_udp, "NoPorts", &snmp_root.udp_NoPorts); + arl_expect(arl_udp, "InErrors", &snmp_root.udp_InErrors); + arl_expect(arl_udp, "OutDatagrams", &snmp_root.udp_OutDatagrams); + arl_expect(arl_udp, "RcvbufErrors", &snmp_root.udp_RcvbufErrors); + arl_expect(arl_udp, "SndbufErrors", &snmp_root.udp_SndbufErrors); + arl_expect(arl_udp, "InCsumErrors", &snmp_root.udp_InCsumErrors); + arl_expect(arl_udp, "IgnoredMulti", &snmp_root.udp_IgnoredMulti); + + arl_udplite = arl_create("snmp/Udplite", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_udplite, "InDatagrams", &snmp_root.udplite_InDatagrams); + arl_expect(arl_udplite, "NoPorts", &snmp_root.udplite_NoPorts); + arl_expect(arl_udplite, "InErrors", &snmp_root.udplite_InErrors); + arl_expect(arl_udplite, "OutDatagrams", &snmp_root.udplite_OutDatagrams); + arl_expect(arl_udplite, "RcvbufErrors", &snmp_root.udplite_RcvbufErrors); + arl_expect(arl_udplite, "SndbufErrors", &snmp_root.udplite_SndbufErrors); + arl_expect(arl_udplite, "InCsumErrors", &snmp_root.udplite_InCsumErrors); + arl_expect(arl_udplite, "IgnoredMulti", &snmp_root.udplite_IgnoredMulti); + + tcp_max_connections_var = rrdvar_custom_host_variable_create(localhost, "tcp_max_connections"); } if(unlikely(!ff)) { @@ -364,9 +227,7 @@ int do_proc_net_snmp(int update_every, usec_t dt) { if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time size_t lines = procfile_lines(ff), l; - size_t words; - - RRDSET *st; + size_t words, w; for(l = 0; l < lines ;l++) { char *key = procfile_lineword(ff, l, 0); @@ -386,101 +247,170 @@ int do_proc_net_snmp(int update_every, usec_t dt) { continue; } - // see also http://net-snmp.sourceforge.net/docs/mibs/ip.html - parse_line_pair(ff, ip_data, h, l); + arl_begin(arl_ip); + for(w = 1; w < words ; w++) { + if (unlikely(arl_check(arl_ip, procfile_lineword(ff, h, w), procfile_lineword(ff, l, w)) != 0)) + break; + } // -------------------------------------------------------------------- if(do_ip_packets) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".packets"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "packets", NULL, "packets", NULL, "IPv4 Packets" - , "packets/s", 3000, update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InDelivers", "delivered", 1, 1, RRD_ALGORITHM_INCREMENTAL); + static RRDSET *st = NULL; + static RRDDIM *rd_InReceives = NULL, + *rd_OutRequests = NULL, + *rd_ForwDatagrams = NULL, + *rd_InDelivers = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "packets" + , NULL + , "packets" + , NULL + , "IPv4 Packets" + , "packets/s" + , "proc" + , "net/snmp" + , 2450 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InReceives = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutRequests = rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ForwDatagrams = rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDelivers = rrddim_add(st, "InDelivers", "delivered", 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "OutRequests", *ip_OutRequests); - rrddim_set(st, "InReceives", *ip_InReceives); - rrddim_set(st, "ForwDatagrams", *ip_ForwDatagrams); - rrddim_set(st, "InDelivers", *ip_InDelivers); + rrddim_set_by_pointer(st, rd_OutRequests, (collected_number)snmp_root.ip_OutRequests); + rrddim_set_by_pointer(st, rd_InReceives, (collected_number)snmp_root.ip_InReceives); + rrddim_set_by_pointer(st, rd_ForwDatagrams, (collected_number)snmp_root.ip_ForwDatagrams); + rrddim_set_by_pointer(st, rd_InDelivers, (collected_number)snmp_root.ip_InDelivers); rrdset_done(st); } // -------------------------------------------------------------------- if(do_ip_fragsout) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".fragsout"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "fragsout", NULL, "fragments", NULL - , "IPv4 Fragments Sent", "packets/s", 3010, update_every - , RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_FragOKs = NULL, + *rd_FragFails = NULL, + *rd_FragCreates = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "fragsout" + , NULL + , "fragments" + , NULL + , "IPv4 Fragments Sent" + , "packets/s" + , "proc" + , "net/snmp" + , 3020 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "FragFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "FragCreates", "created", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_FragOKs = rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_FragFails = rrddim_add(st, "FragFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_FragCreates = rrddim_add(st, "FragCreates", "created", 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "FragOKs", *ip_FragOKs); - rrddim_set(st, "FragFails", *ip_FragFails); - rrddim_set(st, "FragCreates", *ip_FragCreates); + rrddim_set_by_pointer(st, rd_FragOKs, (collected_number)snmp_root.ip_FragOKs); + rrddim_set_by_pointer(st, rd_FragFails, (collected_number)snmp_root.ip_FragFails); + rrddim_set_by_pointer(st, rd_FragCreates, (collected_number)snmp_root.ip_FragCreates); rrdset_done(st); } // -------------------------------------------------------------------- if(do_ip_fragsin) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".fragsin"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "fragsin", NULL, "fragments", NULL - , "IPv4 Fragments Reassembly", "packets/s", 3011, update_every - , RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_ReasmOKs = NULL, + *rd_ReasmFails = NULL, + *rd_ReasmReqds = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "fragsin" + , NULL + , "fragments" + , NULL + , "IPv4 Fragments Reassembly" + , "packets/s" + , "proc" + , "net/snmp" + , 3030 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "ReasmFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "ReasmReqds", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ReasmOKs = rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ReasmFails = rrddim_add(st, "ReasmFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ReasmReqds = rrddim_add(st, "ReasmReqds", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "ReasmOKs", *ip_ReasmOKs); - rrddim_set(st, "ReasmFails", *ip_ReasmFails); - rrddim_set(st, "ReasmReqds", *ip_ReasmReqds); + rrddim_set_by_pointer(st, rd_ReasmOKs, (collected_number)snmp_root.ip_ReasmOKs); + rrddim_set_by_pointer(st, rd_ReasmFails, (collected_number)snmp_root.ip_ReasmFails); + rrddim_set_by_pointer(st, rd_ReasmReqds, (collected_number)snmp_root.ip_ReasmReqds); rrdset_done(st); } // -------------------------------------------------------------------- if(do_ip_errors) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".errors"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "errors", NULL, "errors", NULL, "IPv4 Errors" - , "packets/s", 3002, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InDiscards = NULL, + *rd_OutDiscards = NULL, + *rd_InHdrErrors = NULL, + *rd_OutNoRoutes = NULL, + *rd_InAddrErrors = NULL, + *rd_InUnknownProtos = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "errors" + , NULL + , "errors" + , NULL + , "IPv4 Errors" + , "packets/s" + , "proc" + , "net/snmp" + , 2470 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDiscards = rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDiscards = rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InHdrErrors = rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutNoRoutes = rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InAddrErrors = rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InUnknownProtos = rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InDiscards", *ip_InDiscards); - rrddim_set(st, "OutDiscards", *ip_OutDiscards); - rrddim_set(st, "InHdrErrors", *ip_InHdrErrors); - rrddim_set(st, "InAddrErrors", *ip_InAddrErrors); - rrddim_set(st, "InUnknownProtos", *ip_InUnknownProtos); - rrddim_set(st, "OutNoRoutes", *ip_OutNoRoutes); + rrddim_set_by_pointer(st, rd_InDiscards, (collected_number)snmp_root.ip_InDiscards); + rrddim_set_by_pointer(st, rd_OutDiscards, (collected_number)snmp_root.ip_OutDiscards); + rrddim_set_by_pointer(st, rd_InHdrErrors, (collected_number)snmp_root.ip_InHdrErrors); + rrddim_set_by_pointer(st, rd_InAddrErrors, (collected_number)snmp_root.ip_InAddrErrors); + rrddim_set_by_pointer(st, rd_InUnknownProtos, (collected_number)snmp_root.ip_InUnknownProtos); + rrddim_set_by_pointer(st, rd_OutNoRoutes, (collected_number)snmp_root.ip_OutNoRoutes); rrdset_done(st); } } @@ -498,43 +428,81 @@ int do_proc_net_snmp(int update_every, usec_t dt) { continue; } - parse_line_pair(ff, icmp_data, h, l); + arl_begin(arl_icmp); + for(w = 1; w < words ; w++) { + if (unlikely(arl_check(arl_icmp, procfile_lineword(ff, h, w), procfile_lineword(ff, l, w)) != 0)) + break; + } // -------------------------------------------------------------------- if(do_icmp_packets) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".icmp"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "icmp", NULL, "icmp", NULL, "IPv4 ICMP Packets" - , "packets/s", 2602, update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + { + static RRDSET *st_packets = NULL; + static RRDDIM *rd_InMsgs = NULL, + *rd_OutMsgs = NULL; + + if(unlikely(!st_packets)) { + st_packets = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "icmp" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Packets" + , "packets/s" + , "proc" + , "net/snmp" + , 2602 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InMsgs = rrddim_add(st_packets, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutMsgs = rrddim_add(st_packets, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st_packets); + + rrddim_set_by_pointer(st_packets, rd_InMsgs, (collected_number)snmp_root.icmp_InMsgs); + rrddim_set_by_pointer(st_packets, rd_OutMsgs, (collected_number)snmp_root.icmp_OutMsgs); + + rrdset_done(st_packets); } - else rrdset_next(st); - - rrddim_set(st, "InMsgs", *icmp_InMsgs); - rrddim_set(st, "OutMsgs", *icmp_OutMsgs); - - rrdset_done(st); - - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".icmp_errors"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "icmp_errors", NULL, "icmp", NULL - , "IPv4 ICMP Errors", "packets/s", 2603, update_every - , RRDSET_TYPE_LINE); - rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + { + static RRDSET *st_errors = NULL; + static RRDDIM *rd_InErrors = NULL, + *rd_OutErrors = NULL, + *rd_InCsumErrors = NULL; + + if(unlikely(!st_errors)) { + st_errors = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "icmp_errors" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Errors" + , "packets/s" + , "proc" + , "net/snmp" + , 2603 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InErrors = rrddim_add(st_errors, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutErrors = rrddim_add(st_errors, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCsumErrors = rrddim_add(st_errors, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st_errors); + + rrddim_set_by_pointer(st_errors, rd_InErrors, (collected_number)snmp_root.icmp_InErrors); + rrddim_set_by_pointer(st_errors, rd_OutErrors, (collected_number)snmp_root.icmp_OutErrors); + rrddim_set_by_pointer(st_errors, rd_InCsumErrors, (collected_number)snmp_root.icmp_InCsumErrors); + + rrdset_done(st_errors); } - else rrdset_next(st); - - rrddim_set(st, "InErrors", *icmp_InErrors); - rrddim_set(st, "OutErrors", *icmp_OutErrors); - rrddim_set(st, "InCsumErrors", *icmp_InCsumErrors); - - rrdset_done(st); } } else if(unlikely(hash == hash_icmpmsg && strcmp(key, "IcmpMsg") == 0)) { @@ -545,25 +513,102 @@ int do_proc_net_snmp(int update_every, usec_t dt) { break; } - parse_line_pair(ff, icmpmsg_data, h, l); + words = procfile_linewords(ff, l); + if(words < 3) { + error("Cannot read /proc/net/snmp IcmpMsg line. Expected 3+ params, read %zu.", words); + continue; + } + + arl_begin(arl_icmpmsg); + for(w = 1; w < words ; w++) { + if (unlikely(arl_check(arl_icmpmsg, procfile_lineword(ff, h, w), procfile_lineword(ff, l, w)) != 0)) + break; + } // -------------------------------------------------------------------- if(do_icmpmsg) { - int i; - - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".icmpmsg"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "icmpmsg", NULL, "icmp", NULL, "IPv4 ICMP Messages" - , "packets/s", 2604, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InEchoReps = NULL, + *rd_OutEchoReps = NULL, + *rd_InDestUnreachs = NULL, + *rd_OutDestUnreachs = NULL, + *rd_InRedirects = NULL, + *rd_OutRedirects = NULL, + *rd_InEchos = NULL, + *rd_OutEchos = NULL, + *rd_InRouterAdvert = NULL, + *rd_OutRouterAdvert = NULL, + *rd_InRouterSelect = NULL, + *rd_OutRouterSelect = NULL, + *rd_InTimeExcds = NULL, + *rd_OutTimeExcds = NULL, + *rd_InParmProbs = NULL, + *rd_OutParmProbs = NULL, + *rd_InTimestamps = NULL, + *rd_OutTimestamps = NULL, + *rd_InTimestampReps = NULL, + *rd_OutTimestampReps = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "icmpmsg" + , NULL + , "icmp" + , NULL + , "IPv4 ICMP Messages" + , "packets/s" + , "proc" + , "net/snmp" + , 2604 + , update_every + , RRDSET_TYPE_LINE + ); - for(i = 0; icmpmsg_data[i].name ;i++) - rrddim_add(st, icmpmsg_data[i].name, icmpmsg_data[i].label, icmpmsg_data[i].multiplier, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InEchoReps = rrddim_add(st, "InType0", "InEchoReps", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutEchoReps = rrddim_add(st, "OutType0", "OutEchoReps", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDestUnreachs = rrddim_add(st, "InType3", "InDestUnreachs", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDestUnreachs = rrddim_add(st, "OutType3", "OutDestUnreachs", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InRedirects = rrddim_add(st, "InType5", "InRedirects", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutRedirects = rrddim_add(st, "OutType5", "OutRedirects", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InEchos = rrddim_add(st, "InType8", "InEchos", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutEchos = rrddim_add(st, "OutType8", "OutEchos", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InRouterAdvert = rrddim_add(st, "InType9", "InRouterAdvert", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutRouterAdvert = rrddim_add(st, "OutType9", "OutRouterAdvert", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InRouterSelect = rrddim_add(st, "InType10", "InRouterSelect", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutRouterSelect = rrddim_add(st, "OutType10", "OutRouterSelect", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InTimeExcds = rrddim_add(st, "InType11", "InTimeExcds", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutTimeExcds = rrddim_add(st, "OutType11", "OutTimeExcds", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InParmProbs = rrddim_add(st, "InType12", "InParmProbs", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutParmProbs = rrddim_add(st, "OutType12", "OutParmProbs", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InTimestamps = rrddim_add(st, "InType13", "InTimestamps", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutTimestamps = rrddim_add(st, "OutType13", "OutTimestamps", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InTimestampReps = rrddim_add(st, "InType14", "InTimestampReps", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutTimestampReps = rrddim_add(st, "OutType14", "OutTimestampReps", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - for(i = 0; icmpmsg_data[i].name ;i++) - rrddim_set(st, icmpmsg_data[i].name, icmpmsg_data[i].value); + rrddim_set_by_pointer(st, rd_InEchoReps, (collected_number)snmp_root.icmpmsg_InEchoReps); + rrddim_set_by_pointer(st, rd_OutEchoReps, (collected_number)snmp_root.icmpmsg_OutEchoReps); + rrddim_set_by_pointer(st, rd_InDestUnreachs, (collected_number)snmp_root.icmpmsg_InDestUnreachs); + rrddim_set_by_pointer(st, rd_OutDestUnreachs, (collected_number)snmp_root.icmpmsg_OutDestUnreachs); + rrddim_set_by_pointer(st, rd_InRedirects, (collected_number)snmp_root.icmpmsg_InRedirects); + rrddim_set_by_pointer(st, rd_OutRedirects, (collected_number)snmp_root.icmpmsg_OutRedirects); + rrddim_set_by_pointer(st, rd_InEchos, (collected_number)snmp_root.icmpmsg_InEchos); + rrddim_set_by_pointer(st, rd_OutEchos, (collected_number)snmp_root.icmpmsg_OutEchos); + rrddim_set_by_pointer(st, rd_InRouterAdvert, (collected_number)snmp_root.icmpmsg_InRouterAdvert); + rrddim_set_by_pointer(st, rd_OutRouterAdvert, (collected_number)snmp_root.icmpmsg_OutRouterAdvert); + rrddim_set_by_pointer(st, rd_InRouterSelect, (collected_number)snmp_root.icmpmsg_InRouterSelect); + rrddim_set_by_pointer(st, rd_OutRouterSelect, (collected_number)snmp_root.icmpmsg_OutRouterSelect); + rrddim_set_by_pointer(st, rd_InTimeExcds, (collected_number)snmp_root.icmpmsg_InTimeExcds); + rrddim_set_by_pointer(st, rd_OutTimeExcds, (collected_number)snmp_root.icmpmsg_OutTimeExcds); + rrddim_set_by_pointer(st, rd_InParmProbs, (collected_number)snmp_root.icmpmsg_InParmProbs); + rrddim_set_by_pointer(st, rd_OutParmProbs, (collected_number)snmp_root.icmpmsg_OutParmProbs); + rrddim_set_by_pointer(st, rd_InTimestamps, (collected_number)snmp_root.icmpmsg_InTimestamps); + rrddim_set_by_pointer(st, rd_OutTimestamps, (collected_number)snmp_root.icmpmsg_OutTimestamps); + rrddim_set_by_pointer(st, rd_InTimestampReps, (collected_number)snmp_root.icmpmsg_InTimestampReps); + rrddim_set_by_pointer(st, rd_OutTimestampReps, (collected_number)snmp_root.icmpmsg_OutTimestampReps); rrdset_done(st); } @@ -582,69 +627,132 @@ int do_proc_net_snmp(int update_every, usec_t dt) { continue; } - parse_line_pair(ff, tcp_data, h, l); + arl_begin(arl_tcp); + for(w = 1; w < words ; w++) { + if (unlikely(arl_check(arl_tcp, procfile_lineword(ff, h, w), procfile_lineword(ff, l, w)) != 0)) + break; + } + + // -------------------------------------------------------------------- + + if(snmp_root.tcp_MaxConn != last_max_connections) { + last_max_connections = snmp_root.tcp_MaxConn; + rrdvar_custom_host_variable_set(localhost, tcp_max_connections_var, last_max_connections); + } // -------------------------------------------------------------------- // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html if(do_tcp_sockets) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".tcpsock"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "tcpsock", NULL, "tcp", NULL, "IPv4 TCP Connections" - , "active connections", 2500, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_CurrEstab = NULL; - rrddim_add(st, "CurrEstab", "connections", 1, 1, RRD_ALGORITHM_ABSOLUTE); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "tcpsock" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Connections" + , "active connections" + , "proc" + , "net/snmp" + , 2501 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_CurrEstab = rrddim_add(st, "CurrEstab", "connections", 1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(st); - rrddim_set(st, "CurrEstab", *tcp_CurrEstab); + rrddim_set_by_pointer(st, rd_CurrEstab, (collected_number)snmp_root.tcp_CurrEstab); rrdset_done(st); } // -------------------------------------------------------------------- if(do_tcp_packets) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".tcppackets"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "tcppackets", NULL, "tcp", NULL, "IPv4 TCP Packets" - , "packets/s", 2600, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InSegs = NULL, + *rd_OutSegs = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "tcppackets" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Packets" + , "packets/s" + , "proc" + , "net/snmp" + , 2510 + , update_every + , RRDSET_TYPE_LINE + ); - rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutSegs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InSegs = rrddim_add(st, "InSegs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutSegs = rrddim_add(st, "OutSegs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InSegs", *tcp_InSegs); - rrddim_set(st, "OutSegs", *tcp_OutSegs); + rrddim_set_by_pointer(st, rd_InSegs, (collected_number)snmp_root.tcp_InSegs); + rrddim_set_by_pointer(st, rd_OutSegs, (collected_number)snmp_root.tcp_OutSegs); rrdset_done(st); } // -------------------------------------------------------------------- if(do_tcp_errors) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".tcperrors"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "tcperrors", NULL, "tcp", NULL, "IPv4 TCP Errors" - , "packets/s", 2700, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InErrs = NULL, + *rd_InCsumErrors = NULL, + *rd_RetransSegs = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "tcperrors" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Errors" + , "packets/s" + , "proc" + , "net/snmp" + , 2520 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "InErrs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "RetransSegs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InErrs = rrddim_add(st, "InErrs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_RetransSegs = rrddim_add(st, "RetransSegs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InErrs", *tcp_InErrs); - rrddim_set(st, "InCsumErrors", *tcp_InCsumErrors); - rrddim_set(st, "RetransSegs", *tcp_RetransSegs); + rrddim_set_by_pointer(st, rd_InErrs, (collected_number)snmp_root.tcp_InErrs); + rrddim_set_by_pointer(st, rd_InCsumErrors, (collected_number)snmp_root.tcp_InCsumErrors); + rrddim_set_by_pointer(st, rd_RetransSegs, (collected_number)snmp_root.tcp_RetransSegs); rrdset_done(st); } // -------------------------------------------------------------------- if(do_tcp_handshake) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".tcphandshake"); - if(!st) { + static RRDSET *st = NULL; + static RRDDIM *rd_EstabResets = NULL, + *rd_OutRsts = NULL, + *rd_ActiveOpens = NULL, + *rd_PassiveOpens = NULL, + *rd_AttemptFails = NULL, + *rd_TCPSynRetrans = NULL; + + if(unlikely(!st)) { st = rrdset_create_localhost( RRD_TYPE_NET_SNMP , "tcphandshake" @@ -653,27 +761,29 @@ int do_proc_net_snmp(int update_every, usec_t dt) { , NULL , "IPv4 TCP Handshake Issues" , "events/s" - , 2900 + , "proc" + , "net/snmp" + , 2530 , update_every , RRDSET_TYPE_LINE ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutRsts", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "TCPSynRetrans", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_EstabResets = rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutRsts = rrddim_add(st, "OutRsts", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ActiveOpens = rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_PassiveOpens = rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_AttemptFails = rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_TCPSynRetrans = rrddim_add(st, "TCPSynRetrans", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "EstabResets", *tcp_EstabResets); - rrddim_set(st, "OutRsts", *tcp_OutRsts); - rrddim_set(st, "ActiveOpens", *tcp_ActiveOpens); - rrddim_set(st, "PassiveOpens", *tcp_PassiveOpens); - rrddim_set(st, "AttemptFails", *tcp_AttemptFails); - rrddim_set(st, "TCPSynRetrans", tcpext_TCPSynRetrans); + rrddim_set_by_pointer(st, rd_EstabResets, (collected_number)snmp_root.tcp_EstabResets); + rrddim_set_by_pointer(st, rd_OutRsts, (collected_number)snmp_root.tcp_OutRsts); + rrddim_set_by_pointer(st, rd_ActiveOpens, (collected_number)snmp_root.tcp_ActiveOpens); + rrddim_set_by_pointer(st, rd_PassiveOpens, (collected_number)snmp_root.tcp_PassiveOpens); + rrddim_set_by_pointer(st, rd_AttemptFails, (collected_number)snmp_root.tcp_AttemptFails); + rrddim_set_by_pointer(st, rd_TCPSynRetrans, tcpext_TCPSynRetrans); rrdset_done(st); } } @@ -691,51 +801,89 @@ int do_proc_net_snmp(int update_every, usec_t dt) { continue; } - parse_line_pair(ff, udp_data, h, l); + arl_begin(arl_udp); + for(w = 1; w < words ; w++) { + if (unlikely(arl_check(arl_udp, procfile_lineword(ff, h, w), procfile_lineword(ff, l, w)) != 0)) + break; + } // -------------------------------------------------------------------- // see http://net-snmp.sourceforge.net/docs/mibs/udp.html if(do_udp_packets) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".udppackets"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "udppackets", NULL, "udp", NULL, "IPv4 UDP Packets" - , "packets/s", 2601, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InDatagrams = NULL, + *rd_OutDatagrams = NULL; - rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "udppackets" + , NULL + , "udp" + , NULL + , "IPv4 UDP Packets" + , "packets/s" + , "proc" + , "net/snmp" + , 2602 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InDatagrams = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDatagrams = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InDatagrams", *udp_InDatagrams); - rrddim_set(st, "OutDatagrams", *udp_OutDatagrams); + rrddim_set_by_pointer(st, rd_InDatagrams, (collected_number)snmp_root.udp_InDatagrams); + rrddim_set_by_pointer(st, rd_OutDatagrams, (collected_number)snmp_root.udp_OutDatagrams); rrdset_done(st); } // -------------------------------------------------------------------- if(do_udp_errors) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".udperrors"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "udperrors", NULL, "udp", NULL, "IPv4 UDP Errors" - , "events/s", 2701, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_RcvbufErrors = NULL, + *rd_SndbufErrors = NULL, + *rd_InErrors = NULL, + *rd_NoPorts = NULL, + *rd_InCsumErrors = NULL, + *rd_IgnoredMulti = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "udperrors" + , NULL + , "udp" + , NULL + , "IPv4 UDP Errors" + , "events/s" + , "proc" + , "net/snmp" + , 2701 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_IgnoredMulti = rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InErrors", *udp_InErrors); - rrddim_set(st, "NoPorts", *udp_NoPorts); - rrddim_set(st, "RcvbufErrors", *udp_RcvbufErrors); - rrddim_set(st, "SndbufErrors", *udp_SndbufErrors); - rrddim_set(st, "InCsumErrors", *udp_InCsumErrors); - rrddim_set(st, "IgnoredMulti", *udp_IgnoredMulti); + rrddim_set_by_pointer(st, rd_InErrors, (collected_number)snmp_root.udp_InErrors); + rrddim_set_by_pointer(st, rd_NoPorts, (collected_number)snmp_root.udp_NoPorts); + rrddim_set_by_pointer(st, rd_RcvbufErrors, (collected_number)snmp_root.udp_RcvbufErrors); + rrddim_set_by_pointer(st, rd_SndbufErrors, (collected_number)snmp_root.udp_SndbufErrors); + rrddim_set_by_pointer(st, rd_InCsumErrors, (collected_number)snmp_root.udp_InCsumErrors); + rrddim_set_by_pointer(st, rd_IgnoredMulti, (collected_number)snmp_root.udp_IgnoredMulti); rrdset_done(st); } } @@ -753,48 +901,87 @@ int do_proc_net_snmp(int update_every, usec_t dt) { continue; } - parse_line_pair(ff, udplite_data, h, l); + arl_begin(arl_udplite); + for(w = 1; w < words ; w++) { + if (unlikely(arl_check(arl_udplite, procfile_lineword(ff, h, w), procfile_lineword(ff, l, w)) != 0)) + break; + } // -------------------------------------------------------------------- if(do_udplite_packets) { - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".udplite"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "udplite", NULL, "udplite", NULL - , "IPv4 UDPLite Packets", "packets/s", 2603, update_every - , RRDSET_TYPE_LINE); - - rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + { + static RRDSET *st = NULL; + static RRDDIM *rd_InDatagrams = NULL, + *rd_OutDatagrams = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "udplite" + , NULL + , "udplite" + , NULL + , "IPv4 UDPLite Packets" + , "packets/s" + , "proc" + , "net/snmp" + , 2603 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InDatagrams = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDatagrams = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_InDatagrams, (collected_number)snmp_root.udplite_InDatagrams); + rrddim_set_by_pointer(st, rd_OutDatagrams, (collected_number)snmp_root.udplite_OutDatagrams); + rrdset_done(st); } - else rrdset_next(st); - - rrddim_set(st, "InDatagrams", *udplite_InDatagrams); - rrddim_set(st, "OutDatagrams", *udplite_OutDatagrams); - rrdset_done(st); - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".udplite_errors"); - if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "udplite_errors", NULL, "udplite", NULL - , "IPv4 UDPLite Errors", "packets/s", 2604, update_every - , RRDSET_TYPE_LINE); - - rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + { + static RRDSET *st = NULL; + static RRDDIM *rd_RcvbufErrors = NULL, + *rd_SndbufErrors = NULL, + *rd_InErrors = NULL, + *rd_NoPorts = NULL, + *rd_InCsumErrors = NULL, + *rd_IgnoredMulti = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "udplite_errors" + , NULL + , "udplite" + , NULL + , "IPv4 UDPLite Errors" + , "packets/s" + , "proc" + , "net/snmp" + , 2604 + , update_every + , RRDSET_TYPE_LINE); + + rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_IgnoredMulti = rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_NoPorts, (collected_number)snmp_root.udplite_NoPorts); + rrddim_set_by_pointer(st, rd_InErrors, (collected_number)snmp_root.udplite_InErrors); + rrddim_set_by_pointer(st, rd_InCsumErrors, (collected_number)snmp_root.udplite_InCsumErrors); + rrddim_set_by_pointer(st, rd_RcvbufErrors, (collected_number)snmp_root.udplite_RcvbufErrors); + rrddim_set_by_pointer(st, rd_SndbufErrors, (collected_number)snmp_root.udplite_SndbufErrors); + rrddim_set_by_pointer(st, rd_IgnoredMulti, (collected_number)snmp_root.udplite_IgnoredMulti); + rrdset_done(st); } - else rrdset_next(st); - - rrddim_set(st, "NoPorts", *udplite_NoPorts); - rrddim_set(st, "InErrors", *udplite_InErrors); - rrddim_set(st, "InCsumErrors", *udplite_InCsumErrors); - rrddim_set(st, "RcvbufErrors", *udplite_RcvbufErrors); - rrddim_set(st, "SndbufErrors", *udplite_SndbufErrors); - rrddim_set(st, "IgnoredMulti", *udplite_IgnoredMulti); - rrdset_done(st); } } } diff --git a/src/proc_net_snmp6.c b/src/proc_net_snmp6.c index 6649b7afe..bd71b391a 100644 --- a/src/proc_net_snmp6.c +++ b/src/proc_net_snmp6.c @@ -272,24 +272,37 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { procfile_lineword(ff, l, 1)))) break; } - RRDSET *st; - // -------------------------------------------------------------------- if(do_bandwidth == CONFIG_BOOLEAN_YES || (do_bandwidth == CONFIG_BOOLEAN_AUTO && (Ip6InOctets || Ip6OutOctets))) { do_bandwidth = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost("system.ipv6"); - if(unlikely(!st)) { - st = rrdset_create_localhost("system", "ipv6", NULL, "network", NULL, "IPv6 Bandwidth", "kilobits/s", 500 - , update_every, RRDSET_TYPE_AREA); + static RRDSET *st = NULL; + static RRDDIM *rd_received = NULL, + *rd_sent = NULL; - rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + "system" + , "ipv6" + , NULL + , "network" + , NULL + , "IPv6 Bandwidth" + , "kilobits/s" + , "proc" + , "net/snmp6" + , 502 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_received = rrddim_add(st, "InOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "OutOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Ip6OutOctets); - rrddim_set(st, "received", Ip6InOctets); + rrddim_set_by_pointer(st, rd_received, Ip6InOctets); + rrddim_set_by_pointer(st, rd_sent, Ip6OutOctets); rrdset_done(st); } @@ -297,22 +310,39 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_ip_packets == CONFIG_BOOLEAN_YES || (do_ip_packets == CONFIG_BOOLEAN_AUTO && (Ip6InReceives || Ip6OutRequests || Ip6InDelivers || Ip6OutForwDatagrams))) { do_ip_packets = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".packets"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "packets", NULL, "packets", NULL, "IPv6 Packets" - , "packets/s", 3000, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_received = NULL, + *rd_sent = NULL, + *rd_forwarded = NULL, + *rd_delivers = NULL; - rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "forwarded", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "delivers", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "packets" + , NULL + , "packets" + , NULL + , "IPv6 Packets" + , "packets/s" + , "proc" + , "net/snmp6" + , 3000 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_received = rrddim_add(st, "InReceives", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "OutRequests", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_forwarded = rrddim_add(st, "OutForwDatagrams", "forwarded", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_delivers = rrddim_add(st, "InDelivers", "delivers", 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Ip6OutRequests); - rrddim_set(st, "received", Ip6InReceives); - rrddim_set(st, "forwarded", Ip6OutForwDatagrams); - rrddim_set(st, "delivers", Ip6InDelivers); + rrddim_set_by_pointer(st, rd_received, Ip6InReceives); + rrddim_set_by_pointer(st, rd_sent, Ip6OutRequests); + rrddim_set_by_pointer(st, rd_forwarded, Ip6OutForwDatagrams); + rrddim_set_by_pointer(st, rd_delivers, Ip6InDelivers); rrdset_done(st); } @@ -320,21 +350,37 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_ip_fragsout == CONFIG_BOOLEAN_YES || (do_ip_fragsout == CONFIG_BOOLEAN_AUTO && (Ip6FragOKs || Ip6FragFails || Ip6FragCreates))) { do_ip_fragsout = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".fragsout"); + static RRDSET *st = NULL; + static RRDDIM *rd_ok = NULL, + *rd_failed = NULL, + *rd_all = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "fragsout", NULL, "fragments", NULL, "IPv6 Fragments Sent" - , "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "fragsout" + , NULL + , "fragments6" + , NULL + , "IPv6 Fragments Sent" + , "packets/s" + , "proc" + , "net/snmp6" + , 3011 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "all", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ok = rrddim_add(st, "FragOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_failed = rrddim_add(st, "FragFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_all = rrddim_add(st, "FragCreates", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "ok", Ip6FragOKs); - rrddim_set(st, "failed", Ip6FragFails); - rrddim_set(st, "all", Ip6FragCreates); + rrddim_set_by_pointer(st, rd_ok, Ip6FragOKs); + rrddim_set_by_pointer(st, rd_failed, Ip6FragFails); + rrddim_set_by_pointer(st, rd_all, Ip6FragCreates); rrdset_done(st); } @@ -348,24 +394,40 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Ip6ReasmReqds ))) { do_ip_fragsin = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".fragsin"); + + static RRDSET *st = NULL; + static RRDDIM *rd_ok = NULL, + *rd_failed = NULL, + *rd_timeout = NULL, + *rd_all = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "fragsin", NULL, "fragments", NULL - , "IPv6 Fragments Reassembly", "packets/s", 3011, update_every - , RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "fragsin" + , NULL + , "fragments6" + , NULL + , "IPv6 Fragments Reassembly" + , "packets/s" + , "proc" + , "net/snmp6" + , 3012 + , update_every + , RRDSET_TYPE_LINE); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "ok", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "timeout", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "all", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ok = rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_failed = rrddim_add(st, "ReasmFails", "failed", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_timeout = rrddim_add(st, "ReasmTimeout", "timeout", -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_all = rrddim_add(st, "ReasmReqds", "all", 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "ok", Ip6ReasmOKs); - rrddim_set(st, "failed", Ip6ReasmFails); - rrddim_set(st, "timeout", Ip6ReasmTimeout); - rrddim_set(st, "all", Ip6ReasmReqds); + rrddim_set_by_pointer(st, rd_ok, Ip6ReasmOKs); + rrddim_set_by_pointer(st, rd_failed, Ip6ReasmFails); + rrddim_set_by_pointer(st, rd_timeout, Ip6ReasmTimeout); + rrddim_set_by_pointer(st, rd_all, Ip6ReasmReqds); rrdset_done(st); } @@ -383,37 +445,55 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Ip6InNoRoutes ))) { do_ip_errors = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".errors"); + static RRDSET *st = NULL; + static RRDDIM *rd_InDiscards = NULL, + *rd_OutDiscards = NULL, + *rd_InHdrErrors = NULL, + *rd_InAddrErrors = NULL, + *rd_InUnknownProtos = NULL, + *rd_InTooBigErrors = NULL, + *rd_InTruncatedPkts = NULL, + *rd_InNoRoutes = NULL, + *rd_OutNoRoutes = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "errors", NULL, "errors", NULL, "IPv6 Errors", "packets/s" - , 3002, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "errors" + , NULL + , "errors" + , NULL + , "IPv6 Errors" + , "packets/s" + , "proc" + , "net/snmp6" + , 3002 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - - rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InTooBigErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InTruncatedPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InNoRoutes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - - rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDiscards = rrddim_add(st, "InDiscards", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDiscards = rrddim_add(st, "OutDiscards", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InHdrErrors = rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InAddrErrors = rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InUnknownProtos = rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InTooBigErrors = rrddim_add(st, "InTooBigErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InTruncatedPkts = rrddim_add(st, "InTruncatedPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InNoRoutes = rrddim_add(st, "InNoRoutes", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutNoRoutes = rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InDiscards", Ip6InDiscards); - rrddim_set(st, "OutDiscards", Ip6OutDiscards); - - rrddim_set(st, "InHdrErrors", Ip6InHdrErrors); - rrddim_set(st, "InAddrErrors", Ip6InAddrErrors); - rrddim_set(st, "InUnknownProtos", Ip6InUnknownProtos); - rrddim_set(st, "InTooBigErrors", Ip6InTooBigErrors); - rrddim_set(st, "InTruncatedPkts", Ip6InTruncatedPkts); - rrddim_set(st, "InNoRoutes", Ip6InNoRoutes); - - rrddim_set(st, "OutNoRoutes", Ip6OutNoRoutes); + rrddim_set_by_pointer(st, rd_InDiscards, Ip6InDiscards); + rrddim_set_by_pointer(st, rd_OutDiscards, Ip6OutDiscards); + rrddim_set_by_pointer(st, rd_InHdrErrors, Ip6InHdrErrors); + rrddim_set_by_pointer(st, rd_InAddrErrors, Ip6InAddrErrors); + rrddim_set_by_pointer(st, rd_InUnknownProtos, Ip6InUnknownProtos); + rrddim_set_by_pointer(st, rd_InTooBigErrors, Ip6InTooBigErrors); + rrddim_set_by_pointer(st, rd_InTruncatedPkts, Ip6InTruncatedPkts); + rrddim_set_by_pointer(st, rd_InNoRoutes, Ip6InNoRoutes); + rrddim_set_by_pointer(st, rd_OutNoRoutes, Ip6OutNoRoutes); rrdset_done(st); } @@ -421,18 +501,33 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_udp_packets == CONFIG_BOOLEAN_YES || (do_udp_packets == CONFIG_BOOLEAN_AUTO && (Udp6InDatagrams || Udp6OutDatagrams))) { do_udp_packets = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".udppackets"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "udppackets", NULL, "udp", NULL, "IPv6 UDP Packets" - , "packets/s", 3601, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_received = NULL, + *rd_sent = NULL; - rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "udppackets" + , NULL + , "udp6" + , NULL + , "IPv6 UDP Packets" + , "packets/s" + , "proc" + , "net/snmp6" + , 3601 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_received = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "received", Udp6InDatagrams); - rrddim_set(st, "sent", Udp6OutDatagrams); + rrddim_set_by_pointer(st, rd_received, Udp6InDatagrams); + rrddim_set_by_pointer(st, rd_sent, Udp6OutDatagrams); rrdset_done(st); } @@ -448,27 +543,46 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Udp6IgnoredMulti ))) { do_udp_errors = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".udperrors"); + static RRDSET *st = NULL; + static RRDDIM *rd_RcvbufErrors = NULL, + *rd_SndbufErrors = NULL, + *rd_InErrors = NULL, + *rd_NoPorts = NULL, + *rd_InCsumErrors = NULL, + *rd_IgnoredMulti = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "udperrors", NULL, "udp", NULL, "IPv6 UDP Errors" - , "events/s", 3701, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "udperrors" + , NULL + , "udp6" + , NULL + , "IPv6 UDP Errors" + , "events/s" + , "proc" + , "net/snmp6" + , 3701 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_IgnoredMulti = rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InErrors", Udp6InErrors); - rrddim_set(st, "NoPorts", Udp6NoPorts); - rrddim_set(st, "RcvbufErrors", Udp6RcvbufErrors); - rrddim_set(st, "SndbufErrors", Udp6SndbufErrors); - rrddim_set(st, "InCsumErrors", Udp6InCsumErrors); - rrddim_set(st, "IgnoredMulti", Udp6IgnoredMulti); + rrddim_set_by_pointer(st, rd_RcvbufErrors, Udp6RcvbufErrors); + rrddim_set_by_pointer(st, rd_SndbufErrors, Udp6SndbufErrors); + rrddim_set_by_pointer(st, rd_InErrors, Udp6InErrors); + rrddim_set_by_pointer(st, rd_NoPorts, Udp6NoPorts); + rrddim_set_by_pointer(st, rd_InCsumErrors, Udp6InCsumErrors); + rrddim_set_by_pointer(st, rd_IgnoredMulti, Udp6IgnoredMulti); rrdset_done(st); } @@ -476,18 +590,33 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_udplite_packets == CONFIG_BOOLEAN_YES || (do_udplite_packets == CONFIG_BOOLEAN_AUTO && (UdpLite6InDatagrams || UdpLite6OutDatagrams))) { do_udplite_packets = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".udplitepackets"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "udplitepackets", NULL, "udplite", NULL - , "IPv6 UDPlite Packets", "packets/s", 3601, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_received = NULL, + *rd_sent = NULL; - rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "udplitepackets" + , NULL + , "udplite6" + , NULL + , "IPv6 UDPlite Packets" + , "packets/s" + , "proc" + , "net/snmp6" + , 3602 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_received = rrddim_add(st, "InDatagrams", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_sent = rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "received", UdpLite6InDatagrams); - rrddim_set(st, "sent", UdpLite6OutDatagrams); + rrddim_set_by_pointer(st, rd_received, UdpLite6InDatagrams); + rrddim_set_by_pointer(st, rd_sent, UdpLite6OutDatagrams); rrdset_done(st); } @@ -503,25 +632,43 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || UdpLite6InCsumErrors ))) { do_udplite_errors = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".udpliteerrors"); + static RRDSET *st = NULL; + static RRDDIM *rd_RcvbufErrors = NULL, + *rd_SndbufErrors = NULL, + *rd_InErrors = NULL, + *rd_NoPorts = NULL, + *rd_InCsumErrors = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "udpliteerrors", NULL, "udplite", NULL - , "IPv6 UDP Lite Errors", "events/s", 3701, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "udpliteerrors" + , NULL + , "udplite6" + , NULL + , "IPv6 UDP Lite Errors" + , "events/s" + , "proc" + , "net/snmp6" + , 3701 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_RcvbufErrors = rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_SndbufErrors = rrddim_add(st, "SndbufErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_NoPorts = rrddim_add(st, "NoPorts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InErrors", UdpLite6InErrors); - rrddim_set(st, "NoPorts", UdpLite6NoPorts); - rrddim_set(st, "RcvbufErrors", UdpLite6RcvbufErrors); - rrddim_set(st, "SndbufErrors", UdpLite6SndbufErrors); - rrddim_set(st, "InCsumErrors", UdpLite6InCsumErrors); + rrddim_set_by_pointer(st, rd_InErrors, UdpLite6InErrors); + rrddim_set_by_pointer(st, rd_NoPorts, UdpLite6NoPorts); + rrddim_set_by_pointer(st, rd_RcvbufErrors, UdpLite6RcvbufErrors); + rrddim_set_by_pointer(st, rd_SndbufErrors, UdpLite6SndbufErrors); + rrddim_set_by_pointer(st, rd_InCsumErrors, UdpLite6InCsumErrors); rrdset_done(st); } @@ -529,20 +676,34 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_mcast == CONFIG_BOOLEAN_YES || (do_mcast == CONFIG_BOOLEAN_AUTO && (Ip6OutMcastOctets || Ip6InMcastOctets))) { do_mcast = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".mcast"); + static RRDSET *st = NULL; + static RRDDIM *rd_Ip6InMcastOctets = NULL, + *rd_Ip6OutMcastOctets = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "mcast", NULL, "multicast", NULL - , "IPv6 Multicast Bandwidth", "kilobits/s", 9000, update_every - , RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "mcast" + , NULL + , "multicast6" + , NULL + , "IPv6 Multicast Bandwidth" + , "kilobits/s" + , "proc" + , "net/snmp6" + , 9000 + , update_every + , RRDSET_TYPE_AREA + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6InMcastOctets = rrddim_add(st, "InMcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6OutMcastOctets = rrddim_add(st, "OutMcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Ip6OutMcastOctets); - rrddim_set(st, "received", Ip6InMcastOctets); + rrddim_set_by_pointer(st, rd_Ip6InMcastOctets, Ip6InMcastOctets); + rrddim_set_by_pointer(st, rd_Ip6OutMcastOctets, Ip6OutMcastOctets); rrdset_done(st); } @@ -550,20 +711,34 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_bcast == CONFIG_BOOLEAN_YES || (do_bcast == CONFIG_BOOLEAN_AUTO && (Ip6OutBcastOctets || Ip6InBcastOctets))) { do_bcast = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".bcast"); + static RRDSET *st = NULL; + static RRDDIM *rd_Ip6InBcastOctets = NULL, + *rd_Ip6OutBcastOctets = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "bcast", NULL, "broadcast", NULL - , "IPv6 Broadcast Bandwidth", "kilobits/s", 8000, update_every - , RRDSET_TYPE_AREA); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "bcast" + , NULL + , "broadcast6" + , NULL + , "IPv6 Broadcast Bandwidth" + , "kilobits/s" + , "proc" + , "net/snmp6" + , 8000 + , update_every + , RRDSET_TYPE_AREA + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6InBcastOctets = rrddim_add(st, "InBcastOctets", "received", 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6OutBcastOctets = rrddim_add(st, "OutBcastOctets", "sent", -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Ip6OutBcastOctets); - rrddim_set(st, "received", Ip6InBcastOctets); + rrddim_set_by_pointer(st, rd_Ip6InBcastOctets, Ip6InBcastOctets); + rrddim_set_by_pointer(st, rd_Ip6OutBcastOctets, Ip6OutBcastOctets); rrdset_done(st); } @@ -571,19 +746,34 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_mcast_p == CONFIG_BOOLEAN_YES || (do_mcast_p == CONFIG_BOOLEAN_AUTO && (Ip6OutMcastPkts || Ip6InMcastPkts))) { do_mcast_p = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".mcastpkts"); + static RRDSET *st = NULL; + static RRDDIM *rd_Ip6InMcastPkts = NULL, + *rd_Ip6OutMcastPkts = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "mcastpkts", NULL, "multicast", NULL - , "IPv6 Multicast Packets", "packets/s", 9500, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "mcastpkts" + , NULL + , "multicast6" + , NULL + , "IPv6 Multicast Packets" + , "packets/s" + , "proc" + , "net/snmp6" + , 9500 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6InMcastPkts = rrddim_add(st, "InMcastPkts", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Ip6OutMcastPkts = rrddim_add(st, "OutMcastPkts", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Ip6OutMcastPkts); - rrddim_set(st, "received", Ip6InMcastPkts); + rrddim_set_by_pointer(st, rd_Ip6InMcastPkts, Ip6InMcastPkts); + rrddim_set_by_pointer(st, rd_Ip6OutMcastPkts, Ip6OutMcastPkts); rrdset_done(st); } @@ -591,18 +781,33 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_icmp == CONFIG_BOOLEAN_YES || (do_icmp == CONFIG_BOOLEAN_AUTO && (Icmp6InMsgs || Icmp6OutMsgs))) { do_icmp = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmp"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmp", NULL, "icmp", NULL, "IPv6 ICMP Messages" - , "messages/s", 10000, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_Icmp6InMsgs = NULL, + *rd_Icmp6OutMsgs = NULL; - rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmp" + , NULL + , "icmp6" + , NULL + , "IPv6 ICMP Messages" + , "messages/s" + , "proc" + , "net/snmp6" + , 10000 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_Icmp6InMsgs = rrddim_add(st, "InMsgs", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Icmp6OutMsgs = rrddim_add(st, "OutMsgs", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Icmp6InMsgs); - rrddim_set(st, "received", Icmp6OutMsgs); + rrddim_set_by_pointer(st, rd_Icmp6InMsgs, Icmp6InMsgs); + rrddim_set_by_pointer(st, rd_Icmp6OutMsgs, Icmp6OutMsgs); rrdset_done(st); } @@ -610,18 +815,33 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_icmp_redir == CONFIG_BOOLEAN_YES || (do_icmp_redir == CONFIG_BOOLEAN_AUTO && (Icmp6InRedirects || Icmp6OutRedirects))) { do_icmp_redir = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmpredir"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmpredir", NULL, "icmp", NULL, "IPv6 ICMP Redirects" - , "redirects/s", 10050, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_Icmp6InRedirects = NULL, + *rd_Icmp6OutRedirects = NULL; - rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmpredir" + , NULL + , "icmp6" + , NULL + , "IPv6 ICMP Redirects" + , "redirects/s" + , "proc" + , "net/snmp6" + , 10050 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_Icmp6InRedirects = rrddim_add(st, "InRedirects", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_Icmp6OutRedirects = rrddim_add(st, "OutRedirects", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Icmp6InRedirects); - rrddim_set(st, "received", Icmp6OutRedirects); + rrddim_set_by_pointer(st, rd_Icmp6InRedirects, Icmp6InRedirects); + rrddim_set_by_pointer(st, rd_Icmp6OutRedirects, Icmp6OutRedirects); rrdset_done(st); } @@ -642,37 +862,60 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Icmp6OutParmProblems ))) { do_icmp_errors = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmperrors"); + static RRDSET *st = NULL; + static RRDDIM *rd_InErrors = NULL, + *rd_OutErrors = NULL, + *rd_InCsumErrors = NULL, + *rd_InDestUnreachs = NULL, + *rd_InPktTooBigs = NULL, + *rd_InTimeExcds = NULL, + *rd_InParmProblems = NULL, + *rd_OutDestUnreachs = NULL, + *rd_OutPktTooBigs = NULL, + *rd_OutTimeExcds = NULL, + *rd_OutParmProblems = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmperrors", NULL, "icmp", NULL, "IPv6 ICMP Errors" - , "errors/s", 10100, update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - - rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InDestUnreachs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InPktTooBigs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InTimeExcds", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InParmProblems", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutDestUnreachs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutPktTooBigs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutTimeExcds", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutParmProblems", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmperrors" + , NULL + , "icmp6" + , NULL + , "IPv6 ICMP Errors" + , "errors/s" + , "proc" + , "net/snmp6" + , 10100 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InErrors = rrddim_add(st, "InErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutErrors = rrddim_add(st, "OutErrors", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCsumErrors = rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InDestUnreachs = rrddim_add(st, "InDestUnreachs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InPktTooBigs = rrddim_add(st, "InPktTooBigs", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InTimeExcds = rrddim_add(st, "InTimeExcds", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InParmProblems = rrddim_add(st, "InParmProblems", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutDestUnreachs = rrddim_add(st, "OutDestUnreachs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutPktTooBigs = rrddim_add(st, "OutPktTooBigs", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutTimeExcds = rrddim_add(st, "OutTimeExcds", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutParmProblems = rrddim_add(st, "OutParmProblems", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InErrors", Icmp6InErrors); - rrddim_set(st, "OutErrors", Icmp6OutErrors); - rrddim_set(st, "InCsumErrors", Icmp6InCsumErrors); - rrddim_set(st, "InDestUnreachs", Icmp6InDestUnreachs); - rrddim_set(st, "InPktTooBigs", Icmp6InPktTooBigs); - rrddim_set(st, "InTimeExcds", Icmp6InTimeExcds); - rrddim_set(st, "InParmProblems", Icmp6InParmProblems); - rrddim_set(st, "OutDestUnreachs", Icmp6OutDestUnreachs); - rrddim_set(st, "OutPktTooBigs", Icmp6OutPktTooBigs); - rrddim_set(st, "OutTimeExcds", Icmp6OutTimeExcds); - rrddim_set(st, "OutParmProblems", Icmp6OutParmProblems); + rrddim_set_by_pointer(st, rd_InErrors, Icmp6InErrors); + rrddim_set_by_pointer(st, rd_OutErrors, Icmp6OutErrors); + rrddim_set_by_pointer(st, rd_InCsumErrors, Icmp6InCsumErrors); + rrddim_set_by_pointer(st, rd_InDestUnreachs, Icmp6InDestUnreachs); + rrddim_set_by_pointer(st, rd_InPktTooBigs, Icmp6InPktTooBigs); + rrddim_set_by_pointer(st, rd_InTimeExcds, Icmp6InTimeExcds); + rrddim_set_by_pointer(st, rd_InParmProblems, Icmp6InParmProblems); + rrddim_set_by_pointer(st, rd_OutDestUnreachs, Icmp6OutDestUnreachs); + rrddim_set_by_pointer(st, rd_OutPktTooBigs, Icmp6OutPktTooBigs); + rrddim_set_by_pointer(st, rd_OutTimeExcds, Icmp6OutTimeExcds); + rrddim_set_by_pointer(st, rd_OutParmProblems, Icmp6OutParmProblems); rrdset_done(st); } @@ -686,22 +929,39 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Icmp6OutEchoReplies ))) { do_icmp_echos = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmpechos"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmpechos", NULL, "icmp", NULL, "IPv6 ICMP Echo" - , "messages/s", 10200, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InEchos = NULL, + *rd_OutEchos = NULL, + *rd_InEchoReplies = NULL, + *rd_OutEchoReplies = NULL; - rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InEchoReplies", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutEchoReplies", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmpechos" + , NULL + , "icmp6" + , NULL + , "IPv6 ICMP Echo" + , "messages/s" + , "proc" + , "net/snmp6" + , 10200 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InEchos = rrddim_add(st, "InEchos", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutEchos = rrddim_add(st, "OutEchos", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InEchoReplies = rrddim_add(st, "InEchoReplies", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutEchoReplies = rrddim_add(st, "OutEchoReplies", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InEchos", Icmp6InEchos); - rrddim_set(st, "OutEchos", Icmp6OutEchos); - rrddim_set(st, "InEchoReplies", Icmp6InEchoReplies); - rrddim_set(st, "OutEchoReplies", Icmp6OutEchoReplies); + rrddim_set_by_pointer(st, rd_InEchos, Icmp6InEchos); + rrddim_set_by_pointer(st, rd_OutEchos, Icmp6OutEchos); + rrddim_set_by_pointer(st, rd_InEchoReplies, Icmp6InEchoReplies); + rrddim_set_by_pointer(st, rd_OutEchoReplies, Icmp6OutEchoReplies); rrdset_done(st); } @@ -717,27 +977,44 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Icmp6OutGroupMembReductions ))) { do_icmp_groupmemb = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".groupmemb"); + static RRDSET *st = NULL; + static RRDDIM *rd_InQueries = NULL, + *rd_OutQueries = NULL, + *rd_InResponses = NULL, + *rd_OutResponses = NULL, + *rd_InReductions = NULL, + *rd_OutReductions = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "groupmemb", NULL, "icmp", NULL - , "IPv6 ICMP Group Membership", "messages/s", 10300, update_every - , RRDSET_TYPE_LINE); - - rrddim_add(st, "InQueries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutQueries", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InResponses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutResponses", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InReductions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutReductions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "groupmemb" + , NULL + , "icmp6" + , NULL + , "IPv6 ICMP Group Membership" + , "messages/s" + , "proc" + , "net/snmp6" + , 10300 + , update_every + , RRDSET_TYPE_LINE); + + rd_InQueries = rrddim_add(st, "InQueries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutQueries = rrddim_add(st, "OutQueries", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InResponses = rrddim_add(st, "InResponses", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutResponses = rrddim_add(st, "OutResponses", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InReductions = rrddim_add(st, "InReductions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutReductions = rrddim_add(st, "OutReductions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InQueries", Icmp6InGroupMembQueries); - rrddim_set(st, "OutQueries", Icmp6OutGroupMembQueries); - rrddim_set(st, "InResponses", Icmp6InGroupMembResponses); - rrddim_set(st, "OutResponses", Icmp6OutGroupMembResponses); - rrddim_set(st, "InReductions", Icmp6InGroupMembReductions); - rrddim_set(st, "OutReductions", Icmp6OutGroupMembReductions); + rrddim_set_by_pointer(st, rd_InQueries, Icmp6InGroupMembQueries); + rrddim_set_by_pointer(st, rd_OutQueries, Icmp6OutGroupMembQueries); + rrddim_set_by_pointer(st, rd_InResponses, Icmp6InGroupMembResponses); + rrddim_set_by_pointer(st, rd_OutResponses, Icmp6OutGroupMembResponses); + rrddim_set_by_pointer(st, rd_InReductions, Icmp6InGroupMembReductions); + rrddim_set_by_pointer(st, rd_OutReductions, Icmp6OutGroupMembReductions); rrdset_done(st); } @@ -751,22 +1028,39 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Icmp6OutRouterAdvertisements ))) { do_icmp_router = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmprouter"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmprouter", NULL, "icmp", NULL, "IPv6 Router Messages" - , "messages/s", 10400, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InSolicits = NULL, + *rd_OutSolicits = NULL, + *rd_InAdvertisements = NULL, + *rd_OutAdvertisements = NULL; - rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmprouter" + , NULL + , "icmp6" + , NULL + , "IPv6 Router Messages" + , "messages/s" + , "proc" + , "net/snmp6" + , 10400 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InSolicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutSolicits = rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InAdvertisements = rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutAdvertisements = rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InSolicits", Icmp6InRouterSolicits); - rrddim_set(st, "OutSolicits", Icmp6OutRouterSolicits); - rrddim_set(st, "InAdvertisements", Icmp6InRouterAdvertisements); - rrddim_set(st, "OutAdvertisements", Icmp6OutRouterAdvertisements); + rrddim_set_by_pointer(st, rd_InSolicits, Icmp6InRouterSolicits); + rrddim_set_by_pointer(st, rd_OutSolicits, Icmp6OutRouterSolicits); + rrddim_set_by_pointer(st, rd_InAdvertisements, Icmp6InRouterAdvertisements); + rrddim_set_by_pointer(st, rd_OutAdvertisements, Icmp6OutRouterAdvertisements); rrdset_done(st); } @@ -780,23 +1074,39 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Icmp6OutNeighborAdvertisements ))) { do_icmp_neighbor = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmpneighbor"); + static RRDSET *st = NULL; + static RRDDIM *rd_InSolicits = NULL, + *rd_OutSolicits = NULL, + *rd_InAdvertisements = NULL, + *rd_OutAdvertisements = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmpneighbor", NULL, "icmp", NULL - , "IPv6 Neighbor Messages", "messages/s", 10500, update_every - , RRDSET_TYPE_LINE); - - rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmpneighbor" + , NULL + , "icmp6" + , NULL + , "IPv6 Neighbor Messages" + , "messages/s" + , "proc" + , "net/snmp6" + , 10500 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InSolicits = rrddim_add(st, "InSolicits", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutSolicits = rrddim_add(st, "OutSolicits", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InAdvertisements = rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutAdvertisements = rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InSolicits", Icmp6InNeighborSolicits); - rrddim_set(st, "OutSolicits", Icmp6OutNeighborSolicits); - rrddim_set(st, "InAdvertisements", Icmp6InNeighborAdvertisements); - rrddim_set(st, "OutAdvertisements", Icmp6OutNeighborAdvertisements); + rrddim_set_by_pointer(st, rd_InSolicits, Icmp6InNeighborSolicits); + rrddim_set_by_pointer(st, rd_OutSolicits, Icmp6OutNeighborSolicits); + rrddim_set_by_pointer(st, rd_InAdvertisements, Icmp6InNeighborAdvertisements); + rrddim_set_by_pointer(st, rd_OutAdvertisements, Icmp6OutNeighborAdvertisements); rrdset_done(st); } @@ -804,18 +1114,33 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { if(do_icmp_mldv2 == CONFIG_BOOLEAN_YES || (do_icmp_mldv2 == CONFIG_BOOLEAN_AUTO && (Icmp6InMLDv2Reports || Icmp6OutMLDv2Reports))) { do_icmp_mldv2 = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmpmldv2"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmpmldv2", NULL, "icmp", NULL, "IPv6 ICMP MLDv2 Reports" - , "reports/s", 10600, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InMLDv2Reports = NULL, + *rd_OutMLDv2Reports = NULL; - rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmpmldv2" + , NULL + , "icmp6" + , NULL + , "IPv6 ICMP MLDv2 Reports" + , "reports/s" + , "proc" + , "net/snmp6" + , 10600 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InMLDv2Reports = rrddim_add(st, "InMLDv2Reports", "received", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutMLDv2Reports = rrddim_add(st, "OutMLDv2Reports", "sent", -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "sent", Icmp6InMLDv2Reports); - rrddim_set(st, "received", Icmp6OutMLDv2Reports); + rrddim_set_by_pointer(st, rd_InMLDv2Reports, Icmp6InMLDv2Reports); + rrddim_set_by_pointer(st, rd_OutMLDv2Reports, Icmp6OutMLDv2Reports); rrdset_done(st); } @@ -835,34 +1160,57 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Icmp6OutType143 ))) { do_icmp_types = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".icmptypes"); + static RRDSET *st = NULL; + static RRDDIM *rd_InType1 = NULL, + *rd_InType128 = NULL, + *rd_InType129 = NULL, + *rd_InType136 = NULL, + *rd_OutType1 = NULL, + *rd_OutType128 = NULL, + *rd_OutType129 = NULL, + *rd_OutType133 = NULL, + *rd_OutType135 = NULL, + *rd_OutType143 = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "icmptypes", NULL, "icmp", NULL, "IPv6 ICMP Types" - , "messages/s", 10700, update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "InType1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InType128", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InType129", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InType136", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutType1", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutType128", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutType129", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutType133", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutType135", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutType143", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "icmptypes" + , NULL + , "icmp6" + , NULL + , "IPv6 ICMP Types" + , "messages/s" + , "proc" + , "net/snmp6" + , 10700 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InType1 = rrddim_add(st, "InType1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InType128 = rrddim_add(st, "InType128", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InType129 = rrddim_add(st, "InType129", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InType136 = rrddim_add(st, "InType136", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutType1 = rrddim_add(st, "OutType1", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutType128 = rrddim_add(st, "OutType128", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutType129 = rrddim_add(st, "OutType129", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutType133 = rrddim_add(st, "OutType133", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutType135 = rrddim_add(st, "OutType135", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_OutType143 = rrddim_add(st, "OutType143", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InType1", Icmp6InType1); - rrddim_set(st, "InType128", Icmp6InType128); - rrddim_set(st, "InType129", Icmp6InType129); - rrddim_set(st, "InType136", Icmp6InType136); - rrddim_set(st, "OutType1", Icmp6OutType1); - rrddim_set(st, "OutType128", Icmp6OutType128); - rrddim_set(st, "OutType129", Icmp6OutType129); - rrddim_set(st, "OutType133", Icmp6OutType133); - rrddim_set(st, "OutType135", Icmp6OutType135); - rrddim_set(st, "OutType143", Icmp6OutType143); + rrddim_set_by_pointer(st, rd_InType1, Icmp6InType1); + rrddim_set_by_pointer(st, rd_InType128, Icmp6InType128); + rrddim_set_by_pointer(st, rd_InType129, Icmp6InType129); + rrddim_set_by_pointer(st, rd_InType136, Icmp6InType136); + rrddim_set_by_pointer(st, rd_OutType1, Icmp6OutType1); + rrddim_set_by_pointer(st, rd_OutType128, Icmp6OutType128); + rrddim_set_by_pointer(st, rd_OutType129, Icmp6OutType129); + rrddim_set_by_pointer(st, rd_OutType133, Icmp6OutType133); + rrddim_set_by_pointer(st, rd_OutType135, Icmp6OutType135); + rrddim_set_by_pointer(st, rd_OutType143, Icmp6OutType143); rrdset_done(st); } @@ -876,22 +1224,39 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { || Ip6InCEPkts ))) { do_ect = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_SNMP6 ".ect"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP6, "ect", NULL, "packets", NULL, "IPv6 ECT Packets" - , "packets/s", 10800, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_InNoECTPkts = NULL, + *rd_InECT1Pkts = NULL, + *rd_InECT0Pkts = NULL, + *rd_InCEPkts = NULL; - rrddim_add(st, "InNoECTPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InECT1Pkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InECT0Pkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "InCEPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP6 + , "ect" + , NULL + , "packets" + , NULL + , "IPv6 ECT Packets" + , "packets/s" + , "proc" + , "net/snmp6" + , 10800 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_InNoECTPkts = rrddim_add(st, "InNoECTPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InECT1Pkts = rrddim_add(st, "InECT1Pkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InECT0Pkts = rrddim_add(st, "InECT0Pkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_InCEPkts = rrddim_add(st, "InCEPkts", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "InNoECTPkts", Ip6InNoECTPkts); - rrddim_set(st, "InECT1Pkts", Ip6InECT1Pkts); - rrddim_set(st, "InECT0Pkts", Ip6InECT0Pkts); - rrddim_set(st, "InCEPkts", Ip6InCEPkts); + rrddim_set_by_pointer(st, rd_InNoECTPkts, Ip6InNoECTPkts); + rrddim_set_by_pointer(st, rd_InECT1Pkts, Ip6InECT1Pkts); + rrddim_set_by_pointer(st, rd_InECT0Pkts, Ip6InECT0Pkts); + rrddim_set_by_pointer(st, rd_InCEPkts, Ip6InCEPkts); rrdset_done(st); } diff --git a/src/proc_net_sockstat.c b/src/proc_net_sockstat.c new file mode 100644 index 000000000..2ca4061b5 --- /dev/null +++ b/src/proc_net_sockstat.c @@ -0,0 +1,514 @@ +#include "common.h" + +static struct proc_net_sockstat { + kernel_uint_t sockets_used; + + kernel_uint_t tcp_inuse; + kernel_uint_t tcp_orphan; + kernel_uint_t tcp_tw; + kernel_uint_t tcp_alloc; + kernel_uint_t tcp_mem; + + kernel_uint_t udp_inuse; + kernel_uint_t udp_mem; + + kernel_uint_t udplite_inuse; + + kernel_uint_t raw_inuse; + + kernel_uint_t frag_inuse; + kernel_uint_t frag_memory; +} sockstat_root = { 0 }; + + +static int read_tcp_mem(void) { + static char *filename = NULL; + static RRDVAR *tcp_mem_low_threshold = NULL, + *tcp_mem_pressure_threshold = NULL, + *tcp_mem_high_threshold = NULL; + + if(unlikely(!tcp_mem_low_threshold)) { + tcp_mem_low_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_low"); + tcp_mem_pressure_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_pressure"); + tcp_mem_high_threshold = rrdvar_custom_host_variable_create(localhost, "tcp_mem_high"); + } + + if(unlikely(!filename)) { + char buffer[FILENAME_MAX + 1]; + snprintfz(buffer, FILENAME_MAX, "%s/proc/sys/net/ipv4/tcp_mem", netdata_configured_host_prefix); + filename = strdupz(buffer); + } + + char buffer[200 + 1], *start, *end; + if(read_file(filename, buffer, 200) != 0) return 1; + buffer[200] = '\0'; + + unsigned long long low = 0, pressure = 0, high = 0; + + start = buffer; + low = strtoull(start, &end, 10); + + start = end; + pressure = strtoull(start, &end, 10); + + start = end; + high = strtoull(start, &end, 10); + + // fprintf(stderr, "TCP MEM low = %llu, pressure = %llu, high = %llu\n", low, pressure, high); + + rrdvar_custom_host_variable_set(localhost, tcp_mem_low_threshold, low * sysconf(_SC_PAGESIZE) / 1024); + rrdvar_custom_host_variable_set(localhost, tcp_mem_pressure_threshold, pressure * sysconf(_SC_PAGESIZE) / 1024); + rrdvar_custom_host_variable_set(localhost, tcp_mem_high_threshold, high * sysconf(_SC_PAGESIZE) / 1024); + + return 0; +} + +static kernel_uint_t read_tcp_max_orphans(void) { + static char *filename = NULL; + static RRDVAR *tcp_max_orphans_var = NULL; + + if(unlikely(!filename)) { + char buffer[FILENAME_MAX + 1]; + snprintfz(buffer, FILENAME_MAX, "%s/proc/sys/net/ipv4/tcp_max_orphans", netdata_configured_host_prefix); + filename = strdupz(buffer); + } + + unsigned long long tcp_max_orphans = 0; + if(read_single_number_file(filename, &tcp_max_orphans) == 0) { + + if(unlikely(!tcp_max_orphans_var)) + tcp_max_orphans_var = rrdvar_custom_host_variable_create(localhost, "tcp_max_orphans"); + + rrdvar_custom_host_variable_set(localhost, tcp_max_orphans_var, tcp_max_orphans); + return tcp_max_orphans; + } + + return 0; +} + +int do_proc_net_sockstat(int update_every, usec_t dt) { + (void)dt; + + static procfile *ff = NULL; + + static uint32_t hash_sockets = 0, + hash_raw = 0, + hash_frag = 0, + hash_tcp = 0, + hash_udp = 0, + hash_udplite = 0; + + static long long update_constants_every = 60, update_constants_count = 0; + + static ARL_BASE *arl_sockets = NULL; + static ARL_BASE *arl_tcp = NULL; + static ARL_BASE *arl_udp = NULL; + static ARL_BASE *arl_udplite = NULL; + static ARL_BASE *arl_raw = NULL; + static ARL_BASE *arl_frag = NULL; + + static int do_sockets = -1, do_tcp_sockets = -1, do_tcp_mem = -1, do_udp_sockets = -1, do_udp_mem = -1, do_udplite_sockets = -1, do_raw_sockets = -1, do_frag_sockets = -1, do_frag_mem = -1; + + static char *keys[7] = { NULL }; + static uint32_t hashes[7] = { 0 }; + static ARL_BASE *bases[7] = { NULL }; + + if(unlikely(!arl_sockets)) { + do_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 sockets", CONFIG_BOOLEAN_AUTO); + do_tcp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 TCP sockets", CONFIG_BOOLEAN_AUTO); + do_tcp_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 TCP memory", CONFIG_BOOLEAN_AUTO); + do_udp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 UDP sockets", CONFIG_BOOLEAN_AUTO); + do_udp_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 UDP memory", CONFIG_BOOLEAN_AUTO); + do_udplite_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 UDPLITE sockets", CONFIG_BOOLEAN_AUTO); + do_raw_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 RAW sockets", CONFIG_BOOLEAN_AUTO); + do_frag_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG sockets", CONFIG_BOOLEAN_AUTO); + do_frag_mem = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat", "ipv4 FRAG memory", CONFIG_BOOLEAN_AUTO); + + update_constants_every = config_get_number("plugin:proc:/proc/net/sockstat", "update constants every", update_constants_every); + update_constants_count = update_constants_every; + + arl_sockets = arl_create("sockstat/sockets", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_sockets, "used", &sockstat_root.sockets_used); + + arl_tcp = arl_create("sockstat/TCP", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_tcp, "inuse", &sockstat_root.tcp_inuse); + arl_expect(arl_tcp, "orphan", &sockstat_root.tcp_orphan); + arl_expect(arl_tcp, "tw", &sockstat_root.tcp_tw); + arl_expect(arl_tcp, "alloc", &sockstat_root.tcp_alloc); + arl_expect(arl_tcp, "mem", &sockstat_root.tcp_mem); + + arl_udp = arl_create("sockstat/UDP", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_udp, "inuse", &sockstat_root.udp_inuse); + arl_expect(arl_udp, "mem", &sockstat_root.udp_mem); + + arl_udplite = arl_create("sockstat/UDPLITE", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_udplite, "inuse", &sockstat_root.udplite_inuse); + + arl_raw = arl_create("sockstat/RAW", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_raw, "inuse", &sockstat_root.raw_inuse); + + arl_frag = arl_create("sockstat/FRAG", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_frag, "inuse", &sockstat_root.frag_inuse); + arl_expect(arl_frag, "memory", &sockstat_root.frag_memory); + + hash_sockets = simple_hash("sockets"); + hash_tcp = simple_hash("TCP"); + hash_udp = simple_hash("UDP"); + hash_udplite = simple_hash("UDPLITE"); + hash_raw = simple_hash("RAW"); + hash_frag = simple_hash("FRAG"); + + keys[0] = "sockets"; hashes[0] = hash_sockets; bases[0] = arl_sockets; + keys[1] = "TCP"; hashes[1] = hash_tcp; bases[1] = arl_tcp; + keys[2] = "UDP"; hashes[2] = hash_udp; bases[2] = arl_udp; + keys[3] = "UDPLITE"; hashes[3] = hash_udplite; bases[3] = arl_udplite; + keys[4] = "RAW"; hashes[4] = hash_raw; bases[4] = arl_raw; + keys[5] = "FRAG"; hashes[5] = hash_frag; bases[5] = arl_frag; + keys[6] = NULL; // terminator + } + + update_constants_count += update_every; + if(unlikely(update_constants_count > update_constants_every)) { + read_tcp_max_orphans(); + read_tcp_mem(); + update_constants_count = 0; + } + + if(unlikely(!ff)) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/sockstat"); + ff = procfile_open(config_get("plugin:proc:/proc/net/sockstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; + } + + ff = procfile_readall(ff); + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time + + size_t lines = procfile_lines(ff), l; + + for(l = 0; l < lines ;l++) { + size_t words = procfile_linewords(ff, l); + char *key = procfile_lineword(ff, l, 0); + uint32_t hash = simple_hash(key); + + int k; + for(k = 0; keys[k] ; k++) { + if(unlikely(hash == hashes[k] && strcmp(key, keys[k]) == 0)) { + // fprintf(stderr, "KEY: '%s', l=%zu, w=1, words=%zu\n", key, l, words); + ARL_BASE *arl = bases[k]; + arl_begin(arl); + size_t w = 1; + + while(w + 1 < words) { + char *name = procfile_lineword(ff, l, w); w++; + char *value = procfile_lineword(ff, l, w); w++; + // fprintf(stderr, " > NAME '%s', VALUE '%s', l=%zu, w=%zu, words=%zu\n", name, value, l, w, words); + if(unlikely(arl_check(arl, name, value) != 0)) + break; + } + + break; + } + } + } + + // ------------------------------------------------------------------------ + + if(do_sockets == CONFIG_BOOLEAN_YES || (do_sockets == CONFIG_BOOLEAN_AUTO && sockstat_root.sockets_used)) { + do_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_used = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_sockets" + , NULL + , "sockets" + , NULL + , "IPv4 Sockets Used" + , "sockets" + , "proc" + , "net/sockstat" + , 2400 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_used = rrddim_add(st, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_used, (collected_number)sockstat_root.sockets_used); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO && (sockstat_root.tcp_inuse || sockstat_root.tcp_orphan || sockstat_root.tcp_tw || sockstat_root.tcp_alloc))) { + do_tcp_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL, + *rd_orphan = NULL, + *rd_timewait = NULL, + *rd_alloc = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_tcp_sockets" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Sockets" + , "sockets" + , "proc" + , "net/sockstat" + , 2500 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_alloc = rrddim_add(st, "alloc", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_orphan = rrddim_add(st, "orphan", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_timewait = rrddim_add(st, "timewait", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.tcp_inuse); + rrddim_set_by_pointer(st, rd_orphan, (collected_number)sockstat_root.tcp_orphan); + rrddim_set_by_pointer(st, rd_timewait, (collected_number)sockstat_root.tcp_tw); + rrddim_set_by_pointer(st, rd_alloc, (collected_number)sockstat_root.tcp_alloc); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_tcp_mem == CONFIG_BOOLEAN_YES || (do_tcp_mem == CONFIG_BOOLEAN_AUTO && sockstat_root.tcp_mem)) { + do_tcp_mem = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_mem = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_tcp_mem" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Sockets Memory" + , "KB" + , "proc" + , "net/sockstat" + , 2540 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_mem = rrddim_add(st, "mem", NULL, sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_mem, (collected_number)sockstat_root.tcp_mem); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_udp_sockets == CONFIG_BOOLEAN_YES || (do_udp_sockets == CONFIG_BOOLEAN_AUTO && sockstat_root.udp_inuse)) { + do_udp_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_udp_sockets" + , NULL + , "udp" + , NULL + , "IPv4 UDP Sockets" + , "sockets" + , "proc" + , "net/sockstat" + , 2600 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.udp_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_udp_mem == CONFIG_BOOLEAN_YES || (do_udp_mem == CONFIG_BOOLEAN_AUTO && sockstat_root.udp_mem)) { + do_udp_mem = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_mem = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_udp_mem" + , NULL + , "udp" + , NULL + , "IPv4 UDP Sockets Memory" + , "KB" + , "proc" + , "net/sockstat" + , 2603 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_mem = rrddim_add(st, "mem", NULL, sysconf(_SC_PAGESIZE), 1024, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_mem, (collected_number)sockstat_root.udp_mem); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_udplite_sockets == CONFIG_BOOLEAN_YES || (do_udplite_sockets == CONFIG_BOOLEAN_AUTO && sockstat_root.udplite_inuse)) { + do_udplite_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_udplite_sockets" + , NULL + , "udplite" + , NULL + , "IPv4 UDPLITE Sockets" + , "sockets" + , "proc" + , "net/sockstat" + , 2602 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.udplite_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_raw_sockets == CONFIG_BOOLEAN_YES || (do_raw_sockets == CONFIG_BOOLEAN_AUTO && sockstat_root.raw_inuse)) { + do_raw_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_raw_sockets" + , NULL + , "raw" + , NULL + , "IPv4 RAW Sockets" + , "sockets" + , "proc" + , "net/sockstat" + , 3010 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.raw_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_frag_sockets == CONFIG_BOOLEAN_YES || (do_frag_sockets == CONFIG_BOOLEAN_AUTO && sockstat_root.frag_inuse)) { + do_frag_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_frag_sockets" + , NULL + , "fragments" + , NULL + , "IPv4 FRAG Sockets" + , "fragments" + , "proc" + , "net/sockstat" + , 3010 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat_root.frag_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_frag_mem == CONFIG_BOOLEAN_YES || (do_frag_mem == CONFIG_BOOLEAN_AUTO && sockstat_root.frag_memory)) { + do_frag_mem = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_mem = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv4" + , "sockstat_frag_mem" + , NULL + , "fragments" + , NULL + , "IPv4 FRAG Sockets Memory" + , "KB" + , "proc" + , "net/sockstat" + , 3020 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_mem = rrddim_add(st, "mem", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_mem, (collected_number)sockstat_root.frag_memory); + rrdset_done(st); + } + + return 0; +} + diff --git a/src/proc_net_sockstat6.c b/src/proc_net_sockstat6.c new file mode 100644 index 000000000..97175ccf7 --- /dev/null +++ b/src/proc_net_sockstat6.c @@ -0,0 +1,269 @@ +#include "common.h" + +static struct proc_net_sockstat6 { + kernel_uint_t tcp6_inuse; + kernel_uint_t udp6_inuse; + kernel_uint_t udplite6_inuse; + kernel_uint_t raw6_inuse; + kernel_uint_t frag6_inuse; +} sockstat6_root = { 0 }; + +int do_proc_net_sockstat6(int update_every, usec_t dt) { + (void)dt; + + static procfile *ff = NULL; + + static uint32_t hash_raw = 0, + hash_frag = 0, + hash_tcp = 0, + hash_udp = 0, + hash_udplite = 0; + + static ARL_BASE *arl_tcp = NULL; + static ARL_BASE *arl_udp = NULL; + static ARL_BASE *arl_udplite = NULL; + static ARL_BASE *arl_raw = NULL; + static ARL_BASE *arl_frag = NULL; + + static int do_tcp_sockets = -1, do_udp_sockets = -1, do_udplite_sockets = -1, do_raw_sockets = -1, do_frag_sockets = -1; + + static char *keys[6] = { NULL }; + static uint32_t hashes[6] = { 0 }; + static ARL_BASE *bases[6] = { NULL }; + + if(unlikely(!arl_tcp)) { + do_tcp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 TCP sockets", CONFIG_BOOLEAN_AUTO); + do_udp_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 UDP sockets", CONFIG_BOOLEAN_AUTO); + do_udplite_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 UDPLITE sockets", CONFIG_BOOLEAN_AUTO); + do_raw_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 RAW sockets", CONFIG_BOOLEAN_AUTO); + do_frag_sockets = config_get_boolean_ondemand("plugin:proc:/proc/net/sockstat6", "ipv6 FRAG sockets", CONFIG_BOOLEAN_AUTO); + + arl_tcp = arl_create("sockstat6/TCP6", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_tcp, "inuse", &sockstat6_root.tcp6_inuse); + + arl_udp = arl_create("sockstat6/UDP6", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_udp, "inuse", &sockstat6_root.udp6_inuse); + + arl_udplite = arl_create("sockstat6/UDPLITE6", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_udplite, "inuse", &sockstat6_root.udplite6_inuse); + + arl_raw = arl_create("sockstat6/RAW6", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_raw, "inuse", &sockstat6_root.raw6_inuse); + + arl_frag = arl_create("sockstat6/FRAG6", arl_callback_str2kernel_uint_t, 60); + arl_expect(arl_frag, "inuse", &sockstat6_root.frag6_inuse); + + hash_tcp = simple_hash("TCP6"); + hash_udp = simple_hash("UDP6"); + hash_udplite = simple_hash("UDPLITE6"); + hash_raw = simple_hash("RAW6"); + hash_frag = simple_hash("FRAG6"); + + keys[0] = "TCP6"; hashes[0] = hash_tcp; bases[0] = arl_tcp; + keys[1] = "UDP6"; hashes[1] = hash_udp; bases[1] = arl_udp; + keys[2] = "UDPLITE6"; hashes[2] = hash_udplite; bases[2] = arl_udplite; + keys[3] = "RAW6"; hashes[3] = hash_raw; bases[3] = arl_raw; + keys[4] = "FRAG6"; hashes[4] = hash_frag; bases[4] = arl_frag; + keys[5] = NULL; // terminator + } + + if(unlikely(!ff)) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/net/sockstat6"); + ff = procfile_open(config_get("plugin:proc:/proc/net/sockstat6", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; + } + + ff = procfile_readall(ff); + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time + + size_t lines = procfile_lines(ff), l; + + for(l = 0; l < lines ;l++) { + size_t words = procfile_linewords(ff, l); + char *key = procfile_lineword(ff, l, 0); + uint32_t hash = simple_hash(key); + + int k; + for(k = 0; keys[k] ; k++) { + if(unlikely(hash == hashes[k] && strcmp(key, keys[k]) == 0)) { + // fprintf(stderr, "KEY: '%s', l=%zu, w=1, words=%zu\n", key, l, words); + ARL_BASE *arl = bases[k]; + arl_begin(arl); + size_t w = 1; + + while(w + 1 < words) { + char *name = procfile_lineword(ff, l, w); w++; + char *value = procfile_lineword(ff, l, w); w++; + // fprintf(stderr, " > NAME '%s', VALUE '%s', l=%zu, w=%zu, words=%zu\n", name, value, l, w, words); + if(unlikely(arl_check(arl, name, value) != 0)) + break; + } + + break; + } + } + } + + // ------------------------------------------------------------------------ + + if(do_tcp_sockets == CONFIG_BOOLEAN_YES || (do_tcp_sockets == CONFIG_BOOLEAN_AUTO && (sockstat6_root.tcp6_inuse))) { + do_tcp_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv6" + , "sockstat6_tcp_sockets" + , NULL + , "tcp6" + , NULL + , "IPv6 TCP Sockets" + , "sockets" + , "proc" + , "net/sockstat6" + , 3599 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.tcp6_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_udp_sockets == CONFIG_BOOLEAN_YES || (do_udp_sockets == CONFIG_BOOLEAN_AUTO && sockstat6_root.udp6_inuse)) { + do_udp_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv6" + , "sockstat6_udp_sockets" + , NULL + , "udp6" + , NULL + , "IPv6 UDP Sockets" + , "sockets" + , "proc" + , "net/sockstat6" + , 3600 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.udp6_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_udplite_sockets == CONFIG_BOOLEAN_YES || (do_udplite_sockets == CONFIG_BOOLEAN_AUTO && sockstat6_root.udplite6_inuse)) { + do_udplite_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv6" + , "sockstat6_udplite_sockets" + , NULL + , "udplite6" + , NULL + , "IPv6 UDPLITE Sockets" + , "sockets" + , "proc" + , "net/sockstat6" + , 3601 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.udplite6_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_raw_sockets == CONFIG_BOOLEAN_YES || (do_raw_sockets == CONFIG_BOOLEAN_AUTO && sockstat6_root.raw6_inuse)) { + do_raw_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv6" + , "sockstat6_raw_sockets" + , NULL + , "raw6" + , NULL + , "IPv6 RAW Sockets" + , "sockets" + , "proc" + , "net/sockstat6" + , 3700 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.raw6_inuse); + rrdset_done(st); + } + + // ------------------------------------------------------------------------ + + if(do_frag_sockets == CONFIG_BOOLEAN_YES || (do_frag_sockets == CONFIG_BOOLEAN_AUTO && sockstat6_root.frag6_inuse)) { + do_frag_sockets = CONFIG_BOOLEAN_YES; + + static RRDSET *st = NULL; + static RRDDIM *rd_inuse = NULL; + + if(unlikely(!st)) { + st = rrdset_create_localhost( + "ipv6" + , "sockstat6_frag_sockets" + , NULL + , "fragments6" + , NULL + , "IPv6 FRAG Sockets" + , "fragments" + , "proc" + , "net/sockstat6" + , 3010 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_inuse = rrddim_add(st, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set_by_pointer(st, rd_inuse, (collected_number)sockstat6_root.frag6_inuse); + rrdset_done(st); + } + + return 0; +} diff --git a/src/proc_net_softnet_stat.c b/src/proc_net_softnet_stat.c index b03a43c52..f3c117e16 100644 --- a/src/proc_net_softnet_stat.c +++ b/src/proc_net_softnet_stat.c @@ -79,8 +79,20 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("system", "softnet_stat"); if(unlikely(!st)) { - st = rrdset_create_localhost("system", "softnet_stat", NULL, "softnet_stat", "system.softnet_stat", "System softnet_stat" - , "events/s", 955, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "system" + , "softnet_stat" + , NULL + , "softnet_stat" + , "system.softnet_stat" + , "System softnet_stat" + , "events/s" + , "proc" + , "net/softnet_stat" + , 955 + , update_every + , RRDSET_TYPE_LINE + ); for(w = 0; w < allocated_columns ;w++) if(unlikely(softnet_column_name(w))) rrddim_add(st, softnet_column_name(w), NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -103,8 +115,20 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) { char title[100+1]; snprintfz(title, 100, "CPU%zu softnet_stat", l); - st = rrdset_create_localhost("cpu", id, NULL, "softnet_stat", "cpu.softnet_stat", title, "events/s", 4101 + l - , update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + "cpu" + , id + , NULL + , "softnet_stat" + , "cpu.softnet_stat" + , title + , "events/s" + , "proc" + , "net/softnet_stat" + , 4101 + l + , update_every + , RRDSET_TYPE_LINE + ); for(w = 0; w < allocated_columns ;w++) if(unlikely(softnet_column_name(w))) rrddim_add(st, softnet_column_name(w), NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); diff --git a/src/proc_net_stat_conntrack.c b/src/proc_net_stat_conntrack.c index e04b80a3e..363fbc199 100644 --- a/src/proc_net_stat_conntrack.c +++ b/src/proc_net_stat_conntrack.c @@ -122,136 +122,225 @@ int do_proc_net_stat_conntrack(int update_every, usec_t dt) { unsigned long long max; if(likely(!read_single_number_file(nf_conntrack_max_filename, &max))) - rrdvar_custom_host_variable_set(rrdvar_max, max); + rrdvar_custom_host_variable_set(localhost, rrdvar_max, max); } - RRDSET *st; - // -------------------------------------------------------------------- if(do_sockets) { - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_sockets"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_sockets", NULL - , RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Connections" - , "active connections", 3000, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM *rd_connections = NULL; - rrddim_add(st, "connections", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_CONNTRACK "_sockets" + , NULL + , RRD_TYPE_NET_STAT_CONNTRACK + , NULL + , "Connection Tracker Connections" + , "active connections" + , "proc" + , "net/stat/nf_conntrack" + , 3000 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_connections = rrddim_add(st, "connections", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(st); - rrddim_set(st, "connections", aentries); + rrddim_set_by_pointer(st, rd_connections, aentries); rrdset_done(st); } // -------------------------------------------------------------------- if(do_new) { - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_new"); - if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_new", NULL - , RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker New Connections" - , "connections/s", 3001, update_every, RRDSET_TYPE_LINE); + static RRDSET *st = NULL; + static RRDDIM + *rd_new = NULL, + *rd_ignore = NULL, + *rd_invalid = NULL; - rrddim_add(st, "new", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "ignore", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "invalid", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + if(unlikely(!st)) { + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_CONNTRACK "_new" + , NULL + , RRD_TYPE_NET_STAT_CONNTRACK + , NULL + , "Connection Tracker New Connections" + , "connections/s" + , "proc" + , "net/stat/nf_conntrack" + , 3001 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_new = rrddim_add(st, "new", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_ignore = rrddim_add(st, "ignore", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_invalid = rrddim_add(st, "invalid", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "new", anew); - rrddim_set(st, "ignore", aignore); - rrddim_set(st, "invalid", ainvalid); + rrddim_set_by_pointer(st, rd_new, anew); + rrddim_set_by_pointer(st, rd_ignore, aignore); + rrddim_set_by_pointer(st, rd_invalid, ainvalid); rrdset_done(st); } // -------------------------------------------------------------------- if(do_changes) { - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_changes"); + static RRDSET *st = NULL; + static RRDDIM + *rd_inserted = NULL, + *rd_deleted = NULL, + *rd_delete_list = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_changes", NULL - , RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Changes", "changes/s" - , 3002, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_CONNTRACK "_changes" + , NULL + , RRD_TYPE_NET_STAT_CONNTRACK + , NULL + , "Connection Tracker Changes" + , "changes/s" + , "proc" + , "net/stat/nf_conntrack" + , 3002 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "inserted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "deleted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "delete_list", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_inserted = rrddim_add(st, "inserted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_deleted = rrddim_add(st, "deleted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_delete_list = rrddim_add(st, "delete_list", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "inserted", ainsert); - rrddim_set(st, "deleted", adelete); - rrddim_set(st, "delete_list", adelete_list); + rrddim_set_by_pointer(st, rd_inserted, ainsert); + rrddim_set_by_pointer(st, rd_deleted, adelete); + rrddim_set_by_pointer(st, rd_delete_list, adelete_list); rrdset_done(st); } // -------------------------------------------------------------------- if(do_expect) { - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_expect"); + static RRDSET *st = NULL; + static RRDDIM *rd_created = NULL, + *rd_deleted = NULL, + *rd_new = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_expect", NULL - , RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Expectations" - , "expectations/s", 3003, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_CONNTRACK "_expect" + , NULL + , RRD_TYPE_NET_STAT_CONNTRACK + , NULL + , "Connection Tracker Expectations" + , "expectations/s" + , "proc" + , "net/stat/nf_conntrack" + , 3003 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "created", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "deleted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "new", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_created = rrddim_add(st, "created", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_deleted = rrddim_add(st, "deleted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_new = rrddim_add(st, "new", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "created", aexpect_create); - rrddim_set(st, "deleted", aexpect_delete); - rrddim_set(st, "new", aexpect_new); + rrddim_set_by_pointer(st, rd_created, aexpect_create); + rrddim_set_by_pointer(st, rd_deleted, aexpect_delete); + rrddim_set_by_pointer(st, rd_new, aexpect_new); rrdset_done(st); } // -------------------------------------------------------------------- if(do_search) { - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_search"); + static RRDSET *st = NULL; + static RRDDIM *rd_searched = NULL, + *rd_restarted = NULL, + *rd_found = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_search", NULL - , RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Searches" - , "searches/s", 3010, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_CONNTRACK "_search" + , NULL + , RRD_TYPE_NET_STAT_CONNTRACK + , NULL + , "Connection Tracker Searches" + , "searches/s" + , "proc" + , "net/stat/nf_conntrack" + , 3010 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "searched", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "restarted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "found", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_searched = rrddim_add(st, "searched", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_restarted = rrddim_add(st, "restarted", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_found = rrddim_add(st, "found", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "searched", asearched); - rrddim_set(st, "restarted", asearch_restart); - rrddim_set(st, "found", afound); + rrddim_set_by_pointer(st, rd_searched, asearched); + rrddim_set_by_pointer(st, rd_restarted, asearch_restart); + rrddim_set_by_pointer(st, rd_found, afound); rrdset_done(st); } // -------------------------------------------------------------------- if(do_errors) { - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_errors"); + static RRDSET *st = NULL; + static RRDDIM *rd_icmp_error = NULL, + *rd_insert_failed = NULL, + *rd_drop = NULL, + *rd_early_drop = NULL; + if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_errors", NULL - , RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Errors", "events/s" - , 3005, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_CONNTRACK "_errors" + , NULL + , RRD_TYPE_NET_STAT_CONNTRACK + , NULL + , "Connection Tracker Errors" + , "events/s" + , "proc" + , "net/stat/nf_conntrack" + , 3005 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "icmp_error", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "insert_failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "drop", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "early_drop", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_icmp_error = rrddim_add(st, "icmp_error", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_insert_failed = rrddim_add(st, "insert_failed", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_drop = rrddim_add(st, "drop", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_early_drop = rrddim_add(st, "early_drop", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "icmp_error", aicmp_error); - rrddim_set(st, "insert_failed", ainsert_failed); - rrddim_set(st, "drop", adrop); - rrddim_set(st, "early_drop", aearly_drop); + rrddim_set_by_pointer(st, rd_icmp_error, aicmp_error); + rrddim_set_by_pointer(st, rd_insert_failed, ainsert_failed); + rrddim_set_by_pointer(st, rd_drop, adrop); + rrddim_set_by_pointer(st, rd_early_drop, aearly_drop); rrdset_done(st); } diff --git a/src/proc_net_stat_synproxy.c b/src/proc_net_stat_synproxy.c index 5a1fc30eb..0d6f6ee03 100644 --- a/src/proc_net_stat_synproxy.c +++ b/src/proc_net_stat_synproxy.c @@ -53,18 +53,27 @@ int do_proc_net_stat_synproxy(int update_every, usec_t dt) { unsigned long long events = entries + syn_received + cookie_invalid + cookie_valid + cookie_retrans + conn_reopened; - RRDSET *st; - // -------------------------------------------------------------------- if((do_entries == CONFIG_BOOLEAN_AUTO && events) || do_entries == CONFIG_BOOLEAN_YES) { do_entries = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_entries"); + static RRDSET *st = NULL; if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_entries", NULL - , RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY Entries Used", "entries", 3304 - , update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_SYNPROXY "_entries" + , NULL + , RRD_TYPE_NET_STAT_SYNPROXY + , NULL + , "SYNPROXY Entries Used" + , "entries" + , "proc" + , "net/stat/synproxy" + , 3304 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "entries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -79,11 +88,22 @@ int do_proc_net_stat_synproxy(int update_every, usec_t dt) { if((do_syns == CONFIG_BOOLEAN_AUTO && events) || do_syns == CONFIG_BOOLEAN_YES) { do_syns = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_syn_received"); + static RRDSET *st = NULL; if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_syn_received", NULL - , RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY SYN Packets received", "SYN/s" - , 3301, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_SYNPROXY "_syn_received" + , NULL + , RRD_TYPE_NET_STAT_SYNPROXY + , NULL + , "SYNPROXY SYN Packets received" + , "SYN/s" + , "proc" + , "net/stat/synproxy" + , 3301 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } @@ -98,11 +118,22 @@ int do_proc_net_stat_synproxy(int update_every, usec_t dt) { if((do_reopened == CONFIG_BOOLEAN_AUTO && events) || do_reopened == CONFIG_BOOLEAN_YES) { do_reopened = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_conn_reopened"); + static RRDSET *st = NULL; if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_conn_reopened", NULL - , RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY Connections Reopened" - , "connections/s", 3303, update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_SYNPROXY "_conn_reopened" + , NULL + , RRD_TYPE_NET_STAT_SYNPROXY + , NULL + , "SYNPROXY Connections Reopened" + , "connections/s" + , "proc" + , "net/stat/synproxy" + , 3303 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "reopened", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } @@ -117,11 +148,22 @@ int do_proc_net_stat_synproxy(int update_every, usec_t dt) { if((do_cookies == CONFIG_BOOLEAN_AUTO && events) || do_cookies == CONFIG_BOOLEAN_YES) { do_cookies = CONFIG_BOOLEAN_YES; - st = rrdset_find_localhost(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_cookies"); + static RRDSET *st = NULL; if(unlikely(!st)) { - st = rrdset_create_localhost(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_cookies", NULL - , RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY TCP Cookies", "cookies/s", 3302 - , update_every, RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_STAT_NETFILTER + , RRD_TYPE_NET_STAT_SYNPROXY "_cookies" + , NULL + , RRD_TYPE_NET_STAT_SYNPROXY + , NULL + , "SYNPROXY TCP Cookies" + , "cookies/s" + , "proc" + , "net/stat/synproxy" + , 3302 + , update_every + , RRDSET_TYPE_LINE + ); rrddim_add(st, "valid", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(st, "invalid", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); diff --git a/src/proc_softirqs.c b/src/proc_softirqs.c index 352407a4f..a1b9947e0 100644 --- a/src/proc_softirqs.c +++ b/src/proc_softirqs.c @@ -136,6 +136,8 @@ int do_proc_softirqs(int update_every, usec_t dt) { , NULL , "System softirqs" , "softirqs/s" + , "proc" + , "softirqs" , 950 , update_every , RRDSET_TYPE_STACKED @@ -212,6 +214,8 @@ int do_proc_softirqs(int update_every, usec_t dt) { , "cpu.softirqs" , title , "softirqs/s" + , "proc" + , "softirqs" , 3000 + c , update_every , RRDSET_TYPE_STACKED diff --git a/src/proc_spl_kstat_zfs.c b/src/proc_spl_kstat_zfs.c index dee7a6b3d..9d489d8e4 100644 --- a/src/proc_spl_kstat_zfs.c +++ b/src/proc_spl_kstat_zfs.c @@ -3,7 +3,7 @@ #define ZFS_PROC_ARCSTATS "/proc/spl/kstat/zfs/arcstats" -struct arcstats arcstats = { 0 }; +extern struct arcstats arcstats; int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt) { (void)dt; @@ -11,7 +11,7 @@ int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt) { static procfile *ff = NULL; static ARL_BASE *arl_base = NULL; - l2exist = -1; + arcstats.l2exist = -1; if(unlikely(!arl_base)) { arl_base = arl_create("arcstats", NULL, 60); @@ -135,19 +135,19 @@ int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt) { const char *key = procfile_lineword(ff, l, 0); const char *value = procfile_lineword(ff, l, 2); - if(unlikely(l2exist == -1)) { + if(unlikely(arcstats.l2exist == -1)) { if(key[0] == 'l' && key[1] == '2' && key[2] == '_') - l2exist = 1; + arcstats.l2exist = 1; } if(unlikely(arl_check(arl_base, key, value))) break; } - if(unlikely(l2exist == -1)) - l2exist = 0; + if(unlikely(arcstats.l2exist == -1)) + arcstats.l2exist = 0; - generate_charts_arcstats(update_every); - generate_charts_arc_summary(update_every); + generate_charts_arcstats("proc", update_every); + generate_charts_arc_summary("proc", update_every); return 0; } diff --git a/src/proc_stat.c b/src/proc_stat.c index 40bf5cfad..907b659d0 100644 --- a/src/proc_stat.c +++ b/src/proc_stat.c @@ -287,6 +287,8 @@ int do_proc_stat(int update_every, usec_t dt) { , context , title , "percentage" + , "proc" + , "stat" , priority , update_every , RRDSET_TYPE_STACKED @@ -337,6 +339,8 @@ int do_proc_stat(int update_every, usec_t dt) { , NULL , "CPU Interrupts" , "interrupts/s" + , "proc" + , "stat" , 900 , update_every , RRDSET_TYPE_LINE @@ -367,6 +371,8 @@ int do_proc_stat(int update_every, usec_t dt) { , NULL , "CPU Context Switches" , "context switches/s" + , "proc" + , "stat" , 800 , update_every , RRDSET_TYPE_LINE @@ -406,6 +412,8 @@ int do_proc_stat(int update_every, usec_t dt) { , NULL , "Started Processes" , "processes/s" + , "proc" + , "stat" , 700 , update_every , RRDSET_TYPE_LINE @@ -436,6 +444,8 @@ int do_proc_stat(int update_every, usec_t dt) { , NULL , "System Processes" , "processes" + , "proc" + , "stat" , 600 , update_every , RRDSET_TYPE_LINE @@ -468,6 +478,8 @@ int do_proc_stat(int update_every, usec_t dt) { , "cpu.core_throttling" , "Core Thermal Throttling Events" , "events/s" + , "proc" + , "stat" , 5001 , update_every , RRDSET_TYPE_LINE @@ -496,6 +508,8 @@ int do_proc_stat(int update_every, usec_t dt) { , "cpu.package_throttling" , "Package Thermal Throttling Events" , "events/s" + , "proc" + , "stat" , 5002 , update_every , RRDSET_TYPE_LINE @@ -524,6 +538,8 @@ int do_proc_stat(int update_every, usec_t dt) { , "cpu.scaling_cur_freq" , "Per CPU Core, Current CPU Scaling Frequency" , "MHz" + , "proc" + , "stat" , 5003 , update_every , RRDSET_TYPE_LINE diff --git a/src/proc_sys_kernel_random_entropy_avail.c b/src/proc_sys_kernel_random_entropy_avail.c index 267ea271a..ca4d7657c 100644 --- a/src/proc_sys_kernel_random_entropy_avail.c +++ b/src/proc_sys_kernel_random_entropy_avail.c @@ -29,6 +29,8 @@ int do_proc_sys_kernel_random_entropy_avail(int update_every, usec_t dt) { , NULL , "Available Entropy" , "entropy" + , "proc" + , "sys/kernel/random/entropy_avail" , 1000 , update_every , RRDSET_TYPE_LINE diff --git a/src/proc_uptime.c b/src/proc_uptime.c index fb05b5cb3..8f4b90291 100644 --- a/src/proc_uptime.c +++ b/src/proc_uptime.c @@ -50,6 +50,8 @@ int do_proc_uptime(int update_every, usec_t dt) { , NULL , "System Uptime" , "seconds" + , "proc" + , "uptime" , 1000 , update_every , RRDSET_TYPE_LINE diff --git a/src/proc_vmstat.c b/src/proc_vmstat.c index 2382116f1..72ca3818f 100644 --- a/src/proc_vmstat.c +++ b/src/proc_vmstat.c @@ -102,6 +102,8 @@ int do_proc_vmstat(int update_every, usec_t dt) { , NULL , "Swap I/O" , "kilobytes/s" + , "proc" + , "vmstat" , 250 , update_every , RRDSET_TYPE_AREA @@ -126,13 +128,15 @@ int do_proc_vmstat(int update_every, usec_t dt) { if(unlikely(!st_io)) { st_io = rrdset_create_localhost( "system" - , "io" + , "pgpgio" , NULL , "disk" , NULL - , "Disk I/O" + , "Memory Paged from/to disk" , "kilobytes/s" - , 150 + , "proc" + , "vmstat" + , 151 , update_every , RRDSET_TYPE_AREA ); @@ -162,6 +166,8 @@ int do_proc_vmstat(int update_every, usec_t dt) { , NULL , "Memory Page Faults" , "page faults/s" + , "proc" + , "vmstat" , 500 , update_every , RRDSET_TYPE_LINE @@ -205,6 +211,8 @@ int do_proc_vmstat(int update_every, usec_t dt) { , NULL , "NUMA events" , "events/s" + , "proc" + , "vmstat" , 800 , update_every , RRDSET_TYPE_LINE diff --git a/src/registry.c b/src/registry.c index 76e3fa4d2..9d382e86f 100644 --- a/src/registry.c +++ b/src/registry.c @@ -323,11 +323,21 @@ void registry_statistics(void) { static RRDSET *sts = NULL, *stc = NULL, *stm = NULL; - if(!sts) sts = rrdset_find_localhost("netdata.registry_sessions"); - if(!sts) { - sts = rrdset_create_localhost("netdata", "registry_sessions", NULL, "registry", NULL - , "NetData Registry Sessions", "session", 131000, localhost->rrd_update_every - , RRDSET_TYPE_LINE); + if(unlikely(!sts)) { + sts = rrdset_create_localhost( + "netdata" + , "registry_sessions" + , NULL + , "registry" + , NULL + , "NetData Registry Sessions" + , "session" + , "registry" + , "stats" + , 131000 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); rrddim_add(sts, "sessions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -338,10 +348,21 @@ void registry_statistics(void) { // ------------------------------------------------------------------------ - if(!stc) stc = rrdset_find_localhost("netdata.registry_entries"); - if(!stc) { - stc = rrdset_create_localhost("netdata", "registry_entries", NULL, "registry", NULL, "NetData Registry Entries" - , "entries", 131100, localhost->rrd_update_every, RRDSET_TYPE_LINE); + if(unlikely(!stc)) { + stc = rrdset_create_localhost( + "netdata" + , "registry_entries" + , NULL + , "registry" + , NULL + , "NetData Registry Entries" + , "entries" + , "registry" + , "stats" + , 131100 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); rrddim_add(stc, "persons", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(stc, "machines", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -360,10 +381,21 @@ void registry_statistics(void) { // ------------------------------------------------------------------------ - if(!stm) stm = rrdset_find_localhost("netdata.registry_mem"); - if(!stm) { - stm = rrdset_create_localhost("netdata", "registry_mem", NULL, "registry", NULL, "NetData Registry Memory", "KB" - , 131300, localhost->rrd_update_every, RRDSET_TYPE_STACKED); + if(unlikely(!stm)) { + stm = rrdset_create_localhost( + "netdata" + , "registry_mem" + , NULL + , "registry" + , NULL + , "NetData Registry Memory" + , "KB" + , "registry" + , "stats" + , 131300 + , localhost->rrd_update_every + , RRDSET_TYPE_STACKED + ); rrddim_add(stm, "persons", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(stm, "machines", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); diff --git a/src/rrd.h b/src/rrd.h index 1ebb89e5a..02e608081 100644 --- a/src/rrd.h +++ b/src/rrd.h @@ -87,7 +87,7 @@ struct rrdfamily { size_t use_count; - avl_tree_lock variables_root_index; + avl_tree_lock rrdvar_root_index; }; typedef struct rrdfamily RRDFAMILY; @@ -266,9 +266,9 @@ struct rrdset { char *units; // units of measurement char *context; // the template of this data set - uint32_t hash_context; + uint32_t hash_context; // the hash of the chart's context - RRDSET_TYPE chart_type; + RRDSET_TYPE chart_type; // line, area, stacked int update_every; // every how many seconds is this updated? @@ -282,7 +282,7 @@ struct rrdset { int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored // netdata will interpolate values for gaps lower than this - long priority; + long priority; // the sorting priority of this chart // ------------------------------------------------------------------------ @@ -300,7 +300,11 @@ struct rrdset { time_t last_accessed_time; // the last time this RRDSET has been accessed time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream - size_t unused[8]; + + char *plugin_name; // the name of the plugin that generated this + char *module_name; // the name of the plugin module that generated this + + size_t unused[6]; uint32_t hash; // a simple hash on the id, to speed up searching // we first compare hashes, and only if the hashes are equal we do string comparisons @@ -315,20 +319,20 @@ struct rrdset { total_number collected_total; // used internally to calculate percentages total_number last_collected_total; // used internally to calculate percentages - RRDFAMILY *rrdfamily; - struct rrdhost *rrdhost; + RRDFAMILY *rrdfamily; // pointer to RRDFAMILY this chart belongs to + struct rrdhost *rrdhost; // pointer to RRDHOST this chart belongs to struct rrdset *next; // linking of rrdsets // ------------------------------------------------------------------------ // local variables - calculated_number green; - calculated_number red; + calculated_number green; // green threshold for this chart + calculated_number red; // red threshold for this chart - avl_tree_lock variables_root_index; - RRDSETVAR *variables; - RRDCALC *alarms; + avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart + RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs) + RRDCALC *alarms; // RRDCALC linked list for this chart // ------------------------------------------------------------------------ // members for checking the data when loading from disk @@ -368,9 +372,9 @@ typedef struct rrdset RRDSET; // and may lead to missing information. typedef enum rrdhost_flags { - RRDHOST_ORPHAN = 1 << 0, // this host is orphan (not receiving data) - RRDHOST_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts - RRDHOST_DELETE_ORPHAN_HOST = 1 << 2 // delete the entire host when orphan + RRDHOST_FLAG_ORPHAN = 1 << 0, // this host is orphan (not receiving data) + RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts + RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2 // delete the entire host when orphan } RRDHOST_FLAGS; #ifdef HAVE_C___ATOMIC @@ -410,6 +414,7 @@ struct rrdhost { const char *os; // the O/S type of the host const char *tags; // tags for this host + const char *timezone; // the timezone of the host uint32_t flags; // flags about this RRDHOST @@ -424,17 +429,26 @@ struct rrdhost { // ------------------------------------------------------------------------ // streaming of data to remote hosts - rrdpush - int rrdpush_enabled:1; // 1 when this host sends metrics to another netdata - char *rrdpush_destination; // where to send metrics to - char *rrdpush_api_key; // the api key at the receiving netdata - volatile int rrdpush_connected:1; // 1 when the sender is ready to push metrics - volatile int rrdpush_spawn:1; // 1 when the sender thread has been spawn - volatile int rrdpush_error_shown:1; // 1 when we have logged a communication error - int rrdpush_socket; // the fd of the socket to the remote host, or -1 - pthread_t rrdpush_thread; // the sender thread - netdata_mutex_t rrdpush_mutex; // exclusive access to rrdpush_buffer - int rrdpush_pipe[2]; // collector to sender thread communication - BUFFER *rrdpush_buffer; // collector fills it, sender sends them + int rrdpush_send_enabled:1; // 1 when this host sends metrics to another netdata + char *rrdpush_send_destination; // where to send metrics to + char *rrdpush_send_api_key; // the api key at the receiving netdata + + // the following are state information for the threading + // streaming metrics from this netdata to an upstream netdata + volatile int rrdpush_sender_spawn:1; // 1 when the sender thread has been spawn + pthread_t rrdpush_sender_thread; // the sender thread + + volatile int rrdpush_sender_connected:1; // 1 when the sender is ready to push metrics + int rrdpush_sender_socket; // the fd of the socket to the remote host, or -1 + + volatile int rrdpush_sender_error_shown:1; // 1 when we have logged a communication error + volatile int rrdpush_sender_join:1; // 1 when we have to join the sending thread + + // metrics may be collected asynchronously + // these synchronize all the threads willing the write to our sending buffer + netdata_mutex_t rrdpush_sender_buffer_mutex; // exclusive access to rrdpush_sender_buffer + int rrdpush_sender_pipe[2]; // collector to sender thread signaling + BUFFER *rrdpush_sender_buffer; // collector fills it, sender sends it // ------------------------------------------------------------------------ @@ -483,11 +497,14 @@ struct rrdhost { netdata_rwlock_t rrdhost_rwlock; // lock for this RRDHOST (protects rrdset_root linked list) + // ------------------------------------------------------------------------ + // indexes + avl_tree_lock rrdset_root_index; // the host's charts index (by id) avl_tree_lock rrdset_root_index_name; // the host's charts index (by name) avl_tree_lock rrdfamily_root_index; // the host's chart families index - avl_tree_lock variables_root_index; // the host's chart variables index + avl_tree_lock rrdvar_root_index; // the host's chart variables index struct rrdhost *next; }; @@ -532,6 +549,7 @@ extern RRDHOST *rrdhost_find_or_create( , const char *registry_hostname , const char *guid , const char *os + , const char *timezone , const char *tags , int update_every , long history @@ -560,8 +578,8 @@ extern void __rrd_check_wrlock(const char *file, const char *function, const uns #else #define rrdhost_check_rdlock(host) (void)0 #define rrdhost_check_wrlock(host) (void)0 -#define rrdset_check_rdlock(host) (void)0 -#define rrdset_check_wrlock(host) (void)0 +#define rrdset_check_rdlock(st) (void)0 +#define rrdset_check_wrlock(st) (void)0 #define rrd_check_rdlock() (void)0 #define rrd_check_wrlock() (void)0 #endif @@ -569,7 +587,7 @@ extern void __rrd_check_wrlock(const char *file, const char *function, const uns // ---------------------------------------------------------------------------- // RRDSET functions -extern void rrdset_set_name(RRDSET *st, const char *name); +extern int rrdset_set_name(RRDSET *st, const char *name); extern RRDSET *rrdset_create_custom(RRDHOST *host , const char *type @@ -579,26 +597,30 @@ extern RRDSET *rrdset_create_custom(RRDHOST *host , const char *context , const char *title , const char *units + , const char *plugin + , const char *module , long priority , int update_every , RRDSET_TYPE chart_type , RRD_MEMORY_MODE memory_mode , long history_entries); -#define rrdset_create(host, type, id, name, family, context, title, units, priority, update_every, chart_type) \ - rrdset_create_custom(host, type, id, name, family, context, title, units, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries) +#define rrdset_create(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \ + rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries) -#define rrdset_create_localhost(type, id, name, family, context, title, units, priority, update_every, chart_type) \ - rrdset_create(localhost, type, id, name, family, context, title, units, priority, update_every, chart_type) +#define rrdset_create_localhost(type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \ + rrdset_create(localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) extern void rrdhost_free_all(void); extern void rrdhost_save_all(void); extern void rrdhost_cleanup_all(void); -extern void rrdhost_cleanup_orphan_hosts(RRDHOST *protected); +extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected); extern void rrdhost_free(RRDHOST *host); -extern void rrdhost_save(RRDHOST *host); -extern void rrdhost_delete(RRDHOST *host); +extern void rrdhost_save_charts(RRDHOST *host); +extern void rrdhost_delete_charts(RRDHOST *host); + +extern int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now); extern void rrdset_update_heterogeneous_flag(RRDSET *st); @@ -678,7 +700,6 @@ extern collected_number rrddim_set(RRDSET *st, const char *id, collected_number extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries); - // ---------------------------------------------------------------------------- // RRD internal functions diff --git a/src/rrd2json.c b/src/rrd2json.c index 84d904485..00d7a552f 100644 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -13,6 +13,8 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions "\t\t\t\"context\": \"%s\",\n" "\t\t\t\"title\": \"%s (%s)\",\n" "\t\t\t\"priority\": %ld,\n" + "\t\t\t\"plugin\": \"%s\",\n" + "\t\t\t\"module\": \"%s\",\n" "\t\t\t\"enabled\": %s,\n" "\t\t\t\"units\": \"%s\",\n" "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n" @@ -29,6 +31,8 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions , st->context , st->title, st->name , st->priority + , st->plugin_name?st->plugin_name:"" + , st->module_name?st->module_name:"" , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?"true":"false" , st->units , st->name @@ -48,13 +52,14 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions memory += rd->memsize; - buffer_sprintf(wb, - "%s" - "\t\t\t\t\"%s\": { \"name\": \"%s\" }" - , dimensions?",\n":"" - , rd->id - , rd->name - ); + buffer_sprintf( + wb + , "%s" + "\t\t\t\t\"%s\": { \"name\": \"%s\" }" + , dimensions ? ",\n" : "" + , rd->id + , rd->name + ); dimensions++; } @@ -67,8 +72,33 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions buffer_strcat(wb, ",\n\t\t\t\"red\": "); buffer_rrd_value(wb, st->red); + buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n"); + size_t alarms = 0; + RRDCALC *rc; + for(rc = st->alarms; rc ; rc = rc->rrdset_next) { + + buffer_sprintf( + wb + , "%s" + "\t\t\t\t\"%s\": {\n" + "\t\t\t\t\t\"id\": %u,\n" + "\t\t\t\t\t\"status\": \"%s\",\n" + "\t\t\t\t\t\"units\": \"%s\",\n" + "\t\t\t\t\t\"update_every\": %d\n" + "\t\t\t\t}" + , (alarms) ? ",\n" : "" + , rc->name + , rc->id + , rrdcalc_status2string(rc->status) + , rc->units + , rc->update_every + ); + + alarms++; + } + buffer_sprintf(wb, - "\n\t\t}" + "\n\t\t\t}\n\t\t}" ); rrdset_unlock(st); @@ -92,6 +122,7 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) { "\t\"hostname\": \"%s\"" ",\n\t\"version\": \"%s\"" ",\n\t\"os\": \"%s\"" + ",\n\t\"timezone\": \"%s\"" ",\n\t\"update_every\": %d" ",\n\t\"history\": %ld" ",\n\t\"custom_info\": \"%s\"" @@ -99,6 +130,7 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) { , host->hostname , program_version , host->os + , host->timezone , host->rrd_update_every , host->rrd_history_entries , custom_dashboard_info_js_filename @@ -126,7 +158,8 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) { } rrdhost_unlock(host); - buffer_sprintf(wb, "\n\t}" + buffer_sprintf(wb + , "\n\t}" ",\n\t\"charts_count\": %zu" ",\n\t\"dimensions_count\": %zu" ",\n\t\"alarms_count\": %zu" @@ -142,24 +175,31 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) { if(unlikely(rrd_hosts_available > 1)) { rrd_rdlock(); + + size_t found = 0; RRDHOST *h; rrdhost_foreach_read(h) { - buffer_sprintf(wb, - "%s\n\t\t{" - "\n\t\t\t\"hostname\": \"%s\"" - "\n\t\t}" - , (h != localhost) ? "," : "" - , h->hostname - ); + if(!rrdhost_should_be_removed(h, host, now)) { + buffer_sprintf(wb + , "%s\n\t\t{" + "\n\t\t\t\"hostname\": \"%s\"" + "\n\t\t}" + , (found > 0) ? "," : "" + , h->hostname + ); + + found++; + } } + rrd_unlock(); } else { - buffer_sprintf(wb, - "\n\t\t{" - "\n\t\t\t\"hostname\": \"%s\"" - "\n\t\t}" - , host->hostname + buffer_sprintf(wb + , "\n\t\t{" + "\n\t\t\t\"hostname\": \"%s\"" + "\n\t\t}" + , host->hostname ); } diff --git a/src/rrd2json.h b/src/rrd2json.h index a3b9c9509..7212c0b3d 100644 --- a/src/rrd2json.h +++ b/src/rrd2json.h @@ -61,6 +61,7 @@ #define RRDR_OPTION_LABEL_QUOTES 0x00000400 // in CSV output, wrap header labels in double quotes #define RRDR_OPTION_PERCENTAGE 0x00000800 // give values as percentage of total #define RRDR_OPTION_NOT_ALIGNED 0x00001000 // do not align charts for persistant timeframes +#define RRDR_OPTION_DISPLAY_ABS 0x00002000 // for badges, display the absolute value, but calculate colors with sign extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); extern void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb); diff --git a/src/rrdcalc.c b/src/rrdcalc.c index bb90a4c6d..4177733b0 100644 --- a/src/rrdcalc.c +++ b/src/rrdcalc.c @@ -4,7 +4,7 @@ // ---------------------------------------------------------------------------- // RRDCALC management -inline const char *rrdcalc_status2string(int status) { +inline const char *rrdcalc_status2string(RRDCALC_STATUS status) { switch(status) { case RRDCALC_STATUS_REMOVED: return "REMOVED"; @@ -62,15 +62,15 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { st->red = rc->red; } - rc->local = rrdvar_create_and_index("local", &st->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value); - rc->family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value); + rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value); + rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value); char fullname[RRDVAR_MAX_LENGTH + 1]; snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name); - rc->hostid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value); + rc->hostid = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value); snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name); - rc->hostname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value); + rc->hostname = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value); if(!rc->units) rc->units = strdupz(st->units); @@ -173,16 +173,16 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { rc->rrdset_prev = rc->rrdset_next = NULL; - rrdvar_free(st->rrdhost, &st->variables_root_index, rc->local); + rrdvar_free(st->rrdhost, &st->rrdvar_root_index, rc->local); rc->local = NULL; - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rc->family); + rrdvar_free(st->rrdhost, &st->rrdfamily->rrdvar_root_index, rc->family); rc->family = NULL; - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostid); + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rc->hostid); rc->hostid = NULL; - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostname); + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rc->hostname); rc->hostname = NULL; rc->rrdset = NULL; @@ -375,8 +375,27 @@ inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *c return rc; } -void rrdcalc_free(RRDHOST *host, RRDCALC *rc) { - if(!rc) return; +void rrdcalc_free(RRDCALC *rc) { + if(unlikely(!rc)) return; + + expression_free(rc->calculation); + expression_free(rc->warning); + expression_free(rc->critical); + + freez(rc->name); + freez(rc->chart); + freez(rc->family); + freez(rc->dimensions); + freez(rc->exec); + freez(rc->recipient); + freez(rc->source); + freez(rc->units); + freez(rc->info); + freez(rc); +} + +void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) { + if(unlikely(!rc)) return; debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname); @@ -398,18 +417,5 @@ void rrdcalc_free(RRDHOST *host, RRDCALC *rc) { error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname); } - expression_free(rc->calculation); - expression_free(rc->warning); - expression_free(rc->critical); - - freez(rc->name); - freez(rc->chart); - freez(rc->family); - freez(rc->dimensions); - freez(rc->exec); - freez(rc->recipient); - freez(rc->source); - freez(rc->units); - freez(rc->info); - freez(rc); + rrdcalc_free(rc); } diff --git a/src/rrdcalctemplate.c b/src/rrdcalctemplate.c index b5d2c7d61..4ec24cd21 100644 --- a/src/rrdcalctemplate.c +++ b/src/rrdcalctemplate.c @@ -22,25 +22,9 @@ void rrdcalctemplate_link_matching(RRDSET *st) { } } -inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) { +inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) { if(unlikely(!rt)) return; - debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname); - - if(host->templates == rt) { - host->templates = rt->next; - } - else { - RRDCALCTEMPLATE *t; - for (t = host->templates; t && t->next != rt; t = t->next ) ; - if(t) { - t->next = rt->next; - rt->next = NULL; - } - else - error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname); - } - expression_free(rt->calculation); expression_free(rt->warning); expression_free(rt->critical); @@ -59,4 +43,26 @@ inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) { freez(rt); } +inline void rrdcalctemplate_unlink_and_free(RRDHOST *host, RRDCALCTEMPLATE *rt) { + if(unlikely(!rt)) return; + + debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname); + + if(host->templates == rt) { + host->templates = rt->next; + } + else { + RRDCALCTEMPLATE *t; + for (t = host->templates; t && t->next != rt; t = t->next ) ; + if(t) { + t->next = rt->next; + rt->next = NULL; + } + else + error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname); + } + + rrdcalctemplate_free(rt); +} + diff --git a/src/rrddim.c b/src/rrddim.c index 8df548397..6477a1cbe 100644 --- a/src/rrddim.c +++ b/src/rrddim.c @@ -155,7 +155,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte reset = 1; } else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) { - error("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd->last_collected_time) / USEC_PER_SEC, rd->entries * rd->update_every); + info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd->last_collected_time) / USEC_PER_SEC, rd->entries * rd->update_every); memset(rd, 0, size); reset = 1; } @@ -262,9 +262,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte } if(st->rrdhost->health_enabled) { - rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, 0); - rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, 0); - rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, 0); + rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT); + rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT); + rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); } rrdset_unlock(st); diff --git a/src/rrddimvar.c b/src/rrddimvar.c index f6eb6d8ef..0d20815bf 100644 --- a/src/rrddimvar.c +++ b/src/rrddimvar.c @@ -13,38 +13,38 @@ static inline void rrddimvar_free_variables(RRDDIMVAR *rs) { // CHART VARIABLES FOR THIS DIMENSION - rrdvar_free(st->rrdhost, &st->variables_root_index, rs->var_local_id); + rrdvar_free(st->rrdhost, &st->rrdvar_root_index, rs->var_local_id); rs->var_local_id = NULL; - rrdvar_free(st->rrdhost, &st->variables_root_index, rs->var_local_name); + rrdvar_free(st->rrdhost, &st->rrdvar_root_index, rs->var_local_name); rs->var_local_name = NULL; // FAMILY VARIABLES FOR THIS DIMENSION - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_id); + rrdvar_free(st->rrdhost, &st->rrdfamily->rrdvar_root_index, rs->var_family_id); rs->var_family_id = NULL; - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_name); + rrdvar_free(st->rrdhost, &st->rrdfamily->rrdvar_root_index, rs->var_family_name); rs->var_family_name = NULL; - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_contextid); + rrdvar_free(st->rrdhost, &st->rrdfamily->rrdvar_root_index, rs->var_family_contextid); rs->var_family_contextid = NULL; - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_contextname); + rrdvar_free(st->rrdhost, &st->rrdfamily->rrdvar_root_index, rs->var_family_contextname); rs->var_family_contextname = NULL; // HOST VARIABLES FOR THIS DIMENSION - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartidid); + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rs->var_host_chartidid); rs->var_host_chartidid = NULL; - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartidname); + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rs->var_host_chartidname); rs->var_host_chartidname = NULL; - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartnameid); + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rs->var_host_chartnameid); rs->var_host_chartnameid = NULL; - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartnamename); + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rs->var_host_chartnamename); rs->var_host_chartnamename = NULL; // KEYS @@ -115,8 +115,8 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $id // - $name - rs->var_local_id = rrdvar_create_and_index("local", &st->variables_root_index, rs->key_id, rs->type, rs->value); - rs->var_local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->key_name, rs->type, rs->value); + rs->var_local_id = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_id, rs->type, rs->value); + rs->var_local_name = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_name, rs->type, rs->value); // FAMILY VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -127,10 +127,10 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $chart-context.id // - $chart-context.name - rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_id, rs->type, rs->value); - rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_name, rs->type, rs->value); - rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_contextid, rs->type, rs->value); - rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_contextname, rs->type, rs->value); + rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, rs->value); + rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, rs->value); + rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, rs->value); + rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, rs->value); // HOST VARIABLES FOR THIS DIMENSION // ----------------------------------- @@ -141,13 +141,13 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { // - $chart-name.id // - $chart-name.name - rs->var_host_chartidid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullidid, rs->type, rs->value); - rs->var_host_chartidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullidname, rs->type, rs->value); - rs->var_host_chartnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullnameid, rs->type, rs->value); - rs->var_host_chartnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullnamename, rs->type, rs->value); + rs->var_host_chartidid = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, rs->key_fullidid, rs->type, rs->value); + rs->var_host_chartidname = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, rs->key_fullidname, rs->type, rs->value); + rs->var_host_chartnameid = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, rs->key_fullnameid, rs->type, rs->value); + rs->var_host_chartnamename = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, rs->key_fullnamename, rs->type, rs->value); } -RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options) { +RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options) { RRDSET *st = rd->rrdset; debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:""); diff --git a/src/rrdfamily.c b/src/rrdfamily.c index fc203e915..905ae480b 100644 --- a/src/rrdfamily.c +++ b/src/rrdfamily.c @@ -30,11 +30,11 @@ RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id) { rc->hash_family = simple_hash(rc->family); // initialize the variables index - avl_init_lock(&rc->variables_root_index, rrdvar_compare); + avl_init_lock(&rc->rrdvar_root_index, rrdvar_compare); RRDFAMILY *ret = rrdfamily_index_add(host, rc); if(ret != rc) - fatal("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", rc->family, (ret)?ret->family:"NONE"); + error("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", rc->family, (ret)?ret->family:"NONE"); } rc->use_count++; @@ -46,13 +46,14 @@ void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc) { if(!rc->use_count) { RRDFAMILY *ret = rrdfamily_index_del(host, rc); if(ret != rc) - fatal("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", rc->family, (ret)?ret->family:"NONE"); - - if(rc->variables_root_index.avl_tree.root != NULL) - fatal("RRDFAMILY: INTERNAL ERROR: Variables index of RRDFAMILY '%s' that is freed, is not empty.", rc->family); - - freez((void *)rc->family); - freez(rc); + error("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", rc->family, (ret)?ret->family:"NONE"); + else { + debug(D_RRD_CALLS, "RRDFAMILY: Cleaning up remaining family variables for host '%s', family '%s'", host->hostname, rc->family); + rrdvar_free_remaining_variables(host, &rc->rrdvar_root_index); + + freez((void *) rc->family); + freez(rc); + } } } diff --git a/src/rrdhost.c b/src/rrdhost.c index ff8aa5617..831cc46c8 100644 --- a/src/rrdhost.c +++ b/src/rrdhost.c @@ -86,6 +86,15 @@ static inline void rrdhost_init_os(RRDHOST *host, const char *os) { freez(old); } +static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone) { + if(host->timezone && timezone && !strcmp(host->timezone, timezone)) + return; + + void *old = (void *)host->timezone; + host->timezone = strdupz((timezone && *timezone)?timezone:"unknown"); + freez(old); +} + static inline void rrdhost_init_machine_guid(RRDHOST *host, const char *machine_guid) { strncpy(host->machine_guid, machine_guid, GUID_LEN); host->machine_guid[GUID_LEN] = '\0'; @@ -100,6 +109,7 @@ RRDHOST *rrdhost_create(const char *hostname, const char *registry_hostname, const char *guid, const char *os, + const char *timezone, const char *tags, int update_every, long entries, @@ -120,33 +130,34 @@ RRDHOST *rrdhost_create(const char *hostname, host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries); host->rrd_memory_mode = memory_mode; host->health_enabled = (memory_mode == RRD_MEMORY_MODE_NONE)? 0 : health_enabled; - host->rrdpush_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key); - host->rrdpush_destination = (host->rrdpush_enabled)?strdupz(rrdpush_destination):NULL; - host->rrdpush_api_key = (host->rrdpush_enabled)?strdupz(rrdpush_api_key):NULL; + host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key); + host->rrdpush_send_destination = (host->rrdpush_send_enabled)?strdupz(rrdpush_destination):NULL; + host->rrdpush_send_api_key = (host->rrdpush_send_enabled)?strdupz(rrdpush_api_key):NULL; - host->rrdpush_pipe[0] = -1; - host->rrdpush_pipe[1] = -1; - host->rrdpush_socket = -1; + host->rrdpush_sender_pipe[0] = -1; + host->rrdpush_sender_pipe[1] = -1; + host->rrdpush_sender_socket = -1; - netdata_mutex_init(&host->rrdpush_mutex); + netdata_mutex_init(&host->rrdpush_sender_buffer_mutex); netdata_rwlock_init(&host->rrdhost_rwlock); rrdhost_init_hostname(host, hostname); rrdhost_init_machine_guid(host, guid); rrdhost_init_os(host, os); + rrdhost_init_timezone(host, timezone); rrdhost_init_tags(host, tags); host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname); avl_init_lock(&(host->rrdset_root_index), rrdset_compare); avl_init_lock(&(host->rrdset_root_index_name), rrdset_compare_name); avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare); - avl_init_lock(&(host->variables_root_index), rrdvar_compare); + avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare); if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1)) - rrdhost_flag_set(host, RRDHOST_DELETE_OBSOLETE_CHARTS); + rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost) - rrdhost_flag_set(host, RRDHOST_DELETE_ORPHAN_HOST); + rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST); // ------------------------------------------------------------------------ @@ -251,7 +262,8 @@ RRDHOST *rrdhost_create(const char *hostname, } else { info("Host '%s' (at registry as '%s') with guid '%s' initialized" - ", os %s" + ", os '%s'" + ", timezone '%s'" ", tags '%s'" ", update every %d" ", memory mode %s" @@ -268,13 +280,14 @@ RRDHOST *rrdhost_create(const char *hostname, , host->registry_hostname , host->machine_guid , host->os + , host->timezone , (host->tags)?host->tags:"" , host->rrd_update_every , rrd_memory_mode_name(host->rrd_memory_mode) , host->rrd_history_entries - , host->rrdpush_enabled?"enabled":"disabled" - , host->rrdpush_destination?host->rrdpush_destination:"" - , host->rrdpush_api_key?host->rrdpush_api_key:"" + , host->rrdpush_send_enabled?"enabled":"disabled" + , host->rrdpush_send_destination?host->rrdpush_send_destination:"" + , host->rrdpush_send_api_key?host->rrdpush_send_api_key:"" , host->health_enabled?"enabled":"disabled" , host->cache_dir , host->varlib_dir @@ -294,6 +307,7 @@ RRDHOST *rrdhost_find_or_create( , const char *registry_hostname , const char *guid , const char *os + , const char *timezone , const char *tags , int update_every , long history @@ -313,6 +327,7 @@ RRDHOST *rrdhost_find_or_create( , registry_hostname , guid , os + , timezone , tags , update_every , history @@ -346,16 +361,18 @@ RRDHOST *rrdhost_find_or_create( // update host tags rrdhost_init_tags(host, tags); } - rrd_unlock(); - rrdhost_cleanup_orphan_hosts(host); + rrdhost_cleanup_orphan_hosts_nolock(host); + + rrd_unlock(); return host; } -static inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) { +inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) { if(host != protected && host != localhost + && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN) && !host->connected_senders && host->senders_disconnected_time && host->senders_disconnected_time + rrdhost_free_orphan_time < now) @@ -364,11 +381,9 @@ static inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, t return 0; } -void rrdhost_cleanup_orphan_hosts(RRDHOST *protected) { +void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected) { time_t now = now_realtime_sec(); - rrd_wrlock(); - RRDHOST *host; restart_after_removal: @@ -376,17 +391,15 @@ restart_after_removal: if(rrdhost_should_be_removed(host, protected, now)) { info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid); - if(rrdset_flag_check(host, RRDHOST_DELETE_ORPHAN_HOST) && rrdset_flag_check(host, RRDHOST_ORPHAN)) - rrdhost_delete(host); + if(rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST)) + rrdhost_delete_charts(host); else - rrdhost_save(host); + rrdhost_save_charts(host); rrdhost_free(host); goto restart_after_removal; } } - - rrd_unlock(); } // ---------------------------------------------------------------------------- @@ -406,6 +419,7 @@ void rrd_init(char *hostname) { , registry_get_this_machine_hostname() , registry_get_this_machine_guid() , os_type + , netdata_configured_timezone , config_get(CONFIG_SECTION_BACKEND, "host tags", "") , default_rrd_update_every , default_rrd_history_entries @@ -473,12 +487,19 @@ void rrdhost_free(RRDHOST *host) { // ------------------------------------------------------------------------ // release its children resources - while(host->rrdset_root) rrdset_free(host->rrdset_root); + while(host->rrdset_root) + rrdset_free(host->rrdset_root); - while(host->alarms) rrdcalc_free(host, host->alarms); - while(host->templates) rrdcalctemplate_free(host, host->templates); - health_alarm_log_free(host); + while(host->alarms) + rrdcalc_unlink_and_free(host, host->alarms); + while(host->templates) + rrdcalctemplate_unlink_and_free(host, host->templates); + + debug(D_RRD_CALLS, "RRDHOST: Cleaning up remaining host variables for host '%s'", host->hostname); + rrdvar_free_remaining_variables(host, &host->rrdvar_root_index); + + health_alarm_log_free(host); // ------------------------------------------------------------------------ // remove it from the indexes @@ -508,10 +529,11 @@ void rrdhost_free(RRDHOST *host) { freez((void *)host->tags); freez((void *)host->os); + freez((void *)host->timezone); freez(host->cache_dir); freez(host->varlib_dir); - freez(host->rrdpush_api_key); - freez(host->rrdpush_destination); + freez(host->rrdpush_send_api_key); + freez(host->rrdpush_send_destination); freez(host->health_default_exec); freez(host->health_default_recipient); freez(host->health_log_filename); @@ -534,7 +556,7 @@ void rrdhost_free_all(void) { // ---------------------------------------------------------------------------- // RRDHOST - save host files -void rrdhost_save(RRDHOST *host) { +void rrdhost_save_charts(RRDHOST *host) { if(!host) return; info("Saving/Closing database of host '%s'...", host->hostname); @@ -557,7 +579,7 @@ void rrdhost_save(RRDHOST *host) { // ---------------------------------------------------------------------------- // RRDHOST - delete host files -void rrdhost_delete(RRDHOST *host) { +void rrdhost_delete_charts(RRDHOST *host) { if(!host) return; info("Deleting database of host '%s'...", host->hostname); @@ -582,12 +604,13 @@ void rrdhost_delete(RRDHOST *host) { // ---------------------------------------------------------------------------- // RRDHOST - cleanup host files -void rrdhost_cleanup(RRDHOST *host) { +void rrdhost_cleanup_charts(RRDHOST *host) { if(!host) return; info("Cleaning up database of host '%s'...", host->hostname); RRDSET *st; + uint32_t rrdhost_delete_obsolete_charts = rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); // we get a write lock // to ensure only one thread is saving the database @@ -596,7 +619,7 @@ void rrdhost_cleanup(RRDHOST *host) { rrdset_foreach_write(st, host) { rrdset_rdlock(st); - if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS)) + if(rrdhost_delete_obsolete_charts && rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)) rrdset_delete(st); else rrdset_save(st); @@ -618,7 +641,7 @@ void rrdhost_save_all(void) { RRDHOST *host; rrdhost_foreach_read(host) - rrdhost_save(host); + rrdhost_save_charts(host); rrd_unlock(); } @@ -633,10 +656,10 @@ void rrdhost_cleanup_all(void) { RRDHOST *host; rrdhost_foreach_read(host) { - if(host != localhost && rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS) && !host->connected_senders) - rrdhost_delete(host); + if(host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS) && !host->connected_senders) + rrdhost_delete_charts(host); else - rrdhost_cleanup(host); + rrdhost_cleanup_charts(host); } rrd_unlock(); @@ -651,6 +674,8 @@ void rrdhost_cleanup_obsolete_charts(RRDHOST *host) { RRDSET *st; + uint32_t rrdhost_delete_obsolete_charts = rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); + restart_after_removal: rrdset_foreach_write(st, host) { if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) @@ -661,7 +686,7 @@ restart_after_removal: rrdset_rdlock(st); - if(rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS)) + if(rrdhost_delete_obsolete_charts) rrdset_delete(st); else rrdset_save(st); diff --git a/src/rrdpush.c b/src/rrdpush.c index c1d052fd8..2d10c3ca9 100644 --- a/src/rrdpush.c +++ b/src/rrdpush.c @@ -59,72 +59,100 @@ int rrdpush_init() { // this is for the first iterations of each chart unsigned int remote_clock_resync_iterations = 60; -#define rrdpush_lock(host) netdata_mutex_lock(&((host)->rrdpush_mutex)) -#define rrdpush_unlock(host) netdata_mutex_unlock(&((host)->rrdpush_mutex)) +#define rrdpush_buffer_lock(host) netdata_mutex_lock(&((host)->rrdpush_sender_buffer_mutex)) +#define rrdpush_buffer_unlock(host) netdata_mutex_unlock(&((host)->rrdpush_sender_buffer_mutex)) // checks if the current chart definition has been sent static inline int need_to_send_chart_definition(RRDSET *st) { + rrdset_check_rdlock(st); + if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_EXPOSED_UPSTREAM)))) return 1; RRDDIM *rd; rrddim_foreach_read(rd, st) - if(!rd->exposed) + if(unlikely(!rd->exposed)) return 1; return 0; } // sends the current chart definition -static inline void send_chart_definition(RRDSET *st) { +static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) { + RRDHOST *host = st->rrdhost; + rrdset_flag_set(st, RRDSET_FLAG_EXPOSED_UPSTREAM); - buffer_sprintf(st->rrdhost->rrdpush_buffer, "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s\"\n" - , st->id - , st->name - , st->title - , st->units - , st->family - , st->context - , rrdset_type_name(st->chart_type) - , st->priority - , st->update_every - , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":"" - , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":"" - , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":"" + // send the chart + buffer_sprintf( + host->rrdpush_sender_buffer + , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s\" \"%s\" \"%s\"\n" + , st->id + , st->name + , st->title + , st->units + , st->family + , st->context + , rrdset_type_name(st->chart_type) + , st->priority + , st->update_every + , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":"" + , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":"" + , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":"" + , (st->plugin_name)?st->plugin_name:"" + , (st->module_name)?st->module_name:"" ); + // send the dimensions RRDDIM *rd; rrddim_foreach_read(rd, st) { - buffer_sprintf(st->rrdhost->rrdpush_buffer, "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s\"\n" - , rd->id - , rd->name - , rrd_algorithm_name(rd->algorithm) - , rd->multiplier - , rd->divisor - , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":"" - , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":"" + buffer_sprintf( + host->rrdpush_sender_buffer + , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s\"\n" + , rd->id + , rd->name + , rrd_algorithm_name(rd->algorithm) + , rd->multiplier + , rd->divisor + , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":"" + , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":"" ); rd->exposed = 1; } + // send the chart local custom variables + RRDSETVAR *rs; + for(rs = st->variables; rs ;rs = rs->next) { + if(unlikely(rs->options && RRDVAR_OPTION_ALLOCATED)) { + calculated_number *value = (calculated_number *) rs->value; + + buffer_sprintf( + host->rrdpush_sender_buffer + , "VARIABLE CHART %s = " CALCULATED_NUMBER_FORMAT "\n" + , rs->variable + , *value + ); + } + } + st->upstream_resync_time = st->last_collected_time.tv_sec + (remote_clock_resync_iterations * st->update_every); } // sends the current chart dimensions -static inline void send_chart_metrics(RRDSET *st) { - buffer_sprintf(st->rrdhost->rrdpush_buffer, "BEGIN \"%s\" %llu\n", st->id, (st->upstream_resync_time > st->last_collected_time.tv_sec)?st->usec_since_last_update:0); +static inline void rrdpush_send_chart_metrics_nolock(RRDSET *st) { + buffer_sprintf(st->rrdhost->rrdpush_sender_buffer, "BEGIN \"%s\" %llu\n", st->id, (st->upstream_resync_time > st->last_collected_time.tv_sec)?st->usec_since_last_update:0); RRDDIM *rd; rrddim_foreach_read(rd, st) { if(rd->updated && rd->exposed) - buffer_sprintf(st->rrdhost->rrdpush_buffer, "SET \"%s\" = " COLLECTED_NUMBER_FORMAT "\n" - , rd->id - , rd->collected_value + buffer_sprintf(st->rrdhost->rrdpush_sender_buffer + , "SET \"%s\" = " COLLECTED_NUMBER_FORMAT "\n" + , rd->id + , rd->collected_value ); } - buffer_strcat(st->rrdhost->rrdpush_buffer, "END\n"); + buffer_strcat(st->rrdhost->rrdpush_sender_buffer, "END\n"); } static void rrdpush_sender_thread_spawn(RRDHOST *host); @@ -133,9 +161,9 @@ void rrdset_push_chart_definition(RRDSET *st) { RRDHOST *host = st->rrdhost; rrdset_rdlock(st); - rrdpush_lock(host); - send_chart_definition(st); - rrdpush_unlock(host); + rrdpush_buffer_lock(host); + rrdpush_send_chart_definition_nolock(st); + rrdpush_buffer_unlock(host); rrdset_unlock(st); } @@ -145,40 +173,81 @@ void rrdset_done_push(RRDSET *st) { if(unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ENABLED))) return; - rrdpush_lock(host); + rrdpush_buffer_lock(host); - if(unlikely(host->rrdpush_enabled && !host->rrdpush_spawn)) + if(unlikely(host->rrdpush_send_enabled && !host->rrdpush_sender_spawn)) rrdpush_sender_thread_spawn(host); - if(unlikely(!host->rrdpush_buffer || !host->rrdpush_connected)) { - if(unlikely(!host->rrdpush_error_shown)) + if(unlikely(!host->rrdpush_sender_buffer || !host->rrdpush_sender_connected)) { + if(unlikely(!host->rrdpush_sender_error_shown)) error("STREAM %s [send]: not ready - discarding collected metrics.", host->hostname); - host->rrdpush_error_shown = 1; + host->rrdpush_sender_error_shown = 1; - rrdpush_unlock(host); + rrdpush_buffer_unlock(host); return; } - else if(unlikely(host->rrdpush_error_shown)) { - info("STREAM %s [send]: ready - sending metrics...", host->hostname); - host->rrdpush_error_shown = 0; + else if(unlikely(host->rrdpush_sender_error_shown)) { + info("STREAM %s [send]: sending metrics...", host->hostname); + host->rrdpush_sender_error_shown = 0; } if(need_to_send_chart_definition(st)) - send_chart_definition(st); + rrdpush_send_chart_definition_nolock(st); - send_chart_metrics(st); + rrdpush_send_chart_metrics_nolock(st); // signal the sender there are more data - if(write(host->rrdpush_pipe[PIPE_WRITE], " ", 1) == -1) + if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1) error("STREAM %s [send]: cannot write to internal pipe", host->hostname); - rrdpush_unlock(host); + rrdpush_buffer_unlock(host); } // ---------------------------------------------------------------------------- // rrdpush sender thread +static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, RRDVAR *rv) { + calculated_number *value = (calculated_number *)rv->value; + + buffer_sprintf( + host->rrdpush_sender_buffer + , "VARIABLE HOST %s = " CALCULATED_NUMBER_FORMAT "\n" + , rv->name + , *value + ); + + debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " CALCULATED_NUMBER_FORMAT, rv->name, *value); +} + +void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) { + if(host->rrdpush_send_enabled && host->rrdpush_sender_spawn && host->rrdpush_sender_connected) { + rrdpush_buffer_lock(host); + rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv); + rrdpush_buffer_unlock(host); + } +} + +static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr, void *host_ptr) { + RRDVAR *rv = (RRDVAR *)rrdvar_ptr; + RRDHOST *host = (RRDHOST *)host_ptr; + + if(unlikely(rv->type == RRDVAR_TYPE_CALCULATED_ALLOCATED)) { + rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv); + + // return 1, so that the traversal will return the number of variables sent + return 1; + } + + // returning a negative number will break the traversal + return 0; +} + +static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) { + int ret = rrdvar_callback_for_all_host_variables(host, rrdpush_sender_thread_custom_host_variables_callback, host); + debug(D_STREAM, "RRDVAR sent %d VARIABLES", ret); +} + // resets all the chart, so that their definitions // will be resent to the central netdata static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) { @@ -186,6 +255,7 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) { RRDSET *st; rrdset_foreach_read(st, host) { + rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM); st->upstream_resync_time = 0; @@ -202,68 +272,198 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) { } static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) { - rrdpush_lock(host); + rrdpush_buffer_lock(host); - if(buffer_strlen(host->rrdpush_buffer)) - error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", host->hostname, buffer_strlen(host->rrdpush_buffer)); + if(buffer_strlen(host->rrdpush_sender_buffer)) + error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", host->hostname, buffer_strlen(host->rrdpush_sender_buffer)); - buffer_flush(host->rrdpush_buffer); + buffer_flush(host->rrdpush_sender_buffer); rrdpush_sender_thread_reset_all_charts(host); + rrdpush_sender_thread_send_custom_host_variables(host); - rrdpush_unlock(host); + rrdpush_buffer_unlock(host); } -static void rrdpush_sender_thread_cleanup_locked_all(RRDHOST *host) { - host->rrdpush_connected = 0; +void rrdpush_sender_thread_stop(RRDHOST *host) { + rrdpush_buffer_lock(host); + rrdhost_wrlock(host); + + pthread_t thr = 0; - if(host->rrdpush_socket != -1) { - close(host->rrdpush_socket); - host->rrdpush_socket = -1; + if(host->rrdpush_sender_spawn) { + info("STREAM %s [send]: signaling sending thread to stop...", host->hostname); + + // signal the thread that we want to join it + host->rrdpush_sender_join = 1; + + // copy the thread id, so that we will be waiting for the right one + // even if a new one has been spawn + thr = host->rrdpush_sender_thread; + + // signal it to cancel + int ret = pthread_cancel(host->rrdpush_sender_thread); + if(ret != 0) + error("STREAM %s [send]: pthread_cancel() returned error.", host->hostname); } - // close the pipe - if(host->rrdpush_pipe[PIPE_READ] != -1) { - close(host->rrdpush_pipe[PIPE_READ]); - host->rrdpush_pipe[PIPE_READ] = -1; + rrdhost_unlock(host); + rrdpush_buffer_unlock(host); + + if(thr != 0) { + info("STREAM %s [send]: waiting for the sending thread to stop...", host->hostname); + + void *result; + int ret = pthread_join(thr, &result); + if(ret != 0) + error("STREAM %s [send]: pthread_join() returned error.", host->hostname); + + info("STREAM %s [send]: sending thread has exited.", host->hostname); + } +} + +static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) { + host->rrdpush_sender_connected = 0; + + if(host->rrdpush_sender_socket != -1) { + close(host->rrdpush_sender_socket); + host->rrdpush_sender_socket = -1; + } +} + +static int rrdpush_sender_thread_connect_to_master(RRDHOST *host, int default_port, int timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) { + struct timeval tv = { + .tv_sec = timeout, + .tv_usec = 0 + }; + + // make sure the socket is closed + rrdpush_sender_thread_close_socket(host); + + debug(D_STREAM, "STREAM: Attempting to connect..."); + info("STREAM %s [send to %s]: connecting...", host->hostname, host->rrdpush_send_destination); + + host->rrdpush_sender_socket = connect_to_one_of( + host->rrdpush_send_destination + , default_port + , &tv + , reconnects_counter + , connected_to + , connected_to_size + ); + + if(unlikely(host->rrdpush_sender_socket == -1)) { + error("STREAM %s [send to %s]: failed to connect", host->hostname, host->rrdpush_send_destination); + return 0; + } + + info("STREAM %s [send to %s]: initializing communication...", host->hostname, connected_to); + + #define HTTP_HEADER_SIZE 8192 + char http[HTTP_HEADER_SIZE + 1]; + snprintfz(http, HTTP_HEADER_SIZE, + "STREAM key=%s&hostname=%s®istry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&timezone=%s&tags=%s HTTP/1.1\r\n" + "User-Agent: netdata-push-service/%s\r\n" + "Accept: */*\r\n\r\n" + , host->rrdpush_send_api_key + , host->hostname + , host->registry_hostname + , host->machine_guid + , default_rrd_update_every + , host->os + , host->timezone + , (host->tags)?host->tags:"" + , program_version + ); + + if(send_timeout(host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) { + error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", host->hostname, connected_to); + rrdpush_sender_thread_close_socket(host); + return 0; } - if(host->rrdpush_pipe[PIPE_WRITE] != -1) { - close(host->rrdpush_pipe[PIPE_WRITE]); - host->rrdpush_pipe[PIPE_WRITE] = -1; + info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, connected_to); + + if(recv_timeout(host->rrdpush_sender_socket, http, HTTP_HEADER_SIZE, 0, timeout) == -1) { + error("STREAM %s [send to %s]: remote netdata does not respond.", host->hostname, connected_to); + rrdpush_sender_thread_close_socket(host); + return 0; + } + + if(strncmp(http, START_STREAMING_PROMPT, strlen(START_STREAMING_PROMPT)) != 0) { + error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", host->hostname, connected_to); + rrdpush_sender_thread_close_socket(host); + return 0; } - buffer_free(host->rrdpush_buffer); - host->rrdpush_buffer = NULL; + info("STREAM %s [send to %s]: established communication - ready to send metrics...", host->hostname, connected_to); - host->rrdpush_spawn = 0; + if(sock_setnonblock(host->rrdpush_sender_socket) < 0) + error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, connected_to); + + if(sock_enlarge_out(host->rrdpush_sender_socket) < 0) + error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, connected_to); + + debug(D_STREAM, "STREAM: Connected on fd %d...", host->rrdpush_sender_socket); + + return 1; } -void rrdpush_sender_thread_stop(RRDHOST *host) { - rrdpush_lock(host); +static void rrdpush_sender_thread_cleanup_callback(void *ptr) { + RRDHOST *host = (RRDHOST *)ptr; + + rrdpush_buffer_lock(host); rrdhost_wrlock(host); - if(host->rrdpush_spawn) { - info("STREAM %s [send]: stopping sending thread...", host->hostname); - pthread_cancel(host->rrdpush_thread); - rrdpush_sender_thread_cleanup_locked_all(host); + info("STREAM %s [send]: sending thread cleans up...", host->hostname); + + rrdpush_sender_thread_close_socket(host); + + // close the pipe + if(host->rrdpush_sender_pipe[PIPE_READ] != -1) { + close(host->rrdpush_sender_pipe[PIPE_READ]); + host->rrdpush_sender_pipe[PIPE_READ] = -1; + } + + if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1) { + close(host->rrdpush_sender_pipe[PIPE_WRITE]); + host->rrdpush_sender_pipe[PIPE_WRITE] = -1; } + buffer_free(host->rrdpush_sender_buffer); + host->rrdpush_sender_buffer = NULL; + + if(!host->rrdpush_sender_join) { + info("STREAM %s [send]: sending thread detaches itself.", host->hostname); + if(pthread_detach(pthread_self())) + error("STREAM %s [send]: pthread_detach() failed.", host->hostname); + } + + host->rrdpush_sender_spawn = 0; + + info("STREAM %s [send]: sending thread now exits.", host->hostname); + rrdhost_unlock(host); - rrdpush_unlock(host); + rrdpush_buffer_unlock(host); } void *rrdpush_sender_thread(void *ptr) { RRDHOST *host = (RRDHOST *)ptr; - info("STREAM %s [send]: thread created (task id %d)", host->hostname, gettid()); + if(!host->rrdpush_send_enabled || !host->rrdpush_send_destination || !*host->rrdpush_send_destination || !host->rrdpush_send_api_key || !*host->rrdpush_send_api_key) { + error("STREAM %s [send]: thread created (task id %d), but host has streaming disabled.", host->hostname, gettid()); + pthread_exit(NULL); + return NULL; + } - if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) - error("STREAM %s [send]: cannot set pthread cancel type to DEFERRED.", host->hostname); + info("STREAM %s [send]: thread created (task id %d)", host->hostname, gettid()); if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname); + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("STREAM %s [send]: cannot set pthread cancel type to DEFERRED.", host->hostname); + int timeout = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 60); int default_port = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "default port", 19999); size_t max_size = (size_t)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "buffer size bytes", 1024 * 1024); @@ -271,24 +471,17 @@ void *rrdpush_sender_thread(void *ptr) { remote_clock_resync_iterations = (unsigned int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "initial clock resync iterations", remote_clock_resync_iterations); char connected_to[CONNECTED_TO_SIZE + 1] = ""; - if(!host->rrdpush_enabled || !host->rrdpush_destination || !*host->rrdpush_destination || !host->rrdpush_api_key || !*host->rrdpush_api_key) - goto cleanup; - // initialize rrdpush globals - host->rrdpush_buffer = buffer_create(1); - host->rrdpush_connected = 0; - if(pipe(host->rrdpush_pipe) == -1) fatal("STREAM %s [send]: cannot create required pipe.", host->hostname); + host->rrdpush_sender_buffer = buffer_create(1); + host->rrdpush_sender_connected = 0; + if(pipe(host->rrdpush_sender_pipe) == -1) fatal("STREAM %s [send]: cannot create required pipe.", host->hostname); // initialize local variables size_t begin = 0; size_t reconnects_counter = 0; size_t sent_bytes = 0; - size_t sent_connection = 0; + size_t sent_bytes_on_this_connection = 0; - struct timeval tv = { - .tv_sec = timeout, - .tv_usec = 0 - }; time_t last_sent_t = 0; struct pollfd fds[2], *ifd, *ofd; @@ -297,260 +490,206 @@ void *rrdpush_sender_thread(void *ptr) { ifd = &fds[0]; ofd = &fds[1]; - for(; host->rrdpush_enabled && !netdata_exit ;) { - debug(D_STREAM, "STREAM: Checking if we need to timeout the connection..."); - if(host->rrdpush_socket != -1 && now_monotonic_sec() - last_sent_t > timeout) { - error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, timeout, sent_connection); - close(host->rrdpush_socket); - host->rrdpush_socket = -1; - } + size_t not_connected_loops = 0; - if(unlikely(host->rrdpush_socket == -1)) { - debug(D_STREAM, "STREAM: Attempting to connect..."); + pthread_cleanup_push(rrdpush_sender_thread_cleanup_callback, host); - // stop appending data into rrdpush_buffer - // they will be lost, so there is no point to do it - host->rrdpush_connected = 0; + for(; host->rrdpush_send_enabled && !netdata_exit ;) { + // check for outstanding cancellation requests + pthread_testcancel(); - info("STREAM %s [send to %s]: connecting...", host->hostname, host->rrdpush_destination); - host->rrdpush_socket = connect_to_one_of(host->rrdpush_destination, default_port, &tv, &reconnects_counter, connected_to, CONNECTED_TO_SIZE); + // if we don't have socket open, lets wait a bit + if(unlikely(host->rrdpush_sender_socket == -1)) { + if(not_connected_loops == 0 && sent_bytes_on_this_connection > 0) { + // fast re-connection on first disconnect + sleep_usec(USEC_PER_MS * 500); // milliseconds + } + else { + // slow re-connection on repeating errors + sleep_usec(USEC_PER_SEC * reconnect_delay); // seconds + } - if(unlikely(host->rrdpush_socket == -1)) { - error("STREAM %s [send to %s]: failed to connect", host->hostname, host->rrdpush_destination); - sleep(reconnect_delay); - continue; - } + if(rrdpush_sender_thread_connect_to_master(host, default_port, timeout, &reconnects_counter, connected_to, CONNECTED_TO_SIZE)) { + last_sent_t = now_monotonic_sec(); - info("STREAM %s [send to %s]: initializing communication...", host->hostname, connected_to); + // reset the buffer, to properly send charts and metrics + rrdpush_sender_thread_data_flush(host); - #define HTTP_HEADER_SIZE 8192 - char http[HTTP_HEADER_SIZE + 1]; - snprintfz(http, HTTP_HEADER_SIZE, - "STREAM key=%s&hostname=%s®istry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&tags=%s HTTP/1.1\r\n" - "User-Agent: netdata-push-service/%s\r\n" - "Accept: */*\r\n\r\n" - , host->rrdpush_api_key - , host->hostname - , host->registry_hostname - , host->machine_guid - , default_rrd_update_every - , host->os - , (host->tags)?host->tags:"" - , program_version - ); + // make sure the next reconnection will be immediate + not_connected_loops = 0; - if(send_timeout(host->rrdpush_socket, http, strlen(http), 0, timeout) == -1) { - close(host->rrdpush_socket); - host->rrdpush_socket = -1; - error("STREAM %s [send to %s]: failed to send http header to netdata", host->hostname, connected_to); - sleep(reconnect_delay); - continue; - } + // reset the bytes we have sent for this session + sent_bytes_on_this_connection = 0; - info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, connected_to); + // let the data collection threads know we are ready + host->rrdpush_sender_connected = 1; + } + else { + // increase the failed connections counter + not_connected_loops++; - if(recv_timeout(host->rrdpush_socket, http, HTTP_HEADER_SIZE, 0, timeout) == -1) { - close(host->rrdpush_socket); - host->rrdpush_socket = -1; - error("STREAM %s [send to %s]: failed to initialize communication", host->hostname, connected_to); - sleep(reconnect_delay); - continue; - } + // reset the number of bytes sent + sent_bytes_on_this_connection = 0; + } - if(strncmp(http, START_STREAMING_PROMPT, strlen(START_STREAMING_PROMPT))) { - close(host->rrdpush_socket); - host->rrdpush_socket = -1; - error("STREAM %s [send to %s]: server is not replying properly.", host->hostname, connected_to); - sleep(reconnect_delay); + // loop through continue; } - - info("STREAM %s [send to %s]: established communication - sending metrics...", host->hostname, connected_to); - last_sent_t = now_monotonic_sec(); - - if(sock_setnonblock(host->rrdpush_socket) < 0) - error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, connected_to); - - if(sock_enlarge_out(host->rrdpush_socket) < 0) - error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, connected_to); - - rrdpush_sender_thread_data_flush(host); - sent_connection = 0; - - // allow appending data into rrdpush_buffer - host->rrdpush_connected = 1; - - debug(D_STREAM, "STREAM: Connected on fd %d...", host->rrdpush_socket); - } - - ifd->fd = host->rrdpush_pipe[PIPE_READ]; - ifd->events = POLLIN; - ifd->revents = 0; - - ofd->fd = host->rrdpush_socket; - ofd->revents = 0; - if(ofd->fd != -1 && begin < buffer_strlen(host->rrdpush_buffer)) { - debug(D_STREAM, "STREAM: Requesting data output on streaming socket %d...", ofd->fd); - ofd->events = POLLOUT; - fdmax = 2; - } - else { - debug(D_STREAM, "STREAM: Not requesting data output on streaming socket %d (nothing to send now)...", ofd->fd); - ofd->events = 0; - fdmax = 1; - } - - debug(D_STREAM, "STREAM: Waiting for poll() events (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer)); - if(netdata_exit) break; - int retval = poll(fds, fdmax, 1000); - if(netdata_exit) break; - - if(unlikely(retval == -1)) { - debug(D_STREAM, "STREAM: poll() failed (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer)); - - if(errno == EAGAIN || errno == EINTR) { - debug(D_STREAM, "STREAM: poll() failed with EAGAIN or EINTR..."); - continue; + else if(unlikely(now_monotonic_sec() - last_sent_t > timeout)) { + error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, timeout, sent_bytes_on_this_connection); + rrdpush_sender_thread_close_socket(host); } - error("STREAM %s [send to %s]: failed to poll().", host->hostname, connected_to); - close(host->rrdpush_socket); - host->rrdpush_socket = -1; - break; - } - else if(likely(retval)) { - if (ifd->revents & POLLIN || ifd->revents & POLLPRI) { - debug(D_STREAM, "STREAM: Data added to send buffer (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer)); + ifd->fd = host->rrdpush_sender_pipe[PIPE_READ]; + ifd->events = POLLIN; + ifd->revents = 0; - char buffer[1000 + 1]; - if (read(host->rrdpush_pipe[PIPE_READ], buffer, 1000) == -1) - error("STREAM %s [send to %s]: cannot read from internal pipe.", host->hostname, connected_to); + ofd->fd = host->rrdpush_sender_socket; + ofd->revents = 0; + if(ofd->fd != -1 && begin < buffer_strlen(host->rrdpush_sender_buffer)) { + debug(D_STREAM, "STREAM: Requesting data output on streaming socket %d...", ofd->fd); + ofd->events = POLLOUT; + fdmax = 2; + } + else { + debug(D_STREAM, "STREAM: Not requesting data output on streaming socket %d (nothing to send now)...", ofd->fd); + ofd->events = 0; + fdmax = 1; } - if (ofd->revents & POLLOUT) { - if (begin < buffer_strlen(host->rrdpush_buffer)) { - debug(D_STREAM, "STREAM: Sending data (current buffer length %zu bytes, begin = %zu)...", buffer_strlen(host->rrdpush_buffer), begin); + debug(D_STREAM, "STREAM: Waiting for poll() events (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_sender_buffer)); + if(unlikely(netdata_exit)) break; + int retval = poll(fds, fdmax, 1000); + if(unlikely(netdata_exit)) break; - // BEGIN RRDPUSH LOCKED SESSION + if(unlikely(retval == -1)) { + debug(D_STREAM, "STREAM: poll() failed (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_sender_buffer)); - // during this session, data collectors - // will not be able to append data to our buffer - // but the socket is in non-blocking mode - // so, we will not block at send() + if(errno == EAGAIN || errno == EINTR) { + debug(D_STREAM, "STREAM: poll() failed with EAGAIN or EINTR..."); + } + else { + error("STREAM %s [send to %s]: failed to poll(). Closing socket.", host->hostname, connected_to); + rrdpush_sender_thread_close_socket(host); + } - if (pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) - error("STREAM %s [send]: cannot set pthread cancel state to DISABLE.", host->hostname); + continue; + } + else if(likely(retval)) { + if (ifd->revents & POLLIN || ifd->revents & POLLPRI) { + debug(D_STREAM, "STREAM: Data added to send buffer (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_sender_buffer)); - debug(D_STREAM, "STREAM: Getting exclusive lock on host..."); - rrdpush_lock(host); + char buffer[1000 + 1]; + if (read(host->rrdpush_sender_pipe[PIPE_READ], buffer, 1000) == -1) + error("STREAM %s [send to %s]: cannot read from internal pipe.", host->hostname, connected_to); + } - debug(D_STREAM, "STREAM: Sending data, starting from %zu, size %zu...", begin, buffer_strlen(host->rrdpush_buffer)); - ssize_t ret = send(host->rrdpush_socket, &host->rrdpush_buffer->buffer[begin], buffer_strlen(host->rrdpush_buffer) - begin, MSG_DONTWAIT); - if (unlikely(ret == -1)) { - if (errno != EAGAIN && errno != EINTR && errno != EWOULDBLOCK) { - debug(D_STREAM, "STREAM: Send failed - closing socket..."); - error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection); - close(host->rrdpush_socket); - host->rrdpush_socket = -1; - } - else { - debug(D_STREAM, "STREAM: Send failed - will retry..."); + if (ofd->revents & POLLOUT) { + if (begin < buffer_strlen(host->rrdpush_sender_buffer)) { + debug(D_STREAM, "STREAM: Sending data (current buffer length %zu bytes, begin = %zu)...", buffer_strlen(host->rrdpush_sender_buffer), begin); + + // BEGIN RRDPUSH LOCKED SESSION + + // during this session, data collectors + // will not be able to append data to our buffer + // but the socket is in non-blocking mode + // so, we will not block at send() + + if (pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) + error("STREAM %s [send]: cannot set pthread cancel state to DISABLE.", host->hostname); + + debug(D_STREAM, "STREAM: Getting exclusive lock on host..."); + rrdpush_buffer_lock(host); + + debug(D_STREAM, "STREAM: Sending data, starting from %zu, size %zu...", begin, buffer_strlen(host->rrdpush_sender_buffer)); + ssize_t ret = send(host->rrdpush_sender_socket, &host->rrdpush_sender_buffer->buffer[begin], buffer_strlen(host->rrdpush_sender_buffer) - begin, MSG_DONTWAIT); + if (unlikely(ret == -1)) { + if (errno != EAGAIN && errno != EINTR && errno != EWOULDBLOCK) { + debug(D_STREAM, "STREAM: Send failed - closing socket..."); + error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_bytes_on_this_connection); + rrdpush_sender_thread_close_socket(host); + } + else { + debug(D_STREAM, "STREAM: Send failed - will retry..."); + } } - } - else if (likely(ret > 0)) { - // DEBUG - dump the scring to see it - //char c = host->rrdpush_buffer->buffer[begin + ret]; - //host->rrdpush_buffer->buffer[begin + ret] = '\0'; - //debug(D_STREAM, "STREAM: sent from %zu to %zd:\n%s\n", begin, ret, &host->rrdpush_buffer->buffer[begin]); - //host->rrdpush_buffer->buffer[begin + ret] = c; - - sent_connection += ret; - sent_bytes += ret; - begin += ret; - - if (begin == buffer_strlen(host->rrdpush_buffer)) { - // we send it all - - debug(D_STREAM, "STREAM: Sent %zd bytes (the whole buffer)...", ret); - buffer_flush(host->rrdpush_buffer); - begin = 0; + else if (likely(ret > 0)) { + // DEBUG - dump the string to see it + //char c = host->rrdpush_sender_buffer->buffer[begin + ret]; + //host->rrdpush_sender_buffer->buffer[begin + ret] = '\0'; + //debug(D_STREAM, "STREAM: sent from %zu to %zd:\n%s\n", begin, ret, &host->rrdpush_sender_buffer->buffer[begin]); + //host->rrdpush_sender_buffer->buffer[begin + ret] = c; + + sent_bytes_on_this_connection += ret; + sent_bytes += ret; + begin += ret; + + if (begin == buffer_strlen(host->rrdpush_sender_buffer)) { + // we send it all + + debug(D_STREAM, "STREAM: Sent %zd bytes (the whole buffer)...", ret); + buffer_flush(host->rrdpush_sender_buffer); + begin = 0; + } + else { + debug(D_STREAM, "STREAM: Sent %zd bytes (part of the data buffer)...", ret); + } + + last_sent_t = now_monotonic_sec(); } else { - debug(D_STREAM, "STREAM: Sent %zd bytes (part of the data buffer)...", ret); + debug(D_STREAM, "STREAM: send() returned %zd - closing the socket...", ret); + error("STREAM %s [send to %s]: failed to send metrics (send() returned %zd) - closing connection - we have sent %zu bytes on this connection.", + host->hostname, connected_to, ret, sent_bytes_on_this_connection); + rrdpush_sender_thread_close_socket(host); } - last_sent_t = now_monotonic_sec(); + debug(D_STREAM, "STREAM: Releasing exclusive lock on host..."); + rrdpush_buffer_unlock(host); + + if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname); + + // END RRDPUSH LOCKED SESSION } else { - debug(D_STREAM, "STREAM: send() returned %zd - closing the socket...", ret); - error("STREAM %s [send to %s]: failed to send metrics (send() returned %zd) - closing connection - we have sent %zu bytes on this connection.", - host->hostname, connected_to, ret, sent_connection); - close(host->rrdpush_socket); - host->rrdpush_socket = -1; + debug(D_STREAM, "STREAM: we have sent the entire buffer, but we received POLLOUT..."); } - - debug(D_STREAM, "STREAM: Releasing exclusive lock on host..."); - rrdpush_unlock(host); - - if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) - error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname); - - // END RRDPUSH LOCKED SESSION - } - else { - debug(D_STREAM, "STREAM: we have sent the entire buffer, but we received POLLOUT..."); } - } - if(unlikely(ofd->revents & POLLERR)) { - debug(D_STREAM, "STREAM: Send failed (POLLERR) - closing socket..."); - error("STREAM %s [send to %s]: connection reports errors (POLLERR), closing it - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection); - close(host->rrdpush_socket); - host->rrdpush_socket = -1; - } - else if(unlikely(ofd->revents & POLLHUP)) { - debug(D_STREAM, "STREAM: Send failed (POLLHUP) - closing socket..."); - error("STREAM %s [send to %s]: connection closed by remote end (POLLHUP) - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection); - close(host->rrdpush_socket); - host->rrdpush_socket = -1; + if(host->rrdpush_sender_socket != -1) { + char *error = NULL; + + if (unlikely(ofd->revents & POLLERR)) + error = "socket reports errors (POLLERR)"; + + else if (unlikely(ofd->revents & POLLHUP)) + error = "connection closed by remote end (POLLHUP)"; + + else if (unlikely(ofd->revents & POLLNVAL)) + error = "connection is invalid (POLLNVAL)"; + + if(unlikely(error)) { + debug(D_STREAM, "STREAM: %s - closing socket...", error); + error("STREAM %s [send to %s]: %s - reopening socket - we have sent %zu bytes on this connection.", host->hostname, connected_to, error, sent_bytes_on_this_connection); + rrdpush_sender_thread_close_socket(host); + } + } } - else if(unlikely(ofd->revents & POLLNVAL)) { - debug(D_STREAM, "STREAM: Send failed (POLLNVAL) - closing socket..."); - error("STREAM %s [send to %s]: connection is invalid (POLLNVAL), closing it - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection); - close(host->rrdpush_socket); - host->rrdpush_socket = -1; + else { + debug(D_STREAM, "STREAM: poll() timed out."); } - } - else { - debug(D_STREAM, "STREAM: poll() timed out."); - } - // protection from overflow - if(buffer_strlen(host->rrdpush_buffer) > max_size) { - debug(D_STREAM, "STREAM: Buffer is too big (%zu bytes), bigger than the max (%zu) - flushing it...", buffer_strlen(host->rrdpush_buffer), max_size); - errno = 0; - error("STREAM %s [send to %s]: too many data pending - buffer is %zu bytes long, %zu unsent - we have sent %zu bytes in total, %zu on this connection. Closing connection to flush the data.", host->hostname, connected_to, host->rrdpush_buffer->len, host->rrdpush_buffer->len - begin, sent_bytes, sent_connection); - if(host->rrdpush_socket != -1) { - close(host->rrdpush_socket); - host->rrdpush_socket = -1; + // protection from overflow + if(buffer_strlen(host->rrdpush_sender_buffer) > max_size) { + debug(D_STREAM, "STREAM: Buffer is too big (%zu bytes), bigger than the max (%zu) - flushing it...", buffer_strlen(host->rrdpush_sender_buffer), max_size); + errno = 0; + error("STREAM %s [send to %s]: too many data pending - buffer is %zu bytes long, %zu unsent - we have sent %zu bytes in total, %zu on this connection. Closing connection to flush the data.", host->hostname, connected_to, host->rrdpush_sender_buffer->len, host->rrdpush_sender_buffer->len - begin, sent_bytes, sent_bytes_on_this_connection); + rrdpush_sender_thread_close_socket(host); } } - } - -cleanup: - debug(D_WEB_CLIENT, "STREAM %s [send]: sending thread exits.", host->hostname); - - if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) - error("STREAM %s [send]: cannot set pthread cancel state to DISABLE.", host->hostname); - - rrdpush_lock(host); - rrdhost_wrlock(host); - rrdpush_sender_thread_cleanup_locked_all(host); - rrdhost_unlock(host); - rrdpush_unlock(host); - if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) - error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname); + pthread_cleanup_pop(1); pthread_exit(NULL); return NULL; @@ -560,7 +699,11 @@ cleanup: // ---------------------------------------------------------------------------- // rrdpush receiver thread -static int rrdpush_receive(int fd, const char *key, const char *hostname, const char *registry_hostname, const char *machine_guid, const char *os, const char *tags, int update_every, char *client_ip, char *client_port) { +static void log_stream_connection(const char *client_ip, const char *client_port, const char *api_key, const char *machine_guid, const char *host, const char *msg) { + log_access("STREAM: %d '[%s]:%s' '%s' host '%s' api key '%s' machine guid '%s'", gettid(), client_ip, client_port, msg, host, api_key, machine_guid); +} + +static int rrdpush_receive(int fd, const char *key, const char *hostname, const char *registry_hostname, const char *machine_guid, const char *os, const char *timezone, const char *tags, int update_every, char *client_ip, char *client_port) { RRDHOST *host; int history = default_rrd_history_entries; RRD_MEMORY_MODE mode = default_rrd_memory_mode; @@ -606,6 +749,7 @@ static int rrdpush_receive(int fd, const char *key, const char *hostname, const , registry_hostname , machine_guid , os + , timezone , tags , update_every , history @@ -618,6 +762,7 @@ static int rrdpush_receive(int fd, const char *key, const char *hostname, const if(!host) { close(fd); + log_stream_connection(client_ip, client_port, key, machine_guid, hostname, "FAILED - CANNOT ACQUIRE HOST"); error("STREAM %s [receive from [%s]:%s]: failed to find/create host structure.", hostname, client_ip, client_port); return 1; } @@ -656,6 +801,7 @@ static int rrdpush_receive(int fd, const char *key, const char *hostname, const info("STREAM %s [receive from [%s]:%s]: initializing communication...", host->hostname, client_ip, client_port); if(send_timeout(fd, START_STREAMING_PROMPT, strlen(START_STREAMING_PROMPT), 0, 60) != strlen(START_STREAMING_PROMPT)) { + log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "FAILED - CANNOT REPLY"); error("STREAM %s [receive from [%s]:%s]: cannot send ready command.", host->hostname, client_ip, client_port); close(fd); return 0; @@ -668,6 +814,7 @@ static int rrdpush_receive(int fd, const char *key, const char *hostname, const // convert the socket to a FILE * FILE *fp = fdopen(fd, "r"); if(!fp) { + log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "FAILED - SOCKET ERROR"); error("STREAM %s [receive from [%s]:%s]: failed to get a FILE for FD %d.", host->hostname, client_ip, client_port, fd); close(fd); return 0; @@ -677,8 +824,9 @@ static int rrdpush_receive(int fd, const char *key, const char *hostname, const if(host->connected_senders > 0) info("STREAM %s [receive from [%s]:%s]: multiple streaming connections for the same host detected. If multiple netdata are pushing metrics for the same charts, at the same time, the result is unexpected.", host->hostname, client_ip, client_port); + rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN); host->connected_senders++; - rrdhost_flag_clear(host, RRDHOST_ORPHAN); + host->senders_disconnected_time = 0; if(health_enabled != CONFIG_BOOLEAN_NO) { if(alarms_delay > 0) { host->health_delay_up_to = now_realtime_sec() + alarms_delay; @@ -692,20 +840,25 @@ static int rrdpush_receive(int fd, const char *key, const char *hostname, const // call the plugins.d processor to receive the metrics info("STREAM %s [receive from [%s]:%s]: receiving metrics...", host->hostname, client_ip, client_port); + log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "CONNECTED"); + size_t count = pluginsd_process(host, &cd, fp, 1); - error("STREAM %s [receive from [%s]:%s]: disconnected (completed updates %zu).", host->hostname, client_ip, client_port, count); + + log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "DISCONNECTED"); + error("STREAM %s [receive from [%s]:%s]: disconnected (completed %zu updates).", host->hostname, client_ip, client_port, count); rrdhost_wrlock(host); host->senders_disconnected_time = now_realtime_sec(); host->connected_senders--; if(!host->connected_senders) { - rrdhost_flag_set(host, RRDHOST_ORPHAN); + rrdhost_flag_set(host, RRDHOST_FLAG_ORPHAN); if(health_enabled == CONFIG_BOOLEAN_AUTO) host->health_enabled = 0; } rrdhost_unlock(host); - rrdpush_sender_thread_stop(host); + if(host->connected_senders == 0) + rrdpush_sender_thread_stop(host); // cleanup fclose(fp); @@ -720,6 +873,7 @@ struct rrdpush_thread { char *registry_hostname; char *machine_guid; char *os; + char *timezone; char *tags; char *client_ip; char *client_port; @@ -737,7 +891,7 @@ static void *rrdpush_receiver_thread(void *ptr) { info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid()); - rrdpush_receive(rpt->fd, rpt->key, rpt->hostname, rpt->registry_hostname, rpt->machine_guid, rpt->os, rpt->tags, rpt->update_every, rpt->client_ip, rpt->client_port); + rrdpush_receive(rpt->fd, rpt->key, rpt->hostname, rpt->registry_hostname, rpt->machine_guid, rpt->os, rpt->timezone, rpt->tags, rpt->update_every, rpt->client_ip, rpt->client_port); info("STREAM %s [receive from [%s]:%s]: receive thread ended (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid()); freez(rpt->key); @@ -745,6 +899,7 @@ static void *rrdpush_receiver_thread(void *ptr) { freez(rpt->registry_hostname); freez(rpt->machine_guid); freez(rpt->os); + freez(rpt->timezone); freez(rpt->tags); freez(rpt->client_ip); freez(rpt->client_port); @@ -757,25 +912,30 @@ static void *rrdpush_receiver_thread(void *ptr) { static void rrdpush_sender_thread_spawn(RRDHOST *host) { rrdhost_wrlock(host); - if(!host->rrdpush_spawn) { - if(pthread_create(&host->rrdpush_thread, NULL, rrdpush_sender_thread, (void *) host)) + if(!host->rrdpush_sender_spawn) { + if(pthread_create(&host->rrdpush_sender_thread, NULL, rrdpush_sender_thread, (void *) host)) error("STREAM %s [send]: failed to create new thread for client.", host->hostname); - - else if(pthread_detach(host->rrdpush_thread)) - error("STREAM %s [send]: cannot request detach newly created thread.", host->hostname); - - host->rrdpush_spawn = 1; + else + host->rrdpush_sender_spawn = 1; } rrdhost_unlock(host); } +int rrdpush_receiver_permission_denied(struct web_client *w) { + // we always respond with the same message and error code + // to prevent an attacker from gaining info about the error + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, "You are not permitted to access this. Check the logs for more info."); + return 401; +} + int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url) { (void)host; info("STREAM [receive from [%s]:%s]: new client connection.", w->client_ip, w->client_port); - char *key = NULL, *hostname = NULL, *registry_hostname = NULL, *machine_guid = NULL, *os = "unknown", *tags = NULL; + char *key = NULL, *hostname = NULL, *registry_hostname = NULL, *machine_guid = NULL, *os = "unknown", *timezone = "unknown", *tags = NULL; int update_every = default_rrd_update_every; char buf[GUID_LEN + 1]; @@ -799,6 +959,8 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url update_every = (int)strtoul(value, NULL, 0); else if(!strcmp(name, "os")) os = value; + else if(!strcmp(name, "timezone")) + timezone = value; else if(!strcmp(name, "tags")) tags = value; else @@ -806,52 +968,71 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url } if(!key || !*key) { + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname)?hostname:"-", "ACCESS DENIED - NO KEY"); error("STREAM [receive from [%s]:%s]: request without an API key. Forbidding access.", w->client_ip, w->client_port); - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "You need an API key for this request."); - return 401; + return rrdpush_receiver_permission_denied(w); } if(!hostname || !*hostname) { + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname)?hostname:"-", "ACCESS DENIED - NO HOSTNAME"); error("STREAM [receive from [%s]:%s]: request without a hostname. Forbidding access.", w->client_ip, w->client_port); - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "You need to send a hostname too."); - return 400; + return rrdpush_receiver_permission_denied(w); } if(!machine_guid || !*machine_guid) { + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname)?hostname:"-", "ACCESS DENIED - NO MACHINE GUID"); error("STREAM [receive from [%s]:%s]: request without a machine GUID. Forbidding access.", w->client_ip, w->client_port); - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "You need to send a machine GUID too."); - return 400; + return rrdpush_receiver_permission_denied(w); } if(regenerate_guid(key, buf) == -1) { + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname)?hostname:"-", "ACCESS DENIED - INVALID KEY"); error("STREAM [receive from [%s]:%s]: API key '%s' is not valid GUID (use the command uuidgen to generate one). Forbidding access.", w->client_ip, w->client_port, key); - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Your API key is invalid."); - return 401; + return rrdpush_receiver_permission_denied(w); } if(regenerate_guid(machine_guid, buf) == -1) { + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname)?hostname:"-", "ACCESS DENIED - INVALID MACHINE GUID"); error("STREAM [receive from [%s]:%s]: machine GUID '%s' is not GUID. Forbidding access.", w->client_ip, w->client_port, machine_guid); - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Your machine GUID is invalid."); - return 404; + return rrdpush_receiver_permission_denied(w); } if(!appconfig_get_boolean(&stream_config, key, "enabled", 0)) { + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname)?hostname:"-", "ACCESS DENIED - KEY NOT ENABLED"); error("STREAM [receive from [%s]:%s]: API key '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, key); - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Your API key is not permitted access."); - return 401; + return rrdpush_receiver_permission_denied(w); + } + + { + SIMPLE_PATTERN *key_allow_from = simple_pattern_create(appconfig_get(&stream_config, key, "allow from", "*"), SIMPLE_PATTERN_EXACT); + if(key_allow_from) { + if(!simple_pattern_matches(key_allow_from, w->client_ip)) { + simple_pattern_free(key_allow_from); + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname) ? hostname : "-", "ACCESS DENIED - KEY NOT ALLOWED FROM THIS IP"); + error("STREAM [receive from [%s]:%s]: API key '%s' is not permitted from this IP. Forbidding access.", w->client_ip, w->client_port, key); + return rrdpush_receiver_permission_denied(w); + } + simple_pattern_free(key_allow_from); + } } if(!appconfig_get_boolean(&stream_config, machine_guid, "enabled", 1)) { + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname)?hostname:"-", "ACCESS DENIED - MACHINE GUID NOT ENABLED"); error("STREAM [receive from [%s]:%s]: machine GUID '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, machine_guid); - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Your machine guide is not permitted access."); - return 404; + return rrdpush_receiver_permission_denied(w); + } + + { + SIMPLE_PATTERN *machine_allow_from = simple_pattern_create(appconfig_get(&stream_config, machine_guid, "allow from", "*"), SIMPLE_PATTERN_EXACT); + if(machine_allow_from) { + if(!simple_pattern_matches(machine_allow_from, w->client_ip)) { + simple_pattern_free(machine_allow_from); + log_stream_connection(w->client_ip, w->client_port, (key && *key)?key:"-", (machine_guid && *machine_guid)?machine_guid:"-", (hostname && *hostname) ? hostname : "-", "ACCESS DENIED - MACHINE GUID NOT ALLOWED FROM THIS IP"); + error("STREAM [receive from [%s]:%s]: Machine GUID '%s' is not permitted from this IP. Forbidding access.", w->client_ip, w->client_port, machine_guid); + return rrdpush_receiver_permission_denied(w); + } + simple_pattern_free(machine_allow_from); + } } struct rrdpush_thread *rpt = mallocz(sizeof(struct rrdpush_thread)); @@ -861,6 +1042,7 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url rpt->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname); rpt->machine_guid = strdupz(machine_guid); rpt->os = strdupz(os); + rpt->timezone = strdupz(timezone); rpt->tags = (tags)?strdupz(tags):NULL; rpt->client_ip = strdupz(w->client_ip); rpt->client_port = strdupz(w->client_port); diff --git a/src/rrdpush.h b/src/rrdpush.h index c3c7f4a54..bf3d9435c 100644 --- a/src/rrdpush.h +++ b/src/rrdpush.h @@ -14,4 +14,6 @@ extern void *rrdpush_sender_thread(void *ptr); extern int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url); extern void rrdpush_sender_thread_stop(RRDHOST *host); +extern void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv); + #endif //NETDATA_RRDPUSH_H diff --git a/src/rrdset.c b/src/rrdset.c index c5168f02e..8504d1cb7 100644 --- a/src/rrdset.c +++ b/src/rrdset.c @@ -135,11 +135,11 @@ char *rrdset_strncpyz_name(char *to, const char *from, size_t length) { return to; } -void rrdset_set_name(RRDSET *st, const char *name) { +int rrdset_set_name(RRDSET *st, const char *name) { if(unlikely(st->name && !strcmp(st->name, name))) - return; + return 1; - debug(D_RRD_CALLS, "rrdset_set_name() old: %s, new: %s", st->name, name); + debug(D_RRD_CALLS, "rrdset_set_name() old: '%s', new: '%s'", st->name?st->name:"", name); char b[CONFIG_MAX_VALUE + 1]; char n[RRD_ID_LENGTH_MAX + 1]; @@ -147,6 +147,11 @@ void rrdset_set_name(RRDSET *st, const char *name) { snprintfz(n, RRD_ID_LENGTH_MAX, "%s.%s", st->type, name); rrdset_strncpyz_name(b, n, CONFIG_MAX_VALUE); + if(rrdset_index_find_name(st->rrdhost, b, 0)) { + error("RRDSET: chart name '%s' on host '%s' already exists.", b, st->rrdhost->hostname); + return 0; + } + if(st->name) { rrdset_index_del_name(st->rrdhost, st); st->name = config_set_default(st->config_section, "name", b); @@ -166,6 +171,8 @@ void rrdset_set_name(RRDSET *st, const char *name) { if(unlikely(rrdset_index_add_name(st->rrdhost, st) != st)) error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name); + + return 1; } inline void rrdset_is_obsolete(RRDSET *st) { @@ -175,7 +182,7 @@ inline void rrdset_is_obsolete(RRDSET *st) { // the chart will not get more updates (data collection) // so, we have to push its definition now - if(unlikely(st->rrdhost->rrdpush_enabled)) + if(unlikely(st->rrdhost->rrdpush_send_enabled)) rrdset_push_chart_definition(st); } } @@ -309,6 +316,9 @@ void rrdset_free(RRDSET *st) { rrdfamily_free(st->rrdhost, st->rrdfamily); + debug(D_RRD_CALLS, "RRDSET: Cleaning up remaining chart variables for host '%s', chart '%s'", st->rrdhost->hostname, st->id); + rrdvar_free_remaining_variables(st->rrdhost, &st->rrdvar_root_index); + // ------------------------------------------------------------------------ // unlink it from the host @@ -334,6 +344,8 @@ void rrdset_free(RRDSET *st) { // free directly allocated members freez(st->config_section); + freez(st->plugin_name); + freez(st->module_name); switch(st->rrd_memory_mode) { case RRD_MEMORY_MODE_SAVE: @@ -416,6 +428,8 @@ RRDSET *rrdset_create_custom( , const char *context , const char *title , const char *units + , const char *plugin + , const char *module , long priority , int update_every , RRDSET_TYPE chart_type @@ -423,12 +437,30 @@ RRDSET *rrdset_create_custom( , long history_entries ) { if(!type || !type[0]) { - fatal("Cannot create rrd stats without a type."); + fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'." + , (id && *id)?id:"" + , (name && *name)?name:"" + , (family && *family)?family:"" + , (context && *context)?context:"" + , (title && *title)?title:"" + , (units && *units)?units:"" + , (plugin && *plugin)?plugin:"" + , (module && *module)?module:"" + ); return NULL; } if(!id || !id[0]) { - fatal("Cannot create rrd stats without an id."); + fatal("Cannot create rrd stats without an id: type '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'." + , type + , (name && *name)?name:"" + , (family && *family)?family:"" + , (context && *context)?context:"" + , (title && *title)?title:"" + , (units && *units)?units:"" + , (plugin && *plugin)?plugin:"" + , (module && *module)?module:"" + ); return NULL; } @@ -495,7 +527,7 @@ RRDSET *rrdset_create_custom( if(st) { memset(&st->avl, 0, sizeof(avl)); memset(&st->avlname, 0, sizeof(avl)); - memset(&st->variables_root_index, 0, sizeof(avl_tree_lock)); + memset(&st->rrdvar_root_index, 0, sizeof(avl_tree_lock)); memset(&st->dimensions_index, 0, sizeof(avl_tree_lock)); memset(&st->rrdset_rwlock, 0, sizeof(netdata_rwlock_t)); @@ -507,6 +539,8 @@ RRDSET *rrdset_create_custom( st->units = NULL; st->context = NULL; st->cache_dir = NULL; + st->plugin_name = NULL; + st->module_name = NULL; st->dimensions = NULL; st->rrdfamily = NULL; st->rrdhost = NULL; @@ -564,6 +598,9 @@ RRDSET *rrdset_create_custom( st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC; } + st->plugin_name = plugin?strdup(plugin):NULL; + st->module_name = module?strdup(module):NULL; + st->config_section = strdup(config_section); st->rrdhost = host; st->memsize = size; @@ -624,12 +661,16 @@ RRDSET *rrdset_create_custom( st->upstream_resync_time = 0; avl_init_lock(&st->dimensions_index, rrddim_compare); - avl_init_lock(&st->variables_root_index, rrdvar_compare); + avl_init_lock(&st->rrdvar_root_index, rrdvar_compare); netdata_rwlock_init(&st->rrdset_rwlock); - if(name && *name) rrdset_set_name(st, name); - else rrdset_set_name(st, id); + if(name && *name && rrdset_set_name(st, name)) + // we did set the name + ; + else + // could not use the name, use the id + rrdset_set_name(st, id); st->title = config_get(st->config_section, "title", title); json_fix_string(st->title); @@ -640,11 +681,11 @@ RRDSET *rrdset_create_custom( host->rrdset_root = st; if(host->health_enabled) { - rrdsetvar_create(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, 0); - rrdsetvar_create(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, 0); - rrdsetvar_create(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, 0); - rrdsetvar_create(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, 0); - rrdsetvar_create(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, 0); + rrdsetvar_create(st, "last_collected_t", RRDVAR_TYPE_TIME_T, &st->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); + rrdsetvar_create(st, "collected_total_raw", RRDVAR_TYPE_TOTAL, &st->last_collected_total, RRDVAR_OPTION_DEFAULT); + rrdsetvar_create(st, "green", RRDVAR_TYPE_CALCULATED, &st->green, RRDVAR_OPTION_DEFAULT); + rrdsetvar_create(st, "red", RRDVAR_TYPE_CALCULATED, &st->red, RRDVAR_OPTION_DEFAULT); + rrdsetvar_create(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, RRDVAR_OPTION_DEFAULT); } if(unlikely(rrdset_index_add(host, st) != st)) @@ -1029,7 +1070,7 @@ void rrdset_done(RRDSET *st) { if(unlikely(netdata_exit)) return; if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) { - if(unlikely(st->rrdhost->rrdpush_enabled)) + if(unlikely(st->rrdhost->rrdpush_send_enabled)) rrdset_done_push_exclusive(st); return; @@ -1154,7 +1195,7 @@ void rrdset_done(RRDSET *st) { } st->counter_done++; - if(unlikely(st->rrdhost->rrdpush_enabled)) + if(unlikely(st->rrdhost->rrdpush_send_enabled)) rrdset_done_push(st); #ifdef NETDATA_INTERNAL_CHECKS diff --git a/src/rrdsetvar.c b/src/rrdsetvar.c index 03d7aeced..280156a8a 100644 --- a/src/rrdsetvar.c +++ b/src/rrdsetvar.c @@ -8,29 +8,29 @@ static inline void rrdsetvar_free_variables(RRDSETVAR *rs) { RRDSET *st = rs->rrdset; + // ------------------------------------------------------------------------ // CHART - - rrdvar_free(st->rrdhost, &st->variables_root_index, rs->var_local); + rrdvar_free(st->rrdhost, &st->rrdvar_root_index, rs->var_local); rs->var_local = NULL; + // ------------------------------------------------------------------------ // FAMILY - - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family); + rrdvar_free(st->rrdhost, &st->rrdfamily->rrdvar_root_index, rs->var_family); rs->var_family = NULL; - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host); - rs->var_host = NULL; + rrdvar_free(st->rrdhost, &st->rrdfamily->rrdvar_root_index, rs->var_family_name); + rs->var_family_name = NULL; + // ------------------------------------------------------------------------ // HOST + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rs->var_host); + rs->var_host = NULL; - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_name); - rs->var_family_name = NULL; - - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_name); + rrdvar_free(st->rrdhost, &st->rrdhost->rrdvar_root_index, rs->var_host_name); rs->var_host_name = NULL; + // ------------------------------------------------------------------------ // KEYS - freez(rs->key_fullid); rs->key_fullid = NULL; @@ -39,10 +39,14 @@ static inline void rrdsetvar_free_variables(RRDSETVAR *rs) { } static inline void rrdsetvar_create_variables(RRDSETVAR *rs) { - rrdsetvar_free_variables(rs); - RRDSET *st = rs->rrdset; + // ------------------------------------------------------------------------ + // free the old ones (if any) + + rrdsetvar_free_variables(rs); + + // ------------------------------------------------------------------------ // KEYS char buffer[RRDVAR_MAX_LENGTH + 1]; @@ -52,27 +56,27 @@ static inline void rrdsetvar_create_variables(RRDSETVAR *rs) { snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable); rs->key_fullname = strdupz(buffer); + // ------------------------------------------------------------------------ // CHART + rs->var_local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->variable, rs->type, rs->value); - rs->var_local = rrdvar_create_and_index("local", &st->variables_root_index, rs->variable, rs->type, rs->value); - + // ------------------------------------------------------------------------ // FAMILY + rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, rs->value); + rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, rs->value); - rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_fullid, rs->type, rs->value); - rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_fullname, rs->type, rs->value); - + // ------------------------------------------------------------------------ // HOST - - rs->var_host = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullid, rs->type, rs->value); - rs->var_host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullname, rs->type, rs->value); - + rs->var_host = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, rs->key_fullid, rs->type, rs->value); + rs->var_host_name = rrdvar_create_and_index("host", &st->rrdhost->rrdvar_root_index, rs->key_fullname, rs->type, rs->value); } -RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options) { +RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options) { debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable); RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR)); rs->variable = strdupz(variable); + rs->hash = simple_hash(rs->variable); rs->type = type; rs->value = value; rs->options = options; @@ -89,11 +93,9 @@ RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *va void rrdsetvar_rename_all(RRDSET *st) { debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name); - RRDSETVAR *rs, *next = st->variables; - while((rs = next)) { - next = rs->next; + RRDSETVAR *rs; + for(rs = st->variables; rs ; rs = rs->next) rrdsetvar_create_variables(rs); - } rrdsetcalc_link_matching(st); } @@ -115,6 +117,63 @@ void rrdsetvar_free(RRDSETVAR *rs) { rrdsetvar_free_variables(rs); freez(rs->variable); + + if(rs->options & RRDVAR_OPTION_ALLOCATED) + freez(rs->value); + freez(rs); } +// -------------------------------------------------------------------------------------------------------------------- +// custom chart variables + +RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name) { + char *n = strdupz(name); + rrdvar_fix_name(n); + uint32_t hash = simple_hash(n); + + rrdset_wrlock(st); + + // find it + RRDSETVAR *rs; + for(rs = st->variables; rs ; rs = rs->next) { + if(hash == rs->hash && strcmp(n, rs->variable) == 0) { + rrdset_unlock(st); + if(rs->options & RRDVAR_OPTION_ALLOCATED) { + free(n); + return rs; + } + else { + error("RRDSETVAR: custom variable '%s' on chart '%s' of host '%s', conflicts with an internal chart variable", n, st->id, st->rrdhost->hostname); + free(n); + return NULL; + } + } + } + + // not found, allocate one + + calculated_number *v = mallocz(sizeof(calculated_number)); + *v = NAN; + + rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED); + rrdset_unlock(st); + + free(n); + return rs; +} + +void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value) { + if(unlikely(!(rs->options & RRDVAR_OPTION_ALLOCATED))) { + error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value); + } + else { + calculated_number *v = rs->value; + if(*v != value) { + *v = value; + + // mark the chart to be sent upstream + rrdset_flag_clear(rs->rrdset, RRDSET_FLAG_EXPOSED_UPSTREAM); + } + } +} diff --git a/src/rrdvar.c b/src/rrdvar.c index 2223d7c9a..9119b5384 100644 --- a/src/rrdvar.c +++ b/src/rrdvar.c @@ -56,14 +56,17 @@ inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) { if(tree) { debug(D_VARIABLES, "Deleting variable '%s'", rv->name); if(unlikely(!rrdvar_index_del(tree, rv))) - error("Attempted to delete variable '%s' from host '%s', but it is not found.", rv->name, host->hostname); + error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rv->name, host->hostname); } + if(rv->type == RRDVAR_TYPE_CALCULATED_ALLOCATED) + freez(rv->value); + freez(rv->name); freez(rv); } -inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, void *value) { +inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, void *value) { char *variable = strdupz(name); rrdvar_fix_name(variable); uint32_t hash = simple_hash(variable); @@ -81,7 +84,8 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, c RRDVAR *ret = rrdvar_index_add(tree, rv); if(unlikely(ret != rv)) { debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", variable, scope); - rrdvar_free(NULL, NULL, rv); + freez(rv); + freez(variable); rv = NULL; } else @@ -101,61 +105,64 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, c return rv; } +void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock) { + // FIXME: this is not bullet proof - avl should support some means to destroy it + // with a callback for each item already in the index + + RRDVAR *rv, *last = NULL; + while((rv = (RRDVAR *)tree_lock->avl_tree.root)) { + if(unlikely(rv == last)) { + error("RRDVAR: INTERNAL ERROR: Cannot cleanup tree of RRDVARs"); + break; + } + last = rv; + rrdvar_free(host, tree_lock, rv); + } +} + // ---------------------------------------------------------------------------- -// CUSTOM VARIABLES +// CUSTOM HOST VARIABLES -RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) { +inline int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data) { + return avl_traverse_lock(&host->rrdvar_root_index, callback, data); +} + +static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *tree_lock, const char *name) { calculated_number *v = callocz(1, sizeof(calculated_number)); *v = NAN; - RRDVAR *rv = rrdvar_create_and_index("host", &host->variables_root_index, name, RRDVAR_TYPE_CALCULATED_ALLOCATED, v); + + RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED_ALLOCATED, v); if(unlikely(!rv)) { free(v); - error("Requested variable '%s' already exists - possibly 2 plugins will be updating it at the same time", name); + debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", name); char *variable = strdupz(name); rrdvar_fix_name(variable); uint32_t hash = simple_hash(variable); - rv = rrdvar_index_find(&host->variables_root_index, variable, hash); + rv = rrdvar_index_find(tree_lock, variable, hash); + + freez(variable); } return rv; } -void rrdvar_custom_host_variable_destroy(RRDHOST *host, const char *name) { - char *variable = strdupz(name); - rrdvar_fix_name(variable); - uint32_t hash = simple_hash(variable); - - RRDVAR *rv = rrdvar_index_find(&host->variables_root_index, variable, hash); - freez(variable); - - if(!rv) { - error("Attempted to remove variable '%s' from host '%s', but it does not exist.", name, host->hostname); - return; - } - - if(rv->type != RRDVAR_TYPE_CALCULATED_ALLOCATED) { - error("Attempted to remove variable '%s' from host '%s', but it does not a custom allocated variable.", name, host->hostname); - return; - } - - if(!rrdvar_index_del(&host->variables_root_index, rv)) { - error("Attempted to remove variable '%s' from host '%s', but it cannot be found.", name, host->hostname); - return; - } - - freez(rv->name); - freez(rv->value); - freez(rv); +RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) { + return rrdvar_custom_variable_create("host", &host->rrdvar_root_index, name); } -void rrdvar_custom_host_variable_set(RRDVAR *rv, calculated_number value) { +void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value) { if(rv->type != RRDVAR_TYPE_CALCULATED_ALLOCATED) error("requested to set variable '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rv->name, value); else { calculated_number *v = rv->value; - *v = value; + if(*v != value) { + *v = value; + + // if the host is streaming, send this variable upstream immediately + rrdpush_sender_send_this_host_variable_now(host, rv); + } } } @@ -191,7 +198,7 @@ static calculated_number rrdvar2number(RRDVAR *rv) { } default: - error("I don't know how to convert RRDVAR type %d to calculated_number", rv->type); + error("I don't know how to convert RRDVAR type %u to calculated_number", rv->type); return NAN; } } @@ -202,19 +209,19 @@ int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, cal if(!st) return 0; - rv = rrdvar_index_find(&st->variables_root_index, variable, hash); + rv = rrdvar_index_find(&st->rrdvar_root_index, variable, hash); if(rv) { *result = rrdvar2number(rv); return 1; } - rv = rrdvar_index_find(&st->rrdfamily->variables_root_index, variable, hash); + rv = rrdvar_index_find(&st->rrdfamily->rrdvar_root_index, variable, hash); if(rv) { *result = rrdvar2number(rv); return 1; } - rv = rrdvar_index_find(&st->rrdhost->variables_root_index, variable, hash); + rv = rrdvar_index_find(&st->rrdhost->rrdvar_root_index, variable, hash); if(rv) { *result = rrdvar2number(rv); return 1; @@ -253,13 +260,13 @@ void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf) { }; buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", st->id, st->name, st->context); - avl_traverse_lock(&st->variables_root_index, single_variable2json, (void *)&helper); + avl_traverse_lock(&st->rrdvar_root_index, single_variable2json, (void *)&helper); buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", st->family); helper.counter = 0; - avl_traverse_lock(&st->rrdfamily->variables_root_index, single_variable2json, (void *)&helper); + avl_traverse_lock(&st->rrdfamily->rrdvar_root_index, single_variable2json, (void *)&helper); buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", st->rrdhost->hostname); helper.counter = 0; - avl_traverse_lock(&st->rrdhost->variables_root_index, single_variable2json, (void *)&helper); + avl_traverse_lock(&st->rrdhost->rrdvar_root_index, single_variable2json, (void *)&helper); buffer_strcat(buf, "\n\t}\n}\n"); } diff --git a/src/signals.h b/src/signals.h index d8611edea..2fdd36552 100644 --- a/src/signals.h +++ b/src/signals.h @@ -4,7 +4,7 @@ extern void signals_init(void); extern void signals_block(void); extern void signals_unblock(void); -extern void signals_handle(void); extern void signals_reset(void); +extern void signals_handle(void) NORETURN; #endif //NETDATA_SIGNALS_H diff --git a/src/simple_pattern.c b/src/simple_pattern.c index f72a42d06..469ea396f 100644 --- a/src/simple_pattern.c +++ b/src/simple_pattern.c @@ -12,12 +12,13 @@ struct simple_pattern { struct simple_pattern *next; }; -static inline struct simple_pattern *parse_pattern(const char *str, SIMPLE_PREFIX_MODE default_mode) { +static inline struct simple_pattern *parse_pattern(char *str, SIMPLE_PREFIX_MODE default_mode) { + // fprintf(stderr, "PARSING PATTERN: '%s'\n", str); + SIMPLE_PREFIX_MODE mode; struct simple_pattern *child = NULL; - char *buf = strdupz(str); - char *s = buf, *c = buf; + char *s = str, *c = str; // skip asterisks in front while(*c == '*') c++; @@ -64,8 +65,6 @@ static inline struct simple_pattern *parse_pattern(const char *str, SIMPLE_PREFI m->child = child; - freez(buf); - return m; } @@ -74,51 +73,63 @@ SIMPLE_PATTERN *simple_pattern_create(const char *list, SIMPLE_PREFIX_MODE defau if(unlikely(!list || !*list)) return root; - char *buf = strdupz(list); - if(buf && *buf) { - char *s = buf; + char *buf = mallocz(strlen(list) + 1); + const char *s = list; - while(s && *s) { - char negative = 0; + while(s && *s) { + buf[0] = '\0'; + char *c = buf; - // skip all spaces - while(isspace(*s)) s++; + char negative = 0; - if(*s == '!') { - negative = 1; - s++; - } + // skip all spaces + while(isspace(*s)) + s++; - // empty string - if(unlikely(!*s)) break; + if(*s == '!') { + negative = 1; + s++; + } - // find the next space - char *c = s; - while(*c && !isspace(*c)) c++; + // empty string + if(unlikely(!*s)) + break; - // find the next word - char *n; - if(likely(*c)) n = c + 1; - else n = NULL; + // find the next space + char escape = 0; + while(*s) { + if(*s == '\\' && !escape) { + escape = 1; + s++; + } + else { + if (isspace(*s) && !escape) { + s++; + break; + } - // terminate our string - *c = '\0'; + *c++ = *s++; + escape = 0; + } + } - struct simple_pattern *m = parse_pattern(s, default_mode); - m->negative = negative; + // terminate our string + *c = '\0'; - if(likely(n)) *c = ' '; + // if we matched the empty string, skip it + if(unlikely(!*buf)) + continue; - // link it at the end - if(unlikely(!root)) - root = last = m; - else { - last->next = m; - last = m; - } + // fprintf(stderr, "FOUND PATTERN: '%s'\n", buf); + struct simple_pattern *m = parse_pattern(buf, default_mode); + m->negative = negative; - // prepare for next loop - s = n; + // link it at the end + if(unlikely(!root)) + root = last = m; + else { + last->next = m; + last = m; } } @@ -126,7 +137,28 @@ SIMPLE_PATTERN *simple_pattern_create(const char *list, SIMPLE_PREFIX_MODE defau return (SIMPLE_PATTERN *)root; } -static inline int match_pattern(struct simple_pattern *m, const char *str, size_t len) { +static inline char *add_wildcarded(const char *matched, size_t matched_size, char *wildcarded, size_t *wildcarded_size) { + //if(matched_size) { + // char buf[matched_size + 1]; + // strncpyz(buf, matched, matched_size); + // fprintf(stderr, "ADD WILDCARDED '%s' of length %zu\n", buf, matched_size); + //} + + if(unlikely(wildcarded && *wildcarded_size && matched && *matched && matched_size)) { + size_t wss = *wildcarded_size - 1; + size_t len = (matched_size < wss)?matched_size:wss; + if(likely(len)) { + strncpyz(wildcarded, matched, len); + + *wildcarded_size -= len; + return &wildcarded[len]; + } + } + + return wildcarded; +} + +static inline int match_pattern(struct simple_pattern *m, const char *str, size_t len, char *wildcarded, size_t *wildcarded_size) { char *s; if(m->len <= len) { @@ -134,20 +166,28 @@ static inline int match_pattern(struct simple_pattern *m, const char *str, size_ case SIMPLE_PATTERN_SUBSTRING: if(!m->len) return 1; if((s = strstr(str, m->match))) { - if(!m->child) return 1; - return match_pattern(m->child, &s[m->len], len - (s - str) - m->len); + wildcarded = add_wildcarded(str, s - str, wildcarded, wildcarded_size); + if(!m->child) { + wildcarded = add_wildcarded(&s[m->len], len - (&s[m->len] - str), wildcarded, wildcarded_size); + return 1; + } + return match_pattern(m->child, &s[m->len], len - (s - str) - m->len, wildcarded, wildcarded_size); } break; case SIMPLE_PATTERN_PREFIX: if(unlikely(strncmp(str, m->match, m->len) == 0)) { - if(!m->child) return 1; - return match_pattern(m->child, &str[m->len], len - m->len); + if(!m->child) { + wildcarded = add_wildcarded(&str[m->len], len - m->len, wildcarded, wildcarded_size); + return 1; + } + return match_pattern(m->child, &str[m->len], len - m->len, wildcarded, wildcarded_size); } break; case SIMPLE_PATTERN_SUFFIX: if(unlikely(strcmp(&str[len - m->len], m->match) == 0)) { + wildcarded = add_wildcarded(str, len - m->len, wildcarded, wildcarded_size); if(!m->child) return 1; return 0; } @@ -166,17 +206,26 @@ static inline int match_pattern(struct simple_pattern *m, const char *str, size_ return 0; } -int simple_pattern_matches(SIMPLE_PATTERN *list, const char *str) { +int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) { struct simple_pattern *m, *root = (struct simple_pattern *)list; if(unlikely(!root || !str || !*str)) return 0; size_t len = strlen(str); - for(m = root; m ; m = m->next) - if(match_pattern(m, str, len)) { - if(m->negative) return 0; + for(m = root; m ; m = m->next) { + char *ws = wildcarded; + size_t wss = wildcarded_size; + if(unlikely(ws)) *ws = '\0'; + + if (match_pattern(m, str, len, ws, &wss)) { + + //if(ws && wss) + // fprintf(stderr, "FINAL WILDCARDED '%s' of length %zu\n", ws, strlen(ws)); + + if (m->negative) return 0; return 1; } + } return 0; } diff --git a/src/simple_pattern.h b/src/simple_pattern.h index 3768c5089..60a25f493 100644 --- a/src/simple_pattern.h +++ b/src/simple_pattern.h @@ -15,8 +15,11 @@ typedef void SIMPLE_PATTERN; // should be considered PREFIX matches. extern SIMPLE_PATTERN *simple_pattern_create(const char *list, SIMPLE_PREFIX_MODE default_mode); +// test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*' +extern int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size); + // test if string str is matched from the pattern -extern int simple_pattern_matches(SIMPLE_PATTERN *list, const char *str); +#define simple_pattern_matches(list, str) simple_pattern_matches_extract(list, str, NULL, 0) // free a simple_pattern that was created with simple_pattern_create() // list can be NULL, in which case, this does nothing. diff --git a/src/socket.c b/src/socket.c index d28df81a6..906ab33dd 100644 --- a/src/socket.c +++ b/src/socket.c @@ -857,7 +857,7 @@ int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) { // -------------------------------------------------------------------------------------------------------------------- // accept_socket() - accept a socket and store client IP and port -int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize) { +int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize, SIMPLE_PATTERN *access_list) { struct sockaddr_storage sadr; socklen_t addrlen = sizeof(sadr); @@ -873,6 +873,13 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien client_port[portsize - 1] = '\0'; switch (((struct sockaddr *)&sadr)->sa_family) { + case AF_UNIX: + debug(D_LISTENER, "New UNIX domain web client from %s on socket %d.", client_ip, fd); + // set the port - certain versions of libc return garbage on unix sockets + strncpy(client_port, "UNIX", portsize); + client_port[portsize - 1] = '\0'; + break; + case AF_INET: debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); break; @@ -881,7 +888,8 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien if (strncmp(client_ip, "::ffff:", 7) == 0) { memmove(client_ip, &client_ip[7], strlen(&client_ip[7]) + 1); debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); - } else + } + else debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd); break; @@ -889,6 +897,22 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd); break; } + + if(access_list) { + if(!strcmp(client_ip, "127.0.0.1") || !strcmp(client_ip, "::1")) { + strncpy(client_ip, "localhost", ipsize); + client_ip[ipsize - 1] = '\0'; + } + + if(unlikely(!simple_pattern_matches(access_list, client_ip))) { + errno = 0; + debug(D_LISTENER, "Permission denied for client '%s', port '%s'", client_ip, client_port); + error("DENIED ACCESS to client '%s'", client_ip); + close(nfd); + nfd = -1; + errno = EPERM; + } + } } #ifdef HAVE_ACCEPT4 else if(errno == ENOSYS) @@ -928,8 +952,8 @@ struct poll { struct pollinfo *inf; struct pollinfo *first_free; - void *(*add_callback)(int fd, short int *events); - void (*del_callback)(int fd, void *data); + void *(*add_callback)(int fd, int socktype, short int *events); + void (*del_callback)(int fd, int socktype, void *data); int (*rcv_callback)(int fd, int socktype, void *data, short int *events); int (*snd_callback)(int fd, int socktype, void *data, short int *events); }; @@ -984,7 +1008,7 @@ static inline struct pollinfo *poll_add_fd(struct poll *p, int fd, int socktype, p->max = pi->slot; if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) { - pi->data = p->add_callback(fd, &pf->events); + pi->data = p->add_callback(fd, pi->socktype, &pf->events); } if(pi->flags & POLLINFO_FLAG_SERVER_SOCKET) { @@ -1003,7 +1027,7 @@ static inline void poll_close_fd(struct poll *p, struct pollinfo *pi) { if(unlikely(pf->fd == -1)) return; if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) { - p->del_callback(pf->fd, pi->data); + p->del_callback(pf->fd, pi->socktype, pi->data); } close(pf->fd); @@ -1036,14 +1060,16 @@ static inline void poll_close_fd(struct poll *p, struct pollinfo *pi) { debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); } -static void *add_callback_default(int fd, short int *events) { +static void *add_callback_default(int fd, int socktype, short int *events) { (void)fd; + (void)socktype; (void)events; return NULL; } -static void del_callback_default(int fd, void *data) { +static void del_callback_default(int fd, int socktype, void *data) { (void)fd; + (void)socktype; (void)data; if(data) @@ -1100,10 +1126,11 @@ void poll_events_cleanup(void *data) { } void poll_events(LISTEN_SOCKETS *sockets - , void *(*add_callback)(int fd, short int *events) - , void (*del_callback)(int fd, void *data) + , void *(*add_callback)(int fd, int socktype, short int *events) + , void (*del_callback)(int fd, int socktype, void *data) , int (*rcv_callback)(int fd, int socktype, void *data, short int *events) , int (*snd_callback)(int fd, int socktype, void *data, short int *events) + , SIMPLE_PATTERN *access_list , void *data ) { int retval; @@ -1182,7 +1209,7 @@ void poll_events(LISTEN_SOCKETS *sockets char client_port[NI_MAXSERV + 1]; debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", i, fd); - nfd = accept_socket(fd, SOCK_NONBLOCK, client_ip, NI_MAXHOST + 1, client_port, NI_MAXSERV + 1); + nfd = accept_socket(fd, SOCK_NONBLOCK, client_ip, NI_MAXHOST + 1, client_port, NI_MAXSERV + 1, access_list); if (nfd < 0) { // accept failed @@ -1212,6 +1239,8 @@ void poll_events(LISTEN_SOCKETS *sockets debug(D_POLLFD, "POLLFD: LISTENER: reading data from UDP slot %zu (fd %d)", i, fd); + // FIXME: access_list is not applied to UDP + p.rcv_callback(fd, pi->socktype, pi->data, &pf->events); break; } diff --git a/src/socket.h b/src/socket.h index 8ca7288c9..08b8518b9 100644 --- a/src/socket.h +++ b/src/socket.h @@ -35,7 +35,7 @@ extern int sock_setreuse_port(int fd, int reuse); extern int sock_enlarge_in(int fd); extern int sock_enlarge_out(int fd); -extern int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize); +extern int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize, SIMPLE_PATTERN *access_list); #ifndef HAVE_ACCEPT4 extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags); @@ -52,10 +52,11 @@ extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flag extern void poll_events(LISTEN_SOCKETS *sockets - , void *(*add_callback)(int fd, short int *events) - , void (*del_callback)(int fd, void *data) + , void *(*add_callback)(int fd, int socktype, short int *events) + , void (*del_callback)(int fd, int socktype, void *data) , int (*rcv_callback)(int fd, int socktype, void *data, short int *events) , int (*snd_callback)(int fd, int socktype, void *data, short int *events) + , SIMPLE_PATTERN *access_list , void *data ); diff --git a/src/statsd.c b/src/statsd.c index 08ce3e2f5..39041ca88 100644 --- a/src/statsd.c +++ b/src/statsd.c @@ -54,6 +54,8 @@ typedef struct statsd_histogram_extensions { collected_number last_stddev; collected_number last_sum; + int zeroed; + RRDDIM *rd_min; RRDDIM *rd_max; RRDDIM *rd_percentile; @@ -165,18 +167,21 @@ typedef enum statsd_app_chart_dimension_value_type { } STATSD_APP_CHART_DIM_VALUE_TYPE; typedef struct statsd_app_chart_dimension { - const char *name; - const char *metric; - uint32_t metric_hash; - collected_number multiplier; - collected_number divisor; - STATSD_APP_CHART_DIM_VALUE_TYPE value_type; + const char *name; // the name of this dimension + const char *metric; // the source metric name of this dimension + uint32_t metric_hash; // hash for fast string comparisons + + SIMPLE_PATTERN *metric_pattern; // set when the 'metric' is a simple pattern + + collected_number multiplier; // the multipler of the dimension + collected_number divisor; // the divisor of the dimension + STATSD_APP_CHART_DIM_VALUE_TYPE value_type; // which value to use of the source metric - RRDDIM *rd; - collected_number *value_ptr; - RRD_ALGORITHM algorithm; + RRDDIM *rd; // a pointer to the RRDDIM that has been created for this dimension + collected_number *value_ptr; // a pointer to the source metric value + RRD_ALGORITHM algorithm; // the algorithm of this dimension - struct statsd_app_chart_dimension *next; + struct statsd_app_chart_dimension *next; // the next dimension for this chart } STATSD_APP_CHART_DIM; typedef struct statsd_app_chart { @@ -202,6 +207,7 @@ typedef struct statsd_app { SIMPLE_PATTERN *metrics; STATS_METRIC_OPTIONS default_options; RRD_MEMORY_MODE rrd_memory_mode; + DICTIONARY *dict; long rrd_history_entries; const char *source; @@ -482,7 +488,7 @@ static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value, static inline void statsd_process_timer(STATSD_METRIC *m, const char *value, const char *sampling) { if(unlikely(!value || !*value)) { - error("STATSD: metric of type set, with empty value is ignored."); + error("STATSD: metric of type timer, with empty value is ignored."); return; } @@ -685,8 +691,9 @@ struct statsd_udp { #endif // new TCP client connected -static void *statsd_add_callback(int fd, short int *events) { +static void *statsd_add_callback(int fd, int socktype, short int *events) { (void)fd; + (void)socktype; *events = POLLIN; struct statsd_tcp *data = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1); @@ -697,8 +704,9 @@ static void *statsd_add_callback(int fd, short int *events) { } // TCP client disconnected -static void statsd_del_callback(int fd, void *data) { +static void statsd_del_callback(int fd, int socktype, void *data) { (void)fd; + (void)socktype; if(data) { struct statsd_tcp *t = data; @@ -912,6 +920,7 @@ void *statsd_collector_thread(void *ptr) { , statsd_del_callback , statsd_rcv_callback , statsd_snd_callback + , NULL , (void *)d ); @@ -928,6 +937,83 @@ void *statsd_collector_thread(void *ptr) { #define STATSD_CONF_LINE_MAX 8192 +static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *path, const char *filename) { + if(!type || !*type) type = "last"; + + if(!strcmp(type, "events")) return STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS; + else if(!strcmp(type, "last")) return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST; + else if(!strcmp(type, "min")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MIN; + else if(!strcmp(type, "max")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MAX; + else if(!strcmp(type, "sum")) return STATSD_APP_CHART_DIM_VALUE_TYPE_SUM; + else if(!strcmp(type, "average")) return STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE; + else if(!strcmp(type, "median")) return STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN; + else if(!strcmp(type, "stddev")) return STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV; + else if(!strcmp(type, "percentile")) return STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE; + + error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename); + return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST; +} + +static const char *valuetype2string(STATSD_APP_CHART_DIM_VALUE_TYPE type) { + switch(type) { + case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS: return "events"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST: return "last"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN: return "min"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX: return "max"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM: return "sum"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE: return "average"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN: return "median"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV: return "stddev"; + case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE: return "percentile"; + } + + return "unknown"; +} + +static STATSD_APP_CHART_DIM *add_dimension_to_app_chart( + STATSD_APP *app + , STATSD_APP_CHART *chart + , const char *metric_name + , const char *dim_name + , collected_number multiplier + , collected_number divisor + , STATSD_APP_CHART_DIM_VALUE_TYPE value_type +) { + STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1); + + dim->metric = strdupz(metric_name); + dim->metric_hash = simple_hash(dim->metric); + + dim->name = strdupz((dim_name)?dim_name:""); + dim->multiplier = multiplier; + dim->divisor = divisor; + dim->value_type = value_type; + + if(!dim->multiplier) + dim->multiplier = 1; + + if(!dim->divisor) + dim->divisor = 1; + + // append it to the list of dimension + STATSD_APP_CHART_DIM *tdim; + for(tdim = chart->dimensions; tdim && tdim->next ; tdim = tdim->next) ; + if(!tdim) { + dim->next = chart->dimensions; + chart->dimensions = dim; + } + else { + dim->next = tdim->next; + tdim->next = dim; + } + chart->dimensions_count++; + + debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT, + dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor); + + return dim; +} + int statsd_readfile(const char *path, const char *filename) { debug(D_STATSD, "STATSD configuration reading file '%s/%s'", path, filename); @@ -943,6 +1029,7 @@ int statsd_readfile(const char *path, const char *filename) { STATSD_APP *app = NULL; STATSD_APP_CHART *chart = NULL; + DICTIONARY *dict = NULL; size_t line = 0; char *s; @@ -973,22 +1060,33 @@ int statsd_readfile(const char *path, const char *filename) { app->next = statsd.apps; statsd.apps = app; chart = NULL; + dict = NULL; } else if(app) { - // a new chart - chart = callocz(sizeof(STATSD_APP_CHART), 1); - netdata_fix_chart_id(s); - chart->id = strdupz(s); - chart->name = strdupz(s); - chart->title = strdupz("Statsd chart"); - chart->context = strdupz(s); - chart->family = strdupz("overview"); - chart->units = strdupz("value"); - chart->priority = STATSD_CHART_PRIORITY; - chart->chart_type = RRDSET_TYPE_LINE; - - chart->next = app->charts; - app->charts = chart; + if(!strcmp(s, "dictionary")) { + if(!app->dict) + app->dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + + dict = app->dict; + } + else { + dict = NULL; + + // a new chart + chart = callocz(sizeof(STATSD_APP_CHART), 1); + netdata_fix_chart_id(s); + chart->id = strdupz(s); + chart->name = strdupz(s); + chart->title = strdupz("Statsd chart"); + chart->context = strdupz(s); + chart->family = strdupz("overview"); + chart->units = strdupz("value"); + chart->priority = STATSD_CHART_PRIORITY; + chart->chart_type = RRDSET_TYPE_LINE; + + chart->next = app->charts; + app->charts = chart; + } } else error("STATSD: ignoring line %zu ('%s') of file '%s/%s', [app] is not defined.", line, s, path, filename); @@ -1022,7 +1120,14 @@ int statsd_readfile(const char *path, const char *filename) { continue; } - if(!chart) { + if(unlikely(dict)) { + // parse [dictionary] members + + dictionary_set(dict, name, value, strlen(value) + 1); + } + else if(!chart) { + // parse [app] members + if(!strcmp(name, "name")) { freez((void *)app->name); netdata_fix_chart_name(value); @@ -1056,6 +1161,8 @@ int statsd_readfile(const char *path, const char *filename) { } } else { + // parse [chart] members + if(!strcmp(name, "name")) { freez((void *)chart->name); netdata_fix_chart_id(value); @@ -1086,63 +1193,50 @@ int statsd_readfile(const char *path, const char *filename) { } else if (!strcmp(name, "dimension")) { // metric [name [type [multiplier [divisor]]]] - char *words[5]; - pluginsd_split_words(value, words, 5); - - char *metric_name = words[0]; - char *dim_name = words[1]; - char *type = words[2]; - char *multipler = words[3]; - char *divisor = words[4]; - - STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1); - - dim->metric = strdupz(metric_name); - dim->metric_hash = simple_hash(dim->metric); - - dim->name = strdupz((dim_name && *dim_name)?dim_name:metric_name); - dim->multiplier = (multipler && *multipler)?str2l(multipler):1; - dim->divisor = (divisor && *divisor)?str2l(divisor):1; - - if(!type || !*type) type = "last"; - if(!strcmp(type, "events")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS; - else if(!strcmp(type, "last")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST; - else if(!strcmp(type, "min")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MIN; - else if(!strcmp(type, "max")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MAX; - else if(!strcmp(type, "sum")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_SUM; - else if(!strcmp(type, "average")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE; - else if(!strcmp(type, "median")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN; - else if(!strcmp(type, "stddev")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV; - else if(!strcmp(type, "percentile")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE; - else { - error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename); - dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST; - } + char *words[10]; + pluginsd_split_words(value, words, 10); - if(!dim->multiplier) { - error("STATSD: invalid multiplier value '%s' at line %zu of file '%s/%s'. Using 1.", multipler, line, path, filename); - dim->multiplier = 1; - } - if(!dim->divisor) { - error("STATSD: invalid divisor value '%s' at line %zu of file '%s/%s'. Using 1.", divisor, line, path, filename); - dim->divisor = 1; - } + int pattern = 0; + size_t i = 0; + char *metric_name = words[i++]; - // append it to the list of dimension - STATSD_APP_CHART_DIM *tdim; - for(tdim = chart->dimensions; tdim && tdim->next ; tdim = tdim->next) ; - if(!tdim) { - dim->next = chart->dimensions; - chart->dimensions = dim; + if(strcmp(metric_name, "pattern") == 0) { + metric_name = words[i++]; + pattern = 1; } - else { - dim->next = tdim->next; - tdim->next = dim; + + char *dim_name = words[i++]; + char *type = words[i++]; + char *multipler = words[i++]; + char *divisor = words[i++]; + + if(!pattern) { + if(app->dict) { + if(dim_name && *dim_name) { + char *n = dictionary_get(app->dict, dim_name); + if(n) dim_name = n; + } + else { + dim_name = dictionary_get(app->dict, metric_name); + } + } + + if(!dim_name || !*dim_name) + dim_name = metric_name; } - chart->dimensions_count++; - debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT, - dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor); + STATSD_APP_CHART_DIM *dim = add_dimension_to_app_chart( + app + , chart + , metric_name + , dim_name + , (multipler && *multipler)?str2l(multipler):1 + , (divisor && *divisor)?str2l(divisor):1 + , string2valuetype(type, line, path, filename) + ); + + if(pattern) + dim->metric_pattern = simple_pattern_create(dim->metric, SIMPLE_PATTERN_EXACT); } else { error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id); @@ -1248,19 +1342,21 @@ static inline RRDSET *statsd_private_rrdset_create( statsd.private_charts++; RRDSET *st = rrdset_create_custom( - localhost - , type - , id - , name - , family - , context - , title - , units - , priority - , update_every - , chart_type - , memory_mode - , history + localhost // host + , type // type + , id // id + , name // name + , family // family + , context // context + , title // title + , units // units + , "statsd" // plugin + , NULL // module + , priority // priority + , update_every // update every + , chart_type // chart type + , memory_mode // memory mode + , history // history ); rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST); // rrdset_flag_set(st, RRDSET_FLAG_DEBUG); @@ -1484,6 +1580,22 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char netdata_mutex_lock(&m->histogram.ext->mutex); + if(unlikely(!m->histogram.ext->zeroed)) { + // reset the metrics + // if we collected anything, they will be updated below + // this ensures that we report zeros if nothing is collected + + m->histogram.ext->last_min = 0; + m->histogram.ext->last_max = 0; + m->last = 0; + m->histogram.ext->last_median = 0; + m->histogram.ext->last_stddev = 0; + m->histogram.ext->last_sum = 0; + m->histogram.ext->last_percentile = 0; + + m->histogram.ext->zeroed = 1; + } + int updated = 0; if(m->count && !m->reset && m->histogram.ext->used > 0) { size_t len = m->histogram.ext->used; @@ -1506,11 +1618,11 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT, dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum); + m->histogram.ext->zeroed = 0; m->reset = 1; updated = 1; } - if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) statsd_private_chart_timer_or_histogram(m, dim, family, units); @@ -1540,6 +1652,71 @@ static inline RRD_ALGORITHM statsd_algorithm_for_metric(STATSD_METRIC *m) { } } +static inline void link_metric_to_app_dimension(STATSD_APP *app, STATSD_METRIC *m, STATSD_APP_CHART *chart, STATSD_APP_CHART_DIM *dim) { + if(dim->value_type == STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS) { + dim->value_ptr = &m->events; + dim->algorithm = RRD_ALGORITHM_INCREMENTAL; + } + else if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) { + dim->algorithm = RRD_ALGORITHM_ABSOLUTE; + dim->divisor *= statsd.decimal_detail; + + switch(dim->value_type) { + case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS: + // will never match - added to avoid warning + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST: + case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE: + dim->value_ptr = &m->last; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM: + dim->value_ptr = &m->histogram.ext->last_sum; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN: + dim->value_ptr = &m->histogram.ext->last_min; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX: + dim->value_ptr = &m->histogram.ext->last_max; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN: + dim->value_ptr = &m->histogram.ext->last_median; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE: + dim->value_ptr = &m->histogram.ext->last_percentile; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV: + dim->value_ptr = &m->histogram.ext->last_stddev; + break; + } + } + else { + if (dim->value_type != STATSD_APP_CHART_DIM_VALUE_TYPE_LAST) + error("STATSD: unsupported value type for dimension '%s' of chart '%s' of app '%s' on metric '%s'", dim->name, chart->id, app->name, m->name); + + dim->value_ptr = &m->last; + dim->algorithm = statsd_algorithm_for_metric(m); + + if(m->type == STATSD_METRIC_TYPE_GAUGE) + dim->divisor *= statsd.decimal_detail; + } + + if(unlikely(chart->st && dim->rd)) { + rrddim_set_algorithm(chart->st, dim->rd, dim->algorithm); + rrddim_set_multiplier(chart->st, dim->rd, dim->multiplier); + rrddim_set_divisor(chart->st, dim->rd, dim->divisor); + } + + chart->dimensions_linked_count++; + debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm)); +} + static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC *m) { (void)index; @@ -1565,78 +1742,103 @@ static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC // check if there is a chart in this app, willing to get this metric STATSD_APP_CHART *chart; for(chart = app->charts; chart; chart = chart->next) { + STATSD_APP_CHART_DIM *dim; for(dim = chart->dimensions; dim ; dim = dim->next) { - if(!dim->value_ptr && dim->metric_hash == m->hash && !strcmp(dim->metric, m->name)) { - // we have a match - this metric should be linked to this dimension + if(unlikely(dim->metric_pattern)) { + size_t dim_name_len = strlen(dim->name); + size_t wildcarded_len = dim_name_len + strlen(m->name) + 1; + char wildcarded[wildcarded_len]; - if(dim->value_type == STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS) { - dim->value_ptr = &m->events; - dim->algorithm = RRD_ALGORITHM_INCREMENTAL; - } - else if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) { - dim->algorithm = RRD_ALGORITHM_ABSOLUTE; - dim->divisor *= statsd.decimal_detail; - - switch(dim->value_type) { - case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS: - // will never match - added to avoid warning - break; - - case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST: - case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE: - dim->value_ptr = &m->last; - break; - - case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM: - dim->value_ptr = &m->histogram.ext->last_sum; - break; - - case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN: - dim->value_ptr = &m->histogram.ext->last_min; - break; - - case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX: - dim->value_ptr = &m->histogram.ext->last_max; - break; - - case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN: - dim->value_ptr = &m->histogram.ext->last_median; - break; - - case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE: - dim->value_ptr = &m->histogram.ext->last_percentile; - break; - - case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV: - dim->value_ptr = &m->histogram.ext->last_stddev; - break; - } - } - else { - if (dim->value_type != STATSD_APP_CHART_DIM_VALUE_TYPE_LAST) - error("STATSD: unsupported value type for dimension '%s' of chart '%s' of app '%s' on metric '%s'", dim->name, chart->id, app->name, m->name); + strcpy(wildcarded, dim->name); + char *ws = &wildcarded[dim_name_len]; - dim->value_ptr = &m->last; - dim->algorithm = statsd_algorithm_for_metric(m); + if(simple_pattern_matches_extract(dim->metric_pattern, m->name, ws, wildcarded_len - dim_name_len)) { - if(m->type == STATSD_METRIC_TYPE_GAUGE) - dim->divisor *= statsd.decimal_detail; - } + char *final_name = NULL; - if(unlikely(chart->st && dim->rd)) { - rrddim_set_algorithm(chart->st, dim->rd, dim->algorithm); - rrddim_set_multiplier(chart->st, dim->rd, dim->multiplier); - rrddim_set_divisor(chart->st, dim->rd, dim->divisor); - } + if(app->dict) { + if(likely(*wildcarded)) { + // use the name of the wildcarded string + final_name = dictionary_get(app->dict, wildcarded); + } + + if(unlikely(!final_name)) { + // use the name of the metric + final_name = dictionary_get(app->dict, m->name); + } + } - chart->dimensions_linked_count++; - debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm)); + if(unlikely(!final_name)) + final_name = wildcarded; + + add_dimension_to_app_chart( + app + , chart + , m->name + , final_name + , dim->multiplier + , dim->divisor + , dim->value_type + ); + + // the new dimension is appended to the list + // so, it will be matched and linked later too + } + } + else if(!dim->value_ptr && dim->metric_hash == m->hash && !strcmp(dim->metric, m->name)) { + // we have a match - this metric should be linked to this dimension + link_metric_to_app_dimension(app, m, chart, dim); } } + + } + } + } +} + +static inline RRDDIM *statsd_add_dim_to_app_chart(STATSD_APP *app, STATSD_APP_CHART *chart, STATSD_APP_CHART_DIM *dim) { + (void)app; + + // allow the same statsd metric to be added multiple times to the same chart + + STATSD_APP_CHART_DIM *tdim; + size_t count_same_metric = 0, count_same_metric_value_type = 0; + size_t pos_same_metric_value_type = 0; + + for (tdim = chart->dimensions; tdim && tdim->next; tdim = tdim->next) { + if (dim->metric_hash == tdim->metric_hash && !strcmp(dim->metric, tdim->metric)) { + count_same_metric++; + + if(dim->value_type == tdim->value_type) { + count_same_metric_value_type++; + if (tdim == dim) + pos_same_metric_value_type = count_same_metric_value_type; } } } + + if(count_same_metric > 1) { + // the same metric is found multiple times + + size_t len = strlen(dim->metric) + 100; + char metric[ len + 1 ]; + + if(count_same_metric_value_type > 1) { + // the same metric, with the same value type, is added multiple times + snprintfz(metric, len, "%s_%s%zu", dim->metric, valuetype2string(dim->value_type), pos_same_metric_value_type); + } + else { + // the same metric, with different value type is added + snprintfz(metric, len, "%s_%s", dim->metric, valuetype2string(dim->value_type)); + } + + dim->rd = rrddim_add(chart->st, metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm); + return dim->rd; + } + + dim->rd = rrddim_add(chart->st, dim->metric, dim->name, dim->multiplier, dim->divisor, dim->algorithm); + return dim->rd; } static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *chart) { @@ -1644,19 +1846,21 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch if(!chart->st) { chart->st = rrdset_create_custom( - localhost - , app->name - , chart->id - , chart->name - , chart->family - , chart->context - , chart->title - , chart->units - , chart->priority - , statsd.update_every - , chart->chart_type - , app->rrd_memory_mode - , app->rrd_history_entries + localhost // host + , app->name // type + , chart->id // id + , chart->name // name + , chart->family // family + , chart->context // context + , chart->title // title + , chart->units // units + , "statsd" // plugin + , NULL // module + , chart->priority // priority + , statsd.update_every // update every + , chart->chart_type // chart type + , app->rrd_memory_mode // memory mode + , app->rrd_history_entries // history ); rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST); @@ -1666,12 +1870,14 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch STATSD_APP_CHART_DIM *dim; for(dim = chart->dimensions; dim ;dim = dim->next) { - if(unlikely(!dim->rd)) - dim->rd = rrddim_add(chart->st, dim->name, NULL, dim->multiplier, dim->divisor, dim->algorithm); + if(likely(!dim->metric_pattern)) { + if (unlikely(!dim->rd)) + statsd_add_dim_to_app_chart(app, chart, dim); - if(unlikely(dim->value_ptr)) { - debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr); - rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr); + if (unlikely(dim->value_ptr)) { + debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr); + rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr); + } } } @@ -1871,6 +2077,8 @@ void *statsd_main(void *ptr) { , NULL , "Metrics in the netdata statsd database" , "metrics" + , "netdata" + , "stats" , 132000 , statsd.update_every , RRDSET_TYPE_STACKED @@ -1890,6 +2098,8 @@ void *statsd_main(void *ptr) { , NULL , "Events processed by the netdata statsd server" , "events/s" + , "netdata" + , "stats" , 132001 , statsd.update_every , RRDSET_TYPE_STACKED @@ -1911,6 +2121,8 @@ void *statsd_main(void *ptr) { , NULL , "Read operations made by the netdata statsd server" , "reads/s" + , "netdata" + , "stats" , 132002 , statsd.update_every , RRDSET_TYPE_STACKED @@ -1926,6 +2138,8 @@ void *statsd_main(void *ptr) { , NULL , "Bytes read by the netdata statsd server" , "kilobits/s" + , "netdata" + , "stats" , 132003 , statsd.update_every , RRDSET_TYPE_STACKED @@ -1941,6 +2155,8 @@ void *statsd_main(void *ptr) { , NULL , "Network packets processed by the netdata statsd server" , "packets/s" + , "netdata" + , "stats" , 132004 , statsd.update_every , RRDSET_TYPE_STACKED @@ -1956,6 +2172,8 @@ void *statsd_main(void *ptr) { , NULL , "Private metric charts created by the netdata statsd server" , "charts" + , "netdata" + , "stats" , 132010 , statsd.update_every , RRDSET_TYPE_AREA diff --git a/src/storage_number.c b/src/storage_number.c index 054941202..3fd22a416 100644 --- a/src/storage_number.c +++ b/src/storage_number.c @@ -39,7 +39,9 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) r += (1 << 30) + (m << 27); // the multiplier m if(n > (calculated_number)0x00ffffff) { + #ifdef NETDATA_INTERNAL_CHECKS error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); + #endif r += 0x00ffffff; return r; } diff --git a/src/storage_number.h b/src/storage_number.h index 3c1b6bab3..616ff881e 100644 --- a/src/storage_number.h +++ b/src/storage_number.h @@ -17,6 +17,9 @@ typedef long double collected_number; #define calculated_number_llrint(x) llrintl(x) #define calculated_number_round(x) roundl(x) #define calculated_number_fabs(x) fabsl(x) +#define calculated_number_epsilon (calculated_number)0.0000001 + +#define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon) typedef uint32_t storage_number; #define STORAGE_NUMBER_FORMAT "%u" diff --git a/src/sys_devices_system_edac_mc.c b/src/sys_devices_system_edac_mc.c index 7ec989434..9eac8a12e 100644 --- a/src/sys_devices_system_edac_mc.c +++ b/src/sys_devices_system_edac_mc.c @@ -140,6 +140,8 @@ int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) { , NULL , "ECC Memory Correctable Errors" , "errors" + , "proc" + , "/sys/devices/system/edac/mc" , 6600 , update_every , RRDSET_TYPE_LINE @@ -176,6 +178,8 @@ int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) { , NULL , "ECC Memory Uncorrectable Errors" , "errors" + , "proc" + , "/sys/devices/system/edac/mc" , 6610 , update_every , RRDSET_TYPE_LINE diff --git a/src/sys_devices_system_node.c b/src/sys_devices_system_node.c index a7690e7b7..86d55b298 100644 --- a/src/sys_devices_system_node.c +++ b/src/sys_devices_system_node.c @@ -105,6 +105,8 @@ int do_proc_sys_devices_system_node(int update_every, usec_t dt) { , NULL , "NUMA events" , "events/s" + , "proc" + , "/sys/devices/system/node" , 1000 , update_every , RRDSET_TYPE_LINE diff --git a/src/sys_fs_cgroup.c b/src/sys_fs_cgroup.c index c047547e7..946831afa 100644 --- a/src/sys_fs_cgroup.c +++ b/src/sys_fs_cgroup.c @@ -151,7 +151,8 @@ void read_cgroup_plugin_configuration() { // ---------------------------------------------------------------- " !*/init.scope " // ignore init.scope - " *.scope " // we need all *.scope for sure + " !/system.slice/run-*.scope " // ignore system.slice/run-XXXX.scope + " *.scope " // we need all other *.scope for sure // ---------------------------------------------------------------- @@ -180,6 +181,7 @@ void read_cgroup_plugin_configuration() { config_get("plugin:cgroups", "search for cgroups in subpaths matching", " !*/init.scope " // ignore init.scope " !*-qemu " // #345 + " !*.libvirt-qemu " // #3010 " !/init.scope " " !/system " " !/systemd " @@ -201,6 +203,9 @@ void read_cgroup_plugin_configuration() { " *docker* " " *lxc* " " *qemu* " + " *.libvirt-qemu " // #3010 + " !*/vcpu* " // libvirtd adds these sub-cgroups + " !*/emulator* " // libvirtd adds these sub-cgroups " !/ " " !*.mount " " !*.partition " @@ -762,7 +767,7 @@ static inline void read_cgroup_network_interfaces(struct cgroup *cg) { } if(!*t) { - error("CGROUP: empty container interface returned by script"); + error("CGROUP: empty guest interface returned by script"); continue; } @@ -1447,6 +1452,8 @@ void update_systemd_services_charts( , "services.cpu" , title , "%" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES , update_every , RRDSET_TYPE_STACKED @@ -1469,6 +1476,8 @@ void update_systemd_services_charts( , (cgroup_used_memory_without_cache) ? "Systemd Services Used Memory without Cache" : "Systemd Services Used Memory" , "MB" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 10 , update_every , RRDSET_TYPE_STACKED @@ -1490,6 +1499,8 @@ void update_systemd_services_charts( , "services.mem_rss" , "Systemd Services RSS Memory" , "MB" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 20 , update_every , RRDSET_TYPE_STACKED @@ -1509,6 +1520,8 @@ void update_systemd_services_charts( , "services.mem_mapped" , "Systemd Services Mapped Memory" , "MB" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 30 , update_every , RRDSET_TYPE_STACKED @@ -1528,6 +1541,8 @@ void update_systemd_services_charts( , "services.mem_cache" , "Systemd Services Cache Memory" , "MB" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 40 , update_every , RRDSET_TYPE_STACKED @@ -1547,6 +1562,8 @@ void update_systemd_services_charts( , "services.mem_writeback" , "Systemd Services Writeback Memory" , "MB" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 50 , update_every , RRDSET_TYPE_STACKED @@ -1566,6 +1583,8 @@ void update_systemd_services_charts( , "services.mem_pgfault" , "Systemd Services Memory Minor Page Faults" , "MB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 60 , update_every , RRDSET_TYPE_STACKED @@ -1584,6 +1603,8 @@ void update_systemd_services_charts( , "services.mem_pgmajfault" , "Systemd Services Memory Major Page Faults" , "MB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 70 , update_every , RRDSET_TYPE_STACKED @@ -1603,6 +1624,8 @@ void update_systemd_services_charts( , "services.mem_pgpgin" , "Systemd Services Memory Charging Activity" , "MB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 80 , update_every , RRDSET_TYPE_STACKED @@ -1622,6 +1645,8 @@ void update_systemd_services_charts( , "services.mem_pgpgout" , "Systemd Services Memory Uncharging Activity" , "MB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 90 , update_every , RRDSET_TYPE_STACKED @@ -1643,6 +1668,8 @@ void update_systemd_services_charts( , "services.mem_failcnt" , "Systemd Services Memory Limit Failures" , "MB" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 110 , update_every , RRDSET_TYPE_STACKED @@ -1664,6 +1691,8 @@ void update_systemd_services_charts( , "services.swap_usage" , "Systemd Services Swap Memory Used" , "MB" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 100 , update_every , RRDSET_TYPE_STACKED @@ -1685,6 +1714,8 @@ void update_systemd_services_charts( , "services.io_read" , "Systemd Services Disk Read Bandwidth" , "KB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 120 , update_every , RRDSET_TYPE_STACKED @@ -1704,6 +1735,8 @@ void update_systemd_services_charts( , "services.io_write" , "Systemd Services Disk Write Bandwidth" , "KB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 130 , update_every , RRDSET_TYPE_STACKED @@ -1725,6 +1758,8 @@ void update_systemd_services_charts( , "services.io_ops_read" , "Systemd Services Disk Read Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 140 , update_every , RRDSET_TYPE_STACKED @@ -1744,6 +1779,8 @@ void update_systemd_services_charts( , "services.io_ops_write" , "Systemd Services Disk Write Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 150 , update_every , RRDSET_TYPE_STACKED @@ -1765,6 +1802,8 @@ void update_systemd_services_charts( , "services.throttle_io_read" , "Systemd Services Throttle Disk Read Bandwidth" , "KB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 160 , update_every , RRDSET_TYPE_STACKED @@ -1784,6 +1823,8 @@ void update_systemd_services_charts( , "services.throttle_io_write" , "Systemd Services Throttle Disk Write Bandwidth" , "KB/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 170 , update_every , RRDSET_TYPE_STACKED @@ -1805,6 +1846,8 @@ void update_systemd_services_charts( , "services.throttle_io_ops_read" , "Systemd Services Throttle Disk Read Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 180 , update_every , RRDSET_TYPE_STACKED @@ -1824,6 +1867,8 @@ void update_systemd_services_charts( , "services.throttle_io_ops_write" , "Systemd Services Throttle Disk Write Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 190 , update_every , RRDSET_TYPE_STACKED @@ -1845,6 +1890,8 @@ void update_systemd_services_charts( , "services.queued_io_ops_read" , "Systemd Services Queued Disk Read Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 200 , update_every , RRDSET_TYPE_STACKED @@ -1864,6 +1911,8 @@ void update_systemd_services_charts( , "services.queued_io_ops_write" , "Systemd Services Queued Disk Write Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 210 , update_every , RRDSET_TYPE_STACKED @@ -1885,6 +1934,8 @@ void update_systemd_services_charts( , "services.merged_io_ops_read" , "Systemd Services Merged Disk Read Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 220 , update_every , RRDSET_TYPE_STACKED @@ -1904,6 +1955,8 @@ void update_systemd_services_charts( , "services.merged_io_ops_write" , "Systemd Services Merged Disk Write Operations" , "operations/s" + , "cgroup" + , "systemd" , CHART_PRIORITY_SYSTEMD_SERVICES + 230 , update_every , RRDSET_TYPE_STACKED @@ -2184,6 +2237,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.cpu" , title , "%" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS , update_every , RRDSET_TYPE_STACKED @@ -2215,6 +2270,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.cpu_per_core" , title , "%" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 100 , update_every , RRDSET_TYPE_STACKED @@ -2247,6 +2304,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.mem" , title , "MB" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 210 , update_every , RRDSET_TYPE_STACKED @@ -2285,6 +2344,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.writeback" , title , "MB" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 300 , update_every , RRDSET_TYPE_AREA @@ -2315,6 +2376,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.mem_activity" , title , "MB/s" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 400 , update_every , RRDSET_TYPE_LINE @@ -2341,6 +2404,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.pgfaults" , title , "MB/s" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 500 , update_every , RRDSET_TYPE_LINE @@ -2369,6 +2434,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.mem_usage" , title , "MB" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 200 , update_every , RRDSET_TYPE_STACKED @@ -2397,6 +2464,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.mem_failcnt" , title , "count" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 250 , update_every , RRDSET_TYPE_LINE @@ -2423,6 +2492,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.io" , title , "KB/s" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 1200 , update_every , RRDSET_TYPE_AREA @@ -2451,6 +2522,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.serviced_ops" , title , "operations/s" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 1200 , update_every , RRDSET_TYPE_LINE @@ -2479,6 +2552,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.throttle_io" , title , "KB/s" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 1200 , update_every , RRDSET_TYPE_AREA @@ -2507,6 +2582,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.throttle_serviced_ops" , title , "operations/s" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 1200 , update_every , RRDSET_TYPE_LINE @@ -2535,6 +2612,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.queued_ops" , title , "operations" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 2000 , update_every , RRDSET_TYPE_LINE @@ -2563,6 +2642,8 @@ void update_cgroup_charts(int update_every) { , "cgroup.merged_ops" , title , "operations/s" + , "cgroup" + , "default" , CHART_PRIORITY_CONTAINERS + 2100 , update_every , RRDSET_TYPE_LINE @@ -2650,6 +2731,8 @@ void *cgroups_main(void *ptr) { , NULL , "NetData CGroups Plugin CPU usage" , "milliseconds/s" + , "cgroup" + , "stats" , 132000 , cgroup_update_every , RRDSET_TYPE_STACKED diff --git a/src/sys_kernel_mm_ksm.c b/src/sys_kernel_mm_ksm.c index 6b04ef280..356315be4 100644 --- a/src/sys_kernel_mm_ksm.c +++ b/src/sys_kernel_mm_ksm.c @@ -102,6 +102,8 @@ int do_sys_kernel_mm_ksm(int update_every, usec_t dt) { , NULL , "Kernel Same Page Merging" , "MB" + , "proc" + , "/sys/kernel/mm/ksm" , 5000 , update_every , RRDSET_TYPE_AREA @@ -140,6 +142,8 @@ int do_sys_kernel_mm_ksm(int update_every, usec_t dt) { , NULL , "Kernel Same Page Merging Savings" , "MB" + , "proc" + , "/sys/kernel/mm/ksm" , 5001 , update_every , RRDSET_TYPE_AREA @@ -172,6 +176,8 @@ int do_sys_kernel_mm_ksm(int update_every, usec_t dt) { , NULL , "Kernel Same Page Merging Effectiveness" , "percentage" + , "proc" + , "/sys/kernel/mm/ksm" , 5002 , update_every , RRDSET_TYPE_LINE diff --git a/src/unit_test.c b/src/unit_test.c index 3c9632119..821063baf 100644 --- a/src/unit_test.c +++ b/src/unit_test.c @@ -1,5 +1,62 @@ #include "common.h" +static int check_rrdcalc_comparisons(void) { + RRDCALC_STATUS a, b; + + // make sure calloc() sets the status to UNINITIALIZED + memset(&a, 0, sizeof(RRDCALC_STATUS)); + if(a != RRDCALC_STATUS_UNINITIALIZED) { + fprintf(stderr, "%s is not zero.\n", rrdcalc_status2string(RRDCALC_STATUS_UNINITIALIZED)); + return 1; + } + + a = RRDCALC_STATUS_REMOVED; + b = RRDCALC_STATUS_UNDEFINED; + if(!(a < b)) { + fprintf(stderr, "%s is not less than %s\n", rrdcalc_status2string(a), rrdcalc_status2string(b)); + return 1; + } + + a = RRDCALC_STATUS_UNDEFINED; + b = RRDCALC_STATUS_UNINITIALIZED; + if(!(a < b)) { + fprintf(stderr, "%s is not less than %s\n", rrdcalc_status2string(a), rrdcalc_status2string(b)); + return 1; + } + + a = RRDCALC_STATUS_UNINITIALIZED; + b = RRDCALC_STATUS_CLEAR; + if(!(a < b)) { + fprintf(stderr, "%s is not less than %s\n", rrdcalc_status2string(a), rrdcalc_status2string(b)); + return 1; + } + + a = RRDCALC_STATUS_CLEAR; + b = RRDCALC_STATUS_RAISED; + if(!(a < b)) { + fprintf(stderr, "%s is not less than %s\n", rrdcalc_status2string(a), rrdcalc_status2string(b)); + return 1; + } + + a = RRDCALC_STATUS_RAISED; + b = RRDCALC_STATUS_WARNING; + if(!(a < b)) { + fprintf(stderr, "%s is not less than %s\n", rrdcalc_status2string(a), rrdcalc_status2string(b)); + return 1; + } + + a = RRDCALC_STATUS_WARNING; + b = RRDCALC_STATUS_CRITICAL; + if(!(a < b)) { + fprintf(stderr, "%s is not less than %s\n", rrdcalc_status2string(a), rrdcalc_status2string(b)); + return 1; + } + + fprintf(stderr, "RRDCALC_STATUSes are sortable.\n"); + + return 0; +} + int check_storage_number(calculated_number n, int debug) { char buffer[100]; uint32_t flags = SN_EXISTS; @@ -973,7 +1030,7 @@ int run_test(struct test *test) snprintfz(name, 100, "unittest-%s", test->name); // create the chart - RRDSET *st = rrdset_create_localhost("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", 1 + RRDSET *st = rrdset_create_localhost("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", "unittest", NULL, 1 , test->update_every, RRDSET_TYPE_LINE); RRDDIM *rd = rrddim_add(st, "dim1", NULL, test->multiplier, test->divisor, test->algorithm); @@ -1062,7 +1119,7 @@ int run_test(struct test *test) static int test_variable_renames(void) { fprintf(stderr, "Creating chart\n"); - RRDSET *st = rrdset_create_localhost("chart", "ID", NULL, "family", "context", "Unit Testing", "a value", 1, 1, RRDSET_TYPE_LINE); + RRDSET *st = rrdset_create_localhost("chart", "ID", NULL, "family", "context", "Unit Testing", "a value", "unittest", NULL, 1, 1, RRDSET_TYPE_LINE); fprintf(stderr, "Created chart with id '%s', name '%s'\n", st->id, st->name); fprintf(stderr, "Creating dimension DIM1\n"); @@ -1106,6 +1163,9 @@ static int test_variable_renames(void) { int run_all_mockup_tests(void) { + if(check_rrdcalc_comparisons()) + return 1; + if(!test_variable_renames()) return 1; @@ -1182,7 +1242,7 @@ int unit_test(long delay, long shift) int do_abst = 0; int do_absi = 0; - RRDSET *st = rrdset_create_localhost("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", 1, 1 + RRDSET *st = rrdset_create_localhost("netdata", name, name, "netdata", NULL, "Unit Testing", "a value", "unittest", NULL, 1, 1 , RRDSET_TYPE_LINE); rrdset_flag_set(st, RRDSET_FLAG_DEBUG); diff --git a/src/web_api_v1.c b/src/web_api_v1.c index 9514f8dbd..02c6b0edd 100644 --- a/src/web_api_v1.c +++ b/src/web_api_v1.c @@ -1,20 +1,105 @@ #include "common.h" -inline int web_client_api_request_v1_data_group(char *name, int def) { - if(!strcmp(name, "average")) - return GROUP_AVERAGE; - - else if(!strcmp(name, "min")) - return GROUP_MIN; - - else if(!strcmp(name, "max")) - return GROUP_MAX; +static struct { + const char *name; + uint32_t hash; + int value; +} api_v1_data_groups[] = { + { "average" , 0 , GROUP_AVERAGE} + , {"min" , 0 , GROUP_MIN} + , {"max" , 0 , GROUP_MAX} + , {"sum" , 0 , GROUP_SUM} + , {"incremental_sum", 0 , GROUP_INCREMENTAL_SUM} + , {"incremental-sum", 0 , GROUP_INCREMENTAL_SUM} + , { NULL, 0, 0} +}; + +static struct { + const char *name; + uint32_t hash; + uint32_t value; +} api_v1_data_options[] = { + { "nonzero" , 0 , RRDR_OPTION_NONZERO} + , {"flip" , 0 , RRDR_OPTION_REVERSED} + , {"reversed" , 0 , RRDR_OPTION_REVERSED} + , {"reverse" , 0 , RRDR_OPTION_REVERSED} + , {"jsonwrap" , 0 , RRDR_OPTION_JSON_WRAP} + , {"min2max" , 0 , RRDR_OPTION_MIN2MAX} + , {"ms" , 0 , RRDR_OPTION_MILLISECONDS} + , {"milliseconds" , 0 , RRDR_OPTION_MILLISECONDS} + , {"abs" , 0 , RRDR_OPTION_ABSOLUTE} + , {"absolute" , 0 , RRDR_OPTION_ABSOLUTE} + , {"absolute_sum" , 0 , RRDR_OPTION_ABSOLUTE} + , {"absolute-sum" , 0 , RRDR_OPTION_ABSOLUTE} + , {"display_absolute", 0 , RRDR_OPTION_DISPLAY_ABS} + , {"display-absolute", 0 , RRDR_OPTION_DISPLAY_ABS} + , {"seconds" , 0 , RRDR_OPTION_SECONDS} + , {"null2zero" , 0 , RRDR_OPTION_NULL2ZERO} + , {"objectrows" , 0 , RRDR_OPTION_OBJECTSROWS} + , {"google_json" , 0 , RRDR_OPTION_GOOGLE_JSON} + , {"google-json" , 0 , RRDR_OPTION_GOOGLE_JSON} + , {"percentage" , 0 , RRDR_OPTION_PERCENTAGE} + , {"unaligned" , 0 , RRDR_OPTION_NOT_ALIGNED} + , { NULL, 0, 0} +}; + +static struct { + const char *name; + uint32_t hash; + uint32_t value; +} api_v1_data_formats[] = { + { DATASOURCE_FORMAT_DATATABLE_JSON , 0 , DATASOURCE_DATATABLE_JSON} + , {DATASOURCE_FORMAT_DATATABLE_JSONP, 0 , DATASOURCE_DATATABLE_JSONP} + , {DATASOURCE_FORMAT_JSON , 0 , DATASOURCE_JSON} + , {DATASOURCE_FORMAT_JSONP , 0 , DATASOURCE_JSONP} + , {DATASOURCE_FORMAT_SSV , 0 , DATASOURCE_SSV} + , {DATASOURCE_FORMAT_CSV , 0 , DATASOURCE_CSV} + , {DATASOURCE_FORMAT_TSV , 0 , DATASOURCE_TSV} + , {"tsv-excel" , 0 , DATASOURCE_TSV} + , {DATASOURCE_FORMAT_HTML , 0 , DATASOURCE_HTML} + , {DATASOURCE_FORMAT_JS_ARRAY , 0 , DATASOURCE_JS_ARRAY} + , {DATASOURCE_FORMAT_SSV_COMMA , 0 , DATASOURCE_SSV_COMMA} + , {DATASOURCE_FORMAT_CSV_JSON_ARRAY , 0 , DATASOURCE_CSV_JSON_ARRAY} + , { NULL, 0, 0} +}; + +static struct { + const char *name; + uint32_t hash; + uint32_t value; +} api_v1_data_google_formats[] = { + // this is not error - when google requests json, it expects javascript + // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source#responseformat + { "json" , 0 , DATASOURCE_DATATABLE_JSONP} + , {"html" , 0 , DATASOURCE_HTML} + , {"csv" , 0 , DATASOURCE_CSV} + , {"tsv-excel", 0 , DATASOURCE_TSV} + , { NULL, 0, 0} +}; + +void web_client_api_v1_init(void) { + int i; + + for(i = 0; api_v1_data_groups[i].name ; i++) + api_v1_data_groups[i].hash = simple_hash(api_v1_data_groups[i].name); + + for(i = 0; api_v1_data_options[i].name ; i++) + api_v1_data_options[i].hash = simple_hash(api_v1_data_options[i].name); + + for(i = 0; api_v1_data_formats[i].name ; i++) + api_v1_data_formats[i].hash = simple_hash(api_v1_data_formats[i].name); + + for(i = 0; api_v1_data_google_formats[i].name ; i++) + api_v1_data_google_formats[i].hash = simple_hash(api_v1_data_google_formats[i].name); +} - else if(!strcmp(name, "sum")) - return GROUP_SUM; +inline int web_client_api_request_v1_data_group(char *name, int def) { + int i; - else if(!strcmp(name, "incremental-sum")) - return GROUP_INCREMENTAL_SUM; + uint32_t hash = simple_hash(name); + for(i = 0; api_v1_data_groups[i].name ; i++) + if(unlikely(hash == api_v1_data_groups[i].hash && !strcmp(name, api_v1_data_groups[i].name))) + return api_v1_data_groups[i].value; return def; } @@ -26,84 +111,41 @@ inline uint32_t web_client_api_request_v1_data_options(char *o) { while(o && *o && (tok = mystrsep(&o, ", |"))) { if(!*tok) continue; - if(!strcmp(tok, "nonzero")) - ret |= RRDR_OPTION_NONZERO; - else if(!strcmp(tok, "flip") || !strcmp(tok, "reversed") || !strcmp(tok, "reverse")) - ret |= RRDR_OPTION_REVERSED; - else if(!strcmp(tok, "jsonwrap")) - ret |= RRDR_OPTION_JSON_WRAP; - else if(!strcmp(tok, "min2max")) - ret |= RRDR_OPTION_MIN2MAX; - else if(!strcmp(tok, "ms") || !strcmp(tok, "milliseconds")) - ret |= RRDR_OPTION_MILLISECONDS; - else if(!strcmp(tok, "abs") || !strcmp(tok, "absolute") || !strcmp(tok, "absolute_sum") || !strcmp(tok, "absolute-sum")) - ret |= RRDR_OPTION_ABSOLUTE; - else if(!strcmp(tok, "seconds")) - ret |= RRDR_OPTION_SECONDS; - else if(!strcmp(tok, "null2zero")) - ret |= RRDR_OPTION_NULL2ZERO; - else if(!strcmp(tok, "objectrows")) - ret |= RRDR_OPTION_OBJECTSROWS; - else if(!strcmp(tok, "google_json")) - ret |= RRDR_OPTION_GOOGLE_JSON; - else if(!strcmp(tok, "percentage")) - ret |= RRDR_OPTION_PERCENTAGE; - else if(!strcmp(tok, "unaligned")) - ret |= RRDR_OPTION_NOT_ALIGNED; + uint32_t hash = simple_hash(tok); + int i; + for(i = 0; api_v1_data_options[i].name ; i++) { + if (unlikely(hash == api_v1_data_options[i].hash && !strcmp(tok, api_v1_data_options[i].name))) { + ret |= api_v1_data_options[i].value; + break; + } + } } return ret; } inline uint32_t web_client_api_request_v1_data_format(char *name) { - if(!strcmp(name, DATASOURCE_FORMAT_DATATABLE_JSON)) // datatable - return DATASOURCE_DATATABLE_JSON; - - else if(!strcmp(name, DATASOURCE_FORMAT_DATATABLE_JSONP)) // datasource - return DATASOURCE_DATATABLE_JSONP; - - else if(!strcmp(name, DATASOURCE_FORMAT_JSON)) // json - return DATASOURCE_JSON; - - else if(!strcmp(name, DATASOURCE_FORMAT_JSONP)) // jsonp - return DATASOURCE_JSONP; - - else if(!strcmp(name, DATASOURCE_FORMAT_SSV)) // ssv - return DATASOURCE_SSV; - - else if(!strcmp(name, DATASOURCE_FORMAT_CSV)) // csv - return DATASOURCE_CSV; - - else if(!strcmp(name, DATASOURCE_FORMAT_TSV) || !strcmp(name, "tsv-excel")) // tsv - return DATASOURCE_TSV; - - else if(!strcmp(name, DATASOURCE_FORMAT_HTML)) // html - return DATASOURCE_HTML; - - else if(!strcmp(name, DATASOURCE_FORMAT_JS_ARRAY)) // array - return DATASOURCE_JS_ARRAY; + uint32_t hash = simple_hash(name); + int i; - else if(!strcmp(name, DATASOURCE_FORMAT_SSV_COMMA)) // ssvcomma - return DATASOURCE_SSV_COMMA; - - else if(!strcmp(name, DATASOURCE_FORMAT_CSV_JSON_ARRAY)) // csvjsonarray - return DATASOURCE_CSV_JSON_ARRAY; + for(i = 0; api_v1_data_formats[i].name ; i++) { + if (unlikely(hash == api_v1_data_formats[i].hash && !strcmp(name, api_v1_data_formats[i].name))) { + return api_v1_data_formats[i].value; + } + } return DATASOURCE_JSON; } inline uint32_t web_client_api_request_v1_data_google_format(char *name) { - if(!strcmp(name, "json")) - return DATASOURCE_DATATABLE_JSONP; - - else if(!strcmp(name, "html")) - return DATASOURCE_HTML; + uint32_t hash = simple_hash(name); + int i; - else if(!strcmp(name, "csv")) - return DATASOURCE_CSV; - - else if(!strcmp(name, "tsv-excel")) - return DATASOURCE_TSV; + for(i = 0; api_v1_data_google_formats[i].name ; i++) { + if (unlikely(hash == api_v1_data_google_formats[i].hash && !strcmp(name, api_v1_data_google_formats[i].name))) { + return api_v1_data_google_formats[i].value; + } + } return DATASOURCE_JSON; } @@ -371,7 +413,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u if(!st) st = rrdset_find_byname(host, chart); if(!st) { buffer_no_cacheable(w->response.data); - buffer_svg(w->response.data, "chart not found", NAN, "", NULL, NULL, -1); + buffer_svg(w->response.data, "chart not found", NAN, "", NULL, NULL, -1, 0); ret = 200; goto cleanup; } @@ -382,7 +424,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u rc = rrdcalc_find(st, alarm); if (!rc) { buffer_no_cacheable(w->response.data); - buffer_svg(w->response.data, "alarm not found", NAN, "", NULL, NULL, -1); + buffer_svg(w->response.data, "alarm not found", NAN, "", NULL, NULL, -1, 0); ret = 200; goto cleanup; } @@ -498,7 +540,9 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u units, label_color, value_color, - precision); + precision, + options + ); ret = 200; } else { @@ -532,7 +576,9 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u units, label_color, value_color, - precision); + precision, + options + ); } cleanup: @@ -816,55 +862,65 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client * #endif /* NETDATA_INTERNAL_CHECKS */ } - if(respect_web_browser_do_not_track_policy && web_client_has_donottrack(w)) { + if(unlikely(respect_web_browser_do_not_track_policy && web_client_has_donottrack(w))) { buffer_flush(w->response.data); buffer_sprintf(w->response.data, "Your web browser is sending 'DNT: 1' (Do Not Track). The registry requires persistent cookies on your browser to work."); return 400; } - if(action == 'A' && (!machine_guid || !machine_url || !url_name)) { - error("Invalid registry request - access requires these parameters: machine ('%s'), url ('%s'), name ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", url_name?url_name:"UNSET"); - buffer_flush(w->response.data); - buffer_strcat(w->response.data, "Invalid registry Access request."); - return 400; - } - else if(action == 'D' && (!machine_guid || !machine_url || !delete_url)) { - error("Invalid registry request - delete requires these parameters: machine ('%s'), url ('%s'), delete_url ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", delete_url?delete_url:"UNSET"); - buffer_flush(w->response.data); - buffer_strcat(w->response.data, "Invalid registry Delete request."); - return 400; + if(unlikely(action == 'H')) { + // HELLO request, dashboard ACL + if(unlikely(!web_client_can_access_dashboard(w))) + return web_client_permission_denied(w); } - else if(action == 'S' && (!machine_guid || !machine_url || !search_machine_guid)) { - error("Invalid registry request - search requires these parameters: machine ('%s'), url ('%s'), for ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", search_machine_guid?search_machine_guid:"UNSET"); - buffer_flush(w->response.data); - buffer_strcat(w->response.data, "Invalid registry Search request."); - return 400; - } - else if(action == 'W' && (!machine_guid || !machine_url || !to_person_guid)) { - error("Invalid registry request - switching identity requires these parameters: machine ('%s'), url ('%s'), to ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", to_person_guid?to_person_guid:"UNSET"); - buffer_flush(w->response.data); - buffer_strcat(w->response.data, "Invalid registry Switch request."); - return 400; + else { + // everything else, registry ACL + if(unlikely(!web_client_can_access_registry(w))) + return web_client_permission_denied(w); } switch(action) { case 'A': + if(unlikely(!machine_guid || !machine_url || !url_name)) { + error("Invalid registry request - access requires these parameters: machine ('%s'), url ('%s'), name ('%s')", machine_guid ? machine_guid : "UNSET", machine_url ? machine_url : "UNSET", url_name ? url_name : "UNSET"); + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "Invalid registry Access request."); + return 400; + } + web_client_enable_tracking_required(w); return registry_request_access_json(host, w, person_guid, machine_guid, machine_url, url_name, now_realtime_sec()); case 'D': + if(unlikely(!machine_guid || !machine_url || !delete_url)) { + error("Invalid registry request - delete requires these parameters: machine ('%s'), url ('%s'), delete_url ('%s')", machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", delete_url?delete_url:"UNSET"); + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "Invalid registry Delete request."); + return 400; + } + web_client_enable_tracking_required(w); return registry_request_delete_json(host, w, person_guid, machine_guid, machine_url, delete_url, now_realtime_sec()); case 'S': + if(unlikely(!machine_guid || !machine_url || !search_machine_guid)) { + error("Invalid registry request - search requires these parameters: machine ('%s'), url ('%s'), for ('%s')", machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", search_machine_guid?search_machine_guid:"UNSET"); + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "Invalid registry Search request."); + return 400; + } + web_client_enable_tracking_required(w); return registry_request_search_json(host, w, person_guid, machine_guid, machine_url, search_machine_guid, now_realtime_sec()); case 'W': + if(unlikely(!machine_guid || !machine_url || !to_person_guid)) { + error("Invalid registry request - switching identity requires these parameters: machine ('%s'), url ('%s'), to ('%s')", machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", to_person_guid?to_person_guid:"UNSET"); + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "Invalid registry Switch request."); + return 400; + } + web_client_enable_tracking_required(w); return registry_request_switch_json(host, w, person_guid, machine_guid, machine_url, to_person_guid, now_realtime_sec()); @@ -878,19 +934,40 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client * } } +static struct api_command { + const char *command; + uint32_t hash; + WEB_CLIENT_ACL acl; + int (*callback)(RRDHOST *host, struct web_client *w, char *url); +} api_commands[] = { + { "data", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_data }, + { "chart", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_chart }, + { "charts", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_charts }, + + // registry checks the ACL by itself, so we allow everything + { "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry }, + + // badges can be fetched with both dashboard and badge permissions + { "badge.svg", 0, WEB_CLIENT_ACL_DASHBOARD|WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge }, + + { "alarms", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarms }, + { "alarm_log", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarm_log }, + { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarm_variables }, + { "allmetrics", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_allmetrics }, + + // terminator + { NULL, 0, WEB_CLIENT_ACL_NONE, NULL }, +}; + inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url) { - static uint32_t hash_data = 0, hash_chart = 0, hash_charts = 0, hash_registry = 0, hash_badge = 0, hash_alarms = 0, hash_alarm_log = 0, hash_alarm_variables = 0, hash_raw = 0; + static int initialized = 0; + int i; + + if(unlikely(initialized == 0)) { + initialized = 1; - if(unlikely(hash_data == 0)) { - hash_data = simple_hash("data"); - hash_chart = simple_hash("chart"); - hash_charts = simple_hash("charts"); - hash_registry = simple_hash("registry"); - hash_badge = simple_hash("badge.svg"); - hash_alarms = simple_hash("alarms"); - hash_alarm_log = simple_hash("alarm_log"); - hash_alarm_variables = simple_hash("alarm_variables"); - hash_raw = simple_hash("allmetrics"); + for(i = 0; api_commands[i].command ; i++) + api_commands[i].hash = simple_hash(api_commands[i].command); } // get the command @@ -899,39 +976,19 @@ inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char * debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, tok); uint32_t hash = simple_hash(tok); - if(hash == hash_data && !strcmp(tok, "data")) - return web_client_api_request_v1_data(host, w, url); - - else if(hash == hash_chart && !strcmp(tok, "chart")) - return web_client_api_request_v1_chart(host, w, url); - - else if(hash == hash_charts && !strcmp(tok, "charts")) - return web_client_api_request_v1_charts(host, w, url); - - else if(hash == hash_registry && !strcmp(tok, "registry")) - return web_client_api_request_v1_registry(host, w, url); - - else if(hash == hash_badge && !strcmp(tok, "badge.svg")) - return web_client_api_request_v1_badge(host, w, url); - - else if(hash == hash_alarms && !strcmp(tok, "alarms")) - return web_client_api_request_v1_alarms(host, w, url); - - else if(hash == hash_alarm_log && !strcmp(tok, "alarm_log")) - return web_client_api_request_v1_alarm_log(host, w, url); + for(i = 0; api_commands[i].command ;i++) { + if(unlikely(hash == api_commands[i].hash && !strcmp(tok, api_commands[i].command))) { + if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl)) + return web_client_permission_denied(w); - else if(hash == hash_alarm_variables && !strcmp(tok, "alarm_variables")) - return web_client_api_request_v1_alarm_variables(host, w, url); - - else if(hash == hash_raw && !strcmp(tok, "allmetrics")) - return web_client_api_request_v1_allmetrics(host, w, url); - - else { - buffer_flush(w->response.data); - buffer_strcat(w->response.data, "Unsupported v1 API command: "); - buffer_strcat_htmlescape(w->response.data, tok); - return 404; + return api_commands[i].callback(host, w, url); + } } + + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "Unsupported v1 API command: "); + buffer_strcat_htmlescape(w->response.data, tok); + return 404; } else { buffer_flush(w->response.data); diff --git a/src/web_api_v1.h b/src/web_api_v1.h index e980edb1d..6f4de1aba 100644 --- a/src/web_api_v1.h +++ b/src/web_api_v1.h @@ -18,4 +18,6 @@ extern int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c extern int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url); extern int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url); +extern void web_client_api_v1_init(void); + #endif //NETDATA_WEB_API_V1_H diff --git a/src/web_buffer_svg.c b/src/web_buffer_svg.c index 287bbd6b8..25128bd32 100644 --- a/src/web_buffer_svg.c +++ b/src/web_buffer_svg.c @@ -382,7 +382,7 @@ static inline char *format_value_with_precision_and_unit(char *value_string, siz calculated_number abs = value; if(isless(value, 0)) { lstop = 1; - abs = -value; + abs = calculated_number_fabs(value); } if(isgreaterequal(abs, 1000)) { @@ -610,16 +610,22 @@ static inline const char *color_map(const char *color) { return color; } +typedef enum color_comparison { + COLOR_COMPARE_EQUAL, + COLOR_COMPARE_NOTEQUAL, + COLOR_COMPARE_LESS, + COLOR_COMPARE_LESSEQUAL, + COLOR_COMPARE_GREATER, + COLOR_COMPARE_GREATEREQUAL, +} BADGE_COLOR_COMPARISON; + static inline void calc_colorz(const char *color, char *final, size_t len, calculated_number value) { - int value_is_null = 0; - if(isnan(value) || isinf(value)) { - value = 0.0; - value_is_null = 1; - } + if(isnan(value) || isinf(value)) + value = NAN; char color_buffer[256 + 1] = ""; char value_buffer[256 + 1] = ""; - char comparison = '>'; + BADGE_COLOR_COMPARISON comparison = COLOR_COMPARE_GREATER; // example input: // colormin|color:null... @@ -633,8 +639,15 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu while(*t && *t != '|') { switch(*t) { + case '!': + if(t[1] == '=') t++; + comparison = COLOR_COMPARE_NOTEQUAL; + dv = value_buffer; + break; + + case '=': case ':': - comparison = '='; + comparison = COLOR_COMPARE_EQUAL; dv = value_buffer; break; @@ -642,11 +655,11 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu case ')': case '>': if(t[1] == '=') { - comparison = ')'; + comparison = COLOR_COMPARE_GREATEREQUAL; t++; } else - comparison = '>'; + comparison = COLOR_COMPARE_GREATER; dv = value_buffer; break; @@ -654,11 +667,15 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu case '(': case '<': if(t[1] == '=') { - comparison = '('; + comparison = COLOR_COMPARE_LESSEQUAL; + t++; + } + else if(t[1] == '>' || t[1] == ')' || t[1] == '}') { + comparison = COLOR_COMPARE_NOTEQUAL; t++; } else - comparison = '<'; + comparison = COLOR_COMPARE_LESS; dv = value_buffer; break; @@ -689,19 +706,28 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu *dc = '\0'; if(dv) { *dv = '\0'; + calculated_number v; - if(value_is_null) { - if(!*value_buffer || !strcmp(value_buffer, "null")) - break; + if(!*value_buffer || !strcmp(value_buffer, "null")) { + v = NAN; } else { - calculated_number v = str2l(value_buffer); + v = str2l(value_buffer); + if(isnan(v) || isinf(v)) + v = NAN; + } - if(comparison == '<' && value < v) break; - else if(comparison == '(' && value <= v) break; - else if(comparison == '>' && value > v) break; - else if(comparison == ')' && value >= v) break; - else if(comparison == '=' && value == v) break; + if(unlikely(isnan(value) || isnan(v))) { + if(isnan(value) && isnan(v)) + break; + } + else { + if (unlikely(comparison == COLOR_COMPARE_LESS && isless(value, v))) break; + else if (unlikely(comparison == COLOR_COMPARE_LESSEQUAL && islessequal(value, v))) break; + else if (unlikely(comparison == COLOR_COMPARE_GREATER && isgreater(value, v))) break; + else if (unlikely(comparison == COLOR_COMPARE_GREATEREQUAL && isgreaterequal(value, v))) break; + else if (unlikely(comparison == COLOR_COMPARE_EQUAL && !islessgreater(value, v))) break; + else if (unlikely(comparison == COLOR_COMPARE_NOTEQUAL && islessgreater(value, v))) break; } } else @@ -726,7 +752,7 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu // colors #define COLOR_STRING_SIZE 100 -void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision) { +void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision, uint32_t options) { char label_buffer[LABEL_STRING_SIZE + 1] , value_color_buffer[COLOR_STRING_SIZE + 1] , value_string[VALUE_STRING_SIZE + 1] @@ -744,7 +770,7 @@ void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const ch value_color = (isnan(value) || isinf(value))?"#999":"#4c1"; calc_colorz(value_color, value_color_buffer, COLOR_STRING_SIZE, value); - format_value_and_unit(value_string, VALUE_STRING_SIZE, value, units, precision); + format_value_and_unit(value_string, VALUE_STRING_SIZE, (options & RRDR_OPTION_DISPLAY_ABS)?calculated_number_fabs(value):value, units, precision); // we need to copy the label, since verdana11_width may write to it strncpyz(label_buffer, label, LABEL_STRING_SIZE); diff --git a/src/web_buffer_svg.h b/src/web_buffer_svg.h index 49f73e445..c09ef7bca 100644 --- a/src/web_buffer_svg.h +++ b/src/web_buffer_svg.h @@ -1,7 +1,7 @@ #ifndef NETDATA_WEB_BUFFER_SVG_H #define NETDATA_WEB_BUFFER_SVG_H 1 -extern void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision); +extern void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const char *units, const char *label_color, const char *value_color, int precision, uint32_t options); extern char *format_value_and_unit(char *value_string, size_t value_string_len, calculated_number value, const char *units, int precision); #endif /* NETDATA_WEB_BUFFER_SVG_H */ diff --git a/src/web_client.c b/src/web_client.c index 6ec3e11e3..e17bac922 100644 --- a/src/web_client.c +++ b/src/web_client.c @@ -8,6 +8,15 @@ int web_client_timeout = DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS; int respect_web_browser_do_not_track_policy = 0; char *web_x_frame_options = NULL; +SIMPLE_PATTERN *web_allow_connections_from = NULL; +SIMPLE_PATTERN *web_allow_streaming_from = NULL; +SIMPLE_PATTERN *web_allow_netdataconf_from = NULL; + +// WEB_CLIENT_ACL +SIMPLE_PATTERN *web_allow_dashboard_from = NULL; +SIMPLE_PATTERN *web_allow_registry_from = NULL; +SIMPLE_PATTERN *web_allow_badges_from = NULL; + #ifdef NETDATA_WITH_ZLIB int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY; #endif /* NETDATA_WITH_ZLIB */ @@ -50,6 +59,31 @@ static inline int web_client_uncrock_socket(struct web_client *w) { return 0; } +inline int web_client_permission_denied(struct web_client *w) { + w->response.data->contenttype = CT_TEXT_PLAIN; + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "You are not allowed to access this resource."); + w->response.code = 403; + return 403; +} + +static void log_connection(struct web_client *w, const char *msg) { + log_access("%llu: %d '[%s]:%s' '%s'", w->id, gettid(), w->client_ip, w->client_port, msg); +} + +static void web_client_update_acl_matches(struct web_client *w) { + w->acl = WEB_CLIENT_ACL_NONE; + + if(!web_allow_dashboard_from || simple_pattern_matches(web_allow_dashboard_from, w->client_ip)) + w->acl |= WEB_CLIENT_ACL_DASHBOARD; + + if(!web_allow_registry_from || simple_pattern_matches(web_allow_registry_from, w->client_ip)) + w->acl |= WEB_CLIENT_ACL_REGISTRY; + + if(!web_allow_badges_from || simple_pattern_matches(web_allow_badges_from, w->client_ip)) + w->acl |= WEB_CLIENT_ACL_BADGE; +} + struct web_client *web_client_create(int listener) { struct web_client *w; @@ -58,12 +92,25 @@ struct web_client *web_client_create(int listener) { w->mode = WEB_CLIENT_MODE_NORMAL; { - w->ifd = accept_socket(listener, SOCK_NONBLOCK, w->client_ip, sizeof(w->client_ip), w->client_port, sizeof(w->client_port)); + w->ifd = accept_socket(listener, SOCK_NONBLOCK, w->client_ip, sizeof(w->client_ip), w->client_port, sizeof(w->client_port), web_allow_connections_from); + + if(unlikely(!*w->client_ip)) strcpy(w->client_ip, "-"); + if(unlikely(!*w->client_port)) strcpy(w->client_port, "-"); + if (w->ifd == -1) { - error("%llu: Cannot accept new incoming connection.", w->id); + if(errno == EPERM) + log_connection(w, "ACCESS DENIED"); + else { + log_connection(w, "CONNECTION FAILED"); + error("%llu: Failed to accept new incoming connection.", w->id); + } + freez(w); return NULL; } + else + log_connection(w, "CONNECTED"); + w->ofd = w->ifd; int flag = 1; @@ -75,6 +122,8 @@ struct web_client *web_client_create(int listener) { error("%llu: Cannot set SO_KEEPALIVE on socket.", w->id); } + web_client_update_acl_matches(w); + w->response.data = buffer_create(INITIAL_WEB_DATA_LENGTH); w->response.header = buffer_create(HTTP_RESPONSE_HEADER_SIZE); w->response.header_output = buffer_create(HTTP_RESPONSE_HEADER_SIZE); @@ -119,18 +168,45 @@ void web_client_reset(struct web_client *w) { // -------------------------------------------------------------------- - // access log - log_access("%llu: (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'", - w->id, - sent, size, -((size > 0) ? ((size - sent) / (double) size * 100.0) : 0.0), - dt_usec(&w->tv_ready, &w->tv_in) / 1000.0, - dt_usec(&tv, &w->tv_ready) / 1000.0, - dt_usec(&tv, &w->tv_in) / 1000.0, - (w->mode == WEB_CLIENT_MODE_FILECOPY) ? "filecopy" : ((w->mode == WEB_CLIENT_MODE_OPTIONS) - ? "options" : "data"), - w->response.code, - w->last_url + const char *mode; + switch(w->mode) { + case WEB_CLIENT_MODE_FILECOPY: + mode = "FILECOPY"; + break; + + case WEB_CLIENT_MODE_OPTIONS: + mode = "OPTIONS"; + break; + + case WEB_CLIENT_MODE_STREAM: + mode = "STREAM"; + break; + + case WEB_CLIENT_MODE_NORMAL: + mode = "DATA"; + break; + + default: + mode = "UNKNOWN"; + break; + } + + // access log + log_access("%llu: %d '[%s]:%s' '%s' (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %d '%s'", + w->id + , gettid() + , w->client_ip + , w->client_port + , mode + , sent + , size + , -((size > 0) ? ((size - sent) / (double) size * 100.0) : 0.0) + , dt_usec(&w->tv_ready, &w->tv_in) / 1000.0 + , dt_usec(&tv, &w->tv_ready) / 1000.0 + , dt_usec(&tv, &w->tv_in) / 1000.0 + , w->response.code + , w->last_url ); } @@ -271,6 +347,9 @@ gid_t web_files_gid(void) { int mysendfile(struct web_client *w, char *filename) { debug(D_WEB_CLIENT, "%llu: Looking for file '%s/%s'", w->id, netdata_configured_web_dir, filename); + if(!web_client_can_access_dashboard(w)) + return web_client_permission_denied(w); + // skip leading slashes while (*filename == '/') filename++; @@ -553,6 +632,13 @@ static inline int check_host_and_call(RRDHOST *host, struct web_client *w, char return func(host, w, url); } +static inline int check_host_and_dashboard_acl_and_call(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) { + if(!web_client_can_access_dashboard(w)) + return web_client_permission_denied(w); + + return check_host_and_call(host, w, url, func); +} + int web_client_api_request(RRDHOST *host, struct web_client *w, char *url) { // get the api version @@ -1082,25 +1168,28 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch } else if(unlikely(hash == hash_data && strcmp(tok, WEB_PATH_DATA) == 0)) { // old API "data" debug(D_WEB_CLIENT_ACCESS, "%llu: old API data request...", w->id); - return check_host_and_call(host, w, url, web_client_api_old_data_request_json); + return check_host_and_dashboard_acl_and_call(host, w, url, web_client_api_old_data_request_json); } else if(unlikely(hash == hash_datasource && strcmp(tok, WEB_PATH_DATASOURCE) == 0)) { // old API "datasource" debug(D_WEB_CLIENT_ACCESS, "%llu: old API datasource request...", w->id); - return check_host_and_call(host, w, url, web_client_api_old_data_request_jsonp); + return check_host_and_dashboard_acl_and_call(host, w, url, web_client_api_old_data_request_jsonp); } else if(unlikely(hash == hash_graph && strcmp(tok, WEB_PATH_GRAPH) == 0)) { // old API "graph" debug(D_WEB_CLIENT_ACCESS, "%llu: old API graph request...", w->id); - return check_host_and_call(host, w, url, web_client_api_old_graph_request); + return check_host_and_dashboard_acl_and_call(host, w, url, web_client_api_old_graph_request); } else if(unlikely(hash == hash_list && strcmp(tok, "list") == 0)) { // old API "list" debug(D_WEB_CLIENT_ACCESS, "%llu: old API list request...", w->id); - return check_host_and_call(host, w, url, web_client_api_old_list_request); + return check_host_and_dashboard_acl_and_call(host, w, url, web_client_api_old_list_request); } else if(unlikely(hash == hash_all_json && strcmp(tok, "all.json") == 0)) { // old API "all.json" debug(D_WEB_CLIENT_ACCESS, "%llu: old API all.json request...", w->id); - return check_host_and_call(host, w, url, web_client_api_old_all_json); + return check_host_and_dashboard_acl_and_call(host, w, url, web_client_api_old_all_json); } else if(unlikely(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0)) { // netdata.conf + if(unlikely(!web_client_can_access_netdataconf(w))) + return web_client_permission_denied(w); + debug(D_WEB_CLIENT_ACCESS, "%llu: generating netdata.conf ...", w->id); w->response.data->contenttype = CT_TEXT_PLAIN; buffer_flush(w->response.data); @@ -1109,6 +1198,9 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch } #ifdef NETDATA_INTERNAL_CHECKS else if(unlikely(hash == hash_exit && strcmp(tok, "exit") == 0)) { + if(unlikely(!web_client_can_access_netdataconf(w))) + return web_client_permission_denied(w); + w->response.data->contenttype = CT_TEXT_PLAIN; buffer_flush(w->response.data); @@ -1122,6 +1214,9 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch return 200; } else if(unlikely(hash == hash_debug && strcmp(tok, "debug") == 0)) { + if(unlikely(!web_client_can_access_netdataconf(w))) + return web_client_permission_denied(w); + buffer_flush(w->response.data); // get the name of the data to show @@ -1159,6 +1254,9 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch return 400; } else if(unlikely(hash == hash_mirror && strcmp(tok, "mirror") == 0)) { + if(unlikely(!web_client_can_access_netdataconf(w))) + return web_client_permission_denied(w); + debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id); // replace the zero bytes with spaces @@ -1189,10 +1287,20 @@ void web_client_process_request(struct web_client *w) { case HTTP_VALIDATION_OK: switch(w->mode) { case WEB_CLIENT_MODE_STREAM: + if(unlikely(!web_client_can_access_stream(w))) { + web_client_permission_denied(w); + return; + } + w->response.code = rrdpush_receiver_thread_spawn(localhost, w, w->decoded_url); return; case WEB_CLIENT_MODE_OPTIONS: + if(unlikely(!web_client_can_access_dashboard(w) && !web_client_can_access_registry(w) && !web_client_can_access_badges(w))) { + web_client_permission_denied(w); + return; + } + w->response.data->contenttype = CT_TEXT_PLAIN; buffer_flush(w->response.data); buffer_strcat(w->response.data, "OK"); @@ -1201,6 +1309,11 @@ void web_client_process_request(struct web_client *w) { case WEB_CLIENT_MODE_FILECOPY: case WEB_CLIENT_MODE_NORMAL: + if(unlikely(!web_client_can_access_dashboard(w) && !web_client_can_access_registry(w) && !web_client_can_access_badges(w))) { + web_client_permission_denied(w); + return; + } + w->response.code = web_client_process_url(localhost, w, w->decoded_url); break; } @@ -1607,8 +1720,6 @@ void *web_client_main(void *ptr) int retval, timeout; nfds_t fdmax = 0; - log_access("%llu: %s port %s connected on thread task id %d", w->id, w->client_ip, w->client_port, gettid()); - for(;;) { if(unlikely(netdata_exit)) break; @@ -1716,9 +1827,11 @@ void *web_client_main(void *ptr) } } + if(w->mode != WEB_CLIENT_MODE_STREAM) + log_connection(w, "DISCONNECTED"); + web_client_reset(w); - log_access("%llu: %s port %s disconnected from thread task id %d", w->id, w->client_ip, w->client_port, gettid()); debug(D_WEB_CLIENT, "%llu: done...", w->id); // close the sockets/files now diff --git a/src/web_client.h b/src/web_client.h index 126a494f0..a07558e1e 100644 --- a/src/web_client.h +++ b/src/web_client.h @@ -44,7 +44,7 @@ typedef enum web_client_flags { //#define web_client_flag_set(w, flag) __atomic_or_fetch(&((w)->flags), flag, __ATOMIC_SEQ_CST) //#define web_client_flag_clear(w, flag) __atomic_and_fetch(&((w)->flags), ~flag, __ATOMIC_SEQ_CST) //#else -#define web_client_flag_check(w, flag) ((w)->flags & flag) +#define web_client_flag_check(w, flag) ((w)->flags & (flag)) #define web_client_flag_set(w, flag) (w)->flags |= flag #define web_client_flag_clear(w, flag) (w)->flags &= ~flag //#endif @@ -77,6 +77,7 @@ typedef enum web_client_flags { #define web_client_set_tcp(w) web_client_flag_set(w, WEB_CLIENT_FLAG_TCP_CLIENT) #define web_client_set_unix(w) web_client_flag_set(w, WEB_CLIENT_FLAG_UNIX_CLIENT) +#define web_client_check_unix(w) web_client_flag_check(w, WEB_CLIENT_FLAG_UNIX_CLIENT) #define web_client_is_corkable(w) web_client_flag_check(w, WEB_CLIENT_FLAG_TCP_CLIENT) @@ -107,11 +108,30 @@ struct response { }; +typedef enum web_client_acl { + WEB_CLIENT_ACL_NONE = 0, + WEB_CLIENT_ACL_NOCHECK = 0, + WEB_CLIENT_ACL_DASHBOARD = 1 << 0, + WEB_CLIENT_ACL_REGISTRY = 1 << 1, + WEB_CLIENT_ACL_BADGE = 1 << 2 +} WEB_CLIENT_ACL; + +#define web_client_can_access_dashboard(w) ((w)->acl & WEB_CLIENT_ACL_DASHBOARD) +#define web_client_can_access_registry(w) ((w)->acl & WEB_CLIENT_ACL_REGISTRY) +#define web_client_can_access_badges(w) ((w)->acl & WEB_CLIENT_ACL_BADGE) + +#define web_client_can_access_stream(w) \ + (!web_allow_streaming_from || simple_pattern_matches(web_allow_streaming_from, (w)->client_ip)) + +#define web_client_can_access_netdataconf(w) \ + (!web_allow_netdataconf_from || simple_pattern_matches(web_allow_netdataconf_from, (w)->client_ip)) + struct web_client { unsigned long long id; WEB_CLIENT_FLAGS flags; // status flags for the client WEB_CLIENT_MODE mode; // the operational mode of the client + WEB_CLIENT_ACL acl; // the access list of the client int tcp_cork; // 1 = we have a cork on the socket @@ -142,10 +162,18 @@ struct web_client { }; extern struct web_client *web_clients; +extern SIMPLE_PATTERN *web_allow_connections_from; +extern SIMPLE_PATTERN *web_allow_dashboard_from; +extern SIMPLE_PATTERN *web_allow_registry_from; +extern SIMPLE_PATTERN *web_allow_badges_from; +extern SIMPLE_PATTERN *web_allow_streaming_from; +extern SIMPLE_PATTERN *web_allow_netdataconf_from; extern uid_t web_files_uid(void); extern uid_t web_files_gid(void); +extern int web_client_permission_denied(struct web_client *w); + extern struct web_client *web_client_create(int listener); extern struct web_client *web_client_free(struct web_client *w); extern ssize_t web_client_send(struct web_client *w); diff --git a/src/web_server.c b/src/web_server.c index 72168d15b..d231cbb5c 100644 --- a/src/web_server.c +++ b/src/web_server.c @@ -168,6 +168,11 @@ void *socket_listen_main_multi_threaded(void *ptr) { continue; } + if(api_sockets.fds_families[i] == AF_UNIX) + web_client_set_unix(w); + else + web_client_set_tcp(w); + if(pthread_create(&w->thread, NULL, web_client_main, w) != 0) { error("%llu: failed to create new thread for web client.", w->id); WEB_CLIENT_IS_OBSOLETE(w); @@ -203,8 +208,8 @@ static inline int single_threaded_link_client(struct web_client *w, fd_set *ifds if(unlikely(web_client_check_obsolete(w) || web_client_check_dead(w) || (!web_client_has_wait_receive(w) && !web_client_has_wait_send(w)))) return 1; - if(unlikely(w->ifd < 0 || w->ifd >= FD_SETSIZE || w->ofd < 0 || w->ofd >= FD_SETSIZE)) { - error("%llu: invalid file descriptor, ifd = %d, ofd = %d (required 0 <= fd < FD_SETSIZE (%d)", w->id, w->ifd, w->ofd, FD_SETSIZE); + if(unlikely(w->ifd < 0 || w->ifd >= (int)FD_SETSIZE || w->ofd < 0 || w->ofd >= (int)FD_SETSIZE)) { + error("%llu: invalid file descriptor, ifd = %d, ofd = %d (required 0 <= fd < FD_SETSIZE (%d)", w->id, w->ifd, w->ofd, (int)FD_SETSIZE); return 1; } @@ -261,7 +266,7 @@ void *socket_listen_main_single_threaded(void *ptr) { fatal("LISTENER: no listen sockets available."); size_t i; - for(i = 0; i < FD_SETSIZE ; i++) + for(i = 0; i < (size_t)FD_SETSIZE ; i++) single_threaded_clients[i] = NULL; fd_set ifds, ofds, efds, rifds, rofds, refds; @@ -271,7 +276,7 @@ void *socket_listen_main_single_threaded(void *ptr) { int fdmax = 0; for(i = 0; i < api_sockets.opened ; i++) { - if (api_sockets.fds[i] < 0 || api_sockets.fds[i] >= FD_SETSIZE) + if (api_sockets.fds[i] < 0 || api_sockets.fds[i] >= (int)FD_SETSIZE) fatal("LISTENER: Listen socket %d is not ready, or invalid.", api_sockets.fds[i]); info("Listening on '%s'", (api_sockets.fds_names[i])?api_sockets.fds_names[i]:"UNKNOWN"); @@ -372,3 +377,142 @@ void *socket_listen_main_single_threaded(void *ptr) { pthread_exit(NULL); return NULL; } + + +#if 0 +// new TCP client connected +static void *web_server_add_callback(int fd, int socktype, short int *events) { + (void)fd; + (void)socktype; + + *events = POLLIN; + + debug(D_WEB_CLIENT_ACCESS, "LISTENER on %d: new connection.", fd); + struct web_client *w = web_client_create(fd); + + if(unlikely(socktype == AF_UNIX)) + web_client_set_unix(w); + else + web_client_set_tcp(w); + + return (void *)w; +} + +// TCP client disconnected +static void web_server_del_callback(int fd, int socktype, void *data) { + (void)fd; + (void)socktype; + + struct web_client *w = (struct web_client *)data; + + if(w) { + if(w->ofd == -1 || fd == w->ofd) { + // we free the client, only if the closing fd + // is the client socket + web_client_free(w); + } + } + + return; +} + +// Receive data +static int web_server_rcv_callback(int fd, int socktype, void *data, short int *events) { + (void)fd; + (void)socktype; + + *events = 0; + + struct web_client *w = (struct web_client *)data; + + if(unlikely(!web_client_has_wait_receive(w))) + return -1; + + if(unlikely(web_client_receive(w) < 0)) + return -1; + + if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) { + if(unlikely(w->ifd != -1 && w->ifd != fd)) { + // FIXME: we switched input fd + // add a new socket to poll_events, with the same + } + else if(unlikely(w->ifd == -1)) { + // FIXME: we closed input fd + // instruct poll_events() to close fd + return -1; + } + } + else { + debug(D_WEB_CLIENT, "%llu: Processing received data.", w->id); + web_client_process_request(w); + } + + if(unlikely(w->ifd == fd && web_client_has_wait_receive(w))) + *events |= POLLIN; + + if(unlikely(w->ofd == fd && web_client_has_wait_send(w))) + *events |= POLLOUT; + + if(unlikely(*events == 0)) + return -1; + + return 0; +} + +static int web_server_snd_callback(int fd, int socktype, void *data, short int *events) { + (void)fd; + (void)socktype; + + struct web_client *w = (struct web_client *)data; + + if(unlikely(!web_client_has_wait_send(w))) + return -1; + + if(unlikely(web_client_send(w) < 0)) + return -1; + + if(unlikely(w->ifd == fd && web_client_has_wait_receive(w))) + *events |= POLLIN; + + if(unlikely(w->ofd == fd && web_client_has_wait_send(w))) + *events |= POLLOUT; + + if(unlikely(*events == 0)) + return -1; + + return 0; +} + +void *socket_listen_main_single_threaded(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + + web_server_mode = WEB_SERVER_MODE_SINGLE_THREADED; + + info("Single-threaded WEB SERVER thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + if(!api_sockets.opened) + fatal("LISTENER: no listen sockets available."); + + poll_events(&api_sockets + , web_server_add_callback + , web_server_del_callback + , web_server_rcv_callback + , web_server_snd_callback + , web_allow_connections_from + , NULL + ); + + debug(D_WEB_CLIENT, "LISTENER: exit!"); + listen_sockets_close(&api_sockets); + + static_thread->enabled = 0; + pthread_exit(NULL); + return NULL; +} +#endif diff --git a/src/zfs_common.c b/src/zfs_common.c index 7fa05b03e..0915416f5 100644 --- a/src/zfs_common.c +++ b/src/zfs_common.c @@ -1,9 +1,9 @@ #include "common.h" #include "zfs_common.h" -extern struct arcstats arcstats; +struct arcstats arcstats = { 0 }; -void generate_charts_arcstats(int update_every) { +void generate_charts_arcstats(const char *plugin, int update_every) { // ARC reads unsigned long long aread = arcstats.hits + arcstats.misses; @@ -24,8 +24,8 @@ void generate_charts_arcstats(int update_every) { unsigned long long mread = mhit + mmiss; // l2 reads - unsigned long long l2hit = arcstats.l2_hits + arcstats.l2_misses; - unsigned long long l2miss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses; + unsigned long long l2hit = arcstats.l2_hits; + unsigned long long l2miss = arcstats.l2_misses; unsigned long long l2read = l2hit + l2miss; // -------------------------------------------------------------------- @@ -46,6 +46,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS ARC Size" , "MB" + , plugin + , "zfs" , 2000 , update_every , RRDSET_TYPE_AREA @@ -68,7 +70,7 @@ void generate_charts_arcstats(int update_every) { // -------------------------------------------------------------------- - if(likely(l2exist)) { + if(likely(arcstats.l2exist)) { static RRDSET *st_l2_size = NULL; static RRDDIM *rd_l2_size = NULL; static RRDDIM *rd_l2_asize = NULL; @@ -82,6 +84,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS L2 ARC Size" , "MB" + , plugin + , "zfs" , 2000 , update_every , RRDSET_TYPE_AREA @@ -117,6 +121,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS Reads" , "reads/s" + , plugin + , "zfs" , 2010 , update_every , RRDSET_TYPE_AREA @@ -127,7 +133,7 @@ void generate_charts_arcstats(int update_every) { rd_pread = rrddim_add(st_reads, "preads", "prefetch", 1, 1, RRD_ALGORITHM_INCREMENTAL); rd_mread = rrddim_add(st_reads, "mreads", "metadata", 1, 1, RRD_ALGORITHM_INCREMENTAL); - if(l2exist) + if(arcstats.l2exist) rd_l2read = rrddim_add(st_reads, "l2reads", "l2", 1, 1, RRD_ALGORITHM_INCREMENTAL); } else @@ -138,7 +144,7 @@ void generate_charts_arcstats(int update_every) { rrddim_set_by_pointer(st_reads, rd_pread, pread); rrddim_set_by_pointer(st_reads, rd_mread, mread); - if(l2exist) + if(arcstats.l2exist) rrddim_set_by_pointer(st_reads, rd_l2read, l2read); rrdset_done(st_reads); @@ -146,7 +152,7 @@ void generate_charts_arcstats(int update_every) { // -------------------------------------------------------------------- - if(likely(l2exist)) { + if(likely(arcstats.l2exist)) { static RRDSET *st_l2bytes = NULL; static RRDDIM *rd_l2_read_bytes = NULL; static RRDDIM *rd_l2_write_bytes = NULL; @@ -160,6 +166,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS ARC L2 Read/Write Rate" , "kilobytes/s" + , plugin + , "zfs" , 2200 , update_every , RRDSET_TYPE_AREA @@ -192,6 +200,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS ARC Hits" , "percentage" + , plugin + , "zfs" , 2020 , update_every , RRDSET_TYPE_STACKED @@ -224,6 +234,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS Demand Hits" , "percentage" + , plugin + , "zfs" , 2030 , update_every , RRDSET_TYPE_STACKED @@ -256,6 +268,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS Prefetch Hits" , "percentage" + , plugin + , "zfs" , 2040 , update_every , RRDSET_TYPE_STACKED @@ -288,6 +302,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS Metadata Hits" , "percentage" + , plugin + , "zfs" , 2050 , update_every , RRDSET_TYPE_STACKED @@ -306,7 +322,7 @@ void generate_charts_arcstats(int update_every) { // -------------------------------------------------------------------- - if(likely(l2exist)) { + if(likely(arcstats.l2exist)) { static RRDSET *st_l2hits = NULL; static RRDDIM *rd_l2hits = NULL; static RRDDIM *rd_l2misses = NULL; @@ -320,6 +336,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS L2 Hits" , "percentage" + , plugin + , "zfs" , 2060 , update_every , RRDSET_TYPE_STACKED @@ -354,6 +372,8 @@ void generate_charts_arcstats(int update_every) { , NULL , "ZFS List Hits" , "hits/s" + , plugin + , "zfs" , 2100 , update_every , RRDSET_TYPE_AREA @@ -375,7 +395,7 @@ void generate_charts_arcstats(int update_every) { } } -void generate_charts_arc_summary(int update_every) { +void generate_charts_arc_summary(const char *plugin, int update_every) { unsigned long long arc_accesses_total = arcstats.hits + arcstats.misses; unsigned long long real_hits = arcstats.mfu_hits + arcstats.mru_hits; unsigned long long real_misses = arc_accesses_total - real_hits; @@ -411,6 +431,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS ARC Size Breakdown" , "percentage" + , plugin + , "zfs" , 2020 , update_every , RRDSET_TYPE_STACKED @@ -448,6 +470,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS Memory Operations" , "operations/s" + , plugin + , "zfs" , 2023 , update_every , RRDSET_TYPE_LINE @@ -492,6 +516,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS Important Operations" , "operations/s" + , plugin + , "zfs" , 2022 , update_every , RRDSET_TYPE_LINE @@ -528,6 +554,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS Actual Cache Hits" , "percentage" + , plugin + , "zfs" , 2019 , update_every , RRDSET_TYPE_STACKED @@ -560,6 +588,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS Data Demand Efficiency" , "percentage" + , plugin + , "zfs" , 2031 , update_every , RRDSET_TYPE_STACKED @@ -592,6 +622,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS Data Prefetch Efficiency" , "percentage" + , plugin + , "zfs" , 2032 , update_every , RRDSET_TYPE_STACKED @@ -624,6 +656,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS ARC Hash Elements" , "elements" + , plugin + , "zfs" , 2300 , update_every , RRDSET_TYPE_LINE @@ -656,6 +690,8 @@ void generate_charts_arc_summary(int update_every) { , NULL , "ZFS ARC Hash Chains" , "chains" + , plugin + , "zfs" , 2310 , update_every , RRDSET_TYPE_LINE diff --git a/src/zfs_common.h b/src/zfs_common.h index 9d3aa7dfb..4494e70ca 100644 --- a/src/zfs_common.h +++ b/src/zfs_common.h @@ -8,6 +8,7 @@ #define ZFS_FAMILY_HASH "hashes" struct arcstats { + // values unsigned long long hits; unsigned long long misses; unsigned long long demand_data_hits; @@ -99,11 +100,12 @@ struct arcstats { unsigned long long arc_meta_min; unsigned long long arc_need_free; unsigned long long arc_sys_free; -}; -int l2exist; + // flags + int l2exist; +}; -void generate_charts_arcstats(int update_every); -void generate_charts_arc_summary(int update_every); +void generate_charts_arcstats(const char *plugin, int update_every); +void generate_charts_arc_summary(const char *plugin, int update_every); #endif //NETDATA_ZFS_COMMON_H diff --git a/system/netdata-openrc.in b/system/netdata-openrc.in index 465e1232c..ce76f7e64 100644 --- a/system/netdata-openrc.in +++ b/system/netdata-openrc.in @@ -23,7 +23,7 @@ extra_started_commands="getconf" pidfile="/run/netdata.pid" -command="${NETDATA_INSTALL_PATH}/usr/sbin/netdata" +command="@sbindir_POST@/netdata" command_background="yes" command_args="-P ${pidfile} ${NETDATA_EXTRA_ARGS}" # start_stop_daemon_args="-u ${NETDATA_OWNER}" @@ -38,13 +38,13 @@ depend() { } start_post() { - if [[ ! -f @sysconfdir_POST@/netdata/netdata.conf ]]; then + if [ ! -f @sysconfdir_POST@/netdata/netdata.conf ]; then ebegin "Downloading default configuration to @sysconfdir_POST@/netdata/netdata.conf" sleep 2 curl -s -o @sysconfdir_POST@/netdata/netdata.conf.new "${NETDATA_CONFIG_URL}" ret=$? - if [[ $ret -eq 0 && -s @sysconfdir_POST@/netdata/netdata.conf.new ]]; then - mv @sysconfdir_POST@/netdata/netdata.conf{.new,} + if [ $ret -eq 0 && -s @sysconfdir_POST@/netdata/netdata.conf.new ]; then + mv @sysconfdir_POST@/netdata/netdata.conf.new @sysconfdir_POST@/netdata/netdata.conf else ret=1 rm @sysconfdir_POST@/netdata/netdata.conf.new 2>/dev/null @@ -57,20 +57,20 @@ stop_post() { local result ret=0 count=0 sigkill=0 ebegin "Waiting for netdata to save its database" - while [[ -f "${pidfile}" ]]; do - if [[ $count -gt ${NETDATA_WAIT_EXIT_TIMEOUT} ]]; then + while [ -f "${pidfile}" ]; do + if [ $count -gt ${NETDATA_WAIT_EXIT_TIMEOUT} ]; then sigkill=1 break fi - count=$[count + 1] + count=$((count + 1)) kill -0 $(cat ${pidfile}) 2>/dev/null ret=$? test $ret -eq 0 && sleep 1 done eend $sigkill - if [[ $sigkill -eq 1 && -f "${pidfile}" ]]; then + if [ $sigkill -eq 1 -a -f "${pidfile}" ]; then ebegin "Netdata is taking too long to exit, forcing it to quit" kill -SIGKILL $(cat ${pidfile}) 2>/dev/null eend $? diff --git a/system/netdata.service.in b/system/netdata.service.in index 1d4af988a..58cdff225 100644 --- a/system/netdata.service.in +++ b/system/netdata.service.in @@ -9,6 +9,9 @@ Group=netdata RuntimeDirectory=netdata RuntimeDirectoryMode=0775 ExecStart=@sbindir_POST@/netdata -P /run/netdata/netdata.pid -D +ExecStartPre=/bin/mkdir -p @localstatedir_POST@/cache/netdata +ExecStartPre=/bin/chown -R netdata:netdata @localstatedir_POST@/cache/netdata +PermissionsStartOnly=true # saving a big db on slow disks may need some time TimeoutStopSec=60 diff --git a/web/Makefile.am b/web/Makefile.am index b587f5a17..02d893174 100644 --- a/web/Makefile.am +++ b/web/Makefile.am @@ -39,6 +39,7 @@ dist_webold_DATA = \ weblibdir=$(webdir)/lib dist_weblib_DATA = \ lib/bootstrap-3.3.7.min.js \ + lib/bootstrap-slider-10.0.0.min.js \ lib/bootstrap-table-1.11.0.min.js \ lib/bootstrap-table-export-1.11.0.min.js \ lib/bootstrap-toggle-2.2.2.min.js \ @@ -46,13 +47,16 @@ dist_weblib_DATA = \ lib/d3-3.5.17.min.js \ lib/dygraph-combined-dd74404.js \ lib/dygraph-smooth-plotter-dd74404.js \ + lib/fontawesome-all-5.0.1.min.js \ lib/gauge-1.3.2.min.js \ lib/jquery-2.2.4.min.js \ lib/jquery.easypiechart-97b5824.min.js \ - lib/perfect-scrollbar-0.6.15.min.js \ lib/jquery.peity-3.2.0.min.js \ lib/jquery.sparkline-2.1.2.min.js \ + lib/lz-string-1.4.4.min.js \ lib/morris-0.5.1.min.js \ + lib/pako-1.0.6.min.js \ + lib/perfect-scrollbar-0.6.15.min.js \ lib/raphael-2.2.4-min.js \ lib/tableExport-1.6.0.min.js \ $(NULL) @@ -63,8 +67,8 @@ dist_webcss_DATA = \ css/bootstrap-3.3.7.css \ css/bootstrap-theme-3.3.7.min.css \ css/bootstrap-slate-flat-3.3.7.css \ + css/bootstrap-slider-10.0.0.min.css \ css/bootstrap-toggle-2.2.2.min.css \ - css/font-awesome.min.css \ css/c3-0.4.11.min.css \ $(NULL) @@ -75,12 +79,6 @@ dist_webfonts_DATA = \ fonts/glyphicons-halflings-regular.ttf \ fonts/glyphicons-halflings-regular.woff \ fonts/glyphicons-halflings-regular.woff2 \ - fonts/FontAwesome.otf \ - fonts/fontawesome-webfont.eot \ - fonts/fontawesome-webfont.svg \ - fonts/fontawesome-webfont.ttf \ - fonts/fontawesome-webfont.woff \ - fonts/fontawesome-webfont.woff2 \ $(NULL) webimagesdir=$(webdir)/images diff --git a/web/Makefile.in b/web/Makefile.in index 42939d2a5..92f5c0887 100644 --- a/web/Makefile.in +++ b/web/Makefile.in @@ -340,6 +340,7 @@ dist_webold_DATA = \ weblibdir = $(webdir)/lib dist_weblib_DATA = \ lib/bootstrap-3.3.7.min.js \ + lib/bootstrap-slider-10.0.0.min.js \ lib/bootstrap-table-1.11.0.min.js \ lib/bootstrap-table-export-1.11.0.min.js \ lib/bootstrap-toggle-2.2.2.min.js \ @@ -347,13 +348,16 @@ dist_weblib_DATA = \ lib/d3-3.5.17.min.js \ lib/dygraph-combined-dd74404.js \ lib/dygraph-smooth-plotter-dd74404.js \ + lib/fontawesome-all-5.0.1.min.js \ lib/gauge-1.3.2.min.js \ lib/jquery-2.2.4.min.js \ lib/jquery.easypiechart-97b5824.min.js \ - lib/perfect-scrollbar-0.6.15.min.js \ lib/jquery.peity-3.2.0.min.js \ lib/jquery.sparkline-2.1.2.min.js \ + lib/lz-string-1.4.4.min.js \ lib/morris-0.5.1.min.js \ + lib/pako-1.0.6.min.js \ + lib/perfect-scrollbar-0.6.15.min.js \ lib/raphael-2.2.4-min.js \ lib/tableExport-1.6.0.min.js \ $(NULL) @@ -364,8 +368,8 @@ dist_webcss_DATA = \ css/bootstrap-3.3.7.css \ css/bootstrap-theme-3.3.7.min.css \ css/bootstrap-slate-flat-3.3.7.css \ + css/bootstrap-slider-10.0.0.min.css \ css/bootstrap-toggle-2.2.2.min.css \ - css/font-awesome.min.css \ css/c3-0.4.11.min.css \ $(NULL) @@ -376,12 +380,6 @@ dist_webfonts_DATA = \ fonts/glyphicons-halflings-regular.ttf \ fonts/glyphicons-halflings-regular.woff \ fonts/glyphicons-halflings-regular.woff2 \ - fonts/FontAwesome.otf \ - fonts/fontawesome-webfont.eot \ - fonts/fontawesome-webfont.svg \ - fonts/fontawesome-webfont.ttf \ - fonts/fontawesome-webfont.woff \ - fonts/fontawesome-webfont.woff2 \ $(NULL) webimagesdir = $(webdir)/images diff --git a/web/css/bootstrap-slider-10.0.0.min.css b/web/css/bootstrap-slider-10.0.0.min.css new file mode 100644 index 000000000..1cf68b5e7 --- /dev/null +++ b/web/css/bootstrap-slider-10.0.0.min.css @@ -0,0 +1,41 @@ +/*! ======================================================= + VERSION 10.0.0 +========================================================= */ +/*! ========================================================= + * bootstrap-slider.js + * + * Maintainers: + * Kyle Kemp + * - Twitter: @seiyria + * - Github: seiyria + * Rohit Kalkur + * - Twitter: @Rovolutionary + * - Github: rovolution + * + * ========================================================= + * + * bootstrap-slider is released under the MIT License + * Copyright (c) 2017 Kyle Kemp, Rohit Kalkur, and contributors + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ========================================================= */.slider{display:inline-block;vertical-align:middle;position:relative}.slider.slider-horizontal{width:210px;height:20px}.slider.slider-horizontal .slider-track{height:10px;width:100%;margin-top:-5px;top:50%;left:0}.slider.slider-horizontal .slider-selection,.slider.slider-horizontal .slider-track-low,.slider.slider-horizontal .slider-track-high{height:100%;top:0;bottom:0}.slider.slider-horizontal .slider-tick,.slider.slider-horizontal .slider-handle{margin-left:-10px}.slider.slider-horizontal .slider-tick.triangle,.slider.slider-horizontal .slider-handle.triangle{position:relative;top:50%;-ms-transform:translateY(-50%);transform:translateY(-50%);border-width:0 10px 10px 10px;width:0;height:0;border-bottom-color:#2e6da4;margin-top:0}.slider.slider-horizontal .slider-tick-container{white-space:nowrap;position:absolute;top:0;left:0;width:100%}.slider.slider-horizontal .slider-tick-label-container{white-space:nowrap;margin-top:20px}.slider.slider-horizontal .slider-tick-label-container .slider-tick-label{padding-top:4px;display:inline-block;text-align:center}.slider.slider-horizontal .tooltip{-ms-transform:translateX(-50%);transform:translateX(-50%)}.slider.slider-horizontal.slider-rtl .slider-track{left:initial;right:0}.slider.slider-horizontal.slider-rtl .slider-tick,.slider.slider-horizontal.slider-rtl .slider-handle{margin-left:initial;margin-right:-10px}.slider.slider-horizontal.slider-rtl .slider-tick-container{left:initial;right:0}.slider.slider-horizontal.slider-rtl .tooltip{-ms-transform:translateX(50%);transform:translateX(50%)}.slider.slider-vertical{height:210px;width:20px}.slider.slider-vertical .slider-track{width:10px;height:100%;left:25%;top:0}.slider.slider-vertical .slider-selection{width:100%;left:0;top:0;bottom:0}.slider.slider-vertical .slider-track-low,.slider.slider-vertical .slider-track-high{width:100%;left:0;right:0}.slider.slider-vertical .slider-tick,.slider.slider-vertical .slider-handle{margin-top:-10px}.slider.slider-vertical .slider-tick.triangle,.slider.slider-vertical .slider-handle.triangle{border-width:10px 0 10px 10px;width:1px;height:1px;border-left-color:#2e6da4;border-right-color:#2e6da4;margin-left:0;margin-right:0}.slider.slider-vertical .slider-tick-label-container{white-space:nowrap}.slider.slider-vertical .slider-tick-label-container .slider-tick-label{padding-left:4px}.slider.slider-vertical .tooltip{-ms-transform:translateY(-50%);transform:translateY(-50%)}.slider.slider-vertical.slider-rtl .slider-track{left:initial;right:25%}.slider.slider-vertical.slider-rtl .slider-selection{left:initial;right:0}.slider.slider-vertical.slider-rtl .slider-tick.triangle,.slider.slider-vertical.slider-rtl .slider-handle.triangle{border-width:10px 10px 10px 0}.slider.slider-vertical.slider-rtl .slider-tick-label-container .slider-tick-label{padding-left:initial;padding-right:4px}.slider.slider-disabled .slider-handle{background-image:-webkit-linear-gradient(top,#dfdfdf 0,#bebebe 100%);background-image:-o-linear-gradient(top,#dfdfdf 0,#bebebe 100%);background-image:linear-gradient(to bottom,#dfdfdf 0,#bebebe 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdfdfdf',endColorstr='#ffbebebe',GradientType=0)}.slider.slider-disabled .slider-track{background-image:-webkit-linear-gradient(top,#e5e5e5 0,#e9e9e9 100%);background-image:-o-linear-gradient(top,#e5e5e5 0,#e9e9e9 100%);background-image:linear-gradient(to bottom,#e5e5e5 0,#e9e9e9 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffe5e5e5',endColorstr='#ffe9e9e9',GradientType=0);cursor:not-allowed}.slider input{display:none}.slider .tooltip.top{margin-top:-36px}.slider .tooltip-inner{white-space:nowrap;max-width:none}.slider .hide{display:none}.slider-track{position:absolute;cursor:pointer;background-image:-webkit-linear-gradient(top,#f5f5f5 0,#f9f9f9 100%);background-image:-o-linear-gradient(top,#f5f5f5 0,#f9f9f9 100%);background-image:linear-gradient(to bottom,#f5f5f5 0,#f9f9f9 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5',endColorstr='#fff9f9f9',GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);border-radius:4px}.slider-selection{position:absolute;background-image:-webkit-linear-gradient(top,#f9f9f9 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#f9f9f9 0,#f5f5f5 100%);background-image:linear-gradient(to bottom,#f9f9f9 0,#f5f5f5 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff9f9f9',endColorstr='#fff5f5f5',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;border-radius:4px}.slider-selection.tick-slider-selection{background-image:-webkit-linear-gradient(top,#8ac1ef 0,#82b3de 100%);background-image:-o-linear-gradient(top,#8ac1ef 0,#82b3de 100%);background-image:linear-gradient(to bottom,#8ac1ef 0,#82b3de 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff8ac1ef',endColorstr='#ff82b3de',GradientType=0)}.slider-track-low,.slider-track-high{position:absolute;background:transparent;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;border-radius:4px}.slider-handle{position:absolute;top:0;width:20px;height:20px;background-color:#337ab7;background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7',endColorstr='#ff2e6da4',GradientType=0);filter:none;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.2),0 1px 2px rgba(0,0,0,.05);box-shadow:inset 0 1px 0 rgba(255,255,255,.2),0 1px 2px rgba(0,0,0,.05);border:0 solid transparent}.slider-handle.round{border-radius:50%}.slider-handle.triangle{background:transparent none}.slider-handle.custom{background:transparent none}.slider-handle.custom::before{line-height:20px;font-size:20px;content:'\2605';color:#726204}.slider-tick{position:absolute;width:20px;height:20px;background-image:-webkit-linear-gradient(top,#f9f9f9 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#f9f9f9 0,#f5f5f5 100%);background-image:linear-gradient(to bottom,#f9f9f9 0,#f5f5f5 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff9f9f9',endColorstr='#fff5f5f5',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;filter:none;opacity:.8;border:0 solid transparent}.slider-tick.round{border-radius:50%}.slider-tick.triangle{background:transparent none}.slider-tick.custom{background:transparent none}.slider-tick.custom::before{line-height:20px;font-size:20px;content:'\2605';color:#726204}.slider-tick.in-selection{background-image:-webkit-linear-gradient(top,#8ac1ef 0,#82b3de 100%);background-image:-o-linear-gradient(top,#8ac1ef 0,#82b3de 100%);background-image:linear-gradient(to bottom,#8ac1ef 0,#82b3de 100%);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff8ac1ef',endColorstr='#ff82b3de',GradientType=0);opacity:1} \ No newline at end of file diff --git a/web/dashboard.css b/web/dashboard.css index 42ffa3ddb..79febaa4f 100644 --- a/web/dashboard.css +++ b/web/dashboard.css @@ -44,6 +44,8 @@ body { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* required for child elements to have absolute position */ position: relative; @@ -54,6 +56,8 @@ body { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* required for child elements to have absolute position */ position: relative; @@ -70,6 +74,8 @@ body { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* required for child elements to have absolute position */ position: relative; @@ -93,6 +99,8 @@ body { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* fix minimum scrollbar issue in firefox */ min-height: 99px; @@ -108,9 +116,9 @@ body { bottom: 0px; right: 0px; height: 15px; - width: 30px; + width: 20px; background-color: White; - font-size: 12px; + font-size: 15px; vertical-align: middle; line-height: 15px; cursor: ns-resize; @@ -286,6 +294,7 @@ body { white-space: nowrap; display: inline-block; cursor: pointer; + -webkit-print-color-adjust: exact; } .netdata-legend-value { diff --git a/web/dashboard.js b/web/dashboard.js index f119a5370..dc7f4ea32 100644 --- a/web/dashboard.js +++ b/web/dashboard.js @@ -45,6 +45,10 @@ * (default: null) */ /*global netdataServer *//* string, the URL of the netdata server to use * (default: the URL the page is hosted at) */ +/*global netdataServerStatic *//* string, the URL of the netdata server to use for static files + * (default: netdataServer) */ +/*global netdataSnapshotData *//* object, a netdata snapshot loaded + * (default: null) */ // ---------------------------------------------------------------------------- // global namespace @@ -123,32 +127,43 @@ var NETDATA = window.NETDATA || {}; else if(NETDATA.serverDefault.slice(-1) !== '/') NETDATA.serverDefault += '/'; + if(typeof netdataServerStatic !== 'undefined' && netdataServerStatic !== null && netdataServerStatic !== '') { + NETDATA.serverStatic = netdataServerStatic; + if(NETDATA.serverStatic.slice(-1) !== '/') + NETDATA.serverStatic += '/'; + } + else { + NETDATA.serverStatic = NETDATA.serverDefault; + } + + // default URLs for all the external files we need // make them RELATIVE so that the whole thing can also be // installed under a web server - NETDATA.jQuery = NETDATA.serverDefault + 'lib/jquery-2.2.4.min.js'; - NETDATA.peity_js = NETDATA.serverDefault + 'lib/jquery.peity-3.2.0.min.js'; - NETDATA.sparkline_js = NETDATA.serverDefault + 'lib/jquery.sparkline-2.1.2.min.js'; - NETDATA.easypiechart_js = NETDATA.serverDefault + 'lib/jquery.easypiechart-97b5824.min.js'; - NETDATA.gauge_js = NETDATA.serverDefault + 'lib/gauge-1.3.2.min.js'; - NETDATA.dygraph_js = NETDATA.serverDefault + 'lib/dygraph-combined-dd74404.js'; - NETDATA.dygraph_smooth_js = NETDATA.serverDefault + 'lib/dygraph-smooth-plotter-dd74404.js'; - NETDATA.raphael_js = NETDATA.serverDefault + 'lib/raphael-2.2.4-min.js'; - NETDATA.c3_js = NETDATA.serverDefault + 'lib/c3-0.4.11.min.js'; - NETDATA.c3_css = NETDATA.serverDefault + 'css/c3-0.4.11.min.css'; - NETDATA.d3_js = NETDATA.serverDefault + 'lib/d3-3.5.17.min.js'; - NETDATA.morris_js = NETDATA.serverDefault + 'lib/morris-0.5.1.min.js'; - NETDATA.morris_css = NETDATA.serverDefault + 'css/morris-0.5.1.css'; + NETDATA.jQuery = NETDATA.serverStatic + 'lib/jquery-2.2.4.min.js'; + NETDATA.peity_js = NETDATA.serverStatic + 'lib/jquery.peity-3.2.0.min.js'; + NETDATA.sparkline_js = NETDATA.serverStatic + 'lib/jquery.sparkline-2.1.2.min.js'; + NETDATA.easypiechart_js = NETDATA.serverStatic + 'lib/jquery.easypiechart-97b5824.min.js'; + NETDATA.gauge_js = NETDATA.serverStatic + 'lib/gauge-1.3.2.min.js'; + NETDATA.dygraph_js = NETDATA.serverStatic + 'lib/dygraph-combined-dd74404.js'; + NETDATA.dygraph_smooth_js = NETDATA.serverStatic + 'lib/dygraph-smooth-plotter-dd74404.js'; + NETDATA.raphael_js = NETDATA.serverStatic + 'lib/raphael-2.2.4-min.js'; + NETDATA.c3_js = NETDATA.serverStatic + 'lib/c3-0.4.11.min.js'; + NETDATA.c3_css = NETDATA.serverStatic + 'css/c3-0.4.11.min.css'; + NETDATA.d3_js = NETDATA.serverStatic + 'lib/d3-3.5.17.min.js'; + NETDATA.morris_js = NETDATA.serverStatic + 'lib/morris-0.5.1.min.js'; + NETDATA.morris_css = NETDATA.serverStatic + 'css/morris-0.5.1.css'; NETDATA.google_js = 'https://www.google.com/jsapi'; NETDATA.themes = { white: { - bootstrap_css: NETDATA.serverDefault + 'css/bootstrap-3.3.7.css', - dashboard_css: NETDATA.serverDefault + 'dashboard.css?v20170725-1', + bootstrap_css: NETDATA.serverStatic + 'css/bootstrap-3.3.7.css', + dashboard_css: NETDATA.serverStatic + 'dashboard.css?v20171208-1', background: '#FFFFFF', foreground: '#000000', grid: '#F0F0F0', axis: '#F0F0F0', + highlight: '#F5F5F5', colors: [ '#3366CC', '#DC3912', '#109618', '#FF9900', '#990099', '#DD4477', '#3B3EAC', '#66AA00', '#0099C6', '#B82E2E', '#AAAA11', '#5574A6', '#994499', '#22AA99', '#6633CC', '#E67300', '#316395', '#8B0707', @@ -160,12 +175,13 @@ var NETDATA = window.NETDATA || {}; gauge_gradient: false }, slate: { - bootstrap_css: NETDATA.serverDefault + 'css/bootstrap-slate-flat-3.3.7.css?v20161229-1', - dashboard_css: NETDATA.serverDefault + 'dashboard.slate.css?v20170725-1', + bootstrap_css: NETDATA.serverStatic + 'css/bootstrap-slate-flat-3.3.7.css?v20161229-1', + dashboard_css: NETDATA.serverStatic + 'dashboard.slate.css?v20171208-1', background: '#272b30', foreground: '#C8C8C8', grid: '#283236', axis: '#283236', + highlight: '#383838', /* colors: [ '#55bb33', '#ff2222', '#0099C6', '#faa11b', '#adbce0', '#DDDD00', '#4178ba', '#f58122', '#a5cc39', '#f58667', '#f5ef89', '#cf93c0', '#a5d18a', '#b8539d', '#3954a3', '#c8a9cf', '#c7de8a', '#fad20a', @@ -202,6 +218,9 @@ var NETDATA = window.NETDATA || {}; // (blue) (red) (orange) (green) (pink) (brown) (purple) (yellow) (gray) //NETDATA.colors = [ '#5DA5DA', '#F15854', '#FAA43A', '#60BD68', '#F17CB0', '#B2912F', '#B276B2', '#DECF3F', '#4D4D4D' ]; + if(typeof netdataSnapshotData === 'undefined') + netdataSnapshotData = null; + if(typeof netdataShowHelp === 'undefined') netdataShowHelp = true; @@ -247,7 +266,6 @@ var NETDATA = window.NETDATA || {}; // if the user does not specify any of these, the following will be used NETDATA.chartDefaults = { - host: NETDATA.serverDefault, // the server to get data from width: '100%', // the chart width - can be null height: '100%', // the chart height - can be null min_width: null, // the chart minimum width - can be null @@ -287,13 +305,33 @@ var NETDATA = window.NETDATA || {}; // rendering the chart that is panned or zoomed). // Used with .current.global_pan_sync_time - last_page_resize: Date.now(), // the timestamp of the last resize request + on_scroll_refresher_stop_until: 0, // timestamp in ms - used to stop evaluating + // charts for some time, after a page scroll + + last_page_resize: Date.now(), // the timestamp of the last resize request last_page_scroll: 0, // the timestamp the last time the page was scrolled + browser_timezone: 'unknown', // timezone detected by javascript + server_timezone: 'unknown', // timezone reported by the server + + force_data_points: 0, // force the number of points to be returned for charts + fake_chart_rendering: false, // when set to true, the dashboard will download data but will not render the charts + + passive_events: null, // true if the browser supports passive events + // the current profile // we may have many... current: { + units: 'auto', // can be 'auto' or 'original' + temperature: 'celsius', // can be 'celsius' or 'fahrenheit' + seconds_as_time: true, // show seconds as DDd:HH:MM:SS ? + timezone: 'default', // the timezone to use, or 'default' + user_set_server_timezone: 'default', // as set by the user on the dashboard + + legend_toolbox: true, // show the legend toolbox on charts + resize_charts: true, // show the resize handler on charts + pixels_per_point: isSlowDevice()?5:1, // the minimum pixels per point for all charts // increase this to speed javascript up // each chart library has its own limit too @@ -318,11 +356,11 @@ var NETDATA = window.NETDATA || {}; idle_lost_focus: 500, // ms - when the window does not have focus, check // if focus has been regained, every this time - global_pan_sync_time: 1000, // ms - when you pan or zoom a chart, the background + global_pan_sync_time: 300, // ms - when you pan or zoom a chart, the background // auto-refreshing of charts is paused for this amount // of time - sync_selection_delay: 1500, // ms - when you pan or zoom a chart, wait this amount + sync_selection_delay: 400, // ms - when you pan or zoom a chart, wait this amount // of time before setting up synchronized selections // on hover. @@ -355,8 +393,6 @@ var NETDATA = window.NETDATA || {}; smooth_plot: (isSlowDevice() === false), // enable smooth plot, where possible - charts_selection_animation_delay: 50, // delay to animate charts when syncing selection - color_fill_opacity_line: 1.0, color_fill_opacity_area: 0.2, color_fill_opacity_stacked: 0.8, @@ -368,7 +404,7 @@ var NETDATA = window.NETDATA || {}; abort_ajax_on_scroll: false, // kill pending ajax page scroll async_on_scroll: false, // sync/async onscroll handler - onscroll_worker_duration_threshold: 30, // time in ms, to consider slow the onscroll handler + onscroll_worker_duration_threshold: 30, // time in ms, for async scroll handler retries_on_data_failures: 3, // how many retries to make if we can't fetch chart data from the server @@ -381,11 +417,13 @@ var NETDATA = window.NETDATA || {}; focus: false, visibility: false, chart_data_url: false, - chart_errors: false, // FIXME: remember to set it to false before merging + chart_errors: true, // FIXME: remember to set it to false before merging chart_timing: false, chart_calls: false, libraries: false, - dygraph: false + dygraph: false, + globalSelectionSync:false, + globalPanAndZoom: false } }; @@ -396,6 +434,114 @@ var NETDATA = window.NETDATA || {}; }; + // ---------------------------------------------------------------------------------------------------------------- + + NETDATA.timeout = { + // by default, these are just wrappers to setTimeout() / clearTimeout() + + step: function(callback) { + return window.setTimeout(callback, 1000 / 60); + }, + + set: function(callback, delay) { + return window.setTimeout(callback, delay); + }, + + clear: function(id) { + return window.clearTimeout(id); + }, + + init: function() { + var custom = true; + + if(window.requestAnimationFrame) { + this.step = function(callback) { + return window.requestAnimationFrame(callback); + }; + + this.clear = function(handle) { + return window.cancelAnimationFrame(handle.value); + }; + } + else if(window.webkitRequestAnimationFrame) { + this.step = function(callback) { + return window.webkitRequestAnimationFrame(callback); + }; + + if(window.webkitCancelAnimationFrame) { + this.clear = function (handle) { + return window.webkitCancelAnimationFrame(handle.value); + }; + } + else if(window.webkitCancelRequestAnimationFrame) { + this.clear = function (handle) { + return window.webkitCancelRequestAnimationFrame(handle.value); + }; + } + } + else if(window.mozRequestAnimationFrame) { + this.step = function(callback) { + return window.mozRequestAnimationFrame(callback); + }; + + this.clear = function(handle) { + return window.mozCancelRequestAnimationFrame(handle.value); + }; + } + else if(window.oRequestAnimationFrame) { + this.step = function(callback) { + return window.oRequestAnimationFrame(callback); + }; + + this.clear = function(handle) { + return window.oCancelRequestAnimationFrame(handle.value); + }; + } + else if(window.msRequestAnimationFrame) { + this.step = function(callback) { + return window.msRequestAnimationFrame(callback); + }; + + this.clear = function(handle) { + return window.msCancelRequestAnimationFrame(handle.value); + }; + } + else + custom = false; + + + if(custom === true) { + // we have installed custom .step() / .clear() functions + // overwrite the .set() too + + this.set = function(callback, delay) { + var that = this; + + var start = Date.now(), + handle = new Object(); + + function loop() { + var current = Date.now(), + delta = current - start; + + if(delta >= delay) { + callback.call(); + } + else { + handle.value = that.step(loop); + } + } + + handle.value = that.step(loop); + return handle; + }; + } + } + }; + + NETDATA.timeout.init(); + + // ---------------------------------------------------------------------------------------------------------------- // local storage options @@ -562,6 +708,8 @@ var NETDATA = window.NETDATA || {}; } } } + + NETDATA.dateTime.init(NETDATA.options.current.timezone); }; // ---------------------------------------------------------------------------------------------------------------- @@ -578,113 +726,111 @@ var NETDATA = window.NETDATA || {}; NETDATA.onresizeCallback(); }; - NETDATA.onscroll_updater_count = 0; - NETDATA.onscroll_updater_running = false; - NETDATA.onscroll_updater_last_run = 0; - NETDATA.onscroll_updater_watchdog = null; - NETDATA.onscroll_updater_max_duration = 0; - NETDATA.onscroll_updater_above_threshold_count = 0; - NETDATA.onscroll_updater = function() { - NETDATA.onscroll_updater_running = true; - NETDATA.onscroll_updater_count++; - var start = Date.now(); - + NETDATA.abort_all_refreshes = function() { var targets = NETDATA.options.targets; var len = targets.length; + while (len--) { + if (targets[len].fetching_data === true) { + if (typeof targets[len].xhr !== 'undefined') { + targets[len].xhr.abort(); + targets[len].running = false; + targets[len].fetching_data = false; + } + } + } + }; + + NETDATA.onscroll_start_delay = function() { + NETDATA.options.last_page_scroll = Date.now(); + + NETDATA.options.on_scroll_refresher_stop_until = + NETDATA.options.last_page_scroll + + ((NETDATA.options.current.async_on_scroll === true) ? 1000 : 0); + }; + + NETDATA.onscroll_end_delay = function() { + NETDATA.options.on_scroll_refresher_stop_until = + Date.now() + + ((NETDATA.options.current.async_on_scroll === true) ? NETDATA.options.current.onscroll_worker_duration_threshold : 0); + }; + + NETDATA.onscroll_updater_timeout_id = undefined; + NETDATA.onscroll_updater = function() { + if(NETDATA.options.targets === null) return; + + //var start = Date.now(); + + // console.log('onscroll_updater() begin'); + + NETDATA.globalSelectionSync.stop(); + // when the user scrolls he sees that we have // hidden all the not-visible charts // using this little function we try to switch // the charts back to visible quickly + if(NETDATA.options.abort_ajax_on_scroll === true) + NETDATA.abort_all_refreshes(); - if(NETDATA.options.abort_ajax_on_scroll === true) { - // we have to cancel pending requests too - - while (len--) { - if (targets[len].fetching_data === true) { - if (typeof targets[len].xhr !== 'undefined') { - targets[len].xhr.abort(); - targets[len].running = false; - targets[len].fetching_data = false; - } - targets[len].isVisible(); - } - } - } - else { - // just find which charts are visible + if(NETDATA.options.current.parallel_refresher === false) { + var targets = NETDATA.options.targets; + var len = targets.length; while (len--) targets[len].isVisible(); } - var end = Date.now(); - // console.log('scroll No ' + NETDATA.onscroll_updater_count + ' calculation took ' + (end - start).toString() + ' ms'); + //var end = Date.now(); + //var dt = end - start; + // console.log('count = ' + targets.length + ', average = ' + Math.round(dt / targets.length).toString() + ', dt = ' + dt); - if(NETDATA.options.current.async_on_scroll === false) { - var dt = end - start; - if(dt > NETDATA.onscroll_updater_max_duration) { - // console.log('max onscroll event handler duration increased to ' + dt); - NETDATA.onscroll_updater_max_duration = dt; - } - - if(dt > NETDATA.options.current.onscroll_worker_duration_threshold) { - // console.log('slow: ' + dt); - NETDATA.onscroll_updater_above_threshold_count++; - - if(NETDATA.onscroll_updater_above_threshold_count > 2 && NETDATA.onscroll_updater_above_threshold_count * 100 / NETDATA.onscroll_updater_count > 2) { - NETDATA.setOption('async_on_scroll', true); - console.log('NETDATA: your browser is slow - enabling asynchronous onscroll event handler.'); - } - } - } - - NETDATA.onscroll_updater_last_run = start; - NETDATA.onscroll_updater_running = false; + NETDATA.onscroll_end_delay(); }; NETDATA.scrollUp = false; NETDATA.scrollY = window.scrollY; NETDATA.onscroll = function() { - // console.log('onscroll'); + //console.log('onscroll() begin'); + + NETDATA.onscroll_start_delay(); + NETDATA.chartRefresherReschedule(); NETDATA.scrollUp = (window.scrollY > NETDATA.scrollY); NETDATA.scrollY = window.scrollY; - NETDATA.options.last_page_scroll = Date.now(); - NETDATA.options.auto_refresher_stop_until = 0; - - if(NETDATA.options.targets === null) return; + if(NETDATA.onscroll_updater_timeout_id) + NETDATA.timeout.clear(NETDATA.onscroll_updater_timeout_id); - if(NETDATA.options.current.async_on_scroll === true) { - // async - if(NETDATA.onscroll_updater_running === false) { - NETDATA.onscroll_updater_running = true; - setTimeout(NETDATA.onscroll_updater, 0); - } - else { - if(NETDATA.onscroll_updater_watchdog !== null) - clearTimeout(NETDATA.onscroll_updater_watchdog); + NETDATA.onscroll_updater_timeout_id = NETDATA.timeout.set(NETDATA.onscroll_updater, 0); + //console.log('onscroll() end'); + }; - NETDATA.onscroll_updater_watchdog = setTimeout(function() { - if(NETDATA.onscroll_updater_running === false && NETDATA.options.last_page_scroll > NETDATA.onscroll_updater_last_run) { - // console.log('watchdog'); - NETDATA.onscroll_updater(); + NETDATA.supportsPassiveEvents = function() { + if(NETDATA.options.passive_events === null) { + var supportsPassive = false; + try { + var opts = Object.defineProperty({}, 'passive', { + get: function () { + supportsPassive = true; } - - NETDATA.onscroll_updater_watchdog = null; - }, 200); + }); + window.addEventListener("test", null, opts); + } catch (e) { + console.log('browser does not support passive events'); } + + NETDATA.options.passive_events = supportsPassive; } - else { - // sync - NETDATA.onscroll_updater(); - } + + // console.log('passive ' + NETDATA.options.passive_events); + return NETDATA.options.passive_events; }; - window.onresize = NETDATA.onresize; - window.onscroll = NETDATA.onscroll; + window.addEventListener('resize', NETDATA.onresize, NETDATA.supportsPassiveEvents() ? { passive: true } : false); + window.addEventListener('scroll', NETDATA.onscroll, NETDATA.supportsPassiveEvents() ? { passive: true } : false); + // window.onresize = NETDATA.onresize; + // window.onscroll = NETDATA.onscroll; // ---------------------------------------------------------------------------------------------------------------- // Error Handling @@ -847,6 +993,7 @@ var NETDATA = window.NETDATA || {}; } }, + // the fallback getFixed: function(min, max) { var key = max; if(min === max) { @@ -882,7 +1029,7 @@ var NETDATA = window.NETDATA || {}; }, testIntlNumberFormat: function() { - var n = 1.12345; + var value = 1.12345; var e1 = "1.12", e2 = "1,12"; var s = ""; @@ -893,7 +1040,7 @@ var NETDATA = window.NETDATA || {}; maximumFractionDigits: 2 }); - s = x.format(n); + s = x.format(value); } catch(e) { s = ""; @@ -904,7 +1051,7 @@ var NETDATA = window.NETDATA || {}; }, testLocaleString: function() { - var n = 1.12345; + var value = 1.12345; var e1 = "1.12", e2 = "1,12"; var s = ""; @@ -993,6 +1140,11 @@ var NETDATA = window.NETDATA || {}; keys: {}, latest: {}, + globalReset: function() { + this.keys = {}; + this.latest = {}; + }, + get: function(state) { if(typeof state.tmp.__commonMin === 'undefined') { // get the commonMin setting @@ -1047,6 +1199,11 @@ var NETDATA = window.NETDATA || {}; keys: {}, latest: {}, + globalReset: function() { + this.keys = {}; + this.latest = {}; + }, + get: function(state) { if(typeof state.tmp.__commonMax === 'undefined') { // get the commonMax setting @@ -1117,14 +1274,11 @@ var NETDATA = window.NETDATA || {}; NETDATA.chartRegistry = { charts: {}, - fixid: function(id) { - return id.replace(/:/g, "_").replace(/\//g, "_"); + globalReset: function() { + this.charts = {}; }, add: function(host, id, data) { - host = this.fixid(host); - id = this.fixid(id); - if(typeof this.charts[host] === 'undefined') this.charts[host] = {}; @@ -1133,9 +1287,6 @@ var NETDATA = window.NETDATA || {}; }, get: function(host, id) { - host = this.fixid(host); - id = this.fixid(id); - if(typeof this.charts[host] === 'undefined') return null; @@ -1151,28 +1302,40 @@ var NETDATA = window.NETDATA || {}; var self = this; - $.ajax({ - url: host + '/api/v1/charts', - async: true, - cache: false, - xhrFields: { withCredentials: true } // required for the cookie - }) - .done(function(data) { + function got_data(h, data, callback) { if(data !== null) { - var h = NETDATA.chartRegistry.fixid(host); self.charts[h] = data.charts; + + // update the server timezone in our options + if(typeof data.timezone === 'string') + NETDATA.options.server_timezone = data.timezone; } - else NETDATA.error(406, host + '/api/v1/charts'); + else NETDATA.error(406, h + '/api/v1/charts'); if(typeof callback === 'function') - return callback(data); - }) - .fail(function() { - NETDATA.error(405, host + '/api/v1/charts'); + callback(data); + } - if(typeof callback === 'function') - return callback(null); - }); + if(netdataSnapshotData !== null) { + got_data(host, netdataSnapshotData.charts, callback); + } + else { + $.ajax({ + url: host + '/api/v1/charts', + async: true, + cache: false, + xhrFields: {withCredentials: true} // required for the cookie + }) + .done(function (data) { + got_data(host, data, callback); + }) + .fail(function () { + NETDATA.error(405, host + '/api/v1/charts'); + + if (typeof callback === 'function') + callback(null); + }); + } } }; @@ -1199,20 +1362,45 @@ var NETDATA = window.NETDATA || {}; callback: null, + globalReset: function() { + this.clearMaster(); + this.seq = 0; + this.master = null; + this.force_after_ms = null; + this.force_before_ms = null; + this.callback = null; + }, + + delay: function() { + if(NETDATA.options.debug.globalPanAndZoom === true) + console.log('globalPanAndZoom.delay()'); + + NETDATA.options.auto_refresher_stop_until = Date.now() + NETDATA.options.current.global_pan_sync_time; + }, + // set a new master setMaster: function(state, after, before) { + this.delay(); + if(NETDATA.options.current.sync_pan_and_zoom === false) return; - if(this.master !== null && this.master !== state) + if(this.master === null) { + if(NETDATA.options.debug.globalPanAndZoom === true) + console.log('globalPanAndZoom.setMaster(' + state.id + ', ' + after + ', ' + before + ') SET MASTER'); + } + else if(this.master !== state) { + if(NETDATA.options.debug.globalPanAndZoom === true) + console.log('globalPanAndZoom.setMaster(' + state.id + ', ' + after + ', ' + before + ') CHANGED MASTER'); + this.master.resetChart(true, true); + } var now = Date.now(); this.master = state; this.seq = now; this.force_after_ms = after; this.force_before_ms = before; - NETDATA.options.auto_refresher_stop_until = now + NETDATA.options.current.global_pan_sync_time; if(typeof this.callback === 'function') this.callback(true, after, before); @@ -1220,6 +1408,9 @@ var NETDATA = window.NETDATA || {}; // clear the master clearMaster: function() { + if(NETDATA.options.debug.globalPanAndZoom === true) + console.log('globalPanAndZoom.clearMaster()'); + if(this.master !== null) { var st = this.master; this.master = null; @@ -1260,6 +1451,86 @@ var NETDATA = window.NETDATA || {}; } }; + // ---------------------------------------------------------------------------------------------------------------- + // global chart underlay (time-frame highlighting) + + NETDATA.globalChartUnderlay = { + callback: null, // what to call when a highlighted range is setup + after: null, // highlight after this time + before: null, // highlight before this time + view_after: null, // the charts after_ms viewport when the highlight was setup + view_before: null, // the charts before_ms viewport, when the highlight was setup + state: null, // the chart the highlight was setup + + isActive: function() { + return (this.after !== null && this.before !== null); + }, + + hasViewport: function() { + return (this.state !== null && this.view_after !== null && this.view_before !== null); + }, + + init: function(state, after, before, view_after, view_before) { + this.state = (typeof state !== 'undefined') ? state : null; + this.after = (typeof after !== 'undefined' && after !== null && after > 0) ? after : null; + this.before = (typeof before !== 'undefined' && before !== null && before > 0) ? before : null; + this.view_after = (typeof view_after !== 'undefined' && view_after !== null && view_after > 0) ? view_after : null; + this.view_before = (typeof view_before !== 'undefined' && view_before !== null && view_before > 0) ? view_before : null; + }, + + setup: function() { + if(this.isActive() === true) { + if (this.state === null) + this.state = NETDATA.options.targets[0]; + + if (typeof this.callback === 'function') + this.callback(true, this.after, this.before); + } + else { + if (typeof this.callback === 'function') + this.callback(false, 0, 0); + } + }, + + set: function(state, after, before, view_after, view_before) { + if(after > before) { + var t = after; + after = before; + before = t; + } + + this.init(state, after, before, view_after, view_before); + + if (this.hasViewport() === true) + NETDATA.globalPanAndZoom.setMaster(this.state, this.view_after, this.view_before); + + this.setup(); + }, + + clear: function() { + this.after = null; + this.before = null; + this.state = null; + this.view_after = null; + this.view_before = null; + + if(typeof this.callback === 'function') + this.callback(false, 0, 0); + }, + + focus: function() { + if(this.isActive() === true && this.hasViewport() === true) { + if(this.state === null) + this.state = NETDATA.options.targets[0]; + + if(NETDATA.globalPanAndZoom.isMaster(this.state) === true) + NETDATA.globalPanAndZoom.clearMaster(); + + NETDATA.globalPanAndZoom.setMaster(this.state, this.view_after, this.view_before, true); + } + } + }; + // ---------------------------------------------------------------------------------------------------------------- // dimensions selection @@ -1290,7 +1561,7 @@ var NETDATA = window.NETDATA || {}; if(this.name_div !== name_div) { this.name_div = name_div; this.name_div.title = this.label; - this.name_div.style.color = this.color; + this.name_div.style.setProperty('color', this.color, 'important'); if(this.selected === false) this.name_div.className = 'netdata-legend-name not-selected'; else @@ -1300,7 +1571,7 @@ var NETDATA = window.NETDATA || {}; if(this.value_div !== value_div) { this.value_div = value_div; this.value_div.title = this.label; - this.value_div.style.color = this.color; + this.value_div.style.setProperty('color', this.color, 'important'); if(this.selected === false) this.value_div.className = 'netdata-legend-value not-selected'; else @@ -1463,6 +1734,422 @@ var NETDATA = window.NETDATA || {}; }; + // ---------------------------------------------------------------------------------------------------------------- + // date/time conversion + + NETDATA.dateTime = { + using_timezone: false, + + // these are the old netdata functions + // we fallback to these, if the new ones fail + + localeDateStringNative: function(d) { + return d.toLocaleDateString(); + }, + + localeTimeStringNative: function(d) { + return d.toLocaleTimeString(); + }, + + xAxisTimeStringNative: function(d) { + return NETDATA.zeropad(d.getHours()) + ":" + + NETDATA.zeropad(d.getMinutes()) + ":" + + NETDATA.zeropad(d.getSeconds()); + }, + + // initialize the new date/time conversion + // functions. + // if this fails, we fallback to the above + init: function(timezone) { + //console.log('init with timezone: ' + timezone); + + // detect browser timezone + try { + NETDATA.options.browser_timezone = Intl.DateTimeFormat().resolvedOptions().timeZone; + } + catch(e) { + console.log('failed to detect browser timezone: ' + e.toString()); + NETDATA.options.browser_timezone = 'cannot-detect-it'; + } + + var ret = false; + + try { + var dateOptions ={ + localeMatcher: 'best fit', + formatMatcher: 'best fit', + weekday: 'short', + year: 'numeric', + month: 'short', + day: '2-digit' + }; + + var timeOptions = { + localeMatcher: 'best fit', + hour12: false, + formatMatcher: 'best fit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit' + }; + + var xAxisOptions = { + localeMatcher: 'best fit', + hour12: false, + formatMatcher: 'best fit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit' + }; + + if(typeof timezone === 'string' && timezone !== '' && timezone !== 'default') { + dateOptions.timeZone = timezone; + timeOptions.timeZone = timezone; + timeOptions.timeZoneName = 'short'; + xAxisOptions.timeZone = timezone; + this.using_timezone = true; + } + else { + timezone = 'default'; + this.using_timezone = false; + } + + this.dateFormat = new Intl.DateTimeFormat(navigator.language, dateOptions); + this.timeFormat = new Intl.DateTimeFormat(navigator.language, timeOptions); + this.xAxisFormat = new Intl.DateTimeFormat(navigator.language, xAxisOptions); + + this.localeDateString = function(d) { + return this.dateFormat.format(d); + }; + + this.localeTimeString = function(d) { + return this.timeFormat.format(d); + }; + + this.xAxisTimeString = function(d) { + return this.xAxisFormat.format(d); + }; + + var d = new Date(); + var t = this.dateFormat.format(d) + ' ' + this.timeFormat.format(d) + ' ' + this.xAxisFormat.format(d); + + ret = true; + } + catch(e) { + console.log('Cannot setup Date/Time formatting: ' + e.toString()); + + timezone = 'default'; + this.localeDateString = this.localeDateStringNative; + this.localeTimeString = this.localeTimeStringNative; + this.xAxisTimeString = this.xAxisTimeStringNative; + this.using_timezone = false; + + ret = false; + } + + // save it + //console.log('init setOption timezone: ' + timezone); + NETDATA.setOption('timezone', timezone); + + return ret; + } + }; + NETDATA.dateTime.init(NETDATA.options.current.timezone); + + + // ---------------------------------------------------------------------------------------------------------------- + // units conversion + + NETDATA.unitsConversion = { + keys: {}, // keys for data-common-units + latest: {}, // latest selected units for data-common-units + + globalReset: function() { + this.keys = {}; + this.latest = {}; + }, + + scalableUnits: { + 'kilobits/s': { + 'bits/s': 1 / 1000, + 'kilobits/s': 1, + 'megabits/s': 1000, + 'gigabits/s': 1000000 + }, + 'kilobytes/s': { + 'bytes/s': 1 / 1024, + 'kilobytes/s': 1, + 'megabytes/s': 1024, + 'gigabytes/s': 1024 * 1024 + }, + 'KB/s': { + 'B/s': 1 / 1024, + 'KB/s': 1, + 'MB/s': 1024, + 'GB/s': 1024 * 1024 + }, + 'KB': { + 'B': 1 / 1024, + 'KB': 1, + 'MB': 1024, + 'GB': 1024 * 1024 + }, + 'MB': { + 'KB': 1 / 1024, + 'MB': 1, + 'GB': 1024, + 'TB': 1024 * 1024 + }, + 'GB': { + 'MB': 1 / 1024, + 'GB': 1, + 'TB': 1024 + } + /* + 'seconds': { + 'milliseconds': 0.001, + 'seconds': 1, + 'minutes': 60, + 'hours': 3600, + 'days': 86400 + } + */ + }, + + convertibleUnits: { + 'Celsius': { + 'Fahrenheit': { + check: function() { return NETDATA.options.current.temperature === 'fahrenheit'; }, + convert: function(value) { return value * 9 / 5 + 32; } + } + }, + 'celsius': { + 'fahrenheit': { + check: function() { return NETDATA.options.current.temperature === 'fahrenheit'; }, + convert: function(value) { return value * 9 / 5 + 32; } + } + }, + 'seconds': { + 'time': { + check: function () { return NETDATA.options.current.seconds_as_time; }, + convert: function (seconds) { + seconds = Math.abs(seconds); + + var days = Math.floor(seconds / 86400); + seconds -= days * 86400; + + var hours = Math.floor(seconds / 3600); + seconds -= hours * 3600; + + var minutes = Math.floor(seconds / 60); + seconds -= minutes * 60; + + seconds = Math.round(seconds); + + var ms_txt = ''; + /* + var ms = seconds - Math.floor(seconds); + seconds -= ms; + ms = Math.round(ms * 1000); + + if(ms > 1) { + if(ms < 10) + ms_txt = '.00' + ms.toString(); + else if(ms < 100) + ms_txt = '.0' + ms.toString(); + else + ms_txt = '.' + ms.toString(); + } + */ + + return ((days > 0)?days.toString() + 'd:':'').toString() + + NETDATA.zeropad(hours) + ':' + + NETDATA.zeropad(minutes) + ':' + + NETDATA.zeropad(seconds) + + ms_txt; + } + } + } + }, + + // get a function that converts the units + // + every time units are switched call the callback + get: function(uuid, min, max, units, desired_units, common_units_name, switch_units_callback) { + // validate the parameters + if(typeof units === 'undefined') + units = 'undefined'; + + // check if we support units conversion + if(typeof this.scalableUnits[units] === 'undefined' && typeof this.convertibleUnits[units] === 'undefined') { + // we can't convert these units + //console.log('DEBUG: ' + uuid.toString() + ' can\'t convert units: ' + units.toString()); + return function(value) { return value; }; + } + + // check if the caller wants the original units + if(typeof desired_units === 'undefined' || desired_units === null || desired_units === 'original' || desired_units === units) { + //console.log('DEBUG: ' + uuid.toString() + ' original units wanted'); + switch_units_callback(units); + return function(value) { return value; }; + } + + // now we know we can convert the units + // and the caller wants some kind of conversion + + var tunits = null; + var tdivider = 0; + var x; + + if(typeof this.scalableUnits[units] !== 'undefined') { + // units that can be scaled + // we decide a divider + + // console.log('NETDATA.unitsConversion.get(' + units.toString() + ', ' + desired_units.toString() + ', function()) decide divider with min = ' + min.toString() + ', max = ' + max.toString()); + + if (desired_units === 'auto') { + // the caller wants to auto-scale the units + + // find the absolute maximum value that is rendered on the chart + // based on this we decide the scale + min = Math.abs(min); + max = Math.abs(max); + if (min > max) max = min; + + // find the smallest scale that provides integers + for (x in this.scalableUnits[units]) { + if (this.scalableUnits[units].hasOwnProperty(x)) { + var m = this.scalableUnits[units][x]; + if (m <= max && m > tdivider) { + tunits = x; + tdivider = m; + } + } + } + + if(tunits === null || tdivider <= 0) { + // we couldn't find one + //console.log('DEBUG: ' + uuid.toString() + ' cannot find an auto-scaling candidate for units: ' + units.toString() + ' (max: ' + max.toString() + ')'); + switch_units_callback(units); + return function(value) { return value; }; + } + + if(typeof common_units_name === 'string' && typeof uuid === 'string') { + // the caller wants several charts to have the same units + // data-common-units + + var common_units_key = common_units_name + '-' + units; + + // add our divider into the list of keys + var t = this.keys[common_units_key]; + if(typeof t === 'undefined') { + this.keys[common_units_key] = {}; + t = this.keys[common_units_key]; + } + t[uuid] = { + units: tunits, + divider: tdivider + }; + + // find the max divider of all charts + var common_units = t[uuid]; + for(x in t) { + if (t.hasOwnProperty(x) && t[x].divider > common_units.divider) + common_units = t[x]; + } + + // save our common_max to the latest keys + var latest = this.latest[common_units_key]; + if(typeof latest === 'undefined') { + this.latest[common_units_key] = {}; + latest = this.latest[common_units_key]; + } + latest.units = common_units.units; + latest.divider = common_units.divider; + + tunits = latest.units; + tdivider = latest.divider; + + //console.log('DEBUG: ' + uuid.toString() + ' converted units: ' + units.toString() + ' to units: ' + tunits.toString() + ' with divider ' + tdivider.toString() + ', common-units=' + common_units_name.toString() + ((t[uuid].divider !== tdivider)?' USED COMMON, mine was ' + t[uuid].units:' set common').toString()); + + // apply it to this chart + switch_units_callback(tunits); + return function(value) { + if(tdivider !== latest.divider) { + // another chart switched our common units + // we should switch them too + //console.log('DEBUG: ' + uuid + ' switching units due to a common-units change, from ' + tunits.toString() + ' to ' + latest.units.toString()); + tunits = latest.units; + tdivider = latest.divider; + switch_units_callback(tunits); + } + + return value / tdivider; + } + } + else { + // the caller did not give data-common-units + // this chart auto-scales independently of all others + //console.log('DEBUG: ' + uuid.toString() + ' converted units: ' + units.toString() + ' to units: ' + tunits.toString() + ' with divider ' + tdivider.toString() + ', autonomously'); + + switch_units_callback(tunits); + return function (value) { return value / tdivider; }; + } + } + else { + // the caller wants specific units + + if(typeof this.scalableUnits[units][desired_units] !== 'undefined') { + // all good, set the new units + tdivider = this.scalableUnits[units][desired_units]; + // console.log('DEBUG: ' + uuid.toString() + ' converted units: ' + units.toString() + ' to units: ' + desired_units.toString() + ' with divider ' + tdivider.toString() + ', by reference'); + switch_units_callback(desired_units); + return function (value) { return value / tdivider; }; + } + else { + // oops! switch back to original units + console.log('Units conversion from ' + units.toString() + ' to ' + desired_units.toString() + ' is not supported.'); + switch_units_callback(units); + return function (value) { return value; }; + } + } + } + else if(typeof this.convertibleUnits[units] !== 'undefined') { + // units that can be converted + if(desired_units === 'auto') { + for(x in this.convertibleUnits[units]) { + if (this.convertibleUnits[units].hasOwnProperty(x)) { + if (this.convertibleUnits[units][x].check()) { + //console.log('DEBUG: ' + uuid.toString() + ' converting ' + units.toString() + ' to: ' + x.toString()); + switch_units_callback(x); + return this.convertibleUnits[units][x].convert; + } + } + } + + // none checked ok + //console.log('DEBUG: ' + uuid.toString() + ' no conversion available for ' + units.toString() + ' to: ' + desired_units.toString()); + switch_units_callback(units); + return function (value) { return value; }; + } + else if(typeof this.convertibleUnits[units][desired_units] !== 'undefined') { + switch_units_callback(desired_units); + return this.convertibleUnits[units][desired_units].convert; + } + else { + console.log('Units conversion from ' + units.toString() + ' to ' + desired_units.toString() + ' is not supported.'); + switch_units_callback(units); + return function (value) { return value; }; + } + } + else { + // hm... did we forget to implement the new type? + console.log('Unmatched unit conversion method for units ' + units.toString()); + switch_units_callback(units); + return function (value) { return value; }; + } + } + }; + // ---------------------------------------------------------------------------------------------------------------- // global selection sync @@ -1471,15 +2158,126 @@ var NETDATA = window.NETDATA || {}; dont_sync_before: 0, last_t: 0, slaves: [], + timeout_id: undefined, + + globalReset: function() { + this.stop(); + this.state = null; + this.dont_sync_before = 0; + this.last_t = 0; + this.slaves = []; + this.timeout_id = undefined; + }, + + active: function() { + return (this.state !== null); + }, + + // return true if global selection sync can be enabled now + enabled: function() { + // console.log('enabled()'); + // can we globally apply selection sync? + if(NETDATA.options.current.sync_selection === false) + return false; + + return (this.dont_sync_before <= Date.now()); + }, + + // set the global selection sync master + setMaster: function(state) { + if(this.enabled() === false) { + this.stop(); + return; + } + + if(this.state === state) + return; - stop: function() { if(this.state !== null) - this.state.globalSelectionSyncStop(); + this.stop(); + + if(NETDATA.options.debug.globalSelectionSync === true) + console.log('globalSelectionSync.setMaster(' + state.id + ')'); + + state.selected = true; + this.state = state; + this.last_t = 0; + + // find all slaves + this.slaves = []; + var len = NETDATA.options.targets.length; + while(len--) { + var st = NETDATA.options.targets[len]; + if (this.state !== st && st.globalSelectionSyncIsEligible() === true) + this.slaves.push(st); + } + + // this.delay(100); }, - delay: function() { + // stop global selection sync + stop: function() { if(this.state !== null) { - this.state.globalSelectionSyncDelay(); + if(NETDATA.options.debug.globalSelectionSync === true) + console.log('globalSelectionSync.stop()'); + + var len = this.slaves.length; + while (len--) + this.slaves[len].clearSelection(); + + this.state.clearSelection(); + + this.last_t = 0; + this.slaves = []; + this.state = null; + } + }, + + // delay global selection sync for some time + delay: function(ms) { + if(NETDATA.options.current.sync_selection === true) { + if(NETDATA.options.debug.globalSelectionSync === true) + console.log('globalSelectionSync.delay()'); + + if(typeof ms === 'number') + this.dont_sync_before = Date.now() + ms; + else + this.dont_sync_before = Date.now() + NETDATA.options.current.sync_selection_delay; + } + }, + + __syncSlaves: function() { + if(NETDATA.globalSelectionSync.enabled() === true) { + if(NETDATA.options.debug.globalSelectionSync === true) + console.log('globalSelectionSync.__syncSlaves()'); + + var t = NETDATA.globalSelectionSync.last_t; + var len = NETDATA.globalSelectionSync.slaves.length; + while (len--) + NETDATA.globalSelectionSync.slaves[len].setSelection(t); + + this.timeout_id = undefined; + } + }, + + // sync all the visible charts to the given time + // this is to be called from the chart libraries + sync: function(state, t) { + if(NETDATA.options.current.sync_selection === true) { + if(NETDATA.options.debug.globalSelectionSync === true) + console.log('globalSelectionSync.sync(' + state.id + ', ' + t.toString() + ')'); + + this.setMaster(state); + + if(t === this.last_t) + return; + + this.last_t = t; + + if (this.timeout_id) + NETDATA.timeout.clear(this.timeout_id); + + this.timeout_id = NETDATA.timeout.set(this.__syncSlaves, 0); } } }; @@ -1653,7 +2451,7 @@ var NETDATA = window.NETDATA || {}; return; // string - the netdata server URL, without any path - that.host = NETDATA.dataAttribute(that.element, 'host', NETDATA.chartDefaults.host); + that.host = NETDATA.dataAttribute(that.element, 'host', NETDATA.serverDefault); // make sure the host does not end with / // all netdata API requests use absolute paths @@ -1676,6 +2474,10 @@ var NETDATA = window.NETDATA || {}; that.title = NETDATA.dataAttribute(that.element, 'title', null); // the title of the chart that.units = NETDATA.dataAttribute(that.element, 'units', null); // the units of the chart dimensions + that.units_desired = NETDATA.dataAttribute(that.element, 'desired-units', NETDATA.options.current.units); // the units of the chart dimensions + that.units_current = that.units; + that.units_common = NETDATA.dataAttribute(that.element, 'common-units', null); + that.append_options = NETDATA.dataAttribute(that.element, 'append-options', null); // additional options to pass to netdata that.override_options = NETDATA.dataAttribute(that.element, 'override-options', null); // override options to pass to netdata @@ -1723,6 +2525,7 @@ var NETDATA = window.NETDATA || {}; that.element_chart = null; // the element with the chart that.element_legend = null; // the element with the legend of the chart (if created by us) that.element_legend_childs = { + content: null, hidden: null, title_date: null, title_time: null, @@ -1742,7 +2545,7 @@ var NETDATA = window.NETDATA || {}; that.log('destroyDOM()'); // that.element.className = 'netdata-message icon'; - // that.element.innerHTML = ' netdata'; + // that.element.innerHTML = ' netdata'; that.element.innerHTML = ''; that.element_message = null; that.element_legend = null; @@ -1797,6 +2600,10 @@ var NETDATA = window.NETDATA || {}; that.element.style.min_width = NETDATA.chartDefaults.min_width; }; + var invisibleSearchableText = function() { + return '' + that.id + ''; + } + /* init() private * initialize state variables * destroy all (possibly) created state elements @@ -1806,6 +2613,7 @@ var NETDATA = window.NETDATA || {}; if(that.enabled === false) return; runtimeInit(); + that.element.innerHTML = invisibleSearchableText(); that.tm.last_initialized = Date.now(); that.setMode('auto'); @@ -1868,19 +2676,19 @@ var NETDATA = window.NETDATA || {}; var icon; if(that.chart !== null) { if(that.chart.chart_type === 'line') - icon = ''; + icon = ''; else - icon = ''; + icon = ''; } else - icon = ''; + icon = ''; - showMessageIcon(icon + ' netdata'); + showMessageIcon(icon + ' netdata' + invisibleSearchableText()); }; var showLoading = function() { if(that.chart_created === false) { - showMessageIcon(' netdata'); + showMessageIcon(' netdata'); return true; } return false; @@ -1896,13 +2704,34 @@ var NETDATA = window.NETDATA || {}; if(isHidden() === true) return; if(that.chart_created === true) { + if(NETDATA.options.current.show_help === true) { + if(that.element_legend_childs.toolbox !== null) { + $(that.element_legend_childs.toolbox_left).popover('hide'); + $(that.element_legend_childs.toolbox_reset).popover('hide'); + $(that.element_legend_childs.toolbox_right).popover('hide'); + $(that.element_legend_childs.toolbox_zoomin).popover('hide'); + $(that.element_legend_childs.toolbox_zoomout).popover('hide'); + } + + if(that.element_legend_childs.resize_handler !== null) + $(that.element_legend_childs.resize_handler).popover('hide'); + + if(that.element_legend_childs.content !== null) + $(that.element_legend_childs.content).popover('hide'); + } + if(NETDATA.options.current.destroy_on_hide === true) { + // that.log('hideChart() init'); + // we should destroy it init('force'); } else { + // that.log('hideChart()'); + showRendering(); that.element_chart.style.display = 'none'; + that.element.style.willChange = 'auto'; if(that.element_legend !== null) that.element_legend.style.display = 'none'; if(that.element_legend_childs.toolbox !== null) that.element_legend_childs.toolbox.style.display = 'none'; if(that.element_legend_childs.resize_handler !== null) that.element_legend_childs.resize_handler.style.display = 'none'; @@ -1928,11 +2757,14 @@ var NETDATA = window.NETDATA || {}; that.updates_since_last_unhide = 0; if(that.chart_created === false) { + // that.log('unhideChart() init'); // we need to re-initialize it, to show our background // logo in bootstrap tabs, until the chart loads init('force'); } else { + // that.log('unhideChart()'); + that.element.style.willChange = 'transform'; that.tm.last_unhidden = Date.now(); that.element_chart.style.display = ''; if(that.element_legend !== null) that.element_legend.style.display = ''; @@ -1943,16 +2775,34 @@ var NETDATA = window.NETDATA || {}; } }; - var canBeRendered = function() { - return (isHidden() === false && that.isVisible(true) === true); + var canBeRendered = function(uncached_visibility) { + return ( + ( + NETDATA.options.page_is_visible === true || + NETDATA.options.current.stop_updates_when_focus_is_lost === false || + that.updates_since_last_unhide === 0 + ) + && isHidden() === false && that.isVisible(uncached_visibility) === true + ); }; // https://github.com/petkaantonov/bluebird/wiki/Optimization-killers var callChartLibraryUpdateSafely = function(data) { var status; - if(canBeRendered() === false) - return false; + // we should not do this here + // if we prevent rendering the chart then: + // 1. globalSelectionSync will be wrong + // 2. globalPanAndZoom will be wrong + //if(canBeRendered(true) === false) + // return false; + + if(NETDATA.options.fake_chart_rendering === true) + return true; + + that.updates_counter++; + that.updates_since_last_unhide++; + that.updates_since_last_creation++; if(NETDATA.options.debug.chart_errors === true) status = that.library.update(that, data); @@ -1977,8 +2827,19 @@ var NETDATA = window.NETDATA || {}; var callChartLibraryCreateSafely = function(data) { var status; - if(canBeRendered() === false) - return false; + // we should not do this here + // if we prevent rendering the chart then: + // 1. globalSelectionSync will be wrong + // 2. globalPanAndZoom will be wrong + //if(canBeRendered(true) === false) + // return false; + + if(NETDATA.options.fake_chart_rendering === true) + return true; + + that.updates_counter++; + that.updates_since_last_unhide++; + that.updates_since_last_creation++; if(NETDATA.options.debug.chart_errors === true) status = that.library.create(that, data); @@ -2049,6 +2910,21 @@ var NETDATA = window.NETDATA || {}; resizeChart(); }; + this.resizeForPrint = function() { + if(typeof this.element_legend_childs !== 'undefined' && this.element_legend_childs.perfect_scroller !== null) { + var current = this.element.clientHeight; + var optimal = current + + this.element_legend_childs.perfect_scroller.scrollHeight + - this.element_legend_childs.perfect_scroller.clientHeight; + + if(optimal > current) { + // this.log('resized'); + this.element.style.height = optimal + 'px'; + this.library.resize(this); + } + } + }; + this.resizeHandler = function(e) { e.preventDefault(); @@ -2178,7 +3054,7 @@ var NETDATA = window.NETDATA || {}; var noDataToShow = function() { - showMessageIcon(' empty'); + showMessageIcon(' empty'); that.legendUpdateDOM(); that.tm.last_autorefreshed = Date.now(); // that.data_update_every = 30 * 1000; @@ -2216,74 +3092,6 @@ var NETDATA = window.NETDATA || {}; // ---------------------------------------------------------------------------------------------------------------- // global selection sync - // prevent to global selection sync for some time - this.globalSelectionSyncDelay = function(ms) { - if(NETDATA.options.current.sync_selection === false) - return; - - if(typeof ms === 'number') - NETDATA.globalSelectionSync.dont_sync_before = Date.now() + ms; - else - NETDATA.globalSelectionSync.dont_sync_before = Date.now() + NETDATA.options.current.sync_selection_delay; - }; - - // can we globally apply selection sync? - this.globalSelectionSyncAbility = function() { - if(NETDATA.options.current.sync_selection === false) - return false; - - return (NETDATA.globalSelectionSync.dont_sync_before <= Date.now()); - }; - - this.globalSelectionSyncIsMaster = function() { - return (NETDATA.globalSelectionSync.state === this); - }; - - // this chart is the master of the global selection sync - this.globalSelectionSyncBeMaster = function() { - // am I the master? - if(this.globalSelectionSyncIsMaster()) { - if(this.debug === true) - this.log('sync: I am the master already.'); - - return; - } - - if(NETDATA.globalSelectionSync.state) { - if(this.debug === true) - this.log('sync: I am not the sync master. Resetting global sync.'); - - this.globalSelectionSyncStop(); - } - - // become the master - if(this.debug === true) - this.log('sync: becoming sync master.'); - - this.selected = true; - NETDATA.globalSelectionSync.state = this; - - // find the all slaves - var targets = NETDATA.options.targets; - var len = targets.length; - while(len--) { - var st = targets[len]; - - if(st === this) { - if(this.debug === true) - st.log('sync: not adding me to sync'); - } - else if(st.globalSelectionSyncIsEligible()) { - if(this.debug === true) - st.log('sync: adding to sync as slave'); - - st.globalSelectionSyncBeSlave(); - } - } - - // this.globalSelectionSyncDelay(100); - }; - // can the chart participate to the global selection sync as a slave? this.globalSelectionSyncIsEligible = function() { return (this.enabled === true @@ -2293,61 +3101,6 @@ var NETDATA = window.NETDATA || {}; && this.chart_created === true); }; - // this chart becomes a slave of the global selection sync - this.globalSelectionSyncBeSlave = function() { - if(NETDATA.globalSelectionSync.state !== this) - NETDATA.globalSelectionSync.slaves.push(this); - }; - - // sync all the visible charts to the given time - // this is to be called from the chart libraries - this.globalSelectionSync = function(t) { - if(this.globalSelectionSyncAbility() === false) - return; - - if(this.globalSelectionSyncIsMaster() === false) { - if(this.debug === true) - this.log('sync: trying to be sync master.'); - - this.globalSelectionSyncBeMaster(); - - if(this.globalSelectionSyncAbility() === false) - return; - } - - NETDATA.globalSelectionSync.last_t = t; - $.each(NETDATA.globalSelectionSync.slaves, function(i, st) { - st.setSelection(t); - }); - }; - - // stop syncing all charts to the given time - this.globalSelectionSyncStop = function() { - if(NETDATA.globalSelectionSync.slaves.length) { - if(this.debug === true) - this.log('sync: cleaning up...'); - - $.each(NETDATA.globalSelectionSync.slaves, function(i, st) { - if(st === that) { - if(that.debug === true) - st.log('sync: not adding me to sync stop'); - } - else { - if(that.debug === true) - st.log('sync: removed slave from sync'); - - st.clearSelection(); - } - }); - - NETDATA.globalSelectionSync.last_t = 0; - NETDATA.globalSelectionSync.slaves = []; - NETDATA.globalSelectionSync.state = null; - } - - this.clearSelection(); - }; - this.setSelection = function(t) { if(typeof this.library.setSelection === 'function') this.selected = (this.library.setSelection(this, t) === true); @@ -2376,6 +3129,8 @@ var NETDATA = window.NETDATA || {}; return this.selected; }; + // ---------------------------------------------------------------------------------------------------------------- + // find if a timestamp (ms) is shown in the current chart this.timeIsVisible = function(t) { return (t >= this.data_after && t <= this.data_before); @@ -2448,10 +3203,13 @@ var NETDATA = window.NETDATA || {}; } }; - this.updateChartPanOrZoom = function(after, before) { + this.updateChartPanOrZoom = function(after, before, callback) { var logme = 'updateChartPanOrZoom(' + after + ', ' + before + '): '; var ret = true; + NETDATA.globalPanAndZoom.delay(); + NETDATA.globalSelectionSync.delay(); + if(this.debug === true) this.log(logme); @@ -2521,9 +3279,58 @@ var NETDATA = window.NETDATA || {}; this.current.force_after_ms = after; this.current.force_before_ms = before; NETDATA.globalPanAndZoom.setMaster(this, after, before); + + if(ret === true && typeof callback === 'function') + callback(); + return ret; }; + this.updateChartPanOrZoomAsyncTimeOutId = undefined; + this.updateChartPanOrZoomAsync = function(after, before, callback) { + NETDATA.globalPanAndZoom.delay(); + NETDATA.globalSelectionSync.delay(); + + if(NETDATA.globalPanAndZoom.isMaster(this) === false) { + this.pauseChart(); + NETDATA.globalPanAndZoom.setMaster(this, after, before); + // NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.setMaster(this); + } + + if(this.updateChartPanOrZoomAsyncTimeOutId) + NETDATA.timeout.clear(this.updateChartPanOrZoomAsyncTimeOutId); + + NETDATA.timeout.set(function() { + that.updateChartPanOrZoomAsyncTimeOutId = undefined; + that.updateChartPanOrZoom(after, before, callback); + }, 0); + }; + + var __unitsConversionLastUnits = undefined; + var __unitsConversionLastUnitsDesired = undefined; + var __unitsConversionLastMin = undefined; + var __unitsConversionLastMax = undefined; + var __unitsConversion = function(value) { return value; }; + this.unitsConversionSetup = function(min, max) { + if(this.units !== __unitsConversionLastUnits + || this.units_desired !== __unitsConversionLastUnitsDesired + || min !== __unitsConversionLastMin + || max !== __unitsConversionLastMax) { + + __unitsConversionLastUnits = this.units; + __unitsConversionLastUnitsDesired = this.units_desired; + __unitsConversionLastMin = min; + __unitsConversionLastMax = max; + + __unitsConversion = NETDATA.unitsConversion.get(this.uuid, min, max, this.units, this.units_desired, this.units_common, function (units) { + // console.log('switching units from ' + that.units.toString() + ' to ' + units.toString()); + that.units_current = units; + that.legendSetUnitsString(that.units_current); + }); + } + }; + var __legendFormatValueChartDecimalsLastMin = undefined; var __legendFormatValueChartDecimalsLastMax = undefined; var __legendFormatValueChartDecimals = -1; @@ -2532,6 +3339,15 @@ var NETDATA = window.NETDATA || {}; if(min === __legendFormatValueChartDecimalsLastMin && max === __legendFormatValueChartDecimalsLastMax) return; + this.unitsConversionSetup(min, max); + if(__unitsConversion !== null) { + min = __unitsConversion(min); + max = __unitsConversion(max); + + if(typeof min !== 'number' || typeof max !== 'number') + return; + } + __legendFormatValueChartDecimalsLastMin = min; __legendFormatValueChartDecimalsLastMax = max; @@ -2576,7 +3392,13 @@ var NETDATA = window.NETDATA || {}; }; this.legendFormatValue = function(value) { - if(typeof value !== 'number') return '-'; + if(typeof value !== 'number') + return '-'; + + value = __unitsConversion(value); + + if(typeof value !== 'number') + return value; if(__intlNumberFormat !== null) return __intlNumberFormat.format(value); @@ -2619,11 +3441,11 @@ var NETDATA = window.NETDATA || {}; s = r = this.legendFormatValue(value); if(typeof series.last === 'number') { - if(v > series.last) s += ''; - else if(v < series.last) s += ''; - else s += ''; + if(v > series.last) s += ''; + else if(v < series.last) s += ''; + else s += ''; } - else s += ''; + else s += ''; series.last = v; } @@ -2647,22 +3469,22 @@ var NETDATA = window.NETDATA || {}; if(series.user !== null) series.user.innerText = s; }; - this.__legendSetDateString = function(date) { - if(date !== this.tmp.__last_shown_legend_date) { + this.legendSetDateString = function(date) { + if(this.element_legend_childs.title_date !== null && date !== this.tmp.__last_shown_legend_date) { this.element_legend_childs.title_date.innerText = date; this.tmp.__last_shown_legend_date = date; } }; - this.__legendSetTimeString = function(time) { - if(time !== this.tmp.__last_shown_legend_time) { + this.legendSetTimeString = function(time) { + if(this.element_legend_childs.title_time !== null && time !== this.tmp.__last_shown_legend_time) { this.element_legend_childs.title_time.innerText = time; this.tmp.__last_shown_legend_time = time; } }; - this.__legendSetUnitsString = function(units) { - if(units !== this.tmp.__last_shown_legend_units) { + this.legendSetUnitsString = function(units) { + if(this.element_legend_childs.title_units !== null && units !== this.tmp.__last_shown_legend_units) { this.element_legend_childs.title_units.innerText = units; this.tmp.__last_shown_legend_units = units; } @@ -2683,29 +3505,19 @@ var NETDATA = window.NETDATA || {}; if(this.legendSetDateLast.ms !== ms) { var d = new Date(ms); this.legendSetDateLast.ms = ms; - this.legendSetDateLast.date = d.toLocaleDateString(); - this.legendSetDateLast.time = d.toLocaleTimeString(); + this.legendSetDateLast.date = NETDATA.dateTime.localeDateString(d); + this.legendSetDateLast.time = NETDATA.dateTime.localeTimeString(d); } - if(this.element_legend_childs.title_date !== null) - this.__legendSetDateString(this.legendSetDateLast.date); - - if(this.element_legend_childs.title_time !== null) - this.__legendSetTimeString(this.legendSetDateLast.time); - - if(this.element_legend_childs.title_units !== null) - this.__legendSetUnitsString(this.units) + this.legendSetDateString(this.legendSetDateLast.date); + this.legendSetTimeString(this.legendSetDateLast.time); + this.legendSetUnitsString(this.units_current) }; this.legendShowUndefined = function() { - if(this.element_legend_childs.title_date !== null) - this.__legendSetDateString(' '); - - if(this.element_legend_childs.title_time !== null) - this.__legendSetTimeString(this.chart.name); - - if(this.element_legend_childs.title_units !== null) - this.__legendSetUnitsString(' '); + this.legendSetDateString(this.legendPluginModuleString(false)); + this.legendSetTimeString(this.chart.context.toString()); + // this.legendSetUnitsString(' '); if(this.data && this.element_legend_childs.series !== null) { var labels = this.data.dimension_names; @@ -2869,6 +3681,42 @@ var NETDATA = window.NETDATA || {}; return this.colors; }; + this.legendPluginModuleString = function(withContext) { + var str = ' '; + var context = ''; + + if(typeof this.chart !== 'undefined') { + if(withContext && typeof this.chart.context === 'string') + context = this.chart.context; + + if (typeof this.chart.plugin === 'string' && this.chart.plugin !== '') { + str = this.chart.plugin; + if (typeof this.chart.module === 'string' && this.chart.module !== '') { + str += '/' + this.chart.module; + } + + if (withContext && context !== '') + str += ', ' + context; + } + else if (withContext && context !== '') + str = context; + } + + return str; + }; + + this.legendResolutionTooltip = function () { + if(!this.chart) return ''; + + var collected = this.chart.update_every; + var viewed = (this.data)?this.data.view_update_every:collected; + + if(collected === viewed) + return "resolution " + NETDATA.seconds4human(collected); + + return "resolution " + NETDATA.seconds4human(viewed) + ", collected every " + NETDATA.seconds4human(collected); + }; + this.legendUpdateDOM = function() { var needed = false, dim, keys, len, i; @@ -2961,7 +3809,7 @@ var NETDATA = window.NETDATA || {}; label.name.innerHTML = '
'; var text = document.createTextNode(' ' + name); @@ -2998,14 +3846,14 @@ var NETDATA = window.NETDATA || {}; this.element_legend_childs = { content: content, - resize_handler: document.createElement('div'), - toolbox: document.createElement('div'), - toolbox_left: document.createElement('div'), - toolbox_right: document.createElement('div'), - toolbox_reset: document.createElement('div'), - toolbox_zoomin: document.createElement('div'), - toolbox_zoomout: document.createElement('div'), - toolbox_volume: document.createElement('div'), + resize_handler: null, + toolbox: null, + toolbox_left: null, + toolbox_right: null, + toolbox_reset: null, + toolbox_zoomin: null, + toolbox_zoomout: null, + toolbox_volume: null, title_date: document.createElement('span'), title_time: document.createElement('span'), title_units: document.createElement('span'), @@ -3013,7 +3861,15 @@ var NETDATA = window.NETDATA || {}; series: {} }; - if(this.library.toolboxPanAndZoom !== null) { + if(NETDATA.options.current.legend_toolbox === true && this.library.toolboxPanAndZoom !== null) { + this.element_legend_childs.toolbox = document.createElement('div'); + this.element_legend_childs.toolbox_left = document.createElement('div'); + this.element_legend_childs.toolbox_right = document.createElement('div'); + this.element_legend_childs.toolbox_reset = document.createElement('div'); + this.element_legend_childs.toolbox_zoomin = document.createElement('div'); + this.element_legend_childs.toolbox_zoomout = document.createElement('div'); + this.element_legend_childs.toolbox_volume = document.createElement('div'); + var get_pan_and_zoom_step = function(event) { if (event.ctrlKey) return NETDATA.options.current.pan_and_zoom_factor * NETDATA.options.current.pan_and_zoom_factor_multiplier_control; @@ -3032,7 +3888,7 @@ var NETDATA = window.NETDATA || {}; this.element.appendChild(this.element_legend_childs.toolbox); this.element_legend_childs.toolbox_left.className += ' netdata-legend-toolbox-button'; - this.element_legend_childs.toolbox_left.innerHTML = ''; + this.element_legend_childs.toolbox_left.innerHTML = ''; this.element_legend_childs.toolbox.appendChild(this.element_legend_childs.toolbox_left); this.element_legend_childs.toolbox_left.onclick = function(e) { e.preventDefault(); @@ -3052,12 +3908,12 @@ var NETDATA = window.NETDATA || {}; placement: 'bottom', delay: { show: NETDATA.options.current.show_help_delay_show_ms, hide: NETDATA.options.current.show_help_delay_hide_ms }, title: 'Pan Left', - content: 'Pan the chart to the left. You can also drag it with your mouse or your finger (on touch devices).
Help, can be disabled from the settings.' + content: 'Pan the chart to the left. You can also drag it with your mouse or your finger (on touch devices).
Help can be disabled from the settings.' }); this.element_legend_childs.toolbox_reset.className += ' netdata-legend-toolbox-button'; - this.element_legend_childs.toolbox_reset.innerHTML = ''; + this.element_legend_childs.toolbox_reset.innerHTML = ''; this.element_legend_childs.toolbox.appendChild(this.element_legend_childs.toolbox_reset); this.element_legend_childs.toolbox_reset.onclick = function(e) { e.preventDefault(); @@ -3072,11 +3928,11 @@ var NETDATA = window.NETDATA || {}; placement: 'bottom', delay: { show: NETDATA.options.current.show_help_delay_show_ms, hide: NETDATA.options.current.show_help_delay_hide_ms }, title: 'Chart Reset', - content: 'Reset all the charts to their default auto-refreshing state. You can also double click the chart contents with your mouse or your finger (on touch devices).
Help, can be disabled from the settings.' + content: 'Reset all the charts to their default auto-refreshing state. You can also double click the chart contents with your mouse or your finger (on touch devices).
Help can be disabled from the settings.' }); this.element_legend_childs.toolbox_right.className += ' netdata-legend-toolbox-button'; - this.element_legend_childs.toolbox_right.innerHTML = ''; + this.element_legend_childs.toolbox_right.innerHTML = ''; this.element_legend_childs.toolbox.appendChild(this.element_legend_childs.toolbox_right); this.element_legend_childs.toolbox_right.onclick = function(e) { e.preventDefault(); @@ -3100,7 +3956,7 @@ var NETDATA = window.NETDATA || {}; this.element_legend_childs.toolbox_zoomin.className += ' netdata-legend-toolbox-button'; - this.element_legend_childs.toolbox_zoomin.innerHTML = ''; + this.element_legend_childs.toolbox_zoomin.innerHTML = ''; this.element_legend_childs.toolbox.appendChild(this.element_legend_childs.toolbox_zoomin); this.element_legend_childs.toolbox_zoomin.onclick = function(e) { e.preventDefault(); @@ -3118,11 +3974,11 @@ var NETDATA = window.NETDATA || {}; placement: 'bottom', delay: { show: NETDATA.options.current.show_help_delay_show_ms, hide: NETDATA.options.current.show_help_delay_hide_ms }, title: 'Chart Zoom In', - content: 'Zoom in the chart. You can also press SHIFT and select an area of the chart to zoom in. On Chrome and Opera, you can press the SHIFT or the ALT keys and then use the mouse wheel to zoom in or out.
Help, can be disabled from the settings.' + content: 'Zoom in the chart. You can also press SHIFT and select an area of the chart, or press SHIFT or ALT and use the mouse wheel or 2-finger touchpad scroll to zoom in or out.
Help, can be disabled from the settings.' }); this.element_legend_childs.toolbox_zoomout.className += ' netdata-legend-toolbox-button'; - this.element_legend_childs.toolbox_zoomout.innerHTML = ''; + this.element_legend_childs.toolbox_zoomout.innerHTML = ''; this.element_legend_childs.toolbox.appendChild(this.element_legend_childs.toolbox_zoomout); this.element_legend_childs.toolbox_zoomout.onclick = function(e) { e.preventDefault(); @@ -3141,11 +3997,11 @@ var NETDATA = window.NETDATA || {}; placement: 'bottom', delay: { show: NETDATA.options.current.show_help_delay_show_ms, hide: NETDATA.options.current.show_help_delay_hide_ms }, title: 'Chart Zoom Out', - content: 'Zoom out the chart. On Chrome and Opera, you can also press the SHIFT or the ALT keys and then use the mouse wheel to zoom in or out.
Help, can be disabled from the settings.' + content: 'Zoom out the chart. You can also press SHIFT or ALT and use the mouse wheel, or 2-finger touchpad scroll to zoom in or out.
Help, can be disabled from the settings.' }); //this.element_legend_childs.toolbox_volume.className += ' netdata-legend-toolbox-button'; - //this.element_legend_childs.toolbox_volume.innerHTML = ''; + //this.element_legend_childs.toolbox_volume.innerHTML = ''; //this.element_legend_childs.toolbox_volume.title = 'Visible Volume'; //this.element_legend_childs.toolbox.appendChild(this.element_legend_childs.toolbox_volume); //this.element_legend_childs.toolbox_volume.onclick = function(e) { @@ -3153,41 +4009,44 @@ var NETDATA = window.NETDATA || {}; //alert('clicked toolbox_volume on ' + that.id); //} } - else { - this.element_legend_childs.toolbox = null; - this.element_legend_childs.toolbox_left = null; - this.element_legend_childs.toolbox_reset = null; - this.element_legend_childs.toolbox_right = null; - this.element_legend_childs.toolbox_zoomin = null; - this.element_legend_childs.toolbox_zoomout = null; - this.element_legend_childs.toolbox_volume = null; - } - this.element_legend_childs.resize_handler.className += " netdata-legend-resize-handler"; - this.element_legend_childs.resize_handler.innerHTML = ''; - this.element.appendChild(this.element_legend_childs.resize_handler); - if(NETDATA.options.current.show_help === true) - $(this.element_legend_childs.resize_handler).popover({ - container: "body", - animation: false, - html: true, - trigger: 'hover', - placement: 'bottom', - delay: { show: NETDATA.options.current.show_help_delay_show_ms, hide: NETDATA.options.current.show_help_delay_hide_ms }, - title: 'Chart Resize', - content: 'Drag this point with your mouse or your finger (on touch devices), to resize the chart vertically. You can also double click it or double tap it to reset between 2 states: the default and the one that fits all the values.
Help, can be disabled from the settings.' - }); + if(NETDATA.options.current.resize_charts === true) { + this.element_legend_childs.resize_handler = document.createElement('div'); + + this.element_legend_childs.resize_handler.className += " netdata-legend-resize-handler"; + this.element_legend_childs.resize_handler.innerHTML = ''; + this.element.appendChild(this.element_legend_childs.resize_handler); + if (NETDATA.options.current.show_help === true) + $(this.element_legend_childs.resize_handler).popover({ + container: "body", + animation: false, + html: true, + trigger: 'hover', + placement: 'bottom', + delay: { + show: NETDATA.options.current.show_help_delay_show_ms, + hide: NETDATA.options.current.show_help_delay_hide_ms + }, + title: 'Chart Resize', + content: 'Drag this point with your mouse or your finger (on touch devices), to resize the chart vertically. You can also double click it or double tap it to reset between 2 states: the default and the one that fits all the values.
Help, can be disabled from the settings.' + }); - // mousedown event - this.element_legend_childs.resize_handler.onmousedown = - function(e) { + // mousedown event + this.element_legend_childs.resize_handler.onmousedown = + function (e) { + that.resizeHandler(e); + }; + + // touchstart event + this.element_legend_childs.resize_handler.addEventListener('touchstart', function (e) { that.resizeHandler(e); - }; + }, false); + } - // touchstart event - this.element_legend_childs.resize_handler.addEventListener('touchstart', function(e) { - that.resizeHandler(e); - }, false); + if(this.chart) { + this.element_legend_childs.title_date.title = this.legendPluginModuleString(true); + this.element_legend_childs.title_time.title = this.legendResolutionTooltip(); + } this.element_legend_childs.title_date.className += " netdata-legend-title-date"; this.element_legend.appendChild(this.element_legend_childs.title_date); @@ -3202,6 +4061,7 @@ var NETDATA = window.NETDATA || {}; this.element_legend.appendChild(document.createElement('br')); this.element_legend_childs.title_units.className += " netdata-legend-title-units"; + this.element_legend_childs.title_units.innerText = this.units_current; this.element_legend.appendChild(this.element_legend_childs.title_units); this.tmp.__last_shown_legend_units = undefined; @@ -3213,6 +4073,8 @@ var NETDATA = window.NETDATA || {}; content.className = 'netdata-legend-series-content'; this.element_legend_childs.perfect_scroller.appendChild(content); + this.element_legend_childs.content = content; + if(NETDATA.options.current.show_help === true) $(content).popover({ container: "body", @@ -3351,37 +4213,62 @@ var NETDATA = window.NETDATA || {}; return ret; }; - this.chartURL = function() { - var after, before, points_multiplier = 1; - if(NETDATA.globalPanAndZoom.isActive() && NETDATA.globalPanAndZoom.isMaster(this) === false) { - this.tm.pan_and_zoom_seq = NETDATA.globalPanAndZoom.seq; + this.chartDataUniqueID = function() { + return this.id + ',' + this.library_name + ',' + this.dimensions + ',' + this.chartURLOptions(); + }; - after = Math.round(NETDATA.globalPanAndZoom.force_after_ms / 1000); - before = Math.round(NETDATA.globalPanAndZoom.force_before_ms / 1000); - this.view_after = after * 1000; - this.view_before = before * 1000; + this.chartURLOptions = function() { + var ret = ''; - this.requested_padding = null; - points_multiplier = 1; - } - else if(this.current.force_before_ms !== null && this.current.force_after_ms !== null) { - this.tm.pan_and_zoom_seq = 0; + if(this.override_options !== null) + ret = this.override_options.toString(); + else + ret = this.library.options(this); - before = Math.round(this.current.force_before_ms / 1000); - after = Math.round(this.current.force_after_ms / 1000); - this.view_after = after * 1000; - this.view_before = before * 1000; + if(this.append_options !== null) + ret += '|' + this.append_options.toString(); + + ret += '|jsonwrap'; + + if(NETDATA.options.current.eliminate_zero_dimensions === true) + ret += '|nonzero'; + + return ret; + }; + + this.chartURL = function() { + var after, before, points_multiplier = 1; + if(NETDATA.globalPanAndZoom.isActive()) { + if(this.current.force_before_ms !== null && this.current.force_after_ms !== null) { + this.tm.pan_and_zoom_seq = 0; + + before = Math.round(this.current.force_before_ms / 1000); + after = Math.round(this.current.force_after_ms / 1000); + this.view_after = after * 1000; + this.view_before = before * 1000; + + if(NETDATA.options.current.pan_and_zoom_data_padding === true) { + this.requested_padding = Math.round((before - after) / 2); + after -= this.requested_padding; + before += this.requested_padding; + this.requested_padding *= 1000; + points_multiplier = 2; + } - if(NETDATA.options.current.pan_and_zoom_data_padding === true) { - this.requested_padding = Math.round((before - after) / 2); - after -= this.requested_padding; - before += this.requested_padding; - this.requested_padding *= 1000; - points_multiplier = 2; + this.current.force_before_ms = null; + this.current.force_after_ms = null; } + else { + this.tm.pan_and_zoom_seq = NETDATA.globalPanAndZoom.seq; + + after = Math.round(NETDATA.globalPanAndZoom.force_after_ms / 1000); + before = Math.round(NETDATA.globalPanAndZoom.force_before_ms / 1000); + this.view_after = after * 1000; + this.view_before = before * 1000; - this.current.force_before_ms = null; - this.current.force_after_ms = null; + this.requested_padding = null; + points_multiplier = 1; + } } else { this.tm.pan_and_zoom_seq = 0; @@ -3398,26 +4285,22 @@ var NETDATA = window.NETDATA || {}; this.requested_after = after * 1000; this.requested_before = before * 1000; - this.data_points = this.points || Math.round(this.chartWidth() / this.chartPixelsPerPoint()); + var data_points; + if(NETDATA.options.force_data_points !== 0) { + data_points = NETDATA.options.force_data_points; + this.data_points = data_points; + } + else { + this.data_points = this.points || Math.round(this.chartWidth() / this.chartPixelsPerPoint()); + data_points = this.data_points * points_multiplier; + } // build the data URL this.data_url = this.host + this.chart.data_url; this.data_url += "&format=" + this.library.format(); - this.data_url += "&points=" + (this.data_points * points_multiplier).toString(); + this.data_url += "&points=" + (data_points).toString(); this.data_url += "&group=" + this.method; - - if(this.override_options !== null) - this.data_url += "&options=" + this.override_options.toString(); - else - this.data_url += "&options=" + this.library.options(this); - - this.data_url += '|jsonwrap'; - - if(NETDATA.options.current.eliminate_zero_dimensions === true) - this.data_url += '|nonzero'; - - if(this.append_options !== null) - this.data_url += '|' + this.append_options.toString(); + this.data_url += "&options=" + this.chartURLOptions(); if(after) this.data_url += "&after=" + after.toString(); @@ -3429,7 +4312,7 @@ var NETDATA = window.NETDATA || {}; this.data_url += "&dimensions=" + this.dimensions; if(NETDATA.options.debug.chart_data_url === true || this.debug === true) - this.log('chartURL(): ' + this.data_url + ' WxH:' + this.chartWidth() + 'x' + this.chartHeight() + ' points: ' + this.data_points + ' library: ' + this.library_name); + this.log('chartURL(): ' + this.data_url + ' WxH:' + this.chartWidth() + 'x' + this.chartHeight() + ' points: ' + data_points.toString() + ' library: ' + this.library_name); }; this.redrawChart = function() { @@ -3445,19 +4328,24 @@ var NETDATA = window.NETDATA || {}; resizeChart(); this.data = data; - this.updates_counter++; - this.updates_since_last_unhide++; - this.updates_since_last_creation++; var started = Date.now(); + var view_update_every = data.view_update_every * 1000; + + + if(this.data_update_every !== view_update_every) { + if(this.element_legend_childs.title_time) + this.element_legend_childs.title_time.title = this.legendResolutionTooltip(); + } // if the result is JSON, find the latest update-every - this.data_update_every = data.view_update_every * 1000; + this.data_update_every = view_update_every; this.data_after = data.after * 1000; this.data_before = data.before * 1000; this.netdata_first = data.first_entry * 1000; this.netdata_last = data.last_entry * 1000; this.data_points = data.points; + data.state = this; if(NETDATA.options.current.pan_and_zoom_data_padding === true && this.requested_padding !== null) { @@ -3549,6 +4437,58 @@ var NETDATA = window.NETDATA || {}; this.refresh_dt_element.innerText = this.refresh_dt_ms.toString(); }; + this.getSnapshotData = function(key) { + if(this.debug === true) + this.log('updating from snapshot: ' + key); + + if(typeof netdataSnapshotData.data[key] === 'undefined') { + this.log('snapshot does not include data for key "' + key + '"'); + return null; + } + + if(typeof netdataSnapshotData.data[key] !== 'string') { + this.log('snapshot data for key "' + key + '" is not string'); + return null; + } + + var uncompressed; + try { + uncompressed = netdataSnapshotData.uncompress(netdataSnapshotData.data[key]); + + if(uncompressed === null) { + this.log('uncompressed snapshot data for key ' + key + ' is null'); + return null; + } + + if(typeof uncompressed === 'undefined') { + this.log('uncompressed snapshot data for key ' + key + ' is undefined'); + return null; + } + } + catch(e) { + this.log('decompression of snapshot data for key ' + key + ' failed'); + console.log(e); + uncompressed = null; + } + + if(typeof uncompressed !== 'string') { + this.log('uncompressed snapshot data for key ' + key + ' is not string'); + return null; + } + + var data; + try { + data = JSON.parse(uncompressed); + } + catch(e) { + this.log('parsing snapshot data for key ' + key + ' failed'); + console.log(e); + data = null; + } + + return data; + }; + this.updateChart = function(callback) { if(this.debug === true) this.log('updateChart()'); @@ -3558,7 +4498,7 @@ var NETDATA = window.NETDATA || {}; this.log('I am already updating...'); if(typeof callback === 'function') - return callback(); + return callback(false, 'already running'); return; } @@ -3570,14 +4510,14 @@ var NETDATA = window.NETDATA || {}; this.log('I am not enabled'); if(typeof callback === 'function') - return callback(); + return callback(false, 'not enabled'); return; } if(canBeRendered() === false) { if(typeof callback === 'function') - return callback(); + return callback(false, 'cannot be rendered'); return; } @@ -3600,7 +4540,7 @@ var NETDATA = window.NETDATA || {}; error('chart library "' + this.library_name + '" is not available.'); if(typeof callback === 'function') - return callback(); + return callback(false, 'library not available'); return; } @@ -3609,17 +4549,40 @@ var NETDATA = window.NETDATA || {}; this.clearSelection(); this.chartURL(); - if(this.debug === true) - this.log('updating from ' + this.data_url); - NETDATA.statistics.refreshes_total++; NETDATA.statistics.refreshes_active++; if(NETDATA.statistics.refreshes_active > NETDATA.statistics.refreshes_active_max) NETDATA.statistics.refreshes_active_max = NETDATA.statistics.refreshes_active; + var ok = false; this.fetching_data = true; + if(netdataSnapshotData !== null) { + var key = this.chartDataUniqueID(); + var data = this.getSnapshotData(key); + if (data !== null) { + ok = true; + this.updateChartWithData(data); + } + else { + ok = false; + error('cannot get data from snapshot for key: "' + key + '"'); + that.tm.last_autorefreshed = Date.now(); + } + + NETDATA.statistics.refreshes_active--; + this.fetching_data = false; + + if(typeof callback === 'function') + callback(ok, 'snapshot'); + + return; + } + + if(this.debug === true) + this.log('updating from ' + this.data_url); + this.xhr = $.ajax( { url: this.data_url, cache: false, @@ -3633,6 +4596,7 @@ var NETDATA = window.NETDATA || {}; .done(function(data) { that.xhr = undefined; that.retries_on_data_failures = 0; + ok = true; if(that.debug === true) that.log('data received. updating chart.'); @@ -3662,11 +4626,14 @@ var NETDATA = window.NETDATA || {}; that.fetching_data = false; if(typeof callback === 'function') - return callback(); + return callback(ok, 'download'); }); }; var __isVisible = function() { + if(NETDATA.options.current.update_only_visible === false) + return true; + // tolerance is the number of pixels a chart can be off-screen // to consider it as visible and refresh it as if was visible var tolerance = 0; @@ -3697,6 +4664,7 @@ var NETDATA = window.NETDATA || {}; this.tmp.___isVisible___ = __isVisible(); if(this.tmp.___isVisible___ === true) unhideChart(); else hideChart(); + return this.tmp.___isVisible___; }; @@ -3705,18 +4673,16 @@ var NETDATA = window.NETDATA || {}; }; this.canBeAutoRefreshed = function() { - var now = Date.now(); - - if(this.running === true) { + if(this.enabled === false) { if(this.debug === true) - this.log('I am already running'); + this.log('canBeAutoRefreshed() -> not enabled'); return false; } - if(this.enabled === false) { + if(this.running === true) { if(this.debug === true) - this.log('I am not enabled'); + this.log('canBeAutoRefreshed() -> already running'); return false; } @@ -3724,99 +4690,112 @@ var NETDATA = window.NETDATA || {}; if(this.library === null || this.library.enabled === false) { error('charting library "' + this.library_name + '" is not available'); if(this.debug === true) - this.log('My chart library ' + this.library_name + ' is not available'); + this.log('canBeAutoRefreshed() -> chart library ' + this.library_name + ' is not available'); return false; } if(this.isVisible() === false) { if(NETDATA.options.debug.visibility === true || this.debug === true) - this.log('I am not visible'); + this.log('canBeAutoRefreshed() -> not visible'); return false; } + var now = Date.now(); + if(this.current.force_update_at !== 0 && this.current.force_update_at < now) { if(this.debug === true) - this.log('timed force update detected - allowing this update'); + this.log('canBeAutoRefreshed() -> timed force update - allowing this update'); this.current.force_update_at = 0; return true; } - if(this.isAutoRefreshable() === true) { - // allow the first update, even if the page is not visible - if(this.updates_counter && this.updates_since_last_unhide && NETDATA.options.page_is_visible === false) { - // if(NETDATA.options.debug.focus === true || this.debug === true) - // this.log('canBeAutoRefreshed(): page does not have focus'); + if(this.isAutoRefreshable() === false) { + if(this.debug === true) + this.log('canBeAutoRefreshed() -> not auto-refreshable'); - return false; - } + return false; + } + + // allow the first update, even if the page is not visible + if(NETDATA.options.page_is_visible === false && this.updates_counter && this.updates_since_last_unhide) { + if(NETDATA.options.debug.focus === true || this.debug === true) + this.log('canBeAutoRefreshed() -> not the first update, and page does not have focus'); + + return false; + } + + if(this.needsRecreation() === true) { + if(this.debug === true) + this.log('canBeAutoRefreshed() -> needs re-creation.'); + + return true; + } - if(this.needsRecreation() === true) { + if(NETDATA.options.auto_refresher_stop_until >= now) { + if(this.debug === true) + this.log('canBeAutoRefreshed() -> stopped until is in future.'); + + return false; + } + + // options valid only for autoRefresh() + if(NETDATA.globalPanAndZoom.isActive()) { + if(NETDATA.globalPanAndZoom.shouldBeAutoRefreshed(this)) { if(this.debug === true) - this.log('canBeAutoRefreshed(): needs re-creation.'); + this.log('canBeAutoRefreshed(): global panning: I need an update.'); return true; } + else { + if(this.debug === true) + this.log('canBeAutoRefreshed(): global panning: I am already up to date.'); - // options valid only for autoRefresh() - if(NETDATA.options.auto_refresher_stop_until === 0 || NETDATA.options.auto_refresher_stop_until < now) { - if(NETDATA.globalPanAndZoom.isActive()) { - if(NETDATA.globalPanAndZoom.shouldBeAutoRefreshed(this)) { - if(this.debug === true) - this.log('canBeAutoRefreshed(): global panning: I need an update.'); - - return true; - } - else { - if(this.debug === true) - this.log('canBeAutoRefreshed(): global panning: I am already up to date.'); - - return false; - } - } + return false; + } + } - if(this.selected === true) { - if(this.debug === true) - this.log('canBeAutoRefreshed(): I have a selection in place.'); + if(this.selected === true) { + if(this.debug === true) + this.log('canBeAutoRefreshed(): I have a selection in place.'); - return false; - } + return false; + } - if(this.paused === true) { - if(this.debug === true) - this.log('canBeAutoRefreshed(): I am paused.'); + if(this.paused === true) { + if(this.debug === true) + this.log('canBeAutoRefreshed(): I am paused.'); - return false; - } + return false; + } - if(now - this.tm.last_autorefreshed >= this.data_update_every) { - if(this.debug === true) - this.log('canBeAutoRefreshed(): It is time to update me.'); + if(now - this.tm.last_autorefreshed >= this.data_update_every) { + if(this.debug === true) + this.log('canBeAutoRefreshed(): It is time to update me.'); - return true; - } - } + return true; } return false; }; this.autoRefresh = function(callback) { - if(this.canBeAutoRefreshed() === true && this.running === false) { - var state = this; + var state = that; + + if(state.canBeAutoRefreshed() === true && state.running === false) { state.running = true; state.updateChart(function() { state.running = false; - if(typeof callback !== 'undefined') + if(typeof callback === 'function') return callback(); }); } else { - if(typeof callback !== 'undefined') + if(typeof callback === 'function') return callback(); } }; @@ -3831,8 +4810,10 @@ var NETDATA = window.NETDATA || {}; if(this.title === null) this.title = chart.title; - if(this.units === null) + if(this.units === null) { this.units = chart.units; + this.units_current = this.units; + } }; // fetch the chart description from the netdata server @@ -3844,6 +4825,15 @@ var NETDATA = window.NETDATA || {}; if(typeof callback === 'function') return callback(); } + else if(netdataSnapshotData !== null) { + // console.log(this); + // console.log(NETDATA.chartRegistry); + NETDATA.error(404, 'host: ' + this.host + ', chart: ' + this.id); + error('chart not found in snapshot'); + + if(typeof callback === 'function') + return callback(); + } else { this.chart_url = "/api/v1/chart?chart=" + this.id; @@ -3882,7 +4872,7 @@ var NETDATA = window.NETDATA || {}; NETDATA.resetAllCharts = function(state) { // first clear the global selection sync // to make sure no chart is in selected state - state.globalSelectionSyncStop(); + NETDATA.globalSelectionSync.stop(); // there are 2 possibilities here // a. state is the global Pan and Zoom master @@ -4036,18 +5026,86 @@ var NETDATA = window.NETDATA || {}; NETDATA.options.pause = false; }; + NETDATA.seconds4human = function (seconds, options) { + var default_options = { + now: 'now', + space: ' ', + negative_suffix: 'ago', + day: 'day', + days: 'days', + hour: 'hour', + hours: 'hours', + minute: 'min', + minutes: 'mins', + second: 'sec', + seconds: 'secs', + and: 'and' + }; + + if(typeof options !== 'object') + options = default_options; + else { + var x; + for(x in default_options) { + if(typeof options[x] !== 'string') + options[x] = default_options[x]; + } + } + + if(typeof seconds === 'string') + seconds = parseInt(seconds, 10); + + if(seconds === 0) + return options.now; + + var suffix = ''; + if(seconds < 0) { + seconds = -seconds; + if(options.negative_suffix !== '') suffix = options.space + options.negative_suffix; + } + + var days = Math.floor(seconds / 86400); + seconds -= (days * 86400); + + var hours = Math.floor(seconds / 3600); + seconds -= (hours * 3600); + + var minutes = Math.floor(seconds / 60); + seconds -= (minutes * 60); + + var strings = []; + + if(days > 1) strings.push(days.toString() + options.space + options.days); + else if(days === 1) strings.push(days.toString() + options.space + options.day); + + if(hours > 1) strings.push(hours.toString() + options.space + options.hours); + else if(hours === 1) strings.push(hours.toString() + options.space + options.hour); + + if(minutes > 1) strings.push(minutes.toString() + options.space + options.minutes); + else if(minutes === 1) strings.push(minutes.toString() + options.space + options.minute); + + if(seconds > 1) strings.push(Math.floor(seconds).toString() + options.space + options.seconds); + else if(seconds === 1) strings.push(Math.floor(seconds).toString() + options.space + options.second); + + if(strings.length === 1) + return strings.pop() + suffix; + + var last = strings.pop(); + return strings.join(", ") + " " + options.and + " " + last + suffix; + }; + // ---------------------------------------------------------------------------------------------------------------- // this is purely sequential charts refresher // it is meant to be autonomous - NETDATA.chartRefresherNoParallel = function(index) { + NETDATA.chartRefresherNoParallel = function(index, callback) { if(NETDATA.options.debug.main_loop === true) console.log('NETDATA.chartRefresherNoParallel(' + index + ')'); if(NETDATA.options.updated_dom === true) { // the dom has been updated // get the dom parts again - NETDATA.parseDom(NETDATA.chartRefresher); + NETDATA.parseDom(callback); return; } if(index >= NETDATA.options.targets.length) { @@ -4055,10 +5113,7 @@ var NETDATA = window.NETDATA || {}; console.log('waiting to restart main loop...'); NETDATA.options.auto_refresher_fast_weight = 0; - - setTimeout(function() { - NETDATA.chartRefresher(); - }, NETDATA.options.current.idle_between_loops); + callback(); } else { var state = NETDATA.options.targets[index]; @@ -4067,19 +5122,22 @@ var NETDATA = window.NETDATA || {}; if(NETDATA.options.debug.main_loop === true) console.log('fast rendering...'); - setTimeout(function() { - state.autoRefresh(function () { - NETDATA.chartRefresherNoParallel(++index); - }); - }, 0); + if(state.isVisible() === true) + NETDATA.timeout.set(function() { + state.autoRefresh(function () { + NETDATA.chartRefresherNoParallel(++index, callback); + }); + }, 0); + else + NETDATA.chartRefresherNoParallel(++index, callback); } else { if(NETDATA.options.debug.main_loop === true) console.log('waiting for next refresh...'); NETDATA.options.auto_refresher_fast_weight = 0; - setTimeout(function() { + NETDATA.timeout.set(function() { state.autoRefresh(function() { - NETDATA.chartRefresherNoParallel(++index); + NETDATA.chartRefresherNoParallel(++index, callback); }); }, NETDATA.options.current.idle_between_charts); } @@ -4091,27 +5149,92 @@ var NETDATA = window.NETDATA || {}; }; // the default refresher + NETDATA.chartRefresherLastRun = 0; + NETDATA.chartRefresherRunsAfterParseDom = 0; + NETDATA.chartRefresherTimeoutId = undefined; + + NETDATA.chartRefresherReschedule = function() { + if(NETDATA.options.current.async_on_scroll === true) { + if(NETDATA.chartRefresherTimeoutId) + NETDATA.timeout.clear(NETDATA.chartRefresherTimeoutId); + NETDATA.chartRefresherTimeoutId = NETDATA.timeout.set(NETDATA.chartRefresher, NETDATA.options.current.onscroll_worker_duration_threshold); + //console.log('chartRefresherReschedule()'); + } + }; + NETDATA.chartRefresher = function() { - // console.log('auto-refresher...'); + // console.log('chartRefresher() begin ' + (Date.now() - NETDATA.chartRefresherLastRun).toString() + ' ms since last run'); + + if(NETDATA.options.page_is_visible === false + && NETDATA.options.current.stop_updates_when_focus_is_lost === true + && NETDATA.chartRefresherLastRun > NETDATA.options.last_page_resize + && NETDATA.chartRefresherLastRun > NETDATA.options.last_page_scroll + && NETDATA.chartRefresherRunsAfterParseDom > 10 + ) { + setTimeout( + NETDATA.chartRefresher, + NETDATA.options.current.idle_lost_focus + ); + + // console.log('chartRefresher() page without focus, will run in ' + NETDATA.options.current.idle_lost_focus.toString() + ' ms, ' + NETDATA.chartRefresherRunsAfterParseDom.toString()); + return; + } + NETDATA.chartRefresherRunsAfterParseDom++; + + var now = Date.now(); + NETDATA.chartRefresherLastRun = now; + + if( now < NETDATA.options.on_scroll_refresher_stop_until ) { + NETDATA.chartRefresherTimeoutId = NETDATA.timeout.set( + NETDATA.chartRefresher, + NETDATA.chartRefresherWaitTime() + ); + + // console.log('chartRefresher() end1 will run in ' + NETDATA.chartRefresherWaitTime().toString() + ' ms'); + return; + } + + if( now < NETDATA.options.auto_refresher_stop_until ) { + NETDATA.chartRefresherTimeoutId = NETDATA.timeout.set( + NETDATA.chartRefresher, + NETDATA.chartRefresherWaitTime() + ); + + // console.log('chartRefresher() end2 will run in ' + NETDATA.chartRefresherWaitTime().toString() + ' ms'); + return; + } if(NETDATA.options.pause === true) { // console.log('auto-refresher is paused'); - setTimeout(NETDATA.chartRefresher, - NETDATA.chartRefresherWaitTime()); + NETDATA.chartRefresherTimeoutId = NETDATA.timeout.set( + NETDATA.chartRefresher, + NETDATA.chartRefresherWaitTime() + ); + + // console.log('chartRefresher() end3 will run in ' + NETDATA.chartRefresherWaitTime().toString() + ' ms'); return; } if(typeof NETDATA.options.pauseCallback === 'function') { // console.log('auto-refresher is calling pauseCallback'); + NETDATA.options.pause = true; NETDATA.options.pauseCallback(); NETDATA.chartRefresher(); + + // console.log('chartRefresher() end4 (nested)'); return; } if(NETDATA.options.current.parallel_refresher === false) { // console.log('auto-refresher is calling chartRefresherNoParallel(0)'); - NETDATA.chartRefresherNoParallel(0); + NETDATA.chartRefresherNoParallel(0, function() { + NETDATA.chartRefresherTimeoutId = NETDATA.timeout.set( + NETDATA.chartRefresher, + NETDATA.options.current.idle_between_loops + ); + }); + // console.log('chartRefresher() end5 (no parallel, nested)'); return; } @@ -4120,53 +5243,66 @@ var NETDATA = window.NETDATA || {}; // get the dom parts again // console.log('auto-refresher is calling parseDom()'); NETDATA.parseDom(NETDATA.chartRefresher); + // console.log('chartRefresher() end6 (parseDom)'); return; } - var parallel = []; - var targets = NETDATA.options.targets; - var len = targets.length; - var state; - while(len--) { - state = targets[len]; - if(state.isVisible() === false || state.running === true) - continue; - - if(state.library.initialized === false) { - if(state.library.enabled === true) { - state.library.initialize(NETDATA.chartRefresher); - return; - } - else { - state.error('chart library "' + state.library_name + '" is not enabled.'); + if(NETDATA.globalSelectionSync.active() === false) { + var parallel = []; + var targets = NETDATA.options.targets; + var len = targets.length; + var state; + while(len--) { + state = targets[len]; + if(state.isVisible() === false || state.running === true) + continue; + + if(state.library.initialized === false) { + if(state.library.enabled === true) { + state.library.initialize(NETDATA.chartRefresher); + //console.log('chartRefresher() end6 (library init)'); + return; + } + else { + state.error('chart library "' + state.library_name + '" is not enabled.'); + } } + + if(NETDATA.scrollUp === true) + parallel.unshift(state); + else + parallel.push(state); } - if(NETDATA.scrollUp === true) - parallel.unshift(state); - else - parallel.push(state); - } + len = parallel.length; + while (len--) { + state = parallel[len]; + // console.log('auto-refresher executing in parallel for ' + parallel.length.toString() + ' charts'); + // this will execute the jobs in parallel - if(parallel.length > 0) { - // console.log('auto-refresher executing in parallel for ' + parallel.length.toString() + ' charts'); - // this will execute the jobs in parallel - $(parallel).each(function() { - this.autoRefresh(); - }) + if (state.running === false) + NETDATA.timeout.set(state.autoRefresh, 0); + } + //else { + // console.log('auto-refresher nothing to do'); + //} } - //else { - // console.log('auto-refresher nothing to do'); - //} // run the next refresh iteration - setTimeout(NETDATA.chartRefresher, - NETDATA.chartRefresherWaitTime()); + NETDATA.chartRefresherTimeoutId = NETDATA.timeout.set( + NETDATA.chartRefresher, + NETDATA.chartRefresherWaitTime() + ); + + //console.log('chartRefresher() completed in ' + (Date.now() - now).toString() + ' ms'); }; NETDATA.parseDom = function(callback) { + //console.log('parseDom()'); + NETDATA.options.last_page_scroll = Date.now(); NETDATA.options.updated_dom = false; + NETDATA.chartRefresherRunsAfterParseDom = 0; var targets = $('div[data-netdata]'); //.filter(':visible'); @@ -4181,14 +5317,26 @@ var NETDATA = window.NETDATA || {}; NETDATA.options.targets.push(NETDATA.chartState(targets[len])); } + if(NETDATA.globalChartUnderlay.isActive() === true) + NETDATA.globalChartUnderlay.setup(); + else + NETDATA.globalChartUnderlay.clear(); + if(typeof callback === 'function') return callback(); }; // this is the main function - where everything starts + NETDATA.started = false; NETDATA.start = function() { // this should be called only once + if(NETDATA.started === true) { + console.log('netdata is already started'); + return; + } + + NETDATA.started = true; NETDATA.options.page_is_visible = true; $(window).blur(function() { @@ -4240,6 +5388,18 @@ var NETDATA = window.NETDATA || {}; netdataCallback(); }; + NETDATA.globalReset = function() { + NETDATA.globalSelectionSync.globalReset(); + NETDATA.globalPanAndZoom.globalReset(); + NETDATA.chartRegistry.globalReset(); + NETDATA.commonMin.globalReset(); + NETDATA.commonMax.globalReset(); + NETDATA.unitsConversion.globalReset(); + NETDATA.options.targets = null; + NETDATA.parseDom(); + NETDATA.unpause(); + }; + // ---------------------------------------------------------------------------------------------------------------- // peity @@ -4375,7 +5535,7 @@ var NETDATA = window.NETDATA || {}; var tooltipClassname = NETDATA.dataAttribute(state.element, 'sparkline-tooltipclassname', undefined); var tooltipFormat = NETDATA.dataAttribute(state.element, 'sparkline-tooltipformat', undefined); var tooltipPrefix = NETDATA.dataAttribute(state.element, 'sparkline-tooltipprefix', undefined); - var tooltipSuffix = NETDATA.dataAttribute(state.element, 'sparkline-tooltipsuffix', ' ' + state.units); + var tooltipSuffix = NETDATA.dataAttribute(state.element, 'sparkline-tooltipsuffix', ' ' + state.units_current); var tooltipSkipNull = NETDATA.dataAttributeBoolean(state.element, 'sparkline-tooltipskipnull', true); var tooltipValueLookups = NETDATA.dataAttribute(state.element, 'sparkline-tooltipvaluelookups', undefined); var tooltipFormatFieldlist = NETDATA.dataAttribute(state.element, 'sparkline-tooltipformatfieldlist', undefined); @@ -4444,6 +5604,7 @@ var NETDATA = window.NETDATA || {}; }; $(state.element_chart).sparkline(data.result, state.sparkline_options); + return true; }; @@ -4462,10 +5623,11 @@ var NETDATA = window.NETDATA || {}; before = state.netdata_last; state.setMode('zoom'); - state.globalSelectionSyncStop(); - state.globalSelectionSyncDelay(); + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); state.tmp.dygraph_user_action = true; state.tmp.dygraph_force_zoom = true; + // state.log('toolboxPanAndZoom'); state.updateChartPanOrZoom(after, before); NETDATA.globalPanAndZoom.setMaster(state, after, before); }; @@ -4473,15 +5635,17 @@ var NETDATA = window.NETDATA || {}; NETDATA.dygraphSetSelection = function(state, t) { if(typeof state.tmp.dygraph_instance !== 'undefined') { var r = state.calculateRowForTime(t); - if(r !== -1) + if(r !== -1) { state.tmp.dygraph_instance.setSelection(r); + return true; + } else { state.tmp.dygraph_instance.clearSelection(); state.legendShowUndefined(); } } - return true; + return false; }; NETDATA.dygraphClearSelection = function(state) { @@ -4562,6 +5726,10 @@ var NETDATA = window.NETDATA || {}; visibility: state.dimensions_visibility.selected2BooleanArray(state.data.dimension_names) }; + if(!NETDATA.chartLibraries.dygraph.isSparkline(state)) { + options.ylabel = state.units_current; // (state.units_desired === 'auto')?"":state.units_current; + } + if(state.tmp.dygraph_force_zoom === true) { if(NETDATA.options.debug.dygraph === true || state.debug === true) state.log('dygraphChartUpdate() forced zoom update'); @@ -4602,6 +5770,12 @@ var NETDATA = window.NETDATA || {}; } } + if(netdataSnapshotData !== null && NETDATA.globalPanAndZoom.isActive() === true && NETDATA.globalPanAndZoom.isMaster(state) === false) { + // pan and zoom on snapshots + options.dateWindow = [ NETDATA.globalPanAndZoom.force_after_ms, NETDATA.globalPanAndZoom.force_before_ms ]; + options.isZoomedIgnoreProgrammaticZoom = true; + } + dygraph.updateOptions(options); var redraw = false; @@ -4662,7 +5836,7 @@ var NETDATA = window.NETDATA || {}; xRangePad: NETDATA.dataAttribute(state.element, 'dygraph-xrangepad', 0), yRangePad: NETDATA.dataAttribute(state.element, 'dygraph-yrangepad', 1), valueRange: NETDATA.dataAttribute(state.element, 'dygraph-valuerange', [ null, null ]), - ylabel: state.units, + ylabel: state.units_current, // (state.units_desired === 'auto')?"":state.units_current, yLabelWidth: NETDATA.dataAttribute(state.element, 'dygraph-ylabelwidth', 12), // the function to plot the chart @@ -4724,7 +5898,7 @@ var NETDATA = window.NETDATA || {}; ticker: Dygraph.dateTicker, axisLabelFormatter: function (d, gran) { void(gran); - return NETDATA.zeropad(d.getHours()) + ":" + NETDATA.zeropad(d.getMinutes()) + ":" + NETDATA.zeropad(d.getSeconds()); + return NETDATA.dateTime.xAxisTimeString(d); } }, y: { @@ -4737,11 +5911,41 @@ var NETDATA = window.NETDATA || {}; this.axes_[0].extremeRange[1] ); - return state.legendFormatValue(y); + var old_units = this.user_attrs_.ylabel; + var v = state.legendFormatValue(y); + var new_units = state.units_current; + + if(state.units_desired === 'auto' && typeof old_units !== 'undefined' && new_units !== old_units && !NETDATA.chartLibraries.dygraph.isSparkline(state)) { + // console.log(this); + // state.log('units discrepancy: old = ' + old_units + ', new = ' + new_units); + var len = this.plugins_.length; + while(len--) { + // console.log(this.plugins_[len]); + if(typeof this.plugins_[len].plugin.ylabel_div_ !== 'undefined' + && this.plugins_[len].plugin.ylabel_div_ !== null + && typeof this.plugins_[len].plugin.ylabel_div_.children !== 'undefined' + && this.plugins_[len].plugin.ylabel_div_.children !== null + && typeof this.plugins_[len].plugin.ylabel_div_.children[0].children !== 'undefined' + && this.plugins_[len].plugin.ylabel_div_.children[0].children !== null + ) { + this.plugins_[len].plugin.ylabel_div_.children[0].children[0].innerHTML = new_units; + this.user_attrs_.ylabel = new_units; + break; + } + } + + if(len < 0) + state.log('units discrepancy, but cannot find dygraphs div to change: old = ' + old_units + ', new = ' + new_units); + } + + return v; } } }, legendFormatter: function(data) { + if(state.tmp.dygraph_mouse_down === true) + return; + var elements = state.element_legend_childs; // if the hidden div is not there @@ -4763,6 +5967,16 @@ var NETDATA = window.NETDATA || {}; return ''; }, drawCallback: function(dygraph, is_initial) { + + // the user has panned the chart and this is called to re-draw the chart + // 1. refresh this chart by adding data to it + // 2. notify all the other charts about the update they need + + // to prevent an infinite loop (feedback), we use + // state.tmp.dygraph_user_action + // - when true, this is initiated by a user + // - when false, this is feedback + if(state.current.name !== 'auto' && state.tmp.dygraph_user_action === true) { state.tmp.dygraph_user_action = false; @@ -4771,20 +5985,27 @@ var NETDATA = window.NETDATA || {}; var before = Math.round(x_range[1]); if(NETDATA.options.debug.dygraph === true) - state.log('dygraphDrawCallback(dygraph, ' + is_initial + '): ' + (after / 1000).toString() + ' - ' + (before / 1000).toString()); + state.log('dygraphDrawCallback(dygraph, ' + is_initial + '): mode ' + state.current.name + ' ' + (after / 1000).toString() + ' - ' + (before / 1000).toString()); + //console.log(state); if(before <= state.netdata_last && after >= state.netdata_first) + // update only when we are within the data limits state.updateChartPanOrZoom(after, before); } }, zoomCallback: function(minDate, maxDate, yRanges) { + + // the user has selected a range on the chart + // 1. refresh this chart by adding data to it + // 2. notify all the other charts about the update they need + void(yRanges); if(NETDATA.options.debug.dygraph === true) - state.log('dygraphZoomCallback()'); + state.log('dygraphZoomCallback(): ' + state.current.name); - state.globalSelectionSyncStop(); - state.globalSelectionSyncDelay(); + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); state.setMode('zoom'); // refresh it to the greatest possible zoom level @@ -4795,9 +6016,6 @@ var NETDATA = window.NETDATA || {}; highlightCallback: function(event, x, points, row, seriesName) { void(seriesName); - if(NETDATA.options.debug.dygraph === true || state.debug === true) - state.log('dygraphHighlightCallback()'); - state.pauseChart(); // there is a bug in dygraph when the chart is zoomed enough @@ -4807,7 +6025,8 @@ var NETDATA = window.NETDATA || {}; // var t = state.data_after + row * state.data_update_every; // console.log('row = ' + row + ', x = ' + x + ', t = ' + t + ' ' + ((t === x)?'SAME':(Math.abs(x-t)<=state.data_update_every)?'SIMILAR':'DIFFERENT') + ', rows in db: ' + state.data_points + ' visible(x) = ' + state.timeIsVisible(x) + ' visible(t) = ' + state.timeIsVisible(t) + ' r(x) = ' + state.calculateRowForTime(x) + ' r(t) = ' + state.calculateRowForTime(t) + ' range: ' + state.data_after + ' - ' + state.data_before + ' real: ' + state.data.after + ' - ' + state.data.before + ' every: ' + state.data_update_every); - state.globalSelectionSync(x); + if(state.tmp.dygraph_mouse_down !== true) + NETDATA.globalSelectionSync.sync(state, x); // fix legend zIndex using the internal structures of dygraph legend module // this works, but it is a hack! @@ -4816,11 +6035,41 @@ var NETDATA = window.NETDATA || {}; unhighlightCallback: function(event) { void(event); + if(state.tmp.dygraph_mouse_down === true) + return; + if(NETDATA.options.debug.dygraph === true || state.debug === true) state.log('dygraphUnhighlightCallback()'); state.unpauseChart(); - state.globalSelectionSyncStop(); + NETDATA.globalSelectionSync.stop(); + }, + underlayCallback: function(canvas, area, g) { + + // the chart is about to be drawn + // this function renders global highlighted time-frame + + if(NETDATA.globalChartUnderlay.isActive()) { + var after = NETDATA.globalChartUnderlay.after; + var before = NETDATA.globalChartUnderlay.before; + + if(after < state.view_after) + after = state.view_after; + + if(before > state.view_before) + before = state.view_before; + + if(after < before) { + var bottom_left = g.toDomCoords(after, -20); + var top_right = g.toDomCoords(before, +20); + + var left = bottom_left[0]; + var right = top_right[0]; + + canvas.fillStyle = NETDATA.themes.current.highlight; + canvas.fillRect(left, area.y, right - left, area.h); + } + } }, interactionModel : { mousedown: function(event, dygraph, context) { @@ -4828,37 +6077,73 @@ var NETDATA = window.NETDATA || {}; state.log('interactionModel.mousedown()'); state.tmp.dygraph_user_action = true; - state.globalSelectionSyncStop(); if(NETDATA.options.debug.dygraph === true) state.log('dygraphMouseDown()'); - // Right-click should not initiate a zoom. + // Right-click should not initiate anything. if(event.button && event.button === 2) return; + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + + state.tmp.dygraph_mouse_down = true; context.initializeMouseDown(event, dygraph, context); + //console.log(event); if(event.button && event.button === 1) { - if (event.altKey || event.shiftKey) { + if (event.shiftKey) { + //console.log('middle mouse button dragging (PAN)'); + state.setMode('pan'); - state.globalSelectionSyncDelay(); + // NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_highlight_after = null; Dygraph.startPan(event, dygraph, context); } + else if(event.altKey || event.ctrlKey || event.metaKey) { + //console.log('middle mouse button highlight'); + + if (!(event.offsetX && event.offsetY)) { + event.offsetX = event.layerX - event.target.offsetLeft; + event.offsetY = event.layerY - event.target.offsetTop; + } + state.tmp.dygraph_highlight_after = dygraph.toDataXCoord(event.offsetX); + Dygraph.startZoom(event, dygraph, context); + } else { + //console.log('middle mouse button selection for zoom (ZOOM)'); + state.setMode('zoom'); - state.globalSelectionSyncDelay(); + // NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_highlight_after = null; Dygraph.startZoom(event, dygraph, context); } } else { - if (event.altKey || event.shiftKey) { + if (event.shiftKey) { + //console.log('left mouse button selection for zoom (ZOOM)'); + state.setMode('zoom'); - state.globalSelectionSyncDelay(); + // NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_highlight_after = null; + Dygraph.startZoom(event, dygraph, context); + } + else if(event.altKey || event.ctrlKey || event.metaKey) { + //console.log('left mouse button highlight'); + + if (!(event.offsetX && event.offsetY)) { + event.offsetX = event.layerX - event.target.offsetLeft; + event.offsetY = event.layerY - event.target.offsetTop; + } + state.tmp.dygraph_highlight_after = dygraph.toDataXCoord(event.offsetX); Dygraph.startZoom(event, dygraph, context); } else { + //console.log('left mouse button dragging (PAN)'); + state.setMode('pan'); - state.globalSelectionSyncDelay(); + // NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_highlight_after = null; Dygraph.startPan(event, dygraph, context); } } @@ -4867,35 +6152,98 @@ var NETDATA = window.NETDATA || {}; if(NETDATA.options.debug.dygraph === true || state.debug === true) state.log('interactionModel.mousemove()'); - if(context.isPanning) { + if(state.tmp.dygraph_highlight_after !== null) { + //console.log('highlight selection...'); + + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + + state.tmp.dygraph_user_action = true; + Dygraph.moveZoom(event, dygraph, context); + event.preventDefault(); + } + else if(context.isPanning) { + //console.log('panning...'); + + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_user_action = true; - state.globalSelectionSyncStop(); - state.globalSelectionSyncDelay(); + //NETDATA.globalSelectionSync.stop(); + //NETDATA.globalSelectionSync.delay(); state.setMode('pan'); context.is2DPan = false; Dygraph.movePan(event, dygraph, context); } else if(context.isZooming) { + //console.log('zooming...'); + + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_user_action = true; - state.globalSelectionSyncStop(); - state.globalSelectionSyncDelay(); + //NETDATA.globalSelectionSync.stop(); + //NETDATA.globalSelectionSync.delay(); state.setMode('zoom'); Dygraph.moveZoom(event, dygraph, context); } }, mouseup: function(event, dygraph, context) { + state.tmp.dygraph_mouse_down = false; + if(NETDATA.options.debug.dygraph === true || state.debug === true) state.log('interactionModel.mouseup()'); - if (context.isPanning) { + if(state.tmp.dygraph_highlight_after !== null) { + //console.log('done highlight selection'); + + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + + if (!(event.offsetX && event.offsetY)){ + event.offsetX = event.layerX - event.target.offsetLeft; + event.offsetY = event.layerY - event.target.offsetTop; + } + + NETDATA.globalChartUnderlay.set(state + , state.tmp.dygraph_highlight_after + , dygraph.toDataXCoord(event.offsetX) + , state.view_after + , state.view_before + ); + + state.tmp.dygraph_highlight_after = null; + + context.isZooming = false; + dygraph.clearZoomRect_(); + dygraph.drawGraph_(false); + + // refresh all the charts immediately + NETDATA.options.auto_refresher_stop_until = 0; + } + else if (context.isPanning) { + //console.log('done panning'); + + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_user_action = true; - state.globalSelectionSyncDelay(); Dygraph.endPan(event, dygraph, context); + + // refresh all the charts immediately + NETDATA.options.auto_refresher_stop_until = 0; } else if (context.isZooming) { + //console.log('done zomming'); + + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_user_action = true; - state.globalSelectionSyncDelay(); Dygraph.endZoom(event, dygraph, context); + + // refresh all the charts immediately + NETDATA.options.auto_refresher_stop_until = 0; } }, click: function(event, dygraph, context) { @@ -4984,8 +6332,8 @@ var NETDATA = window.NETDATA || {}; if(event.altKey || event.shiftKey) { state.tmp.dygraph_user_action = true; - state.globalSelectionSyncStop(); - state.globalSelectionSyncDelay(); + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); // http://dygraphs.com/gallery/interaction-api.js var normal_def; @@ -5024,13 +6372,16 @@ var NETDATA = window.NETDATA || {}; } state.setMode('zoom'); - if(state.updateChartPanOrZoom(after, before) === true) + state.updateChartPanOrZoom(after, before, function() { dygraph.updateOptions({ dateWindow: [ after, before ] }); + }); event.preventDefault(); } }, touchstart: function(event, dygraph, context) { + state.tmp.dygraph_mouse_down = true; + if(NETDATA.options.debug.dygraph === true || state.debug === true) state.log('interactionModel.touchstart()'); @@ -5038,6 +6389,9 @@ var NETDATA = window.NETDATA || {}; state.setMode('zoom'); state.pauseChart(); + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + Dygraph.defaultInteractionModel.touchstart(event, dygraph, context); // we overwrite the touch directions at the end, to overwrite @@ -5056,25 +6410,39 @@ var NETDATA = window.NETDATA || {}; if(NETDATA.options.debug.dygraph === true || state.debug === true) state.log('interactionModel.touchmove()'); + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_user_action = true; Dygraph.defaultInteractionModel.touchmove(event, dygraph, context); state.dygraph_last_touch_move = Date.now(); }, touchend: function(event, dygraph, context) { + state.tmp.dygraph_mouse_down = false; + if(NETDATA.options.debug.dygraph === true || state.debug === true) state.log('interactionModel.touchend()'); + NETDATA.globalSelectionSync.stop(); + NETDATA.globalSelectionSync.delay(); + state.tmp.dygraph_user_action = true; Dygraph.defaultInteractionModel.touchend(event, dygraph, context); // if it didn't move, it is a selection if(state.dygraph_last_touch_move === 0 && state.dygraph_last_touch_page_x !== 0) { + NETDATA.globalSelectionSync.dont_sync_before = 0; + NETDATA.globalSelectionSync.setMaster(state); + // internal api of dygraph var pct = (state.dygraph_last_touch_page_x - (dygraph.plotter_.area.x + state.element.getBoundingClientRect().left)) / dygraph.plotter_.area.w; - var t = Math.round(state.data_after + (state.data_before - state.data_after) * pct); - if(NETDATA.dygraphSetSelection(state, t) === true) - state.globalSelectionSync(t); + console.log('pct: ' + pct.toString()); + + var t = Math.round(state.view_after + (state.view_before - state.view_after) * pct); + if(NETDATA.dygraphSetSelection(state, t) === true) { + NETDATA.globalSelectionSync.sync(state, t); + } } // if it was double tap within double click time, reset the charts @@ -5089,6 +6457,9 @@ var NETDATA = window.NETDATA || {}; // remember the timestamp of the last touch end state.dygraph_last_touch_end = now; + + // refresh all the charts immediately + NETDATA.options.auto_refresher_stop_until = 0; } } }; @@ -5114,12 +6485,19 @@ var NETDATA = window.NETDATA || {}; } else state.tmp.dygraph_smooth_eligible = false; + if(netdataSnapshotData !== null && NETDATA.globalPanAndZoom.isActive() === true && NETDATA.globalPanAndZoom.isMaster(state) === false) { + // pan and zoom on snapshots + state.tmp.dygraph_options.dateWindow = [ NETDATA.globalPanAndZoom.force_after_ms, NETDATA.globalPanAndZoom.force_before_ms ]; + state.tmp.dygraph_options.isZoomedIgnoreProgrammaticZoom = true; + } + state.tmp.dygraph_instance = new Dygraph(state.element_chart, data.result.data, state.tmp.dygraph_options); state.tmp.dygraph_force_zoom = false; state.tmp.dygraph_user_action = false; state.tmp.dygraph_last_rendered = Date.now(); + state.tmp.dygraph_highlight_after = null; if(state.tmp.dygraph_options.valueRange[0] === null && state.tmp.dygraph_options.valueRange[1] === null) { if (typeof state.tmp.dygraph_instance.axes_[0].extremeRange !== 'undefined') { @@ -5356,7 +6734,7 @@ var NETDATA = window.NETDATA || {}; type: 'timeseries', tick: { format: function(x) { - return NETDATA.zeropad(x.getHours()) + ":" + NETDATA.zeropad(x.getMinutes()) + ":" + NETDATA.zeropad(x.getSeconds()); + return NETDATA.dateTime.xAxisTimeString(x); } } } @@ -5496,7 +6874,7 @@ var NETDATA = window.NETDATA || {}; } }, vAxis: { - title: state.units, + title: state.units_current, viewWindowMode: 'pretty', minValue: -0.1, maxValue: 0.1, @@ -5578,6 +6956,8 @@ var NETDATA = window.NETDATA || {}; if(min > value) min = value; if(max < value) max = value; + state.legendFormatValueDecimalsFromMinMax(min, max); + if(state.tmp.easyPieChartMin === null && min > 0) min = 0; if(state.tmp.easyPieChartMax === null && max < 0) max = 0; @@ -5635,8 +7015,8 @@ var NETDATA = window.NETDATA || {}; NETDATA.easypiechartClearSelection = function(state) { if(typeof state.tmp.easyPieChartEvent !== 'undefined') { - if(state.tmp.easyPieChartEvent.timer !== undefined) { - clearTimeout(state.tmp.easyPieChartEvent.timer); + if(state.tmp.easyPieChartEvent.timer) { + NETDATA.timeout.clear(state.tmp.easyPieChartEvent.timer); } state.tmp.easyPieChartEvent.timer = undefined; @@ -5682,10 +7062,10 @@ var NETDATA = window.NETDATA || {}; if(state.tmp.easyPieChartEvent.timer === undefined) { state.tmp.easyPieChart_instance.disableAnimation(); - state.tmp.easyPieChartEvent.timer = setTimeout(function() { + state.tmp.easyPieChartEvent.timer = NETDATA.timeout.set(function() { state.tmp.easyPieChartEvent.timer = undefined; state.tmp.easyPieChart_instance.update(state.tmp.easyPieChartEvent.pcent); - }, NETDATA.options.current.charts_selection_animation_delay); + }, 0); } return true; @@ -5731,10 +7111,6 @@ var NETDATA = window.NETDATA || {}; else state.tmp.easyPieChartMax = max; - var pcent = NETDATA.easypiechartPercentFromValueMinMax(state, value, min, max); - - chart.data('data-percent', pcent); - var size = state.chartWidth(); var stroke = Math.floor(size / 22); if(stroke < 3) stroke = 2; @@ -5762,7 +7138,7 @@ var NETDATA = window.NETDATA || {}; var unittop = Math.round(valuetop + (valuefontsize + unitfontsize) + (size / 40)); state.tmp.easyPieChartUnits = document.createElement('span'); state.tmp.easyPieChartUnits.className = 'easyPieChartUnits'; - state.tmp.easyPieChartUnits.innerText = state.units; + state.tmp.easyPieChartUnits.innerText = state.units_current; state.tmp.easyPieChartUnits.style.fontSize = unitfontsize + 'px'; state.tmp.easyPieChartUnits.style.top = unittop.toString() + 'px'; state.element_chart.appendChild(state.tmp.easyPieChartUnits); @@ -5777,6 +7153,9 @@ var NETDATA = window.NETDATA || {}; barColor = tmp; } + var pcent = NETDATA.easypiechartPercentFromValueMinMax(state, value, min, max); + chart.data('data-percent', pcent); + chart.easyPieChart({ barColor: barColor, trackColor: NETDATA.dataAttribute(state.element, 'easypiechart-trackcolor', NETDATA.themes.current.easypiechart_track), @@ -5801,6 +7180,16 @@ var NETDATA = window.NETDATA || {}; if(animate === false) state.tmp.easyPieChart_instance.disableAnimation(); state.tmp.easyPieChart_instance.update(pcent); if(animate === false) state.tmp.easyPieChart_instance.enableAnimation(); + + state.legendSetUnitsString = function(units) { + if(typeof state.tmp.easyPieChartUnits !== 'undefined') + state.tmp.easyPieChartUnits.innerText = units; + }; + state.legendShowUndefined = function() { + if(typeof state.tmp.easyPieChart_instance !== 'undefined') + NETDATA.easypiechartClearSelection(state); + }; + return true; }; @@ -5861,6 +7250,8 @@ var NETDATA = window.NETDATA || {}; else if(min === max) max = min + 1; + state.legendFormatValueDecimalsFromMinMax(min, max); + // gauge.js has an issue if the needle // is smaller than min or larger than max // when we set the new values @@ -5901,8 +7292,8 @@ var NETDATA = window.NETDATA || {}; NETDATA.gaugeClearSelection = function(state) { if(typeof state.tmp.gaugeEvent !== 'undefined') { - if(state.tmp.gaugeEvent.timer !== undefined) { - clearTimeout(state.tmp.gaugeEvent.timer); + if(state.tmp.gaugeEvent.timer) { + NETDATA.timeout.clear(state.tmp.gaugeEvent.timer); } state.tmp.gaugeEvent.timer = undefined; @@ -5955,10 +7346,10 @@ var NETDATA = window.NETDATA || {}; if(state.tmp.gaugeEvent.timer === undefined) { NETDATA.gaugeAnimation(state, false); - state.tmp.gaugeEvent.timer = setTimeout(function() { + state.tmp.gaugeEvent.timer = NETDATA.timeout.set(function() { state.tmp.gaugeEvent.timer = undefined; NETDATA.gaugeSet(state, state.tmp.gaugeEvent.value, state.tmp.gaugeEvent.min, state.tmp.gaugeEvent.max); - }, NETDATA.options.current.charts_selection_animation_delay); + }, 0); } return true; @@ -5968,10 +7359,8 @@ var NETDATA = window.NETDATA || {}; var value, min, max; if(NETDATA.globalPanAndZoom.isActive() === true || state.isAutoRefreshable() === false) { - value = 0; - min = 0; - max = 1; NETDATA.gaugeSetLabels(state, null, null, null); + state.tmp.gauge_instance.set(0); } else { value = data.result[0]; @@ -5985,10 +7374,10 @@ var NETDATA = window.NETDATA || {}; if(state.tmp.gaugeMin === null && min > 0) min = 0; if(state.tmp.gaugeMax === null && max < 0) max = 0; + NETDATA.gaugeSet(state, value, min, max); NETDATA.gaugeSetLabels(state, value, min, max); } - NETDATA.gaugeSet(state, value, min, max); return true; }; @@ -6003,7 +7392,7 @@ var NETDATA = window.NETDATA || {}; var strokeColor = NETDATA.dataAttribute(state.element, 'gauge-stroke-color', NETDATA.themes.current.gauge_stroke); var startColor = NETDATA.dataAttribute(state.element, 'gauge-start-color', state.chartCustomColors()[0]); var stopColor = NETDATA.dataAttribute(state.element, 'gauge-stop-color', void 0); - var generateGradient = NETDATA.dataAttributeBoolean(state.element, 'gauge-generate-gradient', false); + var generateGradient = NETDATA.dataAttribute(state.element, 'gauge-generate-gradient', false); if(min === null) { min = NETDATA.commonMin.get(state); @@ -6122,7 +7511,7 @@ var NETDATA = window.NETDATA || {}; var unitfontsize = Math.round(titlefontsize * 0.9); state.tmp.gaugeChartUnits = document.createElement('span'); state.tmp.gaugeChartUnits.className = 'gaugeChartUnits'; - state.tmp.gaugeChartUnits.innerText = state.units; + state.tmp.gaugeChartUnits.innerText = state.units_current; state.tmp.gaugeChartUnits.style.fontSize = unitfontsize + 'px'; state.element_chart.appendChild(state.tmp.gaugeChartUnits); @@ -6161,6 +7550,20 @@ var NETDATA = window.NETDATA || {}; NETDATA.gaugeSet(state, value, min, max); NETDATA.gaugeSetLabels(state, value, min, max); NETDATA.gaugeAnimation(state, true); + + state.legendSetUnitsString = function(units) { + if(typeof state.tmp.gaugeChartUnits !== 'undefined') { + state.tmp.gaugeChartUnits.innerText = units; + state.tmp.___gaugeOld__.valueLabel = null; + state.tmp.___gaugeOld__.minLabel = null; + state.tmp.___gaugeOld__.maxLabel = null; + } + }; + state.legendShowUndefined = function() { + if(typeof state.tmp.gauge_instance !== 'undefined') + NETDATA.gaugeClearSelection(state); + }; + return true; }; @@ -6173,7 +7576,7 @@ var NETDATA = window.NETDATA || {}; create: NETDATA.dygraphChartCreate, update: NETDATA.dygraphChartUpdate, resize: function(state) { - if(typeof state.tmp.dygraph_instance.resize === 'function') + if(typeof state.tmp.dygraph_instance !== 'undefined' && typeof state.tmp.dygraph_instance.resize === 'function') state.tmp.dygraph_instance.resize(); }, setSelection: NETDATA.dygraphSetSelection, @@ -6394,7 +7797,7 @@ var NETDATA = window.NETDATA || {}; NETDATA.requiredJs = [ { - url: NETDATA.serverDefault + 'lib/bootstrap-3.3.7.min.js', + url: NETDATA.serverStatic + 'lib/bootstrap-3.3.7.min.js', async: false, isAlreadyLoaded: function() { // check if bootstrap is loaded @@ -6406,7 +7809,12 @@ var NETDATA = window.NETDATA || {}; } }, { - url: NETDATA.serverDefault + 'lib/perfect-scrollbar-0.6.15.min.js', + url: NETDATA.serverStatic + 'lib/fontawesome-all-5.0.1.min.js', + async: true, + isAlreadyLoaded: function() { return false; } + }, + { + url: NETDATA.serverStatic + 'lib/perfect-scrollbar-0.6.15.min.js', isAlreadyLoaded: function() { return false; } } ]; @@ -6418,10 +7826,6 @@ var NETDATA = window.NETDATA || {}; return (typeof netdataNoBootstrap !== 'undefined' && netdataNoBootstrap === true); } }, - { - url: NETDATA.serverDefault + 'css/font-awesome.min.css?v4.7.0', - isAlreadyLoaded: function() { return false; } - }, { url: NETDATA.themes.current.dashboard_css, isAlreadyLoaded: function() { return false; } @@ -6619,7 +8023,7 @@ var NETDATA = window.NETDATA || {}; body: entry.hostname + ' - ' + entry.chart + ' (' + entry.family + ') - ' + status + ': ' + entry.info, tag: tag, requireInteraction: interaction, - icon: NETDATA.serverDefault + icon, + icon: NETDATA.serverStatic + icon, data: data }); @@ -6753,6 +8157,9 @@ var NETDATA = window.NETDATA || {}; }, update_forever: function() { + if(netdataShowAlarms !== true || netdataSnapshotData !== null) + return; + NETDATA.alarms.get('active', function(data) { if(data !== null) { NETDATA.alarms.current = data; diff --git a/web/dashboard.slate.css b/web/dashboard.slate.css index 7445d532c..00e0d0dce 100644 --- a/web/dashboard.slate.css +++ b/web/dashboard.slate.css @@ -58,6 +58,8 @@ code { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* required for child elements to have absolute position */ position: relative; @@ -68,6 +70,8 @@ code { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* required for child elements to have absolute position */ position: relative; @@ -84,6 +88,8 @@ code { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* required for child elements to have absolute position */ position: relative; @@ -107,6 +113,8 @@ code { display: inline-block; overflow: hidden; + transform: translate3d(0,0,0); + /* fix minimum scrollbar issue in firefox */ min-height: 99px; @@ -122,9 +130,9 @@ code { bottom: 0px; right: 0px; height: 15px; - width: 30px; + width: 20px; background-color: #272b30; - font-size: 12px; + font-size: 15px; vertical-align: middle; line-height: 15px; cursor: ns-resize; @@ -300,6 +308,7 @@ code { white-space: nowrap; display: inline-block; cursor: pointer; + -webkit-print-color-adjust: exact; } .netdata-legend-value { diff --git a/web/dashboard_info.js b/web/dashboard_info.js index bb2f95991..a3c48640c 100644 --- a/web/dashboard_info.js +++ b/web/dashboard_info.js @@ -9,142 +9,142 @@ var netdataDashboard = window.netdataDashboard || {}; netdataDashboard.menu = { 'system': { title: 'System Overview', - icon: '', + icon: '', info: 'Overview of the key system metrics.' }, 'services': { title: 'systemd Services', - icon: '', + icon: '', info: 'Resources utilization of systemd services. netdata monitors all systemd services via cgroups (the resources accounting used by containers). ' }, 'ap': { title: 'Access Points', - icon: '', + icon: '', info: 'Performance metrics for the access points (i.e. wireless interfaces in AP mode) found on the system.' }, 'tc': { title: 'Quality of Service', - icon: '', + icon: '', info: 'Netdata collects and visualizes tc class utilization using its tc-helper plugin. If you also use FireQOS for setting up QoS, netdata automatically collects interface and class names. If your QoS configuration includes overheads calculation, the values shown here will include these overheads (the total bandwidth for the same interface as reported in the Network Interfaces section, will be lower than the total bandwidth reported here). QoS data collection may have a slight time difference compared to the interface (QoS data collection uses a BASH script, so a shift in data collection of a few milliseconds should be justified).' }, 'net': { title: 'Network Interfaces', - icon: '', + icon: '', info: 'Performance metrics for network interfaces.' }, 'ipv4': { title: 'IPv4 Networking', - icon: '', + icon: '', info: 'Metrics for the IPv4 stack of the system. Internet Protocol version 4 (IPv4) is the fourth version of the Internet Protocol (IP). It is one of the core protocols of standards-based internetworking methods in the Internet. IPv4 is a connectionless protocol for use on packet-switched networks. It operates on a best effort delivery model, in that it does not guarantee delivery, nor does it assure proper sequencing or avoidance of duplicate delivery. These aspects, including data integrity, are addressed by an upper layer transport protocol, such as the Transmission Control Protocol (TCP).' }, 'ipv6': { title: 'IPv6 Networking', - icon: '', + icon: '', info: 'Metrics for the IPv6 stack of the system. Internet Protocol version 6 (IPv6) is the most recent version of the Internet Protocol (IP), the communications protocol that provides an identification and location system for computers on networks and routes traffic across the Internet. IPv6 was developed by the Internet Engineering Task Force (IETF) to deal with the long-anticipated problem of IPv4 address exhaustion. IPv6 is intended to replace IPv4.' }, 'ipvs': { title: 'IP Virtual Server', - icon: '', + icon: '', info: 'IPVS (IP Virtual Server) implements transport-layer load balancing inside the Linux kernel, so called Layer-4 switching. IPVS running on a host acts as a load balancer at the front of a cluster of real servers, it can direct requests for TCP/UDP based services to the real servers, and makes services of the real servers to appear as a virtual service on a single IP address.' }, 'netfilter': { title: 'Firewall (netfilter)', - icon: '', + icon: '', info: 'Performance metrics of the netfilter components.' }, 'ipfw': { title: 'Firewall (ipfw)', - icon: '', + icon: '', info: 'Counters and memory usage for the ipfw rules.' }, 'cpu': { title: 'CPUs', - icon: '', + icon: '', info: 'Detailed information for each CPU of the system. A summary of the system for all CPUs can be found at the System Overview section.' }, 'mem': { title: 'Memory', - icon: '', + icon: '', info: 'Detailed information about the memory management of the system.' }, 'disk': { title: 'Disks', - icon: '', + icon: '', info: 'Charts with performance information for all the system disks. Special care has been given to present disk performance metrics in a way compatible with iostat -x. netdata by default prevents rendering performance charts for individual partitions and unmounted virtual disks. Disabled charts can still be enabled by configuring the relative settings in the netdata configuration file.' }, 'sensors': { title: 'Sensors', - icon: '', + icon: '', info: 'Readings of the configured system sensors.' }, 'ipmi': { title: 'IPMI', - icon: '', + icon: '', info: 'The Intelligent Platform Management Interface (IPMI) is a set of computer interface specifications for an autonomous computer subsystem that provides management and monitoring capabilities independently of the host system\'s CPU, firmware (BIOS or UEFI) and operating system.' }, 'samba': { title: 'Samba', - icon: "", + icon: '', info: 'Performance metrics of the Samba file share operations of this system. Samba is a implementation of Windows services, including Windows SMB protocol file shares.' }, 'nfsd': { title: 'NFS Server', - icon: '', + icon: '', info: 'Performance metrics of the Network File Server. NFS is a distributed file system protocol, allowing a user on a client computer to access files over a network, much like local storage is accessed. NFS, like many other protocols, builds on the Open Network Computing Remote Procedure Call (ONC RPC) system. The NFS is an open standard defined in Request for Comments (RFC).' }, 'nfs': { title: 'NFS Client', - icon: '', + icon: '', info: 'Performance metrics of the NFS operations of this system, acting as an NFS client.' }, 'zfs': { title: 'ZFS filesystem', - icon: '', + icon: '', info: 'Performance metrics of the ZFS filesystem. The following charts visualize all metrics reported by arcstat.py and arc_summary.py.' }, 'apps': { title: 'Applications', - icon: '', + icon: '', info: 'Per application statistics are collected using netdata\'s apps.plugin. This plugin walks through all processes and aggregates statistics for applications of interest, defined in /etc/netdata/apps_groups.conf (the default is here). The plugin internally builds a process tree (much like ps fax does), and groups processes together (evaluating both child and parent processes) so that the result is always a chart with a predefined set of dimensions (of course, only application groups found running are reported). The reported values are compatible with top, although the netdata plugin counts also the resources of exited children (unlike top which shows only the resources of the currently running processes). So for processes like shell scripts, the reported values include the resources used by the commands these scripts run within each timeframe.', height: 1.5 }, 'users': { title: 'Users', - icon: '', + icon: '', info: 'Per user statistics are collected using netdata\'s apps.plugin. This plugin walks through all processes and aggregates statistics per user. The reported values are compatible with top, although the netdata plugin counts also the resources of exited children (unlike top which shows only the resources of the currently running processes). So for processes like shell scripts, the reported values include the resources used by the commands these scripts run within each timeframe.', height: 1.5 }, 'groups': { title: 'User Groups', - icon: '', + icon: '', info: 'Per user group statistics are collected using netdata\'s apps.plugin. This plugin walks through all processes and aggregates statistics per user group. The reported values are compatible with top, although the netdata plugin counts also the resources of exited children (unlike top which shows only the resources of the currently running processes). So for processes like shell scripts, the reported values include the resources used by the commands these scripts run within each timeframe.', height: 1.5 }, 'netdata': { title: 'Netdata Monitoring', - icon: '', + icon: '', info: 'Performance metrics for the operation of netdata itself and its plugins.' }, @@ -155,163 +155,175 @@ netdataDashboard.menu = { 'cgroup': { title: '', - icon: '', + icon: '', info: 'Container resource utilization metrics. Netdata reads this information from cgroups (abbreviated from control groups), a Linux kernel feature that limits and accounts resource usage (CPU, memory, disk I/O, network, etc.) of a collection of processes. cgroups together with namespaces (that offer isolation between processes) provide what we usually call: containers.' }, 'cgqemu': { title: '', - icon: '', + icon: '', info: 'QEMU virtual machine resource utilization metrics. QEMU (short for Quick Emulator) is a free and open-source hosted hypervisor that performs hardware virtualization.' }, 'fping': { title: 'fping', - icon: '', + icon: '', info: 'Network latency statistics, via fping. fping is a program to send ICMP echo probes to network hosts, similar to ping, but much better performing when pinging multiple hosts. fping versions after 3.15 can be directly used as netdata plugins.' }, 'memcached': { title: 'memcached', - icon: '', + icon: '', info: 'Performance metrics for memcached. Memcached is a general-purpose distributed memory caching system. It is often used to speed up dynamic database-driven websites by caching data and objects in RAM to reduce the number of times an external data source (such as a database or API) must be read.' }, 'mysql': { title: 'MySQL', - icon: '', + icon: '', info: 'Performance metrics for mysql, the open-source relational database management system (RDBMS).' }, 'postgres': { title: 'Postgres', - icon: '', + icon: '', info: 'Performance metrics for PostgresSQL, the object-relational database (ORDBMS).' }, 'redis': { title: 'Redis', - icon: '', + icon: '', info: 'Performance metrics for redis. Redis (REmote DIctionary Server) is a software project that implements data structure servers. It is open-source, networked, in-memory, and stores keys with optional durability.' }, 'retroshare': { title: 'RetroShare', - icon: '', + icon: '', info: 'Performance metrics for RetroShare. RetroShare is open source software for encrypted filesharing, serverless email, instant messaging, online chat, and BBS, based on a friend-to-friend network built on GNU Privacy Guard (GPG).' }, 'ipfs': { title: 'IPFS', - icon: '', + icon: '', info: 'Performance metrics for the InterPlanetary File System (IPFS), a content-addressable, peer-to-peer hypermedia distribution protocol.' }, 'phpfpm': { title: 'PHP-FPM', - icon: '', + icon: '', info: 'Performance metrics for PHP-FPM, an alternative FastCGI implementation for PHP.' }, 'postfix': { title: 'postfix', - icon: '', + icon: '', info: undefined }, 'dovecot': { title: 'Dovecot', - icon: '', + icon: '', info: undefined }, 'hddtemp': { title: 'HDD Temp', - icon: '', + icon: '', info: undefined }, 'nginx': { title: 'nginx', - icon: '', + icon: '', info: undefined }, 'apache': { title: 'Apache', - icon: '', + icon: '', info: undefined }, 'lighttpd': { title: 'Lighttpd', - icon: '', + icon: '', info: undefined }, 'web_log': { title: undefined, - icon: '', + icon: '', info: 'Information extracted from a server log file. web_log plugin incrementally parses the server log file to provide, in real-time, a break down of key server performance metrics. For web servers, an extended log file format may optionally be used (for nginx and apache) offering timing information and bandwidth for both requests and responses. web_log plugin may also be configured to provide a break down of requests per URL pattern (check /etc/netdata/python.d/web_log.conf).' }, 'named': { title: 'named', - icon: '', + icon: '', info: undefined }, 'squid': { title: 'squid', - icon: '', + icon: '', info: undefined }, 'nut': { title: 'UPS', - icon: '', + icon: '', info: undefined }, 'apcupsd': { title: 'UPS', - icon: '', + icon: '', info: undefined }, 'smawebbox': { title: 'Solar Power', - icon: '', + icon: '', info: undefined }, 'fronius': { title: 'Fronius', - icon: '', + icon: '', info: undefined }, 'stiebeleltron': { title: 'Stiebel Eltron', - icon: '', + icon: '', info: undefined }, 'snmp': { title: 'SNMP', - icon: '', + icon: '', info: undefined }, 'go_expvar': { title: 'Go - expvars', - icon: '', + icon: '', info: 'Statistics about running Go applications exposed by the expvar package.' }, 'chrony': { - icon: '', + icon: '', info: 'chronyd parameters about the system’s clock performance.' + }, + + 'couchdb': { + icon: '', + info: 'Performance metrics for CouchDB, the open-source, JSON document-based database with an HTTP API and multi-master replication.' + }, + + + 'beanstalk': { + title: 'Beanstalkd', + icon: '', + info: 'Provides statistics on the beanstalkd server and any tubes available on that server using data pulled from beanstalkc' } }; @@ -434,6 +446,30 @@ netdataDashboard.submenu = { 'go_expvar.memstats': { title: 'Memory statistics', info: 'Go runtime memory statistics. See runtime.MemStats documentation for more info about each chart and the values.' + }, + + 'couchdb.dbactivity': { + title: 'DB activity', + info: 'Overall database reads and writes for the entire server. This includes any external HTTP traffic, as well as internal replication traffic performed in a cluster to ensure node consistency.' + }, + + 'couchdb.httptraffic': { + title: 'HTTP traffic breakdown', + info: 'All HTTP traffic, broken down by type of request (GET, PUT, POST, etc.) and response status code (200, 201, 4xx, etc.)

Any 5xx errors here indicate a likely CouchDB bug; check the logfile for further information.' + }, + + 'couchdb.ops': { + title: 'Server operations' + }, + + 'couchdb.perdbstats': { + title: 'Per-DB statistics', + info: 'Statistics per database. This includes 3 size graphs per database: active (the size of live data in the database), external (the uncompressed size of the database contents), and file (the size of the file on disk, exclusive of any views and indexes). It also includes the number of documents and number of deleted documents per database.' + }, + + 'couchdb.erlang': { + title: 'Erlang statistics', + info: 'Detailed information about the status of the Erlang VM that hosts CouchDB. These are intended for advanced users only. High values of the peak message queue (>10e6) generally indicate an overload condition.' } }; @@ -468,7 +504,18 @@ netdataDashboard.context = { }, 'system.io': { - info: 'Total Disk I/O, for all disks. You can get detailed information about each disk at the Disks section and per application Disk usage at the Applications Monitoring section.' + info: function(os) { + var s = 'Total Disk I/O, for all physical disks. You can get detailed information about each disk at the Disks section and per application Disk usage at the Applications Monitoring section.'; + + if(os === 'linux') + return s + ' Physical are all the disks that are listed in /sys/block, but do not exist in /sys/devices/virtual/block.'; + else + return s; + } + }, + + 'system.pgpgio': { + info: 'Memory paged from/to disk. This is usually the total disk I/O of the system.' }, 'system.swapio': { @@ -518,6 +565,17 @@ netdataDashboard.context = { info: 'Idle jitter is calculated by netdata. A thread is spawned that requests to sleep for a few microseconds. When the system wakes it up, it measures how many microseconds have passed. The difference between the requested and the actual duration of the sleep, is the idle jitter. This number is useful in real-time environments, where CPU jitter can affect the quality of the service (like VoIP media gateways).' }, + 'system.net': { + info: function(os) { + var s = 'Total bandwidth of all physical network interfaces. This does not include lo, VPNs, network bridges, IFB devices, bond interfaces, etc. Only the bandwidth of physical network interfaces is aggregated.'; + + if(os === 'linux') + return s + ' Physical are all the network interfaces that are listed in /proc/net/dev, but do not exist in /sys/devices/virtual/net.'; + else + return s; + } + }, + 'system.ipv4': { info: 'Total IPv4 Traffic.' }, @@ -1151,6 +1209,63 @@ netdataDashboard.context = { } ] }, + + // ------------------------------------------------------------------------ + // beanstalkd + // system charts + 'beanstalk.cpu_usage': { + info: 'Amount of CPU Time for user and system used by beanstalkd.' + }, + + // This is also a per-tube stat + 'beanstalk.jobs_rate': { + info: 'The rate of jobs processed by the beanstalkd served.' + }, + + 'beanstalk.connections_rate': { + info: 'Tthe rate of connections opened to beanstalkd.' + }, + + 'beanstalk.commands_rate': { + info: 'The rate of commands received by beanstalkd.' + }, + + 'beanstalk.current_tubes': { + info: 'Total number of current tubes on the server including the default tube (which always exists).' + }, + + 'beanstalk.current_jobs': { + info: 'Current number of jobs in all tubes grouped by status: urgent, ready, reserved, delayed and buried.' + }, + + 'beanstalk.current_connections': { + info: 'Current number of connections group by connection type: written, producers, workers, waiting.' + }, + + 'beanstalk.binlog': { + info: 'The rate of records written to binlog and migrated as part of compaction.' + }, + + 'beanstalk.uptime': { + info: 'Total time beanstalkd server has been up for.' + }, + + // tube charts + 'beanstalk.jobs': { + info: 'Number of jobs currently in the tube grouped by status: urgent, ready, reserved, delayed and buried.' + }, + + 'beanstalk.connections': { + info: 'The current number of connections to this tube grouped by connection type; using, waiting and watching.' + }, + + 'beanstalk.commands': { + info: 'The rate of delete and pause commands executed by beanstalkd.' + }, + + 'beanstalk.pause': { + info: 'Shows info on how long the tube has been paused for, and how long is left remaining on the pause.' + }, // ------------------------------------------------------------------------ // web_log @@ -1534,5 +1649,17 @@ netdataDashboard.context = { info: 'The estimated error bound on the frequency.', height: 0.5, colors: NETDATA.colors[5] + }, + + 'couchdb.active_tasks': { + info: 'Active tasks running on this CouchDB cluster. Four types of tasks currently exist: indexer (view building), replication, database compaction and view compaction.' + }, + + 'couchdb.replicator_jobs': { + info: 'Detailed breakdown of any replication jobs in progress on this node. For more information, see the replicator documentation.' + }, + + 'couchdb.open_files': { + info: 'Count of all files held open by CouchDB. If this value seems pegged at 1024 or 4096, your server process is probably hitting the open file handle limit and needs to be increased.' } }; diff --git a/web/index.html b/web/index.html index cd8239d26..e3c039091 100644 --- a/web/index.html +++ b/web/index.html @@ -46,6 +46,12 @@