summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/Makefile.am16
-rw-r--r--src/Makefile.in315
-rw-r--r--src/appconfig.c55
-rw-r--r--src/appconfig.h7
-rw-r--r--src/apps_plugin.c213
-rw-r--r--src/backend_prometheus.c397
-rw-r--r--src/backend_prometheus.h11
-rw-r--r--src/backends.c311
-rw-r--r--src/backends.h26
-rw-r--r--src/clocks.c4
-rw-r--r--src/clocks.h2
-rw-r--r--src/common.c266
-rw-r--r--src/common.h12
-rw-r--r--src/daemon.c56
-rw-r--r--src/eval.c2
-rw-r--r--src/freebsd_devstat.c662
-rw-r--r--src/freebsd_getifaddrs.c494
-rw-r--r--src/freebsd_getmntinfo.c293
-rw-r--r--src/freebsd_ipfw.c360
-rw-r--r--src/freebsd_kstat_zfs.c212
-rw-r--r--src/freebsd_sysctl.c1283
-rw-r--r--src/freeipmi_plugin.c13
-rw-r--r--src/health.c15
-rw-r--r--src/health_config.c55
-rw-r--r--src/inlined.h97
-rw-r--r--src/log.c176
-rw-r--r--src/log.h12
-rw-r--r--src/main.c100
-rw-r--r--src/plugin_freebsd.c6
-rw-r--r--src/plugin_freebsd.h8
-rw-r--r--src/plugin_idlejitter.c72
-rw-r--r--src/plugin_proc.c5
-rw-r--r--src/plugin_proc.h4
-rw-r--r--src/plugin_proc_diskspace.c41
-rw-r--r--src/plugin_tc.c6
-rw-r--r--src/plugins_d.c99
-rw-r--r--src/plugins_d.h10
-rw-r--r--src/proc_diskstats.c173
-rw-r--r--src/proc_loadavg.c5
-rw-r--r--src/proc_net_dev.c14
-rw-r--r--src/proc_net_netstat.c8
-rw-r--r--src/proc_net_snmp.c37
-rw-r--r--src/proc_net_snmp6.c4
-rw-r--r--src/proc_net_softnet_stat.c4
-rw-r--r--src/proc_spl_kstat_zfs.c153
-rw-r--r--src/proc_vmstat.c20
-rw-r--r--src/registry.c2
-rw-r--r--src/registry.h2
-rw-r--r--src/registry_init.c2
-rw-r--r--src/registry_internals.c4
-rw-r--r--src/rrd.c11
-rw-r--r--src/rrd.h98
-rw-r--r--src/rrd2json.c83
-rw-r--r--src/rrd2json.h24
-rw-r--r--src/rrd2json_api_old.c4
-rw-r--r--src/rrdcalc.c2
-rw-r--r--src/rrdcalctemplate.c2
-rw-r--r--src/rrddim.c175
-rw-r--r--src/rrdhost.c154
-rw-r--r--src/rrdpush.c249
-rw-r--r--src/rrdpush.h2
-rw-r--r--src/rrdset.c921
-rw-r--r--src/socket.c1006
-rw-r--r--src/socket.h55
-rw-r--r--src/statistical.c459
-rw-r--r--src/statistical.h19
-rw-r--r--src/statsd.c2041
-rw-r--r--src/statsd.h9
-rw-r--r--src/storage_number.h1
-rw-r--r--src/sys_fs_cgroup.c48
-rw-r--r--src/unit_test.c56
-rw-r--r--src/unit_test.h1
-rw-r--r--src/web_api_v1.c42
-rw-r--r--src/web_client.c54
-rw-r--r--src/web_client.h1
-rw-r--r--src/web_server.c366
-rw-r--r--src/web_server.h26
-rw-r--r--src/zfs_common.c677
-rw-r--r--src/zfs_common.h109
79 files changed, 9754 insertions, 3085 deletions
diff --git a/src/Makefile.am b/src/Makefile.am
index 1c1dd3385..601d3204f 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -44,6 +44,8 @@ netdata_SOURCES = \
appconfig.h \
avl.c \
avl.h \
+ backend_prometheus.c \
+ backend_prometheus.h \
backends.c \
backends.h \
clocks.c \
@@ -119,6 +121,10 @@ netdata_SOURCES = \
simple_pattern.h \
socket.c \
socket.h \
+ statistical.c \
+ statistical.h \
+ statsd.c \
+ statsd.h \
storage_number.c \
storage_number.h \
sys_devices_system_edac_mc.c \
@@ -146,6 +152,13 @@ netdata_SOURCES += \
plugin_freebsd.c \
plugin_freebsd.h \
freebsd_sysctl.c \
+ freebsd_getmntinfo.c \
+ freebsd_getifaddrs.c \
+ freebsd_devstat.c \
+ zfs_common.c \
+ zfs_common.h \
+ freebsd_kstat_zfs.c \
+ freebsd_ipfw.c \
$(NULL)
else
if MACOS
@@ -178,6 +191,9 @@ netdata_SOURCES += \
proc_net_softnet_stat.c \
proc_net_stat_conntrack.c \
proc_net_stat_synproxy.c \
+ zfs_common.c \
+ zfs_common.h \
+ proc_spl_kstat_zfs.c \
proc_stat.c \
proc_sys_kernel_random_entropy_avail.c \
proc_vmstat.c \
diff --git a/src/Makefile.in b/src/Makefile.in
index d7229d18a..3ce869b0d 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -1,9 +1,8 @@
-# Makefile.in generated by automake 1.11.3 from Makefile.am.
+# Makefile.in generated by automake 1.14.1 from Makefile.am.
# @configure_input@
-# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
-# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
-# Foundation, Inc.
+# Copyright (C) 1994-2013 Free Software Foundation, Inc.
+
# This Makefile.in is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
@@ -17,6 +16,51 @@
VPATH = @srcdir@
+am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)'
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
pkgdatadir = $(datadir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
pkglibdir = $(libdir)/@PACKAGE@
@@ -43,6 +87,13 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2)
@FREEBSD_TRUE@ plugin_freebsd.c \
@FREEBSD_TRUE@ plugin_freebsd.h \
@FREEBSD_TRUE@ freebsd_sysctl.c \
+@FREEBSD_TRUE@ freebsd_getmntinfo.c \
+@FREEBSD_TRUE@ freebsd_getifaddrs.c \
+@FREEBSD_TRUE@ freebsd_devstat.c \
+@FREEBSD_TRUE@ zfs_common.c \
+@FREEBSD_TRUE@ zfs_common.h \
+@FREEBSD_TRUE@ freebsd_kstat_zfs.c \
+@FREEBSD_TRUE@ freebsd_ipfw.c \
@FREEBSD_TRUE@ $(NULL)
@FREEBSD_FALSE@@MACOS_TRUE@am__append_4 = \
@@ -74,6 +125,9 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2)
@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.c \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.c \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.c \
+@FREEBSD_FALSE@@MACOS_FALSE@ zfs_common.c \
+@FREEBSD_FALSE@@MACOS_FALSE@ zfs_common.h \
+@FREEBSD_FALSE@@MACOS_FALSE@ proc_spl_kstat_zfs.c \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_stat.c \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_sys_kernel_random_entropy_avail.c \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_vmstat.c \
@@ -86,9 +140,9 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2)
@FREEBSD_TRUE@ $(NULL)
subdir = src
-DIST_COMMON = $(dist_cache_DATA) $(dist_log_DATA) \
- $(dist_registry_DATA) $(dist_varlib_DATA) \
- $(srcdir)/Makefile.am $(srcdir)/Makefile.in
+DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
+ $(top_srcdir)/depcomp $(dist_cache_DATA) $(dist_log_DATA) \
+ $(dist_registry_DATA) $(dist_varlib_DATA)
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/ax_c___atomic.m4 \
$(top_srcdir)/m4/ax_c__generic.m4 $(top_srcdir)/m4/ax_c_lto.m4 \
@@ -127,8 +181,9 @@ freeipmi_plugin_OBJECTS = $(am_freeipmi_plugin_OBJECTS)
freeipmi_plugin_DEPENDENCIES = $(am__DEPENDENCIES_1)
am__netdata_SOURCES_DIST = adaptive_resortable_list.c \
adaptive_resortable_list.h appconfig.c appconfig.h avl.c avl.h \
- backends.c backends.h clocks.c clocks.h common.c common.h \
- daemon.c daemon.h dictionary.c dictionary.h eval.c eval.h \
+ backend_prometheus.c backend_prometheus.h backends.c \
+ backends.h clocks.c clocks.h common.c common.h daemon.c \
+ daemon.h dictionary.c dictionary.h eval.c eval.h \
global_statistics.c global_statistics.h health.c health.h \
health_config.c health_json.c health_log.c inlined.h locks.h \
log.c log.h main.c main.h plugin_checks.c plugin_checks.h \
@@ -144,25 +199,34 @@ am__netdata_SOURCES_DIST = adaptive_resortable_list.c \
rrdcalctemplate.c rrddim.c rrddimvar.c rrdfamily.c rrdhost.c \
rrdpush.c rrdpush.h rrdset.c rrdsetvar.c rrdvar.c \
simple_pattern.c simple_pattern.h socket.c socket.h \
- storage_number.c storage_number.h sys_devices_system_edac_mc.c \
+ statistical.c statistical.h statsd.c statsd.h storage_number.c \
+ storage_number.h sys_devices_system_edac_mc.c \
sys_devices_system_node.c sys_fs_cgroup.c unit_test.c \
unit_test.h url.c url.h web_api_old.c web_api_old.h \
web_api_v1.c web_api_v1.h web_buffer.c web_buffer.h \
web_buffer_svg.c web_buffer_svg.h web_client.c web_client.h \
web_server.c web_server.h plugin_freebsd.c plugin_freebsd.h \
- freebsd_sysctl.c plugin_macos.c plugin_macos.h macos_sysctl.c \
- macos_mach_smi.c macos_fw.c ipc.c ipc.h plugin_proc.c \
- plugin_proc.h plugin_proc_diskspace.c plugin_proc_diskspace.h \
+ freebsd_sysctl.c freebsd_getmntinfo.c freebsd_getifaddrs.c \
+ freebsd_devstat.c zfs_common.c zfs_common.h \
+ freebsd_kstat_zfs.c freebsd_ipfw.c plugin_macos.c \
+ plugin_macos.h macos_sysctl.c macos_mach_smi.c macos_fw.c \
+ ipc.c ipc.h plugin_proc.c plugin_proc.h \
+ plugin_proc_diskspace.c plugin_proc_diskspace.h \
proc_diskstats.c proc_interrupts.c proc_softirqs.c \
proc_loadavg.c proc_meminfo.c proc_net_dev.c \
proc_net_ip_vs_stats.c proc_net_netstat.c proc_net_rpc_nfs.c \
proc_net_rpc_nfsd.c proc_net_snmp.c proc_net_snmp6.c \
proc_net_softnet_stat.c proc_net_stat_conntrack.c \
- proc_net_stat_synproxy.c proc_stat.c \
+ proc_net_stat_synproxy.c proc_spl_kstat_zfs.c proc_stat.c \
proc_sys_kernel_random_entropy_avail.c proc_vmstat.c \
proc_uptime.c sys_kernel_mm_ksm.c
@FREEBSD_TRUE@am__objects_2 = plugin_freebsd.$(OBJEXT) \
-@FREEBSD_TRUE@ freebsd_sysctl.$(OBJEXT)
+@FREEBSD_TRUE@ freebsd_sysctl.$(OBJEXT) \
+@FREEBSD_TRUE@ freebsd_getmntinfo.$(OBJEXT) \
+@FREEBSD_TRUE@ freebsd_getifaddrs.$(OBJEXT) \
+@FREEBSD_TRUE@ freebsd_devstat.$(OBJEXT) zfs_common.$(OBJEXT) \
+@FREEBSD_TRUE@ freebsd_kstat_zfs.$(OBJEXT) \
+@FREEBSD_TRUE@ freebsd_ipfw.$(OBJEXT)
@FREEBSD_FALSE@@MACOS_TRUE@am__objects_3 = plugin_macos.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_TRUE@ macos_sysctl.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_TRUE@ macos_mach_smi.$(OBJEXT) \
@@ -185,15 +249,17 @@ am__netdata_SOURCES_DIST = adaptive_resortable_list.c \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.$(OBJEXT) \
+@FREEBSD_FALSE@@MACOS_FALSE@ zfs_common.$(OBJEXT) \
+@FREEBSD_FALSE@@MACOS_FALSE@ proc_spl_kstat_zfs.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_stat.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_sys_kernel_random_entropy_avail.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_vmstat.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_FALSE@ proc_uptime.$(OBJEXT) \
@FREEBSD_FALSE@@MACOS_FALSE@ sys_kernel_mm_ksm.$(OBJEXT)
am_netdata_OBJECTS = adaptive_resortable_list.$(OBJEXT) \
- appconfig.$(OBJEXT) avl.$(OBJEXT) backends.$(OBJEXT) \
- clocks.$(OBJEXT) common.$(OBJEXT) daemon.$(OBJEXT) \
- dictionary.$(OBJEXT) eval.$(OBJEXT) \
+ appconfig.$(OBJEXT) avl.$(OBJEXT) backend_prometheus.$(OBJEXT) \
+ backends.$(OBJEXT) clocks.$(OBJEXT) common.$(OBJEXT) \
+ daemon.$(OBJEXT) dictionary.$(OBJEXT) eval.$(OBJEXT) \
global_statistics.$(OBJEXT) health.$(OBJEXT) \
health_config.$(OBJEXT) health_json.$(OBJEXT) \
health_log.$(OBJEXT) log.$(OBJEXT) main.$(OBJEXT) \
@@ -210,6 +276,7 @@ am_netdata_OBJECTS = adaptive_resortable_list.$(OBJEXT) \
rrddimvar.$(OBJEXT) rrdfamily.$(OBJEXT) rrdhost.$(OBJEXT) \
rrdpush.$(OBJEXT) rrdset.$(OBJEXT) rrdsetvar.$(OBJEXT) \
rrdvar.$(OBJEXT) simple_pattern.$(OBJEXT) socket.$(OBJEXT) \
+ statistical.$(OBJEXT) statsd.$(OBJEXT) \
storage_number.$(OBJEXT) sys_devices_system_edac_mc.$(OBJEXT) \
sys_devices_system_node.$(OBJEXT) sys_fs_cgroup.$(OBJEXT) \
unit_test.$(OBJEXT) url.$(OBJEXT) web_api_old.$(OBJEXT) \
@@ -220,18 +287,43 @@ am_netdata_OBJECTS = adaptive_resortable_list.$(OBJEXT) \
netdata_OBJECTS = $(am_netdata_OBJECTS)
netdata_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
depcomp = $(SHELL) $(top_srcdir)/depcomp
am__depfiles_maybe = depfiles
am__mv = mv -f
COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo " CC " $@;
+am__v_CC_1 =
CCLD = $(CC)
LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo " CCLD " $@;
+am__v_CCLD_1 =
SOURCES = $(apps_plugin_SOURCES) $(freeipmi_plugin_SOURCES) \
$(netdata_SOURCES)
DIST_SOURCES = $(am__apps_plugin_SOURCES_DIST) \
$(freeipmi_plugin_SOURCES) $(am__netdata_SOURCES_DIST)
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
am__vpath_adj = case $$p in \
$(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
@@ -261,11 +353,29 @@ am__uninstall_files_from_dir = { \
}
DATA = $(dist_cache_DATA) $(dist_log_DATA) $(dist_registry_DATA) \
$(dist_varlib_DATA)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates. Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+ BEGIN { nonempty = 0; } \
+ { items[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique. This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+ list='$(am__tagged_files)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | $(am__uniquify_input)`
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
@@ -437,8 +547,9 @@ dist_registry_DATA = .keep
dist_log_DATA = .keep
netdata_SOURCES = adaptive_resortable_list.c \
adaptive_resortable_list.h appconfig.c appconfig.h avl.c avl.h \
- backends.c backends.h clocks.c clocks.h common.c common.h \
- daemon.c daemon.h dictionary.c dictionary.h eval.c eval.h \
+ backend_prometheus.c backend_prometheus.h backends.c \
+ backends.h clocks.c clocks.h common.c common.h daemon.c \
+ daemon.h dictionary.c dictionary.h eval.c eval.h \
global_statistics.c global_statistics.h health.c health.h \
health_config.c health_json.c health_log.c inlined.h locks.h \
log.c log.h main.c main.h plugin_checks.c plugin_checks.h \
@@ -454,7 +565,8 @@ netdata_SOURCES = adaptive_resortable_list.c \
rrdcalctemplate.c rrddim.c rrddimvar.c rrdfamily.c rrdhost.c \
rrdpush.c rrdpush.h rrdset.c rrdsetvar.c rrdvar.c \
simple_pattern.c simple_pattern.h socket.c socket.h \
- storage_number.c storage_number.h sys_devices_system_edac_mc.c \
+ statistical.c statistical.h statsd.c statsd.h storage_number.c \
+ storage_number.h sys_devices_system_edac_mc.c \
sys_devices_system_node.c sys_fs_cgroup.c unit_test.c \
unit_test.h url.c url.h web_api_old.c web_api_old.h \
web_api_v1.c web_api_v1.h web_buffer.c web_buffer.h \
@@ -525,14 +637,18 @@ $(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
$(am__aclocal_m4_deps):
install-pluginsPROGRAMS: $(plugins_PROGRAMS)
@$(NORMAL_INSTALL)
- test -z "$(pluginsdir)" || $(MKDIR_P) "$(DESTDIR)$(pluginsdir)"
@list='$(plugins_PROGRAMS)'; test -n "$(pluginsdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(pluginsdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(pluginsdir)" || exit 1; \
+ fi; \
for p in $$list; do echo "$$p $$p"; done | \
sed 's/$(EXEEXT)$$//' | \
- while read p p1; do if test -f $$p; \
- then echo "$$p"; echo "$$p"; else :; fi; \
+ while read p p1; do if test -f $$p \
+ ; then echo "$$p"; echo "$$p"; else :; fi; \
done | \
- sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \
+ sed -e 'p;s,.*/,,;n;h' \
+ -e 's|.*|.|' \
-e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \
sed 'N;N;N;s,\n, ,g' | \
$(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \
@@ -553,7 +669,8 @@ uninstall-pluginsPROGRAMS:
@list='$(plugins_PROGRAMS)'; test -n "$(pluginsdir)" || list=; \
files=`for p in $$list; do echo "$$p"; done | \
sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \
- -e 's/$$/$(EXEEXT)/' `; \
+ -e 's/$$/$(EXEEXT)/' \
+ `; \
test -n "$$list" || exit 0; \
echo " ( cd '$(DESTDIR)$(pluginsdir)' && rm -f" $$files ")"; \
cd "$(DESTDIR)$(pluginsdir)" && rm -f $$files
@@ -562,14 +679,18 @@ clean-pluginsPROGRAMS:
-test -z "$(plugins_PROGRAMS)" || rm -f $(plugins_PROGRAMS)
install-sbinPROGRAMS: $(sbin_PROGRAMS)
@$(NORMAL_INSTALL)
- test -z "$(sbindir)" || $(MKDIR_P) "$(DESTDIR)$(sbindir)"
@list='$(sbin_PROGRAMS)'; test -n "$(sbindir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(sbindir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(sbindir)" || exit 1; \
+ fi; \
for p in $$list; do echo "$$p $$p"; done | \
sed 's/$(EXEEXT)$$//' | \
- while read p p1; do if test -f $$p; \
- then echo "$$p"; echo "$$p"; else :; fi; \
+ while read p p1; do if test -f $$p \
+ ; then echo "$$p"; echo "$$p"; else :; fi; \
done | \
- sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \
+ sed -e 'p;s,.*/,,;n;h' \
+ -e 's|.*|.|' \
-e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \
sed 'N;N;N;s,\n, ,g' | \
$(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \
@@ -590,22 +711,26 @@ uninstall-sbinPROGRAMS:
@list='$(sbin_PROGRAMS)'; test -n "$(sbindir)" || list=; \
files=`for p in $$list; do echo "$$p"; done | \
sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \
- -e 's/$$/$(EXEEXT)/' `; \
+ -e 's/$$/$(EXEEXT)/' \
+ `; \
test -n "$$list" || exit 0; \
echo " ( cd '$(DESTDIR)$(sbindir)' && rm -f" $$files ")"; \
cd "$(DESTDIR)$(sbindir)" && rm -f $$files
clean-sbinPROGRAMS:
-test -z "$(sbin_PROGRAMS)" || rm -f $(sbin_PROGRAMS)
+
apps.plugin$(EXEEXT): $(apps_plugin_OBJECTS) $(apps_plugin_DEPENDENCIES) $(EXTRA_apps_plugin_DEPENDENCIES)
@rm -f apps.plugin$(EXEEXT)
- $(LINK) $(apps_plugin_OBJECTS) $(apps_plugin_LDADD) $(LIBS)
+ $(AM_V_CCLD)$(LINK) $(apps_plugin_OBJECTS) $(apps_plugin_LDADD) $(LIBS)
+
freeipmi.plugin$(EXEEXT): $(freeipmi_plugin_OBJECTS) $(freeipmi_plugin_DEPENDENCIES) $(EXTRA_freeipmi_plugin_DEPENDENCIES)
@rm -f freeipmi.plugin$(EXEEXT)
- $(LINK) $(freeipmi_plugin_OBJECTS) $(freeipmi_plugin_LDADD) $(LIBS)
+ $(AM_V_CCLD)$(LINK) $(freeipmi_plugin_OBJECTS) $(freeipmi_plugin_LDADD) $(LIBS)
+
netdata$(EXEEXT): $(netdata_OBJECTS) $(netdata_DEPENDENCIES) $(EXTRA_netdata_DEPENDENCIES)
@rm -f netdata$(EXEEXT)
- $(LINK) $(netdata_OBJECTS) $(netdata_LDADD) $(LIBS)
+ $(AM_V_CCLD)$(LINK) $(netdata_OBJECTS) $(netdata_LDADD) $(LIBS)
mostlyclean-compile:
-rm -f *.$(OBJEXT)
@@ -617,12 +742,18 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/appconfig.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/apps_plugin.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/avl.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/backend_prometheus.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/backends.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/clocks.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/daemon.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dictionary.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/eval.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_devstat.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_getifaddrs.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_getmntinfo.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_ipfw.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_kstat_zfs.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_sysctl.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freeipmi_plugin.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/global_statistics.Po@am__quote@
@@ -662,6 +793,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_stat_synproxy.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_self_mountinfo.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_softirqs.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_spl_kstat_zfs.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_stat.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_sys_kernel_random_entropy_avail.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_uptime.Po@am__quote@
@@ -690,6 +822,8 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rrdvar.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/simple_pattern.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/socket.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statistical.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statsd.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/storage_number.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_devices_system_edac_mc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_devices_system_node.Po@am__quote@
@@ -703,24 +837,28 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/web_buffer_svg.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/web_client.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/web_server.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/zfs_common.Po@am__quote@
.c.o:
-@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
-@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
-@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCC_FALSE@ $(COMPILE) -c $<
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
.c.obj:
-@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
-@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
-@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'`
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
install-dist_cacheDATA: $(dist_cache_DATA)
@$(NORMAL_INSTALL)
- test -z "$(cachedir)" || $(MKDIR_P) "$(DESTDIR)$(cachedir)"
@list='$(dist_cache_DATA)'; test -n "$(cachedir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(cachedir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(cachedir)" || exit 1; \
+ fi; \
for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
echo "$$d$$p"; \
@@ -737,8 +875,11 @@ uninstall-dist_cacheDATA:
dir='$(DESTDIR)$(cachedir)'; $(am__uninstall_files_from_dir)
install-dist_logDATA: $(dist_log_DATA)
@$(NORMAL_INSTALL)
- test -z "$(logdir)" || $(MKDIR_P) "$(DESTDIR)$(logdir)"
@list='$(dist_log_DATA)'; test -n "$(logdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(logdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(logdir)" || exit 1; \
+ fi; \
for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
echo "$$d$$p"; \
@@ -755,8 +896,11 @@ uninstall-dist_logDATA:
dir='$(DESTDIR)$(logdir)'; $(am__uninstall_files_from_dir)
install-dist_registryDATA: $(dist_registry_DATA)
@$(NORMAL_INSTALL)
- test -z "$(registrydir)" || $(MKDIR_P) "$(DESTDIR)$(registrydir)"
@list='$(dist_registry_DATA)'; test -n "$(registrydir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(registrydir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(registrydir)" || exit 1; \
+ fi; \
for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
echo "$$d$$p"; \
@@ -773,8 +917,11 @@ uninstall-dist_registryDATA:
dir='$(DESTDIR)$(registrydir)'; $(am__uninstall_files_from_dir)
install-dist_varlibDATA: $(dist_varlib_DATA)
@$(NORMAL_INSTALL)
- test -z "$(varlibdir)" || $(MKDIR_P) "$(DESTDIR)$(varlibdir)"
@list='$(dist_varlib_DATA)'; test -n "$(varlibdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(varlibdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(varlibdir)" || exit 1; \
+ fi; \
for p in $$list; do \
if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
echo "$$d$$p"; \
@@ -790,26 +937,15 @@ uninstall-dist_varlibDATA:
files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
dir='$(DESTDIR)$(varlibdir)'; $(am__uninstall_files_from_dir)
-ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
- list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
- unique=`for i in $$list; do \
- if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
- done | \
- $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
- END { if (nonempty) { for (i in files) print i; }; }'`; \
- mkid -fID $$unique
-tags: TAGS
-
-TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
- $(TAGS_FILES) $(LISP)
+ID: $(am__tagged_files)
+ $(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
set x; \
here=`pwd`; \
- list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
- unique=`for i in $$list; do \
- if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
- done | \
- $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
- END { if (nonempty) { for (i in files) print i; }; }'`; \
+ $(am__define_uniq_tagged_files); \
shift; \
if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
test -n "$$unique" || unique=$$empty_fix; \
@@ -821,15 +957,11 @@ TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
$$unique; \
fi; \
fi
-ctags: CTAGS
-CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
- $(TAGS_FILES) $(LISP)
- list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
- unique=`for i in $$list; do \
- if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
- done | \
- $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
- END { if (nonempty) { for (i in files) print i; }; }'`; \
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ $(am__define_uniq_tagged_files); \
test -z "$(CTAGS_ARGS)$$unique" \
|| $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
$$unique
@@ -838,6 +970,21 @@ GTAGS:
here=`$(am__cd) $(top_builddir) && pwd` \
&& $(am__cd) $(top_srcdir) \
&& gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+ list='$(am__tagged_files)'; \
+ case "$(srcdir)" in \
+ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+ *) sdir=$(subdir)/$(srcdir) ;; \
+ esac; \
+ for i in $$list; do \
+ if test -f "$$i"; then \
+ echo "$(subdir)/$$i"; \
+ else \
+ echo "$$sdir/$$i"; \
+ fi; \
+ done >> $(top_builddir)/cscope.files
distclean-tags:
-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
@@ -986,20 +1133,20 @@ uninstall-am: uninstall-dist_cacheDATA uninstall-dist_logDATA \
.MAKE: install-am install-strip
-.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
- clean-pluginsPROGRAMS clean-sbinPROGRAMS ctags distclean \
- distclean-compile distclean-generic distclean-tags distdir dvi \
- dvi-am html html-am info info-am install install-am \
- install-data install-data-am install-dist_cacheDATA \
- install-dist_logDATA install-dist_registryDATA \
- install-dist_varlibDATA install-dvi install-dvi-am \
- install-exec install-exec-am install-html install-html-am \
- install-info install-info-am install-man install-pdf \
- install-pdf-am install-pluginsPROGRAMS install-ps \
+.PHONY: CTAGS GTAGS TAGS all all-am check check-am clean clean-generic \
+ clean-pluginsPROGRAMS clean-sbinPROGRAMS cscopelist-am ctags \
+ ctags-am distclean distclean-compile distclean-generic \
+ distclean-tags distdir dvi dvi-am html html-am info info-am \
+ install install-am install-data install-data-am \
+ install-dist_cacheDATA install-dist_logDATA \
+ install-dist_registryDATA install-dist_varlibDATA install-dvi \
+ install-dvi-am install-exec install-exec-am install-html \
+ install-html-am install-info install-info-am install-man \
+ install-pdf install-pdf-am install-pluginsPROGRAMS install-ps \
install-ps-am install-sbinPROGRAMS install-strip installcheck \
installcheck-am installdirs maintainer-clean \
maintainer-clean-generic mostlyclean mostlyclean-compile \
- mostlyclean-generic pdf pdf-am ps ps-am tags uninstall \
+ mostlyclean-generic pdf pdf-am ps ps-am tags tags-am uninstall \
uninstall-am uninstall-dist_cacheDATA uninstall-dist_logDATA \
uninstall-dist_registryDATA uninstall-dist_varlibDATA \
uninstall-pluginsPROGRAMS uninstall-sbinPROGRAMS
diff --git a/src/appconfig.c b/src/appconfig.c
index 71ff4b75e..91c4c5c54 100644
--- a/src/appconfig.c
+++ b/src/appconfig.c
@@ -135,6 +135,7 @@ static inline struct section *appconfig_section_create(struct config *root, cons
struct section *co = callocz(1, sizeof(struct section));
co->name = strdupz(section);
co->hash = simple_hash(co->name);
+ netdata_mutex_init(&co->mutex);
avl_init_lock(&co->values_index, appconfig_option_compare);
@@ -213,7 +214,8 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam
if(!co_new) co_new = appconfig_section_create(root, section_new);
config_section_wrlock(co_old);
- config_section_wrlock(co_new);
+ if(co_old != co_new)
+ config_section_wrlock(co_new);
cv_old = appconfig_option_index_find(co_old, name_old, 0);
if(!cv_old) goto cleanup;
@@ -250,7 +252,8 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam
ret = 0;
cleanup:
- config_section_unlock(co_new);
+ if(co_old != co_new)
+ config_section_unlock(co_new);
config_section_unlock(co_old);
return ret;
}
@@ -294,6 +297,17 @@ long long appconfig_get_number(struct config *root, const char *section, const c
return strtoll(s, NULL, 0);
}
+long double appconfig_get_float(struct config *root, const char *section, const char *name, long double value)
+{
+ char buffer[100], *s;
+ sprintf(buffer, "%0.5Lf", value);
+
+ s = appconfig_get(root, section, name, buffer);
+ if(!s) return value;
+
+ return str2ld(s, NULL);
+}
+
int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value)
{
char *s;
@@ -337,7 +351,7 @@ const char *appconfig_set_default(struct config *root, const char *section, cons
{
struct config_option *cv;
- debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value);
+ debug(D_CONFIG, "request to set default config in section '%s', name '%s', value '%s'", section, name, value);
struct section *co = appconfig_section_find(root, section);
if(!co) return appconfig_set(root, section, name, value);
@@ -393,6 +407,16 @@ long long appconfig_set_number(struct config *root, const char *section, const c
return value;
}
+long double appconfig_set_float(struct config *root, const char *section, const char *name, long double value)
+{
+ char buffer[100];
+ sprintf(buffer, "%0.5Lf", value);
+
+ appconfig_set(root, section, name, buffer);
+
+ return value;
+}
+
int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value)
{
char *s;
@@ -417,11 +441,11 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME;
- debug(D_CONFIG, "Opening config file '%s'", filename);
+ debug(D_CONFIG, "CONFIG: opening config file '%s'", filename);
FILE *fp = fopen(filename, "r");
if(!fp) {
- error("Cannot open file '%s'", filename);
+ error("CONFIG: cannot open file '%s'", filename);
return 0;
}
@@ -430,8 +454,8 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
line++;
s = trim(buffer);
- if(!s) {
- debug(D_CONFIG, "Ignoring line %d, it is empty.", line);
+ if(!s || *s == '#') {
+ debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename);
continue;
}
@@ -449,14 +473,14 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
if(!co) {
// line outside a section
- error("Ignoring line %d ('%s'), it is outside all sections.", line, s);
+ error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename);
continue;
}
char *name = s;
char *value = strchr(s, '=');
if(!value) {
- error("Ignoring line %d ('%s'), there is no = in it.", line, s);
+ error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename);
continue;
}
*value = '\0';
@@ -465,12 +489,12 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
name = trim(name);
value = trim(value);
- if(!name) {
- error("Ignoring line %d, name is empty.", line);
+ if(!name || *name == '#') {
+ error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename);
continue;
}
if(!value) {
- debug(D_CONFIG, "Ignoring line %d, value is empty.", line);
+ debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', value is empty.", line, filename);
continue;
}
@@ -479,12 +503,12 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used)
if(!cv) cv = appconfig_value_create(co, name, value);
else {
if(((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) {
- debug(D_CONFIG, "Line %d, overwriting '%s/%s'.", line, co->name, cv->name);
+ debug(D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name);
freez(cv->value);
cv->value = strdupz(value);
}
else
- debug(D_CONFIG, "Ignoring line %d, '%s/%s' is already present and used.", line, co->name, cv->name);
+ debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.", line, filename, co->name, cv->name);
}
cv->flags |= CONFIG_VALUE_LOADED;
}
@@ -531,6 +555,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
for(co = root->sections; co ; co = co->next) {
if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)
|| !strcmp(co->name, CONFIG_SECTION_WEB)
+ || !strcmp(co->name, CONFIG_SECTION_STATSD)
|| !strcmp(co->name, CONFIG_SECTION_PLUGINS)
|| !strcmp(co->name, CONFIG_SECTION_REGISTRY)
|| !strcmp(co->name, CONFIG_SECTION_HEALTH)
@@ -558,7 +583,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
if(only_changed && !changed) continue;
if(!used) {
- buffer_sprintf(wb, "\n# node '%s' is not used.", co->name);
+ buffer_sprintf(wb, "\n# section '%s' is not used.", co->name);
}
buffer_sprintf(wb, "\n[%s]\n", co->name);
diff --git a/src/appconfig.h b/src/appconfig.h
index 45cc8cfd5..b8c2ee80c 100644
--- a/src/appconfig.h
+++ b/src/appconfig.h
@@ -5,6 +5,7 @@
#define CONFIG_SECTION_GLOBAL "global"
#define CONFIG_SECTION_WEB "web"
+#define CONFIG_SECTION_STATSD "statsd"
#define CONFIG_SECTION_PLUGINS "plugins"
#define CONFIG_SECTION_REGISTRY "registry"
#define CONFIG_SECTION_HEALTH "health"
@@ -34,12 +35,14 @@ extern int appconfig_load(struct config *root, char *filename, int overwrite_use
extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value);
extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value);
+extern long double appconfig_get_float(struct config *root, const char *section, const char *name, long double value);
extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value);
extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value);
extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value);
extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value);
extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value);
+extern long double appconfig_set_float(struct config *root, const char *section, const char *name, long double value);
extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value);
extern int appconfig_exists(struct config *root, const char *section, const char *name);
@@ -53,12 +56,14 @@ extern void appconfig_generate(struct config *root, BUFFER *wb, int only_changed
#define config_load(filename, overwrite_used) appconfig_load(&netdata_config, filename, overwrite_used)
#define config_get(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value)
#define config_get_number(section, name, value) appconfig_get_number(&netdata_config, section, name, value)
+#define config_get_float(section, name, value) appconfig_get_float(&netdata_config, section, name, value)
#define config_get_boolean(section, name, value) appconfig_get_boolean(&netdata_config, section, name, value)
#define config_get_boolean_ondemand(section, name, value) appconfig_get_boolean_ondemand(&netdata_config, section, name, value)
-#define config_set(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value)
+#define config_set(section, name, default_value) appconfig_set(&netdata_config, section, name, default_value)
#define config_set_default(section, name, value) appconfig_set_default(&netdata_config, section, name, value)
#define config_set_number(section, name, value) appconfig_set_number(&netdata_config, section, name, value)
+#define config_set_float(section, name, value) appconfig_set_float(&netdata_config, section, name, value)
#define config_set_boolean(section, name, value) appconfig_set_boolean(&netdata_config, section, name, value)
#define config_exists(section, name) appconfig_exists(&netdata_config, section, name)
diff --git a/src/apps_plugin.c b/src/apps_plugin.c
index b1bf06bee..ecb6aaeac 100644
--- a/src/apps_plugin.c
+++ b/src/apps_plugin.c
@@ -34,8 +34,7 @@
#define MAX_COMPARE_NAME 100
#define MAX_NAME 100
-#define MAX_CMDLINE 1024
-
+#define MAX_CMDLINE 16384
// ----------------------------------------------------------------------------
// the rates we are going to send to netdata will have this detail a value of:
@@ -109,11 +108,12 @@ static size_t
// metric.
// the total system time, as reported by /proc/stat
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
static kernel_uint_t
global_utime = 0,
global_stime = 0,
global_gtime = 0;
-
+#endif
// the normalization ratios, as calculated by normalize_utilization()
double utime_fix_ratio = 1.0,
@@ -127,7 +127,6 @@ double utime_fix_ratio = 1.0,
cminflt_fix_ratio = 1.0,
cmajflt_fix_ratio = 1.0;
-
// ----------------------------------------------------------------------------
// target
//
@@ -223,7 +222,7 @@ size_t
struct pid_stat {
int32_t pid;
char comm[MAX_COMPARE_NAME + 1];
- char cmdline[MAX_CMDLINE + 1];
+ char *cmdline;
uint32_t log_thrown;
@@ -438,7 +437,7 @@ static struct target *get_users_target(uid_t uid) {
w->idhash = simple_hash(w->id);
struct passwd *pw = getpwuid(uid);
- if(!pw)
+ if(!pw || !pw->pw_name || !*pw->pw_name)
snprintfz(w->name, MAX_NAME, "%u", uid);
else
snprintfz(w->name, MAX_NAME, "%s", pw->pw_name);
@@ -471,7 +470,7 @@ struct target *get_groups_target(gid_t gid)
w->idhash = simple_hash(w->id);
struct group *gr = getgrgid(gid);
- if(!gr)
+ if(!gr || !gr->gr_name || !*gr->gr_name)
snprintfz(w->name, MAX_NAME, "%u", gid);
else
snprintfz(w->name, MAX_NAME, "%s", gr->gr_name);
@@ -698,6 +697,7 @@ static inline void del_pid_entry(pid_t pid) {
freez(p->statm_filename);
freez(p->io_filename);
freez(p->cmdline_filename);
+ freez(p->cmdline);
freez(p);
all_pids[pid] = NULL;
@@ -768,7 +768,7 @@ static inline void assign_target_to_pid(struct pid_stat *p) {
if(unlikely(( (!w->starts_with && !w->ends_with && w->comparehash == hash && !strcmp(w->compare, p->comm))
|| (w->starts_with && !w->ends_with && !strncmp(w->compare, p->comm, w->comparelen))
|| (!w->starts_with && w->ends_with && pclen >= w->comparelen && !strcmp(w->compare, &p->comm[pclen - w->comparelen]))
- || (proc_pid_cmdline_is_needed && w->starts_with && w->ends_with && strstr(p->cmdline, w->compare))
+ || (proc_pid_cmdline_is_needed && w->starts_with && w->ends_with && p->cmdline && strstr(p->cmdline, w->compare))
))) {
if(w->target) p->target = w->target;
@@ -787,6 +787,7 @@ static inline void assign_target_to_pid(struct pid_stat *p) {
// update pids from proc
static inline int read_proc_pid_cmdline(struct pid_stat *p) {
+ static char cmdline[MAX_CMDLINE + 1];
#ifdef __FreeBSD__
size_t i, bytes = MAX_CMDLINE;
@@ -796,7 +797,7 @@ static inline int read_proc_pid_cmdline(struct pid_stat *p) {
mib[1] = KERN_PROC;
mib[2] = KERN_PROC_ARGS;
mib[3] = p->pid;
- if (unlikely(sysctl(mib, 4, p->cmdline, &bytes, NULL, 0)))
+ if (unlikely(sysctl(mib, 4, cmdline, &bytes, NULL, 0)))
goto cleanup;
#else
if(unlikely(!p->cmdline_filename)) {
@@ -808,15 +809,17 @@ static inline int read_proc_pid_cmdline(struct pid_stat *p) {
int fd = open(p->cmdline_filename, O_RDONLY, 0666);
if(unlikely(fd == -1)) goto cleanup;
- ssize_t i, bytes = read(fd, p->cmdline, MAX_CMDLINE);
+ ssize_t i, bytes = read(fd, cmdline, MAX_CMDLINE);
close(fd);
if(unlikely(bytes < 0)) goto cleanup;
#endif
- p->cmdline[bytes] = '\0';
+ cmdline[bytes] = '\0';
for(i = 0; i < bytes ; i++)
- if(unlikely(!p->cmdline[i])) p->cmdline[i] = ' ';
+ if(unlikely(!cmdline[i])) cmdline[i] = ' ';
+
+ p->cmdline = strdupz(cmdline);
if(unlikely(debug))
fprintf(stderr, "Read file '%s' contents: %s\n", p->cmdline_filename, p->cmdline);
@@ -825,7 +828,7 @@ static inline int read_proc_pid_cmdline(struct pid_stat *p) {
cleanup:
// copy the command to the command line
- strncpyz(p->cmdline, p->comm, MAX_CMDLINE);
+ p->cmdline = strdupz(p->comm);
return 0;
}
@@ -1157,24 +1160,13 @@ cleanup:
#endif
}
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
static inline int read_proc_stat() {
-#ifdef __FreeBSD__
- long cp_time[CPUSTATES];
- static kernel_uint_t utime_raw = 0, stime_raw = 0, ntime_raw = 0;
-
- if (unlikely(CPUSTATES != 5)) {
- error("FREEBSD: There are %d CPU states (5 was expected)", CPUSTATES);
- goto cleanup;
- }
- if (unlikely(GETSYSCTL_BY_NAME("kern.cp_time", cp_time))) goto cleanup;
-#else
static char filename[FILENAME_MAX + 1] = "";
static procfile *ff = NULL;
static kernel_uint_t utime_raw = 0, stime_raw = 0, gtime_raw = 0, gntime_raw = 0, ntime_raw = 0;
-#endif
static usec_t collected_usec = 0, last_collected_usec = 0;
-#ifndef __FreeBSD__
if(unlikely(!ff)) {
snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix);
ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
@@ -1183,7 +1175,6 @@ static inline int read_proc_stat() {
ff = procfile_readall(ff);
if(unlikely(!ff)) goto cleanup;
-#endif
last_collected_usec = collected_usec;
collected_usec = now_monotonic_usec();
@@ -1193,25 +1184,13 @@ static inline int read_proc_stat() {
// temporary - it is added global_ntime;
kernel_uint_t global_ntime = 0;
-#ifdef __FreeBSD__
- incremental_rate(global_utime, utime_raw, cp_time[0], collected_usec, last_collected_usec);
- incremental_rate(global_ntime, ntime_raw, cp_time[1], collected_usec, last_collected_usec);
- incremental_rate(global_stime, stime_raw, cp_time[2], collected_usec, last_collected_usec);
-#else
incremental_rate(global_utime, utime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 1)), collected_usec, last_collected_usec);
incremental_rate(global_ntime, ntime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 2)), collected_usec, last_collected_usec);
incremental_rate(global_stime, stime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 3)), collected_usec, last_collected_usec);
incremental_rate(global_gtime, gtime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 10)), collected_usec, last_collected_usec);
-#endif
global_utime += global_ntime;
-#ifdef __FreeBSD__
- if(enable_guest_charts) {
- enable_guest_charts = 0;
- info("Guest charts aren't supported by FreeBSD");
- }
-#else
if(enable_guest_charts) {
// temporary - it is added global_ntime;
kernel_uint_t global_gntime = 0;
@@ -1224,7 +1203,6 @@ static inline int read_proc_stat() {
// remove guest time from user time
global_utime -= (global_utime > global_gtime) ? global_gtime : global_utime;
}
-#endif
if(unlikely(global_iterations_counter == 1)) {
global_utime = 0;
@@ -1240,7 +1218,11 @@ cleanup:
global_gtime = 0;
return 0;
}
-
+#else
+static inline int read_proc_stat() {
+ return 0;
+}
+#endif
// ----------------------------------------------------------------------------
@@ -1751,7 +1733,6 @@ static inline int print_process_and_parents(struct pid_stat *p, usec_t time) {
}
static inline void print_process_tree(struct pid_stat *p, char *msg) {
- log_date(stderr);
fprintf(stderr, "%s: process %s (%d, %s) with parents:\n", msg, p->comm, p->pid, p->updated?"running":"exited");
print_process_and_parents(p, p->stat_collected_usec);
}
@@ -1860,7 +1841,6 @@ static inline void process_exited_processes() {
continue;
if(unlikely(debug)) {
- log_date(stderr);
fprintf(stderr, "Absorb %s (%d %s total resources: utime=" KERNEL_UINT_FORMAT " stime=" KERNEL_UINT_FORMAT " gtime=" KERNEL_UINT_FORMAT " minflt=" KERNEL_UINT_FORMAT " majflt=" KERNEL_UINT_FORMAT ")\n"
, p->comm
, p->pid
@@ -2620,14 +2600,13 @@ static inline void send_END(void) {
fprintf(stdout, "END\n");
}
-static usec_t send_resource_usage_to_netdata() {
+void send_resource_usage_to_netdata(usec_t dt) {
static struct timeval last = { 0, 0 };
static struct rusage me_last;
struct timeval now;
struct rusage me;
- usec_t usec;
usec_t cpuuser;
usec_t cpusyst;
@@ -2635,10 +2614,6 @@ static usec_t send_resource_usage_to_netdata() {
now_monotonic_timeval(&last);
getrusage(RUSAGE_SELF, &me_last);
- // the first time, give a zero to allow
- // netdata calibrate to the current time
- // usec = update_every * USEC_PER_SEC;
- usec = 0ULL;
cpuuser = 0;
cpusyst = 0;
}
@@ -2646,7 +2621,6 @@ static usec_t send_resource_usage_to_netdata() {
now_monotonic_timeval(&now);
getrusage(RUSAGE_SELF, &me);
- usec = dt_usec(&now, &last);
cpuuser = me.ru_utime.tv_sec * USEC_PER_SEC + me.ru_utime.tv_usec;
cpusyst = me.ru_stime.tv_sec * USEC_PER_SEC + me.ru_stime.tv_usec;
@@ -2658,38 +2632,45 @@ static usec_t send_resource_usage_to_netdata() {
if(unlikely(!created_charts)) {
created_charts = 1;
- fprintf(stdout
- , "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' apps.plugin netdata.apps_cpu stacked 140000 %1$d\n"
- "DIMENSION user '' incremental 1 1000\n"
- "DIMENSION system '' incremental 1 1000\n"
- "CHART netdata.apps_sizes '' 'Apps Plugin Files' 'files/s' apps.plugin netdata.apps_sizes line 140001 %1$d\n"
- "DIMENSION calls '' incremental 1 1\n"
- "DIMENSION files '' incremental 1 1\n"
- "DIMENSION pids '' absolute 1 1\n"
- "DIMENSION fds '' absolute 1 1\n"
- "DIMENSION targets '' absolute 1 1\n"
- "DIMENSION new_pids 'new pids' incremental 1 1\n"
- "CHART netdata.apps_fix '' 'Apps Plugin Normalization Ratios' 'percentage' apps.plugin netdata.apps_fix line 140002 %1$d\n"
- "DIMENSION utime '' absolute 1 %2$llu\n"
- "DIMENSION stime '' absolute 1 %2$llu\n"
- "DIMENSION gtime '' absolute 1 %2$llu\n"
- "DIMENSION minflt '' absolute 1 %2$llu\n"
- "DIMENSION majflt '' absolute 1 %2$llu\n"
+ fprintf(stdout,
+ "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' apps.plugin netdata.apps_cpu stacked 140000 %1$d\n"
+ "DIMENSION user '' incremental 1 1000\n"
+ "DIMENSION system '' incremental 1 1000\n"
+ "CHART netdata.apps_sizes '' 'Apps Plugin Files' 'files/s' apps.plugin netdata.apps_sizes line 140001 %1$d\n"
+ "DIMENSION calls '' incremental 1 1\n"
+ "DIMENSION files '' incremental 1 1\n"
+ "DIMENSION pids '' absolute 1 1\n"
+ "DIMENSION fds '' absolute 1 1\n"
+ "DIMENSION targets '' absolute 1 1\n"
+ "DIMENSION new_pids 'new pids' incremental 1 1\n"
+ , update_every
+ );
+
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+ fprintf(stdout,
+ "CHART netdata.apps_fix '' 'Apps Plugin Normalization Ratios' 'percentage' apps.plugin netdata.apps_fix line 140002 %1$d\n"
+ "DIMENSION utime '' absolute 1 %2$llu\n"
+ "DIMENSION stime '' absolute 1 %2$llu\n"
+ "DIMENSION gtime '' absolute 1 %2$llu\n"
+ "DIMENSION minflt '' absolute 1 %2$llu\n"
+ "DIMENSION majflt '' absolute 1 %2$llu\n"
, update_every
, RATES_DETAIL
);
if(include_exited_childs)
- fprintf(stdout
- , "CHART netdata.apps_children_fix '' 'Apps Plugin Exited Children Normalization Ratios' 'percentage' apps.plugin netdata.apps_children_fix line 140003 %1$d\n"
- "DIMENSION cutime '' absolute 1 %2$llu\n"
- "DIMENSION cstime '' absolute 1 %2$llu\n"
- "DIMENSION cgtime '' absolute 1 %2$llu\n"
- "DIMENSION cminflt '' absolute 1 %2$llu\n"
- "DIMENSION cmajflt '' absolute 1 %2$llu\n"
+ fprintf(stdout,
+ "CHART netdata.apps_children_fix '' 'Apps Plugin Exited Children Normalization Ratios' 'percentage' apps.plugin netdata.apps_children_fix line 140003 %1$d\n"
+ "DIMENSION cutime '' absolute 1 %2$llu\n"
+ "DIMENSION cstime '' absolute 1 %2$llu\n"
+ "DIMENSION cgtime '' absolute 1 %2$llu\n"
+ "DIMENSION cminflt '' absolute 1 %2$llu\n"
+ "DIMENSION cmajflt '' absolute 1 %2$llu\n"
, update_every
, RATES_DETAIL
);
+#endif
+
}
fprintf(stdout,
@@ -2705,31 +2686,35 @@ static usec_t send_resource_usage_to_netdata() {
"SET targets = %zu\n"
"SET new_pids = %zu\n"
"END\n"
- "BEGIN netdata.apps_fix %llu\n"
- "SET utime = %u\n"
- "SET stime = %u\n"
- "SET gtime = %u\n"
- "SET minflt = %u\n"
- "SET majflt = %u\n"
- "END\n"
- , usec
+ , dt
, cpuuser
, cpusyst
- , usec
+ , dt
, calls_counter
, file_counter
, all_pids_count
, all_files_len
, apps_groups_targets_count
, targets_assignment_counter
- , usec
- , (unsigned int)(utime_fix_ratio * 100 * RATES_DETAIL)
- , (unsigned int)(stime_fix_ratio * 100 * RATES_DETAIL)
- , (unsigned int)(gtime_fix_ratio * 100 * RATES_DETAIL)
- , (unsigned int)(minflt_fix_ratio * 100 * RATES_DETAIL)
- , (unsigned int)(majflt_fix_ratio * 100 * RATES_DETAIL)
);
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
+ fprintf(stdout,
+ "BEGIN netdata.apps_fix %llu\n"
+ "SET utime = %u\n"
+ "SET stime = %u\n"
+ "SET gtime = %u\n"
+ "SET minflt = %u\n"
+ "SET majflt = %u\n"
+ "END\n"
+ , dt
+ , (unsigned int)(utime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(stime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(gtime_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(minflt_fix_ratio * 100 * RATES_DETAIL)
+ , (unsigned int)(majflt_fix_ratio * 100 * RATES_DETAIL)
+ );
+
if(include_exited_childs)
fprintf(stdout,
"BEGIN netdata.apps_children_fix %llu\n"
@@ -2739,17 +2724,17 @@ static usec_t send_resource_usage_to_netdata() {
"SET cminflt = %u\n"
"SET cmajflt = %u\n"
"END\n"
- , usec
+ , dt
, (unsigned int)(cutime_fix_ratio * 100 * RATES_DETAIL)
, (unsigned int)(cstime_fix_ratio * 100 * RATES_DETAIL)
, (unsigned int)(cgtime_fix_ratio * 100 * RATES_DETAIL)
, (unsigned int)(cminflt_fix_ratio * 100 * RATES_DETAIL)
, (unsigned int)(cmajflt_fix_ratio * 100 * RATES_DETAIL)
);
-
- return usec;
+#endif
}
+#if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
static void normalize_utilization(struct target *root) {
struct target *w;
@@ -2895,25 +2880,30 @@ static void normalize_utilization(struct target *root) {
);
}
}
+#else // ALL_PIDS_ARE_READ_INSTANTLY == 1
+static void normalize_utilization(struct target *root) {
+ (void)root;
+}
+#endif // ALL_PIDS_ARE_READ_INSTANTLY
-static void send_collected_data_to_netdata(struct target *root, const char *type, usec_t usec) {
+static void send_collected_data_to_netdata(struct target *root, const char *type, usec_t dt) {
struct target *w;
- send_BEGIN(type, "cpu", usec);
+ send_BEGIN(type, "cpu", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, (kernel_uint_t)(w->utime * utime_fix_ratio) + (kernel_uint_t)(w->stime * stime_fix_ratio) + (kernel_uint_t)(w->gtime * gtime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cutime * cutime_fix_ratio) + (kernel_uint_t)(w->cstime * cstime_fix_ratio) + (kernel_uint_t)(w->cgtime * cgtime_fix_ratio)):0ULL));
}
send_END();
- send_BEGIN(type, "cpu_user", usec);
+ send_BEGIN(type, "cpu_user", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, (kernel_uint_t)(w->utime * utime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cutime * cutime_fix_ratio)):0ULL));
}
send_END();
- send_BEGIN(type, "cpu_system", usec);
+ send_BEGIN(type, "cpu_system", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, (kernel_uint_t)(w->stime * stime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cstime * cstime_fix_ratio)):0ULL));
@@ -2921,7 +2911,7 @@ static void send_collected_data_to_netdata(struct target *root, const char *type
send_END();
if(show_guest_time) {
- send_BEGIN(type, "cpu_guest", usec);
+ send_BEGIN(type, "cpu_guest", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, (kernel_uint_t)(w->gtime * gtime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cgtime * cgtime_fix_ratio)):0ULL));
@@ -2929,42 +2919,42 @@ static void send_collected_data_to_netdata(struct target *root, const char *type
send_END();
}
- send_BEGIN(type, "threads", usec);
+ send_BEGIN(type, "threads", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, w->num_threads);
}
send_END();
- send_BEGIN(type, "processes", usec);
+ send_BEGIN(type, "processes", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, w->processes);
}
send_END();
- send_BEGIN(type, "mem", usec);
+ send_BEGIN(type, "mem", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, (w->statm_resident > w->statm_share)?(w->statm_resident - w->statm_share):0ULL);
}
send_END();
- send_BEGIN(type, "vmem", usec);
+ send_BEGIN(type, "vmem", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, w->statm_size);
}
send_END();
- send_BEGIN(type, "minor_faults", usec);
+ send_BEGIN(type, "minor_faults", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, (kernel_uint_t)(w->minflt * minflt_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cminflt * cminflt_fix_ratio)):0ULL));
}
send_END();
- send_BEGIN(type, "major_faults", usec);
+ send_BEGIN(type, "major_faults", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, (kernel_uint_t)(w->majflt * majflt_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cmajflt * cmajflt_fix_ratio)):0ULL));
@@ -2972,14 +2962,14 @@ static void send_collected_data_to_netdata(struct target *root, const char *type
send_END();
#ifndef __FreeBSD__
- send_BEGIN(type, "lreads", usec);
+ send_BEGIN(type, "lreads", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, w->io_logical_bytes_read);
}
send_END();
- send_BEGIN(type, "lwrites", usec);
+ send_BEGIN(type, "lwrites", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, w->io_logical_bytes_written);
@@ -2987,14 +2977,14 @@ static void send_collected_data_to_netdata(struct target *root, const char *type
send_END();
#endif
- send_BEGIN(type, "preads", usec);
+ send_BEGIN(type, "preads", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, w->io_storage_bytes_read);
}
send_END();
- send_BEGIN(type, "pwrites", usec);
+ send_BEGIN(type, "pwrites", dt);
for (w = root; w ; w = w->next) {
if(unlikely(w->exposed))
send_SET(w->name, w->io_storage_bytes_written);
@@ -3002,21 +2992,21 @@ static void send_collected_data_to_netdata(struct target *root, const char *type
send_END();
if(enable_file_charts) {
- send_BEGIN(type, "files", usec);
+ send_BEGIN(type, "files", dt);
for (w = root; w; w = w->next) {
if (unlikely(w->exposed))
send_SET(w->name, w->openfiles);
}
send_END();
- send_BEGIN(type, "sockets", usec);
+ send_BEGIN(type, "sockets", dt);
for (w = root; w; w = w->next) {
if (unlikely(w->exposed))
send_SET(w->name, w->opensockets);
}
send_END();
- send_BEGIN(type, "pipes", usec);
+ send_BEGIN(type, "pipes", dt);
for (w = root; w; w = w->next) {
if (unlikely(w->exposed))
send_SET(w->name, w->openpipes);
@@ -3472,8 +3462,9 @@ int main(int argc, char **argv) {
static int profiling_count=0;
profiling_count++;
if(unlikely(profiling_count > 1000)) exit(0);
+ usec_t dt = update_every * USEC_PER_SEC;
#else
- heartbeat_next(&hb, step);
+ usec_t dt = heartbeat_next(&hb, step);
#endif
if(!collect_data_for_all_processes()) {
@@ -3485,7 +3476,7 @@ int main(int argc, char **argv) {
calculate_netdata_statistics();
normalize_utilization(apps_groups_root_target);
- usec_t dt = send_resource_usage_to_netdata();
+ send_resource_usage_to_netdata(dt);
// this is smart enough to show only newly added apps, when needed
send_charts_updates_to_netdata(apps_groups_root_target, "apps", "Apps");
diff --git a/src/backend_prometheus.c b/src/backend_prometheus.c
new file mode 100644
index 000000000..88ec2c65b
--- /dev/null
+++ b/src/backend_prometheus.c
@@ -0,0 +1,397 @@
+#include "common.h"
+
+// ----------------------------------------------------------------------------
+// PROMETHEUS
+// /api/v1/allmetrics?format=prometheus
+
+static struct prometheus_server {
+ const char *server;
+ uint32_t hash;
+ time_t last_access;
+ struct prometheus_server *next;
+} *prometheus_server_root = NULL;
+
+static inline time_t prometheus_server_last_access(const char *server, time_t now) {
+ uint32_t hash = simple_hash(server);
+
+ struct prometheus_server *ps;
+ for(ps = prometheus_server_root; ps ;ps = ps->next) {
+ if (hash == ps->hash && !strcmp(server, ps->server)) {
+ time_t last = ps->last_access;
+ ps->last_access = now;
+ return last;
+ }
+ }
+
+ ps = callocz(1, sizeof(struct prometheus_server));
+ ps->server = strdupz(server);
+ ps->hash = hash;
+ ps->last_access = now;
+ ps->next = prometheus_server_root;
+ prometheus_server_root = ps;
+
+ return 0;
+}
+
+static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) {
+ size_t n;
+
+ for(n = 0; *s && n < usable ; d++, s++, n++) {
+ register char c = *s;
+
+ if(!isalnum(c)) *d = '_';
+ else *d = c;
+ }
+ *d = '\0';
+
+ return n;
+}
+
+static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable) {
+ size_t n;
+
+ // make sure we can escape one character without overflowing the buffer
+ usable--;
+
+ for(n = 0; *s && n < usable ; d++, s++, n++) {
+ register char c = *s;
+
+ if(unlikely(c == '"' || c == '\\' || c == '\n')) {
+ *d++ = '\\';
+ n++;
+ }
+ *d = c;
+ }
+ *d = '\0';
+
+ return n;
+}
+
+static inline char *prometheus_units_copy(char *d, const char *s, size_t usable) {
+ const char *sorig = s;
+ char *ret = d;
+ size_t n;
+
+ *d++ = '_';
+ for(n = 1; *s && n < usable ; d++, s++, n++) {
+ register char c = *s;
+
+ if(!isalnum(c)) *d = '_';
+ else *d = c;
+ }
+
+ if(n == 2 && sorig[0] == '%') {
+ n = 0;
+ d = ret;
+ s = "_percent";
+ for( ; *s && n < usable ; n++) *d++ = *s++;
+ }
+ else if(n > 3 && sorig[n-3] == '/' && sorig[n-2] == 's') {
+ n = n - 2;
+ d -= 2;
+ s = "_persec";
+ for( ; *s && n < usable ; n++) *d++ = *s++;
+ }
+
+ *d = '\0';
+
+ return ret;
+}
+
+
+#define PROMETHEUS_ELEMENT_MAX 256
+#define PROMETHEUS_LABELS_MAX 1024
+
+static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, uint32_t options, time_t after, time_t before, int allhosts, int help, int types, int names) {
+ rrdhost_rdlock(host);
+
+ char hostname[PROMETHEUS_ELEMENT_MAX + 1];
+ prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
+
+ char labels[PROMETHEUS_LABELS_MAX + 1] = "";
+ if(allhosts) {
+ if(host->tags && *(host->tags))
+ buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);
+
+ snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
+ }
+ else {
+ if(host->tags && *(host->tags))
+ buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
+ }
+
+ // for each chart
+ RRDSET *st;
+ rrdset_foreach_read(st, host) {
+ char chart[PROMETHEUS_ELEMENT_MAX + 1];
+ char context[PROMETHEUS_ELEMENT_MAX + 1];
+ char family[PROMETHEUS_ELEMENT_MAX + 1];
+ char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
+
+ prometheus_label_copy(chart, (names && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);
+ prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
+ prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
+
+ if(likely(backends_can_send_rrdset(options, st))) {
+ rrdset_rdlock(st);
+
+ int as_collected = ((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED);
+ int homogeneus = 1;
+ if(as_collected) {
+ if(rrdset_flag_check(st, RRDSET_FLAG_HOMEGENEOUS_CHECK))
+ rrdset_update_heterogeneous_flag(st);
+
+ if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS))
+ homogeneus = 0;
+ }
+ else {
+ if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE)
+ prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX);
+ }
+
+ if(unlikely(help))
+ buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n"
+ , (homogeneus)?"homogeneus":"heterogeneous"
+ , (names && st->name) ? st->name : st->id
+ , st->context
+ , st->family
+ , st->units
+ );
+
+ // for each dimension
+ RRDDIM *rd;
+ rrddim_foreach_read(rd, st) {
+ if(rd->collections_counter) {
+ char dimension[PROMETHEUS_ELEMENT_MAX + 1];
+ char *suffix = "";
+
+ if (as_collected) {
+ // we need as-collected / raw data
+
+ const char *t = "gauge", *h = "gives";
+ if(rd->algorithm == RRD_ALGORITHM_INCREMENTAL ||
+ rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) {
+ t = "counter";
+ h = "delta gives";
+ suffix = "_total";
+ }
+
+ if(homogeneus) {
+ // all the dimensions of the chart, has the same algorithm, multiplier and divisor
+ // we add all dimensions as labels
+
+ prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
+
+ if(unlikely(help))
+ buffer_sprintf(wb
+ , "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"
+ , prefix
+ , context
+ , suffix
+ , (names && st->name) ? st->name : st->id
+ , st->context
+ , st->family
+ , (names && rd->name) ? rd->name : rd->id
+ , rd->multiplier
+ , rd->divisor
+ , h
+ , st->units
+ , t
+ );
+
+ if(unlikely(types))
+ buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n"
+ , prefix
+ , context
+ , suffix
+ , t
+ );
+
+ buffer_sprintf(wb
+ , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"
+ , prefix
+ , context
+ , suffix
+ , chart
+ , family
+ , dimension
+ , labels
+ , rd->last_collected_value
+ , timeval_msec(&rd->last_collected_time)
+ );
+ }
+ else {
+ // the dimensions of the chart, do not have the same algorithm, multiplier or divisor
+ // we create a metric per dimension
+
+ prometheus_name_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
+
+ if(unlikely(help))
+ buffer_sprintf(wb
+ , "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"
+ , prefix
+ , context
+ , dimension
+ , suffix
+ , (names && st->name) ? st->name : st->id
+ , st->context
+ , st->family
+ , (names && rd->name) ? rd->name : rd->id
+ , rd->multiplier
+ , rd->divisor
+ , h
+ , st->units
+ , t
+ );
+
+ if(unlikely(types))
+ buffer_sprintf(wb, "# COMMENT TYPE %s_%s_%s%s %s\n"
+ , prefix
+ , context
+ , dimension
+ , suffix
+ , t
+ );
+
+ buffer_sprintf(wb
+ , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"
+ , prefix
+ , context
+ , dimension
+ , suffix
+ , chart
+ , family
+ , labels
+ , rd->last_collected_value
+ , timeval_msec(&rd->last_collected_time)
+ );
+ }
+ }
+ else {
+ // we need average or sum of the data
+
+ time_t first_t = after, last_t = before;
+ calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);
+
+ if(!isnan(value) && !isinf(value)) {
+
+ if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE)
+ suffix = "_average";
+ else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)
+ suffix = "_sum";
+
+ prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);
+
+ if (unlikely(help))
+ buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n"
+ , prefix
+ , context
+ , units
+ , suffix
+ , (names && rd->name) ? rd->name : rd->id
+ , st->units
+ , (unsigned long long)first_t
+ , (unsigned long long)last_t
+ );
+
+ if (unlikely(types))
+ buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n"
+ , prefix
+ , context
+ , units
+ , suffix
+ );
+
+ buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n"
+ , prefix
+ , context
+ , units
+ , suffix
+ , chart
+ , family
+ , dimension
+ , labels
+ , value
+ , last_t * MSEC_PER_SEC
+ );
+ }
+ }
+ }
+ }
+
+ rrdset_unlock(st);
+ }
+ }
+
+ rrdhost_unlock(host);
+}
+
+static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, uint32_t options, const char *server, time_t now, int help) {
+ if(!server || !*server) server = "default";
+
+ time_t after = prometheus_server_last_access(server, now);
+
+ int first_seen = 0;
+ if(!after) {
+ after = now - backend_update_every;
+ first_seen = 1;
+ }
+
+ if(after > now) {
+ // oops! this should never happen
+ after = now - backend_update_every;
+ }
+
+ if(help) {
+ int show_range = 1;
+ char *mode;
+ if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) {
+ mode = "as collected";
+ show_range = 0;
+ }
+ else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE)
+ mode = "average";
+ else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)
+ mode = "sum";
+ else
+ mode = "unknown";
+
+ buffer_sprintf(wb, "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s"
+ , host->hostname
+ , (first_seen)?"FIRST SEEN ":""
+ , server
+ , mode
+ , (unsigned long)((first_seen)?0:(now - after))
+ , (first_seen)?"never":"seconds ago"
+ );
+
+ if(show_range)
+ buffer_sprintf(wb, ", time range %lu to %lu", (unsigned long)after, (unsigned long)now);
+
+ buffer_strcat(wb, "\n\n");
+ }
+
+ return after;
+}
+
+void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names) {
+ time_t before = now_realtime_sec();
+
+ // we start at the point we had stopped before
+ time_t after = prometheus_preparation(host, wb, options, server, before, help);
+
+ rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 0, help, types, names);
+}
+
+void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names) {
+ time_t before = now_realtime_sec();
+
+ // we start at the point we had stopped before
+ time_t after = prometheus_preparation(host, wb, options, server, before, help);
+
+ rrd_rdlock();
+ rrdhost_foreach_read(host) {
+ rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 1, help, types, names);
+ }
+ rrd_unlock();
+}
diff --git a/src/backend_prometheus.h b/src/backend_prometheus.h
new file mode 100644
index 000000000..53dddb0d2
--- /dev/null
+++ b/src/backend_prometheus.h
@@ -0,0 +1,11 @@
+//
+// Created by costa on 09/07/17.
+//
+
+#ifndef NETDATA_BACKEND_PROMETHEUS_H
+#define NETDATA_BACKEND_PROMETHEUS_H
+
+extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names);
+extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names);
+
+#endif //NETDATA_BACKEND_PROMETHEUS_H
diff --git a/src/backends.c b/src/backends.c
index 3e385cab5..eed4ab409 100644
--- a/src/backends.c
+++ b/src/backends.c
@@ -22,47 +22,81 @@
// 5. repeats the above forever.
//
-#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001
-#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002
-#define BACKEND_SOURCE_DATA_SUM 0x00000004
-
+const char *backend_prefix = "netdata";
+int backend_send_names = 1;
+int backend_update_every = 10;
+uint32_t backend_options = BACKEND_SOURCE_DATA_AVERAGE;
// ----------------------------------------------------------------------------
// helper functions for backends
+static inline size_t backend_name_copy(char *d, const char *s, size_t usable) {
+ size_t n;
+
+ for(n = 0; *s && n < usable ; d++, s++, n++) {
+ char c = *s;
+
+ if(c != '.' && !isalnum(c)) *d = '_';
+ else *d = c;
+ }
+ *d = '\0';
+
+ return n;
+}
+
// calculate the SUM or AVERAGE of a dimension, for any timeframe
// may return NAN if the database does not have any value in the give timeframe
-static inline calculated_number backend_calculate_value_from_stored_data(
+inline calculated_number backend_calculate_value_from_stored_data(
RRDSET *st // the chart
, RRDDIM *rd // the dimension
, time_t after // the start timestamp
, time_t before // the end timestamp
, uint32_t options // BACKEND_SOURCE_* bitmap
+ , time_t *first_timestamp // the first point of the database used in this response
+ , time_t *last_timestamp // the timestamp that should be reported to backend
) {
// find the edges of the rrd database for this chart
time_t first_t = rrdset_first_entry_t(st);
time_t last_t = rrdset_last_entry_t(st);
+ time_t update_every = st->update_every;
- if(unlikely(before < first_t || after > last_t))
- // the chart has not been updated in the wanted timeframe
- return NAN;
+ // step back a little, to make sure we have complete data collection
+ // for all metrics
+ after -= update_every * 2;
+ before -= update_every * 2;
// align the time-frame
- // for 'after' also skip the first value by adding st->update_every
- after = after - after % st->update_every + st->update_every;
- before = before - before % st->update_every;
+ after = after - (after % update_every);
+ before = before - (before % update_every);
- if(unlikely(after < first_t))
- after = first_t;
+ // for before, loose another iteration
+ // the latest point will be reported the next time
+ before -= update_every;
if(unlikely(after > before))
- // this can happen when st->update_every > before - after
- before = after;
+ // this can happen when update_every > before - after
+ after = before;
+
+ if(unlikely(after < first_t))
+ after = first_t;
if(unlikely(before > last_t))
before = last_t;
+ if(unlikely(before < first_t || after > last_t)) {
+ // the chart has not been updated in the wanted timeframe
+ debug(D_BACKEND, "BACKEND: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu",
+ st->rrdhost->hostname, st->id, rd->id,
+ (unsigned long)after, (unsigned long)before,
+ (unsigned long)first_t, (unsigned long)last_t
+ );
+ return NAN;
+ }
+
+ *first_timestamp = after;
+ *last_timestamp = before;
+
size_t counter = 0;
calculated_number sum = 0;
@@ -88,10 +122,15 @@ static inline calculated_number backend_calculate_value_from_stored_data(
counter++;
}
- if(unlikely(!counter))
+ if(unlikely(!counter)) {
+ debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu",
+ st->rrdhost->hostname, st->id, rd->id,
+ (unsigned long)after, (unsigned long)before
+ );
return NAN;
+ }
- if(unlikely(options & BACKEND_SOURCE_DATA_SUM))
+ if(unlikely((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM))
return sum;
return sum / (calculated_number)counter;
@@ -113,7 +152,7 @@ static inline int discard_response(BUFFER *b, const char *backend) {
}
*d = '\0';
- info("Received %zu bytes from %s backend. Ignoring them. Sample: '%s'", buffer_strlen(b), backend, sample);
+ info("BACKEND: received %zu bytes from %s backend. Ignoring them. Sample: '%s'", buffer_strlen(b), backend, sample);
buffer_flush(b);
return 0;
}
@@ -138,13 +177,18 @@ static inline int format_dimension_collected_graphite_plaintext(
(void)before;
(void)options;
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
+ backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
+ backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
+
buffer_sprintf(
b
, "%s.%s.%s.%s " COLLECTED_NUMBER_FORMAT " %u\n"
, prefix
, hostname
- , st->id
- , rd->id
+ , chart_name
+ , dimension_name
, rd->last_collected_value
, (uint32_t)rd->last_collected_time.tv_sec
);
@@ -165,7 +209,13 @@ static inline int format_dimension_stored_graphite_plaintext(
) {
(void)host;
- calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options);
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
+ backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
+ backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
+
+ time_t first_t = after, last_t = before;
+ calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);
if(!isnan(value)) {
@@ -174,10 +224,10 @@ static inline int format_dimension_stored_graphite_plaintext(
, "%s.%s.%s.%s " CALCULATED_NUMBER_FORMAT " %u\n"
, prefix
, hostname
- , st->id
- , rd->id
+ , chart_name
+ , dimension_name
, value
- , (uint32_t) before
+ , (uint32_t) last_t
);
return 1;
@@ -209,15 +259,22 @@ static inline int format_dimension_collected_opentsdb_telnet(
(void)before;
(void)options;
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
+ backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
+ backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
+
buffer_sprintf(
b
- , "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s\n"
+ , "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n"
, prefix
- , st->id
- , rd->id
+ , chart_name
+ , dimension_name
, (uint32_t)rd->last_collected_time.tv_sec
, rd->last_collected_value
, hostname
+ , (host->tags)?" ":""
+ , (host->tags)?host->tags:""
);
return 1;
@@ -236,19 +293,27 @@ static inline int format_dimension_stored_opentsdb_telnet(
) {
(void)host;
- calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options);
+ time_t first_t = after, last_t = before;
+ calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);
+
+ char chart_name[RRD_ID_LENGTH_MAX + 1];
+ char dimension_name[RRD_ID_LENGTH_MAX + 1];
+ backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);
+ backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);
if(!isnan(value)) {
buffer_sprintf(
b
- , "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s\n"
+ , "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n"
, prefix
- , st->id
- , rd->id
- , (uint32_t) before
+ , chart_name
+ , dimension_name
+ , (uint32_t) last_t
, value
, hostname
+ , (host->tags)?" ":""
+ , (host->tags)?host->tags:""
);
return 1;
@@ -329,7 +394,8 @@ static inline int format_dimension_stored_json_plaintext(
) {
(void)host;
- calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options);
+ time_t first_t = after, last_t = before;
+ calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);
if(!isnan(value)) {
buffer_sprintf(b, "{"
@@ -362,7 +428,7 @@ static inline int format_dimension_stored_json_plaintext(
rd->name,
value,
- (uint32_t)before
+ (uint32_t) last_t
);
return 1;
@@ -378,6 +444,56 @@ static inline int process_json_response(BUFFER *b) {
// ----------------------------------------------------------------------------
// the backend thread
+static SIMPLE_PATTERN *charts_pattern = NULL;
+
+inline int backends_can_send_rrdset(uint32_t options, RRDSET *st) {
+ if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE)))
+ return 0;
+
+ if(unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) {
+ // we have not checked this chart
+ if(simple_pattern_matches(charts_pattern, st->id) || simple_pattern_matches(charts_pattern, st->name))
+ rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND);
+ else {
+ rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE);
+ debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, st->rrdhost->hostname);
+ return 0;
+ }
+ }
+
+ if(unlikely(!rrdset_is_available_for_backends(st))) {
+ debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.", st->id, st->rrdhost->hostname);
+ return 0;
+ }
+
+ if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {
+ debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, st->rrdhost->hostname, rrd_memory_mode_name(st->rrdhost->rrd_memory_mode));
+ return 0;
+ }
+
+ return 1;
+}
+
+inline uint32_t backend_parse_data_source(const char *source, uint32_t mode) {
+ if(!strcmp(source, "raw") || !strcmp(source, "as collected") || !strcmp(source, "as-collected") || !strcmp(source, "as_collected") || !strcmp(source, "ascollected")) {
+ mode |= BACKEND_SOURCE_DATA_AS_COLLECTED;
+ mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AS_COLLECTED);
+ }
+ else if(!strcmp(source, "average")) {
+ mode |= BACKEND_SOURCE_DATA_AVERAGE;
+ mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AVERAGE);
+ }
+ else if(!strcmp(source, "sum") || !strcmp(source, "volume")) {
+ mode |= BACKEND_SOURCE_DATA_SUM;
+ mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_SUM);
+ }
+ else {
+ error("BACKEND: invalid data source method '%s'.", source);
+ }
+
+ return mode;
+}
+
void *backends_main(void *ptr) {
int default_port = 0;
int sock = -1;
@@ -387,13 +503,13 @@ void *backends_main(void *ptr) {
int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, uint32_t) = NULL;
int (*backend_response_checker)(BUFFER *) = NULL;
- info("BACKEND thread created with task id %d", gettid());
+ info("BACKEND: thread created with task id %d", gettid());
if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
- error("Cannot set pthread cancel type to DEFERRED.");
+ error("BACKEND: cannot set pthread cancel type to DEFERRED.");
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
- error("Cannot set pthread cancel state to ENABLE.");
+ error("BACKEND: cannot set pthread cancel state to ENABLE.");
// ------------------------------------------------------------------------
// collect configuration options
@@ -402,45 +518,35 @@ void *backends_main(void *ptr) {
.tv_sec = 0,
.tv_usec = 0
};
- uint32_t options;
int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0);
const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average");
const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");
const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost");
- const char *prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");
+ backend_prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");
const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname);
- int frequency = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", 10);
+ backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", backend_update_every);
int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10);
- long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", frequency * 2 * 1000);
+ long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", backend_update_every * 2 * 1000);
+ backend_send_names = config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", backend_send_names);
+
+ charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), SIMPLE_PATTERN_EXACT);
+
// ------------------------------------------------------------------------
// validate configuration options
// and prepare for sending data to our backend
- if(!enabled || frequency < 1)
- goto cleanup;
-
- if(!strcmp(source, "as collected")) {
- options = BACKEND_SOURCE_DATA_AS_COLLECTED;
- }
- else if(!strcmp(source, "average")) {
- options = BACKEND_SOURCE_DATA_AVERAGE;
- }
- else if(!strcmp(source, "sum") || !strcmp(source, "volume")) {
- options = BACKEND_SOURCE_DATA_SUM;
- }
- else {
- error("Invalid data source method '%s' for backend given. Disabling backed.", source);
- goto cleanup;
- }
+ backend_options = backend_parse_data_source(source, backend_options);
if(timeoutms < 1) {
- error("BACKED invalid timeout %ld ms given. Assuming %d ms.", timeoutms, frequency * 2 * 1000);
- timeoutms = frequency * 2 * 1000;
+ error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, backend_update_every * 2 * 1000);
+ timeoutms = backend_update_every * 2 * 1000;
}
timeout.tv_sec = (timeoutms * 1000) / 1000000;
timeout.tv_usec = (timeoutms * 1000) % 1000000;
+ if(!enabled || backend_update_every < 1)
+ goto cleanup;
// ------------------------------------------------------------------------
// select the backend type
@@ -450,7 +556,7 @@ void *backends_main(void *ptr) {
default_port = 2003;
backend_response_checker = process_graphite_response;
- if(options == BACKEND_SOURCE_DATA_AS_COLLECTED)
+ if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_graphite_plaintext;
else
backend_request_formatter = format_dimension_stored_graphite_plaintext;
@@ -461,7 +567,7 @@ void *backends_main(void *ptr) {
default_port = 4242;
backend_response_checker = process_opentsdb_response;
- if(options == BACKEND_SOURCE_DATA_AS_COLLECTED)
+ if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_opentsdb_telnet;
else
backend_request_formatter = format_dimension_stored_opentsdb_telnet;
@@ -472,19 +578,19 @@ void *backends_main(void *ptr) {
default_port = 5448;
backend_response_checker = process_json_response;
- if (options == BACKEND_SOURCE_DATA_AS_COLLECTED)
+ if ((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)
backend_request_formatter = format_dimension_collected_json_plaintext;
else
backend_request_formatter = format_dimension_stored_json_plaintext;
}
else {
- error("Unknown backend type '%s'", type);
+ error("BACKEND: Unknown backend type '%s'", type);
goto cleanup;
}
if(backend_request_formatter == NULL || backend_response_checker == NULL) {
- error("backend is misconfigured - disabling it.");
+ error("BACKEND: backend is misconfigured - disabling it.");
goto cleanup;
}
@@ -509,18 +615,18 @@ void *backends_main(void *ptr) {
chart_backend_reconnects = 0,
chart_backend_latency = 0;
- RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, frequency, RRDSET_TYPE_LINE);
+ RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, backend_update_every, RRDSET_TYPE_LINE);
rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
- RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, frequency, RRDSET_TYPE_AREA);
+ RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, backend_update_every, RRDSET_TYPE_AREA);
rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
- RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, frequency, RRDSET_TYPE_LINE);
+ RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, backend_update_every, RRDSET_TYPE_LINE);
rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -534,11 +640,11 @@ void *backends_main(void *ptr) {
*
* issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
*
- RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, frequency, RRDSET_TYPE_AREA);
+ RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, backend_update_every, RRDSET_TYPE_AREA);
rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
*/
- RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, frequency, RRDSET_TYPE_STACKED);
+ RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, backend_update_every, RRDSET_TYPE_STACKED);
rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
@@ -546,9 +652,9 @@ void *backends_main(void *ptr) {
// ------------------------------------------------------------------------
// prepare the backend main loop
- info("BACKEND configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, frequency, hostname, prefix);
+ info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, backend_update_every, hostname, backend_prefix);
- usec_t step_ut = frequency * USEC_PER_SEC;
+ usec_t step_ut = backend_update_every * USEC_PER_SEC;
time_t after = now_realtime_sec();
int failures = 0;
heartbeat_t hb;
@@ -558,9 +664,10 @@ void *backends_main(void *ptr) {
// ------------------------------------------------------------------------
// Wait for the next iteration point.
+
heartbeat_next(&hb, step_ut);
time_t before = now_realtime_sec();
-
+ debug(D_BACKEND, "BACKEND: preparing buffer for timeframe %lu to %lu", (unsigned long)after, (unsigned long)before);
// ------------------------------------------------------------------------
// add to the buffer the data we need to send to the backend
@@ -568,33 +675,59 @@ void *backends_main(void *ptr) {
int pthreadoldcancelstate;
if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &pthreadoldcancelstate) != 0))
- error("Cannot set pthread cancel state to DISABLE.");
+ error("BACKEND: cannot set pthread cancel state to DISABLE.");
+
+ size_t count_hosts = 0;
+ size_t count_charts_total = 0;
+ size_t count_dims_total = 0;
rrd_rdlock();
RRDHOST *host;
rrdhost_foreach_read(host) {
- if(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE)
- continue;
-
rrdhost_rdlock(host);
+ count_hosts++;
+ size_t count_charts = 0;
+ size_t count_dims = 0;
+ size_t count_dims_skipped = 0;
+
+ const char *__hostname = (host == localhost)?hostname:host->hostname;
+
RRDSET *st;
rrdset_foreach_read(st, host) {
- rrdset_rdlock(st);
+ if(likely(backends_can_send_rrdset(backend_options, st))) {
+ rrdset_rdlock(st);
+
+ count_charts++;
+
+ RRDDIM *rd;
+ rrddim_foreach_read(rd, st) {
+ if (likely(rd->last_collected_time.tv_sec >= after)) {
+ chart_buffered_metrics += backend_request_formatter(b, backend_prefix, host, __hostname, st, rd, after, before, backend_options);
+ count_dims++;
+ }
+ else {
+ debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before);
+ count_dims_skipped++;
+ }
+ }
- RRDDIM *rd;
- rrddim_foreach_read(rd, st) {
- if(rd->last_collected_time.tv_sec >= after)
- chart_buffered_metrics += backend_request_formatter(b, prefix, host, (host == localhost)?hostname:host->hostname, st, rd, after, before, options);
+ rrdset_unlock(st);
}
- rrdset_unlock(st);
}
+
+ debug(D_BACKEND, "BACKEND: sending host '%s', metrics of %zu dimensions, of %zu charts. Skipped %zu dimensions.", __hostname, count_dims, count_charts, count_dims_skipped);
+ count_charts_total += count_charts;
+ count_dims_total += count_dims;
+
rrdhost_unlock(host);
}
rrd_unlock();
+ debug(D_BACKEND, "BACKEND: buffer has %zu bytes, added metrics for %zu dimensions, of %zu charts, from %zu hosts", buffer_strlen(b), count_dims_total, count_charts_total, count_hosts);
+
if(unlikely(pthread_setcancelstate(pthreadoldcancelstate, NULL) != 0))
- error("Cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate);
+ error("BACKEND: cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate);
// ------------------------------------------------------------------------
@@ -639,14 +772,14 @@ void *backends_main(void *ptr) {
chart_receptions++;
}
else if(r == 0) {
- error("Backend '%s' closed the socket", destination);
+ error("BACKEND: '%s' closed the socket", destination);
close(sock);
sock = -1;
}
else {
// failed to receive data
if(errno != EAGAIN && errno != EWOULDBLOCK) {
- error("Cannot receive data from backend '%s'.", destination);
+ error("BACKEND: cannot receive data from backend '%s'.", destination);
}
}
}
@@ -698,7 +831,7 @@ void *backends_main(void *ptr) {
}
else {
// oops! we couldn't send (all or some of the) data
- error("Failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written);
+ error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written);
chart_transmission_failures++;
if(written != -1)
@@ -713,7 +846,7 @@ void *backends_main(void *ptr) {
}
}
else {
- error("Failed to update database backend '%s'", destination);
+ error("BACKEND: failed to update database backend '%s'", destination);
chart_transmission_failures++;
// increment the counter we check for data loss
@@ -723,7 +856,7 @@ void *backends_main(void *ptr) {
if(failures > buffer_on_failures) {
// too bad! we are going to lose data
chart_lost_bytes += buffer_strlen(b);
- error("Reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination);
+ error("BACKEND: reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination);
buffer_flush(b);
failures = 0;
chart_data_lost_events++;
@@ -781,7 +914,7 @@ cleanup:
buffer_free(b);
buffer_free(response);
- info("BACKEND thread exiting");
+ info("BACKEND: thread exiting");
static_thread->enabled = 0;
pthread_exit(NULL);
diff --git a/src/backends.h b/src/backends.h
index 61122a1d0..e882f3db1 100644
--- a/src/backends.h
+++ b/src/backends.h
@@ -1,6 +1,30 @@
#ifndef NETDATA_BACKENDS_H
#define NETDATA_BACKENDS_H 1
-void *backends_main(void *ptr);
+#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001
+#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002
+#define BACKEND_SOURCE_DATA_SUM 0x00000004
+
+#define BACKEND_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM)
+
+extern int backend_send_names;
+extern int backend_update_every;
+extern uint32_t backend_options;
+extern const char *backend_prefix;
+
+extern void *backends_main(void *ptr);
+
+extern int backends_can_send_rrdset(uint32_t options, RRDSET *st);
+extern uint32_t backend_parse_data_source(const char *source, uint32_t mode);
+
+extern calculated_number backend_calculate_value_from_stored_data(
+ RRDSET *st // the chart
+ , RRDDIM *rd // the dimension
+ , time_t after // the start timestamp
+ , time_t before // the end timestamp
+ , uint32_t options // BACKEND_SOURCE_* bitmap
+ , time_t *first_timestamp // the timestamp of the first point used in this response
+ , time_t *last_timestamp // the timestamp that should be reported to backend
+);
#endif /* NETDATA_BACKENDS_H */
diff --git a/src/clocks.c b/src/clocks.c
index 879ebf911..8f2aa740e 100644
--- a/src/clocks.c
+++ b/src/clocks.c
@@ -115,8 +115,8 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick)
if(likely(*hb != 0ULL)) {
usec_t dt = now - *hb;
*hb = now;
- if(unlikely(dt / tick > 1))
- error("heartbeat missed between %llu usec and %llu usec", *hb, now);
+ if(unlikely(dt >= tick + tick / 2))
+ error("heartbeat missed %llu microseconds", dt - tick);
return dt;
}
else {
diff --git a/src/clocks.h b/src/clocks.h
index 197b5431f..ca5715254 100644
--- a/src/clocks.h
+++ b/src/clocks.h
@@ -55,6 +55,8 @@ typedef usec_t heartbeat_t;
#define USEC_PER_SEC 1000000ULL
#define MSEC_PER_SEC 1000ULL
+#define USEC_PER_MS 1000ULL
+
#ifndef HAVE_CLOCK_GETTIME
/* Fallback function for POSIX.1-2001 clock_gettime() function.
*
diff --git a/src/common.c b/src/common.c
index 88fcf85bc..aa75c198d 100644
--- a/src/common.c
+++ b/src/common.c
@@ -226,6 +226,13 @@ void json_escape_string(char *dst, const char *src, size_t size) {
*d = '\0';
}
+void json_fix_string(char *s) {
+ for( ; *s ;s++) {
+ if(unlikely(*s == '\\')) *s = '/';
+ else if(unlikely(*s == '"')) *s = '\'';
+ }
+}
+
int sleep_usec(usec_t usec) {
#ifndef NETDATA_WITH_USLEEP
@@ -895,9 +902,8 @@ char *mystrsep(char **ptr, char *s) {
char *trim(char *s) {
// skip leading spaces
- // and 'comments' as well!?
while (*s && isspace(*s)) s++;
- if (!*s || *s == '#') return NULL;
+ if (!*s) return NULL;
// skip tailing spaces
// this way is way faster. Writes only one NUL char.
@@ -913,105 +919,163 @@ char *trim(char *s) {
return s;
}
-void *mymmap(const char *filename, size_t size, int flags, int ksm) {
-#ifndef MADV_MERGEABLE
- (void)ksm;
-#endif
- static int log_madvise_1 = 1;
-#ifdef MADV_MERGEABLE
- static int log_madvise_2 = 1, log_madvise_3 = 1;
-#endif
- void *mem = NULL;
+inline char *trim_all(char *buffer) {
+ char *d = buffer, *s = buffer;
+
+ // skip spaces
+ while(isspace(*s)) s++;
+
+ while(*s) {
+ // copy the non-space part
+ while(*s && !isspace(*s)) *d++ = *s++;
+
+ // add a space if we have to
+ if(*s && isspace(*s)) {
+ *d++ = ' ';
+ s++;
+ }
+
+ // skip spaces
+ while(isspace(*s)) s++;
+ }
+
+ *d = '\0';
+
+ if(d > buffer) {
+ d--;
+ if(isspace(*d)) *d = '\0';
+ }
+
+ if(!buffer[0]) return NULL;
+ return buffer;
+}
+
+static int memory_file_open(const char *filename, size_t size) {
+ // info("memory_file_open('%s', %zu", filename, size);
- errno = 0;
int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664);
if (fd != -1) {
if (lseek(fd, size, SEEK_SET) == (off_t) size) {
if (write(fd, "", 1) == 1) {
if (ftruncate(fd, size))
error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size);
+ }
+ else error("Cannot write to file '%s' at position %zu.", filename, size);
+ }
+ else error("Cannot seek file '%s' to size %zu.", filename, size);
+ }
+ else error("Cannot create/open file '%s'.", filename);
-#ifdef MADV_MERGEABLE
- if (flags & MAP_SHARED || !enable_ksm || !ksm) {
-#endif
- mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0);
- if (mem == MAP_FAILED) {
- error("Cannot allocate SHARED memory for file '%s'.", filename);
- mem = NULL;
- }
- else {
+ return fd;
+}
+
+// mmap_shared is used for memory mode = map
+static void *memory_file_mmap(const char *filename, size_t size, int flags) {
+ // info("memory_file_mmap('%s', %zu", filename, size);
+ static int log_madvise = 1;
+
+ int fd = -1;
+ if(filename) {
+ fd = memory_file_open(filename, size);
+ if(fd == -1) return MAP_FAILED;
+ }
+
+ void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0);
+ if (mem != MAP_FAILED) {
#ifdef NETDATA_LOG_ALLOCATIONS
- mmap_accounting(size);
+ mmap_accounting(size);
#endif
- int advise = MADV_SEQUENTIAL | MADV_DONTFORK;
- if (flags & MAP_SHARED) advise |= MADV_WILLNEED;
-
- if (madvise(mem, size, advise) != 0 && log_madvise_1) {
- error("Cannot advise the kernel about the memory usage of file '%s'.", filename);
- log_madvise_1--;
- }
- }
+ int advise = MADV_SEQUENTIAL | MADV_DONTFORK;
+ if (flags & MAP_SHARED) advise |= MADV_WILLNEED;
+
+ if (madvise(mem, size, advise) != 0 && log_madvise) {
+ error("Cannot advise the kernel about shared memory usage.");
+ log_madvise--;
+ }
+ }
+
+ if(fd != -1)
+ close(fd);
+
+ return mem;
+}
+
#ifdef MADV_MERGEABLE
- }
- else {
-/*
- // test - load the file into memory
- mem = calloc(1, size);
- if(mem) {
- if(lseek(fd, 0, SEEK_SET) == 0) {
- if(read(fd, mem, size) != (ssize_t)size)
- error("Cannot read from file '%s'", filename);
- }
- else
- error("Cannot seek to beginning of file '%s'.", filename);
- }
-*/
- mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0);
- if (mem == MAP_FAILED) {
- error("Cannot allocate PRIVATE ANONYMOUS memory for KSM for file '%s'.", filename);
- mem = NULL;
- }
- else {
+static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) {
+ // info("memory_file_mmap_ksm('%s', %zu", filename, size);
+ static int log_madvise_2 = 1, log_madvise_3 = 1;
+
+ int fd = -1;
+ if(filename) {
+ fd = memory_file_open(filename, size);
+ if(fd == -1) return MAP_FAILED;
+ }
+
+ void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0);
+ if (mem != MAP_FAILED) {
#ifdef NETDATA_LOG_ALLOCATIONS
- mmap_accounting(size);
-#endif
- if (lseek(fd, 0, SEEK_SET) == 0) {
- if (read(fd, mem, size) != (ssize_t) size)
- error("Cannot read from file '%s'", filename);
- } else
- error("Cannot seek to beginning of file '%s'.", filename);
-
- // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE
- if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) {
- error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.",
- filename);
- log_madvise_2--;
- }
-
- if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) {
- error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.",
- filename);
- log_madvise_3--;
- }
- }
- }
+ mmap_accounting(size);
#endif
+ if(fd != -1) {
+ if (lseek(fd, 0, SEEK_SET) == 0) {
+ if (read(fd, mem, size) != (ssize_t) size)
+ error("Cannot read from file '%s'", filename);
}
- else
- error("Cannot write to file '%s' at position %zu.", filename, size);
+ else error("Cannot seek to beginning of file '%s'.", filename);
}
- else
- error("Cannot seek file '%s' to size %zu.", filename, size);
- close(fd);
+ // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE
+ if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) {
+ error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.", filename);
+ log_madvise_2--;
+ }
+
+ if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) {
+ error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.", filename);
+ log_madvise_3--;
+ }
}
+
+ if(fd != -1)
+ close(fd);
+
+ return mem;
+}
+#else
+static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) {
+ // info("memory_file_mmap_ksm FALLBACK ('%s', %zu", filename, size);
+
+ if(filename)
+ return memory_file_mmap(filename, size, flags);
+
+ // when KSM is not available and no filename is given (memory mode = ram),
+ // we just report failure
+ return MAP_FAILED;
+}
+#endif
+
+void *mymmap(const char *filename, size_t size, int flags, int ksm) {
+ void *mem = NULL;
+
+ if (filename && (flags & MAP_SHARED || !enable_ksm || !ksm))
+ // memory mode = map | save
+ // when KSM is not enabled
+ // MAP_SHARED is used for memory mode = map (no KSM possible)
+ mem = memory_file_mmap(filename, size, flags);
+
else
- error("Cannot create/open file '%s'.", filename);
+ // memory mode = save | ram
+ // when KSM is enabled
+ // for memory mode = ram, the filename is NULL
+ mem = memory_file_mmap_ksm(filename, size, flags);
+ if(mem == MAP_FAILED) return NULL;
+
+ errno = 0;
return mem;
}
-int savememory(const char *filename, void *mem, size_t size) {
+int memory_file_save(const char *filename, void *mem, size_t size) {
char tmpfilename[FILENAME_MAX + 1];
snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid());
@@ -1228,3 +1292,47 @@ unsigned long end_tsc(void) {
return (((unsigned long)d << 32) | (unsigned long)a) - tsc;
}
*/
+
+int recursively_delete_dir(const char *path, const char *reason) {
+ DIR *dir = opendir(path);
+ if(!dir) {
+ error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path);
+ return -1;
+ }
+
+ int ret = 0;
+ struct dirent *de = NULL;
+ while((de = readdir(dir))) {
+ if(de->d_type == DT_DIR
+ && (
+ (de->d_name[0] == '.' && de->d_name[1] == '\0')
+ || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ ))
+ continue;
+
+ char fullpath[FILENAME_MAX + 1];
+ snprintfz(fullpath, FILENAME_MAX, "%s/%s", path, de->d_name);
+
+ if(de->d_type == DT_DIR) {
+ int r = recursively_delete_dir(fullpath, reason);
+ if(r > 0) ret += r;
+ continue;
+ }
+
+ info("Deleting %s file '%s'", reason?reason:"", fullpath);
+ if(unlikely(unlink(fullpath) == -1))
+ error("Cannot delete %s file '%s'", reason?reason:"", fullpath);
+ else
+ ret++;
+ }
+
+ info("Deleting empty directory '%s'", path);
+ if(unlikely(rmdir(path) == -1))
+ error("Cannot delete empty directory '%s'", path);
+ else
+ ret++;
+
+ closedir(dir);
+
+ return ret;
+}
diff --git a/src/common.h b/src/common.h
index b82c078fa..efeebf16f 100644
--- a/src/common.h
+++ b/src/common.h
@@ -40,6 +40,7 @@
#include <strings.h>
#include <arpa/inet.h>
#include <netinet/tcp.h>
+#include <sys/ioctl.h>
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
@@ -201,12 +202,14 @@
#define NETDATA_OS_TYPE "linux"
#endif /* __FreeBSD__, __APPLE__*/
+#include "statistical.h"
#include "socket.h"
#include "eval.h"
#include "health.h"
#include "rrd.h"
#include "plugin_tc.h"
#include "plugins_d.h"
+#include "statsd.h"
#include "rrd2json.h"
#include "rrd2json_api_old.h"
#include "web_client.h"
@@ -217,6 +220,7 @@
#include "unit_test.h"
#include "ipc.h"
#include "backends.h"
+#include "backend_prometheus.h"
#include "inlined.h"
#include "adaptive_resortable_list.h"
#include "rrdpush.h"
@@ -238,7 +242,8 @@ extern void netdata_fix_chart_name(char *s);
extern void strreverse(char* begin, char* end);
extern char *mystrsep(char **ptr, char *s);
-extern char *trim(char *s);
+extern char *trim(char *s); // remove leading and trailing spaces; may return NULL
+extern char *trim_all(char *buffer); // like trim(), but also remove duplicate spaces inside the string; may return NULL
extern int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args);
extern int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4);
@@ -265,9 +270,10 @@ extern void freez(void *ptr);
#endif
extern void json_escape_string(char *dst, const char *src, size_t size);
+extern void json_fix_string(char *s);
extern void *mymmap(const char *filename, size_t size, int flags, int ksm);
-extern int savememory(const char *filename, void *mem, size_t size);
+extern int memory_file_save(const char *filename, void *mem, size_t size);
extern int fd_is_valid(int fd);
@@ -289,6 +295,8 @@ extern pid_t get_system_pid_max(void);
extern unsigned int hz;
extern void get_system_HZ(void);
+extern int recursively_delete_dir(const char *path, const char *reason);
+
extern volatile sig_atomic_t netdata_exit;
extern const char *os_type;
diff --git a/src/daemon.c b/src/daemon.c
index 42b04c401..bc02446e0 100644
--- a/src/daemon.c
+++ b/src/daemon.c
@@ -73,8 +73,9 @@ void create_needed_dir(const char *dir, uid_t uid, gid_t gid)
error("Cannot create directory '%s'", dir);
}
-int become_user(const char *username, int pid_fd)
-{
+int become_user(const char *username, int pid_fd) {
+ int am_i_root = (getuid() == 0)?1:0;
+
struct passwd *pw = getpwnam(username);
if(!pw) {
error("User %s is not present.", username);
@@ -94,12 +95,12 @@ int become_user(const char *username, int pid_fd)
int ngroups = (int)sysconf(_SC_NGROUPS_MAX);
gid_t *supplementary_groups = NULL;
- if(ngroups) {
+ if(ngroups > 0) {
supplementary_groups = mallocz(sizeof(gid_t) * ngroups);
if(getgrouplist(username, gid, supplementary_groups, &ngroups) == -1) {
- error("Cannot get supplementary groups of user '%s'.", username);
- freez(supplementary_groups);
- supplementary_groups = NULL;
+ if(am_i_root)
+ error("Cannot get supplementary groups of user '%s'.", username);
+
ngroups = 0;
}
}
@@ -109,14 +110,17 @@ int become_user(const char *username, int pid_fd)
chown_open_file(stdaccess_fd, uid, gid);
chown_open_file(pid_fd, uid, gid);
- if(supplementary_groups && ngroups) {
- if(setgroups(ngroups, supplementary_groups) == -1)
- error("Cannot set supplementary groups for user '%s'", username);
-
- freez(supplementary_groups);
+ if(supplementary_groups && ngroups > 0) {
+ if(setgroups((size_t)ngroups, supplementary_groups) == -1) {
+ if(am_i_root)
+ error("Cannot set supplementary groups for user '%s'", username);
+ }
ngroups = 0;
}
+ if(supplementary_groups)
+ freez(supplementary_groups);
+
#ifdef __APPLE__
if(setregid(gid, gid) != 0) {
#else
@@ -155,22 +159,42 @@ int become_user(const char *username, int pid_fd)
return(0);
}
+#ifndef OOM_SCORE_ADJ_MAX
+#define OOM_SCORE_ADJ_MAX 1000
+#endif
+#ifndef OOM_SCORE_ADJ_MIN
+#define OOM_SCORE_ADJ_MIN -1000
+#endif
+
static void oom_score_adj(void) {
- int score = (int)config_get_number(CONFIG_SECTION_GLOBAL, "OOM score", 1000);
+ char buf[10 + 1];
+ snprintfz(buf, 10, "%d", OOM_SCORE_ADJ_MAX);
+
+ // check the environment
+ char *s = getenv("OOMScoreAdjust");
+ if(!s || !*s) s = buf;
+
+ // check netdata.conf configuration
+ s = config_get(CONFIG_SECTION_GLOBAL, "OOM score", s);
+ if(!s || !*s) s = buf;
+
+ if(!isdigit(*s) && *s != '-' && *s != '+') {
+ info("Out-Of-Memory score not changed due to setting: '%s'", s);
+ return;
+ }
int done = 0;
int fd = open("/proc/self/oom_score_adj", O_WRONLY);
if(fd != -1) {
- char buf[10 + 1];
- ssize_t len = snprintfz(buf, 10, "%d", score);
+ ssize_t len = strlen(s);
if(len > 0 && write(fd, buf, (size_t)len) == len) done = 1;
close(fd);
}
if(!done)
- error("Cannot adjust my Out-Of-Memory score to %d.", score);
+ error("Cannot adjust my Out-Of-Memory score to '%s'.", s);
else
- debug(D_SYSTEM, "Adjusted my Out-Of-Memory score to %d.", score);
+ info("Adjusted my Out-Of-Memory score to '%s'.", s);
}
static void process_nice_level(void) {
diff --git a/src/eval.c b/src/eval.c
index 122959ce4..9248109b0 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -723,7 +723,7 @@ static inline int parse_variable(const char **string, char *buffer, size_t len)
static inline int parse_constant(const char **string, calculated_number *number) {
char *end = NULL;
- calculated_number n = strtold(*string, &end);
+ calculated_number n = str2ld(*string, &end);
if(unlikely(!end || *string == end)) {
*number = 0;
return 0;
diff --git a/src/freebsd_devstat.c b/src/freebsd_devstat.c
new file mode 100644
index 000000000..5b0687d5b
--- /dev/null
+++ b/src/freebsd_devstat.c
@@ -0,0 +1,662 @@
+#include "common.h"
+
+#include <sys/devicestat.h>
+
+struct disk {
+ char *name;
+ uint32_t hash;
+ size_t len;
+
+ // flags
+ int configured;
+ int enabled;
+ int updated;
+
+ int do_io;
+ int do_ops;
+ int do_qops;
+ int do_util;
+ int do_iotime;
+ int do_await;
+ int do_avagsz;
+ int do_svctm;
+
+
+ // data for differential charts
+
+ struct prev_dstat {
+ collected_number bytes_read;
+ collected_number bytes_write;
+ collected_number operations_read;
+ collected_number operations_write;
+ collected_number duration_read_ms;
+ collected_number duration_write_ms;
+ collected_number busy_time_ms;
+ } prev_dstat;
+
+ // charts and dimensions
+
+ RRDSET *st_io;
+ RRDDIM *rd_io_in;
+ RRDDIM *rd_io_out;
+
+ RRDSET *st_ops;
+ RRDDIM *rd_ops_in;
+ RRDDIM *rd_ops_out;
+
+ RRDSET *st_qops;
+ RRDDIM *rd_qops;
+
+ RRDSET *st_util;
+ RRDDIM *rd_util;
+
+ RRDSET *st_iotime;
+ RRDDIM *rd_iotime_in;
+ RRDDIM *rd_iotime_out;
+
+ RRDSET *st_await;
+ RRDDIM *rd_await_in;
+ RRDDIM *rd_await_out;
+
+ RRDSET *st_avagsz;
+ RRDDIM *rd_avagsz_in;
+ RRDDIM *rd_avagsz_out;
+
+ RRDSET *st_svctm;
+ RRDDIM *rd_svctm;
+
+ struct disk *next;
+};
+
+static struct disk *disks_root = NULL, *disks_last_used = NULL;
+
+static size_t disks_added = 0, disks_found = 0;
+
+static void disk_free(struct disk *dm) {
+ if (likely(dm->st_io))
+ rrdset_is_obsolete(dm->st_io);
+ if (likely(dm->st_ops))
+ rrdset_is_obsolete(dm->st_ops);
+ if (likely(dm->st_qops))
+ rrdset_is_obsolete(dm->st_qops);
+ if (likely(dm->st_util))
+ rrdset_is_obsolete(dm->st_util);
+ if (likely(dm->st_iotime))
+ rrdset_is_obsolete(dm->st_iotime);
+ if (likely(dm->st_await))
+ rrdset_is_obsolete(dm->st_await);
+ if (likely(dm->st_avagsz))
+ rrdset_is_obsolete(dm->st_avagsz);
+ if (likely(dm->st_svctm))
+ rrdset_is_obsolete(dm->st_svctm);
+
+ disks_added--;
+ freez(dm->name);
+ freez(dm);
+}
+
+static void disks_cleanup() {
+ if (likely(disks_found == disks_added)) return;
+
+ struct disk *dm = disks_root, *last = NULL;
+ while(dm) {
+ if (unlikely(!dm->updated)) {
+ // info("Removing disk '%s', linked after '%s'", dm->name, last?last->name:"ROOT");
+
+ if (disks_last_used == dm)
+ disks_last_used = last;
+
+ struct disk *t = dm;
+
+ if (dm == disks_root || !last)
+ disks_root = dm = dm->next;
+
+ else
+ last->next = dm = dm->next;
+
+ t->next = NULL;
+ disk_free(t);
+ }
+ else {
+ last = dm;
+ dm->updated = 0;
+ dm = dm->next;
+ }
+ }
+}
+
+static struct disk *get_disk(const char *name) {
+ struct disk *dm;
+
+ uint32_t hash = simple_hash(name);
+
+ // search it, from the last position to the end
+ for(dm = disks_last_used ; dm ; dm = dm->next) {
+ if (unlikely(hash == dm->hash && !strcmp(name, dm->name))) {
+ disks_last_used = dm->next;
+ return dm;
+ }
+ }
+
+ // search it from the beginning to the last position we used
+ for(dm = disks_root ; dm != disks_last_used ; dm = dm->next) {
+ if (unlikely(hash == dm->hash && !strcmp(name, dm->name))) {
+ disks_last_used = dm->next;
+ return dm;
+ }
+ }
+
+ // create a new one
+ dm = callocz(1, sizeof(struct disk));
+ dm->name = strdupz(name);
+ dm->hash = simple_hash(dm->name);
+ dm->len = strlen(dm->name);
+ disks_added++;
+
+ // link it to the end
+ if (disks_root) {
+ struct disk *e;
+ for(e = disks_root; e->next ; e = e->next) ;
+ e->next = dm;
+ }
+ else
+ disks_root = dm;
+
+ return dm;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// kern.devstat
+
+int do_kern_devstat(int update_every, usec_t dt) {
+
+#define DELAULT_EXLUDED_DISKS ""
+#define CONFIG_SECTION_KERN_DEVSTAT "plugin:freebsd:kern.devstat"
+#define BINTIME_SCALE 5.42101086242752217003726400434970855712890625e-17 // this is 1000/2^64
+
+ static int enable_new_disks = -1;
+ static int enable_pass_devices = -1, do_system_io = -1, do_io = -1, do_ops = -1, do_qops = -1, do_util = -1,
+ do_iotime = -1, do_await = -1, do_avagsz = -1, do_svctm = -1;
+ static SIMPLE_PATTERN *excluded_disks = NULL;
+
+ if (unlikely(enable_new_disks == -1)) {
+ enable_new_disks = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT,
+ "enable new disks detected at runtime", CONFIG_BOOLEAN_AUTO);
+
+ enable_pass_devices = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT,
+ "performance metrics for pass devices", CONFIG_BOOLEAN_AUTO);
+
+ do_system_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "total bandwidth for all disks",
+ CONFIG_BOOLEAN_YES);
+
+ do_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "bandwidth for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_ops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "operations for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_qops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "queued operations for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_util = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "utilization percentage for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_iotime = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "i/o time for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_await = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o time for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_avagsz = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o bandwidth for all disks",
+ CONFIG_BOOLEAN_AUTO);
+ do_svctm = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average service time for all disks",
+ CONFIG_BOOLEAN_AUTO);
+
+ excluded_disks = simple_pattern_create(
+ config_get(CONFIG_SECTION_KERN_DEVSTAT, "disable by default disks matching", DELAULT_EXLUDED_DISKS)
+ , SIMPLE_PATTERN_EXACT
+ );
+ }
+
+ if (likely(do_system_io || do_io || do_ops || do_qops || do_util || do_iotime || do_await || do_avagsz || do_svctm)) {
+ static int mib_numdevs[3] = {0, 0, 0};
+ int numdevs;
+ int common_error = 0;
+
+ if (unlikely(GETSYSCTL_SIMPLE("kern.devstat.numdevs", mib_numdevs, numdevs))) {
+ common_error = 1;
+ } else {
+ static int mib_devstat[3] = {0, 0, 0};
+ static void *devstat_data = NULL;
+ static int old_numdevs = 0;
+
+ if (unlikely(numdevs != old_numdevs)) {
+ devstat_data = reallocz(devstat_data, sizeof(long) + sizeof(struct devstat) *
+ numdevs); // there is generation number before devstat structures
+ old_numdevs = numdevs;
+ }
+ if (unlikely(GETSYSCTL_WSIZE("kern.devstat.all", mib_devstat, devstat_data,
+ sizeof(long) + sizeof(struct devstat) * numdevs))) {
+ common_error = 1;
+ } else {
+ struct devstat *dstat;
+ int i;
+ collected_number total_disk_kbytes_read = 0;
+ collected_number total_disk_kbytes_write = 0;
+
+ disks_found = 0;
+
+ dstat = devstat_data + sizeof(long); // skip generation number
+
+ for (i = 0; i < numdevs; i++) {
+ if (likely(do_system_io)) {
+ if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) {
+ total_disk_kbytes_read += dstat[i].bytes[DEVSTAT_READ] / KILO_FACTOR;
+ total_disk_kbytes_write += dstat[i].bytes[DEVSTAT_WRITE] / KILO_FACTOR;
+ }
+ }
+
+ if (unlikely(!enable_pass_devices))
+ if ((dstat[i].device_type & DEVSTAT_TYPE_PASS) == DEVSTAT_TYPE_PASS)
+ continue;
+
+ if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) {
+ char disk[DEVSTAT_NAME_LEN + MAX_INT_DIGITS + 1];
+ struct cur_dstat {
+ collected_number duration_read_ms;
+ collected_number duration_write_ms;
+ collected_number busy_time_ms;
+ } cur_dstat;
+
+ sprintf(disk, "%s%d", dstat[i].device_name, dstat[i].unit_number);
+
+ struct disk *dm = get_disk(disk);
+ dm->updated = 1;
+ disks_found++;
+
+ if(unlikely(!dm->configured)) {
+ char var_name[4096 + 1];
+
+ // this is the first time we see this disk
+
+ // remember we configured it
+ dm->configured = 1;
+
+ dm->enabled = enable_new_disks;
+
+ if (likely(dm->enabled))
+ dm->enabled = !simple_pattern_matches(excluded_disks, disk);
+
+ snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_KERN_DEVSTAT, disk);
+ dm->enabled = config_get_boolean_ondemand(var_name, "enabled", dm->enabled);
+
+ dm->do_io = config_get_boolean_ondemand(var_name, "bandwidth", do_io);
+ dm->do_ops = config_get_boolean_ondemand(var_name, "operations", do_ops);
+ dm->do_qops = config_get_boolean_ondemand(var_name, "queued operations", do_qops);
+ dm->do_util = config_get_boolean_ondemand(var_name, "utilization percentage", do_util);
+ dm->do_iotime = config_get_boolean_ondemand(var_name, "i/o time", do_iotime);
+ dm->do_await = config_get_boolean_ondemand(var_name, "average completed i/o time",
+ do_await);
+ dm->do_avagsz = config_get_boolean_ondemand(var_name, "average completed i/o bandwidth",
+ do_avagsz);
+ dm->do_svctm = config_get_boolean_ondemand(var_name, "average service time", do_svctm);
+
+ // initialise data for differential charts
+
+ dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ];
+ dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE];
+ dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ];
+ dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE];
+ dm->prev_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000
+ + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
+ dm->prev_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000
+ + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
+ dm->prev_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000
+ + dstat[i].busy_time.frac * BINTIME_SCALE;
+ }
+
+ cur_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000
+ + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
+ cur_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000
+ + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
+ cur_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE;
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_io == CONFIG_BOOLEAN_YES || (dm->do_io == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].bytes[DEVSTAT_READ] || dstat[i].bytes[DEVSTAT_WRITE]))) {
+ if (unlikely(!dm->st_io)) {
+ dm->st_io = rrdset_create_localhost("disk",
+ disk,
+ NULL,
+ disk,
+ "disk.io",
+ "Disk I/O Bandwidth",
+ "kilobytes/s",
+ 2000,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ dm->rd_io_in = rrddim_add(dm->st_io, "reads", NULL, 1, KILO_FACTOR,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_io_out = rrddim_add(dm->st_io, "writes", NULL, -1, KILO_FACTOR,
+ RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(dm->st_io);
+
+ rrddim_set_by_pointer(dm->st_io, dm->rd_io_in, dstat[i].bytes[DEVSTAT_READ]);
+ rrddim_set_by_pointer(dm->st_io, dm->rd_io_out, dstat[i].bytes[DEVSTAT_WRITE]);
+ rrdset_done(dm->st_io);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_ops == CONFIG_BOOLEAN_YES || (dm->do_ops == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
+ if (unlikely(!dm->st_ops)) {
+ dm->st_ops = rrdset_create_localhost("disk_ops",
+ disk,
+ NULL,
+ disk,
+ "disk.ops",
+ "Disk Completed I/O Operations",
+ "operations/s",
+ 2001,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_ops, RRDSET_FLAG_DETAIL);
+
+ dm->rd_ops_in = rrddim_add(dm->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_ops_out = rrddim_add(dm->st_ops, "writes", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(dm->st_ops);
+
+ rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_in, dstat[i].operations[DEVSTAT_READ]);
+ rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_out, dstat[i].operations[DEVSTAT_WRITE]);
+ rrdset_done(dm->st_ops);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_qops == CONFIG_BOOLEAN_YES || (dm->do_qops == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].start_count || dstat[i].end_count))) {
+ if (unlikely(!dm->st_qops)) {
+ dm->st_qops = rrdset_create_localhost("disk_qops",
+ disk,
+ NULL,
+ disk,
+ "disk.qops",
+ "Disk Current I/O Operations",
+ "operations",
+ 2002,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_qops, RRDSET_FLAG_DETAIL);
+
+ dm->rd_qops = rrddim_add(dm->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ } else
+ rrdset_next(dm->st_qops);
+
+ rrddim_set_by_pointer(dm->st_qops, dm->rd_qops, dstat[i].start_count - dstat[i].end_count);
+ rrdset_done(dm->st_qops);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_util == CONFIG_BOOLEAN_YES || (dm->do_util == CONFIG_BOOLEAN_AUTO &&
+ cur_dstat.busy_time_ms)) {
+ if (unlikely(!dm->st_util)) {
+ dm->st_util = rrdset_create_localhost("disk_util",
+ disk,
+ NULL,
+ disk,
+ "disk.util",
+ "Disk Utilization Time",
+ "% of time working",
+ 2004,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(dm->st_util, RRDSET_FLAG_DETAIL);
+
+ dm->rd_util = rrddim_add(dm->st_util, "utilization", NULL, 1, 10,
+ RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(dm->st_util);
+
+ rrddim_set_by_pointer(dm->st_util, dm->rd_util, cur_dstat.busy_time_ms);
+ rrdset_done(dm->st_util);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_iotime == CONFIG_BOOLEAN_YES || (dm->do_iotime == CONFIG_BOOLEAN_AUTO &&
+ (cur_dstat.duration_read_ms || cur_dstat.duration_write_ms))) {
+ if (unlikely(!dm->st_iotime)) {
+ dm->st_iotime = rrdset_create_localhost("disk_iotime",
+ disk,
+ NULL,
+ disk,
+ "disk.iotime",
+ "Disk Total I/O Time",
+ "milliseconds/s",
+ 2022,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_iotime, RRDSET_FLAG_DETAIL);
+
+ dm->rd_iotime_in = rrddim_add(dm->st_iotime, "reads", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ dm->rd_iotime_out = rrddim_add(dm->st_iotime, "writes", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(dm->st_iotime);
+
+ rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_in, cur_dstat.duration_read_ms);
+ rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_out, cur_dstat.duration_write_ms);
+ rrdset_done(dm->st_iotime);
+ }
+
+ // --------------------------------------------------------------------
+ // calculate differential charts
+ // only if this is not the first time we run
+
+ if (likely(dt)) {
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_await == CONFIG_BOOLEAN_YES || (dm->do_await == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
+ if (unlikely(!dm->st_await)) {
+ dm->st_await = rrdset_create_localhost("disk_await",
+ disk,
+ NULL,
+ disk,
+ "disk.await",
+ "Average Completed I/O Operation Time",
+ "ms per operation",
+ 2005,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_await, RRDSET_FLAG_DETAIL);
+
+ dm->rd_await_in = rrddim_add(dm->st_await, "reads", NULL, 1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ dm->rd_await_out = rrddim_add(dm->st_await, "writes", NULL, -1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ } else
+ rrdset_next(dm->st_await);
+
+ rrddim_set_by_pointer(dm->st_await, dm->rd_await_in,
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) ?
+ (cur_dstat.duration_read_ms - dm->prev_dstat.duration_read_ms) /
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) :
+ 0);
+ rrddim_set_by_pointer(dm->st_await, dm->rd_await_out,
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) ?
+ (cur_dstat.duration_write_ms - dm->prev_dstat.duration_write_ms) /
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) :
+ 0);
+ rrdset_done(dm->st_await);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_avagsz == CONFIG_BOOLEAN_YES || (dm->do_avagsz == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
+ if (unlikely(!dm->st_avagsz)) {
+ dm->st_avagsz = rrdset_create_localhost("disk_avgsz",
+ disk,
+ NULL,
+ disk,
+ "disk.avgsz",
+ "Average Completed I/O Operation Bandwidth",
+ "kilobytes per operation",
+ 2006,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rrdset_flag_set(dm->st_avagsz, RRDSET_FLAG_DETAIL);
+
+ dm->rd_avagsz_in = rrddim_add(dm->st_avagsz, "reads", NULL, 1, KILO_FACTOR,
+ RRD_ALGORITHM_ABSOLUTE);
+ dm->rd_avagsz_out = rrddim_add(dm->st_avagsz, "writes", NULL, -1, KILO_FACTOR,
+ RRD_ALGORITHM_ABSOLUTE);
+ } else
+ rrdset_next(dm->st_avagsz);
+
+ rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_in,
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) ?
+ (dstat[i].bytes[DEVSTAT_READ] - dm->prev_dstat.bytes_read) /
+ (dstat[i].operations[DEVSTAT_READ] -
+ dm->prev_dstat.operations_read) :
+ 0);
+ rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_out,
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) ?
+ (dstat[i].bytes[DEVSTAT_WRITE] - dm->prev_dstat.bytes_write) /
+ (dstat[i].operations[DEVSTAT_WRITE] -
+ dm->prev_dstat.operations_write) :
+ 0);
+ rrdset_done(dm->st_avagsz);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(dm->do_svctm == CONFIG_BOOLEAN_YES || (dm->do_svctm == CONFIG_BOOLEAN_AUTO &&
+ (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
+ if (unlikely(!dm->st_svctm)) {
+ dm->st_svctm = rrdset_create_localhost("disk_svctm",
+ disk,
+ NULL,
+ disk,
+ "disk.svctm",
+ "Average Service Time",
+ "ms per operation",
+ 2007,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(dm->st_svctm, RRDSET_FLAG_DETAIL);
+
+ dm->rd_svctm = rrddim_add(dm->st_svctm, "svctm", NULL, 1, 1,
+ RRD_ALGORITHM_ABSOLUTE);
+ } else
+ rrdset_next(dm->st_svctm);
+
+ rrddim_set_by_pointer(dm->st_svctm, dm->rd_svctm,
+ ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) +
+ (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) ?
+ (cur_dstat.busy_time_ms - dm->prev_dstat.busy_time_ms) /
+ ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) +
+ (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) :
+ 0);
+ rrdset_done(dm->st_svctm);
+ }
+
+ // --------------------------------------------------------------------
+
+ dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ];
+ dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE];
+ dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ];
+ dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE];
+ dm->prev_dstat.duration_read_ms = cur_dstat.duration_read_ms;
+ dm->prev_dstat.duration_write_ms = cur_dstat.duration_write_ms;
+ dm->prev_dstat.busy_time_ms = cur_dstat.busy_time_ms;
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ if (likely(do_system_io)) {
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "io",
+ NULL,
+ "disk",
+ NULL,
+ "Disk I/O",
+ "kilobytes/s",
+ 150,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(st);
+
+ rrddim_set_by_pointer(st, rd_in, total_disk_kbytes_read);
+ rrddim_set_by_pointer(st, rd_out, total_disk_kbytes_write);
+ rrdset_done(st);
+ }
+ }
+ }
+ if (unlikely(common_error)) {
+ do_system_io = 0;
+ error("DISABLED: system.io chart");
+ do_io = 0;
+ error("DISABLED: disk.* charts");
+ do_ops = 0;
+ error("DISABLED: disk_ops.* charts");
+ do_qops = 0;
+ error("DISABLED: disk_qops.* charts");
+ do_util = 0;
+ error("DISABLED: disk_util.* charts");
+ do_iotime = 0;
+ error("DISABLED: disk_iotime.* charts");
+ do_await = 0;
+ error("DISABLED: disk_await.* charts");
+ do_avagsz = 0;
+ error("DISABLED: disk_avgsz.* charts");
+ do_svctm = 0;
+ error("DISABLED: disk_svctm.* charts");
+ error("DISABLED: kern.devstat module");
+ return 1;
+ }
+ } else {
+ error("DISABLED: kern.devstat module");
+ return 1;
+ }
+
+ disks_cleanup();
+
+ return 0;
+}
diff --git a/src/freebsd_getifaddrs.c b/src/freebsd_getifaddrs.c
new file mode 100644
index 000000000..7355fac9e
--- /dev/null
+++ b/src/freebsd_getifaddrs.c
@@ -0,0 +1,494 @@
+#include "common.h"
+
+#include <ifaddrs.h>
+
+struct network_interface {
+ char *name;
+ uint32_t hash;
+ size_t len;
+
+ // flags
+ int configured;
+ int enabled;
+ int updated;
+
+ int do_bandwidth;
+ int do_packets;
+ int do_errors;
+ int do_drops;
+ int do_events;
+
+ // charts and dimensions
+
+ RRDSET *st_bandwidth;
+ RRDDIM *rd_bandwidth_in;
+ RRDDIM *rd_bandwidth_out;
+
+ RRDSET *st_packets;
+ RRDDIM *rd_packets_in;
+ RRDDIM *rd_packets_out;
+ RRDDIM *rd_packets_m_in;
+ RRDDIM *rd_packets_m_out;
+
+ RRDSET *st_errors;
+ RRDDIM *rd_errors_in;
+ RRDDIM *rd_errors_out;
+
+ RRDSET *st_drops;
+ RRDDIM *rd_drops_in;
+ RRDDIM *rd_drops_out;
+
+ RRDSET *st_events;
+ RRDDIM *rd_events_coll;
+
+ struct network_interface *next;
+};
+
+static struct network_interface *network_interfaces_root = NULL, *network_interfaces_last_used = NULL;
+
+static size_t network_interfaces_added = 0, network_interfaces_found = 0;
+
+static void network_interface_free(struct network_interface *ifm) {
+ if (likely(ifm->st_bandwidth))
+ rrdset_is_obsolete(ifm->st_bandwidth);
+ if (likely(ifm->st_packets))
+ rrdset_is_obsolete(ifm->st_packets);
+ if (likely(ifm->st_errors))
+ rrdset_is_obsolete(ifm->st_errors);
+ if (likely(ifm->st_drops))
+ rrdset_is_obsolete(ifm->st_drops);
+ if (likely(ifm->st_events))
+ rrdset_is_obsolete(ifm->st_events);
+
+ network_interfaces_added--;
+ freez(ifm->name);
+ freez(ifm);
+}
+
+static void network_interfaces_cleanup() {
+ if (likely(network_interfaces_found == network_interfaces_added)) return;
+
+ struct network_interface *ifm = network_interfaces_root, *last = NULL;
+ while(ifm) {
+ if (unlikely(!ifm->updated)) {
+ // info("Removing network interface '%s', linked after '%s'", ifm->name, last?last->name:"ROOT");
+
+ if (network_interfaces_last_used == ifm)
+ network_interfaces_last_used = last;
+
+ struct network_interface *t = ifm;
+
+ if (ifm == network_interfaces_root || !last)
+ network_interfaces_root = ifm = ifm->next;
+
+ else
+ last->next = ifm = ifm->next;
+
+ t->next = NULL;
+ network_interface_free(t);
+ }
+ else {
+ last = ifm;
+ ifm->updated = 0;
+ ifm = ifm->next;
+ }
+ }
+}
+
+static struct network_interface *get_network_interface(const char *name) {
+ struct network_interface *ifm;
+
+ uint32_t hash = simple_hash(name);
+
+ // search it, from the last position to the end
+ for(ifm = network_interfaces_last_used ; ifm ; ifm = ifm->next) {
+ if (unlikely(hash == ifm->hash && !strcmp(name, ifm->name))) {
+ network_interfaces_last_used = ifm->next;
+ return ifm;
+ }
+ }
+
+ // search it from the beginning to the last position we used
+ for(ifm = network_interfaces_root ; ifm != network_interfaces_last_used ; ifm = ifm->next) {
+ if (unlikely(hash == ifm->hash && !strcmp(name, ifm->name))) {
+ network_interfaces_last_used = ifm->next;
+ return ifm;
+ }
+ }
+
+ // create a new one
+ ifm = callocz(1, sizeof(struct network_interface));
+ ifm->name = strdupz(name);
+ ifm->hash = simple_hash(ifm->name);
+ ifm->len = strlen(ifm->name);
+ network_interfaces_added++;
+
+ // link it to the end
+ if (network_interfaces_root) {
+ struct network_interface *e;
+ for(e = network_interfaces_root; e->next ; e = e->next) ;
+ e->next = ifm;
+ }
+ else
+ network_interfaces_root = ifm;
+
+ return ifm;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// getifaddrs
+
+int do_getifaddrs(int update_every, usec_t dt) {
+ (void)dt;
+
+#define DELAULT_EXLUDED_INTERFACES "lo*"
+#define CONFIG_SECTION_GETIFADDRS "plugin:freebsd:getifaddrs"
+
+ static int enable_new_interfaces = -1;
+ static int do_bandwidth_ipv4 = -1, do_bandwidth_ipv6 = -1, do_bandwidth = -1, do_packets = -1,
+ do_errors = -1, do_drops = -1, do_events = -1;
+ static SIMPLE_PATTERN *excluded_interfaces = NULL;
+
+ if (unlikely(enable_new_interfaces == -1)) {
+ enable_new_interfaces = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS,
+ "enable new interfaces detected at runtime",
+ CONFIG_BOOLEAN_AUTO);
+
+ do_bandwidth_ipv4 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv4 interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_bandwidth_ipv6 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv6 interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_bandwidth = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "bandwidth for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_packets = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "packets for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_errors = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "errors for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_drops = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "drops for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+ do_events = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "collisions for all interfaces",
+ CONFIG_BOOLEAN_AUTO);
+
+ excluded_interfaces = simple_pattern_create(
+ config_get(CONFIG_SECTION_GETIFADDRS, "disable by default interfaces matching",
+ DELAULT_EXLUDED_INTERFACES)
+ , SIMPLE_PATTERN_EXACT
+ );
+ }
+
+ if (likely(do_bandwidth_ipv4 || do_bandwidth_ipv6 || do_bandwidth || do_packets || do_errors ||
+ do_drops || do_events)) {
+ struct ifaddrs *ifap;
+
+ if (unlikely(getifaddrs(&ifap))) {
+ error("FREEBSD: getifaddrs() failed");
+ do_bandwidth_ipv4 = 0;
+ error("DISABLED: system.ipv4 chart");
+ do_bandwidth_ipv6 = 0;
+ error("DISABLED: system.ipv6 chart");
+ do_bandwidth = 0;
+ error("DISABLED: net.* charts");
+ do_packets = 0;
+ error("DISABLED: net_packets.* charts");
+ do_errors = 0;
+ error("DISABLED: net_errors.* charts");
+ do_drops = 0;
+ error("DISABLED: net_drops.* charts");
+ do_events = 0;
+ error("DISABLED: net_events.* charts");
+ error("DISABLED: getifaddrs module");
+ return 1;
+ } else {
+#define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s)
+ struct ifaddrs *ifa;
+ struct iftot {
+ u_long ift_ibytes;
+ u_long ift_obytes;
+ } iftot = {0, 0};
+
+ // --------------------------------------------------------------------
+
+ if (likely(do_bandwidth_ipv4)) {
+ iftot.ift_ibytes = iftot.ift_obytes = 0;
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_INET)
+ continue;
+ iftot.ift_ibytes += IFA_DATA(ibytes);
+ iftot.ift_obytes += IFA_DATA(obytes);
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "ipv4",
+ NULL,
+ "network",
+ NULL,
+ "IPv4 Bandwidth",
+ "kilobits/s",
+ 500,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "InOctets", "received", 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "OutOctets", "sent", -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(st);
+
+ rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes);
+ rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ if (likely(do_bandwidth_ipv6)) {
+ iftot.ift_ibytes = iftot.ift_obytes = 0;
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_INET6)
+ continue;
+ iftot.ift_ibytes += IFA_DATA(ibytes);
+ iftot.ift_obytes += IFA_DATA(obytes);
+ }
+
+ static RRDSET *st = NULL;
+ static RRDDIM *rd_in = NULL, *rd_out = NULL;
+
+ if (unlikely(!st)) {
+ st = rrdset_create_localhost("system",
+ "ipv6",
+ NULL,
+ "network",
+ NULL,
+ "IPv6 Bandwidth",
+ "kilobits/s",
+ 500,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ rd_in = rrddim_add(st, "received", NULL, 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
+ rd_out = rrddim_add(st, "sent", NULL, -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(st);
+
+ rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes);
+ rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes);
+ rrdset_done(st);
+ }
+
+ // --------------------------------------------------------------------
+
+ network_interfaces_found = 0;
+
+ for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr->sa_family != AF_LINK)
+ continue;
+
+ struct network_interface *ifm = get_network_interface(ifa->ifa_name);
+ ifm->updated = 1;
+ network_interfaces_found++;
+
+ if (unlikely(!ifm->configured)) {
+ char var_name[4096 + 1];
+
+ // this is the first time we see this network interface
+
+ // remember we configured it
+ ifm->configured = 1;
+
+ ifm->enabled = enable_new_interfaces;
+
+ if (likely(ifm->enabled))
+ ifm->enabled = !simple_pattern_matches(excluded_interfaces, ifa->ifa_name);
+
+ snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETIFADDRS, ifa->ifa_name);
+ ifm->enabled = config_get_boolean_ondemand(var_name, "enabled", ifm->enabled);
+
+ if (unlikely(ifm->enabled == CONFIG_BOOLEAN_NO))
+ continue;
+
+ ifm->do_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", do_bandwidth);
+ ifm->do_packets = config_get_boolean_ondemand(var_name, "packets", do_packets);
+ ifm->do_errors = config_get_boolean_ondemand(var_name, "errors", do_errors);
+ ifm->do_drops = config_get_boolean_ondemand(var_name, "drops", do_drops);
+ ifm->do_events = config_get_boolean_ondemand(var_name, "events", do_events);
+ }
+
+ if (unlikely(!ifm->enabled))
+ continue;
+
+ // --------------------------------------------------------------------
+
+ if (ifm->do_bandwidth == CONFIG_BOOLEAN_YES || (ifm->do_bandwidth == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(ibytes) || IFA_DATA(obytes)))) {
+ if (unlikely(!ifm->st_bandwidth)) {
+ ifm->st_bandwidth = rrdset_create_localhost("net",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.net",
+ "Bandwidth",
+ "kilobits/s",
+ 7000,
+ update_every,
+ RRDSET_TYPE_AREA
+ );
+
+ ifm->rd_bandwidth_in = rrddim_add(ifm->st_bandwidth, "received", NULL, 8, KILO_FACTOR,
+ RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_bandwidth_out = rrddim_add(ifm->st_bandwidth, "sent", NULL, -8, KILO_FACTOR,
+ RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(ifm->st_bandwidth);
+
+ rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_in, IFA_DATA(ibytes));
+ rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_out, IFA_DATA(obytes));
+ rrdset_done(ifm->st_bandwidth);
+ }
+
+ // --------------------------------------------------------------------
+
+ if (ifm->do_packets == CONFIG_BOOLEAN_YES || (ifm->do_packets == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(ipackets) || IFA_DATA(opackets) || IFA_DATA(imcasts) || IFA_DATA(omcasts)))) {
+ if (unlikely(!ifm->st_packets)) {
+ ifm->st_packets = rrdset_create_localhost("net_packets",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.packets",
+ "Packets",
+ "packets/s",
+ 7001,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_packets, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_packets_in = rrddim_add(ifm->st_packets, "received", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_packets_out = rrddim_add(ifm->st_packets, "sent", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_packets_m_in = rrddim_add(ifm->st_packets, "multicast_received", NULL, 1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_packets_m_out = rrddim_add(ifm->st_packets, "multicast_sent", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(ifm->st_packets);
+
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_in, IFA_DATA(ipackets));
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_out, IFA_DATA(opackets));
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_in, IFA_DATA(imcasts));
+ rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_out, IFA_DATA(omcasts));
+ rrdset_done(ifm->st_packets);
+ }
+
+ // --------------------------------------------------------------------
+
+ if (ifm->do_errors == CONFIG_BOOLEAN_YES || (ifm->do_errors == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(ierrors) || IFA_DATA(oerrors)))) {
+ if (unlikely(!ifm->st_errors)) {
+ ifm->st_errors = rrdset_create_localhost("net_errors",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.errors",
+ "Interface Errors",
+ "errors/s",
+ 7002,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_errors, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_errors_in = rrddim_add(ifm->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ ifm->rd_errors_out = rrddim_add(ifm->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(ifm->st_errors);
+
+ rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_in, IFA_DATA(ierrors));
+ rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_out, IFA_DATA(oerrors));
+ rrdset_done(ifm->st_errors);
+ }
+ // --------------------------------------------------------------------
+
+ if (ifm->do_drops == CONFIG_BOOLEAN_YES || (ifm->do_drops == CONFIG_BOOLEAN_AUTO &&
+ (IFA_DATA(iqdrops)
+ #if __FreeBSD__ >= 11
+ || IFA_DATA(oqdrops)
+#endif
+ ))) {
+ if (unlikely(!ifm->st_drops)) {
+ ifm->st_drops = rrdset_create_localhost("net_drops",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.drops",
+ "Interface Drops",
+ "drops/s",
+ 7003,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_drops, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_drops_in = rrddim_add(ifm->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#if __FreeBSD__ >= 11
+ ifm->rd_drops_out = rrddim_add(ifm->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ } else
+ rrdset_next(ifm->st_drops);
+
+ rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_in, IFA_DATA(iqdrops));
+#if __FreeBSD__ >= 11
+ rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_out, IFA_DATA(oqdrops));
+#endif
+ rrdset_done(ifm->st_drops);
+ }
+
+ // --------------------------------------------------------------------
+
+ if (ifm->do_events == CONFIG_BOOLEAN_YES || (ifm->do_events == CONFIG_BOOLEAN_AUTO &&
+ IFA_DATA(collisions))) {
+ if (unlikely(!ifm->st_events)) {
+ ifm->st_events = rrdset_create_localhost("net_events",
+ ifa->ifa_name,
+ NULL,
+ ifa->ifa_name,
+ "net.events",
+ "Network Interface Events",
+ "events/s",
+ 7006,
+ update_every,
+ RRDSET_TYPE_LINE
+ );
+
+ rrdset_flag_set(ifm->st_events, RRDSET_FLAG_DETAIL);
+
+ ifm->rd_events_coll = rrddim_add(ifm->st_events, "collisions", NULL, -1, 1,
+ RRD_ALGORITHM_INCREMENTAL);
+ } else
+ rrdset_next(ifm->st_events);
+
+ rrddim_set_by_pointer(ifm->st_events, ifm->rd_events_coll, IFA_DATA(collisions));
+ rrdset_done(ifm->st_events);
+ }
+ }
+
+ freeifaddrs(ifap);
+ }
+ } else {
+ error("DISABLED: getifaddrs module");
+ return 1;
+ }
+
+ network_interfaces_cleanup();
+
+ return 0;
+}
diff --git a/src/freebsd_getmntinfo.c b/src/freebsd_getmntinfo.c
new file mode 100644
index 000000000..e7ca56b57
--- /dev/null
+++ b/src/freebsd_getmntinfo.c
@@ -0,0 +1,293 @@
+#include "common.h"
+
+#include <sys/mount.h>
+
+struct mount_point {
+ char *name;
+ uint32_t hash;
+ size_t len;
+
+ // flags
+ int configured;
+ int enabled;
+ int updated;
+
+ int do_space;
+ int do_inodes;
+
+ size_t collected; // the number of times this has been collected
+
+ // charts and dimensions
+
+ RRDSET *st_space;
+ RRDDIM *rd_space_used;
+ RRDDIM *rd_space_avail;
+ RRDDIM *rd_space_reserved;
+
+ RRDSET *st_inodes;
+ RRDDIM *rd_inodes_used;
+ RRDDIM *rd_inodes_avail;
+
+ struct mount_point *next;
+};
+
+static struct mount_point *mount_points_root = NULL, *mount_points_last_used = NULL;
+
+static size_t mount_points_added = 0, mount_points_found = 0;
+
+static void mount_point_free(struct mount_point *m) {
+ if (likely(m->st_space))
+ rrdset_is_obsolete(m->st_space);
+ if (likely(m->st_inodes))
+ rrdset_is_obsolete(m->st_inodes);
+
+ mount_points_added--;
+ freez(m->name);
+ freez(m);
+}
+
+static void mount_points_cleanup() {
+ if (likely(mount_points_found == mount_points_added)) return;
+
+ struct mount_point *m = mount_points_root, *last = NULL;
+ while(m) {
+ if (unlikely(!m->updated)) {
+ // info("Removing mount point '%s', linked after '%s'", m->name, last?last->name:"ROOT");
+
+ if (mount_points_last_used == m)
+ mount_points_last_used = last;
+
+ struct mount_point *t = m;
+
+ if (m == mount_points_root || !last)
+ mount_points_root = m = m->next;
+
+ else
+ last->next = m = m->next;
+
+ t->next = NULL;
+ mount_point_free(t);
+ }
+ else {
+ last = m;
+ m->updated = 0;
+ m = m->next;
+ }
+ }
+}
+
+static struct mount_point *get_mount_point(const char *name) {
+ struct mount_point *m;
+
+ uint32_t hash = simple_hash(name);
+
+ // search it, from the last position to the end
+ for(m = mount_points_last_used ; m ; m = m->next) {
+ if (unlikely(hash == m->hash && !strcmp(name, m->name))) {
+ mount_points_last_used = m->next;
+ return m;
+ }
+ }
+
+ // search it from the beginning to the last position we used
+ for(m = mount_points_root ; m != mount_points_last_used ; m = m->next) {
+ if (unlikely(hash == m->hash && !strcmp(name, m->name))) {
+ mount_points_last_used = m->next;
+ return m;
+ }
+ }
+
+ // create a new one
+ m = callocz(1, sizeof(struct mount_point));
+ m->name = strdupz(name);
+ m->hash = simple_hash(m->name);
+ m->len = strlen(m->name);
+ mount_points_added++;
+
+ // link it to the end
+ if (mount_points_root) {
+ struct mount_point *e;
+ for(e = mount_points_root; e->next ; e = e->next) ;
+ e->next = m;
+ }
+ else
+ mount_points_root = m;
+
+ return m;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// getmntinfo
+
+int do_getmntinfo(int update_every, usec_t dt) {
+ (void)dt;
+
+#define DELAULT_EXLUDED_PATHS "/proc/*"
+// taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes
+#define DEFAULT_EXCLUDED_FILESYSTEMS "autofs procfs subfs devfs none"
+#define CONFIG_SECTION_GETMNTINFO "plugin:freebsd:getmntinfo"
+
+ static int enable_new_mount_points = -1;
+ static int do_space = -1, do_inodes = -1;
+ static SIMPLE_PATTERN *excluded_mountpoints = NULL;
+ static SIMPLE_PATTERN *excluded_filesystems = NULL;
+
+ if (unlikely(enable_new_mount_points == -1)) {
+ enable_new_mount_points = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO,
+ "enable new mount points detected at runtime",
+ CONFIG_BOOLEAN_AUTO);
+
+ do_space = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "space usage for all disks", CONFIG_BOOLEAN_AUTO);
+ do_inodes = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "inodes usage for all disks", CONFIG_BOOLEAN_AUTO);
+
+ excluded_mountpoints = simple_pattern_create(
+ config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on paths",
+ DELAULT_EXLUDED_PATHS),
+ SIMPLE_PATTERN_EXACT
+ );
+
+ excluded_filesystems = simple_pattern_create(
+ config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on filesystems",
+ DEFAULT_EXCLUDED_FILESYSTEMS),
+ SIMPLE_PATTERN_EXACT
+ );
+ }
+
+ if (likely(do_space || do_inodes)) {
+ struct statfs *mntbuf;
+ int mntsize;
+
+ // there is no mount info in sysctl MIBs
+ if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) {
+ error("FREEBSD: getmntinfo() failed");
+ do_space = 0;
+ error("DISABLED: disk_space.* charts");
+ do_inodes = 0;
+ error("DISABLED: disk_inodes.* charts");
+ error("DISABLED: getmntinfo module");
+ return 1;
+ } else {
+ int i;
+
+ mount_points_found = 0;
+
+ for (i = 0; i < mntsize; i++) {
+ char title[4096 + 1];
+
+ struct mount_point *m = get_mount_point(mntbuf[i].f_mntonname);
+ m->updated = 1;
+ mount_points_found++;
+
+ if (unlikely(!m->configured)) {
+ char var_name[4096 + 1];
+
+ // this is the first time we see this filesystem
+
+ // remember we configured it
+ m->configured = 1;
+
+ m->enabled = enable_new_mount_points;
+
+ if (likely(m->enabled))
+ m->enabled = !(simple_pattern_matches(excluded_mountpoints, mntbuf[i].f_mntonname)
+ || simple_pattern_matches(excluded_filesystems, mntbuf[i].f_fstypename));
+
+ snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETMNTINFO, mntbuf[i].f_mntonname);
+ m->enabled = config_get_boolean_ondemand(var_name, "enabled", m->enabled);
+
+ if (unlikely(m->enabled == CONFIG_BOOLEAN_NO))
+ continue;
+
+ m->do_space = config_get_boolean_ondemand(var_name, "space usage", do_space);
+ m->do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", do_inodes);
+ }
+
+ if (unlikely(!m->enabled))
+ continue;
+
+ if (unlikely(mntbuf[i].f_flags & MNT_RDONLY && !m->collected))
+ continue;
+
+ // --------------------------------------------------------------------------
+
+ int rendered = 0;
+
+ if (m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_blocks > 2))) {
+ if (unlikely(!m->st_space)) {
+ snprintfz(title, 4096, "Disk Space Usage for %s [%s]",
+ mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
+ m->st_space = rrdset_create_localhost("disk_space",
+ mntbuf[i].f_mntonname,
+ NULL,
+ mntbuf[i].f_mntonname,
+ "disk.space",
+ title,
+ "GB",
+ 2023,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL,
+ mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_space_used = rrddim_add(m->st_space, "used", NULL,
+ mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root",
+ mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
+ } else
+ rrdset_next(m->st_space);
+
+ rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number) mntbuf[i].f_bavail);
+ rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number) (mntbuf[i].f_blocks -
+ mntbuf[i].f_bfree));
+ rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number) (mntbuf[i].f_bfree -
+ mntbuf[i].f_bavail));
+ rrdset_done(m->st_space);
+
+ rendered++;
+ }
+
+ // --------------------------------------------------------------------------
+
+ if (m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_files > 1))) {
+ if (unlikely(!m->st_inodes)) {
+ snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]",
+ mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
+ m->st_inodes = rrdset_create_localhost("disk_inodes",
+ mntbuf[i].f_mntonname,
+ NULL,
+ mntbuf[i].f_mntonname,
+ "disk.inodes",
+ title,
+ "Inodes",
+ 2024,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+
+ m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ } else
+ rrdset_next(m->st_inodes);
+
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number) mntbuf[i].f_ffree);
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number) (mntbuf[i].f_files -
+ mntbuf[i].f_ffree));
+ rrdset_done(m->st_inodes);
+
+ rendered++;
+ }
+
+ if (likely(rendered))
+ m->collected++;
+ }
+ }
+ } else {
+ error("DISABLED: getmntinfo module");
+ return 1;
+ }
+
+ mount_points_cleanup();
+
+ return 0;
+}
diff --git a/src/freebsd_ipfw.c b/src/freebsd_ipfw.c
new file mode 100644
index 000000000..b89650a04
--- /dev/null
+++ b/src/freebsd_ipfw.c
@@ -0,0 +1,360 @@
+#include "common.h"
+
+#include <netinet/ip_fw.h>
+
+#define FREE_MEM_THRESHOLD 10000 // number of unused chunks that trigger memory freeing
+
+#define COMMON_IPFW_ERROR() error("DISABLED: ipfw.packets chart"); \
+ error("DISABLED: ipfw.bytes chart"); \
+ error("DISABLED: ipfw.dyn_active chart"); \
+ error("DISABLED: ipfw.dyn_expired chart"); \
+ error("DISABLED: ipfw.mem chart");
+
+// --------------------------------------------------------------------------------------------------------------------
+// ipfw
+
+int do_ipfw(int update_every, usec_t dt) {
+ (void)dt;
+#if __FreeBSD__ >= 11
+
+ static int do_static = -1, do_dynamic = -1, do_mem = -1;
+
+ if (unlikely(do_static == -1)) {
+ do_static = config_get_boolean("plugin:freebsd:ipfw", "counters for static rules", 1);
+ do_dynamic = config_get_boolean("plugin:freebsd:ipfw", "number of dynamic rules", 1);
+ do_mem = config_get_boolean("plugin:freebsd:ipfw", "allocated memory", 1);
+ }
+
+ // variables for getting ipfw configuration
+
+ int error;
+ static int ipfw_socket = -1;
+ static ipfw_cfg_lheader *cfg = NULL;
+ ip_fw3_opheader *op3 = NULL;
+ static socklen_t *optlen = NULL, cfg_size = 0;
+
+ // variables for static rules handling
+
+ ipfw_obj_ctlv *ctlv = NULL;
+ ipfw_obj_tlv *rbase = NULL;
+ int rcnt = 0;
+
+ int n, seen;
+ struct ip_fw_rule *rule;
+ struct ip_fw_bcounter *cntr;
+ int c = 0;
+
+ char rule_num_str[12];
+
+ // variables for dynamic rules handling
+
+ caddr_t dynbase = NULL;
+ size_t dynsz = 0;
+ size_t readsz = sizeof(*cfg);;
+ int ttype = 0;
+ ipfw_obj_tlv *tlv;
+ ipfw_dyn_rule *dyn_rule;
+ uint16_t rulenum, prev_rulenum = IPFW_DEFAULT_RULE;
+ unsigned srn, static_rules_num = 0;
+ static size_t dyn_rules_num_size = 0;
+
+ static struct dyn_rule_num {
+ uint16_t rule_num;
+ uint32_t active_rules;
+ uint32_t expired_rules;
+ } *dyn_rules_num = NULL;
+
+ uint32_t *dyn_rules_counter;
+
+ if (likely(do_static | do_dynamic | do_mem)) {
+
+ // initialize the smallest ipfw_cfg_lheader possible
+
+ if (unlikely((optlen == NULL) || (cfg == NULL))) {
+ optlen = reallocz(optlen, sizeof(socklen_t));
+ *optlen = cfg_size = 32;
+ cfg = reallocz(cfg, *optlen);
+ }
+
+ // get socket descriptor and initialize ipfw_cfg_lheader structure
+
+ if (unlikely(ipfw_socket == -1))
+ ipfw_socket = socket(AF_INET, SOCK_RAW, IPPROTO_RAW);
+ if (unlikely(ipfw_socket == -1)) {
+ error("FREEBSD: can't get socket for ipfw configuration");
+ error("FREEBSD: run netdata as root to get access to ipfw data");
+ COMMON_IPFW_ERROR();
+ return 1;
+ }
+
+ bzero(cfg, 32);
+ cfg->flags = IPFW_CFG_GET_STATIC | IPFW_CFG_GET_COUNTERS | IPFW_CFG_GET_STATES;
+ op3 = &cfg->opheader;
+ op3->opcode = IP_FW_XGET;
+
+ // get ifpw configuration size than get configuration
+
+ *optlen = cfg_size;
+ error = getsockopt(ipfw_socket, IPPROTO_IP, IP_FW3, op3, optlen);
+ if (error)
+ if (errno != ENOMEM) {
+ error("FREEBSD: ipfw socket reading error");
+ COMMON_IPFW_ERROR();
+ return 1;
+ }
+ if ((cfg->size > cfg_size) || ((cfg_size - cfg->size) > sizeof(struct dyn_rule_num) * FREE_MEM_THRESHOLD)) {
+ *optlen = cfg_size = cfg->size;
+ cfg = reallocz(cfg, *optlen);
+ bzero(cfg, 32);
+ cfg->flags = IPFW_CFG_GET_STATIC | IPFW_CFG_GET_COUNTERS | IPFW_CFG_GET_STATES;
+ op3 = &cfg->opheader;
+ op3->opcode = IP_FW_XGET;
+ error = getsockopt(ipfw_socket, IPPROTO_IP, IP_FW3, op3, optlen);
+ if (error) {
+ error("FREEBSD: ipfw socket reading error");
+ COMMON_IPFW_ERROR();
+ return 1;
+ }
+ }
+
+ // go through static rules configuration structures
+
+ ctlv = (ipfw_obj_ctlv *) (cfg + 1);
+
+ if (cfg->flags & IPFW_CFG_GET_STATIC) {
+ /* We've requested static rules */
+ if (ctlv->head.type == IPFW_TLV_TBLNAME_LIST) {
+ readsz += ctlv->head.length;
+ ctlv = (ipfw_obj_ctlv *) ((caddr_t) ctlv +
+ ctlv->head.length);
+ }
+
+ if (ctlv->head.type == IPFW_TLV_RULE_LIST) {
+ rbase = (ipfw_obj_tlv *) (ctlv + 1);
+ rcnt = ctlv->count;
+ readsz += ctlv->head.length;
+ ctlv = (ipfw_obj_ctlv *) ((caddr_t) ctlv + ctlv->head.length);
+ }
+ }
+
+ if ((cfg->flags & IPFW_CFG_GET_STATES) && (readsz != *optlen)) {
+ /* We may have some dynamic states */
+ dynsz = *optlen - readsz;
+ /* Skip empty header */
+ if (dynsz != sizeof(ipfw_obj_ctlv))
+ dynbase = (caddr_t) ctlv;
+ else
+ dynsz = 0;
+ }
+
+ // --------------------------------------------------------------------
+
+ if (likely(do_mem)) {
+ static RRDSET *st_mem = NULL;
+ static RRDDIM *rd_dyn_mem = NULL;
+ static RRDDIM *rd_stat_mem = NULL;
+
+ if (unlikely(!st_mem)) {
+ st_mem = rrdset_create_localhost("ipfw",
+ "mem",
+ NULL,
+ "memory allocated",
+ NULL,
+ "Memory allocated by rules",
+ "bytes",
+ 3005,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ rrdset_flag_set(st_mem, RRDSET_FLAG_DETAIL);
+
+ rd_dyn_mem = rrddim_add(st_mem, "dynamic", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_stat_mem = rrddim_add(st_mem, "static", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ } else
+ rrdset_next(st_mem);
+
+ rrddim_set_by_pointer(st_mem, rd_dyn_mem, dynsz);
+ rrddim_set_by_pointer(st_mem, rd_stat_mem, *optlen - dynsz);
+ rrdset_done(st_mem);
+ }
+
+ // --------------------------------------------------------------------
+
+ static RRDSET *st_packets = NULL, *st_bytes = NULL;
+ RRDDIM *rd_packets = NULL, *rd_bytes = NULL;
+
+ if (likely(do_static || do_dynamic)) {
+ if (likely(do_static)) {
+ if (unlikely(!st_packets))
+ st_packets = rrdset_create_localhost("ipfw",
+ "packets",
+ NULL,
+ "static rules",
+ NULL,
+ "Packets",
+ "packets/s",
+ 3001,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ else
+ rrdset_next(st_packets);
+
+ if (unlikely(!st_bytes))
+ st_bytes = rrdset_create_localhost("ipfw",
+ "bytes",
+ NULL,
+ "static rules",
+ NULL,
+ "Bytes",
+ "bytes/s",
+ 3002,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ else
+ rrdset_next(st_bytes);
+ }
+
+ for (n = seen = 0; n < rcnt; n++, rbase = (ipfw_obj_tlv *) ((caddr_t) rbase + rbase->length)) {
+ cntr = (struct ip_fw_bcounter *) (rbase + 1);
+ rule = (struct ip_fw_rule *) ((caddr_t) cntr + cntr->size);
+ if (rule->rulenum != prev_rulenum)
+ static_rules_num++;
+ if (rule->rulenum > IPFW_DEFAULT_RULE)
+ break;
+
+ if (likely(do_static)) {
+ sprintf(rule_num_str, "%d_%d", rule->rulenum, rule->id);
+
+ rd_packets = rrddim_find(st_packets, rule_num_str);
+ if (unlikely(!rd_packets))
+ rd_packets = rrddim_add(st_packets, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_set_by_pointer(st_packets, rd_packets, cntr->pcnt);
+
+ rd_bytes = rrddim_find(st_bytes, rule_num_str);
+ if (unlikely(!rd_bytes))
+ rd_bytes = rrddim_add(st_bytes, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_set_by_pointer(st_bytes, rd_bytes, cntr->bcnt);
+ }
+
+ c += rbase->length;
+ seen++;
+ }
+
+ if (likely(do_static)) {
+ rrdset_done(st_packets);
+ rrdset_done(st_bytes);
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ // go through dynamic rules configuration structures
+
+ if (likely(do_dynamic && (dynsz > 0))) {
+ if ((dyn_rules_num_size < sizeof(struct dyn_rule_num) * static_rules_num) ||
+ ((dyn_rules_num_size - sizeof(struct dyn_rule_num) * static_rules_num) >
+ sizeof(struct dyn_rule_num) * FREE_MEM_THRESHOLD)) {
+ dyn_rules_num_size = sizeof(struct dyn_rule_num) * static_rules_num;
+ dyn_rules_num = reallocz(dyn_rules_num, dyn_rules_num_size);
+ }
+ bzero(dyn_rules_num, sizeof(struct dyn_rule_num) * static_rules_num);
+ dyn_rules_num->rule_num = IPFW_DEFAULT_RULE;
+
+ if (dynsz > 0 && ctlv->head.type == IPFW_TLV_DYNSTATE_LIST) {
+ dynbase += sizeof(*ctlv);
+ dynsz -= sizeof(*ctlv);
+ ttype = IPFW_TLV_DYN_ENT;
+ }
+
+ while (dynsz > 0) {
+ tlv = (ipfw_obj_tlv *) dynbase;
+ if (tlv->type != ttype)
+ break;
+
+ dyn_rule = (ipfw_dyn_rule *) (tlv + 1);
+ bcopy(&dyn_rule->rule, &rulenum, sizeof(rulenum));
+
+ for (srn = 0; srn < (static_rules_num - 1); srn++) {
+ if (dyn_rule->expire > 0)
+ dyn_rules_counter = &dyn_rules_num[srn].active_rules;
+ else
+ dyn_rules_counter = &dyn_rules_num[srn].expired_rules;
+ if (dyn_rules_num[srn].rule_num == rulenum) {
+ (*dyn_rules_counter)++;
+ break;
+ }
+ if (dyn_rules_num[srn].rule_num == IPFW_DEFAULT_RULE) {
+ dyn_rules_num[srn].rule_num = rulenum;
+ dyn_rules_num[srn + 1].rule_num = IPFW_DEFAULT_RULE;
+ (*dyn_rules_counter)++;
+ break;
+ }
+ }
+
+ dynsz -= tlv->length;
+ dynbase += tlv->length;
+ }
+
+ // --------------------------------------------------------------------
+
+ static RRDSET *st_active = NULL, *st_expired = NULL;
+ RRDDIM *rd_active = NULL, *rd_expired = NULL;
+
+ if (unlikely(!st_active))
+ st_active = rrdset_create_localhost("ipfw",
+ "active",
+ NULL,
+ "dynamic_rules",
+ NULL,
+ "Active rules",
+ "rules",
+ 3003,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ else
+ rrdset_next(st_active);
+
+ if (unlikely(!st_expired))
+ st_expired = rrdset_create_localhost("ipfw",
+ "expired",
+ NULL,
+ "dynamic_rules",
+ NULL,
+ "Expired rules",
+ "rules",
+ 3004,
+ update_every,
+ RRDSET_TYPE_STACKED
+ );
+ else
+ rrdset_next(st_expired);
+
+ for (srn = 0; (srn < (static_rules_num - 1)) && (dyn_rules_num[srn].rule_num != IPFW_DEFAULT_RULE); srn++) {
+ sprintf(rule_num_str, "%d", dyn_rules_num[srn].rule_num);
+
+ rd_active = rrddim_find(st_active, rule_num_str);
+ if (unlikely(!rd_active))
+ rd_active = rrddim_add(st_active, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_set_by_pointer(st_active, rd_active, dyn_rules_num[srn].active_rules);
+
+ rd_expired = rrddim_find(st_expired, rule_num_str);
+ if (unlikely(!rd_expired))
+ rd_expired = rrddim_add(st_expired, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rrddim_set_by_pointer(st_expired, rd_expired, dyn_rules_num[srn].expired_rules);
+ }
+
+ rrdset_done(st_active);
+ rrdset_done(st_expired);
+ }
+ }
+
+ return 0;
+#else
+ error("FREEBSD: ipfw charts supported for FreeBSD 11.0 and newer releases only");
+ COMMON_IPFW_ERROR();
+ return 1;
+#endif
+}
diff --git a/src/freebsd_kstat_zfs.c b/src/freebsd_kstat_zfs.c
new file mode 100644
index 000000000..17642994e
--- /dev/null
+++ b/src/freebsd_kstat_zfs.c
@@ -0,0 +1,212 @@
+#include "common.h"
+#include "zfs_common.h"
+
+struct arcstats arcstats = { 0 };
+
+// kstat.zfs.misc.arcstats
+int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt) {
+ (void)dt;
+
+ unsigned long long l2_size;
+ size_t uint64_t_size = sizeof(uint64_t);
+ static struct mibs {
+ int hits[5];
+ int misses[5];
+ int demand_data_hits[5];
+ int demand_data_misses[5];
+ int demand_metadata_hits[5];
+ int demand_metadata_misses[5];
+ int prefetch_data_hits[5];
+ int prefetch_data_misses[5];
+ int prefetch_metadata_hits[5];
+ int prefetch_metadata_misses[5];
+ int mru_hits[5];
+ int mru_ghost_hits[5];
+ int mfu_hits[5];
+ int mfu_ghost_hits[5];
+ int deleted[5];
+ int mutex_miss[5];
+ int evict_skip[5];
+ int evict_not_enough[5];
+ int evict_l2_cached[5];
+ int evict_l2_eligible[5];
+ int evict_l2_ineligible[5];
+ int evict_l2_skip[5];
+ int hash_elements[5];
+ int hash_elements_max[5];
+ int hash_collisions[5];
+ int hash_chains[5];
+ int hash_chain_max[5];
+ int p[5];
+ int c[5];
+ int c_min[5];
+ int c_max[5];
+ int size[5];
+ int hdr_size[5];
+ int data_size[5];
+ int metadata_size[5];
+ int other_size[5];
+ int anon_size[5];
+ int anon_evictable_data[5];
+ int anon_evictable_metadata[5];
+ int mru_size[5];
+ int mru_evictable_data[5];
+ int mru_evictable_metadata[5];
+ int mru_ghost_size[5];
+ int mru_ghost_evictable_data[5];
+ int mru_ghost_evictable_metadata[5];
+ int mfu_size[5];
+ int mfu_evictable_data[5];
+ int mfu_evictable_metadata[5];
+ int mfu_ghost_size[5];
+ int mfu_ghost_evictable_data[5];
+ int mfu_ghost_evictable_metadata[5];
+ int l2_hits[5];
+ int l2_misses[5];
+ int l2_feeds[5];
+ int l2_rw_clash[5];
+ int l2_read_bytes[5];
+ int l2_write_bytes[5];
+ int l2_writes_sent[5];
+ int l2_writes_done[5];
+ int l2_writes_error[5];
+ int l2_writes_lock_retry[5];
+ int l2_evict_lock_retry[5];
+ int l2_evict_reading[5];
+ int l2_evict_l1cached[5];
+ int l2_free_on_write[5];
+ int l2_cdata_free_on_write[5];
+ int l2_abort_lowmem[5];
+ int l2_cksum_bad[5];
+ int l2_io_error[5];
+ int l2_size[5];
+ int l2_asize[5];
+ int l2_hdr_size[5];
+ int l2_compress_successes[5];
+ int l2_compress_zeros[5];
+ int l2_compress_failures[5];
+ int memory_throttle_count[5];
+ int duplicate_buffers[5];
+ int duplicate_buffers_size[5];
+ int duplicate_reads[5];
+ int memory_direct_count[5];
+ int memory_indirect_count[5];
+ int arc_no_grow[5];
+ int arc_tempreserve[5];
+ int arc_loaned_bytes[5];
+ int arc_prune[5];
+ int arc_meta_used[5];
+ int arc_meta_limit[5];
+ int arc_meta_max[5];
+ int arc_meta_min[5];
+ int arc_need_free[5];
+ int arc_sys_free[5];
+ } mibs;
+
+ l2exist = -1;
+
+ if(unlikely(sysctlbyname("kstat.zfs.misc.arcstats.l2_size", &l2_size, &uint64_t_size, NULL, 0)))
+ return 0;
+
+ if(likely(l2_size))
+ l2exist = 1;
+ else
+ l2exist = 0;
+
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hits", mibs.hits, arcstats.hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.misses", mibs.misses, arcstats.misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_hits", mibs.demand_data_hits, arcstats.demand_data_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_misses", mibs.demand_data_misses, arcstats.demand_data_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_hits", mibs.demand_metadata_hits, arcstats.demand_metadata_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_misses", mibs.demand_metadata_misses, arcstats.demand_metadata_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_hits", mibs.prefetch_data_hits, arcstats.prefetch_data_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_misses", mibs.prefetch_data_misses, arcstats.prefetch_data_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_hits", mibs.prefetch_metadata_hits, arcstats.prefetch_metadata_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_misses", mibs.prefetch_metadata_misses, arcstats.prefetch_metadata_misses);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_hits", mibs.mru_hits, arcstats.mru_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_hits", mibs.mru_ghost_hits, arcstats.mru_ghost_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_hits", mibs.mfu_hits, arcstats.mfu_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_hits", mibs.mfu_ghost_hits, arcstats.mfu_ghost_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.deleted", mibs.deleted, arcstats.deleted);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mutex_miss", mibs.mutex_miss, arcstats.mutex_miss);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_skip", mibs.evict_skip, arcstats.evict_skip);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_not_enough", mibs.evict_not_enough, arcstats.evict_not_enough);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_cached", mibs.evict_l2_cached, arcstats.evict_l2_cached);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_eligible", mibs.evict_l2_eligible, arcstats.evict_l2_eligible);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_ineligible", mibs.evict_l2_ineligible, arcstats.evict_l2_ineligible);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_skip", mibs.evict_l2_skip, arcstats.evict_l2_skip);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements", mibs.hash_elements, arcstats.hash_elements);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements_max", mibs.hash_elements_max, arcstats.hash_elements_max);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_collisions", mibs.hash_collisions, arcstats.hash_collisions);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chains", mibs.hash_chains, arcstats.hash_chains);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chain_max", mibs.hash_chain_max, arcstats.hash_chain_max);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.p", mibs.p, arcstats.p);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c", mibs.c, arcstats.c);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_min", mibs.c_min, arcstats.c_min);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_max", mibs.c_max, arcstats.c_max);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.size", mibs.size, arcstats.size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hdr_size", mibs.hdr_size, arcstats.hdr_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.data_size", mibs.data_size, arcstats.data_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.metadata_size", mibs.metadata_size, arcstats.metadata_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.other_size", mibs.other_size, arcstats.other_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_size", mibs.anon_size, arcstats.anon_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_data", mibs.anon_evictable_data, arcstats.anon_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_metadata", mibs.anon_evictable_metadata, arcstats.anon_evictable_metadata);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_size", mibs.mru_size, arcstats.mru_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_data", mibs.mru_evictable_data, arcstats.mru_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_metadata", mibs.mru_evictable_metadata, arcstats.mru_evictable_metadata);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_size", mibs.mru_ghost_size, arcstats.mru_ghost_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_data", mibs.mru_ghost_evictable_data, arcstats.mru_ghost_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_metadata", mibs.mru_ghost_evictable_metadata, arcstats.mru_ghost_evictable_metadata);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_size", mibs.mfu_size, arcstats.mfu_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_data", mibs.mfu_evictable_data, arcstats.mfu_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_metadata", mibs.mfu_evictable_metadata, arcstats.mfu_evictable_metadata);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_size", mibs.mfu_ghost_size, arcstats.mfu_ghost_size);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_data", mibs.mfu_ghost_evictable_data, arcstats.mfu_ghost_evictable_data);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_metadata", mibs.mfu_ghost_evictable_metadata, arcstats.mfu_ghost_evictable_metadata);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hits", mibs.l2_hits, arcstats.l2_hits);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_misses", mibs.l2_misses, arcstats.l2_misses);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_feeds", mibs.l2_feeds, arcstats.l2_feeds);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_rw_clash", mibs.l2_rw_clash, arcstats.l2_rw_clash);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_read_bytes", mibs.l2_read_bytes, arcstats.l2_read_bytes);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_write_bytes", mibs.l2_write_bytes, arcstats.l2_write_bytes);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_sent", mibs.l2_writes_sent, arcstats.l2_writes_sent);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_done", mibs.l2_writes_done, arcstats.l2_writes_done);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_error", mibs.l2_writes_error, arcstats.l2_writes_error);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_lock_retry", mibs.l2_writes_lock_retry, arcstats.l2_writes_lock_retry);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_lock_retry", mibs.l2_evict_lock_retry, arcstats.l2_evict_lock_retry);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_reading", mibs.l2_evict_reading, arcstats.l2_evict_reading);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_l1cached", mibs.l2_evict_l1cached, arcstats.l2_evict_l1cached);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_free_on_write", mibs.l2_free_on_write, arcstats.l2_free_on_write);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cdata_free_on_write", mibs.l2_cdata_free_on_write, arcstats.l2_cdata_free_on_write);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_abort_lowmem", mibs.l2_abort_lowmem, arcstats.l2_abort_lowmem);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cksum_bad", mibs.l2_cksum_bad, arcstats.l2_cksum_bad);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_io_error", mibs.l2_io_error, arcstats.l2_io_error);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_size", mibs.l2_size, arcstats.l2_size);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_asize", mibs.l2_asize, arcstats.l2_asize);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hdr_size", mibs.l2_hdr_size, arcstats.l2_hdr_size);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_successes", mibs.l2_compress_successes, arcstats.l2_compress_successes);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_zeros", mibs.l2_compress_zeros, arcstats.l2_compress_zeros);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_failures", mibs.l2_compress_failures, arcstats.l2_compress_failures);
+ GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_throttle_count", mibs.memory_throttle_count, arcstats.memory_throttle_count);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers", mibs.duplicate_buffers, arcstats.duplicate_buffers);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers_size", mibs.duplicate_buffers_size, arcstats.duplicate_buffers_size);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_reads", mibs.duplicate_reads, arcstats.duplicate_reads);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_direct_count", mibs.memory_direct_count, arcstats.memory_direct_count);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_indirect_count", mibs.memory_indirect_count, arcstats.memory_indirect_count);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_no_grow", mibs.arc_no_grow, arcstats.arc_no_grow);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_tempreserve", mibs.arc_tempreserve, arcstats.arc_tempreserve);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_loaned_bytes", mibs.arc_loaned_bytes, arcstats.arc_loaned_bytes);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_prune", mibs.arc_prune, arcstats.arc_prune);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_used", mibs.arc_meta_used, arcstats.arc_meta_used);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_limit", mibs.arc_meta_limit, arcstats.arc_meta_limit);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_max", mibs.arc_meta_max, arcstats.arc_meta_max);
+ // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_min", mibs.arc_meta_min, arcstats.arc_meta_min);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_need_free", mibs.arc_need_free, arcstats.arc_need_free);
+ // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_sys_free", mibs.arc_sys_free, arcstats.arc_sys_free);
+
+ generate_charts_arcstats(update_every);
+ generate_charts_arc_summary(update_every);
+
+ return 0;
+} \ No newline at end of file
diff --git a/src/freebsd_sysctl.c b/src/freebsd_sysctl.c
index 965c1cbbf..d2f0eaa82 100644
--- a/src/freebsd_sysctl.c
+++ b/src/freebsd_sysctl.c
@@ -1,8 +1,6 @@
#include "common.h"
#include <sys/vmmeter.h>
-#include <sys/devicestat.h>
-#include <sys/mount.h>
#include <vm/vm_param.h>
#define _KERNEL
@@ -12,8 +10,6 @@
#undef _KERNEL
#include <net/netisr.h>
-#include <net/if.h>
-#include <ifaddrs.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
@@ -29,12 +25,6 @@
// --------------------------------------------------------------------------------------------------------------------
// common definitions and variables
-#define KILO_FACTOR 1024
-#define MEGA_FACTOR 1048576 // 1024 * 1024
-#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024
-
-#define MAX_INT_DIGITS 10 // maximum number of digits for int
-
int system_pagesize = PAGE_SIZE;
int number_of_cpus = 1;
@@ -311,8 +301,10 @@ int do_kern_cp_times(int update_every, usec_t dt) {
static int mib[2] = {0, 0};
long cp_time[CPUSTATES];
static long *pcpu_cp_time = NULL;
+ static int old_number_of_cpus = 0;
- pcpu_cp_time = reallocz(pcpu_cp_time, sizeof(cp_time) * number_of_cpus);
+ if(unlikely(number_of_cpus != old_number_of_cpus))
+ pcpu_cp_time = reallocz(pcpu_cp_time, sizeof(cp_time) * number_of_cpus);
if (unlikely(GETSYSCTL_WSIZE("kern.cp_times", mib, pcpu_cp_time, sizeof(cp_time) * number_of_cpus))) {
error("DISABLED: cpu.cpuXX charts");
error("DISABLED: kern.cp_times module");
@@ -331,12 +323,10 @@ int do_kern_cp_times(int update_every, usec_t dt) {
RRDDIM *rd_interrupt;
RRDDIM *rd_idle;
} *all_cpu_charts = NULL;
- static int old_number_of_cpus = 0;
if(unlikely(number_of_cpus > old_number_of_cpus)) {
all_cpu_charts = reallocz(all_cpu_charts, sizeof(struct cpu_chart) * number_of_cpus);
memset(&all_cpu_charts[old_number_of_cpus], 0, sizeof(struct cpu_chart) * (number_of_cpus - old_number_of_cpus));
- old_number_of_cpus = number_of_cpus;
}
for (i = 0; i < number_of_cpus; i++) {
@@ -375,6 +365,8 @@ int do_kern_cp_times(int update_every, usec_t dt) {
rrdset_done(all_cpu_charts[i].st);
}
}
+
+ old_number_of_cpus = number_of_cpus;
}
return 0;
@@ -386,7 +378,7 @@ int do_kern_cp_times(int update_every, usec_t dt) {
int do_hw_intcnt(int update_every, usec_t dt) {
(void)dt;
static int mib_hw_intrcnt[2] = {0, 0};
- size_t intrcnt_size = sizeof(mib_hw_intrcnt);
+ size_t intrcnt_size = 0;
unsigned long i;
if (unlikely(GETSYSCTL_SIZE("hw.intrcnt", mib_hw_intrcnt, intrcnt_size))) {
@@ -396,11 +388,13 @@ int do_hw_intcnt(int update_every, usec_t dt) {
return 1;
} else {
unsigned long nintr = 0;
+ static unsigned long old_nintr = 0;
static unsigned long *intrcnt = NULL;
unsigned long long totalintr = 0;
nintr = intrcnt_size / sizeof(u_long);
- intrcnt = reallocz(intrcnt, nintr * sizeof(u_long));
+ if (unlikely(nintr != old_nintr))
+ intrcnt = reallocz(intrcnt, nintr * sizeof(u_long));
if (unlikely(GETSYSCTL_WSIZE("hw.intrcnt", mib_hw_intrcnt, intrcnt, nintr * sizeof(u_long)))) {
error("DISABLED: system.intr chart");
error("DISABLED: system.interrupts chart");
@@ -443,7 +437,8 @@ int do_hw_intcnt(int update_every, usec_t dt) {
static char *intrnames = NULL;
size = nintr * (MAXCOMLEN + 1);
- intrnames = reallocz(intrnames, size);
+ if (unlikely(nintr != old_nintr))
+ intrnames = reallocz(intrnames, size);
if (unlikely(GETSYSCTL_WSIZE("hw.intrnames", mib_hw_intrnames, intrnames, size))) {
error("DISABLED: system.intr chart");
error("DISABLED: system.interrupts chart");
@@ -484,6 +479,8 @@ int do_hw_intcnt(int update_every, usec_t dt) {
rrdset_done(st_interrupts);
}
}
+
+ old_nintr = nintr;
}
return 0;
@@ -931,8 +928,12 @@ int do_kern_ipc_sem(int update_every, usec_t dt) {
return 1;
} else {
static struct semid_kernel *ipc_sem_data = NULL;
+ static int old_semmni = 0;
- ipc_sem_data = reallocz(ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni);
+ if (unlikely(ipc_sem.semmni != old_semmni)) {
+ ipc_sem_data = reallocz(ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni);
+ old_semmni = ipc_sem.semmni;
+ }
if (unlikely(GETSYSCTL_WSIZE("kern.ipc.sema", mib_sema, ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni))) {
error("DISABLED: system.ipc_semaphores chart");
error("DISABLED: system.ipc_semaphore_arrays chart");
@@ -1019,8 +1020,12 @@ int do_kern_ipc_shm(int update_every, usec_t dt) {
return 1;
} else {
static struct shmid_kernel *ipc_shm_data = NULL;
+ static u_long old_shmmni = 0;
- ipc_shm_data = reallocz(ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni);
+ if (unlikely(ipc_shm.shmmni != old_shmmni)) {
+ ipc_shm_data = reallocz(ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni);
+ old_shmmni = ipc_shm.shmmni;
+ }
if (unlikely(
GETSYSCTL_WSIZE("kern.ipc.shmsegs", mib_shmsegs, ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni))) {
error("DISABLED: system.ipc_shared_mem_segs chart");
@@ -1111,8 +1116,12 @@ int do_kern_ipc_msq(int update_every, usec_t dt) {
return 1;
} else {
static struct msqid_kernel *ipc_msq_data = NULL;
+ static int old_msgmni = 0;
- ipc_msq_data = reallocz(ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni);
+ if (unlikely(ipc_msq.msgmni != old_msgmni)) {
+ ipc_msq_data = reallocz(ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni);
+ old_msgmni = ipc_msq.msgmni;
+ }
if (unlikely(
GETSYSCTL_WSIZE("kern.ipc.msqids", mib_msqids, ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni))) {
error("DISABLED: system.ipc_msq_queues chart");
@@ -1259,7 +1268,7 @@ int do_net_isr(int update_every, usec_t dt) {
static int mib_workstream[3] = {0, 0, 0}, mib_work[3] = {0, 0, 0};
int common_error = 0;
- size_t netisr_workstream_size = sizeof(mib_workstream), netisr_work_size = sizeof(mib_work);
+ size_t netisr_workstream_size = 0, netisr_work_size = 0;
unsigned long num_netisr_workstreams = 0, num_netisr_works = 0;
static struct sysctl_netisr_workstream *netisr_workstream = NULL;
static struct sysctl_netisr_work *netisr_work = NULL;
@@ -1276,14 +1285,25 @@ int do_net_isr(int update_every, usec_t dt) {
} else if (unlikely(GETSYSCTL_SIZE("net.isr.work", mib_work, netisr_work_size))) {
common_error = 1;
} else {
+ static size_t old_netisr_workstream_size = 0;
+
num_netisr_workstreams = netisr_workstream_size / sizeof(struct sysctl_netisr_workstream);
- netisr_workstream = reallocz(netisr_workstream, num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream));
+ if (unlikely(netisr_workstream_size != old_netisr_workstream_size)) {
+ netisr_workstream = reallocz(netisr_workstream,
+ num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream));
+ old_netisr_workstream_size = netisr_workstream_size;
+ }
if (unlikely(GETSYSCTL_WSIZE("net.isr.workstream", mib_workstream, netisr_workstream,
num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream)))){
common_error = 1;
} else {
+ static size_t old_netisr_work_size = 0;
+
num_netisr_works = netisr_work_size / sizeof(struct sysctl_netisr_work);
- netisr_work = reallocz(netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work));
+ if (unlikely(netisr_work_size != old_netisr_work_size)) {
+ netisr_work = reallocz(netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work));
+ old_netisr_work_size = netisr_work_size;
+ }
if (unlikely(GETSYSCTL_WSIZE("net.isr.work", mib_work, netisr_work,
num_netisr_works * sizeof(struct sysctl_netisr_work)))){
common_error = 1;
@@ -1301,8 +1321,12 @@ int do_net_isr(int update_every, usec_t dt) {
} else {
unsigned long i, n;
int j;
+ static int old_number_of_cpus = 0;
- netisr_stats = reallocz(netisr_stats, (number_of_cpus + 1) * sizeof(struct netisr_stats));
+ if (unlikely(number_of_cpus != old_number_of_cpus)) {
+ netisr_stats = reallocz(netisr_stats, (number_of_cpus + 1) * sizeof(struct netisr_stats));
+ old_number_of_cpus = number_of_cpus;
+ }
memset(netisr_stats, 0, (number_of_cpus + 1) * sizeof(struct netisr_stats));
for (i = 0; i < num_netisr_workstreams; i++) {
for (n = 0; n < num_netisr_works; n++) {
@@ -2749,1216 +2773,3 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) {
return 0;
}
-
-// --------------------------------------------------------------------------------------------------------------------
-// getmntinfo
-
-int do_getmntinfo(int update_every, usec_t dt) {
- (void)dt;
-
-#define DELAULT_EXLUDED_PATHS "/proc/*"
-// taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes
-#define DEFAULT_EXCLUDED_FILESYSTEMS "autofs procfs subfs devfs none"
-#define CONFIG_SECTION_GETMNTINFO "plugin:freebsd:getmntinfo"
-
- static int do_space = -1, do_inodes = -1;
-
- if (unlikely(do_space == -1)) {
- do_space = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "space usage for all disks", CONFIG_BOOLEAN_AUTO);
- do_inodes = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "inodes usage for all disks", CONFIG_BOOLEAN_AUTO);
- }
-
- if (likely(do_space || do_inodes)) {
- struct statfs *mntbuf;
- int mntsize;
-
- // there is no mount info in sysctl MIBs
- if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) {
- error("FREEBSD: getmntinfo() failed");
- do_space = 0;
- error("DISABLED: disk_space.* charts");
- do_inodes = 0;
- error("DISABLED: disk_inodes.* charts");
- error("DISABLED: getmntinfo module");
- return 1;
- } else {
- // Data to be stored in DICTIONARY mount_points.
- // This DICTIONARY is used to lookup the settings of the mount point on each iteration.
- struct mount_point_metadata {
- int do_space;
- int do_inodes;
-
- size_t collected; // the number of times this has been collected
-
- // charts and dimensions
-
- RRDSET *st_space;
- RRDDIM *rd_space_used;
- RRDDIM *rd_space_avail;
- RRDDIM *rd_space_reserved;
-
- RRDSET *st_inodes;
- RRDDIM *rd_inodes_used;
- RRDDIM *rd_inodes_avail;
- };
- static DICTIONARY *mount_points = NULL;
- static SIMPLE_PATTERN *excluded_mountpoints = NULL;
- static SIMPLE_PATTERN *excluded_filesystems = NULL;
- int i;
-
- if(unlikely(!mount_points)) {
-
- excluded_mountpoints = simple_pattern_create(
- config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on paths",
- DELAULT_EXLUDED_PATHS),
- SIMPLE_PATTERN_EXACT
- );
-
- excluded_filesystems = simple_pattern_create(
- config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on filesystems",
- DEFAULT_EXCLUDED_FILESYSTEMS),
- SIMPLE_PATTERN_EXACT
- );
-
- mount_points = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
- }
-
- for (i = 0; i < mntsize; i++) {
-
- char title[4096 + 1];
- int def_space, def_inodes, iter_space, iter_inodes;
-
- struct mount_point_metadata *m = dictionary_get(mount_points, mntbuf[i].f_mntonname);
- if(unlikely(!m)) {
- char var_name[4096 + 1];
- snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETMNTINFO, mntbuf[i].f_mntonname);
-
- def_space = do_space;
- def_inodes = do_space;
-
- if(unlikely(simple_pattern_matches(excluded_mountpoints, mntbuf[i].f_mntonname))) {
- def_space = CONFIG_BOOLEAN_NO;
- def_inodes = CONFIG_BOOLEAN_NO;
- }
-
- if(unlikely(simple_pattern_matches(excluded_filesystems, mntbuf[i].f_fstypename))) {
- def_space = CONFIG_BOOLEAN_NO;
- def_inodes = CONFIG_BOOLEAN_NO;
- }
-
- iter_space = config_get_boolean_ondemand(var_name, "space usage", def_space);
- iter_inodes = config_get_boolean_ondemand(var_name, "inodes usage", def_inodes);
-
- struct mount_point_metadata mp = {
- .do_space = iter_space,
- .do_inodes = iter_inodes,
-
- .collected = 0,
-
- .st_space = NULL,
- .rd_space_avail = NULL,
- .rd_space_used = NULL,
- .rd_space_reserved = NULL,
-
- .st_inodes = NULL,
- .rd_inodes_avail = NULL,
- .rd_inodes_used = NULL,
- };
-
- m = dictionary_set(mount_points, mntbuf[i].f_mntonname, &mp, sizeof(struct mount_point_metadata));
- }
-
- if(unlikely(m->do_space == CONFIG_BOOLEAN_NO && m->do_inodes == CONFIG_BOOLEAN_NO))
- continue;
-
- if(unlikely(mntbuf[i].f_flags & MNT_RDONLY && !m->collected))
- continue;
-
- // --------------------------------------------------------------------------
-
- int rendered = 0;
-
- if (m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_blocks > 2))) {
- if (unlikely(!m->st_space)) {
- snprintfz(title, 4096, "Disk Space Usage for %s [%s]",
- mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
- m->st_space = rrdset_create_localhost("disk_space",
- mntbuf[i].f_mntonname,
- NULL,
- mntbuf[i].f_mntonname,
- "disk.space",
- title,
- "GB",
- 2023,
- update_every,
- RRDSET_TYPE_STACKED
- );
-
- m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL,
- mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
- m->rd_space_used = rrddim_add(m->st_space, "used", NULL,
- mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
- m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root",
- mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE);
- } else
- rrdset_next(m->st_space);
-
- rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number) mntbuf[i].f_bavail);
- rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number) (mntbuf[i].f_blocks -
- mntbuf[i].f_bfree));
- rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number) (mntbuf[i].f_bfree -
- mntbuf[i].f_bavail));
- rrdset_done(m->st_space);
-
- rendered++;
- }
-
- // --------------------------------------------------------------------------
-
- if (m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_files > 1))) {
- if (unlikely(!m->st_inodes)) {
- snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]",
- mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
- m->st_inodes = rrdset_create_localhost("disk_inodes",
- mntbuf[i].f_mntonname,
- NULL,
- mntbuf[i].f_mntonname,
- "disk.inodes",
- title,
- "Inodes",
- 2024,
- update_every,
- RRDSET_TYPE_STACKED
- );
-
- m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
- m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
- } else
- rrdset_next(m->st_inodes);
-
- rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number) mntbuf[i].f_ffree);
- rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number) (mntbuf[i].f_files -
- mntbuf[i].f_ffree));
- rrdset_done(m->st_inodes);
-
- rendered++;
- }
-
- if(likely(rendered))
- m->collected++;
- }
- }
- } else {
- error("DISABLED: getmntinfo module");
- return 1;
- }
-
- return 0;
-}
-
-// --------------------------------------------------------------------------------------------------------------------
-// getifaddrs
-
-int do_getifaddrs(int update_every, usec_t dt) {
- (void)dt;
-
-#define DELAULT_EXLUDED_INTERFACES "lo*"
-#define CONFIG_SECTION_GETIFADDRS "plugin:freebsd:getifaddrs"
-
- static int do_bandwidth_ipv4 = -1, do_bandwidth_ipv6 = -1, do_bandwidth = -1, do_packets = -1,
- do_errors = -1, do_drops = -1, do_events = -1;
-
- if (unlikely(do_bandwidth_ipv4 == -1)) {
- do_bandwidth_ipv4 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv4 interfaces",
- CONFIG_BOOLEAN_AUTO);
- do_bandwidth_ipv6 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv6 interfaces",
- CONFIG_BOOLEAN_AUTO);
- do_bandwidth = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "bandwidth for all interfaces",
- CONFIG_BOOLEAN_AUTO);
- do_packets = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "packets for all interfaces",
- CONFIG_BOOLEAN_AUTO);
- do_errors = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "errors for all interfaces",
- CONFIG_BOOLEAN_AUTO);
- do_drops = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "drops for all interfaces",
- CONFIG_BOOLEAN_AUTO);
- do_events = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "collisions for all interfaces",
- CONFIG_BOOLEAN_AUTO);
- }
-
- if (likely(do_bandwidth_ipv4 || do_bandwidth_ipv6 || do_bandwidth || do_packets || do_errors ||
- do_drops || do_events)) {
- struct ifaddrs *ifap;
-
- if (unlikely(getifaddrs(&ifap))) {
- error("FREEBSD: getifaddrs() failed");
- do_bandwidth_ipv4 = 0;
- error("DISABLED: system.ipv4 chart");
- do_bandwidth_ipv6 = 0;
- error("DISABLED: system.ipv6 chart");
- do_bandwidth = 0;
- error("DISABLED: net.* charts");
- do_packets = 0;
- error("DISABLED: net_packets.* charts");
- do_errors = 0;
- error("DISABLED: net_errors.* charts");
- do_drops = 0;
- error("DISABLED: net_drops.* charts");
- do_events = 0;
- error("DISABLED: net_events.* charts");
- error("DISABLED: getifaddrs module");
- return 1;
- } else {
- #define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s)
- struct ifaddrs *ifa;
- struct iftot {
- u_long ift_ibytes;
- u_long ift_obytes;
- } iftot = {0, 0};
-
- // --------------------------------------------------------------------
-
- if (likely(do_bandwidth_ipv4)) {
- iftot.ift_ibytes = iftot.ift_obytes = 0;
- for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
- if (ifa->ifa_addr->sa_family != AF_INET)
- continue;
- iftot.ift_ibytes += IFA_DATA(ibytes);
- iftot.ift_obytes += IFA_DATA(obytes);
- }
-
- static RRDSET *st = NULL;
- static RRDDIM *rd_in = NULL, *rd_out = NULL;
-
- if (unlikely(!st)) {
- st = rrdset_create_localhost("system",
- "ipv4",
- NULL,
- "network",
- NULL,
- "IPv4 Bandwidth",
- "kilobits/s",
- 500,
- update_every,
- RRDSET_TYPE_AREA
- );
-
- rd_in = rrddim_add(st, "InOctets", "received", 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
- rd_out = rrddim_add(st, "OutOctets", "sent", -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(st);
-
- rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes);
- rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes);
- rrdset_done(st);
- }
-
- // --------------------------------------------------------------------
-
- if (likely(do_bandwidth_ipv6)) {
- iftot.ift_ibytes = iftot.ift_obytes = 0;
- for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
- if (ifa->ifa_addr->sa_family != AF_INET6)
- continue;
- iftot.ift_ibytes += IFA_DATA(ibytes);
- iftot.ift_obytes += IFA_DATA(obytes);
- }
-
- static RRDSET *st = NULL;
- static RRDDIM *rd_in = NULL, *rd_out = NULL;
-
- if (unlikely(!st)) {
- st = rrdset_create_localhost("system",
- "ipv6",
- NULL,
- "network",
- NULL,
- "IPv6 Bandwidth",
- "kilobits/s",
- 500,
- update_every,
- RRDSET_TYPE_AREA
- );
-
- rd_in = rrddim_add(st, "received", NULL, 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
- rd_out = rrddim_add(st, "sent", NULL, -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(st);
-
- rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes);
- rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes);
- rrdset_done(st);
- }
-
- // --------------------------------------------------------------------
-
- // Data to be stored in DICTIONARY interfaces.
- // This DICTIONARY is used to lookup the settings of the interfaces on each iteration.
- struct interfaces_metadata {
- int do_bandwidth;
- int do_packets;
- int do_errors;
- int do_drops;
- int do_events;
-
- // charts and dimensions
-
- RRDSET *st_bandwidth;
- RRDDIM *rd_bandwidth_in;
- RRDDIM *rd_bandwidth_out;
-
- RRDSET *st_packets;
- RRDDIM *rd_packets_in;
- RRDDIM *rd_packets_out;
- RRDDIM *rd_packets_m_in;
- RRDDIM *rd_packets_m_out;
-
- RRDSET *st_errors;
- RRDDIM *rd_errors_in;
- RRDDIM *rd_errors_out;
-
- RRDSET *st_drops;
- RRDDIM *rd_drops_in;
- RRDDIM *rd_drops_out;
-
- RRDSET *st_events;
- RRDDIM *rd_events_coll;
- };
- static DICTIONARY *interfaces = NULL;
- static SIMPLE_PATTERN *excluded_interfaces = NULL;
-
- if(unlikely(!interfaces)) {
-
- excluded_interfaces = simple_pattern_create(
- config_get(CONFIG_SECTION_GETIFADDRS, "disable by default interfaces matching",
- DELAULT_EXLUDED_INTERFACES)
- , SIMPLE_PATTERN_EXACT
- );
-
- interfaces = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
- }
-
- for (ifa = ifap; ifa; ifa = ifa->ifa_next) {
- if (ifa->ifa_addr->sa_family != AF_LINK)
- continue;
-
- int def_bandwidth, def_packets, def_errors, def_drops, def_events,
- iter_bandwidth, iter_packets, iter_errors, iter_drops, iter_events;
-
- struct interfaces_metadata *ifm = dictionary_get(interfaces, ifa->ifa_name);
- if(unlikely(!ifm)) {
- char var_name[4096 + 1];
- snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETIFADDRS, ifa->ifa_name);
-
- def_bandwidth = do_bandwidth;
- def_packets = do_packets;
- def_errors = do_errors;
- def_drops = do_drops;
- def_events = do_events;
-
- if(unlikely(simple_pattern_matches(excluded_interfaces, ifa->ifa_name))) {
- def_bandwidth = CONFIG_BOOLEAN_NO;
- def_packets = CONFIG_BOOLEAN_NO;
- def_errors = CONFIG_BOOLEAN_NO;
- def_drops = CONFIG_BOOLEAN_NO;
- def_events = CONFIG_BOOLEAN_NO;
- }
-
- iter_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", def_bandwidth);
- iter_packets = config_get_boolean_ondemand(var_name, "packets", def_packets);
- iter_errors = config_get_boolean_ondemand(var_name, "errors", def_errors);
- iter_drops = config_get_boolean_ondemand(var_name, "drops", def_drops);
- iter_events = config_get_boolean_ondemand(var_name, "events", def_events);
-
- struct interfaces_metadata ifmp = {
- .do_bandwidth = iter_bandwidth,
- .do_packets = iter_packets,
- .do_errors = iter_errors,
- .do_drops = iter_drops,
- .do_events = iter_events,
-
- .st_bandwidth = NULL,
- .rd_bandwidth_in = NULL,
- .rd_bandwidth_out = NULL,
-
- .st_packets = NULL,
- .rd_packets_in = NULL,
- .rd_packets_out = NULL,
- .rd_packets_m_in = NULL,
- .rd_packets_m_out = NULL,
-
- .st_errors = NULL,
- .rd_errors_in = NULL,
- .rd_errors_out = NULL,
-
- .st_drops = NULL,
- .rd_drops_in = NULL,
- .rd_drops_out = NULL,
-
- .st_events = NULL,
- .rd_events_coll = NULL,
- };
-
- ifm = dictionary_set(interfaces, ifa->ifa_name, &ifmp, sizeof(struct interfaces_metadata));
- }
-
- // --------------------------------------------------------------------
-
- if (ifm->do_bandwidth == CONFIG_BOOLEAN_YES || (ifm->do_bandwidth == CONFIG_BOOLEAN_AUTO &&
- (IFA_DATA(ibytes) || IFA_DATA(obytes)))) {
- if (unlikely(!ifm->st_bandwidth)) {
- ifm->st_bandwidth = rrdset_create_localhost("net",
- ifa->ifa_name,
- NULL,
- ifa->ifa_name,
- "net.net",
- "Bandwidth",
- "kilobits/s",
- 7000,
- update_every,
- RRDSET_TYPE_AREA
- );
-
- ifm->rd_bandwidth_in = rrddim_add(ifm->st_bandwidth, "received", NULL, 8, KILO_FACTOR,
- RRD_ALGORITHM_INCREMENTAL);
- ifm->rd_bandwidth_out = rrddim_add(ifm->st_bandwidth, "sent", NULL, -8, KILO_FACTOR,
- RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(ifm->st_bandwidth);
-
- rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_in, IFA_DATA(ibytes));
- rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_out, IFA_DATA(obytes));
- rrdset_done(ifm->st_bandwidth);
- }
-
- // --------------------------------------------------------------------
-
- if (ifm->do_packets == CONFIG_BOOLEAN_YES || (ifm->do_packets == CONFIG_BOOLEAN_AUTO &&
- (IFA_DATA(ipackets) || IFA_DATA(opackets) || IFA_DATA(imcasts) || IFA_DATA(omcasts)))) {
- if (unlikely(!ifm->st_packets)) {
- ifm->st_packets = rrdset_create_localhost("net_packets",
- ifa->ifa_name,
- NULL,
- ifa->ifa_name,
- "net.packets",
- "Packets",
- "packets/s",
- 7001,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(ifm->st_packets, RRDSET_FLAG_DETAIL);
-
- ifm->rd_packets_in = rrddim_add(ifm->st_packets, "received", NULL, 1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- ifm->rd_packets_out = rrddim_add(ifm->st_packets, "sent", NULL, -1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- ifm->rd_packets_m_in = rrddim_add(ifm->st_packets, "multicast_received", NULL, 1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- ifm->rd_packets_m_out = rrddim_add(ifm->st_packets, "multicast_sent", NULL, -1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(ifm->st_packets);
-
- rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_in, IFA_DATA(ipackets));
- rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_out, IFA_DATA(opackets));
- rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_in, IFA_DATA(imcasts));
- rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_out, IFA_DATA(omcasts));
- rrdset_done(ifm->st_packets);
- }
-
- // --------------------------------------------------------------------
-
- if (ifm->do_errors == CONFIG_BOOLEAN_YES || (ifm->do_errors == CONFIG_BOOLEAN_AUTO &&
- (IFA_DATA(ierrors) || IFA_DATA(oerrors)))) {
- if (unlikely(!ifm->st_errors)) {
- ifm->st_errors = rrdset_create_localhost("net_errors",
- ifa->ifa_name,
- NULL,
- ifa->ifa_name,
- "net.errors",
- "Interface Errors",
- "errors/s",
- 7002,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(ifm->st_errors, RRDSET_FLAG_DETAIL);
-
- ifm->rd_errors_in = rrddim_add(ifm->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- ifm->rd_errors_out = rrddim_add(ifm->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(ifm->st_errors);
-
- rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_in, IFA_DATA(ierrors));
- rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_out, IFA_DATA(oerrors));
- rrdset_done(ifm->st_errors);
- }
- // --------------------------------------------------------------------
-
- if (ifm->do_drops == CONFIG_BOOLEAN_YES || (ifm->do_drops == CONFIG_BOOLEAN_AUTO &&
- (IFA_DATA(iqdrops) || IFA_DATA(oqdrops)))) {
- if (unlikely(!ifm->st_drops)) {
- ifm->st_drops = rrdset_create_localhost("net_drops",
- ifa->ifa_name,
- NULL,
- ifa->ifa_name,
- "net.drops",
- "Interface Drops",
- "drops/s",
- 7003,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(ifm->st_drops, RRDSET_FLAG_DETAIL);
-
- ifm->rd_drops_in = rrddim_add(ifm->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
-#if __FreeBSD__ >= 11
- ifm->rd_drops_out = rrddim_add(ifm->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
-#endif
- } else
- rrdset_next(ifm->st_drops);
-
- rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_in, IFA_DATA(iqdrops));
-#if __FreeBSD__ >= 11
- rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_out, IFA_DATA(oqdrops));
-#endif
- rrdset_done(ifm->st_drops);
- }
-
- // --------------------------------------------------------------------
-
- if (ifm->do_events == CONFIG_BOOLEAN_YES || (ifm->do_events == CONFIG_BOOLEAN_AUTO &&
- IFA_DATA(collisions))) {
- if (unlikely(!ifm->st_events)) {
- ifm->st_events = rrdset_create_localhost("net_events",
- ifa->ifa_name,
- NULL,
- ifa->ifa_name,
- "net.events",
- "Network Interface Events",
- "events/s",
- 7006,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(ifm->st_events, RRDSET_FLAG_DETAIL);
-
- ifm->rd_events_coll = rrddim_add(ifm->st_events, "collisions", NULL, -1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(ifm->st_events);
-
- rrddim_set_by_pointer(ifm->st_events, ifm->rd_events_coll, IFA_DATA(collisions));
- rrdset_done(ifm->st_events);
- }
- }
-
- freeifaddrs(ifap);
- }
- } else {
- error("DISABLED: getifaddrs module");
- return 1;
- }
-
- return 0;
-}
-
-// --------------------------------------------------------------------------------------------------------------------
-// kern.devstat
-
-int do_kern_devstat(int update_every, usec_t dt) {
-
-#define DELAULT_EXLUDED_DISKS ""
-#define CONFIG_SECTION_KERN_DEVSTAT "plugin:freebsd:kern.devstat"
-#define BINTIME_SCALE 5.42101086242752217003726400434970855712890625e-17 // this is 1000/2^64
-
- static int enable_pass_devices = -1, do_system_io = -1, do_io = -1, do_ops = -1, do_qops = -1, do_util = -1,
- do_iotime = -1, do_await = -1, do_avagsz = -1, do_svctm = -1;
-
- if (unlikely(enable_pass_devices == -1)) {
- enable_pass_devices = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT,
- "performance metrics for pass devices", CONFIG_BOOLEAN_AUTO);
-
- do_system_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "total bandwidth for all disks",
- CONFIG_BOOLEAN_YES);
-
- do_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "bandwidth for all disks",
- CONFIG_BOOLEAN_AUTO);
- do_ops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "operations for all disks",
- CONFIG_BOOLEAN_AUTO);
- do_qops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "queued operations for all disks",
- CONFIG_BOOLEAN_AUTO);
- do_util = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "utilization percentage for all disks",
- CONFIG_BOOLEAN_AUTO);
- do_iotime = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "i/o time for all disks",
- CONFIG_BOOLEAN_AUTO);
- do_await = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o time for all disks",
- CONFIG_BOOLEAN_AUTO);
- do_avagsz = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o bandwidth for all disks",
- CONFIG_BOOLEAN_AUTO);
- do_svctm = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average service time for all disks",
- CONFIG_BOOLEAN_AUTO);
- }
-
- if (likely(do_system_io || do_io || do_ops || do_qops || do_util || do_iotime || do_await || do_avagsz || do_svctm)) {
- static int mib_numdevs[3] = {0, 0, 0};
- int numdevs;
- int common_error = 0;
-
- if (unlikely(GETSYSCTL_SIMPLE("kern.devstat.numdevs", mib_numdevs, numdevs))) {
- common_error = 1;
- } else {
- static int mib_devstat[3] = {0, 0, 0};
- static void *devstat_data = NULL;
-
- devstat_data = reallocz(devstat_data, sizeof(long) + sizeof(struct devstat) * numdevs); // there is generation number before devstat structures
- if (unlikely(GETSYSCTL_WSIZE("kern.devstat.all", mib_devstat, devstat_data,
- sizeof(long) + sizeof(struct devstat) * numdevs))) {
- common_error = 1;
- } else {
- struct devstat *dstat;
- int i;
- collected_number total_disk_kbytes_read = 0;
- collected_number total_disk_kbytes_write = 0;
-
- // Data to be stored in DICTIONARY disks.
- // This DICTIONARY is used to lookup the settings of the disks on each iteration.
- struct disks_metadata {
- int do_io;
- int do_ops;
- int do_qops;
- int do_util;
- int do_iotime;
- int do_await;
- int do_avagsz;
- int do_svctm;
-
-
- // data for differential charts
-
- struct prev_dstat {
- collected_number bytes_read;
- collected_number bytes_write;
- collected_number operations_read;
- collected_number operations_write;
- collected_number duration_read_ms;
- collected_number duration_write_ms;
- collected_number busy_time_ms;
- } prev_dstat;
-
- // charts and dimensions
-
- RRDSET *st_io;
- RRDDIM *rd_io_in;
- RRDDIM *rd_io_out;
-
- RRDSET *st_ops;
- RRDDIM *rd_ops_in;
- RRDDIM *rd_ops_out;
-
- RRDSET *st_qops;
- RRDDIM *rd_qops;
-
- RRDSET *st_util;
- RRDDIM *rd_util;
-
- RRDSET *st_iotime;
- RRDDIM *rd_iotime_in;
- RRDDIM *rd_iotime_out;
-
- RRDSET *st_await;
- RRDDIM *rd_await_in;
- RRDDIM *rd_await_out;
-
- RRDSET *st_avagsz;
- RRDDIM *rd_avagsz_in;
- RRDDIM *rd_avagsz_out;
-
- RRDSET *st_svctm;
- RRDDIM *rd_svctm;
-
- };
- static DICTIONARY *disks = NULL;
- static SIMPLE_PATTERN *excluded_disks = NULL;
-
- if(unlikely(!disks)) {
-
- excluded_disks = simple_pattern_create(
- config_get(CONFIG_SECTION_KERN_DEVSTAT, "disable by default disks matching",
- DELAULT_EXLUDED_DISKS)
- , SIMPLE_PATTERN_EXACT
- );
-
- disks = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
- }
-
- dstat = devstat_data + sizeof(long); // skip generation number
-
- for (i = 0; i < numdevs; i++) {
- if (likely(do_system_io)) {
- if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) {
- total_disk_kbytes_read += dstat[i].bytes[DEVSTAT_READ] / KILO_FACTOR;
- total_disk_kbytes_write += dstat[i].bytes[DEVSTAT_WRITE] / KILO_FACTOR;
- }
- }
-
- if (unlikely(!enable_pass_devices))
- if ((dstat[i].device_type & DEVSTAT_TYPE_PASS) == DEVSTAT_TYPE_PASS)
- continue;
-
- if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) {
- char disk[DEVSTAT_NAME_LEN + MAX_INT_DIGITS + 1];
- int def_io, def_ops, def_qops, def_util, def_iotime, def_await, def_avagsz, def_svctm,
- iter_io, iter_ops, iter_qops, iter_util, iter_iotime, iter_await, iter_avagsz, iter_svctm;
- struct cur_dstat {
- collected_number duration_read_ms;
- collected_number duration_write_ms;
- collected_number busy_time_ms;
- } cur_dstat;
-
- sprintf(disk, "%s%d", dstat[i].device_name, dstat[i].unit_number);
-
- struct disks_metadata *dm = dictionary_get(disks, disk);
- if(unlikely(!dm)) {
- char var_name[4096 + 1];
- snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_KERN_DEVSTAT, disk);
-
- def_io = do_io;
- def_ops = do_ops;
- def_qops = do_qops;
- def_util = do_util;
- def_iotime = do_iotime;
- def_await = do_await;
- def_avagsz = do_avagsz;
- def_svctm = do_svctm;
-
- if(unlikely(simple_pattern_matches(excluded_disks, disk))) {
- def_io = CONFIG_BOOLEAN_NO;
- def_ops = CONFIG_BOOLEAN_NO;
- def_qops = CONFIG_BOOLEAN_NO;
- def_util = CONFIG_BOOLEAN_NO;
- def_iotime = CONFIG_BOOLEAN_NO;
- def_await = CONFIG_BOOLEAN_NO;
- def_avagsz = CONFIG_BOOLEAN_NO;
- def_svctm = CONFIG_BOOLEAN_NO;
- }
-
- iter_io = config_get_boolean_ondemand(var_name, "bandwidth", def_io);
- iter_ops = config_get_boolean_ondemand(var_name, "operations", def_ops);
- iter_qops = config_get_boolean_ondemand(var_name, "queued operations", def_qops);
- iter_util = config_get_boolean_ondemand(var_name, "utilization percentage", def_util);
- iter_iotime = config_get_boolean_ondemand(var_name, "i/o time", def_iotime);
- iter_await = config_get_boolean_ondemand(var_name, "average completed i/o time", def_await);
- iter_avagsz = config_get_boolean_ondemand(var_name, "average completed i/o bandwidth",
- def_avagsz);
- iter_svctm = config_get_boolean_ondemand(var_name, "average service time", def_svctm);
-
- struct disks_metadata dmp = {
- .do_io = iter_io,
- .do_ops = iter_ops,
- .do_qops = iter_qops,
- .do_util = iter_util,
- .do_iotime = iter_iotime,
- .do_await = iter_await,
- .do_avagsz = iter_avagsz,
- .do_svctm = iter_svctm,
-
- .st_io = NULL,
- .rd_io_in = NULL,
- .rd_io_out = NULL,
-
- .st_ops = NULL,
- .rd_ops_in = NULL,
- .rd_ops_out = NULL,
-
- .st_qops = NULL,
- .rd_qops = NULL,
-
- .st_util = NULL,
- .rd_util = NULL,
-
- .st_iotime = NULL,
- .rd_iotime_in = NULL,
- .rd_iotime_out = NULL,
-
- .st_await = NULL,
- .rd_await_in = NULL,
- .rd_await_out = NULL,
-
- .st_avagsz = NULL,
- .rd_avagsz_in = NULL,
- .rd_avagsz_out = NULL,
-
- .st_svctm = NULL,
- .rd_svctm = NULL,
- };
-
- // initialise data for differential charts
-
- dmp.prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ];
- dmp.prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE];
- dmp.prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ];
- dmp.prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE];
- dmp.prev_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000
- + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
- dmp.prev_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000
- + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
- dmp.prev_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000
- + dstat[i].busy_time.frac * BINTIME_SCALE;
-
- dm = dictionary_set(disks, disk, &dmp, sizeof(struct disks_metadata));
- }
-
- cur_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000
- + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
- cur_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000
- + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE;
- cur_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE;
-
- // --------------------------------------------------------------------
-
- if(dm->do_io == CONFIG_BOOLEAN_YES || (dm->do_io == CONFIG_BOOLEAN_AUTO &&
- (dstat[i].bytes[DEVSTAT_READ] || dstat[i].bytes[DEVSTAT_WRITE]))) {
- if (unlikely(!dm->st_io)) {
- dm->st_io = rrdset_create_localhost("disk",
- disk,
- NULL,
- disk,
- "disk.io",
- "Disk I/O Bandwidth",
- "kilobytes/s",
- 2000,
- update_every,
- RRDSET_TYPE_AREA
- );
-
- dm->rd_io_in = rrddim_add(dm->st_io, "reads", NULL, 1, KILO_FACTOR,
- RRD_ALGORITHM_INCREMENTAL);
- dm->rd_io_out = rrddim_add(dm->st_io, "writes", NULL, -1, KILO_FACTOR,
- RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(dm->st_io);
-
- rrddim_set_by_pointer(dm->st_io, dm->rd_io_in, dstat[i].bytes[DEVSTAT_READ]);
- rrddim_set_by_pointer(dm->st_io, dm->rd_io_out, dstat[i].bytes[DEVSTAT_WRITE]);
- rrdset_done(dm->st_io);
- }
-
- // --------------------------------------------------------------------
-
- if(dm->do_ops == CONFIG_BOOLEAN_YES || (dm->do_ops == CONFIG_BOOLEAN_AUTO &&
- (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
- if (unlikely(!dm->st_ops)) {
- dm->st_ops = rrdset_create_localhost("disk_ops",
- disk,
- NULL,
- disk,
- "disk.ops",
- "Disk Completed I/O Operations",
- "operations/s",
- 2001,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(dm->st_ops, RRDSET_FLAG_DETAIL);
-
- dm->rd_ops_in = rrddim_add(dm->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- dm->rd_ops_out = rrddim_add(dm->st_ops, "writes", NULL, -1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(dm->st_ops);
-
- rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_in, dstat[i].operations[DEVSTAT_READ]);
- rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_out, dstat[i].operations[DEVSTAT_WRITE]);
- rrdset_done(dm->st_ops);
- }
-
- // --------------------------------------------------------------------
-
- if(dm->do_qops == CONFIG_BOOLEAN_YES || (dm->do_qops == CONFIG_BOOLEAN_AUTO &&
- (dstat[i].start_count || dstat[i].end_count))) {
- if (unlikely(!dm->st_qops)) {
- dm->st_qops = rrdset_create_localhost("disk_qops",
- disk,
- NULL,
- disk,
- "disk.qops",
- "Disk Current I/O Operations",
- "operations",
- 2002,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(dm->st_qops, RRDSET_FLAG_DETAIL);
-
- dm->rd_qops = rrddim_add(dm->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
- } else
- rrdset_next(dm->st_qops);
-
- rrddim_set_by_pointer(dm->st_qops, dm->rd_qops, dstat[i].start_count - dstat[i].end_count);
- rrdset_done(dm->st_qops);
- }
-
- // --------------------------------------------------------------------
-
- if(dm->do_util == CONFIG_BOOLEAN_YES || (dm->do_util == CONFIG_BOOLEAN_AUTO &&
- cur_dstat.busy_time_ms)) {
- if (unlikely(!dm->st_util)) {
- dm->st_util = rrdset_create_localhost("disk_util",
- disk,
- NULL,
- disk,
- "disk.util",
- "Disk Utilization Time",
- "% of time working",
- 2004,
- update_every,
- RRDSET_TYPE_AREA
- );
-
- rrdset_flag_set(dm->st_util, RRDSET_FLAG_DETAIL);
-
- dm->rd_util = rrddim_add(dm->st_util, "utilization", NULL, 1, 10,
- RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(dm->st_util);
-
- rrddim_set_by_pointer(dm->st_util, dm->rd_util, cur_dstat.busy_time_ms);
- rrdset_done(dm->st_util);
- }
-
- // --------------------------------------------------------------------
-
- if(dm->do_iotime == CONFIG_BOOLEAN_YES || (dm->do_iotime == CONFIG_BOOLEAN_AUTO &&
- (cur_dstat.duration_read_ms || cur_dstat.duration_write_ms))) {
- if (unlikely(!dm->st_iotime)) {
- dm->st_iotime = rrdset_create_localhost("disk_iotime",
- disk,
- NULL,
- disk,
- "disk.iotime",
- "Disk Total I/O Time",
- "milliseconds/s",
- 2022,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(dm->st_iotime, RRDSET_FLAG_DETAIL);
-
- dm->rd_iotime_in = rrddim_add(dm->st_iotime, "reads", NULL, 1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- dm->rd_iotime_out = rrddim_add(dm->st_iotime, "writes", NULL, -1, 1,
- RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(dm->st_iotime);
-
- rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_in, cur_dstat.duration_read_ms);
- rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_out, cur_dstat.duration_write_ms);
- rrdset_done(dm->st_iotime);
- }
-
- // --------------------------------------------------------------------
- // calculate differential charts
- // only if this is not the first time we run
-
- if (likely(dt)) {
-
- // --------------------------------------------------------------------
-
- if(dm->do_await == CONFIG_BOOLEAN_YES || (dm->do_await == CONFIG_BOOLEAN_AUTO &&
- (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
- if (unlikely(!dm->st_await)) {
- dm->st_await = rrdset_create_localhost("disk_await",
- disk,
- NULL,
- disk,
- "disk.await",
- "Average Completed I/O Operation Time",
- "ms per operation",
- 2005,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(dm->st_await, RRDSET_FLAG_DETAIL);
-
- dm->rd_await_in = rrddim_add(dm->st_await, "reads", NULL, 1, 1,
- RRD_ALGORITHM_ABSOLUTE);
- dm->rd_await_out = rrddim_add(dm->st_await, "writes", NULL, -1, 1,
- RRD_ALGORITHM_ABSOLUTE);
- } else
- rrdset_next(dm->st_await);
-
- rrddim_set_by_pointer(dm->st_await, dm->rd_await_in,
- (dstat[i].operations[DEVSTAT_READ] -
- dm->prev_dstat.operations_read) ?
- (cur_dstat.duration_read_ms - dm->prev_dstat.duration_read_ms) /
- (dstat[i].operations[DEVSTAT_READ] -
- dm->prev_dstat.operations_read) :
- 0);
- rrddim_set_by_pointer(dm->st_await, dm->rd_await_out,
- (dstat[i].operations[DEVSTAT_WRITE] -
- dm->prev_dstat.operations_write) ?
- (cur_dstat.duration_write_ms - dm->prev_dstat.duration_write_ms) /
- (dstat[i].operations[DEVSTAT_WRITE] -
- dm->prev_dstat.operations_write) :
- 0);
- rrdset_done(dm->st_await);
- }
-
- // --------------------------------------------------------------------
-
- if(dm->do_avagsz == CONFIG_BOOLEAN_YES || (dm->do_avagsz == CONFIG_BOOLEAN_AUTO &&
- (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
- if (unlikely(!dm->st_avagsz)) {
- dm->st_avagsz = rrdset_create_localhost("disk_avgsz",
- disk,
- NULL,
- disk,
- "disk.avgsz",
- "Average Completed I/O Operation Bandwidth",
- "kilobytes per operation",
- 2006,
- update_every,
- RRDSET_TYPE_AREA
- );
-
- rrdset_flag_set(dm->st_avagsz, RRDSET_FLAG_DETAIL);
-
- dm->rd_avagsz_in = rrddim_add(dm->st_avagsz, "reads", NULL, 1, KILO_FACTOR,
- RRD_ALGORITHM_ABSOLUTE);
- dm->rd_avagsz_out = rrddim_add(dm->st_avagsz, "writes", NULL, -1, KILO_FACTOR,
- RRD_ALGORITHM_ABSOLUTE);
- } else
- rrdset_next(dm->st_avagsz);
-
- rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_in,
- (dstat[i].operations[DEVSTAT_READ] -
- dm->prev_dstat.operations_read) ?
- (dstat[i].bytes[DEVSTAT_READ] - dm->prev_dstat.bytes_read) /
- (dstat[i].operations[DEVSTAT_READ] -
- dm->prev_dstat.operations_read) :
- 0);
- rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_out,
- (dstat[i].operations[DEVSTAT_WRITE] -
- dm->prev_dstat.operations_write) ?
- (dstat[i].bytes[DEVSTAT_WRITE] - dm->prev_dstat.bytes_write) /
- (dstat[i].operations[DEVSTAT_WRITE] -
- dm->prev_dstat.operations_write) :
- 0);
- rrdset_done(dm->st_avagsz);
- }
-
- // --------------------------------------------------------------------
-
- if(dm->do_svctm == CONFIG_BOOLEAN_YES || (dm->do_svctm == CONFIG_BOOLEAN_AUTO &&
- (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) {
- if (unlikely(!dm->st_svctm)) {
- dm->st_svctm = rrdset_create_localhost("disk_svctm",
- disk,
- NULL,
- disk,
- "disk.svctm",
- "Average Service Time",
- "ms per operation",
- 2007,
- update_every,
- RRDSET_TYPE_LINE
- );
-
- rrdset_flag_set(dm->st_svctm, RRDSET_FLAG_DETAIL);
-
- dm->rd_svctm = rrddim_add(dm->st_svctm, "svctm", NULL, 1, 1,
- RRD_ALGORITHM_ABSOLUTE);
- } else
- rrdset_next(dm->st_svctm);
-
- rrddim_set_by_pointer(dm->st_svctm, dm->rd_svctm,
- ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) +
- (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) ?
- (cur_dstat.busy_time_ms - dm->prev_dstat.busy_time_ms) /
- ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) +
- (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) :
- 0);
- rrdset_done(dm->st_svctm);
- }
-
- // --------------------------------------------------------------------
-
- dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ];
- dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE];
- dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ];
- dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE];
- dm->prev_dstat.duration_read_ms = cur_dstat.duration_read_ms;
- dm->prev_dstat.duration_write_ms = cur_dstat.duration_write_ms;
- dm->prev_dstat.busy_time_ms = cur_dstat.busy_time_ms;
- }
- }
- }
-
- // --------------------------------------------------------------------
-
- if (likely(do_system_io)) {
- static RRDSET *st = NULL;
- static RRDDIM *rd_in = NULL, *rd_out = NULL;
-
- if (unlikely(!st)) {
- st = rrdset_create_localhost("system",
- "io",
- NULL,
- "disk",
- NULL,
- "Disk I/O",
- "kilobytes/s",
- 150,
- update_every,
- RRDSET_TYPE_AREA
- );
-
- rd_in = rrddim_add(st, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rd_out = rrddim_add(st, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
- } else
- rrdset_next(st);
-
- rrddim_set_by_pointer(st, rd_in, total_disk_kbytes_read);
- rrddim_set_by_pointer(st, rd_out, total_disk_kbytes_write);
- rrdset_done(st);
- }
- }
- }
- if (unlikely(common_error)) {
- do_system_io = 0;
- error("DISABLED: system.io chart");
- do_io = 0;
- error("DISABLED: disk.* charts");
- do_ops = 0;
- error("DISABLED: disk_ops.* charts");
- do_qops = 0;
- error("DISABLED: disk_qops.* charts");
- do_util = 0;
- error("DISABLED: disk_util.* charts");
- do_iotime = 0;
- error("DISABLED: disk_iotime.* charts");
- do_await = 0;
- error("DISABLED: disk_await.* charts");
- do_avagsz = 0;
- error("DISABLED: disk_avgsz.* charts");
- do_svctm = 0;
- error("DISABLED: disk_svctm.* charts");
- error("DISABLED: kern.devstat module");
- return 1;
- }
- } else {
- error("DISABLED: kern.devstat module");
- return 1;
- }
-
- return 0;
-}
diff --git a/src/freeipmi_plugin.c b/src/freeipmi_plugin.c
index 4459de7ca..146268a53 100644
--- a/src/freeipmi_plugin.c
+++ b/src/freeipmi_plugin.c
@@ -242,7 +242,7 @@ _get_sensor_type_string (int sensor_type)
static int debug = 0;
-static int netdata_update_every = 5;
+static int netdata_update_every = 5; // this is the minimum update frequency
static int netdata_priority = 90000;
static int netdata_do_sel = 1;
@@ -1403,7 +1403,7 @@ int ipmi_detect_speed_secs(struct ipmi_monitoring_ipmi_config *ipmi_config) {
// we find the average in microseconds
// and we round-up to the closest second
- return (( total * 2 / checks / 1000000 ) + 1);
+ return (int)(( total * 2 / checks / 1000000 ) + 1);
}
int main (int argc, char **argv) {
@@ -1426,15 +1426,14 @@ int main (int argc, char **argv) {
int i, freq = 0;
for(i = 1; i < argc ; i++) {
- if(!freq) {
+ if(isdigit(*argv[i]) && !freq) {
int n = atoi(argv[i]);
- if(n > 0) {
+ if(n > 0 && freq < 86400) {
freq = n;
continue;
}
}
-
- if(strcmp("version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) {
+ else if(strcmp("version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) {
printf("freeipmi.plugin %s\n", VERSION);
exit(0);
}
@@ -1568,7 +1567,7 @@ int main (int argc, char **argv) {
freq = ipmi_detect_speed_secs(&ipmi_config);
if(debug) fprintf(stderr, "freeipmi.plugin: IPMI minimum update frequency was calculated to %d seconds.\n", freq);
- if(netdata_update_every < freq) {
+ if(freq > netdata_update_every) {
info("enforcing minimum data collection frequency, calculated to %d seconds.", freq);
netdata_update_every = freq;
}
diff --git a/src/health.c b/src/health.c
index 46b27db6f..cc470f81f 100644
--- a/src/health.c
+++ b/src/health.c
@@ -148,7 +148,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s' '%s' '%s'",
exec,
recipient,
- host->hostname,
+ host->registry_hostname,
ae->unique_id,
ae->alarm_id,
ae->alarm_event_id,
@@ -356,9 +356,16 @@ void *health_main(void *ptr) {
// detect if boottime and realtime have twice the difference
// in which case we assume the system was just waken from hibernation
- if(unlikely(now - last_now > 2 * (now_boottime - last_now_boottime)))
+ if(unlikely(now - last_now > 2 * (now_boottime - last_now_boottime))) {
apply_hibernation_delay = 1;
+ info("Postponing alarm checks for %ld seconds, due to boottime discrepancy (realtime dt: %ld, boottime dt: %ld)."
+ , hibernation_delay
+ , (long)(now - last_now)
+ , (long)(now_boottime - last_now_boottime)
+ );
+ }
+
last_now = now;
last_now_boottime = now_boottime;
@@ -374,11 +381,9 @@ void *health_main(void *ptr) {
if(unlikely(apply_hibernation_delay)) {
- info("Postponing alarm checks for %ld seconds, on host '%s', due to boottime discrepancy (realtime dt: %ld, boottime dt: %ld)."
+ info("Postponing alarm checks for %ld seconds, on host '%s'."
, hibernation_delay
, host->hostname
- , (long)(now - last_now)
- , (long)(now_boottime - last_now_boottime)
);
host->health_delay_up_to = now + hibernation_delay;
diff --git a/src/health_config.c b/src/health_config.c
index ad954cbe1..b4655dc78 100644
--- a/src/health_config.c
+++ b/src/health_config.c
@@ -32,8 +32,8 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
return 0;
}
- if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) {
- error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name);
+ if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->calculation && !rc->warning && !rc->critical) {
+ error("Health configuration for alarm '%s.%s' is useless (no db lookup, no calculation, no warning and no critical expressions)", rc->chart?rc->chart:"NOCHART", rc->name);
return 0;
}
@@ -136,7 +136,7 @@ static inline int health_parse_duration(char *string, int *result) {
}
char *e = NULL;
- calculated_number n = strtold(string, &e);
+ calculated_number n = str2ld(string, &e);
if(e && *e) {
switch (*e) {
case 'Y':
@@ -241,10 +241,10 @@ static inline int health_parse_delay(
if(!given_max) {
if((*delay_max_duration) < (*delay_up_duration) * (*delay_multiplier))
- *delay_max_duration = (*delay_up_duration) * (*delay_multiplier);
+ *delay_max_duration = (int)((*delay_up_duration) * (*delay_multiplier));
if((*delay_max_duration) < (*delay_down_duration) * (*delay_multiplier))
- *delay_max_duration = (*delay_down_duration) * (*delay_multiplier);
+ *delay_max_duration = (int)((*delay_down_duration) * (*delay_multiplier));
}
return 1;
@@ -381,37 +381,6 @@ static inline int health_parse_db_lookup(
return 1;
}
-static inline char *trim_all_spaces(char *buffer) {
- char *d = buffer, *s = buffer;
-
- // skip spaces
- while(isspace(*s)) s++;
-
- while(*s) {
- // copy the non-space part
- while(*s && !isspace(*s)) *d++ = *s++;
-
- // add a space if we have to
- if(*s && isspace(*s)) {
- *d++ = ' ';
- s++;
- }
-
- // skip spaces
- while(isspace(*s)) s++;
- }
-
- *d = '\0';
-
- if(d > buffer) {
- d--;
- if(isspace(*d)) *d = '\0';
- }
-
- if(!buffer[0]) return NULL;
- return buffer;
-}
-
static inline char *health_source_file(size_t line, const char *path, const char *filename) {
char buffer[FILENAME_MAX + 1];
snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
@@ -485,7 +454,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
int stop_appending = !s;
line++;
s = trim(buffer);
- if(!s) continue;
+ if(!s || *s == '#') continue;
append = strlen(s);
if(!stop_appending && s[append - 1] == '\\') {
@@ -509,8 +478,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
s++;
char *value = s;
- key = trim_all_spaces(key);
- value = trim_all_spaces(value);
+ key = trim_all(key);
+ value = trim_all(value);
if(!key) {
error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
@@ -593,7 +562,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
}
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
char *e;
- rc->green = strtold(value, &e);
+ rc->green = str2ld(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
line, path, filename, rc->name, key, e);
@@ -601,7 +570,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
}
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
char *e;
- rc->red = strtold(value, &e);
+ rc->red = str2ld(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
line, path, filename, rc->name, key, e);
@@ -717,7 +686,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
}
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
char *e;
- rt->green = strtold(value, &e);
+ rt->green = str2ld(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
line, path, filename, rt->name, key, e);
@@ -725,7 +694,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
}
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
char *e;
- rt->red = strtold(value, &e);
+ rt->red = str2ld(value, &e);
if(e && *e) {
error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
line, path, filename, rt->name, key, e);
diff --git a/src/inlined.h b/src/inlined.h
index 0dc11c950..f1812ba1c 100644
--- a/src/inlined.h
+++ b/src/inlined.h
@@ -123,6 +123,103 @@ static inline unsigned long long str2ull(const char *s) {
return n;
}
+static inline long long str2ll(const char *s, char **endptr) {
+ int negative = 0;
+
+ if(unlikely(*s == '-')) {
+ s++;
+ negative = 1;
+ }
+ else if(unlikely(*s == '+'))
+ s++;
+
+ long long n = 0;
+ char c;
+ for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+
+ if(unlikely(endptr))
+ *endptr = (char *)s;
+
+ if(unlikely(negative))
+ return -n;
+ else
+ return n;
+}
+
+static inline long double str2ld(const char *s, char **endptr) {
+ int negative = 0;
+ const char *start = s;
+ unsigned long long integer_part = 0;
+ unsigned long decimal_part = 0;
+ size_t decimal_digits = 0;
+
+ switch(*s) {
+ case '-':
+ s++;
+ negative = 1;
+ break;
+
+ case '+':
+ s++;
+ break;
+
+ case 'n':
+ if(s[1] == 'a' && s[2] == 'n') {
+ if(endptr) *endptr = (char *)&s[3];
+ return NAN;
+ }
+ break;
+
+ case 'i':
+ if(s[1] == 'n' && s[2] == 'f') {
+ if(endptr) *endptr = (char *)&s[3];
+ return INFINITY;
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ while (*s >= '0' && *s <= '9') {
+ integer_part = (integer_part * 10) + (*s - '0');
+ s++;
+ }
+
+ if(unlikely(*s == '.')) {
+ decimal_part = 0;
+ s++;
+
+ while (*s >= '0' && *s <= '9') {
+ decimal_part = (decimal_part * 10) + (*s - '0');
+ s++;
+ decimal_digits++;
+ }
+ }
+
+ if(unlikely(*s == 'e' || *s == 'E'))
+ return strtold(start, endptr);
+
+ if(unlikely(endptr))
+ *endptr = (char *)s;
+
+ if(unlikely(negative)) {
+ if(unlikely(decimal_digits))
+ return -((long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits));
+ else
+ return -((long double)integer_part);
+ }
+ else {
+ if(unlikely(decimal_digits))
+ return (long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits);
+ else
+ return (long double)integer_part;
+ }
+}
+
#ifdef NETDATA_STRCMP_OVERRIDE
#ifdef strcmp
#undef strcmp
diff --git a/src/log.c b/src/log.c
index 855ecaee6..b3dfc73d0 100644
--- a/src/log.c
+++ b/src/log.c
@@ -23,6 +23,37 @@ void syslog_init(void) {
}
}
+#define LOG_DATE_LENGTH 26
+
+static inline void log_date(char *buffer, size_t len) {
+ if(unlikely(!buffer || !len))
+ return;
+
+ time_t t;
+ struct tm *tmp, tmbuf;
+
+ t = now_realtime_sec();
+ tmp = localtime_r(&t, &tmbuf);
+
+ if (tmp == NULL) {
+ buffer[0] = '\0';
+ return;
+ }
+
+ if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
+ buffer[0] = '\0';
+
+ buffer[len - 1] = '\0';
+}
+
+static netdata_mutex_t log_mutex = NETDATA_MUTEX_INITIALIZER;
+static inline void log_lock() {
+ netdata_mutex_lock(&log_mutex);
+}
+static inline void log_unlock() {
+ netdata_mutex_unlock(&log_mutex);
+}
+
int open_log_file(int fd, FILE **fp, const char *filename, int *enabled_syslog) {
int f;
@@ -136,8 +167,10 @@ int error_log_limit(int reset) {
if(reset) {
if(prevented) {
- log_date(stderr);
- fprintf(stderr, "%s: Resetting logging for process '%s' (prevented %lu logs in the last %ld seconds).\n"
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stderr, "%s: %s Resetting logging for process '%s' (prevented %lu logs in the last %ld seconds).\n"
+ , date
, program_name
, program_name
, prevented
@@ -155,8 +188,10 @@ int error_log_limit(int reset) {
if(now - start > error_log_throttle_period) {
if(prevented) {
- log_date(stderr);
- fprintf(stderr, "%s: Resuming logging from process '%s' (prevented %lu logs in the last %ld seconds).\n"
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stderr, "%s: %s Resuming logging from process '%s' (prevented %lu logs in the last %ld seconds).\n"
+ , date
, program_name
, program_name
, prevented
@@ -175,8 +210,10 @@ int error_log_limit(int reset) {
if(counter > error_log_errors_per_period) {
if(!prevented) {
- log_date(stderr);
- fprintf(stderr, "%s: Too many logs (%lu logs in %ld seconds, threshold is set to %lu logs in %ld seconds). Preventing more logs from process '%s' for %ld seconds.\n"
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stderr, "%s: %s Too many logs (%lu logs in %ld seconds, threshold is set to %lu logs in %ld seconds). Preventing more logs from process '%s' for %ld seconds.\n"
+ , date
, program_name
, counter
, now - start
@@ -200,37 +237,16 @@ int error_log_limit(int reset) {
}
// ----------------------------------------------------------------------------
-// print the date
-
-// FIXME
-// this should print the date in a buffer the way it
-// is now, logs from multiple threads may be multiplexed
-
-void log_date(FILE *out)
-{
- char outstr[26];
- time_t t;
- struct tm *tmp, tmbuf;
-
- t = now_realtime_sec();
- tmp = localtime_r(&t, &tmbuf);
-
- if (tmp == NULL) return;
- if (unlikely(strftime(outstr, sizeof(outstr), "%Y-%m-%d %H:%M:%S", tmp) == 0)) return;
-
- fprintf(out, "%s: ", outstr);
-}
-
-// ----------------------------------------------------------------------------
// debug log
-void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
-{
+void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
va_list args;
- log_date(stdout);
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+
va_start( args, fmt );
- printf("%s: DEBUG (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function);
+ printf("%s: %s DEBUG (%04lu@%-10.10s:%-15.15s): ", date, program_name, line, file, function);
vprintf(fmt, args);
va_end( args );
putchar('\n');
@@ -254,21 +270,26 @@ void info_int( const char *file, const char *function, const unsigned long line,
// prevent logging too much
if(error_log_limit(0)) return;
- log_date(stderr);
+ if(error_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_INFO, fmt, args );
+ va_end( args );
+ }
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+
+ log_lock();
va_start( args, fmt );
- if(debug_flags) fprintf(stderr, "%s: INFO : (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function);
- else fprintf(stderr, "%s: INFO : ", program_name);
+ if(debug_flags) fprintf(stderr, "%s: %s INFO : (%04lu@%-10.10s:%-15.15s): ", date, program_name, line, file, function);
+ else fprintf(stderr, "%s: %s INFO : ", date, program_name);
vfprintf( stderr, fmt, args );
va_end( args );
fputc('\n', stderr);
- if(error_log_syslog) {
- va_start( args, fmt );
- vsyslog(LOG_INFO, fmt, args );
- va_end( args );
- }
+ log_unlock();
}
// ----------------------------------------------------------------------------
@@ -296,56 +317,67 @@ static const char *strerror_result_string(const char *a, const char *b) { (void)
#error "cannot detect the format of function strerror_r()"
#endif
-void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... )
-{
+void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
+ // save a copy of errno - just in case this function generates a new error
+ int __errno = errno;
+
va_list args;
// prevent logging too much
if(error_log_limit(0)) return;
- log_date(stderr);
+ if(error_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_ERR, fmt, args );
+ va_end( args );
+ }
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+
+ log_lock();
va_start( args, fmt );
- if(debug_flags) fprintf(stderr, "%s: %s: (%04lu@%-10.10s:%-15.15s): ", program_name, prefix, line, file, function);
- else fprintf(stderr, "%s: %s: ", program_name, prefix);
+ if(debug_flags) fprintf(stderr, "%s: %s %s: (%04lu@%-10.10s:%-15.15s): ", date, program_name, prefix, line, file, function);
+ else fprintf(stderr, "%s: %s %s: ", date, program_name, prefix);
vfprintf( stderr, fmt, args );
va_end( args );
- if(errno) {
+ if(__errno) {
char buf[1024];
- fprintf(stderr, " (errno %d, %s)\n", errno, strerror_result(strerror_r(errno, buf, 1023), buf));
+ fprintf(stderr, " (errno %d, %s)\n", __errno, strerror_result(strerror_r(__errno, buf, 1023), buf));
errno = 0;
}
else
fputc('\n', stderr);
+ log_unlock();
+}
+
+void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
+ va_list args;
+
if(error_log_syslog) {
va_start( args, fmt );
- vsyslog(LOG_ERR, fmt, args );
+ vsyslog(LOG_CRIT, fmt, args );
va_end( args );
}
-}
-void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
-{
- va_list args;
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
- log_date(stderr);
+ log_lock();
va_start( args, fmt );
- if(debug_flags) fprintf(stderr, "%s: FATAL: (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function);
- else fprintf(stderr, "%s: FATAL: ", program_name);
+ if(debug_flags) fprintf(stderr, "%s: %s FATAL: (%04lu@%-10.10s:%-15.15s): ", date, program_name, line, file, function);
+ else fprintf(stderr, "%s: %s FATAL: ", date, program_name);
vfprintf( stderr, fmt, args );
va_end( args );
perror(" # ");
fputc('\n', stderr);
- if(error_log_syslog) {
- va_start( args, fmt );
- vsyslog(LOG_CRIT, fmt, args );
- va_end( args );
- }
+ log_unlock();
netdata_cleanup_and_exit(1);
}
@@ -353,23 +385,29 @@ void fatal_int( const char *file, const char *function, const unsigned long line
// ----------------------------------------------------------------------------
// access log
-void log_access( const char *fmt, ... )
-{
+void log_access( const char *fmt, ... ) {
va_list args;
+ if(access_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_INFO, fmt, args );
+ va_end( args );
+ }
+
if(stdaccess) {
- log_date(stdaccess);
+ static netdata_mutex_t access_mutex = NETDATA_MUTEX_INITIALIZER;
+
+ netdata_mutex_lock(&access_mutex);
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stdaccess, "%s: ", date);
va_start( args, fmt );
vfprintf( stdaccess, fmt, args );
va_end( args );
fputc('\n', stdaccess);
- }
- if(access_log_syslog) {
- va_start( args, fmt );
- vsyslog(LOG_INFO, fmt, args );
- va_end( args );
+ netdata_mutex_unlock(&access_mutex);
}
}
-
diff --git a/src/log.h b/src/log.h
index d8ff0654b..c0414df8d 100644
--- a/src/log.h
+++ b/src/log.h
@@ -28,6 +28,10 @@
#define D_CONNECT_TO 0x0000000001000000
#define D_RRDHOST 0x0000000002000000
#define D_LOCKS 0x0000000004000000
+#define D_BACKEND 0x0000000008000000
+#define D_STATSD 0x0000000010000000
+#define D_POLLFD 0x0000000020000000
+#define D_STREAM 0x0000000040000000
#define D_SYSTEM 0x8000000000000000
//#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)
@@ -56,16 +60,22 @@ extern int error_log_limit(int reset);
extern void open_all_log_files();
extern void reopen_all_log_files();
+static inline void debug_dummy(void) {}
+
#define error_log_limit_reset() do { error_log_throttle_period = error_log_throttle_period_backup; error_log_limit(1); } while(0)
#define error_log_limit_unlimited() do { error_log_throttle_period = 0; } while(0)
+#ifdef NETDATA_INTERNAL_CHECKS
#define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
+#else
+#define debug(type, args...) debug_dummy()
+#endif
+
#define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args)
#define infoerr(args...) error_int("INFO", __FILE__, __FUNCTION__, __LINE__, ##args)
#define error(args...) error_int("ERROR", __FILE__, __FUNCTION__, __LINE__, ##args)
#define fatal(args...) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args)
-extern void log_date(FILE *out);
extern void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5);
extern void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5);
extern void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(5, 6);
diff --git a/src/main.c b/src/main.c
index a72585e28..bf5d787ad 100644
--- a/src/main.c
+++ b/src/main.c
@@ -9,8 +9,8 @@ void netdata_cleanup_and_exit(int ret) {
debug(D_EXIT, "Called: netdata_cleanup_and_exit()");
- // save the database
- rrdhost_save_all();
+ // cleanup the database
+ rrdhost_cleanup_all();
// unlink the pid
if(pidfile[0]) {
@@ -56,6 +56,7 @@ struct netdata_static_thread static_threads[] = {
{"web", NULL, NULL, 1, NULL, NULL, socket_listen_main_multi_threaded},
{"web-single-threaded", NULL, NULL, 0, NULL, NULL, socket_listen_main_single_threaded},
{"push-metrics", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread},
+ {"statsd", NULL, NULL, 1, NULL, NULL, statsd_main},
{NULL, NULL, NULL, 0, NULL, NULL, NULL}
};
@@ -66,14 +67,16 @@ void web_server_threading_selection(void) {
int single_threaded = (web_server_mode == WEB_SERVER_MODE_SINGLE_THREADED);
int i;
- for(i = 0; static_threads[i].name ; i++) {
- if(static_threads[i].start_routine == socket_listen_main_multi_threaded)
+ for (i = 0; static_threads[i].name; i++) {
+ if (static_threads[i].start_routine == socket_listen_main_multi_threaded)
static_threads[i].enabled = multi_threaded;
- if(static_threads[i].start_routine == socket_listen_main_single_threaded)
+ if (static_threads[i].start_routine == socket_listen_main_single_threaded)
static_threads[i].enabled = single_threaded;
}
+}
+void web_server_config_options(void) {
web_client_timeout = (int) config_get_number(CONFIG_SECTION_WEB, "disconnect idle clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS);
respect_web_browser_do_not_track_policy = config_get_boolean(CONFIG_SECTION_WEB, "respect do not track policy", respect_web_browser_do_not_track_policy);
@@ -299,6 +302,8 @@ void help(int exitcode) {
" -W stacksize=N Set the stacksize (in bytes).\n\n"
" -W debug_flags=N Set runtime tracing to debug.log.\n\n"
" -W unittest Run internal unittests and exit.\n\n"
+ " -W set section option value\n"
+ " set netdata.conf option from the command line.\n\n"
" -W simple-pattern pattern string\n"
" Check if string matches pattern and exit.\n\n"
);
@@ -405,6 +410,9 @@ static void backwards_compatible_config() {
config_move(CONFIG_SECTION_GLOBAL, "web files group",
CONFIG_SECTION_WEB, "web files group");
+
+ config_move(CONFIG_SECTION_BACKEND, "opentsdb host tags",
+ CONFIG_SECTION_BACKEND, "host tags");
}
static void get_netdata_configured_variables() {
@@ -420,8 +428,6 @@ static void get_netdata_configured_variables() {
netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", buf);
debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname);
- netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", CONFIG_DIR);
-
// ------------------------------------------------------------------------
// get default database size
@@ -502,7 +508,9 @@ void set_global_environment() {
// avoid flood calls to stat(/etc/localtime)
// http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux
- setenv("TZ", ":/etc/localtime", 0);
+ const char *tz = getenv("TZ");
+ if(!tz || !*tz)
+ setenv("TZ", config_get(CONFIG_SECTION_GLOBAL, "TZ environment variable", ":/etc/localtime"), 0);
// set the path we need
char path[1024 + 1], *p = getenv("PATH");
@@ -625,10 +633,12 @@ int main(int argc, char **argv) {
{
char* stacksize_string = "stacksize=";
char* debug_flags_string = "debug_flags=";
+
if(strcmp(optarg, "unittest") == 0) {
- default_rrd_update_every = 1;
- default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
- if(!config_loaded) config_load(NULL, 0);
+ if(unit_test_str2ld()) exit(1);
+ //default_rrd_update_every = 1;
+ //default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
+ //if(!config_loaded) config_load(NULL, 0);
get_netdata_configured_variables();
default_rrd_update_every = 1;
default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
@@ -691,9 +701,71 @@ int main(int argc, char **argv) {
config_set(CONFIG_SECTION_GLOBAL, "debug flags", optarg);
debug_flags = strtoull(optarg, NULL, 0);
}
+ else if(strcmp(optarg, "set") == 0) {
+ if(optind + 3 > argc) {
+ fprintf(stderr, "%s", "\nUSAGE: -W set 'section' 'key' 'value'\n\n"
+ " Overwrites settings of netdata.conf.\n"
+ "\n"
+ " These options interact with: -c netdata.conf\n"
+ " If -c netdata.conf is given on the command line,\n"
+ " before -W set... the user may overwrite command\n"
+ " line parameters at netdata.conf\n"
+ " If -c netdata.conf is given after (or missing)\n"
+ " -W set... the user cannot overwrite the command line\n"
+ " parameters."
+ "\n"
+ );
+ exit(1);
+ }
+ const char *section = argv[optind];
+ const char *key = argv[optind + 1];
+ const char *value = argv[optind + 2];
+ optind += 3;
+
+ // set this one as the default
+ // only if it is not already set in the config file
+ // so the caller can use -c netdata.conf before or
+ // after this parameter to prevent or allow overwriting
+ // variables at netdata.conf
+ config_set_default(section, key, value);
+
+ // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value);
+ }
+ else if(strcmp(optarg, "get") == 0) {
+ if(optind + 3 > argc) {
+ fprintf(stderr, "%s", "\nUSAGE: -W get 'section' 'key' 'value'\n\n"
+ " Prints settings of netdata.conf.\n"
+ "\n"
+ " These options interact with: -c netdata.conf\n"
+ " -c netdata.conf has to be given before -W get.\n"
+ "\n"
+ );
+ exit(1);
+ }
+
+ if(!config_loaded) {
+ fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n");
+ config_load(NULL, 0);
+ }
+
+ backwards_compatible_config();
+ get_netdata_configured_variables();
+
+ const char *section = argv[optind];
+ const char *key = argv[optind + 1];
+ const char *def = argv[optind + 2];
+ const char *value = config_get(section, key, def);
+ printf("%s\n", value);
+ exit(0);
+ }
+ else {
+ fprintf(stderr, "Unknown -W parameter '%s'\n", optarg);
+ help(1);
+ }
}
break;
default: /* ? */
+ fprintf(stderr, "Unknown parameter '%c'\n", opt);
help(1);
break;
}
@@ -875,8 +947,10 @@ int main(int argc, char **argv) {
// --------------------------------------------------------------------
// create the listening sockets
+ web_server_threading_selection();
+
if(web_server_mode != WEB_SERVER_MODE_NONE)
- create_listen_sockets();
+ api_listen_sockets_setup();
}
// initialize the log files
@@ -928,7 +1002,7 @@ int main(int argc, char **argv) {
// ------------------------------------------------------------------------
// spawn the threads
- web_server_threading_selection();
+ web_server_config_options();
for (i = 0; static_threads[i].name != NULL ; i++) {
struct netdata_static_thread *st = &static_threads[i];
diff --git a/src/plugin_freebsd.c b/src/plugin_freebsd.c
index 31ab6e0c4..020fdb41c 100644
--- a/src/plugin_freebsd.c
+++ b/src/plugin_freebsd.c
@@ -53,6 +53,12 @@ static struct freebsd_module {
// network interfaces metrics
{ .name = "getifaddrs", .dim = "getifaddrs", .enabled = 1, .func = do_getifaddrs },
+ // ZFS metrics
+ { .name = "kstat.zfs.misc.arcstats", .dim = "arcstats", .enabled = 1, .func = do_kstat_zfs_misc_arcstats },
+
+ // ipfw metrics
+ { .name = "ipfw", .dim = "ipfw", .enabled = 1, .func = do_ipfw },
+
// the terminator of this array
{ .name = NULL, .dim = NULL, .enabled = 0, .func = NULL }
};
diff --git a/src/plugin_freebsd.h b/src/plugin_freebsd.h
index 166c64338..541bf852f 100644
--- a/src/plugin_freebsd.h
+++ b/src/plugin_freebsd.h
@@ -3,6 +3,12 @@
#include <sys/sysctl.h>
+#define KILO_FACTOR 1024
+#define MEGA_FACTOR 1048576 // 1024 * 1024
+#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024
+
+#define MAX_INT_DIGITS 10 // maximum number of digits for int
+
void *freebsd_main(void *ptr);
extern int freebsd_plugin_init();
@@ -35,6 +41,8 @@ extern int do_net_inet6_icmp6_stats(int update_every, usec_t dt);
extern int do_getifaddrs(int update_every, usec_t dt);
extern int do_getmntinfo(int update_every, usec_t dt);
extern int do_kern_devstat(int update_every, usec_t dt);
+extern int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt);
+extern int do_ipfw(int update_every, usec_t dt);
#define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int))
diff --git a/src/plugin_idlejitter.c b/src/plugin_idlejitter.c
index 2ed78160c..89f490233 100644
--- a/src/plugin_idlejitter.c
+++ b/src/plugin_idlejitter.c
@@ -13,41 +13,71 @@ void *cpuidlejitter_main(void *ptr) {
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
error("Cannot set pthread cancel state to ENABLE.");
- int sleep_ms = (int) config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS);
- if(sleep_ms <= 0) {
+ usec_t sleep_ut = config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS) * USEC_PER_MS;
+ if(sleep_ut <= 0) {
config_set_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS);
- sleep_ms = CPU_IDLEJITTER_SLEEP_TIME_MS;
- }
-
- RRDSET *st = rrdset_find_localhost("system.idlejitter");
- if(!st) {
- st = rrdset_create_localhost("system", "idlejitter", NULL, "processes", NULL, "CPU Idle Jitter"
- , "microseconds lost/s", 9999, localhost->rrd_update_every, RRDSET_TYPE_LINE);
- rrddim_add(st, "jitter", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ sleep_ut = CPU_IDLEJITTER_SLEEP_TIME_MS * USEC_PER_MS;
}
+ RRDSET *st = rrdset_create_localhost(
+ "system"
+ , "idlejitter"
+ , NULL
+ , "processes"
+ , NULL
+ , "CPU Idle Jitter"
+ , "microseconds lost/s"
+ , 9999
+ , localhost->rrd_update_every
+ , RRDSET_TYPE_AREA
+ );
+ RRDDIM *rd_min = rrddim_add(st, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_avg = rrddim_add(st, "average", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ usec_t update_every_ut = localhost->rrd_update_every * USEC_PER_SEC;
struct timeval before, after;
unsigned long long counter;
for(counter = 0; 1 ;counter++) {
- usec_t usec = 0, susec = 0;
+ int iterations = 0;
+ usec_t error_total = 0,
+ error_min = 0,
+ error_max = 0,
+ elapsed = 0;
if(netdata_exit) break;
- while(susec < (localhost->rrd_update_every * USEC_PER_SEC)) {
-
+ while(elapsed < update_every_ut) {
now_monotonic_timeval(&before);
- sleep_usec(sleep_ms * 1000);
+ sleep_usec(sleep_ut);
now_monotonic_timeval(&after);
- // calculate the time it took for a full loop
- usec = dt_usec(&after, &before);
- susec += usec;
+ usec_t dt = dt_usec(&after, &before);
+ elapsed += dt;
+
+ usec_t error = dt - sleep_ut;
+ error_total += error;
+
+ if(unlikely(!iterations))
+ error_min = error;
+ else if(error < error_min)
+ error_min = error;
+
+ if(error > error_max)
+ error_max = error;
+
+ iterations++;
}
- usec -= (sleep_ms * 1000);
- if(counter) rrdset_next(st);
- rrddim_set(st, "jitter", usec);
- rrdset_done(st);
+ if(netdata_exit) break;
+
+ if(iterations) {
+ if (likely(counter)) rrdset_next(st);
+ rrddim_set_by_pointer(st, rd_min, error_min);
+ rrddim_set_by_pointer(st, rd_max, error_max);
+ rrddim_set_by_pointer(st, rd_avg, error_total / iterations);
+ rrdset_done(st);
+ }
}
info("IDLEJITTER thread exiting");
diff --git a/src/plugin_proc.c b/src/plugin_proc.c
index 2ca77491d..e64f57398 100644
--- a/src/plugin_proc.c
+++ b/src/plugin_proc.c
@@ -32,7 +32,7 @@ static struct proc_module {
// network metrics
{ .name = "/proc/net/dev", .dim = "netdev", .func = do_proc_net_dev },
- { .name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat },
+ { .name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat }, // this has to be before /proc/net/snmp, because there is a shared metric
{ .name = "/proc/net/snmp", .dim = "snmp", .func = do_proc_net_snmp },
{ .name = "/proc/net/snmp6", .dim = "snmp6", .func = do_proc_net_snmp6 },
{ .name = "/proc/net/softnet_stat", .dim = "softnet", .func = do_proc_net_softnet_stat },
@@ -49,6 +49,9 @@ static struct proc_module {
{ .name = "/proc/net/rpc/nfsd", .dim = "nfsd", .func = do_proc_net_rpc_nfsd },
{ .name = "/proc/net/rpc/nfs", .dim = "nfs", .func = do_proc_net_rpc_nfs },
+ // ZFS metrics
+ { .name = "/proc/spl/kstat/zfs/arcstats", .dim = "zfs_arcstats", .func = do_proc_spl_kstat_zfs_arcstats },
+
// IPC metrics
{ .name = "ipc", .dim = "ipc", .func = do_ipc },
diff --git a/src/plugin_proc.h b/src/plugin_proc.h
index 5dee7853c..688b23de9 100644
--- a/src/plugin_proc.h
+++ b/src/plugin_proc.h
@@ -25,7 +25,11 @@ extern int do_proc_net_softnet_stat(int update_every, usec_t dt);
extern int do_proc_uptime(int update_every, usec_t dt);
extern int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt);
extern int do_proc_sys_devices_system_node(int update_every, usec_t dt);
+extern int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt);
extern int get_numa_node_count(void);
+// metrics that need to be shared among data collectors
+extern unsigned long long tcpext_TCPSynRetrans;
+
#endif /* NETDATA_PLUGIN_PROC_H */
diff --git a/src/plugin_proc_diskspace.c b/src/plugin_proc_diskspace.c
index 37133e044..750086a2c 100644
--- a/src/plugin_proc_diskspace.c
+++ b/src/plugin_proc_diskspace.c
@@ -45,7 +45,7 @@ struct mount_point_metadata {
static DICTIONARY *dict_mountpoints = NULL;
-#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); st = NULL; } } while(st)
+#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); st = NULL; } } while(st)
int mount_point_cleanup(void *entry, void *data) {
(void)data;
@@ -125,6 +125,33 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
def_inodes = CONFIG_BOOLEAN_NO;
}
+ // check if the mount point is a directory #2407
+ {
+ struct stat bs;
+ if(stat(mi->mount_point, &bs) == -1) {
+ error("DISKSPACE: Cannot stat() mount point '%s' (disk '%s', filesystem '%s', root '%s')."
+ , mi->mount_point
+ , disk
+ , mi->filesystem?mi->filesystem:""
+ , mi->root?mi->root:""
+ );
+ def_space = CONFIG_BOOLEAN_NO;
+ def_inodes = CONFIG_BOOLEAN_NO;
+ }
+ else {
+ if((bs.st_mode & S_IFMT) != S_IFDIR) {
+ error("DISKSPACE: Mount point '%s' (disk '%s', filesystem '%s', root '%s') is not a directory."
+ , mi->mount_point
+ , disk
+ , mi->filesystem?mi->filesystem:""
+ , mi->root?mi->root:""
+ );
+ def_space = CONFIG_BOOLEAN_NO;
+ def_inodes = CONFIG_BOOLEAN_NO;
+ }
+ }
+ }
+
do_space = config_get_boolean_ondemand(var_name, "space usage", def_space);
do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", def_inodes);
@@ -161,7 +188,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
struct statvfs buff_statvfs;
if (statvfs(mi->mount_point, &buff_statvfs) < 0) {
if(!m->shown_error) {
- error("Failed statvfs() for '%s' (disk '%s', filesystem '%s', root '%s')"
+ error("DISKSPACE: failed to statvfs() mount point '%s' (disk '%s', filesystem '%s', root '%s')"
, mi->mount_point
, disk
, mi->filesystem?mi->filesystem:""
@@ -188,7 +215,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(btotal != bavail + breserved_root + bused))
- error("Disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused);
+ error("DISKSPACE: disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused);
#endif
// --------------------------------------------------------------------------
@@ -201,7 +228,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(btotal != bavail + breserved_root + bused))
- error("Disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused);
+ error("DISKSPACE: disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused);
#endif
// --------------------------------------------------------------------------
@@ -294,10 +321,10 @@ void *proc_diskspace_main(void *ptr) {
info("DISKSPACE thread created with task id %d", gettid());
if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
- error("Cannot set pthread cancel type to DEFERRED.");
+ error("DISKSPACE: Cannot set pthread cancel type to DEFERRED.");
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
- error("Cannot set pthread cancel state to ENABLE.");
+ error("DISKSPACE: Cannot set pthread cancel state to ENABLE.");
int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1);
@@ -334,7 +361,7 @@ void *proc_diskspace_main(void *ptr) {
struct mountinfo *mi;
for(mi = disk_mountinfo_root; mi; mi = mi->next) {
- if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND | MOUNTINFO_IS_SAME_DEV | MOUNTINFO_NO_STAT | MOUNTINFO_NO_SIZE)))
+ if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND)))
continue;
do_disk_space_stats(mi, update_every);
diff --git a/src/plugin_tc.c b/src/plugin_tc.c
index 7dcfedb33..6bf5782a9 100644
--- a/src/plugin_tc.c
+++ b/src/plugin_tc.c
@@ -713,7 +713,7 @@ static inline void tc_device_free_all()
tc_device_free(tc_device_root);
}
-#define MAX_WORDS 20
+#define PLUGINSD_MAX_WORDS 20
static inline int tc_space(char c) {
switch(c) {
@@ -779,7 +779,7 @@ void *tc_main(void *ptr) {
RRDSET *stcpu = NULL, *sttime = NULL;
char buffer[TC_LINE_MAX+1] = "";
- char *words[MAX_WORDS] = { NULL };
+ char *words[PLUGINSD_MAX_WORDS] = { NULL };
uint32_t BEGIN_HASH = simple_hash("BEGIN");
uint32_t END_HASH = simple_hash("END");
@@ -822,7 +822,7 @@ void *tc_main(void *ptr) {
buffer[TC_LINE_MAX] = '\0';
// debug(D_TC_LOOP, "TC: read '%s'", buffer);
- tc_split_words(buffer, words, MAX_WORDS);
+ tc_split_words(buffer, words, PLUGINSD_MAX_WORDS);
if(unlikely(!words[0] || !*words[0])) {
// debug(D_TC_LOOP, "empty line");
diff --git a/src/plugins_d.c b/src/plugins_d.c
index 7fa19eaf0..9eb102770 100644
--- a/src/plugins_d.c
+++ b/src/plugins_d.c
@@ -2,8 +2,6 @@
struct plugind *pluginsd_root = NULL;
-#define MAX_WORDS 20
-
static inline int pluginsd_space(char c) {
switch(c) {
case ' ':
@@ -18,7 +16,8 @@ static inline int pluginsd_space(char c) {
}
}
-static int pluginsd_split_words(char *str, char **words, int max_words) {
+// split a text into words, respecting quotes
+inline int pluginsd_split_words(char *str, char **words, int max_words) {
char *s = str, quote = 0;
int i = 0, j;
@@ -95,14 +94,13 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
char line[PLUGINSD_LINE_MAX + 1];
- char *words[MAX_WORDS] = { NULL };
- /* uint32_t HOST_HASH = simple_hash("HOST"); */
- uint32_t BEGIN_HASH = simple_hash("BEGIN");
- uint32_t END_HASH = simple_hash("END");
- uint32_t FLUSH_HASH = simple_hash("FLUSH");
- uint32_t CHART_HASH = simple_hash("CHART");
- uint32_t DIMENSION_HASH = simple_hash("DIMENSION");
- uint32_t DISABLE_HASH = simple_hash("DISABLE");
+ char *words[PLUGINSD_MAX_WORDS] = { NULL };
+ uint32_t BEGIN_HASH = simple_hash(PLUGINSD_KEYWORD_BEGIN);
+ uint32_t END_HASH = simple_hash(PLUGINSD_KEYWORD_END);
+ uint32_t FLUSH_HASH = simple_hash(PLUGINSD_KEYWORD_FLUSH);
+ uint32_t CHART_HASH = simple_hash(PLUGINSD_KEYWORD_CHART);
+ uint32_t DIMENSION_HASH = simple_hash(PLUGINSD_KEYWORD_DIMENSION);
+ uint32_t DISABLE_HASH = simple_hash(PLUGINSD_KEYWORD_DISABLE);
RRDSET *st = NULL;
uint32_t hash;
@@ -130,7 +128,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
// debug(D_PLUGINSD, "PLUGINSD: %s: %s", cd->filename, line);
- int w = pluginsd_split_words(line, words, MAX_WORDS);
+ int w = pluginsd_split_words(line, words, PLUGINSD_MAX_WORDS);
char *s = words[0];
if(unlikely(!s || !*s || !w)) {
// debug(D_PLUGINSD, "PLUGINSD: empty line");
@@ -159,9 +157,18 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) debug(D_PLUGINSD, "PLUGINSD: '%s' is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value?value:"<nothing>");
- if(value) rrddim_set(st, dimension, strtoll(value, NULL, 0));
+ if(value) {
+ RRDDIM *rd = rrddim_find(st, dimension);
+ if(unlikely(!rd)) {
+ error("PLUGINSD: '%s' is requesting a SET to dimension with id '%s' on stats '%s' (%s) on host '%s', which does not exist. Disabling it.", cd->fullfilename, dimension, st->name, st->id, st->rrdhost->hostname);
+ enabled = 0;
+ break;
+ }
+ else
+ rrddim_set_by_pointer(st, rd, strtoll(value, NULL, 0));
+ }
}
- else if(likely(hash == BEGIN_HASH && !strcmp(s, "BEGIN"))) {
+ else if(likely(hash == BEGIN_HASH && !strcmp(s, PLUGINSD_KEYWORD_BEGIN))) {
char *id = words[1];
char *microseconds_txt = words[2];
@@ -191,7 +198,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
else rrdset_next(st);
}
}
- else if(likely(hash == END_HASH && !strcmp(s, "END"))) {
+ else if(likely(hash == END_HASH && !strcmp(s, PLUGINSD_KEYWORD_END))) {
if(unlikely(!st)) {
error("PLUGINSD: '%s' is requesting an END, without a BEGIN on host '%s'. Disabling it.", cd->fullfilename, host->hostname);
enabled = 0;
@@ -205,28 +212,11 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
count++;
}
-/* else if(likely(hash == HOST_HASH && !strcmp(s, "HOST"))) {
- char *guid = words[1];
- char *hostname = words[2];
-
- if(unlikely(!guid || !*guid)) {
- error("PLUGINSD: '%s' is requesting HOST with guid '%s' and hostname '%s', without a guid. Disabling it.", cd->fullfilename, guid?guid:"", hostname?hostname:"");
- enabled = 0;
- break;
- }
- if(unlikely(!hostname || !*hostname)) {
- error("PLUGINSD: '%s' is requesting HOST with guid '%s' and hostname '%s', without a hostname. Disabling it.", cd->fullfilename, guid?guid:"", hostname?hostname:"");
- enabled = 0;
- break;
- }
-
- host = rrdhost_find_or_create(hostname, guid);
- } */
- else if(likely(hash == FLUSH_HASH && !strcmp(s, "FLUSH"))) {
+ else if(likely(hash == FLUSH_HASH && !strcmp(s, PLUGINSD_KEYWORD_FLUSH))) {
debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename);
st = NULL;
}
- else if(likely(hash == CHART_HASH && !strcmp(s, "CHART"))) {
+ else if(likely(hash == CHART_HASH && !strcmp(s, PLUGINSD_KEYWORD_CHART))) {
int noname = 0;
st = NULL;
@@ -247,6 +237,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
char *chart = words[7];
char *priority_s = words[8];
char *update_every_s = words[9];
+ char *options = words[10];
if(unlikely(!type || !*type || !id || !*id)) {
error("PLUGINSD: '%s' is requesting a CHART, without a type.id, on host '%s'. Disabling it.", cd->fullfilename, host->hostname);
@@ -284,8 +275,25 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
cd->update_every = update_every;
}
else debug(D_PLUGINSD, "PLUGINSD: Chart '%s' already exists. Not adding it again.", st->id);
+
+ if(options && *options) {
+ if(strstr(options, "obsolete"))
+ rrdset_is_obsolete(st);
+ else
+ rrdset_isnot_obsolete(st);
+
+ if(strstr(options, "detail"))
+ rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
+
+ if(strstr(options, "store_first"))
+ rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
+ else
+ rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
+ }
}
- else if(likely(hash == DIMENSION_HASH && !strcmp(s, "DIMENSION"))) {
+ else if(likely(hash == DIMENSION_HASH && !strcmp(s, PLUGINSD_KEYWORD_DIMENSION))) {
char *id = words[1];
char *name = words[2];
char *algorithm = words[3];
@@ -326,21 +334,16 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int
, options?options:""
);
- RRDDIM *rd = rrddim_find(st, id);
- if(unlikely(!rd)) {
- rd = rrddim_add(st, id, name, multiplier, divisor, rrd_algorithm_id(algorithm));
- rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
- rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
- if(options && *options) {
- if(strstr(options, "hidden") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
- if(strstr(options, "noreset") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
- if(strstr(options, "nooverflow") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
- }
+ RRDDIM *rd = rrddim_add(st, id, name, multiplier, divisor, rrd_algorithm_id(algorithm));
+ rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
+ rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ if(options && *options) {
+ if(strstr(options, "hidden") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
+ if(strstr(options, "noreset") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
+ if(strstr(options, "nooverflow") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
}
- else if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_PLUGINSD, "PLUGINSD: dimension %s/%s already exists. Not adding it again.", st->id, id);
}
- else if(unlikely(hash == DISABLE_HASH && !strcmp(s, "DISABLE"))) {
+ else if(unlikely(hash == DISABLE_HASH && !strcmp(s, PLUGINSD_KEYWORD_DISABLE))) {
info("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename);
enabled = 0;
break;
diff --git a/src/plugins_d.h b/src/plugins_d.h
index d34c4030c..595a515c4 100644
--- a/src/plugins_d.h
+++ b/src/plugins_d.h
@@ -4,7 +4,16 @@
#define PLUGINSD_FILE_SUFFIX ".plugin"
#define PLUGINSD_FILE_SUFFIX_LEN strlen(PLUGINSD_FILE_SUFFIX)
#define PLUGINSD_CMD_MAX (FILENAME_MAX*2)
+
+#define PLUGINSD_KEYWORD_CHART "CHART"
+#define PLUGINSD_KEYWORD_DIMENSION "DIMENSION"
+#define PLUGINSD_KEYWORD_BEGIN "BEGIN"
+#define PLUGINSD_KEYWORD_END "END"
+#define PLUGINSD_KEYWORD_FLUSH "FLUSH"
+#define PLUGINSD_KEYWORD_DISABLE "DISABLE"
+
#define PLUGINSD_LINE_MAX 1024
+#define PLUGINSD_MAX_WORDS 20
struct plugind {
char id[CONFIG_MAX_NAME+1]; // config node id
@@ -35,5 +44,6 @@ extern struct plugind *pluginsd_root;
extern void *pluginsd_main(void *ptr);
extern size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations);
+extern int pluginsd_split_words(char *str, char **words, int max_words);
#endif /* NETDATA_PLUGINS_D_H */
diff --git a/src/proc_diskstats.c b/src/proc_diskstats.c
index a1b4072dd..d3fed5a6d 100644
--- a/src/proc_diskstats.c
+++ b/src/proc_diskstats.c
@@ -10,7 +10,8 @@
#define DELAULT_EXLUDED_DISKS "loop* ram*"
static struct disk {
- char *disk; // the name of the disk (sda, sdb, etc)
+ char *disk; // the name of the disk (sda, sdb, etc, after being looked up)
+ char *device; // the device of the disk (before being looked up)
unsigned long major;
unsigned long minor;
int sector_size;
@@ -44,12 +45,75 @@ static struct disk {
struct disk *next;
} *disk_root = NULL;
-#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); st = NULL; } } while(st)
+#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); st = NULL; } } while(st)
+
+static char *path_to_get_hw_sector_size = NULL;
+static char *path_to_get_hw_sector_size_partitions = NULL;
+static char *path_to_find_block_device = NULL;
+static char *path_to_device_mapper = NULL;
+
+static inline char *get_disk_name(unsigned long major, unsigned long minor, char *disk) {
+ static int enabled = 1;
+
+ if(!enabled) goto cleanup;
+
+ char filename[FILENAME_MAX + 1];
+ char link[FILENAME_MAX + 1];
+
+ DIR *dir = opendir(path_to_device_mapper);
+ if (!dir) {
+ error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot open directory '%s'. Disabling device-mapper support.", disk, major, minor, path_to_device_mapper);
+ enabled = 0;
+ goto cleanup;
+ }
+
+ struct dirent *de = NULL;
+ while ((de = readdir(dir))) {
+ if(de->d_type != DT_LNK) continue;
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s", path_to_device_mapper, de->d_name);
+ ssize_t len = readlink(filename, link, FILENAME_MAX);
+ if(len <= 0) {
+ error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot read link '%s'.", disk, major, minor, filename);
+ continue;
+ }
+
+ link[len] = '\0';
+ if(link[0] != '/')
+ snprintfz(filename, FILENAME_MAX, "%s/%s", path_to_device_mapper, link);
+ else
+ strncpyz(filename, link, FILENAME_MAX);
+
+ struct stat sb;
+ if(stat(filename, &sb) == -1) {
+ error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot stat() file '%s'.", disk, major, minor, filename);
+ continue;
+ }
+
+ if((sb.st_mode & S_IFMT) != S_IFBLK) {
+ // info("DEVICE-MAPPER ('%s', %lu:%lu): file '%s' is not a block device.", disk, major, minor, filename);
+ continue;
+ }
+
+ if(major(sb.st_rdev) != major || minor(sb.st_rdev) != minor) {
+ // info("DEVICE-MAPPER ('%s', %lu:%lu): filename '%s' does not match %lu:%lu.", disk, major, minor, filename, (unsigned long)major(sb.st_rdev), (unsigned long)minor(sb.st_rdev));
+ continue;
+ }
+
+ // info("DEVICE-MAPPER ('%s', %lu:%lu): filename '%s' matches.", disk, major, minor, filename);
+
+ strncpy(link, de->d_name, FILENAME_MAX);
+ netdata_fix_chart_name(link);
+ disk = link;
+ break;
+ }
+ closedir(dir);
+
+cleanup:
+ return strdupz(disk);
+}
static struct disk *get_disk(unsigned long major, unsigned long minor, char *disk) {
- static char path_to_get_hw_sector_size[FILENAME_MAX + 1] = "";
- static char path_to_get_hw_sector_size_partitions[FILENAME_MAX + 1] = "";
- static char path_find_block_device[FILENAME_MAX + 1] = "";
static struct mountinfo *disk_mountinfo_root = NULL;
struct disk *d;
@@ -66,7 +130,8 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis
// create a new disk structure
d = (struct disk *)callocz(1, sizeof(struct disk));
- d->disk = strdupz(disk);
+ d->disk = get_disk_name(major, minor, disk);
+ d->device = strdupz(disk);
d->major = major;
d->minor = minor;
d->type = DISK_TYPE_PHYSICAL; // Default type. Changed later if not correct.
@@ -83,27 +148,17 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis
last->next = d;
}
- // ------------------------------------------------------------------------
- // find the type of the device
-
- char buffer[FILENAME_MAX + 1];
-
- // get the default path for finding info about the block device
- if(unlikely(!path_find_block_device[0])) {
- snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/%s");
- snprintfz(path_find_block_device, FILENAME_MAX, "%s", config_get(CONFIG_SECTION_DISKSTATS, "path to get block device infos", buffer));
- }
-
// find if it is a partition
// by checking if /sys/dev/block/MAJOR:MINOR/partition is readable.
- snprintfz(buffer, FILENAME_MAX, path_find_block_device, major, minor, "partition");
+ char buffer[FILENAME_MAX + 1];
+ snprintfz(buffer, FILENAME_MAX, path_to_find_block_device, major, minor, "partition");
if(likely(access(buffer, R_OK) == 0)) {
d->type = DISK_TYPE_PARTITION;
}
else {
// find if it is a container
// by checking if /sys/dev/block/MAJOR:MINOR/slaves has entries
- snprintfz(buffer, FILENAME_MAX, path_find_block_device, major, minor, "slaves/");
+ snprintfz(buffer, FILENAME_MAX, path_to_find_block_device, major, minor, "slaves/");
DIR *dirp = opendir(buffer);
if(likely(dirp != NULL)) {
struct dirent *dp;
@@ -143,18 +198,9 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis
// ------------------------------------------------------------------------
// find the disk sector size
- if(unlikely(!path_to_get_hw_sector_size[0])) {
- snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/queue/hw_sector_size");
- snprintfz(path_to_get_hw_sector_size, FILENAME_MAX, "%s", config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size", buffer));
- }
- if(unlikely(!path_to_get_hw_sector_size_partitions[0])) {
- snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/subsystem/%s/../queue/hw_sector_size");
- snprintfz(path_to_get_hw_sector_size_partitions, FILENAME_MAX, "%s", config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size for partitions", buffer));
- }
-
{
char tf[FILENAME_MAX + 1], *t;
- strncpyz(tf, d->disk, FILENAME_MAX);
+ strncpyz(tf, d->device, FILENAME_MAX);
// replace all / with !
for(t = tf; *t ;t++)
@@ -173,15 +219,15 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis
if(likely(tmp)) {
d->sector_size = str2i(tmp);
if(unlikely(d->sector_size <= 0)) {
- error("Invalid sector size %d for device %s in %s. Assuming 512.", d->sector_size, d->disk, buffer);
+ error("Invalid sector size %d for device %s in %s. Assuming 512.", d->sector_size, d->device, buffer);
d->sector_size = 512;
}
}
- else error("Cannot read data for sector size for device %s from %s. Assuming 512.", d->disk, buffer);
+ else error("Cannot read data for sector size for device %s from %s. Assuming 512.", d->device, buffer);
fclose(fpss);
}
- else error("Cannot read sector size for device %s from %s. Assuming 512.", d->disk, buffer);
+ else error("Cannot read sector size for device %s from %s. Assuming 512.", d->device, buffer);
}
return d;
@@ -230,6 +276,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
globals_initialized = 0;
if(unlikely(!globals_initialized)) {
+ globals_initialized = 1;
+
global_enable_new_disks_detected_at_runtime = config_get_boolean(CONFIG_SECTION_DISKSTATS, "enable new disks detected at runtime", global_enable_new_disks_detected_at_runtime);
global_enable_performance_for_physical_disks = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "performance metrics for physical disks", global_enable_performance_for_physical_disks);
global_enable_performance_for_virtual_disks = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "performance metrics for virtual disks", global_enable_performance_for_virtual_disks);
@@ -243,7 +291,19 @@ int do_proc_diskstats(int update_every, usec_t dt) {
global_do_util = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "utilization percentage for all disks", global_do_util);
global_do_backlog = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "backlog for all disks", global_do_backlog);
- globals_initialized = 1;
+ char buffer[FILENAME_MAX + 1];
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/%s");
+ path_to_find_block_device = config_get(CONFIG_SECTION_DISKSTATS, "path to get block device infos", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/queue/hw_sector_size");
+ path_to_get_hw_sector_size = config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/subsystem/%s/../queue/hw_sector_size");
+ path_to_get_hw_sector_size_partitions = config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size for partitions", buffer);
+
+ snprintfz(buffer, FILENAME_MAX, "%s/dev/mapper", netdata_configured_host_prefix);
+ path_to_device_mapper = config_get(CONFIG_SECTION_DISKSTATS, "path to device mapper", buffer);
}
// --------------------------------------------------------------------------
@@ -339,7 +399,7 @@ int do_proc_diskstats(int update_every, usec_t dt) {
// Set its family based on mount point
char *family = d->mount_point;
- if(!family) family = disk;
+ if(!family) family = d->disk;
// --------------------------------------------------------------------------
@@ -359,11 +419,11 @@ int do_proc_diskstats(int update_every, usec_t dt) {
int def_enable = global_enable_new_disks_detected_at_runtime;
- if(def_enable != CONFIG_BOOLEAN_NO && simple_pattern_matches(excluded_disks, disk))
+ if(def_enable != CONFIG_BOOLEAN_NO && (simple_pattern_matches(excluded_disks, d->device) || simple_pattern_matches(excluded_disks, d->disk)))
def_enable = CONFIG_BOOLEAN_NO;
char var_name[4096 + 1];
- snprintfz(var_name, 4096, "plugin:proc:/proc/diskstats:%s", disk);
+ snprintfz(var_name, 4096, "plugin:proc:/proc/diskstats:%s", d->disk);
def_enable = config_get_boolean_ondemand(var_name, "enable", def_enable);
if(unlikely(def_enable == CONFIG_BOOLEAN_NO)) {
@@ -449,8 +509,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_io)) {
d->st_io = rrdset_create_localhost(
RRD_TYPE_DISK
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.io"
, "Disk I/O Bandwidth"
@@ -478,8 +538,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_ops)) {
d->st_ops = rrdset_create_localhost(
"disk_ops"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.ops"
, "Disk Completed I/O Operations"
@@ -509,8 +569,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_qops)) {
d->st_qops = rrdset_create_localhost(
"disk_qops"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.qops"
, "Disk Current I/O Operations"
@@ -538,8 +598,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_backlog)) {
d->st_backlog = rrdset_create_localhost(
"disk_backlog"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.backlog"
, "Disk Backlog"
@@ -567,8 +627,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_util)) {
d->st_util = rrdset_create_localhost(
"disk_util"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.util"
, "Disk Utilization Time"
@@ -596,8 +656,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_mops)) {
d->st_mops = rrdset_create_localhost(
"disk_mops"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.mops"
, "Disk Merged Operations"
@@ -627,8 +687,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_iotime)) {
d->st_iotime = rrdset_create_localhost(
"disk_iotime"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.iotime"
, "Disk Total I/O Time"
@@ -661,8 +721,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_await)) {
d->st_await = rrdset_create_localhost(
"disk_await"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.await"
, "Average Completed I/O Operation Time"
@@ -690,8 +750,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_avgsz)) {
d->st_avgsz = rrdset_create_localhost(
"disk_avgsz"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.avgsz"
, "Average Completed I/O Operation Bandwidth"
@@ -719,8 +779,8 @@ int do_proc_diskstats(int update_every, usec_t dt) {
if(unlikely(!d->st_svctm)) {
d->st_svctm = rrdset_create_localhost(
"disk_svctm"
- , disk
- , NULL
+ , d->device
+ , d->disk
, family
, "disk.svctm"
, "Average Service Time"
@@ -769,6 +829,7 @@ int do_proc_diskstats(int update_every, usec_t dt) {
}
freez(t->disk);
+ freez(t->device);
freez(t->mount_point);
freez(t);
}
diff --git a/src/proc_loadavg.c b/src/proc_loadavg.c
index e7863f114..a48801b37 100644
--- a/src/proc_loadavg.c
+++ b/src/proc_loadavg.c
@@ -68,9 +68,10 @@ int do_proc_loadavg(int update_every, usec_t dt) {
rrddim_set(load_chart, "load5", (collected_number) (load5 * 1000));
rrddim_set(load_chart, "load15", (collected_number) (load15 * 1000));
rrdset_done(load_chart);
- }
- next_loadavg_dt = load_chart->update_every * USEC_PER_SEC;
+ next_loadavg_dt = load_chart->update_every * USEC_PER_SEC;
+ }
+ else next_loadavg_dt = MIN_LOADAVG_UPDATE_EVERY * USEC_PER_SEC;
}
else next_loadavg_dt -= dt;
diff --git a/src/proc_net_dev.c b/src/proc_net_dev.c
index 1b00758d5..ee7588990 100644
--- a/src/proc_net_dev.c
+++ b/src/proc_net_dev.c
@@ -73,13 +73,13 @@ static struct netdev *netdev_root = NULL, *netdev_last_used = NULL;
static size_t netdev_added = 0, netdev_found = 0;
static void netdev_free(struct netdev *d) {
- if(d->st_bandwidth) rrdset_flag_set(d->st_bandwidth, RRDSET_FLAG_OBSOLETE);
- if(d->st_packets) rrdset_flag_set(d->st_packets, RRDSET_FLAG_OBSOLETE);
- if(d->st_errors) rrdset_flag_set(d->st_errors, RRDSET_FLAG_OBSOLETE);
- if(d->st_drops) rrdset_flag_set(d->st_drops, RRDSET_FLAG_OBSOLETE);
- if(d->st_fifo) rrdset_flag_set(d->st_fifo, RRDSET_FLAG_OBSOLETE);
- if(d->st_compressed) rrdset_flag_set(d->st_compressed, RRDSET_FLAG_OBSOLETE);
- if(d->st_events) rrdset_flag_set(d->st_events, RRDSET_FLAG_OBSOLETE);
+ if(d->st_bandwidth) rrdset_is_obsolete(d->st_bandwidth);
+ if(d->st_packets) rrdset_is_obsolete(d->st_packets);
+ if(d->st_errors) rrdset_is_obsolete(d->st_errors);
+ if(d->st_drops) rrdset_is_obsolete(d->st_drops);
+ if(d->st_fifo) rrdset_is_obsolete(d->st_fifo);
+ if(d->st_compressed) rrdset_is_obsolete(d->st_compressed);
+ if(d->st_events) rrdset_is_obsolete(d->st_events);
netdev_added--;
freez(d->name);
diff --git a/src/proc_net_netstat.c b/src/proc_net_netstat.c
index 2677a6c17..322e51d13 100644
--- a/src/proc_net_netstat.c
+++ b/src/proc_net_netstat.c
@@ -1,5 +1,7 @@
#include "common.h"
+unsigned long long tcpext_TCPSynRetrans;
+
static void parse_line_pair(procfile *ff, ARL_BASE *base, size_t header_line, size_t values_line) {
size_t hwords = procfile_linewords(ff, header_line);
size_t vwords = procfile_linewords(ff, values_line);
@@ -94,6 +96,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
// IPv4 TCP memory pressures
static unsigned long long tcpext_TCPMemoryPressures = 0;
+ // shared: tcpext_TCPSynRetrans
+
+
if(unlikely(!arl_ipext)) {
hash_ipext = simple_hash("IpExt");
hash_tcpext = simple_hash("TcpExt");
@@ -191,6 +196,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
if(do_tcpext_memory != CONFIG_BOOLEAN_NO) {
arl_expect(arl_tcpext, "TCPMemoryPressures", &tcpext_TCPMemoryPressures);
}
+
+ // shared metrics
+ arl_expect(arl_tcpext, "TCPSynRetrans", &tcpext_TCPSynRetrans);
}
if(unlikely(!ff)) {
diff --git a/src/proc_net_snmp.c b/src/proc_net_snmp.c
index ba7b40013..7c0fd9b4a 100644
--- a/src/proc_net_snmp.c
+++ b/src/proc_net_snmp.c
@@ -645,24 +645,35 @@ int do_proc_net_snmp(int update_every, usec_t dt) {
if(do_tcp_handshake) {
st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".tcphandshake");
if(!st) {
- st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "tcphandshake", NULL, "tcp", NULL
- , "IPv4 TCP Handshake Issues", "events/s", 2900, update_every
- , RRDSET_TYPE_LINE);
+ st = rrdset_create_localhost(
+ RRD_TYPE_NET_SNMP
+ , "tcphandshake"
+ , NULL
+ , "tcp"
+ , NULL
+ , "IPv4 TCP Handshake Issues"
+ , "events/s"
+ , 2900
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
- rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st, "OutRsts", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "OutRsts", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st, "TCPSynRetrans", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
}
else rrdset_next(st);
- rrddim_set(st, "EstabResets", *tcp_EstabResets);
- rrddim_set(st, "OutRsts", *tcp_OutRsts);
- rrddim_set(st, "ActiveOpens", *tcp_ActiveOpens);
- rrddim_set(st, "PassiveOpens", *tcp_PassiveOpens);
- rrddim_set(st, "AttemptFails", *tcp_AttemptFails);
+ rrddim_set(st, "EstabResets", *tcp_EstabResets);
+ rrddim_set(st, "OutRsts", *tcp_OutRsts);
+ rrddim_set(st, "ActiveOpens", *tcp_ActiveOpens);
+ rrddim_set(st, "PassiveOpens", *tcp_PassiveOpens);
+ rrddim_set(st, "AttemptFails", *tcp_AttemptFails);
+ rrddim_set(st, "TCPSynRetrans", tcpext_TCPSynRetrans);
rrdset_done(st);
}
}
diff --git a/src/proc_net_snmp6.c b/src/proc_net_snmp6.c
index 8c4581c1b..aa9ab2209 100644
--- a/src/proc_net_snmp6.c
+++ b/src/proc_net_snmp6.c
@@ -311,8 +311,8 @@ int do_proc_net_snmp6(int update_every, usec_t dt) {
rrddim_set(st, "sent", Ip6OutRequests);
rrddim_set(st, "received", Ip6InReceives);
- rrddim_set(st, "forwarded", Ip6InDelivers);
- rrddim_set(st, "delivers", Ip6OutForwDatagrams);
+ rrddim_set(st, "forwarded", Ip6OutForwDatagrams);
+ rrddim_set(st, "delivers", Ip6InDelivers);
rrdset_done(st);
}
diff --git a/src/proc_net_softnet_stat.c b/src/proc_net_softnet_stat.c
index 40946a7a5..b03a43c52 100644
--- a/src/proc_net_softnet_stat.c
+++ b/src/proc_net_softnet_stat.c
@@ -79,7 +79,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
st = rrdset_find_bytype_localhost("system", "softnet_stat");
if(unlikely(!st)) {
- st = rrdset_create_localhost("system", "softnet_stat", NULL, "softnet_stat", NULL, "System softnet_stat"
+ st = rrdset_create_localhost("system", "softnet_stat", NULL, "softnet_stat", "system.softnet_stat", "System softnet_stat"
, "events/s", 955, update_every, RRDSET_TYPE_LINE);
for(w = 0; w < allocated_columns ;w++)
if(unlikely(softnet_column_name(w)))
@@ -103,7 +103,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
char title[100+1];
snprintfz(title, 100, "CPU%zu softnet_stat", l);
- st = rrdset_create_localhost("cpu", id, NULL, "softnet_stat", NULL, title, "events/s", 4101 + l
+ st = rrdset_create_localhost("cpu", id, NULL, "softnet_stat", "cpu.softnet_stat", title, "events/s", 4101 + l
, update_every, RRDSET_TYPE_LINE);
for(w = 0; w < allocated_columns ;w++)
if(unlikely(softnet_column_name(w)))
diff --git a/src/proc_spl_kstat_zfs.c b/src/proc_spl_kstat_zfs.c
new file mode 100644
index 000000000..dee7a6b3d
--- /dev/null
+++ b/src/proc_spl_kstat_zfs.c
@@ -0,0 +1,153 @@
+#include "common.h"
+#include "zfs_common.h"
+
+#define ZFS_PROC_ARCSTATS "/proc/spl/kstat/zfs/arcstats"
+
+struct arcstats arcstats = { 0 };
+
+int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt) {
+ (void)dt;
+
+ static procfile *ff = NULL;
+ static ARL_BASE *arl_base = NULL;
+
+ l2exist = -1;
+
+ if(unlikely(!arl_base)) {
+ arl_base = arl_create("arcstats", NULL, 60);
+
+ arl_expect(arl_base, "hits", &arcstats.hits);
+ arl_expect(arl_base, "misses", &arcstats.misses);
+ arl_expect(arl_base, "demand_data_hits", &arcstats.demand_data_hits);
+ arl_expect(arl_base, "demand_data_misses", &arcstats.demand_data_misses);
+ arl_expect(arl_base, "demand_metadata_hits", &arcstats.demand_metadata_hits);
+ arl_expect(arl_base, "demand_metadata_misses", &arcstats.demand_metadata_misses);
+ arl_expect(arl_base, "prefetch_data_hits", &arcstats.prefetch_data_hits);
+ arl_expect(arl_base, "prefetch_data_misses", &arcstats.prefetch_data_misses);
+ arl_expect(arl_base, "prefetch_metadata_hits", &arcstats.prefetch_metadata_hits);
+ arl_expect(arl_base, "prefetch_metadata_misses", &arcstats.prefetch_metadata_misses);
+ arl_expect(arl_base, "mru_hits", &arcstats.mru_hits);
+ arl_expect(arl_base, "mru_ghost_hits", &arcstats.mru_ghost_hits);
+ arl_expect(arl_base, "mfu_hits", &arcstats.mfu_hits);
+ arl_expect(arl_base, "mfu_ghost_hits", &arcstats.mfu_ghost_hits);
+ arl_expect(arl_base, "deleted", &arcstats.deleted);
+ arl_expect(arl_base, "mutex_miss", &arcstats.mutex_miss);
+ arl_expect(arl_base, "evict_skip", &arcstats.evict_skip);
+ arl_expect(arl_base, "evict_not_enough", &arcstats.evict_not_enough);
+ arl_expect(arl_base, "evict_l2_cached", &arcstats.evict_l2_cached);
+ arl_expect(arl_base, "evict_l2_eligible", &arcstats.evict_l2_eligible);
+ arl_expect(arl_base, "evict_l2_ineligible", &arcstats.evict_l2_ineligible);
+ arl_expect(arl_base, "evict_l2_skip", &arcstats.evict_l2_skip);
+ arl_expect(arl_base, "hash_elements", &arcstats.hash_elements);
+ arl_expect(arl_base, "hash_elements_max", &arcstats.hash_elements_max);
+ arl_expect(arl_base, "hash_collisions", &arcstats.hash_collisions);
+ arl_expect(arl_base, "hash_chains", &arcstats.hash_chains);
+ arl_expect(arl_base, "hash_chain_max", &arcstats.hash_chain_max);
+ arl_expect(arl_base, "p", &arcstats.p);
+ arl_expect(arl_base, "c", &arcstats.c);
+ arl_expect(arl_base, "c_min", &arcstats.c_min);
+ arl_expect(arl_base, "c_max", &arcstats.c_max);
+ arl_expect(arl_base, "size", &arcstats.size);
+ arl_expect(arl_base, "hdr_size", &arcstats.hdr_size);
+ arl_expect(arl_base, "data_size", &arcstats.data_size);
+ arl_expect(arl_base, "metadata_size", &arcstats.metadata_size);
+ arl_expect(arl_base, "other_size", &arcstats.other_size);
+ arl_expect(arl_base, "anon_size", &arcstats.anon_size);
+ arl_expect(arl_base, "anon_evictable_data", &arcstats.anon_evictable_data);
+ arl_expect(arl_base, "anon_evictable_metadata", &arcstats.anon_evictable_metadata);
+ arl_expect(arl_base, "mru_size", &arcstats.mru_size);
+ arl_expect(arl_base, "mru_evictable_data", &arcstats.mru_evictable_data);
+ arl_expect(arl_base, "mru_evictable_metadata", &arcstats.mru_evictable_metadata);
+ arl_expect(arl_base, "mru_ghost_size", &arcstats.mru_ghost_size);
+ arl_expect(arl_base, "mru_ghost_evictable_data", &arcstats.mru_ghost_evictable_data);
+ arl_expect(arl_base, "mru_ghost_evictable_metadata", &arcstats.mru_ghost_evictable_metadata);
+ arl_expect(arl_base, "mfu_size", &arcstats.mfu_size);
+ arl_expect(arl_base, "mfu_evictable_data", &arcstats.mfu_evictable_data);
+ arl_expect(arl_base, "mfu_evictable_metadata", &arcstats.mfu_evictable_metadata);
+ arl_expect(arl_base, "mfu_ghost_size", &arcstats.mfu_ghost_size);
+ arl_expect(arl_base, "mfu_ghost_evictable_data", &arcstats.mfu_ghost_evictable_data);
+ arl_expect(arl_base, "mfu_ghost_evictable_metadata", &arcstats.mfu_ghost_evictable_metadata);
+ arl_expect(arl_base, "l2_hits", &arcstats.l2_hits);
+ arl_expect(arl_base, "l2_misses", &arcstats.l2_misses);
+ arl_expect(arl_base, "l2_feeds", &arcstats.l2_feeds);
+ arl_expect(arl_base, "l2_rw_clash", &arcstats.l2_rw_clash);
+ arl_expect(arl_base, "l2_read_bytes", &arcstats.l2_read_bytes);
+ arl_expect(arl_base, "l2_write_bytes", &arcstats.l2_write_bytes);
+ arl_expect(arl_base, "l2_writes_sent", &arcstats.l2_writes_sent);
+ arl_expect(arl_base, "l2_writes_done", &arcstats.l2_writes_done);
+ arl_expect(arl_base, "l2_writes_error", &arcstats.l2_writes_error);
+ arl_expect(arl_base, "l2_writes_lock_retry", &arcstats.l2_writes_lock_retry);
+ arl_expect(arl_base, "l2_evict_lock_retry", &arcstats.l2_evict_lock_retry);
+ arl_expect(arl_base, "l2_evict_reading", &arcstats.l2_evict_reading);
+ arl_expect(arl_base, "l2_evict_l1cached", &arcstats.l2_evict_l1cached);
+ arl_expect(arl_base, "l2_free_on_write", &arcstats.l2_free_on_write);
+ arl_expect(arl_base, "l2_cdata_free_on_write", &arcstats.l2_cdata_free_on_write);
+ arl_expect(arl_base, "l2_abort_lowmem", &arcstats.l2_abort_lowmem);
+ arl_expect(arl_base, "l2_cksum_bad", &arcstats.l2_cksum_bad);
+ arl_expect(arl_base, "l2_io_error", &arcstats.l2_io_error);
+ arl_expect(arl_base, "l2_size", &arcstats.l2_size);
+ arl_expect(arl_base, "l2_asize", &arcstats.l2_asize);
+ arl_expect(arl_base, "l2_hdr_size", &arcstats.l2_hdr_size);
+ arl_expect(arl_base, "l2_compress_successes", &arcstats.l2_compress_successes);
+ arl_expect(arl_base, "l2_compress_zeros", &arcstats.l2_compress_zeros);
+ arl_expect(arl_base, "l2_compress_failures", &arcstats.l2_compress_failures);
+ arl_expect(arl_base, "memory_throttle_count", &arcstats.memory_throttle_count);
+ arl_expect(arl_base, "duplicate_buffers", &arcstats.duplicate_buffers);
+ arl_expect(arl_base, "duplicate_buffers_size", &arcstats.duplicate_buffers_size);
+ arl_expect(arl_base, "duplicate_reads", &arcstats.duplicate_reads);
+ arl_expect(arl_base, "memory_direct_count", &arcstats.memory_direct_count);
+ arl_expect(arl_base, "memory_indirect_count", &arcstats.memory_indirect_count);
+ arl_expect(arl_base, "arc_no_grow", &arcstats.arc_no_grow);
+ arl_expect(arl_base, "arc_tempreserve", &arcstats.arc_tempreserve);
+ arl_expect(arl_base, "arc_loaned_bytes", &arcstats.arc_loaned_bytes);
+ arl_expect(arl_base, "arc_prune", &arcstats.arc_prune);
+ arl_expect(arl_base, "arc_meta_used", &arcstats.arc_meta_used);
+ arl_expect(arl_base, "arc_meta_limit", &arcstats.arc_meta_limit);
+ arl_expect(arl_base, "arc_meta_max", &arcstats.arc_meta_max);
+ arl_expect(arl_base, "arc_meta_min", &arcstats.arc_meta_min);
+ arl_expect(arl_base, "arc_need_free", &arcstats.arc_need_free);
+ arl_expect(arl_base, "arc_sys_free", &arcstats.arc_sys_free);
+ }
+
+ if(unlikely(!ff)) {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, ZFS_PROC_ARCSTATS);
+ ff = procfile_open(config_get("plugin:proc:" ZFS_PROC_ARCSTATS, "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!ff))
+ return 1;
+ }
+
+ ff = procfile_readall(ff);
+ if(unlikely(!ff))
+ return 0; // we return 0, so that we will retry to open it next time
+
+ size_t lines = procfile_lines(ff), l;
+
+ arl_begin(arl_base);
+
+ for(l = 0; l < lines ;l++) {
+ size_t words = procfile_linewords(ff, l);
+ if(unlikely(words < 3)) {
+ if(unlikely(words)) error("Cannot read " ZFS_PROC_ARCSTATS " line %zu. Expected 3 params, read %zu.", l, words);
+ continue;
+ }
+
+ const char *key = procfile_lineword(ff, l, 0);
+ const char *value = procfile_lineword(ff, l, 2);
+
+ if(unlikely(l2exist == -1)) {
+ if(key[0] == 'l' && key[1] == '2' && key[2] == '_')
+ l2exist = 1;
+ }
+
+ if(unlikely(arl_check(arl_base, key, value))) break;
+ }
+
+ if(unlikely(l2exist == -1))
+ l2exist = 0;
+
+ generate_charts_arcstats(update_every);
+ generate_charts_arc_summary(update_every);
+
+ return 0;
+}
diff --git a/src/proc_vmstat.c b/src/proc_vmstat.c
index 847487363..a2416313a 100644
--- a/src/proc_vmstat.c
+++ b/src/proc_vmstat.c
@@ -169,11 +169,11 @@ int do_proc_vmstat(int update_every, usec_t dt) {
// The following stats depend on CONFIG_NUMA_BALANCING in the
// kernel.
- rrddim_add(st_numa, "pte updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st_numa, "huge pte updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st_numa, "hint faults", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st_numa, "hint faults local", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
- rrddim_add(st_numa, "pages migrated", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st_numa, "pte_updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st_numa, "huge_pte_updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st_numa, "hint_faults", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st_numa, "hint_faults_local", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rrddim_add(st_numa, "pages_migrated", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
}
else rrdset_next(st_numa);
@@ -182,11 +182,11 @@ int do_proc_vmstat(int update_every, usec_t dt) {
rrddim_set(st_numa, "interleave", numa_interleave);
rrddim_set(st_numa, "other", numa_other);
- rrddim_set(st_numa, "pte updates", numa_pte_updates);
- rrddim_set(st_numa, "huge pte updates", numa_huge_pte_updates);
- rrddim_set(st_numa, "hint faults", numa_hint_faults);
- rrddim_set(st_numa, "hint faults local", numa_hint_faults_local);
- rrddim_set(st_numa, "pages migrated", numa_pages_migrated);
+ rrddim_set(st_numa, "pte_updates", numa_pte_updates);
+ rrddim_set(st_numa, "huge_pte_updates", numa_huge_pte_updates);
+ rrddim_set(st_numa, "hint_faults", numa_hint_faults);
+ rrddim_set(st_numa, "hint_faults_local", numa_hint_faults_local);
+ rrddim_set(st_numa, "pages_migrated", numa_pages_migrated);
rrdset_done(st_numa);
}
diff --git a/src/registry.c b/src/registry.c
index ed9be9848..76e3fa4d2 100644
--- a/src/registry.c
+++ b/src/registry.c
@@ -45,7 +45,7 @@ static inline void registry_json_header(RRDHOST *host, struct web_client *w, con
buffer_flush(w->response.data);
w->response.data->contenttype = CT_APPLICATION_JSON;
buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"",
- action, status, (host == localhost)?registry.hostname:host->hostname, host->machine_guid);
+ action, status, host->registry_hostname, host->machine_guid);
}
static inline void registry_json_footer(struct web_client *w) {
diff --git a/src/registry.h b/src/registry.h
index 2c4592b92..9aa241564 100644
--- a/src/registry.h
+++ b/src/registry.h
@@ -70,6 +70,8 @@ extern int registry_request_hello_json(RRDHOST *host, struct web_client *w);
extern void registry_statistics(void);
extern char *registry_get_this_machine_guid(void);
+extern char *registry_get_this_machine_hostname(void);
+
extern int regenerate_guid(const char *guid, char *result);
#endif /* NETDATA_REGISTRY_H */
diff --git a/src/registry_init.c b/src/registry_init.c
index 2a41d36ee..654f66d12 100644
--- a/src/registry_init.c
+++ b/src/registry_init.c
@@ -34,7 +34,7 @@ int registry_init(void) {
registry.persons_expiration = config_get_number(CONFIG_SECTION_REGISTRY, "registry expire idle persons days", 365) * 86400;
registry.registry_domain = config_get(CONFIG_SECTION_REGISTRY, "registry domain", "");
registry.registry_to_announce = config_get(CONFIG_SECTION_REGISTRY, "registry to announce", "https://registry.my-netdata.io");
- registry.hostname = config_get(CONFIG_SECTION_REGISTRY, "registry hostname", config_get(CONFIG_SECTION_GLOBAL, "hostname", "localhost"));
+ registry.hostname = config_get(CONFIG_SECTION_REGISTRY, "registry hostname", netdata_configured_hostname);
registry.verify_cookies_redirects = config_get_boolean(CONFIG_SECTION_REGISTRY, "verify browser cookies support", 1);
setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1);
diff --git a/src/registry_internals.c b/src/registry_internals.c
index 9ec91ba40..fd3c295ce 100644
--- a/src/registry_internals.c
+++ b/src/registry_internals.c
@@ -274,6 +274,10 @@ static inline int is_machine_guid_blacklisted(const char *guid) {
return 0;
}
+char *registry_get_this_machine_hostname(void) {
+ return registry.hostname;
+}
+
char *registry_get_this_machine_guid(void) {
static char guid[GUID_LEN + 1] = "";
diff --git a/src/rrd.c b/src/rrd.c
index a9ff6243d..85ce93dd3 100644
--- a/src/rrd.c
+++ b/src/rrd.c
@@ -31,19 +31,28 @@ inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) {
return RRD_MEMORY_MODE_NONE_NAME;
case RRD_MEMORY_MODE_SAVE:
- default:
return RRD_MEMORY_MODE_SAVE_NAME;
+
+ case RRD_MEMORY_MODE_ALLOC:
+ return RRD_MEMORY_MODE_ALLOC_NAME;
}
+
+ return RRD_MEMORY_MODE_SAVE_NAME;
}
RRD_MEMORY_MODE rrd_memory_mode_id(const char *name) {
if(unlikely(!strcmp(name, RRD_MEMORY_MODE_RAM_NAME)))
return RRD_MEMORY_MODE_RAM;
+
else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_MAP_NAME)))
return RRD_MEMORY_MODE_MAP;
+
else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_NONE_NAME)))
return RRD_MEMORY_MODE_NONE;
+ else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_ALLOC_NAME)))
+ return RRD_MEMORY_MODE_ALLOC;
+
return RRD_MEMORY_MODE_SAVE;
}
diff --git a/src/rrd.h b/src/rrd.h
index 2f4f2127f..5bc61dcb8 100644
--- a/src/rrd.h
+++ b/src/rrd.h
@@ -5,7 +5,7 @@
#define UPDATE_EVERY_MAX 3600
#define RRD_DEFAULT_HISTORY_ENTRIES 3600
-#define RRD_HISTORY_ENTRIES_MAX (86400*10)
+#define RRD_HISTORY_ENTRIES_MAX (86400*365)
extern int default_rrd_update_every;
extern int default_rrd_history_entries;
@@ -42,13 +42,15 @@ typedef enum rrd_memory_mode {
RRD_MEMORY_MODE_NONE = 0,
RRD_MEMORY_MODE_RAM = 1,
RRD_MEMORY_MODE_MAP = 2,
- RRD_MEMORY_MODE_SAVE = 3
+ RRD_MEMORY_MODE_SAVE = 3,
+ RRD_MEMORY_MODE_ALLOC = 4
} RRD_MEMORY_MODE;
#define RRD_MEMORY_MODE_NONE_NAME "none"
#define RRD_MEMORY_MODE_RAM_NAME "ram"
#define RRD_MEMORY_MODE_MAP_NAME "map"
#define RRD_MEMORY_MODE_SAVE_NAME "save"
+#define RRD_MEMORY_MODE_ALLOC_NAME "alloc"
extern RRD_MEMORY_MODE default_rrd_memory_mode;
@@ -101,9 +103,15 @@ typedef enum rrddim_flags {
RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = 1 << 1 // do not offer RESET or OVERFLOW info to callers
} RRDDIM_FLAGS;
+#ifdef HAVE_C___ATOMIC
+#define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & flag)
+#define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), flag, __ATOMIC_SEQ_CST)
+#define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~flag, __ATOMIC_SEQ_CST)
+#else
#define rrddim_flag_check(rd, flag) ((rd)->flags & flag)
#define rrddim_flag_set(rd, flag) (rd)->flags |= flag
#define rrddim_flag_clear(rd, flag) (rd)->flags &= ~flag
+#endif
// ----------------------------------------------------------------------------
@@ -210,16 +218,28 @@ typedef struct rrddim RRDDIM;
// and may lead to missing information.
typedef enum rrdset_flags {
- RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart
- RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another
- // (the master data set should be the one that has the same family and is not detail)
- RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart
- RRDSET_FLAG_OBSOLETE = 1 << 3 // this is marked by the collector/module as obsolete
+ RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart
+ RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another
+ // (the master data set should be the one that has the same family and is not detail)
+ RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart
+ RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete
+ RRDSET_FLAG_BACKEND_SEND = 1 << 4, // if set, this chart should be sent to backends
+ RRDSET_FLAG_BACKEND_IGNORE = 1 << 5, // if set, this chart should not be sent to backends
+ RRDSET_FLAG_EXPOSED_UPSTREAM = 1 << 6, // if set, we have sent this chart to netdata master (streaming)
+ RRDSET_FLAG_STORE_FIRST = 1 << 7, // if set, do not eliminate the first collection during interpolation
+ RRDSET_FLAG_HETEROGENEOUS = 1 << 8, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
+ RRDSET_FLAG_HOMEGENEOUS_CHECK= 1 << 9 // if set, the chart should be checked to determine if the dimensions as homogeneous
} RRDSET_FLAGS;
+#ifdef HAVE_C___ATOMIC
+#define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_SEQ_CST) & flag)
+#define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_SEQ_CST)
+#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~flag, __ATOMIC_SEQ_CST)
+#else
#define rrdset_flag_check(st, flag) ((st)->flags & flag)
#define rrdset_flag_set(st, flag) (st)->flags |= flag
#define rrdset_flag_clear(st, flag) (st)->flags &= ~flag
+#endif
struct rrdset {
// ------------------------------------------------------------------------
@@ -279,7 +299,8 @@ struct rrdset {
size_t counter_done; // the number of times rrdset_done() has been called
time_t last_accessed_time; // the last time this RRDSET has been accessed
- size_t unused[9];
+ time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream
+ size_t unused[8];
uint32_t hash; // a simple hash on the id, to speed up searching
// we first compare hashes, and only if the hashes are equal we do string comparisons
@@ -347,14 +368,28 @@ typedef struct rrdset RRDSET;
// and may lead to missing information.
typedef enum rrdhost_flags {
- RRDHOST_ORPHAN = 1 << 0, // this host is orphan
- RRDHOST_DELETE_OBSOLETE_FILES = 1 << 1, // delete files of obsolete charts
- RRDHOST_DELETE_ORPHAN_FILES = 1 << 2 // delete the entire host when orphan
+ RRDHOST_ORPHAN = 1 << 0, // this host is orphan (not receiving data)
+ RRDHOST_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts
+ RRDHOST_DELETE_ORPHAN_HOST = 1 << 2 // delete the entire host when orphan
} RRDHOST_FLAGS;
+#ifdef HAVE_C___ATOMIC
+#define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & flag)
+#define rrdhost_flag_set(host, flag) __atomic_or_fetch(&((host)->flags), flag, __ATOMIC_SEQ_CST)
+#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~flag, __ATOMIC_SEQ_CST)
+#else
#define rrdhost_flag_check(host, flag) ((host)->flags & flag)
#define rrdhost_flag_set(host, flag) (host)->flags |= flag
#define rrdhost_flag_clear(host, flag) (host)->flags &= ~flag
+#endif
+
+#ifdef NETDATA_INTERNAL_CHECKS
+#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \
+ debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, st->name, ##args); } while(0)
+#else
+#define rrdset_debug(st, fmt, args...) debug_dummy()
+#endif
+
// ----------------------------------------------------------------------------
// RRD HOST
@@ -368,10 +403,13 @@ struct rrdhost {
char *hostname; // the hostname of this host
uint32_t hash_hostname; // the hostname hash
+ char *registry_hostname; // the registry hostname for this host
+
char machine_guid[GUID_LEN + 1]; // the unique ID of this host
uint32_t hash_machine_guid; // the hash of the unique ID
- char *os; // the O/S type of the host
+ const char *os; // the O/S type of the host
+ const char *tags; // tags for this host
uint32_t flags; // flags about this RRDHOST
@@ -488,8 +526,10 @@ extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash);
extern RRDHOST *rrdhost_find_or_create(
const char *hostname
+ , const char *registry_hostname
, const char *guid
, const char *os
+ , const char *tags
, int update_every
, long history
, RRD_MEMORY_MODE mode
@@ -528,7 +568,7 @@ extern void __rrd_check_wrlock(const char *file, const char *function, const uns
extern void rrdset_set_name(RRDSET *st, const char *name);
-extern RRDSET *rrdset_create(RRDHOST *host
+extern RRDSET *rrdset_create_custom(RRDHOST *host
, const char *type
, const char *id
, const char *name
@@ -538,18 +578,27 @@ extern RRDSET *rrdset_create(RRDHOST *host
, const char *units
, long priority
, int update_every
- , RRDSET_TYPE chart_type);
+ , RRDSET_TYPE chart_type
+ , RRD_MEMORY_MODE memory_mode
+ , long history_entries);
+
+#define rrdset_create(host, type, id, name, family, context, title, units, priority, update_every, chart_type) \
+ rrdset_create_custom(host, type, id, name, family, context, title, units, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries)
-#define rrdset_create_localhost(type, id, name, family, context, title, units, priority, update_every, chart_type) rrdset_create(localhost, type, id, name, family, context, title, units, priority, update_every, chart_type)
+#define rrdset_create_localhost(type, id, name, family, context, title, units, priority, update_every, chart_type) \
+ rrdset_create(localhost, type, id, name, family, context, title, units, priority, update_every, chart_type)
extern void rrdhost_free_all(void);
extern void rrdhost_save_all(void);
+extern void rrdhost_cleanup_all(void);
-extern void rrdhost_cleanup_orphan(RRDHOST *protected);
+extern void rrdhost_cleanup_orphan_hosts(RRDHOST *protected);
extern void rrdhost_free(RRDHOST *host);
extern void rrdhost_save(RRDHOST *host);
extern void rrdhost_delete(RRDHOST *host);
+extern void rrdset_update_heterogeneous_flag(RRDSET *st);
+
extern RRDSET *rrdset_find(RRDHOST *host, const char *id);
#define rrdset_find_localhost(id) rrdset_find(localhost, id)
@@ -565,8 +614,12 @@ extern void rrdset_next_usec(RRDSET *st, usec_t microseconds);
extern void rrdset_done(RRDSET *st);
+extern void rrdset_is_obsolete(RRDSET *st);
+extern void rrdset_isnot_obsolete(RRDSET *st);
+
// checks if the RRDSET should be offered to viewers
-#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions)
+#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
+#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions)
// get the total duration in seconds of the round robin database
#define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every ))
@@ -604,9 +657,14 @@ extern void rrdset_done(RRDSET *st);
// ----------------------------------------------------------------------------
// RRD DIMENSION functions
-extern RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm);
+extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode);
+#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
+
+extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name);
+extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
+extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier);
+extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor);
-extern void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name);
extern RRDDIM *rrddim_find(RRDSET *st, const char *id);
extern int rrddim_hide(RRDSET *st, const char *id);
@@ -647,7 +705,7 @@ extern void rrdset_reset(RRDSET *st);
extern void rrdset_save(RRDSET *st);
extern void rrdset_delete(RRDSET *st);
-extern void rrdhost_cleanup_obsolete(RRDHOST *host);
+extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);
#endif /* NETDATA_RRD_INTERNALS */
diff --git a/src/rrd2json.c b/src/rrd2json.c
index 4d853930c..98080139c 100644
--- a/src/rrd2json.c
+++ b/src/rrd2json.c
@@ -11,7 +11,7 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions
"\t\t\t\"type\": \"%s\",\n"
"\t\t\t\"family\": \"%s\",\n"
"\t\t\t\"context\": \"%s\",\n"
- "\t\t\t\"title\": \"%s\",\n"
+ "\t\t\t\"title\": \"%s (%s)\",\n"
"\t\t\t\"priority\": %ld,\n"
"\t\t\t\"enabled\": %s,\n"
"\t\t\t\"units\": \"%s\",\n"
@@ -27,7 +27,7 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions
, st->type
, st->family
, st->context
- , st->title
+ , st->title, st->name
, st->priority
, rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?"true":"false"
, st->units
@@ -167,79 +167,6 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) {
}
// ----------------------------------------------------------------------------
-// PROMETHEUS
-// /api/v1/allmetrics?format=prometheus
-
-static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) {
- size_t n;
-
- for(n = 0; *s && n < usable ; d++, s++, n++) {
- register char c = *s;
-
- if(unlikely(!isalnum(c))) *d = '_';
- else *d = c;
- }
- *d = '\0';
-
- return n;
-}
-
-#define PROMETHEUS_ELEMENT_MAX 256
-
-void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb) {
- rrdhost_rdlock(host);
-
- char hostname[PROMETHEUS_ELEMENT_MAX + 1];
- prometheus_name_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
-
- // for each chart
- RRDSET *st;
- rrdset_foreach_read(st, host) {
- char chart[PROMETHEUS_ELEMENT_MAX + 1];
- prometheus_name_copy(chart, st->id, PROMETHEUS_ELEMENT_MAX);
-
- buffer_strcat(wb, "\n");
- if(rrdset_is_available_for_viewers(st)) {
- rrdset_rdlock(st);
-
- // for each dimension
- RRDDIM *rd;
- rrddim_foreach_read(rd, st) {
- if(rd->collections_counter) {
- char dimension[PROMETHEUS_ELEMENT_MAX + 1];
- prometheus_name_copy(dimension, rd->id, PROMETHEUS_ELEMENT_MAX);
-
- // buffer_sprintf(wb, "# HELP %s.%s %s\n", st->id, rd->id, st->units);
-
- switch(rd->algorithm) {
- case RRD_ALGORITHM_INCREMENTAL:
- case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
- buffer_sprintf(wb, "# TYPE %s_%s counter\n", chart, dimension);
- break;
-
- default:
- buffer_sprintf(wb, "# TYPE %s_%s gauge\n", chart, dimension);
- break;
- }
-
- // calculated_number n = (calculated_number)rd->last_collected_value * (calculated_number)(abs(rd->multiplier)) / (calculated_number)(abs(rd->divisor));
- // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n, timeval_msec(&rd->last_collected_time));
-
- buffer_sprintf(wb, "%s_%s{instance=\"%s\"} " COLLECTED_NUMBER_FORMAT " %llu\n",
- chart, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)
- );
-
- }
- }
-
- rrdset_unlock(st);
- }
- }
-
- rrdhost_unlock(host);
-}
-
-// ----------------------------------------------------------------------------
// BASH
// /api/v1/allmetrics?format=bash
@@ -267,7 +194,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) {
rrdset_foreach_read(st, host) {
calculated_number total = 0.0;
char chart[SHELL_ELEMENT_MAX + 1];
- shell_name_copy(chart, st->id, SHELL_ELEMENT_MAX);
+ shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX);
buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name);
if(rrdset_is_available_for_viewers(st)) {
@@ -278,7 +205,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) {
rrddim_foreach_read(rd, st) {
if(rd->collections_counter) {
char dimension[SHELL_ELEMENT_MAX + 1];
- shell_name_copy(dimension, rd->id, SHELL_ELEMENT_MAX);
+ shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX);
calculated_number n = rd->last_stored_value;
@@ -306,7 +233,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) {
if(!rc->rrdset) continue;
char chart[SHELL_ELEMENT_MAX + 1];
- shell_name_copy(chart, rc->rrdset->id, SHELL_ELEMENT_MAX);
+ shell_name_copy(chart, rc->rrdset->name?rc->rrdset->name:rc->rrdset->id, SHELL_ELEMENT_MAX);
char alarm[SHELL_ELEMENT_MAX + 1];
shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
diff --git a/src/rrd2json.h b/src/rrd2json.h
index f2f03c640..a3b9c9509 100644
--- a/src/rrd2json.h
+++ b/src/rrd2json.h
@@ -31,13 +31,15 @@
#define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma"
#define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray"
-#define ALLMETRICS_FORMAT_SHELL "shell"
-#define ALLMETRICS_FORMAT_PROMETHEUS "prometheus"
-#define ALLMETRICS_FORMAT_JSON "json"
+#define ALLMETRICS_FORMAT_SHELL "shell"
+#define ALLMETRICS_FORMAT_PROMETHEUS "prometheus"
+#define ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS "prometheus_all_hosts"
+#define ALLMETRICS_FORMAT_JSON "json"
-#define ALLMETRICS_SHELL 1
-#define ALLMETRICS_PROMETHEUS 2
-#define ALLMETRICS_JSON 3
+#define ALLMETRICS_SHELL 1
+#define ALLMETRICS_PROMETHEUS 2
+#define ALLMETRICS_JSON 3
+#define ALLMETRICS_PROMETHEUS_ALL_HOSTS 4
#define GROUP_UNDEFINED 0
#define GROUP_AVERAGE 1
@@ -65,13 +67,13 @@ extern void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb);
extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb);
extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb);
-extern void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb);
extern int rrdset2anything_api_v1(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points
- , long long after, long long before, int group_method, uint32_t options
- , time_t *latest_timestamp);
+ , long long after, long long before, int group_method, uint32_t options
+ , time_t *latest_timestamp);
+
extern int rrdset2value_api_v1(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points
- , long long after, long long before, int group_method, uint32_t options
- , time_t *db_before, time_t *db_after, int *value_is_null);
+ , long long after, long long before, int group_method, uint32_t options
+ , time_t *db_before, time_t *db_after, int *value_is_null);
#endif /* NETDATA_RRD2JSON_H */
diff --git a/src/rrd2json_api_old.c b/src/rrd2json_api_old.c
index 6710f31cf..003b8626d 100644
--- a/src/rrd2json_api_old.c
+++ b/src/rrd2json_api_old.c
@@ -14,7 +14,7 @@ unsigned long rrdset_info2json_api_old(RRDSET *st, char *options, BUFFER *wb) {
"\t\t\t\"type\": \"%s\",\n"
"\t\t\t\"family\": \"%s\",\n"
"\t\t\t\"context\": \"%s\",\n"
- "\t\t\t\"title\": \"%s\",\n"
+ "\t\t\t\"title\": \"%s (%s)\",\n"
"\t\t\t\"priority\": %ld,\n"
"\t\t\t\"enabled\": %d,\n"
"\t\t\t\"units\": \"%s\",\n"
@@ -37,7 +37,7 @@ unsigned long rrdset_info2json_api_old(RRDSET *st, char *options, BUFFER *wb) {
, st->type
, st->family
, st->context
- , st->title
+ , st->title, st->name
, st->priority
, rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?1:0
, st->units
diff --git a/src/rrdcalc.c b/src/rrdcalc.c
index 1f1845409..bb90a4c6d 100644
--- a/src/rrdcalc.c
+++ b/src/rrdcalc.c
@@ -219,7 +219,7 @@ inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, ui
for(rc = host->alarms; rc ; rc = rc->next) {
if (unlikely(rc->chart && rc->hash == hash_name && rc->hash_chart == hash_chart && !strcmp(name, rc->name) && !strcmp(chart, rc->chart))) {
debug(D_HEALTH, "Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
- error("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
+ info("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
return 1;
}
}
diff --git a/src/rrdcalctemplate.c b/src/rrdcalctemplate.c
index 2c5e2bd16..b5d2c7d61 100644
--- a/src/rrdcalctemplate.c
+++ b/src/rrdcalctemplate.c
@@ -12,7 +12,7 @@ void rrdcalctemplate_link_matching(RRDSET *st) {
&& (!rt->family_pattern || simple_pattern_matches(rt->family_pattern, st->family))) {
RRDCALC *rc = rrdcalc_create(st->rrdhost, rt, st->id);
if(unlikely(!rc))
- error("Health tried to create alarm from template '%s', but it failed", rt->name);
+ info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, st->rrdhost->hostname);
#ifdef NETDATA_INTERNAL_CHECKS
else if(rc->rrdset != st)
diff --git a/src/rrddim.c b/src/rrddim.c
index 54a17522f..e75aa3fd2 100644
--- a/src/rrddim.c
+++ b/src/rrddim.c
@@ -35,9 +35,9 @@ inline RRDDIM *rrddim_find(RRDSET *st, const char *id) {
// ----------------------------------------------------------------------------
// RRDDIM rename a dimension
-inline void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) {
- if(unlikely(!strcmp(rd->name, name)))
- return;
+inline int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) {
+ if(unlikely(!name || !*name || !strcmp(rd->name, name)))
+ return 0;
debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", st->name, rd->name, st->name, name);
@@ -45,18 +45,57 @@ inline void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) {
snprintfz(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
rd->name = config_set_default(st->config_section, varname, name);
rd->hash_name = simple_hash(rd->name);
-
rrddimvar_rename_all(rd);
+ rd->exposed = 0;
+ return 1;
+}
+
+inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm) {
+ if(unlikely(rd->algorithm == algorithm))
+ return 0;
+
+ debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", st->id, rd->name, rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm));
+ rd->algorithm = algorithm;
+ rd->exposed = 0;
+ rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
+ return 1;
+}
+
+inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier) {
+ if(unlikely(rd->multiplier == multiplier))
+ return 0;
+
+ debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->multiplier, multiplier);
+ rd->multiplier = multiplier;
+ rd->exposed = 0;
+ rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
+ return 1;
}
+inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) {
+ if(unlikely(rd->divisor == divisor))
+ return 0;
+
+ debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->divisor, divisor);
+ rd->divisor = divisor;
+ rd->exposed = 0;
+ rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
+ return 1;
+}
// ----------------------------------------------------------------------------
// RRDDIM create a dimension
-RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm) {
+RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) {
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(rd)) {
debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>");
+
+ rrddim_set_name(st, rd, name);
+ rrddim_set_algorithm(st, rd, algorithm);
+ rrddim_set_multiplier(st, rd, multiplier);
+ rrddim_set_divisor(st, rd, divisor);
+
return rd;
}
@@ -71,8 +110,14 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
rrdset_strncpyz_name(filename, id, FILENAME_MAX);
snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
- if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
- rd = (RRDDIM *)mymmap(fullfilename, size, ((st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1);
+ if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_RAM) {
+ rd = (RRDDIM *)mymmap(
+ (memory_mode == RRD_MEMORY_MODE_RAM)?NULL:fullfilename
+ , size
+ , ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE)
+ , 1
+ );
+
if(likely(rd)) {
// we have a file mapped for rd
@@ -83,56 +128,64 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
rd->variables = NULL;
rd->next = NULL;
rd->rrdset = NULL;
+ rd->exposed = 0;
struct timeval now;
now_realtime_timeval(&now);
- if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) {
- errno = 0;
- info("Initializing file %s.", fullfilename);
+ if(memory_mode == RRD_MEMORY_MODE_RAM) {
memset(rd, 0, size);
}
- else if(rd->memsize != size) {
- errno = 0;
- error("File %s does not have the desired size. Clearing it.", fullfilename);
- memset(rd, 0, size);
- }
- else if(rd->multiplier != multiplier) {
- errno = 0;
- error("File %s does not have the same multiplier. Clearing it.", fullfilename);
- memset(rd, 0, size);
- }
- else if(rd->divisor != divisor) {
- errno = 0;
- error("File %s does not have the same divisor. Clearing it.", fullfilename);
- memset(rd, 0, size);
- }
- else if(rd->update_every != st->update_every) {
- errno = 0;
- error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
- memset(rd, 0, size);
- }
- else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) {
- errno = 0;
- error("File %s is too old. Clearing it.", fullfilename);
- memset(rd, 0, size);
+ else {
+ int reset = 0;
+
+ if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) {
+ info("Initializing file %s.", fullfilename);
+ memset(rd, 0, size);
+ reset = 1;
+ }
+ else if(rd->memsize != size) {
+ error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd->memsize);
+ memset(rd, 0, size);
+ reset = 1;
+ }
+ else if(rd->update_every != st->update_every) {
+ error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd->update_every);
+ memset(rd, 0, size);
+ reset = 1;
+ }
+ else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) {
+ error("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd->last_collected_time) / USEC_PER_SEC, rd->entries * rd->update_every);
+ memset(rd, 0, size);
+ reset = 1;
+ }
+
+ if(!reset) {
+ if(rd->algorithm != algorithm) {
+ info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
+ fullfilename, algorithm, rrd_algorithm_name(algorithm), rd->algorithm, rrd_algorithm_name(rd->algorithm));
+ }
+
+ if(rd->multiplier != multiplier) {
+ info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT ". Previous values may be wrong.", fullfilename, multiplier, rd->multiplier);
+ }
+
+ if(rd->divisor != divisor) {
+ info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT ". Previous values may be wrong.", fullfilename, divisor, rd->divisor);
+ }
+ }
}
- if(rd->algorithm && rd->algorithm != algorithm)
- error("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong."
- , fullfilename, algorithm, rrd_algorithm_name(algorithm), rd->algorithm,
- rrd_algorithm_name(rd->algorithm));
-
// make sure we have the right memory mode
// even if we cleared the memory
- rd->rrd_memory_mode = st->rrd_memory_mode;
+ rd->rrd_memory_mode = memory_mode;
}
}
if(unlikely(!rd)) {
// if we didn't manage to get a mmap'd dimension, just create one
rd = callocz(1, size);
- rd->rrd_memory_mode = (st->rrd_memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_RAM;
+ rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
rd->memsize = size;
@@ -161,8 +214,11 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
rd->entries = st->entries;
rd->update_every = st->update_every;
- // prevent incremental calculation spikes
- rd->collections_counter = 0;
+ if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
+ rd->collections_counter = 1;
+ else
+ rd->collections_counter = 0;
+
rd->updated = 0;
rd->flags = 0x00000000;
@@ -173,7 +229,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
rd->collected_volume = 0;
rd->stored_volume = 0;
rd->last_stored_value = 0;
- rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
+ rd->values[st->current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS);
rd->last_collected_time.tv_sec = 0;
rd->last_collected_time.tv_usec = 0;
rd->rrdset = st;
@@ -184,6 +240,23 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
st->dimensions = rd;
else {
RRDDIM *td = st->dimensions;
+
+ if(td->algorithm != rd->algorithm || abs(td->multiplier) != abs(rd->multiplier) || abs(td->divisor) != abs(rd->divisor)) {
+ if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").",
+ rd->name,
+ st->name,
+ st->rrdhost->hostname,
+ rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm),
+ rd->multiplier, td->multiplier,
+ rd->divisor, td->divisor
+ );
+ #endif
+ rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS);
+ }
+ }
+
for(; td->next; td = td->next) ;
td->next = rd;
}
@@ -202,7 +275,6 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
return(rd);
}
-
// ----------------------------------------------------------------------------
// RRDDIM remove / free a dimension
@@ -233,19 +305,16 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
switch(rd->rrd_memory_mode) {
case RRD_MEMORY_MODE_SAVE:
- debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename);
- savememory(rd->cache_filename, rd, rd->memsize);
- // continue to map mode - no break;
-
case RRD_MEMORY_MODE_MAP:
+ case RRD_MEMORY_MODE_RAM:
debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
freez((void *)rd->id);
freez(rd->cache_filename);
munmap(rd, rd->memsize);
break;
+ case RRD_MEMORY_MODE_ALLOC:
case RRD_MEMORY_MODE_NONE:
- case RRD_MEMORY_MODE_RAM:
debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
freez((void *)rd->id);
freez(rd->cache_filename);
@@ -263,7 +332,7 @@ int rrddim_hide(RRDSET *st, const char *id) {
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(!rd)) {
- error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
+ error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, st->rrdhost->hostname);
return 1;
}
@@ -276,7 +345,7 @@ int rrddim_unhide(RRDSET *st, const char *id) {
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(!rd)) {
- error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
+ error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, st->rrdhost->hostname);
return 1;
}
@@ -305,7 +374,7 @@ inline collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_
collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) {
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(!rd)) {
- error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
+ error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, st->rrdhost->hostname);
return 0;
}
diff --git a/src/rrdhost.c b/src/rrdhost.c
index a2310330d..ff8aa5617 100644
--- a/src/rrdhost.c
+++ b/src/rrdhost.c
@@ -58,15 +58,32 @@ RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash) {
// ----------------------------------------------------------------------------
// RRDHOST - internal helpers
+static inline void rrdhost_init_tags(RRDHOST *host, const char *tags) {
+ if(host->tags && tags && !strcmp(host->tags, tags))
+ return;
+
+ void *old = (void *)host->tags;
+ host->tags = (tags && *tags)?strdupz(tags):NULL;
+ freez(old);
+}
+
static inline void rrdhost_init_hostname(RRDHOST *host, const char *hostname) {
- freez(host->hostname);
- host->hostname = strdupz(hostname);
+ if(host->hostname && hostname && !strcmp(host->hostname, hostname))
+ return;
+
+ void *old = host->hostname;
+ host->hostname = strdupz(hostname?hostname:"localhost");
host->hash_hostname = simple_hash(host->hostname);
+ freez(old);
}
static inline void rrdhost_init_os(RRDHOST *host, const char *os) {
- freez(host->os);
+ if(host->os && os && !strcmp(host->os, os))
+ return;
+
+ void *old = (void *)host->os;
host->os = strdupz(os?os:"unknown");
+ freez(old);
}
static inline void rrdhost_init_machine_guid(RRDHOST *host, const char *machine_guid) {
@@ -80,16 +97,18 @@ static inline void rrdhost_init_machine_guid(RRDHOST *host, const char *machine_
// RRDHOST - add a host
RRDHOST *rrdhost_create(const char *hostname,
- const char *guid,
- const char *os,
- int update_every,
- long entries,
- RRD_MEMORY_MODE memory_mode,
- int health_enabled,
- int rrdpush_enabled,
- char *rrdpush_destination,
- char *rrdpush_api_key,
- int is_localhost
+ const char *registry_hostname,
+ const char *guid,
+ const char *os,
+ const char *tags,
+ int update_every,
+ long entries,
+ RRD_MEMORY_MODE memory_mode,
+ int health_enabled,
+ int rrdpush_enabled,
+ char *rrdpush_destination,
+ char *rrdpush_api_key,
+ int is_localhost
) {
debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
@@ -115,6 +134,8 @@ RRDHOST *rrdhost_create(const char *hostname,
rrdhost_init_hostname(host, hostname);
rrdhost_init_machine_guid(host, guid);
rrdhost_init_os(host, os);
+ rrdhost_init_tags(host, tags);
+ host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
avl_init_lock(&(host->rrdset_root_index), rrdset_compare);
avl_init_lock(&(host->rrdset_root_index_name), rrdset_compare_name);
@@ -122,10 +143,10 @@ RRDHOST *rrdhost_create(const char *hostname,
avl_init_lock(&(host->variables_root_index), rrdvar_compare);
if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1))
- rrdhost_flag_set(host, RRDHOST_DELETE_OBSOLETE_FILES);
+ rrdhost_flag_set(host, RRDHOST_DELETE_OBSOLETE_CHARTS);
if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost)
- rrdhost_flag_set(host, RRDHOST_DELETE_ORPHAN_FILES);
+ rrdhost_flag_set(host, RRDHOST_DELETE_ORPHAN_HOST);
// ------------------------------------------------------------------------
@@ -229,8 +250,9 @@ RRDHOST *rrdhost_create(const char *hostname,
host = NULL;
}
else {
- info("Host '%s' with guid '%s' initialized"
+ info("Host '%s' (at registry as '%s') with guid '%s' initialized"
", os %s"
+ ", tags '%s'"
", update every %d"
", memory mode %s"
", history entries %ld"
@@ -243,8 +265,10 @@ RRDHOST *rrdhost_create(const char *hostname,
", alarms default handler '%s'"
", alarms default recipient '%s'"
, host->hostname
+ , host->registry_hostname
, host->machine_guid
, host->os
+ , (host->tags)?host->tags:""
, host->rrd_update_every
, rrd_memory_mode_name(host->rrd_memory_mode)
, host->rrd_history_entries
@@ -267,8 +291,10 @@ RRDHOST *rrdhost_create(const char *hostname,
RRDHOST *rrdhost_find_or_create(
const char *hostname
+ , const char *registry_hostname
, const char *guid
, const char *os
+ , const char *tags
, int update_every
, long history
, RRD_MEMORY_MODE mode
@@ -284,8 +310,10 @@ RRDHOST *rrdhost_find_or_create(
if(!host) {
host = rrdhost_create(
hostname
+ , registry_hostname
, guid
, os
+ , tags
, update_every
, history
, mode
@@ -307,22 +335,25 @@ RRDHOST *rrdhost_find_or_create(
}
if(host->rrd_update_every != update_every)
- error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds.", host->hostname, host->rrd_update_every, update_every);
+ error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
- if(host->rrd_history_entries != history)
- error("Host '%s' has history of %ld entries, but the wanted one is %ld entries.", host->hostname, host->rrd_history_entries, history);
+ if(host->rrd_history_entries < history)
+ error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
if(host->rrd_memory_mode != mode)
- error("Host '%s' has memory mode '%s', but the wanted one is '%s'.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
+ error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
+
+ // update host tags
+ rrdhost_init_tags(host, tags);
}
rrd_unlock();
- rrdhost_cleanup_orphan(host);
+ rrdhost_cleanup_orphan_hosts(host);
return host;
}
-static inline int rrdhost_should_be_deleted(RRDHOST *host, RRDHOST *protected, time_t now) {
+static inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) {
if(host != protected
&& host != localhost
&& !host->connected_senders
@@ -333,7 +364,7 @@ static inline int rrdhost_should_be_deleted(RRDHOST *host, RRDHOST *protected, t
return 0;
}
-void rrdhost_cleanup_orphan(RRDHOST *protected) {
+void rrdhost_cleanup_orphan_hosts(RRDHOST *protected) {
time_t now = now_realtime_sec();
rrd_wrlock();
@@ -342,10 +373,10 @@ void rrdhost_cleanup_orphan(RRDHOST *protected) {
restart_after_removal:
rrdhost_foreach_write(host) {
- if(rrdhost_should_be_deleted(host, protected, now)) {
+ if(rrdhost_should_be_removed(host, protected, now)) {
info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid);
- if(rrdset_flag_check(host, RRDHOST_ORPHAN))
+ if(rrdset_flag_check(host, RRDHOST_DELETE_ORPHAN_HOST) && rrdset_flag_check(host, RRDHOST_ORPHAN))
rrdhost_delete(host);
else
rrdhost_save(host);
@@ -372,8 +403,10 @@ void rrd_init(char *hostname) {
rrd_wrlock();
localhost = rrdhost_create(
hostname
+ , registry_get_this_machine_hostname()
, registry_get_this_machine_guid()
, os_type
+ , config_get(CONFIG_SECTION_BACKEND, "host tags", "")
, default_rrd_update_every
, default_rrd_history_entries
, default_rrd_memory_mode
@@ -473,7 +506,8 @@ void rrdhost_free(RRDHOST *host) {
// ------------------------------------------------------------------------
// free it
- freez(host->os);
+ freez((void *)host->tags);
+ freez((void *)host->os);
freez(host->cache_dir);
freez(host->varlib_dir);
freez(host->rrdpush_api_key);
@@ -482,6 +516,7 @@ void rrdhost_free(RRDHOST *host) {
freez(host->health_default_recipient);
freez(host->health_log_filename);
freez(host->hostname);
+ freez(host->registry_hostname);
rrdhost_unlock(host);
netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock);
netdata_rwlock_destroy(&host->rrdhost_rwlock);
@@ -497,12 +532,12 @@ void rrdhost_free_all(void) {
}
// ----------------------------------------------------------------------------
-// RRDHOST - save
+// RRDHOST - save host files
void rrdhost_save(RRDHOST *host) {
if(!host) return;
- info("Saving database of host '%s'...", host->hostname);
+ info("Saving/Closing database of host '%s'...", host->hostname);
RRDSET *st;
@@ -520,7 +555,7 @@ void rrdhost_save(RRDHOST *host) {
}
// ----------------------------------------------------------------------------
-// RRDHOST - delete files
+// RRDHOST - delete host files
void rrdhost_delete(RRDHOST *host) {
if(!host) return;
@@ -539,9 +574,43 @@ void rrdhost_delete(RRDHOST *host) {
rrdset_unlock(st);
}
+ recursively_delete_dir(host->cache_dir, "left over host");
+
rrdhost_unlock(host);
}
+// ----------------------------------------------------------------------------
+// RRDHOST - cleanup host files
+
+void rrdhost_cleanup(RRDHOST *host) {
+ if(!host) return;
+
+ info("Cleaning up database of host '%s'...", host->hostname);
+
+ RRDSET *st;
+
+ // we get a write lock
+ // to ensure only one thread is saving the database
+ rrdhost_wrlock(host);
+
+ rrdset_foreach_write(st, host) {
+ rrdset_rdlock(st);
+
+ if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS))
+ rrdset_delete(st);
+ else
+ rrdset_save(st);
+
+ rrdset_unlock(st);
+ }
+
+ rrdhost_unlock(host);
+}
+
+
+// ----------------------------------------------------------------------------
+// RRDHOST - save all hosts to disk
+
void rrdhost_save_all(void) {
info("Saving database [%zu hosts(s)]...", rrd_hosts_available);
@@ -554,7 +623,30 @@ void rrdhost_save_all(void) {
rrd_unlock();
}
-void rrdhost_cleanup_obsolete(RRDHOST *host) {
+// ----------------------------------------------------------------------------
+// RRDHOST - save or delete all hosts from disk
+
+void rrdhost_cleanup_all(void) {
+ info("Cleaning up database [%zu hosts(s)]...", rrd_hosts_available);
+
+ rrd_rdlock();
+
+ RRDHOST *host;
+ rrdhost_foreach_read(host) {
+ if(host != localhost && rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS) && !host->connected_senders)
+ rrdhost_delete(host);
+ else
+ rrdhost_cleanup(host);
+ }
+
+ rrd_unlock();
+}
+
+
+// ----------------------------------------------------------------------------
+// RRDHOST - save or delete all the host charts from disk
+
+void rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
time_t now = now_realtime_sec();
RRDSET *st;
@@ -569,7 +661,7 @@ restart_after_removal:
rrdset_rdlock(st);
- if(rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_FILES))
+ if(rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS))
rrdset_delete(st);
else
rrdset_save(st);
diff --git a/src/rrdpush.c b/src/rrdpush.c
index 72e6d8a73..6def90fe5 100644
--- a/src/rrdpush.c
+++ b/src/rrdpush.c
@@ -57,13 +57,16 @@ int rrdpush_init() {
// to its current clock, we send for this many
// iterations a BEGIN line without microseconds
// this is for the first iterations of each chart
-static unsigned int remote_clock_resync_iterations = 60;
+unsigned int remote_clock_resync_iterations = 60;
#define rrdpush_lock(host) netdata_mutex_lock(&((host)->rrdpush_mutex))
#define rrdpush_unlock(host) netdata_mutex_unlock(&((host)->rrdpush_mutex))
// checks if the current chart definition has been sent
static inline int need_to_send_chart_definition(RRDSET *st) {
+ if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_EXPOSED_UPSTREAM))))
+ return 1;
+
RRDDIM *rd;
rrddim_foreach_read(rd, st)
if(!rd->exposed)
@@ -74,7 +77,9 @@ static inline int need_to_send_chart_definition(RRDSET *st) {
// sends the current chart definition
static inline void send_chart_definition(RRDSET *st) {
- buffer_sprintf(st->rrdhost->rrdpush_buffer, "CHART '%s' '%s' '%s' '%s' '%s' '%s' '%s' %ld %d\n"
+ rrdset_flag_set(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
+
+ buffer_sprintf(st->rrdhost->rrdpush_buffer, "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s\"\n"
, st->id
, st->name
, st->title
@@ -84,11 +89,14 @@ static inline void send_chart_definition(RRDSET *st) {
, rrdset_type_name(st->chart_type)
, st->priority
, st->update_every
+ , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":""
+ , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":""
+ , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
);
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
- buffer_sprintf(st->rrdhost->rrdpush_buffer, "DIMENSION '%s' '%s' '%s' " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " '%s %s'\n"
+ buffer_sprintf(st->rrdhost->rrdpush_buffer, "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s\"\n"
, rd->id
, rd->name
, rrd_algorithm_name(rd->algorithm)
@@ -99,11 +107,13 @@ static inline void send_chart_definition(RRDSET *st) {
);
rd->exposed = 1;
}
+
+ st->upstream_resync_time = st->last_collected_time.tv_sec + (remote_clock_resync_iterations * st->update_every);
}
// sends the current chart dimensions
static inline void send_chart_metrics(RRDSET *st) {
- buffer_sprintf(st->rrdhost->rrdpush_buffer, "BEGIN %s %llu\n", st->id, (st->counter_done > remote_clock_resync_iterations)?st->usec_since_last_update:0);
+ buffer_sprintf(st->rrdhost->rrdpush_buffer, "BEGIN %s %llu\n", st->id, (st->upstream_resync_time > st->last_collected_time.tv_sec)?st->usec_since_last_update:0);
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
@@ -117,7 +127,17 @@ static inline void send_chart_metrics(RRDSET *st) {
buffer_strcat(st->rrdhost->rrdpush_buffer, "END\n");
}
-void rrdpush_sender_thread_spawn(RRDHOST *host);
+static void rrdpush_sender_thread_spawn(RRDHOST *host);
+
+void rrdset_push_chart_definition(RRDSET *st) {
+ RRDHOST *host = st->rrdhost;
+
+ rrdset_rdlock(st);
+ rrdpush_lock(host);
+ send_chart_definition(st);
+ rrdpush_unlock(host);
+ rrdset_unlock(st);
+}
void rrdset_done_push(RRDSET *st) {
RRDHOST *host = st->rrdhost;
@@ -167,9 +187,7 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {
RRDSET *st;
rrdset_foreach_read(st, host) {
- // make it re-align the current time
- // on the remote host
- st->counter_done = 0;
+ st->upstream_resync_time = 0;
rrdset_rdlock(st);
@@ -219,8 +237,6 @@ static void rrdpush_sender_thread_cleanup_locked_all(RRDHOST *host) {
host->rrdpush_buffer = NULL;
host->rrdpush_spawn = 0;
-
- rrdhost_flag_set(host, RRDHOST_ORPHAN);
}
void rrdpush_sender_thread_stop(RRDHOST *host) {
@@ -274,6 +290,7 @@ void *rrdpush_sender_thread(void *ptr) {
.tv_usec = 0
};
+ time_t last_sent_t = 0;
struct pollfd fds[2], *ifd, *ofd;
nfds_t fdmax;
@@ -281,8 +298,16 @@ void *rrdpush_sender_thread(void *ptr) {
ofd = &fds[1];
for(; host->rrdpush_enabled && !netdata_exit ;) {
+ debug(D_STREAM, "STREAM: Checking if we need to timeout the connection...");
+ if(host->rrdpush_socket != -1 && now_monotonic_sec() - last_sent_t > timeout) {
+ error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, timeout, sent_connection);
+ close(host->rrdpush_socket);
+ host->rrdpush_socket = -1;
+ }
if(unlikely(host->rrdpush_socket == -1)) {
+ debug(D_STREAM, "STREAM: Attempting to connect...");
+
// stop appending data into rrdpush_buffer
// they will be lost, so there is no point to do it
host->rrdpush_connected = 0;
@@ -298,16 +323,19 @@ void *rrdpush_sender_thread(void *ptr) {
info("STREAM %s [send to %s]: initializing communication...", host->hostname, connected_to);
- char http[1000 + 1];
- snprintfz(http, 1000,
- "STREAM key=%s&hostname=%s&machine_guid=%s&os=%s&update_every=%d HTTP/1.1\r\n"
+ #define HTTP_HEADER_SIZE 8192
+ char http[HTTP_HEADER_SIZE + 1];
+ snprintfz(http, HTTP_HEADER_SIZE,
+ "STREAM key=%s&hostname=%s&registry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&tags=%s HTTP/1.1\r\n"
"User-Agent: netdata-push-service/%s\r\n"
"Accept: */*\r\n\r\n"
, host->rrdpush_api_key
, host->hostname
+ , host->registry_hostname
, host->machine_guid
- , host->os
, default_rrd_update_every
+ , host->os
+ , (host->tags)?host->tags:""
, program_version
);
@@ -321,7 +349,7 @@ void *rrdpush_sender_thread(void *ptr) {
info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, connected_to);
- if(recv_timeout(host->rrdpush_socket, http, 1000, 0, timeout) == -1) {
+ if(recv_timeout(host->rrdpush_socket, http, HTTP_HEADER_SIZE, 0, timeout) == -1) {
close(host->rrdpush_socket);
host->rrdpush_socket = -1;
error("STREAM %s [send to %s]: failed to initialize communication", host->hostname, connected_to);
@@ -338,15 +366,21 @@ void *rrdpush_sender_thread(void *ptr) {
}
info("STREAM %s [send to %s]: established communication - sending metrics...", host->hostname, connected_to);
+ last_sent_t = now_monotonic_sec();
- if(fcntl(host->rrdpush_socket, F_SETFL, O_NONBLOCK) < 0)
+ if(sock_setnonblock(host->rrdpush_socket) < 0)
error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, connected_to);
+ if(sock_enlarge_out(host->rrdpush_socket) < 0)
+ error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, connected_to);
+
rrdpush_sender_thread_data_flush(host);
sent_connection = 0;
// allow appending data into rrdpush_buffer
host->rrdpush_connected = 1;
+
+ debug(D_STREAM, "Connected...");
}
ifd->fd = host->rrdpush_pipe[PIPE_READ];
@@ -356,82 +390,113 @@ void *rrdpush_sender_thread(void *ptr) {
ofd->fd = host->rrdpush_socket;
ofd->revents = 0;
if(begin < buffer_strlen(host->rrdpush_buffer)) {
+ debug(D_STREAM, "STREAM: Requesting data output on streaming socket...");
ofd->events = POLLOUT;
fdmax = 2;
}
else {
+ debug(D_STREAM, "STREAM: Not requesting data output on streaming socket (nothing to send now)...");
ofd->events = 0;
fdmax = 1;
}
+ debug(D_STREAM, "STREAM: Waiting for poll() events (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer));
if(netdata_exit) break;
- int retval = poll(fds, fdmax, timeout * 1000);
+ int retval = poll(fds, fdmax, 1000);
if(netdata_exit) break;
if(unlikely(retval == -1)) {
- if(errno == EAGAIN || errno == EINTR)
+ debug(D_STREAM, "STREAM: poll() failed (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer));
+
+ if(errno == EAGAIN || errno == EINTR) {
+ debug(D_STREAM, "STREAM: poll() failed with EAGAIN or EINTR...");
continue;
+ }
error("STREAM %s [send to %s]: failed to poll().", host->hostname, connected_to);
close(host->rrdpush_socket);
host->rrdpush_socket = -1;
break;
}
- else if(unlikely(!retval)) {
- // timeout
- continue;
- }
-
- if(ifd->revents & POLLIN) {
- char buffer[1000 + 1];
- if(read(host->rrdpush_pipe[PIPE_READ], buffer, 1000) == -1)
- error("STREAM %s [send to %s]: cannot read from internal pipe.", host->hostname, connected_to);
- }
-
- if(ofd->revents & POLLOUT && begin < buffer_strlen(host->rrdpush_buffer)) {
+ else if(likely(retval)) {
+ if (ifd->revents & POLLIN) {
+ debug(D_STREAM, "STREAM: Data added to send buffer (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer));
- // BEGIN RRDPUSH LOCKED SESSION
-
- // during this session, data collectors
- // will not be able to append data to our buffer
- // but the socket is in non-blocking mode
- // so, we will not block at send()
-
- if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0)
- error("STREAM %s [send]: cannot set pthread cancel state to DISABLE.", host->hostname);
-
- rrdpush_lock(host);
+ char buffer[1000 + 1];
+ if (read(host->rrdpush_pipe[PIPE_READ], buffer, 1000) == -1)
+ error("STREAM %s [send to %s]: cannot read from internal pipe.", host->hostname, connected_to);
+ }
- ssize_t ret = send(host->rrdpush_socket, &host->rrdpush_buffer->buffer[begin], buffer_strlen(host->rrdpush_buffer) - begin, MSG_DONTWAIT);
- if(ret == -1) {
- if(errno != EAGAIN && errno != EINTR) {
- error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection);
+ if (ofd->revents & POLLOUT && begin < buffer_strlen(host->rrdpush_buffer)) {
+ debug(D_STREAM, "STREAM: Sending data (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer));
+
+ // BEGIN RRDPUSH LOCKED SESSION
+
+ // during this session, data collectors
+ // will not be able to append data to our buffer
+ // but the socket is in non-blocking mode
+ // so, we will not block at send()
+
+ if (pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0)
+ error("STREAM %s [send]: cannot set pthread cancel state to DISABLE.", host->hostname);
+
+ debug(D_STREAM, "STREAM: Getting exclusive lock on host...");
+ rrdpush_lock(host);
+
+ debug(D_STREAM, "STREAM: Sending data, starting from %zu, size %zu...", begin, buffer_strlen(host->rrdpush_buffer));
+ ssize_t ret = send(host->rrdpush_socket, &host->rrdpush_buffer->buffer[begin], buffer_strlen(host->rrdpush_buffer) - begin, MSG_DONTWAIT);
+ if (unlikely(ret == -1)) {
+ if (errno != EAGAIN && errno != EINTR && errno != EWOULDBLOCK) {
+ debug(D_STREAM, "STREAM: Send failed - closing socket...");
+ error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection);
+ close(host->rrdpush_socket);
+ host->rrdpush_socket = -1;
+ }
+ else {
+ debug(D_STREAM, "STREAM: Send failed - will retry...");
+ }
+ }
+ else if(likely(ret > 0)) {
+ sent_connection += ret;
+ sent_bytes += ret;
+ begin += ret;
+
+ if (begin == buffer_strlen(host->rrdpush_buffer)) {
+ // we send it all
+
+ debug(D_STREAM, "STREAM: Sent %zd bytes (the whole buffer)...", ret);
+ buffer_flush(host->rrdpush_buffer);
+ begin = 0;
+ }
+ else {
+ debug(D_STREAM, "STREAM: Sent %zd bytes (part of the data buffer)...", ret);
+ }
+
+ last_sent_t = now_monotonic_sec();
+ }
+ else {
+ debug(D_STREAM, "STREAM: send() returned %zd - closing the socket...", ret);
+ error("STREAM %s [send to %s]: failed to send metrics (send() returned %zd) - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, ret, sent_connection);
close(host->rrdpush_socket);
host->rrdpush_socket = -1;
}
- }
- else {
- sent_connection += ret;
- sent_bytes += ret;
- begin += ret;
- if(begin == buffer_strlen(host->rrdpush_buffer)) {
- // we send it all
-
- buffer_flush(host->rrdpush_buffer);
- begin = 0;
- }
- }
- rrdpush_unlock(host);
+ debug(D_STREAM, "STREAM: Releasing exclusive lock on host...");
+ rrdpush_unlock(host);
- if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
- error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname);
+ if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
+ error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname);
- // END RRDPUSH LOCKED SESSION
+ // END RRDPUSH LOCKED SESSION
+ }
+ }
+ else {
+ debug(D_STREAM, "STREAM: poll() timed out.");
}
// protection from overflow
- if(host->rrdpush_buffer->len > max_size) {
+ if(buffer_strlen(host->rrdpush_buffer) > max_size) {
+ debug(D_STREAM, "STREAM: Buffer is too big (%zu bytes), bigger than the max (%zu) - flushing it...", buffer_strlen(host->rrdpush_buffer), max_size);
errno = 0;
error("STREAM %s [send to %s]: too many data pending - buffer is %zu bytes long, %zu unsent - we have sent %zu bytes in total, %zu on this connection. Closing connection to flush the data.", host->hostname, connected_to, host->rrdpush_buffer->len, host->rrdpush_buffer->len - begin, sent_bytes, sent_connection);
if(host->rrdpush_socket != -1) {
@@ -464,7 +529,7 @@ cleanup:
// ----------------------------------------------------------------------------
// rrdpush receiver thread
-int rrdpush_receive(int fd, const char *key, const char *hostname, const char *machine_guid, const char *os, int update_every, char *client_ip, char *client_port) {
+static int rrdpush_receive(int fd, const char *key, const char *hostname, const char *registry_hostname, const char *machine_guid, const char *os, const char *tags, int update_every, char *client_ip, char *client_port) {
RRDHOST *host;
int history = default_rrd_history_entries;
RRD_MEMORY_MODE mode = default_rrd_memory_mode;
@@ -499,13 +564,18 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m
rrdpush_api_key = appconfig_get(&stream_config, key, "default proxy api key", rrdpush_api_key);
rrdpush_api_key = appconfig_get(&stream_config, machine_guid, "proxy api key", rrdpush_api_key);
+ tags = appconfig_set_default(&stream_config, machine_guid, "host tags", (tags)?tags:"");
+ if(tags && !*tags) tags = NULL;
+
if(!strcmp(machine_guid, "localhost"))
host = localhost;
else
host = rrdhost_find_or_create(
hostname
+ , registry_hostname
, machine_guid
, os
+ , tags
, update_every
, history
, mode
@@ -522,7 +592,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m
}
#ifdef NETDATA_INTERNAL_CHECKS
- info("STREAM %s [receive from [%s]:%s]: client willing to stream metrics for host '%s' with machine_guid '%s': update every = %d, history = %ld, memory mode = %s, health %s"
+ info("STREAM %s [receive from [%s]:%s]: client willing to stream metrics for host '%s' with machine_guid '%s': update every = %d, history = %ld, memory mode = %s, health %s, tags '%s'"
, hostname
, client_ip
, client_port
@@ -532,6 +602,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m
, host->rrd_history_entries
, rrd_memory_mode_name(host->rrd_memory_mode)
, (health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto")
+ , host->tags
);
#endif // NETDATA_INTERNAL_CHECKS
@@ -560,7 +631,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m
}
// remove the non-blocking flag from the socket
- if(fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) & ~O_NONBLOCK) == -1)
+ if(sock_delnonblock(fd) < 0)
error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", host->hostname, client_ip, client_port, fd);
// convert the socket to a FILE *
@@ -572,7 +643,11 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m
}
rrdhost_wrlock(host);
+ if(host->connected_senders > 0)
+ info("STREAM %s [receive from [%s]:%s]: multiple streaming connections for the same host detected. If multiple netdata are pushing metrics for the same charts, at the same time, the result is unexpected.", host->hostname, client_ip, client_port);
+
host->connected_senders++;
+ rrdhost_flag_clear(host, RRDHOST_ORPHAN);
if(health_enabled != CONFIG_BOOLEAN_NO)
host->health_delay_up_to = now_realtime_sec() + alarms_delay;
rrdhost_unlock(host);
@@ -586,6 +661,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m
host->senders_disconnected_time = now_realtime_sec();
host->connected_senders--;
if(!host->connected_senders) {
+ rrdhost_flag_set(host, RRDHOST_ORPHAN);
if(health_enabled == CONFIG_BOOLEAN_AUTO)
host->health_enabled = 0;
}
@@ -603,14 +679,16 @@ struct rrdpush_thread {
int fd;
char *key;
char *hostname;
+ char *registry_hostname;
char *machine_guid;
char *os;
+ char *tags;
char *client_ip;
char *client_port;
int update_every;
};
-void *rrdpush_receiver_thread(void *ptr) {
+static void *rrdpush_receiver_thread(void *ptr) {
struct rrdpush_thread *rpt = (struct rrdpush_thread *)ptr;
if (pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
@@ -621,13 +699,15 @@ void *rrdpush_receiver_thread(void *ptr) {
info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
- rrdpush_receive(rpt->fd, rpt->key, rpt->hostname, rpt->machine_guid, rpt->os, rpt->update_every, rpt->client_ip, rpt->client_port);
+ rrdpush_receive(rpt->fd, rpt->key, rpt->hostname, rpt->registry_hostname, rpt->machine_guid, rpt->os, rpt->tags, rpt->update_every, rpt->client_ip, rpt->client_port);
info("STREAM %s [receive from [%s]:%s]: receive thread ended (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
freez(rpt->key);
freez(rpt->hostname);
+ freez(rpt->registry_hostname);
freez(rpt->machine_guid);
freez(rpt->os);
+ freez(rpt->tags);
freez(rpt->client_ip);
freez(rpt->client_port);
freez(rpt);
@@ -636,7 +716,7 @@ void *rrdpush_receiver_thread(void *ptr) {
return NULL;
}
-void rrdpush_sender_thread_spawn(RRDHOST *host) {
+static void rrdpush_sender_thread_spawn(RRDHOST *host) {
rrdhost_wrlock(host);
if(!host->rrdpush_spawn) {
@@ -646,7 +726,6 @@ void rrdpush_sender_thread_spawn(RRDHOST *host) {
else if(pthread_detach(host->rrdpush_thread))
error("STREAM %s [send]: cannot request detach newly created thread.", host->hostname);
- rrdhost_flag_clear(host, RRDHOST_ORPHAN);
host->rrdpush_spawn = 1;
}
@@ -658,7 +737,7 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
info("STREAM [receive from [%s]:%s]: new client connection.", w->client_ip, w->client_port);
- char *key = NULL, *hostname = NULL, *machine_guid = NULL, *os = "unknown";
+ char *key = NULL, *hostname = NULL, *registry_hostname = NULL, *machine_guid = NULL, *os = "unknown", *tags = NULL;
int update_every = default_rrd_update_every;
char buf[GUID_LEN + 1];
@@ -674,12 +753,18 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
key = value;
else if(!strcmp(name, "hostname"))
hostname = value;
+ else if(!strcmp(name, "registry_hostname"))
+ registry_hostname = value;
else if(!strcmp(name, "machine_guid"))
machine_guid = value;
else if(!strcmp(name, "update_every"))
update_every = (int)strtoul(value, NULL, 0);
else if(!strcmp(name, "os"))
os = value;
+ else if(!strcmp(name, "tags"))
+ tags = value;
+ else
+ info("STREAM [receive from [%s]:%s]: request has parameter '%s' = '%s', which is not used.", w->client_ip, w->client_port, key, value);
}
if(!key || !*key) {
@@ -704,21 +789,21 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
}
if(regenerate_guid(key, buf) == -1) {
- error("STREAM [receive from [%s]:%s]: API key '%s' is not valid GUID. Forbidding access.", w->client_ip, w->client_port, key);
+ error("STREAM [receive from [%s]:%s]: API key '%s' is not valid GUID (use the command uuidgen to generate one). Forbidding access.", w->client_ip, w->client_port, key);
buffer_flush(w->response.data);
buffer_sprintf(w->response.data, "Your API key is invalid.");
return 401;
}
if(regenerate_guid(machine_guid, buf) == -1) {
- error("STREAM [receive from [%s]:%s]: machine GUID '%s' is not GUID. Forbidding access.", w->client_ip, w->client_port, key);
+ error("STREAM [receive from [%s]:%s]: machine GUID '%s' is not GUID. Forbidding access.", w->client_ip, w->client_port, machine_guid);
buffer_flush(w->response.data);
buffer_sprintf(w->response.data, "Your machine GUID is invalid.");
return 404;
}
if(!appconfig_get_boolean(&stream_config, key, "enabled", 0)) {
- error("STREAM [receive from [%s]:%s]: API key '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, machine_guid);
+ error("STREAM [receive from [%s]:%s]: API key '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, key);
buffer_flush(w->response.data);
buffer_sprintf(w->response.data, "Your API key is not permitted access.");
return 401;
@@ -732,14 +817,16 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
}
struct rrdpush_thread *rpt = mallocz(sizeof(struct rrdpush_thread));
- rpt->fd = w->ifd;
- rpt->key = strdupz(key);
- rpt->hostname = strdupz(hostname);
- rpt->machine_guid = strdupz(machine_guid);
- rpt->os = strdupz(os);
- rpt->client_ip = strdupz(w->client_ip);
- rpt->client_port = strdupz(w->client_port);
- rpt->update_every = update_every;
+ rpt->fd = w->ifd;
+ rpt->key = strdupz(key);
+ rpt->hostname = strdupz(hostname);
+ rpt->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
+ rpt->machine_guid = strdupz(machine_guid);
+ rpt->os = strdupz(os);
+ rpt->tags = (tags)?strdupz(tags):NULL;
+ rpt->client_ip = strdupz(w->client_ip);
+ rpt->client_port = strdupz(w->client_port);
+ rpt->update_every = update_every;
pthread_t thread;
debug(D_SYSTEM, "STREAM [receive from [%s]:%s]: starting receiving thread.", w->client_ip, w->client_port);
diff --git a/src/rrdpush.h b/src/rrdpush.h
index dddbe758b..c3c7f4a54 100644
--- a/src/rrdpush.h
+++ b/src/rrdpush.h
@@ -4,9 +4,11 @@
extern int default_rrdpush_enabled;
extern char *default_rrdpush_destination;
extern char *default_rrdpush_api_key;
+extern unsigned int remote_clock_resync_iterations;
extern int rrdpush_init();
extern void rrdset_done_push(RRDSET *st);
+extern void rrdset_push_chart_definition(RRDSET *st);
extern void *rrdpush_sender_thread(void *ptr);
extern int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url);
diff --git a/src/rrdset.c b/src/rrdset.c
index c847b9690..caa427ff6 100644
--- a/src/rrdset.c
+++ b/src/rrdset.c
@@ -168,6 +168,58 @@ void rrdset_set_name(RRDSET *st, const char *name) {
error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name);
}
+inline void rrdset_is_obsolete(RRDSET *st) {
+ if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
+ rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE);
+ rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
+
+ // the chart will not get more updates (data collection)
+ // so, we have to push its definition now
+ if(unlikely(st->rrdhost->rrdpush_enabled))
+ rrdset_push_chart_definition(st);
+ }
+}
+
+inline void rrdset_isnot_obsolete(RRDSET *st) {
+ if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
+ rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+ rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
+
+ // the chart will be pushed upstream automatically
+ // due to data collection
+ }
+}
+
+inline void rrdset_update_heterogeneous_flag(RRDSET *st) {
+ RRDDIM *rd;
+
+ rrdset_flag_clear(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
+
+ RRD_ALGORITHM algorithm = st->dimensions->algorithm;
+ collected_number multiplier = abs(st->dimensions->multiplier);
+ collected_number divisor = abs(st->dimensions->divisor);
+
+ rrddim_foreach_read(rd, st) {
+ if(algorithm != rd->algorithm || multiplier != abs(rd->multiplier) || divisor != abs(rd->divisor)) {
+ if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").",
+ rd->name,
+ st->name,
+ st->rrdhost->hostname,
+ rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm),
+ rd->multiplier, multiplier,
+ rd->divisor, divisor
+ );
+ #endif
+ rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS);
+ }
+ return;
+ }
+ }
+
+ rrdset_flag_clear(st, RRDSET_FLAG_HETEROGENEOUS);
+}
// ----------------------------------------------------------------------------
// RRDSET - reset a chart
@@ -188,7 +240,7 @@ void rrdset_reset(RRDSET *st) {
rd->last_collected_time.tv_sec = 0;
rd->last_collected_time.tv_usec = 0;
rd->collections_counter = 0;
- memset(rd->values, 0, rd->entries * sizeof(storage_number));
+ // memset(rd->values, 0, rd->entries * sizeof(storage_number));
}
}
@@ -199,7 +251,7 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
if(unlikely(entries < 5)) entries = 5;
if(unlikely(entries > RRD_HISTORY_ENTRIES_MAX)) entries = RRD_HISTORY_ENTRIES_MAX;
- if(unlikely(mode == RRD_MEMORY_MODE_NONE || mode == RRD_MEMORY_MODE_RAM))
+ if(unlikely(mode == RRD_MEMORY_MODE_NONE || mode == RRD_MEMORY_MODE_ALLOC))
return entries;
long page = (size_t)sysconf(_SC_PAGESIZE);
@@ -215,14 +267,18 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
return entries;
}
-static inline void last_collected_time_align(struct timeval *tv, int update_every) {
- tv->tv_sec -= tv->tv_sec % update_every;
- tv->tv_usec = 500000;
+static inline void last_collected_time_align(RRDSET *st) {
+ st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every;
+
+ if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)))
+ st->last_collected_time.tv_usec = 0;
+ else
+ st->last_collected_time.tv_usec = 500000;
}
-static inline void last_updated_time_align(struct timeval *tv, int update_every) {
- tv->tv_sec -= tv->tv_sec % update_every;
- tv->tv_usec = 0;
+static inline void last_updated_time_align(RRDSET *st) {
+ st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every;
+ st->last_updated.tv_usec = 0;
}
// ----------------------------------------------------------------------------
@@ -279,30 +335,36 @@ void rrdset_free(RRDSET *st) {
// free directly allocated members
freez(st->config_section);
- if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
- debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
- munmap(st, st->memsize);
+ switch(st->rrd_memory_mode) {
+ case RRD_MEMORY_MODE_SAVE:
+ case RRD_MEMORY_MODE_MAP:
+ case RRD_MEMORY_MODE_RAM:
+ debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
+ munmap(st, st->memsize);
+ break;
+
+ case RRD_MEMORY_MODE_ALLOC:
+ case RRD_MEMORY_MODE_NONE:
+ freez(st);
+ break;
}
- else
- freez(st);
}
void rrdset_save(RRDSET *st) {
- RRDDIM *rd;
-
rrdset_check_rdlock(st);
// info("Saving chart '%s' ('%s')", st->id, st->name);
if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename);
- savememory(st->cache_filename, st, st->memsize);
+ memory_file_save(st->cache_filename, st, st->memsize);
}
+ RRDDIM *rd;
rrddim_foreach_read(rd, st) {
if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) {
debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename);
- savememory(rd->cache_filename, rd, rd->memsize);
+ memory_file_save(rd->cache_filename, rd, rd->memsize);
}
}
}
@@ -312,19 +374,23 @@ void rrdset_delete(RRDSET *st) {
rrdset_check_rdlock(st);
- // info("Deleting chart '%s' ('%s')", st->id, st->name);
+ info("Deleting chart '%s' ('%s') from disk...", st->id, st->name);
- if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
- debug(D_RRD_STATS, "Deleting stats '%s' to '%s'.", st->name, st->cache_filename);
- unlink(st->cache_filename);
+ if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
+ info("Deleting chart header file '%s'.", st->cache_filename);
+ if(unlikely(unlink(st->cache_filename) == -1))
+ error("Cannot delete chart header file '%s'", st->cache_filename);
}
rrddim_foreach_read(rd, st) {
- if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) {
- debug(D_RRD_STATS, "Deleting dimension '%s' to '%s'.", rd->name, rd->cache_filename);
- unlink(rd->cache_filename);
+ if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
+ info("Deleting dimension file '%s'.", rd->cache_filename);
+ if(unlikely(unlink(rd->cache_filename) == -1))
+ error("Cannot delete dimension file '%s'", rd->cache_filename);
}
}
+
+ recursively_delete_dir(st->cache_dir, "left-over chart");
}
// ----------------------------------------------------------------------------
@@ -333,7 +399,7 @@ void rrdset_delete(RRDSET *st) {
static inline RRDSET *rrdset_find_on_create(RRDHOST *host, const char *fullid) {
RRDSET *st = rrdset_find(host, fullid);
if(unlikely(st)) {
- rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+ rrdset_isnot_obsolete(st);
debug(D_RRD_CALLS, "RRDSET '%s', already exists.", fullid);
return st;
}
@@ -341,7 +407,7 @@ static inline RRDSET *rrdset_find_on_create(RRDHOST *host, const char *fullid) {
return NULL;
}
-RRDSET *rrdset_create(
+RRDSET *rrdset_create_custom(
RRDHOST *host
, const char *type
, const char *id
@@ -353,6 +419,8 @@ RRDSET *rrdset_create(
, long priority
, int update_every
, RRDSET_TYPE chart_type
+ , RRD_MEMORY_MODE memory_mode
+ , long history_entries
) {
if(!type || !type[0]) {
fatal("Cannot create rrd stats without a type.");
@@ -395,11 +463,11 @@ RRDSET *rrdset_create(
// ------------------------------------------------------------------------
// get the options from the config, we need to create it
- long rentries = config_get_number(config_section, "history", host->rrd_history_entries);
- long entries = align_entries_to_pagesize(host->rrd_memory_mode, rentries);
+ long rentries = config_get_number(config_section, "history", history_entries);
+ long entries = align_entries_to_pagesize(memory_mode, rentries);
if(entries != rentries) entries = config_set_number(config_section, "history", entries);
- if(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE && entries != rentries)
+ if(memory_mode == RRD_MEMORY_MODE_NONE && entries != rentries)
entries = config_set_number(config_section, "history", 10);
int enabled = config_get_boolean(config_section, "enabled", 1);
@@ -416,8 +484,14 @@ RRDSET *rrdset_create(
debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
- if(host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
- st = (RRDSET *) mymmap(fullfilename, size, ((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0);
+ if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_RAM) {
+ st = (RRDSET *) mymmap(
+ (memory_mode == RRD_MEMORY_MODE_RAM)?NULL:fullfilename
+ , size
+ , ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE)
+ , 0
+ );
+
if(st) {
memset(&st->avl, 0, sizeof(avl));
memset(&st->avlname, 0, sizeof(avl));
@@ -426,64 +500,68 @@ RRDSET *rrdset_create(
memset(&st->rrdset_rwlock, 0, sizeof(netdata_rwlock_t));
st->name = NULL;
+ st->config_section = NULL;
st->type = NULL;
st->family = NULL;
- st->context = NULL;
st->title = NULL;
st->units = NULL;
+ st->context = NULL;
+ st->cache_dir = NULL;
st->dimensions = NULL;
+ st->rrdfamily = NULL;
+ st->rrdhost = NULL;
st->next = NULL;
st->variables = NULL;
st->alarms = NULL;
st->flags = 0x00000000;
- if(strcmp(st->magic, RRDSET_MAGIC) != 0) {
- errno = 0;
- info("Initializing file %s.", fullfilename);
- memset(st, 0, size);
- }
- else if(strcmp(st->id, fullid) != 0) {
- errno = 0;
- error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
- // munmap(st, size);
- // st = NULL;
+ if(memory_mode == RRD_MEMORY_MODE_RAM) {
memset(st, 0, size);
}
- else if(st->memsize != size || st->entries != entries) {
- errno = 0;
- error("File %s does not have the desired size. Clearing it.", fullfilename);
- memset(st, 0, size);
- }
- else if(st->update_every != update_every) {
- errno = 0;
- error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
- memset(st, 0, size);
- }
- else if((now - st->last_updated.tv_sec) > update_every * entries) {
- errno = 0;
- error("File %s is too old. Clearing it.", fullfilename);
- memset(st, 0, size);
- }
- else if(st->last_updated.tv_sec > now + update_every) {
- errno = 0;
- error("File %s refers to the future. Clearing it.", fullfilename);
- memset(st, 0, size);
- }
-
- // make sure the database is aligned
- if(st->last_updated.tv_sec)
- last_updated_time_align(&st->last_updated, update_every);
+ else {
+ if(strcmp(st->magic, RRDSET_MAGIC) != 0) {
+ info("Initializing file %s.", fullfilename);
+ memset(st, 0, size);
+ }
+ else if(strcmp(st->id, fullid) != 0) {
+ error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
+ // munmap(st, size);
+ // st = NULL;
+ memset(st, 0, size);
+ }
+ else if(st->memsize != size || st->entries != entries) {
+ error("File %s does not have the desired size. Clearing it.", fullfilename);
+ memset(st, 0, size);
+ }
+ else if(st->update_every != update_every) {
+ error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
+ memset(st, 0, size);
+ }
+ else if((now - st->last_updated.tv_sec) > update_every * entries) {
+ error("File %s is too old. Clearing it.", fullfilename);
+ memset(st, 0, size);
+ }
+ else if(st->last_updated.tv_sec > now + update_every) {
+ error("File %s refers to the future. Clearing it.", fullfilename);
+ memset(st, 0, size);
+ }
+ // make sure the database is aligned
+ if(st->last_updated.tv_sec) {
+ st->update_every = update_every;
+ last_updated_time_align(st);
+ }
+ }
// make sure we have the right memory mode
// even if we cleared the memory
- st->rrd_memory_mode = host->rrd_memory_mode;
+ st->rrd_memory_mode = memory_mode;
}
}
if(unlikely(!st)) {
st = callocz(1, size);
- st->rrd_memory_mode = (host->rrd_memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_RAM;
+ st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
st->config_section = strdup(config_section);
@@ -519,6 +597,7 @@ RRDSET *rrdset_create(
rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
rrdset_flag_clear(st, RRDSET_FLAG_DEBUG);
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+ rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
// if(!strcmp(st->id, "disk_util.dm-0")) {
// st->debug = 1;
@@ -536,6 +615,9 @@ RRDSET *rrdset_create(
st->gap_when_lost_iterations_above = (int) (
config_get_number(st->config_section, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS) + 2);
+ st->last_accessed_time = 0;
+ st->upstream_resync_time = 0;
+
avl_init_lock(&st->dimensions_index, rrddim_compare);
avl_init_lock(&st->variables_root_index, rrdvar_compare);
@@ -544,13 +626,8 @@ RRDSET *rrdset_create(
if(name && *name) rrdset_set_name(st, name);
else rrdset_set_name(st, id);
- {
- char varvalue[CONFIG_MAX_VALUE + 1];
- char varvalue2[CONFIG_MAX_VALUE + 1];
- snprintfz(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
- json_escape_string(varvalue2, varvalue, sizeof(varvalue2));
- st->title = config_get(st->config_section, "title", varvalue2);
- }
+ st->title = config_get(st->config_section, "title", title);
+ json_fix_string(st->title);
st->rrdfamily = rrdfamily_create(host, st->family);
@@ -571,7 +648,7 @@ RRDSET *rrdset_create(
rrdsetcalc_link_matching(st);
rrdcalctemplate_link_matching(st);
- rrdhost_cleanup_obsolete(host);
+ rrdhost_cleanup_obsolete_charts(host);
rrdhost_unlock(host);
@@ -583,16 +660,10 @@ RRDSET *rrdset_create(
// RRDSET - data collection iteration control
inline void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds) {
-
- if(unlikely(!st->last_collected_time.tv_sec)) {
- // the first entry
- microseconds = st->update_every * USEC_PER_SEC;
- }
- else if(unlikely(!microseconds)) {
- // no dt given by the plugin
- struct timeval now;
- now_realtime_timeval(&now);
- microseconds = dt_usec(&now, &st->last_collected_time);
+ if(unlikely(!st->last_collected_time.tv_sec || !microseconds || (st->counter % remote_clock_resync_iterations) == 0)) {
+ // call the full next_usec() function
+ rrdset_next_usec(st, microseconds);
+ return;
}
st->usec_since_last_update = microseconds;
@@ -612,52 +683,34 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
}
else {
// microseconds has the time since the last collection
-//#ifdef NETDATA_INTERNAL_CHECKS
-// usec_t now_usec = timeval_usec(&now);
-// usec_t last_usec = timeval_usec(&st->last_collected_time);
-//#endif
susec_t since_last_usec = dt_usec_signed(&now, &st->last_collected_time);
if(unlikely(since_last_usec < 0)) {
// oops! the database is in the future
- error("Database for chart '%s' on host '%s' is %lld microseconds in the future. Adjusting it to current time.", st->id, st->rrdhost->hostname, -since_last_usec);
+ info("RRD database for chart '%s' on host '%s' is %0.5Lf secs in the future. Adjusting it to current time.", st->id, st->rrdhost->hostname, (long double)-since_last_usec / USEC_PER_SEC);
st->last_collected_time.tv_sec = now.tv_sec - st->update_every;
st->last_collected_time.tv_usec = now.tv_usec;
- last_collected_time_align(&st->last_collected_time, st->update_every);
+ last_collected_time_align(st);
st->last_updated.tv_sec = now.tv_sec - st->update_every;
st->last_updated.tv_usec = now.tv_usec;
- last_updated_time_align(&st->last_updated, st->update_every);
+ last_updated_time_align(st);
microseconds = st->update_every * USEC_PER_SEC;
- since_last_usec = st->update_every * USEC_PER_SEC;
- }
-
- // verify the microseconds given is good
- if(unlikely(microseconds > (usec_t)since_last_usec)) {
- debug(D_RRD_CALLS, "dt %llu usec given is too big - it leads %llu usec to the future, for chart '%s' (%s).", microseconds, microseconds - (usec_t)since_last_usec, st->name, st->id);
-
-//#ifdef NETDATA_INTERNAL_CHECKS
-// if(unlikely(last_usec + microseconds > now_usec + 1000))
-// error("dt %llu usec given is too big - it leads %llu usec to the future, for chart '%s' (%s).", microseconds, microseconds - (usec_t)since_last_usec, st->name, st->id);
-//#endif
-
- microseconds = (usec_t)since_last_usec;
}
- else if(unlikely(microseconds < (usec_t)since_last_usec * 0.8)) {
- debug(D_RRD_CALLS, "dt %llu usec given is too small - expected %llu usec up to -20%%, for chart '%s' (%s).", microseconds, (usec_t)since_last_usec, st->name, st->id);
+ else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 10 * USEC_PER_SEC))) {
+ // oops! the database is too far behind
+ info("RRD database for chart '%s' on host '%s' is %0.5Lf secs in the past. Adjusting it to current time.", st->id, st->rrdhost->hostname, (long double)since_last_usec / USEC_PER_SEC);
-//#ifdef NETDATA_INTERNAL_CHECKS
-// error("dt %llu usec given is too small - expected %llu usec up to -20%%, for chart '%s' (%s).", microseconds, (usec_t)since_last_usec, st->name, st->id);
-//#endif
microseconds = (usec_t)since_last_usec;
}
}
- debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds);
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds);
+ rrdset_debug(st, "NEXT: %llu microseconds", microseconds);
+ #endif
st->usec_since_last_update = microseconds;
}
@@ -666,9 +719,17 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
// ----------------------------------------------------------------------------
// RRDSET - process the collected values for all dimensions of a chart
-static inline void rrdset_init_last_collected_time(RRDSET *st) {
+static inline usec_t rrdset_init_last_collected_time(RRDSET *st) {
now_realtime_timeval(&st->last_collected_time);
- last_collected_time_align(&st->last_collected_time, st->update_every);
+ last_collected_time_align(st);
+
+ usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "initialized last collected time to %0.3Lf", (long double)last_collect_ut / USEC_PER_SEC);
+ #endif
+
+ return last_collect_ut;
}
static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
@@ -676,17 +737,42 @@ static inline usec_t rrdset_update_last_collected_time(RRDSET *st) {
usec_t ut = last_collect_ut + st->usec_since_last_update;
st->last_collected_time.tv_sec = (time_t) (ut / USEC_PER_SEC);
st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC);
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "updated last collected time to %0.3Lf", (long double)last_collect_ut / USEC_PER_SEC);
+ #endif
+
return last_collect_ut;
}
-static inline void rrdset_init_last_updated_time(RRDSET *st) {
+static inline usec_t rrdset_init_last_updated_time(RRDSET *st) {
// copy the last collected time to last updated time
st->last_updated.tv_sec = st->last_collected_time.tv_sec;
st->last_updated.tv_usec = st->last_collected_time.tv_usec;
- last_updated_time_align(&st->last_updated, st->update_every);
+
+ if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
+ st->last_updated.tv_sec -= st->update_every;
+
+ last_updated_time_align(st);
+
+ usec_t last_updated_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "initialized last updated time to %0.3Lf", (long double)last_updated_ut / USEC_PER_SEC);
+ #endif
+
+ return last_updated_ut;
}
static inline void rrdset_done_push_exclusive(RRDSET *st) {
+// usec_t update_every_ut = st->update_every * USEC_PER_SEC; // st->update_every in microseconds
+//
+// if(unlikely(st->usec_since_last_update > update_every_ut * remote_clock_resync_iterations)) {
+// error("Chart '%s' was last collected %llu usec before. Resetting it.", st->id, st->usec_since_last_update);
+// rrdset_reset(st);
+// st->usec_since_last_update = update_every_ut;
+// }
+
if(unlikely(!st->last_collected_time.tv_sec)) {
// it is the first entry
// set the last_collected_time to now
@@ -705,6 +791,235 @@ static inline void rrdset_done_push_exclusive(RRDSET *st) {
rrdset_unlock(st);
}
+
+static inline size_t rrdset_done_interpolate(
+ RRDSET *st
+ , usec_t update_every_ut
+ , usec_t last_stored_ut
+ , usec_t next_store_ut
+ , usec_t last_collect_ut
+ , usec_t now_collect_ut
+ , char store_this_entry
+ , uint32_t storage_flags
+) {
+ RRDDIM *rd;
+
+ size_t stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done()
+
+ usec_t first_ut = last_stored_ut, last_ut = 0;
+ ssize_t iterations = (ssize_t)((now_collect_ut - last_stored_ut) / (update_every_ut));
+ if((now_collect_ut % (update_every_ut)) == 0) iterations++;
+
+ size_t counter = st->counter;
+ long current_entry = st->current_entry;
+
+ for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) {
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
+ rrdset_debug(st, "last_stored_ut = %0.3Lf (last updated time)", (long double)last_stored_ut/USEC_PER_SEC);
+ rrdset_debug(st, "next_store_ut = %0.3Lf (next interpolation point)", (long double)next_store_ut/USEC_PER_SEC);
+ #endif
+
+ last_ut = next_store_ut;
+
+ rrddim_foreach_read(rd, st) {
+ calculated_number new_value;
+
+ switch(rd->algorithm) {
+ case RRD_ALGORITHM_INCREMENTAL:
+ new_value = (calculated_number)
+ ( rd->calculated_value
+ * (calculated_number)(next_store_ut - last_collect_ut)
+ / (calculated_number)(now_collect_ut - last_collect_ut)
+ );
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: CALC2 INC "
+ CALCULATED_NUMBER_FORMAT " = "
+ CALCULATED_NUMBER_FORMAT
+ " * (%llu - %llu)"
+ " / (%llu - %llu)"
+ , rd->name
+ , new_value
+ , rd->calculated_value
+ , next_store_ut, last_collect_ut
+ , now_collect_ut, last_collect_ut
+ );
+ #endif
+
+ rd->calculated_value -= new_value;
+ new_value += rd->last_calculated_value;
+ rd->last_calculated_value = 0;
+ new_value /= (calculated_number)st->update_every;
+
+ if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING",
+ rd->name
+ , (calculated_number)(next_store_ut - last_stored_ut)
+ );
+ #endif
+
+ new_value = new_value * (calculated_number)(st->update_every * USEC_PER_SEC) / (calculated_number)(next_store_ut - last_stored_ut);
+ }
+ break;
+
+ case RRD_ALGORITHM_ABSOLUTE:
+ case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
+ case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
+ default:
+ if(iterations == 1) {
+ // this is the last iteration
+ // do not interpolate
+ // just show the calculated value
+
+ new_value = rd->calculated_value;
+ }
+ else {
+ // we have missed an update
+ // interpolate in the middle values
+
+ new_value = (calculated_number)
+ ( ( (rd->calculated_value - rd->last_calculated_value)
+ * (calculated_number)(next_store_ut - last_collect_ut)
+ / (calculated_number)(now_collect_ut - last_collect_ut)
+ )
+ + rd->last_calculated_value
+ );
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: CALC2 DEF "
+ CALCULATED_NUMBER_FORMAT " = ((("
+ "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
+ " * %llu"
+ " / %llu) + " CALCULATED_NUMBER_FORMAT
+ , rd->name
+ , new_value
+ , rd->calculated_value, rd->last_calculated_value
+ , (next_store_ut - first_ut)
+ , (now_collect_ut - first_ut), rd->last_calculated_value
+ );
+ #endif
+ }
+ break;
+ }
+
+ if(unlikely(!store_this_entry)) {
+ rd->values[current_entry] = SN_EMPTY_SLOT; //pack_storage_number(0, SN_NOT_EXISTS);
+ continue;
+ }
+
+ if(likely(rd->updated && rd->collections_counter > 1 && iterations < st->gap_when_lost_iterations_above)) {
+ rd->values[current_entry] = pack_storage_number(new_value, storage_flags );
+ rd->last_stored_value = new_value;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: STORE[%ld] "
+ CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
+ , rd->name
+ , current_entry
+ , unpack_storage_number(rd->values[current_entry]), new_value
+ );
+ #endif
+
+ }
+ else {
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING "
+ , rd->name
+ , current_entry
+ );
+ #endif
+
+ rd->values[current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS);
+ rd->last_stored_value = NAN;
+ }
+
+ stored_entries++;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
+ calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor;
+ calculated_number t2 = unpack_storage_number(rd->values[current_entry]);
+
+ calculated_number accuracy = accuracy_loss(t1, t2);
+ debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
+ , st->id, rd->name
+ , current_entry
+ , t2
+ , get_storage_number_flags(rd->values[current_entry])
+ , t1
+ , accuracy
+ , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
+ );
+
+ rd->collected_volume += t1;
+ rd->stored_volume += t2;
+
+ accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume);
+ debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s"
+ , st->id, rd->name
+ , current_entry
+ , rd->stored_volume
+ , rd->collected_volume
+ , accuracy
+ , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
+ );
+ }
+ #endif
+ }
+ // reset the storage flags for the next point, if any;
+ storage_flags = SN_EXISTS;
+
+ counter++;
+ current_entry = ((current_entry + 1) >= st->entries) ? 0 : current_entry + 1;
+ last_stored_ut = next_store_ut;
+ }
+
+ st->counter = counter;
+ st->current_entry = current_entry;
+
+ if(likely(last_ut)) {
+ st->last_updated.tv_sec = (time_t) (last_ut / USEC_PER_SEC);
+ st->last_updated.tv_usec = 0;
+ }
+
+ return stored_entries;
+}
+
+static inline void rrdset_done_fill_the_gap(RRDSET *st) {
+ usec_t update_every_ut = st->update_every * USEC_PER_SEC;
+ usec_t now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
+
+ long c = 0, entries = st->entries;
+ RRDDIM *rd;
+ rrddim_foreach_read(rd, st) {
+ usec_t next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
+ long current_entry = st->current_entry;
+
+ for(c = 0; c < entries && next_store_ut <= now_collect_ut ; next_store_ut += update_every_ut, c++) {
+ rd->values[current_entry] = SN_EMPTY_SLOT;
+ current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING (FILLED THE GAP)", rd->name, current_entry);
+ #endif
+ }
+ }
+
+ if(c > 0) {
+ c--;
+ st->last_updated.tv_sec += c * st->update_every;
+
+ st->current_entry += c;
+ if(st->current_entry >= st->entries)
+ st->current_entry -= st->entries;
+ }
+}
+
void rrdset_done(RRDSET *st) {
if(unlikely(netdata_exit)) return;
@@ -726,9 +1041,6 @@ void rrdset_done(RRDSET *st) {
store_this_entry = 1, // boolean: 1 = store this entry, 0 = don't store this entry
first_entry = 0; // boolean: 1 = this is the first entry seen for this chart, 0 = all other entries
- unsigned int
- stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done()
-
usec_t
last_collect_ut, // the timestamp in microseconds, of the last collected value
now_collect_ut, // the timestamp in microseconds, of this collected value (this is NOW)
@@ -742,42 +1054,33 @@ void rrdset_done(RRDSET *st) {
// a read lock is OK here
rrdset_rdlock(st);
-/*
- // enable the chart, if it was disabled
- if(unlikely(rrd_delete_unupdated_dimensions) && !st->enabled)
- st->enabled = 1;
-*/
-
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
error("Chart '%s' has the OBSOLETE flag set, but it is collected.", st->id);
- rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
+ rrdset_isnot_obsolete(st);
}
// check if the chart has a long time to be updated
if(unlikely(st->usec_since_last_update > st->entries * update_every_ut)) {
- info("%s: took too long to be updated (%0.3Lf secs). Resetting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
+ info("host '%s', chart %s: took too long to be updated (%0.3Lf secs). Resetting it.", st->rrdhost->hostname, st->name, (long double)st->usec_since_last_update / USEC_PER_SEC);
rrdset_reset(st);
st->usec_since_last_update = update_every_ut;
+ store_this_entry = 0;
first_entry = 1;
}
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "microseconds since last update: %llu", st->usec_since_last_update);
+ #endif
// set last_collected_time
if(unlikely(!st->last_collected_time.tv_sec)) {
// it is the first entry
// set the last_collected_time to now
- rrdset_init_last_collected_time(st);
-
- last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec - update_every_ut;
+ last_collect_ut = rrdset_init_last_collected_time(st) - update_every_ut;
// the first entry should not be stored
store_this_entry = 0;
first_entry = 1;
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s: has not set last_collected_time. Setting it now. Will not store the next entry.", st->name);
}
else {
// it is not the first entry
@@ -795,9 +1098,6 @@ void rrdset_done(RRDSET *st) {
// the first entry should not be stored
store_this_entry = 0;
first_entry = 1;
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s: initializing last_updated to last_collected_time - %llu microseconds. Will not store the next entry.", st->name, st->usec_since_last_update);
}
// check if we will re-write the entire data set
@@ -817,27 +1117,48 @@ void rrdset_done(RRDSET *st) {
// last_stored_ut = the last time we added a value to the storage
// now_collect_ut = the time the current value has been collected
// next_store_ut = the time of the next interpolation point
- last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec;
+ last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
- debug(D_RRD_STATS, "%s: last_collect_ut = %0.3Lf (last collection time)", st->name, (long double)last_collect_ut/1000000.0);
- debug(D_RRD_STATS, "%s: now_collect_ut = %0.3Lf (current collection time)", st->name, (long double)now_collect_ut/1000000.0);
- debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/1000000.0);
- debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/1000000.0);
- }
-
if(unlikely(!st->counter_done)) {
- store_this_entry = 0;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name);
+ // if we have not collected metrics this session (st->counter_done == 0)
+ // and we have collected metrics for this chart in the past (st->counter != 0)
+ // fill the gap (the chart has been just loaded from disk)
+ if(unlikely(st->counter)) {
+ rrdset_done_fill_the_gap(st);
+ last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec;
+ next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC;
+ }
+
+ if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))) {
+ store_this_entry = 1;
+ last_collect_ut = next_store_ut - update_every_ut;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "Fixed first entry.");
+ #endif
+ }
+ else {
+ store_this_entry = 0;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "Will not store the next entry.");
+ #endif
+ }
}
st->counter_done++;
if(unlikely(st->rrdhost->rrdpush_enabled))
rrdset_done_push(st);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "last_collect_ut = %0.3Lf (last collection time)", (long double)last_collect_ut/USEC_PER_SEC);
+ rrdset_debug(st, "now_collect_ut = %0.3Lf (current collection time)", (long double)now_collect_ut/USEC_PER_SEC);
+ rrdset_debug(st, "last_stored_ut = %0.3Lf (last updated time)", (long double)last_stored_ut/USEC_PER_SEC);
+ rrdset_debug(st, "next_store_ut = %0.3Lf (next interpolation point)", (long double)next_store_ut/USEC_PER_SEC);
+ #endif
+
// calculate totals and count the dimensions
int dimensions = 0;
st->collected_total = 0;
@@ -859,18 +1180,19 @@ void rrdset_done(RRDSET *st) {
continue;
}
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: START "
- " last_collected_value = " COLLECTED_NUMBER_FORMAT
- " collected_value = " COLLECTED_NUMBER_FORMAT
- " last_calculated_value = " CALCULATED_NUMBER_FORMAT
- " calculated_value = " CALCULATED_NUMBER_FORMAT
- , st->id, rd->name
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: START "
+ " last_collected_value = " COLLECTED_NUMBER_FORMAT
+ " collected_value = " COLLECTED_NUMBER_FORMAT
+ " last_calculated_value = " CALCULATED_NUMBER_FORMAT
+ " calculated_value = " CALCULATED_NUMBER_FORMAT
+ , rd->name
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
, rd->calculated_value
- );
+ );
+ #endif
switch(rd->algorithm) {
case RRD_ALGORITHM_ABSOLUTE:
@@ -878,18 +1200,20 @@ void rrdset_done(RRDSET *st) {
* (calculated_number)rd->multiplier
/ (calculated_number)rd->divisor;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN "
CALCULATED_NUMBER_FORMAT " = "
COLLECTED_NUMBER_FORMAT
" * " CALCULATED_NUMBER_FORMAT
" / " CALCULATED_NUMBER_FORMAT
- , st->id, rd->name
+ , rd->name
, rd->calculated_value
, rd->collected_value
, (calculated_number)rd->multiplier
, (calculated_number)rd->divisor
- );
+ );
+ #endif
+
break;
case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
@@ -903,16 +1227,18 @@ void rrdset_done(RRDSET *st) {
* (calculated_number)rd->collected_value
/ (calculated_number)st->collected_total;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: CALC PCENT-ROW "
CALCULATED_NUMBER_FORMAT " = 100"
- " * " COLLECTED_NUMBER_FORMAT
+ " * " COLLECTED_NUMBER_FORMAT
" / " COLLECTED_NUMBER_FORMAT
- , st->id, rd->name
+ , rd->name
, rd->calculated_value
, rd->collected_value
, st->collected_total
- );
+ );
+ #endif
+
break;
case RRD_ALGORITHM_INCREMENTAL:
@@ -940,19 +1266,21 @@ void rrdset_done(RRDSET *st) {
* (calculated_number)rd->multiplier
/ (calculated_number)rd->divisor;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: CALC INC PRE "
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: CALC INC PRE "
CALCULATED_NUMBER_FORMAT " = ("
COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
")"
" * " CALCULATED_NUMBER_FORMAT
" / " CALCULATED_NUMBER_FORMAT
- , st->id, rd->name
+ , rd->name
, rd->calculated_value
, rd->collected_value, rd->last_collected_value
, (calculated_number)rd->multiplier
, (calculated_number)rd->divisor
- );
+ );
+ #endif
+
break;
case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
@@ -967,7 +1295,8 @@ void rrdset_done(RRDSET *st) {
debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
, st->name, rd->name
, rd->last_collected_value
- , rd->collected_value);
+ , rd->collected_value
+ );
if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)))
storage_flags = SN_EXISTS_RESET;
@@ -985,16 +1314,18 @@ void rrdset_done(RRDSET *st) {
* (calculated_number)(rd->collected_value - rd->last_collected_value)
/ (calculated_number)(st->collected_total - st->last_collected_total);
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: CALC PCENT-DIFF "
CALCULATED_NUMBER_FORMAT " = 100"
- " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
- " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
- , st->id, rd->name
+ " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
+ " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
+ , rd->name
, rd->calculated_value
, rd->collected_value, rd->last_collected_value
, st->collected_total, st->last_collected_total
- );
+ );
+ #endif
+
break;
default:
@@ -1002,203 +1333,53 @@ void rrdset_done(RRDSET *st) {
// it gets noticed when we add new types
rd->calculated_value = 0;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: CALC "
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: CALC "
CALCULATED_NUMBER_FORMAT " = 0"
- , st->id, rd->name
+ , rd->name
, rd->calculated_value
- );
+ );
+ #endif
+
break;
}
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: PHASE2 "
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: PHASE2 "
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT
- , st->id, rd->name
+ , rd->name
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
, rd->calculated_value
- );
+ );
+ #endif
}
// at this point we have all the calculated values ready
// it is now time to interpolate values on a second boundary
+#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(now_collect_ut < next_store_ut)) {
// this is collected in the same interpolation point
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s: THIS IS IN THE SAME INTERPOLATION POINT", st->name);
-
-//#ifdef NETDATA_INTERNAL_CHECKS
-// info("%s is collected in the same interpolation point: short by %llu microseconds", st->name, next_store_ut - now_collect_ut);
-//#endif
- }
-
- usec_t first_ut = last_stored_ut;
- long long iterations = (now_collect_ut - last_stored_ut) / (update_every_ut);
- if((now_collect_ut % (update_every_ut)) == 0) iterations++;
-
- for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) {
-//#ifdef NETDATA_INTERNAL_CHECKS
-// if(iterations < 0) { error("%s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); }
-//#endif
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
- debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/1000000.0);
- debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/1000000.0);
- }
-
- st->last_updated.tv_sec = (time_t) (next_store_ut / USEC_PER_SEC);
- st->last_updated.tv_usec = 0;
-
- rrddim_foreach_read(rd, st) {
- calculated_number new_value;
-
- switch(rd->algorithm) {
- case RRD_ALGORITHM_INCREMENTAL:
- new_value = (calculated_number)
- ( rd->calculated_value
- * (calculated_number)(next_store_ut - last_collect_ut)
- / (calculated_number)(now_collect_ut - last_collect_ut)
- );
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: CALC2 INC "
- CALCULATED_NUMBER_FORMAT " = "
- CALCULATED_NUMBER_FORMAT
- " * %llu"
- " / %llu"
- , st->id, rd->name
- , new_value
- , rd->calculated_value
- , (next_store_ut - last_stored_ut)
- , (now_collect_ut - last_stored_ut)
- );
-
- rd->calculated_value -= new_value;
- new_value += rd->last_calculated_value;
- rd->last_calculated_value = 0;
- new_value /= (calculated_number)st->update_every;
-
- if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) {
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING",
- st->id, rd->name
- , (calculated_number)(next_store_ut - last_stored_ut)
- );
- new_value = new_value * (calculated_number)(st->update_every * 1000000) / (calculated_number)(next_store_ut - last_stored_ut);
- }
- break;
-
- case RRD_ALGORITHM_ABSOLUTE:
- case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
- case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
- default:
- if(iterations == 1) {
- // this is the last iteration
- // do not interpolate
- // just show the calculated value
-
- new_value = rd->calculated_value;
- }
- else {
- // we have missed an update
- // interpolate in the middle values
-
- new_value = (calculated_number)
- ( ( (rd->calculated_value - rd->last_calculated_value)
- * (calculated_number)(next_store_ut - last_collect_ut)
- / (calculated_number)(now_collect_ut - last_collect_ut)
- )
- + rd->last_calculated_value
- );
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
- CALCULATED_NUMBER_FORMAT " = ((("
- "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
- " * %llu"
- " / %llu) + " CALCULATED_NUMBER_FORMAT
- , st->id, rd->name
- , new_value
- , rd->calculated_value, rd->last_calculated_value
- , (next_store_ut - first_ut)
- , (now_collect_ut - first_ut), rd->last_calculated_value
- );
- }
- break;
- }
-
- if(unlikely(!store_this_entry)) {
- rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
- continue;
- }
-
- if(likely(rd->updated && rd->collections_counter > 1 && iterations < st->gap_when_lost_iterations_above)) {
- rd->values[st->current_entry] = pack_storage_number(new_value, storage_flags );
- rd->last_stored_value = new_value;
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
- CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
- , st->id, rd->name
- , st->current_entry
- , unpack_storage_number(rd->values[st->current_entry]), new_value
- );
- }
- else {
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING "
- , st->id, rd->name
- , st->current_entry
- );
- rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
- rd->last_stored_value = NAN;
- }
-
- stored_entries++;
-
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) {
- calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor;
- calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]);
- calculated_number accuracy = accuracy_loss(t1, t2);
- debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
- , st->id, rd->name
- , st->current_entry
- , t2
- , get_storage_number_flags(rd->values[st->current_entry])
- , t1
- , accuracy
- , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
- );
-
- rd->collected_volume += t1;
- rd->stored_volume += t2;
- accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume);
- debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s"
- , st->id, rd->name
- , st->current_entry
- , rd->stored_volume
- , rd->collected_volume
- , accuracy
- , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
- );
-
- }
- }
- // reset the storage flags for the next point, if any;
- storage_flags = SN_EXISTS;
-
- st->counter++;
- st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
- last_stored_ut = next_store_ut;
+ rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT");
+ info("INTERNAL CHECK: host '%s', chart '%s' is collected in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, st->name, next_store_ut - now_collect_ut);
}
+#endif
+
+ rrdset_done_interpolate(st
+ , update_every_ut
+ , last_stored_ut
+ , next_store_ut
+ , last_collect_ut
+ , now_collect_ut
+ , store_this_entry
+ , storage_flags
+ );
st->last_collected_total = st->collected_total;
@@ -1206,29 +1387,35 @@ void rrdset_done(RRDSET *st) {
if(unlikely(!rd->updated))
continue;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_collected_value, rd->collected_value);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rd->name, rd->last_collected_value, rd->collected_value);
+ #endif
rd->last_collected_value = rd->collected_value;
switch(rd->algorithm) {
case RRD_ALGORITHM_INCREMENTAL:
if(unlikely(!first_entry)) {
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value);
+ #endif
+
rd->last_calculated_value += rd->calculated_value;
}
else {
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s: THIS IS THE FIRST POINT", st->name);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "THIS IS THE FIRST POINT");
+ #endif
}
break;
case RRD_ALGORITHM_ABSOLUTE:
case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:
case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value, rd->calculated_value);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value);
+ #endif
+
rd->last_calculated_value = rd->calculated_value;
break;
}
@@ -1237,18 +1424,20 @@ void rrdset_done(RRDSET *st) {
rd->collected_value = 0;
rd->updated = 0;
- if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
- debug(D_RRD_STATS, "%s/%s: END "
+ #ifdef NETDATA_INTERNAL_CHECKS
+ rrdset_debug(st, "%s: END "
" last_collected_value = " COLLECTED_NUMBER_FORMAT
" collected_value = " COLLECTED_NUMBER_FORMAT
" last_calculated_value = " CALCULATED_NUMBER_FORMAT
" calculated_value = " CALCULATED_NUMBER_FORMAT
- , st->id, rd->name
+ , rd->name
, rd->last_collected_value
, rd->collected_value
, rd->last_calculated_value
, rd->calculated_value
- );
+ );
+ #endif
+
}
// ALL DONE ABOUT THE DATA UPDATE
diff --git a/src/socket.c b/src/socket.c
index 400c1ef4e..2b3821190 100644
--- a/src/socket.c
+++ b/src/socket.c
@@ -1,44 +1,267 @@
#include "common.h"
-// connect_to()
-//
-// definition format:
-//
-// [PROTOCOL:]IP[%INTERFACE][:PORT]
-//
-// PROTOCOL = tcp or udp
-// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6)
-// INTERFACE = for IPv6 only, the network interface to use
-// PORT = port number or service name
+// --------------------------------------------------------------------------------------------------------------------
+// various library calls
-int connect_to(const char *definition, int default_port, struct timeval *timeout) {
+#ifdef __gnu_linux__
+#define LARGE_SOCK_SIZE 33554431 // don't ask why - I found it at brubeck source - I guess it is just a large number
+#else
+#define LARGE_SOCK_SIZE 4096
+#endif
+
+int sock_setnonblock(int fd) {
+ int flags;
+
+ flags = fcntl(fd, F_GETFL);
+ flags |= O_NONBLOCK;
+
+ int ret = fcntl(fd, F_SETFL, flags);
+ if(ret < 0)
+ error("Failed to set O_NONBLOCK on socket %d", fd);
+
+ return ret;
+}
+
+int sock_delnonblock(int fd) {
+ int flags;
+
+ flags = fcntl(fd, F_GETFL);
+ flags &= ~O_NONBLOCK;
+
+ int ret = fcntl(fd, F_SETFL, flags);
+ if(ret < 0)
+ error("Failed to remove O_NONBLOCK on socket %d", fd);
+
+ return ret;
+}
+
+int sock_setreuse(int fd, int reuse) {
+ int ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
+
+ if(ret == -1)
+ error("Failed to set SO_REUSEADDR on socket %d", fd);
+
+ return ret;
+}
+
+int sock_setreuse_port(int fd, int reuse) {
+ int ret = -1;
+#ifdef SO_REUSEPORT
+ ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &reuse, sizeof(reuse));
+ if(ret == -1)
+ error("failed to set SO_REUSEPORT on socket %d", fd);
+#endif
+
+ return ret;
+}
+
+int sock_enlarge_in(int fd) {
+ int ret, bs = LARGE_SOCK_SIZE;
+
+ ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs));
+
+ if(ret == -1)
+ error("Failed to set SO_RCVBUF on socket %d", fd);
+
+ return ret;
+}
+
+int sock_enlarge_out(int fd) {
+ int ret, bs = LARGE_SOCK_SIZE;
+ ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs));
+
+ if(ret == -1)
+ error("Failed to set SO_SNDBUF on socket %d", fd);
+
+ return ret;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// listening sockets
+
+int create_listen_socket4(int socktype, const char *ip, int port, int listen_backlog) {
+ int sock;
+
+ debug(D_LISTENER, "LISTENER: IPv4 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype);
+
+ sock = socket(AF_INET, socktype, 0);
+ if(sock < 0) {
+ error("LISTENER: IPv4 socket() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ sock_setreuse(sock, 1);
+ sock_setreuse_port(sock, 1);
+ sock_setnonblock(sock);
+ sock_enlarge_in(sock);
+
+ struct sockaddr_in name;
+ memset(&name, 0, sizeof(struct sockaddr_in));
+ name.sin_family = AF_INET;
+ name.sin_port = htons (port);
+
+ int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr);
+ if(ret != 1) {
+ error("LISTENER: Failed to convert IP '%s' to a valid IPv4 address.", ip);
+ close(sock);
+ return -1;
+ }
+
+ if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
+ close(sock);
+ error("LISTENER: IPv4 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ if(socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) {
+ close(sock);
+ error("LISTENER: IPv4 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ debug(D_LISTENER, "LISTENER: Listening on IPv4 ip '%s' port %d, socktype %d", ip, port, socktype);
+ return sock;
+}
+
+int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int port, int listen_backlog) {
+ int sock;
+ int ipv6only = 1;
+
+ debug(D_LISTENER, "LISTENER: IPv6 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype);
+
+ sock = socket(AF_INET6, socktype, 0);
+ if (sock < 0) {
+ error("LISTENER: IPv6 socket() on ip '%s' port %d, socktype %d, failed.", ip, port, socktype);
+ return -1;
+ }
+
+ sock_setreuse(sock, 1);
+ sock_setreuse_port(sock, 1);
+ sock_setnonblock(sock);
+ sock_enlarge_in(sock);
+
+ /* IPv6 only */
+ if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0)
+ error("LISTENER: Cannot set IPV6_V6ONLY on ip '%s' port %d, socktype %d.", ip, port, socktype);
+
+ struct sockaddr_in6 name;
+ memset(&name, 0, sizeof(struct sockaddr_in6));
+ name.sin6_family = AF_INET6;
+ name.sin6_port = htons ((uint16_t) port);
+ name.sin6_scope_id = scope_id;
+
+ int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr);
+ if(ret != 1) {
+ error("LISTENER: Failed to convert IP '%s' to a valid IPv6 address.", ip);
+ close(sock);
+ return -1;
+ }
+
+ name.sin6_scope_id = scope_id;
+
+ if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
+ close(sock);
+ error("LISTENER: IPv6 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ if (socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) {
+ close(sock);
+ error("LISTENER: IPv6 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ debug(D_LISTENER, "LISTENER: Listening on IPv6 ip '%s' port %d, socktype %d", ip, port, socktype);
+ return sock;
+}
+
+static inline int listen_sockets_add(LISTEN_SOCKETS *sockets, int fd, int socktype, const char *protocol, const char *ip, int port) {
+ if(sockets->opened >= MAX_LISTEN_FDS) {
+ error("LISTENER: Too many listening sockets. Failed to add listening %s socket at ip '%s' port %d, protocol %s, socktype %d", protocol, ip, port, protocol, socktype);
+ close(fd);
+ return -1;
+ }
+
+ sockets->fds[sockets->opened] = fd;
+
+ char buffer[100 + 1];
+ snprintfz(buffer, 100, "%s:[%s]:%d", protocol, ip, port);
+ sockets->fds_names[sockets->opened] = strdupz(buffer);
+ sockets->fds_types[sockets->opened] = socktype;
+
+ sockets->opened++;
+ return 0;
+}
+
+int listen_sockets_check_is_member(LISTEN_SOCKETS *sockets, int fd) {
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++)
+ if(sockets->fds[i] == fd) return 1;
+
+ return 0;
+}
+
+static inline void listen_sockets_init(LISTEN_SOCKETS *sockets) {
+ size_t i;
+ for(i = 0; i < MAX_LISTEN_FDS ;i++) {
+ sockets->fds[i] = -1;
+ sockets->fds_names[i] = NULL;
+ sockets->fds_types[i] = -1;
+ }
+
+ sockets->opened = 0;
+ sockets->failed = 0;
+}
+
+void listen_sockets_close(LISTEN_SOCKETS *sockets) {
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++) {
+ close(sockets->fds[i]);
+ sockets->fds[i] = -1;
+
+ freez(sockets->fds_names[i]);
+ sockets->fds_names[i] = NULL;
+
+ sockets->fds_types[i] = -1;
+ }
+
+ sockets->opened = 0;
+ sockets->failed = 0;
+}
+
+static inline int bind_to_one(LISTEN_SOCKETS *sockets, const char *definition, int default_port, int listen_backlog) {
+ int added = 0;
struct addrinfo hints;
- struct addrinfo *ai_head = NULL, *ai = NULL;
+ struct addrinfo *result = NULL, *rp = NULL;
char buffer[strlen(definition) + 1];
strcpy(buffer, definition);
- char default_service[10 + 1];
- snprintfz(default_service, 10, "%d", default_port);
+ char buffer2[10 + 1];
+ snprintfz(buffer2, 10, "%d", default_port);
+
+ char *ip = buffer, *port = buffer2, *interface = "";;
- char *host = buffer, *service = default_service, *interface = "";
int protocol = IPPROTO_TCP, socktype = SOCK_STREAM;
- uint32_t scope_id = 0;
+ const char *protocol_str = "tcp";
- if(strncmp(host, "tcp:", 4) == 0) {
- host += 4;
+ if(strncmp(ip, "tcp:", 4) == 0) {
+ ip += 4;
protocol = IPPROTO_TCP;
socktype = SOCK_STREAM;
+ protocol_str = "tcp";
}
- else if(strncmp(host, "udp:", 4) == 0) {
- host += 4;
+ else if(strncmp(ip, "udp:", 4) == 0) {
+ ip += 4;
protocol = IPPROTO_UDP;
socktype = SOCK_DGRAM;
+ protocol_str = "udp";
}
- char *e = host;
+ char *e = ip;
if(*e == '[') {
- e = ++host;
+ e = ++ip;
while(*e && *e != ']') e++;
if(*e == ']') {
*e = '\0';
@@ -57,26 +280,142 @@ int connect_to(const char *definition, int default_port, struct timeval *timeout
}
if(*e == ':') {
+ port = e + 1;
*e = '\0';
- e++;
- service = e;
}
- debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP);
+ uint32_t scope_id = 0;
+ if(*interface) {
+ scope_id = if_nametoindex(interface);
+ if(!scope_id)
+ error("LISTENER: Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface);
+ }
- if(!*host) {
- error("Definition '%s' does not specify a host.", definition);
+ if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all"))
+ ip = NULL;
+
+ if(!*port)
+ port = buffer2;
+
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
+ hints.ai_socktype = socktype;
+ hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */
+ hints.ai_protocol = protocol;
+ hints.ai_canonname = NULL;
+ hints.ai_addr = NULL;
+ hints.ai_next = NULL;
+
+ int r = getaddrinfo(ip, port, &hints, &result);
+ if (r != 0) {
+ error("LISTENER: getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r));
return -1;
}
- if(*interface) {
- scope_id = if_nametoindex(interface);
- if(!scope_id)
- error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface);
+ for (rp = result; rp != NULL; rp = rp->ai_next) {
+ int fd = -1;
+
+ char rip[INET_ADDRSTRLEN + INET6_ADDRSTRLEN] = "INVALID";
+ int rport = default_port;
+
+ switch (rp->ai_addr->sa_family) {
+ case AF_INET: {
+ struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr;
+ inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN);
+ rport = ntohs(sin->sin_port);
+ // info("Attempting to listen on IPv4 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype);
+ fd = create_listen_socket4(socktype, rip, rport, listen_backlog);
+ break;
+ }
+
+ case AF_INET6: {
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr;
+ inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN);
+ rport = ntohs(sin6->sin6_port);
+ // info("Attempting to listen on IPv6 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype);
+ fd = create_listen_socket6(socktype, scope_id, rip, rport, listen_backlog);
+ break;
+ }
+
+ default:
+ debug(D_LISTENER, "LISTENER: Unknown socket family %d", rp->ai_addr->sa_family);
+ break;
+ }
+
+ if (fd == -1) {
+ error("LISTENER: Cannot bind to ip '%s', port %d", rip, rport);
+ sockets->failed++;
+ }
+ else {
+ listen_sockets_add(sockets, fd, socktype, protocol_str, rip, rport);
+ added++;
+ }
}
- if(!*service)
- service = default_service;
+ freeaddrinfo(result);
+
+ return added;
+}
+
+int listen_sockets_setup(LISTEN_SOCKETS *sockets) {
+ listen_sockets_init(sockets);
+
+ sockets->backlog = (int) config_get_number(sockets->config_section, "listen backlog", sockets->backlog);
+
+ int old_port = sockets->default_port;
+ sockets->default_port = (int) config_get_number(sockets->config_section, "default port", sockets->default_port);
+ if(sockets->default_port < 1 || sockets->default_port > 65535) {
+ error("LISTENER: Invalid listen port %d given. Defaulting to %d.", sockets->default_port, old_port);
+ sockets->default_port = (int) config_set_number(sockets->config_section, "default port", old_port);
+ }
+ debug(D_OPTIONS, "LISTENER: Default listen port set to %d.", sockets->default_port);
+
+ char *s = config_get(sockets->config_section, "bind to", sockets->default_bind_to);
+ while(*s) {
+ char *e = s;
+
+ // skip separators, moving both s(tart) and e(nd)
+ while(isspace(*e) || *e == ',') s = ++e;
+
+ // move e(nd) to the first separator
+ while(*e && !isspace(*e) && *e != ',') e++;
+
+ // is there anything?
+ if(!*s || s == e) break;
+
+ char buf[e - s + 1];
+ strncpyz(buf, s, e - s);
+ bind_to_one(sockets, buf, sockets->default_port, sockets->backlog);
+
+ s = e;
+ }
+
+ if(sockets->failed) {
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++)
+ info("LISTENER: Listen socket %s opened successfully.", sockets->fds_names[i]);
+ }
+
+ return (int)sockets->opened;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// connect to another host/port
+
+// _connect_to()
+// protocol IPPROTO_TCP, IPPROTO_UDP
+// socktype SOCK_STREAM, SOCK_DGRAM
+// host the destination hostname or IP address (IPv4 or IPv6) to connect to
+// if it resolves to many IPs, all are tried (IPv4 and IPv6)
+// scope_id the if_index id of the interface to use for connecting (0 = any)
+// (used only under IPv6)
+// service the service name or port to connect to
+// timeout the timeout for establishing a connection
+
+static inline int _connect_to(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout) {
+ struct addrinfo hints;
+ struct addrinfo *ai_head = NULL, *ai = NULL;
memset(&hints, 0, sizeof(hints));
hints.ai_family = PF_UNSPEC; /* Allow IPv4 or IPv6 */
@@ -103,52 +442,52 @@ int connect_to(const char *definition, int default_port, struct timeval *timeout
char servBfr[NI_MAXSERV + 1];
getnameinfo(ai->ai_addr,
- ai->ai_addrlen,
- hostBfr,
- sizeof(hostBfr),
- servBfr,
- sizeof(servBfr),
- NI_NUMERICHOST | NI_NUMERICSERV);
+ ai->ai_addrlen,
+ hostBfr,
+ sizeof(hostBfr),
+ servBfr,
+ sizeof(servBfr),
+ NI_NUMERICHOST | NI_NUMERICSERV);
debug(D_CONNECT_TO, "Address info: host = '%s', service = '%s', ai_flags = 0x%02X, ai_family = %d (PF_INET = %d, PF_INET6 = %d), ai_socktype = %d (SOCK_STREAM = %d, SOCK_DGRAM = %d), ai_protocol = %d (IPPROTO_TCP = %d, IPPROTO_UDP = %d), ai_addrlen = %lu (sockaddr_in = %lu, sockaddr_in6 = %lu)",
- hostBfr,
- servBfr,
- (unsigned int)ai->ai_flags,
- ai->ai_family,
- PF_INET,
- PF_INET6,
- ai->ai_socktype,
- SOCK_STREAM,
- SOCK_DGRAM,
- ai->ai_protocol,
- IPPROTO_TCP,
- IPPROTO_UDP,
- (unsigned long)ai->ai_addrlen,
- (unsigned long)sizeof(struct sockaddr_in),
- (unsigned long)sizeof(struct sockaddr_in6));
+ hostBfr,
+ servBfr,
+ (unsigned int)ai->ai_flags,
+ ai->ai_family,
+ PF_INET,
+ PF_INET6,
+ ai->ai_socktype,
+ SOCK_STREAM,
+ SOCK_DGRAM,
+ ai->ai_protocol,
+ IPPROTO_TCP,
+ IPPROTO_UDP,
+ (unsigned long)ai->ai_addrlen,
+ (unsigned long)sizeof(struct sockaddr_in),
+ (unsigned long)sizeof(struct sockaddr_in6));
switch (ai->ai_addr->sa_family) {
case PF_INET: {
struct sockaddr_in *pSadrIn = (struct sockaddr_in *)ai->ai_addr;
debug(D_CONNECT_TO, "ai_addr = sin_family: %d (AF_INET = %d, AF_INET6 = %d), sin_addr: '%s', sin_port: '%s'",
- pSadrIn->sin_family,
- AF_INET,
- AF_INET6,
- hostBfr,
- servBfr);
+ pSadrIn->sin_family,
+ AF_INET,
+ AF_INET6,
+ hostBfr,
+ servBfr);
break;
}
case PF_INET6: {
struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr;
debug(D_CONNECT_TO,"ai_addr = sin6_family: %d (AF_INET = %d, AF_INET6 = %d), sin6_addr: '%s', sin6_port: '%s', sin6_flowinfo: %u, sin6_scope_id: %u",
- pSadrIn6->sin6_family,
- AF_INET,
- AF_INET6,
- hostBfr,
- servBfr,
- pSadrIn6->sin6_flowinfo,
- pSadrIn6->sin6_scope_id);
+ pSadrIn6->sin6_family,
+ AF_INET,
+ AF_INET6,
+ hostBfr,
+ servBfr,
+ pSadrIn6->sin6_flowinfo,
+ pSadrIn6->sin6_scope_id);
break;
}
@@ -180,6 +519,85 @@ int connect_to(const char *definition, int default_port, struct timeval *timeout
return fd;
}
+// connect_to()
+//
+// definition format:
+//
+// [PROTOCOL:]IP[%INTERFACE][:PORT]
+//
+// PROTOCOL = tcp or udp
+// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6)
+// INTERFACE = for IPv6 only, the network interface to use
+// PORT = port number or service name
+
+int connect_to(const char *definition, int default_port, struct timeval *timeout) {
+ char buffer[strlen(definition) + 1];
+ strcpy(buffer, definition);
+
+ char default_service[10 + 1];
+ snprintfz(default_service, 10, "%d", default_port);
+
+ char *host = buffer, *service = default_service, *interface = "";
+ int protocol = IPPROTO_TCP, socktype = SOCK_STREAM;
+ uint32_t scope_id = 0;
+
+ if(strncmp(host, "tcp:", 4) == 0) {
+ host += 4;
+ protocol = IPPROTO_TCP;
+ socktype = SOCK_STREAM;
+ }
+ else if(strncmp(host, "udp:", 4) == 0) {
+ host += 4;
+ protocol = IPPROTO_UDP;
+ socktype = SOCK_DGRAM;
+ }
+
+ char *e = host;
+ if(*e == '[') {
+ e = ++host;
+ while(*e && *e != ']') e++;
+ if(*e == ']') {
+ *e = '\0';
+ e++;
+ }
+ }
+ else {
+ while(*e && *e != ':' && *e != '%') e++;
+ }
+
+ if(*e == '%') {
+ *e = '\0';
+ e++;
+ interface = e;
+ while(*e && *e != ':') e++;
+ }
+
+ if(*e == ':') {
+ *e = '\0';
+ e++;
+ service = e;
+ }
+
+ debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP);
+
+ if(!*host) {
+ error("Definition '%s' does not specify a host.", definition);
+ return -1;
+ }
+
+ if(*interface) {
+ scope_id = if_nametoindex(interface);
+ if(!scope_id)
+ error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface);
+ }
+
+ if(!*service)
+ service = default_service;
+
+
+ return _connect_to(protocol, socktype, host, scope_id, service, timeout);
+}
+
int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) {
int sock = -1;
@@ -213,6 +631,10 @@ int connect_to_one_of(const char *destination, int default_port, struct timeval
return sock;
}
+
+// --------------------------------------------------------------------------------------------------------------------
+// helpers to send/receive data in one call, in blocking mode, with a timeout
+
ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) {
for(;;) {
struct pollfd fd = {
@@ -274,3 +696,455 @@ ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout)
return send(sockfd, buf, len, flags);
}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// accept4() replacement for systems that do not have one
+
+#ifndef HAVE_ACCEPT4
+int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) {
+ int fd = accept(sock, addr, addrlen);
+ int newflags = 0;
+
+ if (fd < 0) return fd;
+
+ if (flags & SOCK_NONBLOCK) {
+ newflags |= O_NONBLOCK;
+ flags &= ~SOCK_NONBLOCK;
+ }
+
+#ifdef SOCK_CLOEXEC
+#ifdef O_CLOEXEC
+ if (flags & SOCK_CLOEXEC) {
+ newflags |= O_CLOEXEC;
+ flags &= ~SOCK_CLOEXEC;
+ }
+#endif
+#endif
+
+ if (flags) {
+ errno = -EINVAL;
+ return -1;
+ }
+
+ if (fcntl(fd, F_SETFL, newflags) < 0) {
+ int saved_errno = errno;
+ close(fd);
+ errno = saved_errno;
+ return -1;
+ }
+
+ return fd;
+}
+#endif
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// accept_socket() - accept a socket and store client IP and port
+
+int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize) {
+ struct sockaddr_storage sadr;
+ socklen_t addrlen = sizeof(sadr);
+
+ int nfd = accept4(fd, (struct sockaddr *)&sadr, &addrlen, flags);
+ if (nfd >= 0) {
+ if (getnameinfo((struct sockaddr *)&sadr, addrlen, client_ip, (socklen_t)ipsize, client_port, (socklen_t)portsize, NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
+ error("LISTENER: cannot getnameinfo() on received client connection.");
+ strncpyz(client_ip, "UNKNOWN", ipsize - 1);
+ strncpyz(client_port, "UNKNOWN", portsize - 1);
+ }
+
+ client_ip[ipsize - 1] = '\0';
+ client_port[portsize - 1] = '\0';
+
+ switch (((struct sockaddr *)&sadr)->sa_family) {
+ case AF_INET:
+ debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ break;
+
+ case AF_INET6:
+ if (strncmp(client_ip, "::ffff:", 7) == 0) {
+ memmove(client_ip, &client_ip[7], strlen(&client_ip[7]) + 1);
+ debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ } else
+ debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ break;
+
+ default:
+ debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ break;
+ }
+ }
+
+ return nfd;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// poll() based listener
+// this should be the fastest possible listener for up to 100 sockets
+// above 100, an epoll() interface is needed on Linux
+
+#define POLL_FDS_INCREASE_STEP 10
+
+#define POLLINFO_FLAG_SERVER_SOCKET 0x00000001
+#define POLLINFO_FLAG_CLIENT_SOCKET 0x00000002
+
+struct pollinfo {
+ size_t slot;
+ char *client;
+ struct pollinfo *next;
+ uint32_t flags;
+ int socktype;
+
+ void *data;
+};
+
+struct poll {
+ size_t slots;
+ size_t used;
+ size_t min;
+ size_t max;
+ struct pollfd *fds;
+ struct pollinfo *inf;
+ struct pollinfo *first_free;
+
+ void *(*add_callback)(int fd, short int *events);
+ void (*del_callback)(int fd, void *data);
+ int (*rcv_callback)(int fd, int socktype, void *data, short int *events);
+ int (*snd_callback)(int fd, int socktype, void *data, short int *events);
+};
+
+static inline struct pollinfo *poll_add_fd(struct poll *p, int fd, int socktype, short int events, uint32_t flags) {
+ debug(D_POLLFD, "POLLFD: ADD: request to add fd %d, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", fd, p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ if(unlikely(fd < 0)) return NULL;
+
+ if(unlikely(!p->first_free)) {
+ size_t new_slots = p->slots + POLL_FDS_INCREASE_STEP;
+ debug(D_POLLFD, "POLLFD: ADD: increasing size (current = %zu, new = %zu, used = %zu, min = %zu, max = %zu)", p->slots, new_slots, p->used, p->min, p->max);
+
+ p->fds = reallocz(p->fds, sizeof(struct pollfd) * new_slots);
+ p->inf = reallocz(p->inf, sizeof(struct pollinfo) * new_slots);
+
+ ssize_t i;
+ for(i = new_slots - 1; i >= (ssize_t)p->slots ; i--) {
+ debug(D_POLLFD, "POLLFD: ADD: resetting new slot %zd", i);
+ p->fds[i].fd = -1;
+ p->fds[i].events = 0;
+ p->fds[i].revents = 0;
+
+ p->inf[i].slot = (size_t)i;
+ p->inf[i].flags = 0;
+ p->inf[i].socktype = -1;
+ p->inf[i].client = NULL;
+ p->inf[i].data = NULL;
+ p->inf[i].next = p->first_free;
+ p->first_free = &p->inf[i];
+ }
+
+ p->slots = new_slots;
+ }
+
+ struct pollinfo *pi = p->first_free;
+ p->first_free = p->first_free->next;
+
+ debug(D_POLLFD, "POLLFD: ADD: selected slot %zu, next free is %zd", pi->slot, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ struct pollfd *pf = &p->fds[pi->slot];
+ pf->fd = fd;
+ pf->events = events;
+ pf->revents = 0;
+
+ pi->socktype = socktype;
+ pi->flags = flags;
+ pi->next = NULL;
+
+ p->used++;
+ if(unlikely(pi->slot > p->max))
+ p->max = pi->slot;
+
+ if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) {
+ pi->data = p->add_callback(fd, &pf->events);
+ }
+
+ if(pi->flags & POLLINFO_FLAG_SERVER_SOCKET) {
+ p->min = pi->slot;
+ }
+
+ debug(D_POLLFD, "POLLFD: ADD: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ return pi;
+}
+
+static inline void poll_close_fd(struct poll *p, struct pollinfo *pi) {
+ struct pollfd *pf = &p->fds[pi->slot];
+ debug(D_POLLFD, "POLLFD: DEL: request to clear slot %zu (fd %d), old next free was %zd", pi->slot, pf->fd, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ if(unlikely(pf->fd == -1)) return;
+
+ if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) {
+ p->del_callback(pf->fd, pi->data);
+ }
+
+ close(pf->fd);
+ pf->fd = -1;
+ pf->events = 0;
+ pf->revents = 0;
+
+ pi->socktype = -1;
+ pi->flags = 0;
+ pi->data = NULL;
+
+ freez(pi->client);
+ pi->client = NULL;
+
+ pi->next = p->first_free;
+ p->first_free = pi;
+
+ p->used--;
+ if(p->max == pi->slot) {
+ p->max = p->min;
+ ssize_t i;
+ for(i = (ssize_t)pi->slot; i > (ssize_t)p->min ;i--) {
+ if (unlikely(p->fds[i].fd != -1)) {
+ p->max = (size_t)i;
+ break;
+ }
+ }
+ }
+
+ debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+}
+
+static void *add_callback_default(int fd, short int *events) {
+ (void)fd;
+ (void)events;
+
+ return NULL;
+}
+static void del_callback_default(int fd, void *data) {
+ (void)fd;
+ (void)data;
+
+ if(data)
+ error("POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak");
+}
+
+static int rcv_callback_default(int fd, int socktype, void *data, short int *events) {
+ (void)socktype;
+ (void)data;
+ (void)events;
+
+ char buffer[1024 + 1];
+
+ ssize_t rc;
+ do {
+ rc = recv(fd, buffer, 1024, MSG_DONTWAIT);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN) {
+ error("POLLFD: recv() failed.");
+ return -1;
+ }
+ } else if (rc) {
+ // data received
+ info("POLLFD: internal error: discarding %zd bytes received on socket %d", rc, fd);
+ }
+ } while (rc != -1);
+
+ return 0;
+}
+
+static int snd_callback_default(int fd, int socktype, void *data, short int *events) {
+ (void)socktype;
+ (void)data;
+ (void)events;
+
+ *events &= ~POLLOUT;
+
+ info("POLLFD: internal error: nothing to send on socket %d", fd);
+ return 0;
+}
+
+void poll_events_cleanup(void *data) {
+ struct poll *p = (struct poll *)data;
+
+ size_t i;
+ for(i = 0 ; i <= p->max ; i++) {
+ struct pollinfo *pi = &p->inf[i];
+ poll_close_fd(p, pi);
+ }
+
+ freez(p->fds);
+ freez(p->inf);
+}
+
+void poll_events(LISTEN_SOCKETS *sockets
+ , void *(*add_callback)(int fd, short int *events)
+ , void (*del_callback)(int fd, void *data)
+ , int (*rcv_callback)(int fd, int socktype, void *data, short int *events)
+ , int (*snd_callback)(int fd, int socktype, void *data, short int *events)
+ , void *data
+) {
+ int retval;
+
+ struct poll p = {
+ .slots = 0,
+ .used = 0,
+ .max = 0,
+ .fds = NULL,
+ .inf = NULL,
+ .first_free = NULL,
+
+ .add_callback = add_callback?add_callback:add_callback_default,
+ .del_callback = del_callback?del_callback:del_callback_default,
+ .rcv_callback = rcv_callback?rcv_callback:rcv_callback_default,
+ .snd_callback = snd_callback?snd_callback:snd_callback_default
+ };
+
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++) {
+ struct pollinfo *pi = poll_add_fd(&p, sockets->fds[i], sockets->fds_types[i], POLLIN, POLLINFO_FLAG_SERVER_SOCKET);
+ pi->data = data;
+ info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN");
+ }
+
+ int timeout = -1; // wait forever
+
+ pthread_cleanup_push(poll_events_cleanup, &p);
+
+ for(;;) {
+ if(unlikely(netdata_exit)) break;
+
+ debug(D_POLLFD, "POLLFD: LISTENER: Waiting on %zu sockets...", p.max + 1);
+ retval = poll(p.fds, p.max + 1, timeout);
+
+ if(unlikely(retval == -1)) {
+ error("POLLFD: LISTENER: poll() failed.");
+ continue;
+ }
+ else if(unlikely(!retval)) {
+ debug(D_POLLFD, "POLLFD: LISTENER: poll() timeout.");
+ continue;
+ }
+
+ if(unlikely(netdata_exit)) break;
+
+ for(i = 0 ; i <= p.max ; i++) {
+ struct pollfd *pf = &p.fds[i];
+ struct pollinfo *pi = &p.inf[i];
+ int fd = pf->fd;
+ short int revents = pf->revents;
+ pf->revents = 0;
+
+ if(unlikely(fd == -1)) {
+ debug(D_POLLFD, "POLLFD: LISTENER: ignoring slot %zu, it does not have an fd", i);
+ continue;
+ }
+
+ debug(D_POLLFD, "POLLFD: LISTENER: processing events for slot %zu (events = %d, revents = %d)", i, pf->events, revents);
+
+ if(revents & POLLIN || revents & POLLPRI) {
+ // receiving data
+
+ if(likely(pi->flags & POLLINFO_FLAG_SERVER_SOCKET)) {
+ // new connection
+ // debug(D_POLLFD, "POLLFD: LISTENER: accepting connections from slot %zu (fd %d)", i, fd);
+
+ switch(pi->socktype) {
+ case SOCK_STREAM: {
+ // a TCP socket
+ // we accept the connection
+
+ int nfd;
+ do {
+ char client_ip[NI_MAXHOST + 1];
+ char client_port[NI_MAXSERV + 1];
+
+ debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", i, fd);
+ nfd = accept_socket(fd, SOCK_NONBLOCK, client_ip, NI_MAXHOST + 1, client_port, NI_MAXSERV + 1);
+ if (nfd < 0) {
+ // accept failed
+
+ debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", i, fd);
+
+ if(errno != EWOULDBLOCK && errno != EAGAIN)
+ error("POLLFD: LISTENER: accept() failed.");
+
+ break;
+ }
+ else {
+ // accept ok
+ info("POLLFD: LISTENER: client '[%s]:%s' connected to '%s'", client_ip, client_port, sockets->fds_names[i]);
+ poll_add_fd(&p, nfd, SOCK_STREAM, POLLIN, POLLINFO_FLAG_CLIENT_SOCKET);
+
+ // it may have realloced them, so refresh our pointers
+ pf = &p.fds[i];
+ pi = &p.inf[i];
+ }
+ } while (nfd != -1);
+ break;
+ }
+
+ case SOCK_DGRAM: {
+ // a UDP socket
+ // we read data from the server socket
+
+ debug(D_POLLFD, "POLLFD: LISTENER: reading data from UDP slot %zu (fd %d)", i, fd);
+
+ p.rcv_callback(fd, pi->socktype, pi->data, &pf->events);
+ break;
+ }
+
+ default: {
+ error("POLLFD: LISTENER: Unknown socktype %d on slot %zu", pi->socktype, pi->slot);
+ break;
+ }
+ }
+ }
+
+ if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) {
+ // read data from client TCP socket
+ debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", i, fd);
+
+ if (p.rcv_callback(fd, pi->socktype, pi->data, &pf->events) == -1) {
+ poll_close_fd(&p, pi);
+ continue;
+ }
+ }
+ }
+
+ if(unlikely(revents & POLLOUT)) {
+ // sending data
+ debug(D_POLLFD, "POLLFD: LISTENER: sending data to socket on slot %zu (fd %d)", i, fd);
+
+ if (p.snd_callback(fd, pi->socktype, pi->data, &pf->events) == -1) {
+ poll_close_fd(&p, pi);
+ continue;
+ }
+ }
+
+ if(unlikely(revents & POLLERR)) {
+ error("POLLFD: LISTENER: processing POLLERR events for slot %zu (events = %d, revents = %d)", i, pf->events, revents);
+ poll_close_fd(&p, pi);
+ continue;
+ }
+
+ if(unlikely(revents & POLLHUP)) {
+ error("POLLFD: LISTENER: processing POLLHUP events for slot %zu (events = %d, revents = %d)", i, pf->events, pf->revents);
+ poll_close_fd(&p, pi);
+ continue;
+ }
+
+ if(unlikely(revents & POLLNVAL)) {
+ error("POLLFD: LISTENER: processing POLLNVAP events for slot %zu (events = %d, revents = %d)", i, pf->events, revents);
+ poll_close_fd(&p, pi);
+ continue;
+ }
+ }
+ }
+
+ pthread_cleanup_pop(1);
+ debug(D_POLLFD, "POLLFD: LISTENER: cleanup completed");
+}
diff --git a/src/socket.h b/src/socket.h
index 89c154a61..bb95347ab 100644
--- a/src/socket.h
+++ b/src/socket.h
@@ -1,14 +1,61 @@
-//
-// Created by costa on 24/12/2016.
-//
-
#ifndef NETDATA_SOCKET_H
#define NETDATA_SOCKET_H
+#ifndef MAX_LISTEN_FDS
+#define MAX_LISTEN_FDS 50
+#endif
+
+typedef struct listen_sockets {
+ const char *config_section; // the netdata configuration section to read settings from
+ const char *default_bind_to; // the default bind to configuration string
+ int default_port; // the default port to use
+ int backlog; // the default listen backlog to use
+
+ size_t opened; // the number of sockets opened
+ size_t failed; // the number of sockets attempted to open, but failed
+ int fds[MAX_LISTEN_FDS]; // the open sockets
+ char *fds_names[MAX_LISTEN_FDS]; // descriptions for the open sockets
+ int fds_types[MAX_LISTEN_FDS]; // the socktype for the open sockets
+} LISTEN_SOCKETS;
+
+extern int listen_sockets_setup(LISTEN_SOCKETS *sockets);
+extern void listen_sockets_close(LISTEN_SOCKETS *sockets);
+
extern int connect_to(const char *definition, int default_port, struct timeval *timeout);
extern int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size);
extern ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
extern ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
+extern int sock_setnonblock(int fd);
+extern int sock_delnonblock(int fd);
+extern int sock_setreuse(int fd, int reuse);
+extern int sock_setreuse_port(int fd, int reuse);
+extern int sock_enlarge_in(int fd);
+extern int sock_enlarge_out(int fd);
+
+extern int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize);
+
+#ifndef HAVE_ACCEPT4
+extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags);
+
+#ifndef SOCK_NONBLOCK
+#define SOCK_NONBLOCK 00004000
+#endif /* #ifndef SOCK_NONBLOCK */
+
+#ifndef SOCK_CLOEXEC
+#define SOCK_CLOEXEC 02000000
+#endif /* #ifndef SOCK_CLOEXEC */
+
+#endif /* #ifndef HAVE_ACCEPT4 */
+
+
+extern void poll_events(LISTEN_SOCKETS *sockets
+ , void *(*add_callback)(int fd, short int *events)
+ , void (*del_callback)(int fd, void *data)
+ , int (*rcv_callback)(int fd, int socktype, void *data, short int *events)
+ , int (*snd_callback)(int fd, int socktype, void *data, short int *events)
+ , void *data
+);
+
#endif //NETDATA_SOCKET_H
diff --git a/src/statistical.c b/src/statistical.c
new file mode 100644
index 000000000..807bc25ea
--- /dev/null
+++ b/src/statistical.c
@@ -0,0 +1,459 @@
+#include "common.h"
+
+// --------------------------------------------------------------------------------------------------------------------
+
+inline long double sum_and_count(long double *series, size_t entries, size_t *count) {
+ if(unlikely(entries == 0)) {
+ if(likely(count))
+ *count = 0;
+
+ return NAN;
+ }
+
+ if(unlikely(entries == 1)) {
+ if(likely(count))
+ *count = (isnan(series[0])?0:1);
+
+ return series[0];
+ }
+
+ size_t i, c = 0;
+ long double sum = 0;
+
+ for(i = 0; i < entries ; i++) {
+ long double value = series[i];
+ if(unlikely(isnan(value) || isinf(value))) continue;
+ c++;
+ sum += value;
+ }
+
+ if(likely(count))
+ *count = c;
+
+ if(unlikely(c == 0))
+ return NAN;
+
+ return sum;
+}
+
+inline long double sum(long double *series, size_t entries) {
+ return sum_and_count(series, entries, NULL);
+}
+
+inline long double average(long double *series, size_t entries) {
+ size_t count = 0;
+ long double sum = sum_and_count(series, entries, &count);
+
+ if(unlikely(count == 0))
+ return NAN;
+
+ return sum / (long double)count;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+long double moving_average(long double *series, size_t entries, size_t period) {
+ if(unlikely(period <= 0))
+ return 0.0;
+
+ size_t i, count;
+ long double sum = 0, avg = 0;
+ long double p[period];
+
+ for(count = 0; count < period ; count++)
+ p[count] = 0.0;
+
+ for(i = 0, count = 0; i < entries; i++) {
+ long double value = series[i];
+ if(unlikely(isnan(value) || isinf(value))) continue;
+
+ if(unlikely(count < period)) {
+ sum += value;
+ avg = (count == period - 1) ? sum / (long double)period : 0;
+ }
+ else {
+ sum = sum - p[count % period] + value;
+ avg = sum / (long double)period;
+ }
+
+ p[count % period] = value;
+ count++;
+ }
+
+ return avg;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+static int qsort_compare(const void *a, const void *b) {
+ long double *p1 = (long double *)a, *p2 = (long double *)b;
+ long double n1 = *p1, n2 = *p2;
+
+ if(unlikely(isnan(n1) || isnan(n2))) {
+ if(isnan(n1) && !isnan(n2)) return -1;
+ if(!isnan(n1) && isnan(n2)) return 1;
+ return 0;
+ }
+ if(unlikely(isinf(n1) || isinf(n2))) {
+ if(!isinf(n1) && isinf(n2)) return -1;
+ if(isinf(n1) && !isinf(n2)) return 1;
+ return 0;
+ }
+
+ if(unlikely(n1 < n2)) return -1;
+ if(unlikely(n1 > n2)) return 1;
+ return 0;
+}
+
+inline void sort_series(long double *series, size_t entries) {
+ qsort(series, entries, sizeof(long double), qsort_compare);
+}
+
+inline long double *copy_series(long double *series, size_t entries) {
+ long double *copy = mallocz(sizeof(long double) * entries);
+ memcpy(copy, series, sizeof(long double) * entries);
+ return copy;
+}
+
+long double median_on_sorted_series(long double *series, size_t entries) {
+ if(unlikely(entries == 0))
+ return NAN;
+
+ if(unlikely(entries == 1))
+ return series[0];
+
+ if(unlikely(entries == 2))
+ return (series[0] + series[1]) / 2;
+
+ long double avg;
+ if(entries % 2 == 0) {
+ size_t m = entries / 2;
+ avg = (series[m] + series[m + 1]) / 2;
+ }
+ else {
+ avg = series[entries / 2];
+ }
+
+ return avg;
+}
+
+long double median(long double *series, size_t entries) {
+ if(unlikely(entries == 0))
+ return NAN;
+
+ if(unlikely(entries == 1))
+ return series[0];
+
+ if(unlikely(entries == 2))
+ return (series[0] + series[1]) / 2;
+
+ long double *copy = copy_series(series, entries);
+ sort_series(copy, entries);
+
+ long double avg = median_on_sorted_series(copy, entries);
+
+ freez(copy);
+ return avg;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+long double moving_median(long double *series, size_t entries, size_t period) {
+ if(entries <= period)
+ return median(series, entries);
+
+ long double *data = copy_series(series, entries);
+
+ size_t i;
+ for(i = period; i < entries; i++) {
+ data[i - period] = median(&series[i - period], period);
+ }
+
+ long double avg = median(data, entries - period);
+ freez(data);
+ return avg;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+// http://stackoverflow.com/a/15150143/4525767
+long double running_median_estimate(long double *series, size_t entries) {
+ long double median = 0.0f;
+ long double average = 0.0f;
+ size_t i;
+
+ for(i = 0; i < entries ; i++) {
+ long double value = series[i];
+ if(unlikely(isnan(value) || isinf(value))) continue;
+
+ average += ( value - average ) * 0.1f; // rough running average.
+ median += copysignl( average * 0.01, value - median );
+ }
+
+ return median;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+long double standard_deviation(long double *series, size_t entries) {
+ if(unlikely(entries < 1))
+ return NAN;
+
+ if(unlikely(entries == 1))
+ return series[0];
+
+ size_t i, count = 0;
+ long double sum = 0;
+
+ for(i = 0; i < entries ; i++) {
+ long double value = series[i];
+ if(unlikely(isnan(value) || isinf(value))) continue;
+
+ count++;
+ sum += value;
+ }
+
+ if(unlikely(count == 0))
+ return NAN;
+
+ if(unlikely(count == 1))
+ return sum;
+
+ long double average = sum / (long double)count;
+
+ for(i = 0, count = 0, sum = 0; i < entries ; i++) {
+ long double value = series[i];
+ if(unlikely(isnan(value) || isinf(value))) continue;
+
+ count++;
+ sum += powl(value - average, 2);
+ }
+
+ if(unlikely(count == 0))
+ return NAN;
+
+ if(unlikely(count == 1))
+ return average;
+
+ long double variance = sum / (long double)(count - 1); // remove -1 to have a population stddev
+
+ long double stddev = sqrtl(variance);
+ return stddev;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+long double single_exponential_smoothing(long double *series, size_t entries, long double alpha) {
+ size_t i, count = 0;
+ long double level = 0, sum = 0;
+
+ if(unlikely(isnan(alpha)))
+ alpha = 0.3;
+
+ for(i = 0; i < entries ; i++) {
+ long double value = series[i];
+ if(unlikely(isnan(value) || isinf(value))) continue;
+ count++;
+
+ sum += value;
+
+ long double last_level = level;
+ level = alpha * value + (1.0 - alpha) * last_level;
+ }
+
+ return level;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+// http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/
+long double double_exponential_smoothing(long double *series, size_t entries, long double alpha, long double beta, long double *forecast) {
+ size_t i, count = 0;
+ long double level = series[0], trend, sum;
+
+ if(unlikely(isnan(alpha)))
+ alpha = 0.3;
+
+ if(unlikely(isnan(beta)))
+ beta = 0.05;
+
+ if(likely(entries > 1))
+ trend = series[1] - series[0];
+ else
+ trend = 0;
+
+ sum = series[0];
+
+ for(i = 1; i < entries ; i++) {
+ long double value = series[i];
+ if(unlikely(isnan(value) || isinf(value))) continue;
+ count++;
+
+ sum += value;
+
+ long double last_level = level;
+
+ level = alpha * value + (1.0 - alpha) * (level + trend);
+ trend = beta * (level - last_level) + (1.0 - beta) * trend;
+ }
+
+ if(forecast)
+ *forecast = level + trend;
+
+ return level;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+/*
+ * Based on th R implementation
+ *
+ * a: level component
+ * b: trend component
+ * s: seasonal component
+ *
+ * Additive:
+ *
+ * Yhat[t+h] = a[t] + h * b[t] + s[t + 1 + (h - 1) mod p],
+ * a[t] = α (Y[t] - s[t-p]) + (1-α) (a[t-1] + b[t-1])
+ * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1]
+ * s[t] = γ (Y[t] - a[t]) + (1-γ) s[t-p]
+ *
+ * Multiplicative:
+ *
+ * Yhat[t+h] = (a[t] + h * b[t]) * s[t + 1 + (h - 1) mod p],
+ * a[t] = α (Y[t] / s[t-p]) + (1-α) (a[t-1] + b[t-1])
+ * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1]
+ * s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p]
+ */
+static int __HoltWinters(
+ long double *series,
+ int entries, // start_time + h
+
+ long double alpha, // alpha parameter of Holt-Winters Filter.
+ long double beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing.
+ long double gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted.
+
+ int *seasonal,
+ int *period,
+ long double *a, // Start value for level (a[0]).
+ long double *b, // Start value for trend (b[0]).
+ long double *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0])
+
+ /* return values */
+ long double *SSE, // The final sum of squared errors achieved in optimizing
+ long double *level, // Estimated values for the level component (size entries - t + 2)
+ long double *trend, // Estimated values for the trend component (size entries - t + 2)
+ long double *season // Estimated values for the seasonal component (size entries - t + 2)
+)
+{
+ if(unlikely(entries < 4))
+ return 0;
+
+ int start_time = 2;
+
+ long double res = 0, xhat = 0, stmp = 0;
+ int i, i0, s0;
+
+ /* copy start values to the beginning of the vectors */
+ level[0] = *a;
+ if(beta > 0) trend[0] = *b;
+ if(gamma > 0) memcpy(season, s, *period * sizeof(long double));
+
+ for(i = start_time - 1; i < entries; i++) {
+ /* indices for period i */
+ i0 = i - start_time + 2;
+ s0 = i0 + *period - 1;
+
+ /* forecast *for* period i */
+ xhat = level[i0 - 1] + (beta > 0 ? trend[i0 - 1] : 0);
+ stmp = gamma > 0 ? season[s0 - *period] : (*seasonal != 1);
+ if (*seasonal == 1)
+ xhat += stmp;
+ else
+ xhat *= stmp;
+
+ /* Sum of Squared Errors */
+ res = series[i] - xhat;
+ *SSE += res * res;
+
+ /* estimate of level *in* period i */
+ if (*seasonal == 1)
+ level[i0] = alpha * (series[i] - stmp)
+ + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]);
+ else
+ level[i0] = alpha * (series[i] / stmp)
+ + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]);
+
+ /* estimate of trend *in* period i */
+ if (beta > 0)
+ trend[i0] = beta * (level[i0] - level[i0 - 1])
+ + (1 - beta) * trend[i0 - 1];
+
+ /* estimate of seasonal component *in* period i */
+ if (gamma > 0) {
+ if (*seasonal == 1)
+ season[s0] = gamma * (series[i] - level[i0])
+ + (1 - gamma) * stmp;
+ else
+ season[s0] = gamma * (series[i] / level[i0])
+ + (1 - gamma) * stmp;
+ }
+ }
+
+ return 1;
+}
+
+long double holtwinters(long double *series, size_t entries, long double alpha, long double beta, long double gamma, long double *forecast) {
+ if(unlikely(isnan(alpha)))
+ alpha = 0.3;
+
+ if(unlikely(isnan(beta)))
+ beta = 0.05;
+
+ if(unlikely(isnan(gamma)))
+ gamma = 0;
+
+ int seasonal = 0;
+ int period = 0;
+ long double a0 = series[0];
+ long double b0 = 0;
+ long double s[] = {};
+
+ long double errors = 0.0;
+ size_t nb_computations = entries;
+ long double *estimated_level = callocz(nb_computations, sizeof(long double));
+ long double *estimated_trend = callocz(nb_computations, sizeof(long double));
+ long double *estimated_season = callocz(nb_computations, sizeof(long double));
+
+ int ret = __HoltWinters(
+ series,
+ (int)entries,
+ alpha,
+ beta,
+ gamma,
+ &seasonal,
+ &period,
+ &a0,
+ &b0,
+ s,
+ &errors,
+ estimated_level,
+ estimated_trend,
+ estimated_season
+ );
+
+ long double value = estimated_level[nb_computations - 1];
+
+ if(forecast)
+ *forecast = 0.0;
+
+ freez(estimated_level);
+ freez(estimated_trend);
+ freez(estimated_season);
+
+ if(!ret)
+ return 0.0;
+
+ return value;
+}
diff --git a/src/statistical.h b/src/statistical.h
new file mode 100644
index 000000000..844e579bb
--- /dev/null
+++ b/src/statistical.h
@@ -0,0 +1,19 @@
+#ifndef NETDATA_STATISTICAL_H
+#define NETDATA_STATISTICAL_H
+
+extern long double average(long double *series, size_t entries);
+extern long double moving_average(long double *series, size_t entries, size_t period);
+extern long double median(long double *series, size_t entries);
+extern long double moving_median(long double *series, size_t entries, size_t period);
+extern long double running_median_estimate(long double *series, size_t entries);
+extern long double standard_deviation(long double *series, size_t entries);
+extern long double single_exponential_smoothing(long double *series, size_t entries, long double alpha);
+extern long double double_exponential_smoothing(long double *series, size_t entries, long double alpha, long double beta, long double *forecast);
+extern long double holtwinters(long double *series, size_t entries, long double alpha, long double beta, long double gamma, long double *forecast);
+extern long double sum_and_count(long double *series, size_t entries, size_t *count);
+extern long double sum(long double *series, size_t entries);
+extern long double median_on_sorted_series(long double *series, size_t entries);
+extern long double *copy_series(long double *series, size_t entries);
+extern void sort_series(long double *series, size_t entries);
+
+#endif //NETDATA_STATISTICAL_H
diff --git a/src/statsd.c b/src/statsd.c
new file mode 100644
index 000000000..4dd04757b
--- /dev/null
+++ b/src/statsd.c
@@ -0,0 +1,2041 @@
+#include "common.h"
+
+#define STATSD_CHART_PREFIX "statsd"
+#define STATSD_CHART_PRIORITY 90000
+
+// --------------------------------------------------------------------------------------
+
+// #define STATSD_MULTITHREADED 1
+
+#ifdef STATSD_MULTITHREADED
+// DO NOT ENABLE MULTITHREADING - IT IS NOT WELL TESTED
+#define STATSD_AVL_TREE avl_tree_lock
+#define STATSD_AVL_INSERT avl_insert_lock
+#define STATSD_AVL_SEARCH avl_search_lock
+#define STATSD_AVL_INDEX_INIT { .avl_tree = { NULL, statsd_metric_compare }, .rwlock = AVL_LOCK_INITIALIZER }
+#define STATSD_FIRST_PTR_MUTEX netdata_mutex_t first_mutex
+#define STATSD_FIRST_PTR_MUTEX_INIT .first_mutex = NETDATA_MUTEX_INITIALIZER
+#define STATSD_FIRST_PTR_MUTEX_LOCK(index) netdata_mutex_lock(&((index)->first_mutex))
+#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index) netdata_mutex_unlock(&((index)->first_mutex))
+#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DEFAULT
+#else
+#define STATSD_AVL_TREE avl_tree
+#define STATSD_AVL_INSERT avl_insert
+#define STATSD_AVL_SEARCH avl_search
+#define STATSD_AVL_INDEX_INIT { .root = NULL, .compar = statsd_metric_compare }
+#define STATSD_FIRST_PTR_MUTEX
+#define STATSD_FIRST_PTR_MUTEX_INIT
+#define STATSD_FIRST_PTR_MUTEX_LOCK(index)
+#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index)
+#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_SINGLE_THREADED
+#endif
+
+#define STATSD_DECIMAL_DETAIL 1000 // floating point values get multiplied by this, with the same divisor
+
+// --------------------------------------------------------------------------------------------------------------------
+// data specific to each metric type
+
+typedef struct statsd_metric_gauge {
+ long double value;
+} STATSD_METRIC_GAUGE;
+
+typedef struct statsd_metric_counter { // counter and meter
+ long long value;
+} STATSD_METRIC_COUNTER;
+
+typedef struct statsd_histogram_extensions {
+ netdata_mutex_t mutex;
+
+ // average is stored in metric->last
+ collected_number last_min;
+ collected_number last_max;
+ collected_number last_percentile;
+ collected_number last_median;
+ collected_number last_stddev;
+ collected_number last_sum;
+
+ RRDDIM *rd_min;
+ RRDDIM *rd_max;
+ RRDDIM *rd_percentile;
+ RRDDIM *rd_median;
+ RRDDIM *rd_stddev;
+ RRDDIM *rd_sum;
+
+ size_t size;
+ size_t used;
+ long double *values; // dynamic array of values collected
+} STATSD_METRIC_HISTOGRAM_EXTENSIONS;
+
+typedef struct statsd_metric_histogram { // histogram and timer
+ STATSD_METRIC_HISTOGRAM_EXTENSIONS *ext;
+} STATSD_METRIC_HISTOGRAM;
+
+typedef struct statsd_metric_set {
+ DICTIONARY *dict;
+ size_t unique;
+} STATSD_METRIC_SET;
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// this is a metric - for all types of metrics
+
+typedef enum statsd_metric_options {
+ STATSD_METRIC_OPTION_NONE = 0x00000000, // no options set
+ STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED = 0x00000001, // do not update the chart dimension, when this metric is not collected
+ STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED = 0x00000002, // render a private chart for this metric
+ STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED = 0x00000004, // the metric has been checked if it should get private chart or not
+ STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT = 0x00000008, // show the count of events for this private chart
+ STATSD_METRIC_OPTION_CHECKED_IN_APPS = 0x00000010, // set when this metric has been checked agains apps
+} STATS_METRIC_OPTIONS;
+
+typedef enum statsd_metric_type {
+ STATSD_METRIC_TYPE_GAUGE,
+ STATSD_METRIC_TYPE_COUNTER,
+ STATSD_METRIC_TYPE_METER,
+ STATSD_METRIC_TYPE_TIMER,
+ STATSD_METRIC_TYPE_HISTOGRAM,
+ STATSD_METRIC_TYPE_SET
+} STATSD_METRIC_TYPE;
+
+
+typedef struct statsd_metric {
+ avl avl; // indexing
+
+ const char *name; // the name of the metric
+ uint32_t hash; // hash of the name
+
+ STATSD_METRIC_TYPE type;
+
+ // metadata about data collection
+ collected_number events; // the number of times this metric has been collected (never resets)
+ size_t count; // the number of times this metric has been collected since the last flush
+
+ // the actual collected data
+ union {
+ STATSD_METRIC_GAUGE gauge;
+ STATSD_METRIC_COUNTER counter;
+ STATSD_METRIC_HISTOGRAM histogram;
+ STATSD_METRIC_SET set;
+ };
+
+ // chart related members
+ STATS_METRIC_OPTIONS options; // STATSD_METRIC_OPTION_* (bitfield)
+ char reset; // set to 1 to reset this metric to zero
+ collected_number last; // the last value sent to netdata
+ RRDSET *st; // the chart of this metric
+ RRDDIM *rd_value; // the dimension of this metric value
+ RRDDIM *rd_count; // the dimension for the number of events received
+
+ // linking, used for walking through all metrics
+ struct statsd_metric *next;
+} STATSD_METRIC;
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// each type of metric has its own index
+
+typedef struct statsd_index {
+ char *name; // the name of the index of metrics
+ size_t events; // the number of events processed for this index
+ size_t metrics; // the number of metrics in this index
+
+ STATSD_AVL_TREE index; // the AVL tree
+
+ STATSD_METRIC *first; // the linked list of metrics (new metrics are added in front)
+ STATSD_FIRST_PTR_MUTEX; // when mutli-threading is enabled, a lock to protect the linked list
+
+ STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index
+} STATSD_INDEX;
+
+static int statsd_metric_compare(void* a, void* b);
+
+// --------------------------------------------------------------------------------------------------------------------
+// synthetic charts
+
+typedef enum statsd_app_chart_dimension_value_type {
+ STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_LAST,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_SUM,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_MIN,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_MAX,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN,
+ STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV
+} STATSD_APP_CHART_DIM_VALUE_TYPE;
+
+typedef struct statsd_app_chart_dimension {
+ const char *name;
+ const char *metric;
+ uint32_t metric_hash;
+ collected_number multiplier;
+ collected_number divisor;
+ STATSD_APP_CHART_DIM_VALUE_TYPE value_type;
+
+ RRDDIM *rd;
+ collected_number *value_ptr;
+ RRD_ALGORITHM algorithm;
+
+ struct statsd_app_chart_dimension *next;
+} STATSD_APP_CHART_DIM;
+
+typedef struct statsd_app_chart {
+ const char *source;
+ const char *id;
+ const char *name;
+ const char *title;
+ const char *family;
+ const char *context;
+ const char *units;
+ long priority;
+ RRDSET_TYPE chart_type;
+ STATSD_APP_CHART_DIM *dimensions;
+ size_t dimensions_count;
+ size_t dimensions_linked_count;
+
+ RRDSET *st;
+ struct statsd_app_chart *next;
+} STATSD_APP_CHART;
+
+typedef struct statsd_app {
+ const char *name;
+ SIMPLE_PATTERN *metrics;
+ STATS_METRIC_OPTIONS default_options;
+ RRD_MEMORY_MODE rrd_memory_mode;
+ long rrd_history_entries;
+
+ const char *source;
+ STATSD_APP_CHART *charts;
+ struct statsd_app *next;
+} STATSD_APP;
+
+// --------------------------------------------------------------------------------------------------------------------
+// global statsd data
+
+static struct statsd {
+ STATSD_INDEX gauges;
+ STATSD_INDEX counters;
+ STATSD_INDEX timers;
+ STATSD_INDEX histograms;
+ STATSD_INDEX meters;
+ STATSD_INDEX sets;
+ size_t unknown_types;
+ size_t socket_errors;
+ size_t tcp_socket_reads;
+ size_t tcp_packets_received;
+ size_t tcp_bytes_read;
+ size_t udp_socket_reads;
+ size_t udp_packets_received;
+ size_t udp_bytes_read;
+
+ int enabled;
+ int update_every;
+ SIMPLE_PATTERN *charts_for;
+
+ size_t private_charts;
+ size_t max_private_charts;
+ size_t max_private_charts_hard;
+ RRD_MEMORY_MODE private_charts_memory_mode;
+ long private_charts_rrd_history_entries;
+
+ STATSD_APP *apps;
+ size_t recvmmsg_size;
+ size_t histogram_increase_step;
+ double histogram_percentile;
+ char *histogram_percentile_str;
+ int threads;
+ LISTEN_SOCKETS sockets;
+} statsd = {
+ .enabled = 1,
+ .max_private_charts = 200,
+ .max_private_charts_hard = 1000,
+ .recvmmsg_size = 10,
+
+ .gauges = {
+ .name = "gauge",
+ .events = 0,
+ .metrics = 0,
+ .index = STATSD_AVL_INDEX_INIT,
+ .default_options = STATSD_METRIC_OPTION_NONE,
+ .first = NULL,
+ STATSD_FIRST_PTR_MUTEX_INIT
+ },
+ .counters = {
+ .name = "counter",
+ .events = 0,
+ .metrics = 0,
+ .index = STATSD_AVL_INDEX_INIT,
+ .default_options = STATSD_METRIC_OPTION_NONE,
+ .first = NULL,
+ STATSD_FIRST_PTR_MUTEX_INIT
+ },
+ .timers = {
+ .name = "timer",
+ .events = 0,
+ .metrics = 0,
+ .index = STATSD_AVL_INDEX_INIT,
+ .default_options = STATSD_METRIC_OPTION_NONE,
+ .first = NULL,
+ STATSD_FIRST_PTR_MUTEX_INIT
+ },
+ .histograms = {
+ .name = "histogram",
+ .events = 0,
+ .metrics = 0,
+ .index = STATSD_AVL_INDEX_INIT,
+ .default_options = STATSD_METRIC_OPTION_NONE,
+ .first = NULL,
+ STATSD_FIRST_PTR_MUTEX_INIT
+ },
+ .meters = {
+ .name = "meter",
+ .events = 0,
+ .metrics = 0,
+ .index = STATSD_AVL_INDEX_INIT,
+ .default_options = STATSD_METRIC_OPTION_NONE,
+ .first = NULL,
+ STATSD_FIRST_PTR_MUTEX_INIT
+ },
+ .sets = {
+ .name = "set",
+ .events = 0,
+ .metrics = 0,
+ .index = STATSD_AVL_INDEX_INIT,
+ .default_options = STATSD_METRIC_OPTION_NONE,
+ .first = NULL,
+ STATSD_FIRST_PTR_MUTEX_INIT
+ },
+
+ .apps = NULL,
+ .histogram_percentile = 95.0,
+ .histogram_increase_step = 10,
+ .threads = 0,
+ .sockets = {
+ .config_section = CONFIG_SECTION_STATSD,
+ .default_bind_to = "udp:localhost tcp:localhost",
+ .default_port = STATSD_LISTEN_PORT,
+ .backlog = STATSD_LISTEN_BACKLOG
+ },
+};
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd index management - add/find metrics
+
+static int statsd_metric_compare(void* a, void* b) {
+ if(((STATSD_METRIC *)a)->hash < ((STATSD_METRIC *)b)->hash) return -1;
+ else if(((STATSD_METRIC *)a)->hash > ((STATSD_METRIC *)b)->hash) return 1;
+ else return strcmp(((STATSD_METRIC *)a)->name, ((STATSD_METRIC *)b)->name);
+}
+
+static inline STATSD_METRIC *stasd_metric_index_find(STATSD_INDEX *index, const char *name, uint32_t hash) {
+ STATSD_METRIC tmp;
+ tmp.name = name;
+ tmp.hash = (hash)?hash:simple_hash(tmp.name);
+
+ return (STATSD_METRIC *)STATSD_AVL_SEARCH(&index->index, (avl *)&tmp);
+}
+
+static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, const char *name, STATSD_METRIC_TYPE type) {
+ debug(D_STATSD, "searching for metric '%s' under '%s'", name, index->name);
+
+ uint32_t hash = simple_hash(name);
+
+ STATSD_METRIC *m = stasd_metric_index_find(index, name, hash);
+ if(unlikely(!m)) {
+ debug(D_STATSD, "Creating new %s metric '%s'", index->name, name);
+
+ m = (STATSD_METRIC *)callocz(sizeof(STATSD_METRIC), 1);
+ m->name = strdupz(name);
+ m->hash = hash;
+ m->type = type;
+ m->options = index->default_options;
+
+ if(type == STATSD_METRIC_TYPE_HISTOGRAM || type == STATSD_METRIC_TYPE_TIMER) {
+ m->histogram.ext = callocz(sizeof(STATSD_METRIC_HISTOGRAM_EXTENSIONS), 1);
+ netdata_mutex_init(&m->histogram.ext->mutex);
+ }
+ STATSD_METRIC *n = (STATSD_METRIC *)STATSD_AVL_INSERT(&index->index, (avl *)m);
+ if(unlikely(n != m)) {
+ freez((void *)m->histogram.ext);
+ freez((void *)m->name);
+ freez((void *)m);
+ m = n;
+ }
+ else {
+ STATSD_FIRST_PTR_MUTEX_LOCK(index);
+ index->metrics++;
+ m->next = index->first;
+ index->first = m;
+ STATSD_FIRST_PTR_MUTEX_UNLOCK(index);
+ }
+ }
+
+ index->events++;
+ return m;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd parsing numbers
+
+static inline long double statsd_parse_float(const char *v, long double def) {
+ long double value;
+
+ if(likely(v && *v)) {
+ char *e = NULL;
+ value = str2ld(v, &e);
+ if(unlikely(e && *e))
+ error("STATSD: excess data '%s' after value '%s'", e, v);
+ }
+ else
+ value = def;
+
+ return value;
+}
+
+static inline long long statsd_parse_int(const char *v, long long def) {
+ long long value;
+
+ if(likely(v && *v)) {
+ char *e = NULL;
+ value = str2ll(v, &e);
+ if(unlikely(e && *e))
+ error("STATSD: excess data '%s' after value '%s'", e, v);
+ }
+ else
+ value = def;
+
+ return value;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd processors per metric type
+
+static inline void statsd_reset_metric(STATSD_METRIC *m) {
+ m->reset = 0;
+ m->count = 0;
+}
+
+static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, const char *sampling) {
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric '%s' of type gauge, with empty value is ignored.", m->name);
+ return;
+ }
+
+ if(unlikely(m->reset)) {
+ // no need to reset anything specific for gauges
+ statsd_reset_metric(m);
+ }
+
+ if(unlikely(*value == '+' || *value == '-'))
+ m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+ else
+ m->gauge.value = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+
+ m->events++;
+ m->count++;
+}
+
+static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, const char *sampling) {
+ // we accept empty values for counters
+
+ if(unlikely(m->reset)) statsd_reset_metric(m);
+
+ m->counter.value += roundl((long double)statsd_parse_int(value, 1) / statsd_parse_float(sampling, 1.0));
+
+ m->events++;
+ m->count++;
+}
+
+static inline void statsd_process_meter(STATSD_METRIC *m, const char *value, const char *sampling) {
+ // this is the same with the counter
+ statsd_process_counter(m, value, sampling);
+}
+
+static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value, const char *sampling) {
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric '%s' of type histogram, with empty value is ignored.", m->name);
+ return;
+ }
+
+ if(unlikely(m->reset)) {
+ m->histogram.ext->used = 0;
+ statsd_reset_metric(m);
+ }
+
+ if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
+ netdata_mutex_lock(&m->histogram.ext->mutex);
+ m->histogram.ext->size += statsd.histogram_increase_step;
+ m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(long double) * m->histogram.ext->size);
+ netdata_mutex_unlock(&m->histogram.ext->mutex);
+ }
+
+ m->histogram.ext->values[m->histogram.ext->used++] = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+
+ m->events++;
+ m->count++;
+}
+
+static inline void statsd_process_timer(STATSD_METRIC *m, const char *value, const char *sampling) {
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric of type set, with empty value is ignored.");
+ return;
+ }
+
+ // timers are a use case of histogram
+ statsd_process_histogram(m, value, sampling);
+}
+
+static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
+ if(unlikely(!value || !*value)) {
+ error("STATSD: metric of type set, with empty value is ignored.");
+ return;
+ }
+
+ if(unlikely(m->reset)) {
+ if(likely(m->set.dict)) {
+ dictionary_destroy(m->set.dict);
+ m->set.dict = NULL;
+ }
+ statsd_reset_metric(m);
+ }
+
+ if(unlikely(!m->set.dict)) {
+ m->set.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
+ m->set.unique = 0;
+ }
+
+ void *t = dictionary_get(m->set.dict, value);
+ if(unlikely(!t)) {
+ dictionary_set(m->set.dict, value, NULL, 1);
+ m->set.unique++;
+ }
+
+ m->events++;
+ m->count++;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd parsing
+
+static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling) {
+ debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', rate '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)");
+
+ if(unlikely(!name || !*name)) return;
+ if(unlikely(!type || !*type)) type = "m";
+
+ char t0 = type[0], t1 = type[1];
+
+ if(unlikely(t0 == 'g' && t1 == '\0')) {
+ statsd_process_gauge(
+ statsd_find_or_add_metric(&statsd.gauges, name, STATSD_METRIC_TYPE_GAUGE),
+ value, sampling);
+ }
+ else if(unlikely((t0 == 'c' || t0 == 'C') && t1 == '\0')) {
+ // etsy/statsd uses 'c'
+ // brubeck uses 'C'
+ statsd_process_counter(
+ statsd_find_or_add_metric(&statsd.counters, name, STATSD_METRIC_TYPE_COUNTER),
+ value, sampling);
+ }
+ else if(unlikely(t0 == 'm' && t1 == '\0')) {
+ statsd_process_meter(
+ statsd_find_or_add_metric(&statsd.meters, name, STATSD_METRIC_TYPE_METER),
+ value, sampling);
+ }
+ else if(unlikely(t0 == 'h' && t1 == '\0')) {
+ statsd_process_histogram(
+ statsd_find_or_add_metric(&statsd.histograms, name, STATSD_METRIC_TYPE_HISTOGRAM),
+ value, sampling);
+ }
+ else if(unlikely(t0 == 's' && t1 == '\0')) {
+ statsd_process_set(
+ statsd_find_or_add_metric(&statsd.sets, name, STATSD_METRIC_TYPE_SET),
+ value);
+ }
+ else if(unlikely(t0 == 'm' && t1 == 's' && type[2] == '\0')) {
+ statsd_process_timer(
+ statsd_find_or_add_metric(&statsd.timers, name, STATSD_METRIC_TYPE_TIMER),
+ value, sampling);
+ }
+ else {
+ statsd.unknown_types++;
+ error("STATSD: metric '%s' with value '%s' is sent with unknown metric type '%s'", name, value?value:"", type);
+ }
+}
+
+static inline const char *statsd_parse_skip_up_to(const char *s, char d1, char d2) {
+ char c;
+
+ for(c = *s; c && c != d1 && c != d2 && c != '\r' && c != '\n'; c = *++s) ;
+
+ return s;
+}
+
+const char *statsd_parse_skip_spaces(const char *s) {
+ char c;
+
+ for(c = *s; c && ( c == ' ' || c == '\t' || c == '\r' || c == '\n' ); c = *++s) ;
+
+ return s;
+}
+
+static inline const char *statsd_parse_field_trim(const char *start, char *end) {
+ if(unlikely(!start)) {
+ start = end;
+ return start;
+ }
+
+ while(start <= end && (*start == ' ' || *start == '\t'))
+ start++;
+
+ *end = '\0';
+ end--;
+ while(end >= start && (*end == ' ' || *end == '\t'))
+ *end-- = '\0';
+
+ return start;
+}
+
+static inline size_t statsd_process(char *buffer, size_t size, int require_newlines) {
+ buffer[size] = '\0';
+ debug(D_STATSD, "RECEIVED: %zu bytes: '%s'", size, buffer);
+
+ const char *s = buffer;
+ while(*s) {
+ const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL;
+ char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL;
+
+ s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '|');
+ if(name == name_end) {
+ s = statsd_parse_skip_spaces(s);
+ continue;
+ }
+
+ if(likely(*s == ':'))
+ s = value_end = (char *) statsd_parse_skip_up_to(value = ++s, '|', '|');
+
+ if(likely(*s == '|'))
+ s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@');
+
+ if(likely(*s == '|' || *s == '@')) {
+ s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '\r', '\n');
+ if(*sampling == '@') sampling++;
+ }
+
+ // skip everything until the end of the line
+ while(*s && *s != '\n') s++;
+
+ if(unlikely(require_newlines && *s != '\n' && s > buffer)) {
+ // move the remaining data to the beginning
+ size -= (name - buffer);
+ memmove(buffer, name, size);
+ return size;
+ }
+ else
+ s = statsd_parse_skip_spaces(s);
+
+ statsd_process_metric(
+ statsd_parse_field_trim(name, name_end)
+ , statsd_parse_field_trim(value, value_end)
+ , statsd_parse_field_trim(type, type_end)
+ , statsd_parse_field_trim(sampling, sampling_end)
+ );
+ }
+
+ return 0;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd pollfd interface
+
+#define STATSD_TCP_BUFFER_SIZE 65536 // minimize tcp reads
+#define STATSD_UDP_BUFFER_SIZE 9000 // this should be up to MTU
+
+typedef enum {
+ STATSD_SOCKET_DATA_TYPE_TCP,
+ STATSD_SOCKET_DATA_TYPE_UDP
+} STATSD_SOCKET_DATA_TYPE;
+
+struct statsd_tcp {
+ STATSD_SOCKET_DATA_TYPE type;
+ size_t size;
+ size_t len;
+ char buffer[];
+};
+
+#ifdef HAVE_RECVMMSG
+struct statsd_udp {
+ STATSD_SOCKET_DATA_TYPE type;
+ size_t size;
+ struct iovec *iovecs;
+ struct mmsghdr *msgs;
+};
+#else
+struct statsd_udp {
+ STATSD_SOCKET_DATA_TYPE type;
+ char buffer[STATSD_UDP_BUFFER_SIZE];
+};
+#endif
+
+// new TCP client connected
+static void *statsd_add_callback(int fd, short int *events) {
+ (void)fd;
+ *events = POLLIN;
+
+ struct statsd_tcp *data = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1);
+ data->type = STATSD_SOCKET_DATA_TYPE_TCP;
+ data->size = STATSD_TCP_BUFFER_SIZE - 1;
+
+ return data;
+}
+
+// TCP client disconnected
+static void statsd_del_callback(int fd, void *data) {
+ (void)fd;
+
+ if(data) {
+ struct statsd_tcp *t = data;
+ if(t->type == STATSD_SOCKET_DATA_TYPE_TCP) {
+ if(t->len != 0) {
+ statsd.socket_errors++;
+ error("STATSD: client is probably sending unterminated metrics. Closed socket left with '%s'. Trying to process it.", t->buffer);
+ statsd_process(t->buffer, t->len, 0);
+ }
+ }
+ else
+ error("STATSD: internal error: received socket data type is %d, but expected %d", (int)t->type, (int)STATSD_SOCKET_DATA_TYPE_TCP);
+
+ freez(data);
+ }
+
+ return;
+}
+
+// Receive data
+static int statsd_rcv_callback(int fd, int socktype, void *data, short int *events) {
+ *events = POLLIN;
+
+ switch(socktype) {
+ case SOCK_STREAM: {
+ struct statsd_tcp *d = (struct statsd_tcp *)data;
+ if(unlikely(!d)) {
+ error("STATSD: internal error: expected TCP data pointer is NULL");
+ statsd.socket_errors++;
+ return -1;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_TCP)) {
+ error("STATSD: internal error: socket data type should be %d, but it is %d", (int)STATSD_SOCKET_DATA_TYPE_TCP, (int)d->type);
+ statsd.socket_errors++;
+ return -1;
+ }
+#endif
+
+ int ret = 0;
+ ssize_t rc;
+ do {
+ rc = recv(fd, &d->buffer[d->len], d->size - d->len, MSG_DONTWAIT);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
+ error("STATSD: recv() on TCP socket %d failed.", fd);
+ statsd.socket_errors++;
+ ret = -1;
+ }
+ }
+ else if (!rc) {
+ // connection closed
+ debug(D_STATSD, "STATSD: client disconnected.");
+ ret = -1;
+ }
+ else {
+ // data received
+ d->len += rc;
+ statsd.tcp_socket_reads++;
+ statsd.tcp_bytes_read += rc;
+ }
+
+ if(likely(d->len > 0)) {
+ statsd.tcp_packets_received++;
+ d->len = statsd_process(d->buffer, d->len, 1);
+ }
+
+ if(unlikely(ret == -1))
+ return -1;
+
+ } while (rc != -1);
+ break;
+ }
+
+ case SOCK_DGRAM: {
+ struct statsd_udp *d = (struct statsd_udp *)data;
+ if(unlikely(!d)) {
+ error("STATSD: internal error: expected UDP data pointer is NULL");
+ statsd.socket_errors++;
+ return -1;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) {
+ error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_UDP);
+ statsd.socket_errors++;
+ return -1;
+ }
+#endif
+
+#ifdef HAVE_RECVMMSG
+ ssize_t rc;
+ do {
+ rc = recvmmsg(fd, d->msgs, (unsigned int)d->size, MSG_DONTWAIT, NULL);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
+ error("STATSD: recvmmsg() on UDP socket %d failed.", fd);
+ statsd.socket_errors++;
+ return -1;
+ }
+ } else if (rc) {
+ // data received
+ statsd.udp_socket_reads++;
+ statsd.udp_packets_received += rc;
+
+ size_t i;
+ for (i = 0; i < (size_t)rc; ++i) {
+ size_t len = (size_t)d->msgs[i].msg_len;
+ statsd.udp_bytes_read += len;
+ statsd_process(d->msgs[i].msg_hdr.msg_iov->iov_base, len, 0);
+ }
+ }
+ } while (rc != -1);
+
+#else // !HAVE_RECVMMSG
+ ssize_t rc;
+ do {
+ rc = recv(fd, d->buffer, STATSD_UDP_BUFFER_SIZE - 1, MSG_DONTWAIT);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
+ error("STATSD: recv() on UDP socket %d failed.", fd);
+ statsd.socket_errors++;
+ return -1;
+ }
+ } else if (rc) {
+ // data received
+ statsd.udp_socket_reads++;
+ statsd.udp_packets_received++;
+ statsd.udp_bytes_read += rc;
+ statsd_process(d->buffer, (size_t) rc, 0);
+ }
+ } while (rc != -1);
+#endif
+
+ break;
+ }
+
+ default: {
+ error("STATSD: internal error: unknown socktype %d on socket %d", socktype, fd);
+ statsd.socket_errors++;
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static int statsd_snd_callback(int fd, int socktype, void *data, short int *events) {
+ (void)fd;
+ (void)socktype;
+ (void)data;
+ (void)events;
+
+ error("STATSD: snd_callback() called, but we never requested to send data to statsd clients.");
+ return -1;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd child thread to collect metrics from network
+
+void statsd_collector_thread_cleanup(void *data) {
+ struct statsd_udp *d = data;
+
+#ifdef HAVE_RECVMMSG
+ size_t i;
+ for (i = 0; i < d->size; i++)
+ freez(d->iovecs[i].iov_base);
+
+ freez(d->iovecs);
+ freez(d->msgs);
+#endif
+
+ freez(d);
+}
+
+void *statsd_collector_thread(void *ptr) {
+ int id = *((int *)ptr);
+
+ info("STATSD collector thread No %d created with task id %d", id + 1, gettid());
+
+ if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
+ error("Cannot set pthread cancel type to DEFERRED.");
+
+ if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
+ error("Cannot set pthread cancel state to ENABLE.");
+
+ struct statsd_udp *d = callocz(sizeof(struct statsd_udp), 1);
+ pthread_cleanup_push(statsd_collector_thread_cleanup, d);
+
+#ifdef HAVE_RECVMMSG
+ d->type = STATSD_SOCKET_DATA_TYPE_UDP;
+ d->size = statsd.recvmmsg_size;
+ d->iovecs = callocz(sizeof(struct iovec), d->size);
+ d->msgs = callocz(sizeof(struct mmsghdr), d->size);
+
+ size_t i;
+ for (i = 0; i < d->size; i++) {
+ d->iovecs[i].iov_base = mallocz(STATSD_UDP_BUFFER_SIZE);
+ d->iovecs[i].iov_len = STATSD_UDP_BUFFER_SIZE - 1;
+ d->msgs[i].msg_hdr.msg_iov = &d->iovecs[i];
+ d->msgs[i].msg_hdr.msg_iovlen = 1;
+ }
+#endif
+
+ poll_events(&statsd.sockets
+ , statsd_add_callback
+ , statsd_del_callback
+ , statsd_rcv_callback
+ , statsd_snd_callback
+ , (void *)d
+ );
+
+ pthread_cleanup_pop(1);
+
+ debug(D_WEB_CLIENT, "STATSD: exit!");
+ pthread_exit(NULL);
+ return NULL;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd applications configuration files parsing
+
+#define STATSD_CONF_LINE_MAX 8192
+
+int statsd_readfile(const char *path, const char *filename) {
+ debug(D_STATSD, "STATSD configuration reading file '%s/%s'", path, filename);
+
+ char buffer[STATSD_CONF_LINE_MAX + 1];
+
+ FILE *fp = NULL;
+ snprintfz(buffer, STATSD_CONF_LINE_MAX, "%s/%s", path, filename);
+ fp = fopen(buffer, "r");
+ if(!fp) {
+ error("STATSD: cannot open file '%s'.", buffer);
+ return -1;
+ }
+
+ STATSD_APP *app = NULL;
+ STATSD_APP_CHART *chart = NULL;
+
+ size_t line = 0;
+ char *s;
+ while(fgets(buffer, STATSD_CONF_LINE_MAX, fp) != NULL) {
+ buffer[STATSD_CONF_LINE_MAX] = '\0';
+ line++;
+
+ s = trim(buffer);
+ if (!s || *s == '#') {
+ debug(D_STATSD, "STATSD: ignoring line %zu of file '%s/%s', it is empty.", line, path, filename);
+ continue;
+ }
+ debug(D_STATSD, "STATSD: processing line %zu of file '%s/%s': %s", line, path, filename, buffer);
+
+ int len = (int) strlen(s);
+ if (*s == '[' && s[len - 1] == ']') {
+ // new section
+ s[len - 1] = '\0';
+ s++;
+
+ if (!strcmp(s, "app")) {
+ // a new app
+ app = callocz(sizeof(STATSD_APP), 1);
+ app->name = strdupz("unnamed");
+ app->rrd_memory_mode = localhost->rrd_memory_mode;
+ app->rrd_history_entries = localhost->rrd_history_entries;
+
+ app->next = statsd.apps;
+ statsd.apps = app;
+ chart = NULL;
+ }
+ else if(app) {
+ // a new chart
+ chart = callocz(sizeof(STATSD_APP_CHART), 1);
+ netdata_fix_chart_id(s);
+ chart->id = strdupz(s);
+ chart->name = strdupz(s);
+ chart->title = strdupz("Statsd chart");
+ chart->context = strdupz(s);
+ chart->family = strdupz("overview");
+ chart->units = strdupz("value");
+ chart->priority = STATSD_CHART_PRIORITY;
+ chart->chart_type = RRDSET_TYPE_LINE;
+
+ chart->next = app->charts;
+ app->charts = chart;
+ }
+ else
+ error("STATSD: ignoring line %zu ('%s') of file '%s/%s', [app] is not defined.", line, s, path, filename);
+
+ continue;
+ }
+
+ if(!app) {
+ error("STATSD: ignoring line %zu ('%s') of file '%s/%s', it is outside all sections.", line, s, path, filename);
+ continue;
+ }
+
+ char *name = s;
+ char *value = strchr(s, '=');
+ if(!value) {
+ error("STATSD: ignoring line %zu ('%s') of file '%s/%s', there is no = in it.", line, s, path, filename);
+ continue;
+ }
+ *value = '\0';
+ value++;
+
+ name = trim(name);
+ value = trim(value);
+
+ if(!name || *name == '#') {
+ error("STATSD: ignoring line %zu of file '%s/%s', name is empty.", line, path, filename);
+ continue;
+ }
+ if(!value) {
+ debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s/%s', value is empty.", line, path, filename);
+ continue;
+ }
+
+ if(!chart) {
+ if(!strcmp(name, "name")) {
+ freez((void *)app->name);
+ netdata_fix_chart_name(value);
+ app->name = strdupz(value);
+ }
+ else if (!strcmp(name, "metrics")) {
+ simple_pattern_free(app->metrics);
+ app->metrics = simple_pattern_create(value, SIMPLE_PATTERN_EXACT);
+ }
+ else if (!strcmp(name, "private charts")) {
+ if (!strcmp(value, "yes") || !strcmp(value, "on"))
+ app->default_options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ else
+ app->default_options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ }
+ else if (!strcmp(name, "gaps when not collected")) {
+ if (!strcmp(value, "yes") || !strcmp(value, "on"))
+ app->default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+ }
+ else if (!strcmp(name, "memory mode")) {
+ app->rrd_memory_mode = rrd_memory_mode_id(value);
+ }
+ else if (!strcmp(name, "history")) {
+ app->rrd_history_entries = atol(value);
+ if (app->rrd_history_entries < 5)
+ app->rrd_history_entries = 5;
+ }
+ else {
+ error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [app] section.", line, name, path, filename);
+ continue;
+ }
+ }
+ else {
+ if(!strcmp(name, "name")) {
+ freez((void *)chart->name);
+ netdata_fix_chart_id(value);
+ chart->name = strdupz(value);
+ }
+ else if(!strcmp(name, "title")) {
+ freez((void *)chart->title);
+ chart->title = strdupz(value);
+ }
+ else if (!strcmp(name, "family")) {
+ freez((void *)chart->family);
+ chart->family = strdupz(value);
+ }
+ else if (!strcmp(name, "context")) {
+ freez((void *)chart->context);
+ netdata_fix_chart_id(value);
+ chart->context = strdupz(value);
+ }
+ else if (!strcmp(name, "units")) {
+ freez((void *)chart->units);
+ chart->units = strdupz(value);
+ }
+ else if (!strcmp(name, "priority")) {
+ chart->priority = atol(value);
+ }
+ else if (!strcmp(name, "type")) {
+ chart->chart_type = rrdset_type_id(value);
+ }
+ else if (!strcmp(name, "dimension")) {
+ // metric [name [type [multiplier [divisor]]]]
+ char *words[5];
+ pluginsd_split_words(value, words, 5);
+
+ char *metric_name = words[0];
+ char *dim_name = words[1];
+ char *type = words[2];
+ char *multipler = words[3];
+ char *divisor = words[4];
+
+ STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1);
+
+ dim->metric = strdupz(metric_name);
+ dim->metric_hash = simple_hash(dim->metric);
+
+ dim->name = strdupz((dim_name && *dim_name)?dim_name:metric_name);
+ dim->multiplier = (multipler && *multipler)?str2l(multipler):1;
+ dim->divisor = (divisor && *divisor)?str2l(divisor):1;
+
+ if(!type || !*type) type = "last";
+ if(!strcmp(type, "events")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS;
+ else if(!strcmp(type, "last")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
+ else if(!strcmp(type, "min")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MIN;
+ else if(!strcmp(type, "max")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MAX;
+ else if(!strcmp(type, "sum")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_SUM;
+ else if(!strcmp(type, "average")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE;
+ else if(!strcmp(type, "median")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN;
+ else if(!strcmp(type, "stddev")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV;
+ else if(!strcmp(type, "percentile")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE;
+ else {
+ error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename);
+ dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
+ }
+
+ if(!dim->multiplier) {
+ error("STATSD: invalid multiplier value '%s' at line %zu of file '%s/%s'. Using 1.", multipler, line, path, filename);
+ dim->multiplier = 1;
+ }
+ if(!dim->divisor) {
+ error("STATSD: invalid divisor value '%s' at line %zu of file '%s/%s'. Using 1.", divisor, line, path, filename);
+ dim->divisor = 1;
+ }
+
+ // append it to the list of dimension
+ STATSD_APP_CHART_DIM *tdim;
+ for(tdim = chart->dimensions; tdim && tdim->next ; tdim = tdim->next) ;
+ if(!tdim) {
+ dim->next = chart->dimensions;
+ chart->dimensions = dim;
+ }
+ else {
+ dim->next = tdim->next;
+ tdim->next = dim;
+ }
+ chart->dimensions_count++;
+
+ debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT,
+ dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor);
+ }
+ else {
+ error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id);
+ continue;
+ }
+ }
+ }
+
+ fclose(fp);
+ return 0;
+}
+
+static void statsd_readdir(const char *path) {
+ size_t pathlen = strlen(path);
+
+ debug(D_STATSD, "STATSD configuration reading directory '%s'", path);
+
+ DIR *dir = opendir(path);
+ if (!dir) {
+ error("STATSD configuration cannot open directory '%s'.", path);
+ return;
+ }
+
+ struct dirent *de = NULL;
+ while ((de = readdir(dir))) {
+ size_t len = strlen(de->d_name);
+
+ if(de->d_type == DT_DIR
+ && (
+ (de->d_name[0] == '.' && de->d_name[1] == '\0')
+ || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ )) {
+ debug(D_STATSD, "STATSD: ignoring directory '%s'", de->d_name);
+ continue;
+ }
+
+ else if(de->d_type == DT_DIR) {
+ char *s = mallocz(pathlen + strlen(de->d_name) + 2);
+ strcpy(s, path);
+ strcat(s, "/");
+ strcat(s, de->d_name);
+ statsd_readdir(s);
+ freez(s);
+ continue;
+ }
+
+ else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
+ len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
+ statsd_readfile(path, de->d_name);
+ }
+
+ else debug(D_STATSD, "STATSD: ignoring file '%s'", de->d_name);
+ }
+
+ closedir(dir);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// send metrics to netdata - in private charts - called from the main thread
+
+// extract chart type and chart id from metric name
+static inline void statsd_get_metric_type_and_id(STATSD_METRIC *m, char *type, char *id, const char *defid, size_t len) {
+ char *s;
+
+ snprintfz(type, len, "%s_%s_%s", STATSD_CHART_PREFIX, defid, m->name);
+ for(s = type; *s ;s++)
+ if(unlikely(*s == '.')) break;
+
+ if(*s == '.') {
+ *s++ = '\0';
+ strncpyz(id, s, len);
+ }
+ else {
+ strncpyz(id, defid, len);
+ }
+
+ netdata_fix_chart_id(type);
+ netdata_fix_chart_id(id);
+}
+
+static inline RRDSET *statsd_private_rrdset_create(
+ STATSD_METRIC *m
+ , const char *type
+ , const char *id
+ , const char *name
+ , const char *family
+ , const char *context
+ , const char *title
+ , const char *units
+ , long priority
+ , int update_every
+ , RRDSET_TYPE chart_type
+) {
+ RRD_MEMORY_MODE memory_mode = statsd.private_charts_memory_mode;
+ long history = statsd.private_charts_rrd_history_entries;
+
+ if(unlikely(statsd.private_charts >= statsd.max_private_charts)) {
+ debug(D_STATSD, "STATSD: metric '%s' will be charted with memory mode = none, because the maximum number of charts has been reached.", m->name);
+ info("STATSD: metric '%s' will be charted with memory mode = none, because the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts);
+ memory_mode = RRD_MEMORY_MODE_NONE;
+ history = 5;
+ }
+
+ statsd.private_charts++;
+ RRDSET *st = rrdset_create_custom(
+ localhost
+ , type
+ , id
+ , name
+ , family
+ , context
+ , title
+ , units
+ , priority
+ , update_every
+ , chart_type
+ , memory_mode
+ , history
+ );
+ rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
+ // rrdset_flag_set(st, RRDSET_FLAG_DEBUG);
+ return st;
+}
+
+static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
+ debug(D_STATSD, "updating private chart for gauge metric '%s'", m->name);
+
+ if(unlikely(!m->st)) {
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, "gauge", RRD_ID_LENGTH_MAX);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , "gauges" // family (submenu)
+ , m->name // context
+ , m->name // title
+ , "value" // units
+ , STATSD_CHART_PRIORITY
+ , statsd.update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ m->rd_value = rrddim_add(m->st, "gauge", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else rrdset_next(m->st);
+
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) {
+ debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);
+
+ if(unlikely(!m->st)) {
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, dim, RRD_ID_LENGTH_MAX);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , family // family (submenu)
+ , m->name // context
+ , m->name // title
+ , "events/s" // units
+ , STATSD_CHART_PRIORITY
+ , statsd.update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ m->rd_value = rrddim_add(m->st, dim, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else rrdset_next(m->st);
+
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+static inline void statsd_private_chart_set(STATSD_METRIC *m) {
+ debug(D_STATSD, "updating private chart for set metric '%s'", m->name);
+
+ if(unlikely(!m->st)) {
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, "set", RRD_ID_LENGTH_MAX);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , "sets" // family (submenu)
+ , m->name // context
+ , m->name // title
+ , "entries" // units
+ , STATSD_CHART_PRIORITY
+ , statsd.update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ m->rd_value = rrddim_add(m->st, "set", "set size", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else rrdset_next(m->st);
+
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
+ debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);
+
+ if(unlikely(!m->st)) {
+ char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
+ statsd_get_metric_type_and_id(m, type, id, dim, RRD_ID_LENGTH_MAX);
+
+ m->st = statsd_private_rrdset_create(
+ m
+ , type
+ , id
+ , NULL // name
+ , family // family (submenu)
+ , m->name // context
+ , m->name // title
+ , units // units
+ , STATSD_CHART_PRIORITY
+ , statsd.update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ m->histogram.ext->rd_min = rrddim_add(m->st, "min", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_max = rrddim_add(m->st, "max", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+ m->rd_value = rrddim_add(m->st, "average", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_percentile = rrddim_add(m->st, statsd.histogram_percentile_str, NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_median = rrddim_add(m->st, "median", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_stddev = rrddim_add(m->st, "stddev", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+ m->histogram.ext->rd_sum = rrddim_add(m->st, "sum", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE);
+
+ if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+ m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else rrdset_next(m->st);
+
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_min, m->histogram.ext->last_min);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_max, m->histogram.ext->last_max);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_percentile, m->histogram.ext->last_percentile);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_median, m->histogram.ext->last_median);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_stddev, m->histogram.ext->last_stddev);
+ rrddim_set_by_pointer(m->st, m->histogram.ext->rd_sum, m->histogram.ext->last_sum);
+ rrddim_set_by_pointer(m->st, m->rd_value, m->last);
+
+ if(m->rd_count)
+ rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+ rrdset_done(m->st);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// statsd flush metrics
+
+static inline void statsd_flush_gauge(STATSD_METRIC *m) {
+ debug(D_STATSD, "flushing gauge metric '%s'", m->name);
+
+ int updated = 0;
+ if(m->count && !m->reset) {
+ m->last = (collected_number) (m->gauge.value * STATSD_DECIMAL_DETAIL);
+
+ m->reset = 1;
+ updated = 1;
+ }
+
+ if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+ statsd_private_chart_gauge(m);
+}
+
+static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) {
+ debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
+
+ int updated = 0;
+ if(m->count && !m->reset) {
+ m->last = m->counter.value;
+
+ m->reset = 1;
+ updated = 1;
+ }
+
+ if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+ statsd_private_chart_counter_or_meter(m, dim, family);
+}
+
+static inline void statsd_flush_counter(STATSD_METRIC *m) {
+ statsd_flush_counter_or_meter(m, "counter", "counters");
+}
+
+static inline void statsd_flush_meter(STATSD_METRIC *m) {
+ statsd_flush_counter_or_meter(m, "meter", "meters");
+}
+
+static inline void statsd_flush_set(STATSD_METRIC *m) {
+ debug(D_STATSD, "flushing set metric '%s'", m->name);
+
+ int updated = 0;
+ if(m->count && !m->reset) {
+ m->last = (collected_number)m->set.unique;
+
+ m->reset = 1;
+ updated = 1;
+ }
+
+ if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+ statsd_private_chart_set(m);
+}
+
+static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
+ debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
+
+ netdata_mutex_lock(&m->histogram.ext->mutex);
+
+ int updated = 0;
+ if(m->count && !m->reset && m->histogram.ext->used > 0) {
+ size_t len = m->histogram.ext->used;
+ long double *series = m->histogram.ext->values;
+ sort_series(series, len);
+
+ m->histogram.ext->last_min = (collected_number)roundl(series[0] * STATSD_DECIMAL_DETAIL);
+ m->histogram.ext->last_max = (collected_number)roundl(series[len - 1] * STATSD_DECIMAL_DETAIL);
+ m->last = (collected_number)roundl(average(series, len) * STATSD_DECIMAL_DETAIL);
+ m->histogram.ext->last_median = (collected_number)roundl(median_on_sorted_series(series, len) * STATSD_DECIMAL_DETAIL);
+ m->histogram.ext->last_stddev = (collected_number)roundl(standard_deviation(series, len) * STATSD_DECIMAL_DETAIL);
+ m->histogram.ext->last_sum = (collected_number)roundl(sum(series, len) * STATSD_DECIMAL_DETAIL);
+
+ size_t pct_len = (size_t)floor((double)len * statsd.histogram_percentile / 100.0);
+ if(pct_len < 1)
+ m->histogram.ext->last_percentile = (collected_number)(series[0] * STATSD_DECIMAL_DETAIL);
+ else
+ m->histogram.ext->last_percentile = (collected_number)roundl(average(series, pct_len) * STATSD_DECIMAL_DETAIL);
+
+ debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT,
+ dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum);
+
+ m->reset = 1;
+ updated = 1;
+ }
+
+
+ if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+ statsd_private_chart_timer_or_histogram(m, dim, family, units);
+
+ netdata_mutex_unlock(&m->histogram.ext->mutex);
+}
+
+static inline void statsd_flush_timer(STATSD_METRIC *m) {
+ statsd_flush_timer_or_histogram(m, "timer", "timers", "milliseconds");
+}
+
+static inline void statsd_flush_histogram(STATSD_METRIC *m) {
+ statsd_flush_timer_or_histogram(m, "histogram", "histograms", "value");
+}
+
+static inline RRD_ALGORITHM statsd_algorithm_for_metric(STATSD_METRIC *m) {
+ switch(m->type) {
+ default:
+ case STATSD_METRIC_TYPE_GAUGE:
+ case STATSD_METRIC_TYPE_SET:
+ case STATSD_METRIC_TYPE_TIMER:
+ case STATSD_METRIC_TYPE_HISTOGRAM:
+ return RRD_ALGORITHM_ABSOLUTE;
+
+ case STATSD_METRIC_TYPE_METER:
+ case STATSD_METRIC_TYPE_COUNTER:
+ return RRD_ALGORITHM_INCREMENTAL;
+ }
+}
+
+static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC *m) {
+ (void)index;
+
+ STATSD_APP *app;
+ for(app = statsd.apps; app ;app = app->next) {
+ if(unlikely(simple_pattern_matches(app->metrics, m->name))) {
+ debug(D_STATSD, "metric '%s' matches app '%s'", m->name, app->name);
+
+ // the metric should get the options from the app
+
+ if(app->default_options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED)
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ else
+ m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+
+ if(app->default_options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)
+ m->options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+ else
+ m->options &= ~STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED;
+
+ // check if there is a chart in this app, willing to get this metric
+ STATSD_APP_CHART *chart;
+ for(chart = app->charts; chart; chart = chart->next) {
+ STATSD_APP_CHART_DIM *dim;
+ for(dim = chart->dimensions; dim ; dim = dim->next) {
+ if(!dim->value_ptr && dim->metric_hash == m->hash && !strcmp(dim->metric, m->name)) {
+ // we have a match - this metric should be linked to this dimension
+
+ if(dim->value_type == STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS) {
+ dim->value_ptr = &m->events;
+ dim->algorithm = RRD_ALGORITHM_INCREMENTAL;
+ }
+ else if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
+ dim->algorithm = RRD_ALGORITHM_ABSOLUTE;
+ dim->divisor *= STATSD_DECIMAL_DETAIL;
+
+ switch(dim->value_type) {
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS:
+ // will never match - added to avoid warning
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST:
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE:
+ dim->value_ptr = &m->last;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM:
+ dim->value_ptr = &m->histogram.ext->last_sum;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN:
+ dim->value_ptr = &m->histogram.ext->last_min;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX:
+ dim->value_ptr = &m->histogram.ext->last_max;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN:
+ dim->value_ptr = &m->histogram.ext->last_median;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE:
+ dim->value_ptr = &m->histogram.ext->last_percentile;
+ break;
+
+ case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV:
+ dim->value_ptr = &m->histogram.ext->last_stddev;
+ break;
+ }
+ }
+ else {
+ if (dim->value_type != STATSD_APP_CHART_DIM_VALUE_TYPE_LAST)
+ error("STATSD: unsupported value type for dimension '%s' of chart '%s' of app '%s' on metric '%s'", dim->name, chart->id, app->name, m->name);
+
+ dim->value_ptr = &m->last;
+ dim->algorithm = statsd_algorithm_for_metric(m);
+
+ if(m->type == STATSD_METRIC_TYPE_GAUGE)
+ dim->divisor *= STATSD_DECIMAL_DETAIL;
+ }
+
+ if(unlikely(chart->st && dim->rd)) {
+ rrddim_set_algorithm(chart->st, dim->rd, dim->algorithm);
+ rrddim_set_multiplier(chart->st, dim->rd, dim->multiplier);
+ rrddim_set_divisor(chart->st, dim->rd, dim->divisor);
+ }
+
+ chart->dimensions_linked_count++;
+ debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm));
+ }
+ }
+ }
+ }
+ }
+}
+
+static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *chart) {
+ debug(D_STATSD, "updating chart '%s' for app '%s'", chart->id, app->name);
+
+ if(!chart->st) {
+ chart->st = rrdset_create_custom(
+ localhost
+ , app->name
+ , chart->id
+ , chart->name
+ , chart->family
+ , chart->context
+ , chart->title
+ , chart->units
+ , chart->priority
+ , statsd.update_every
+ , chart->chart_type
+ , app->rrd_memory_mode
+ , app->rrd_history_entries
+ );
+
+ rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);
+ // rrdset_flag_set(chart->st, RRDSET_FLAG_DEBUG);
+ }
+ else rrdset_next(chart->st);
+
+ STATSD_APP_CHART_DIM *dim;
+ for(dim = chart->dimensions; dim ;dim = dim->next) {
+ if(unlikely(!dim->rd))
+ dim->rd = rrddim_add(chart->st, dim->name, NULL, dim->multiplier, dim->divisor, dim->algorithm);
+
+ if(unlikely(dim->value_ptr)) {
+ debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr);
+ rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr);
+ }
+ }
+
+ rrdset_done(chart->st);
+ debug(D_STATSD, "completed update of chart '%s' for app '%s'", chart->id, app->name);
+}
+
+static inline void statsd_update_all_app_charts(void) {
+ // debug(D_STATSD, "updating app charts");
+
+ STATSD_APP *app;
+ for(app = statsd.apps; app ;app = app->next) {
+ // debug(D_STATSD, "updating charts for app '%s'", app->name);
+
+ STATSD_APP_CHART *chart;
+ for(chart = app->charts; chart ;chart = chart->next) {
+ if(unlikely(chart->dimensions_linked_count)) {
+ statsd_update_app_chart(app, chart);
+ }
+ }
+ }
+
+ // debug(D_STATSD, "completed update of app charts");
+}
+
+static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_metric)(STATSD_METRIC *)) {
+ STATSD_METRIC *m;
+ for(m = index->first; m ; m = m->next) {
+ if(unlikely(!(m->options & STATSD_METRIC_OPTION_CHECKED_IN_APPS))) {
+ check_if_metric_is_for_app(index, m);
+ m->options |= STATSD_METRIC_OPTION_CHECKED_IN_APPS;
+ }
+
+ if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) {
+ if(statsd.private_charts >= statsd.max_private_charts_hard) {
+ debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name);
+ info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts);
+ m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ }
+ else {
+ if (simple_pattern_matches(statsd.charts_for, m->name)) {
+ debug(D_STATSD, "STATSD: metric '%s' will be charted.", m->name);
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ } else {
+ debug(D_STATSD, "STATSD: metric '%s' will not be charted.", m->name);
+ m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
+ }
+ }
+
+ m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED;
+ }
+
+ flush_metric(m);
+ }
+}
+
+
+// --------------------------------------------------------------------------------------
+// statsd main thread
+
+int statsd_listen_sockets_setup(void) {
+ return listen_sockets_setup(&statsd.sockets);
+}
+
+void statsd_main_cleanup(void *data) {
+ pthread_t *threads = data;
+
+ int i;
+ for(i = 0; i < statsd.threads ;i++)
+ pthread_cancel(threads[i]);
+
+ listen_sockets_close(&statsd.sockets);
+}
+
+void *statsd_main(void *ptr) {
+ (void)ptr;
+
+ info("STATSD main thread created with task id %d", gettid());
+
+ if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
+ error("Cannot set pthread cancel type to DEFERRED.");
+
+ if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
+ error("Cannot set pthread cancel state to ENABLE.");
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd configuration
+
+ statsd.enabled = config_get_boolean(CONFIG_SECTION_STATSD, "enabled", statsd.enabled);
+
+ statsd.update_every = default_rrd_update_every;
+ statsd.update_every = (int)config_get_number(CONFIG_SECTION_STATSD, "update every (flushInterval)", statsd.update_every);
+ if(statsd.update_every < default_rrd_update_every) {
+ error("STATSD: minimum flush interval %d given, but the minimum is the update every of netdata. Using %d", statsd.update_every, default_rrd_update_every);
+ statsd.update_every = default_rrd_update_every;
+ }
+
+#ifdef HAVE_RECVMMSG
+ statsd.recvmmsg_size = (size_t)config_get_number(CONFIG_SECTION_STATSD, "udp messages to process at once", (long long)statsd.recvmmsg_size);
+#endif
+
+ statsd.charts_for = simple_pattern_create(config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), SIMPLE_PATTERN_EXACT);
+ statsd.max_private_charts = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts allowed", (long long)statsd.max_private_charts);
+ statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts * 5);
+ statsd.private_charts_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_STATSD, "private charts memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
+ statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
+
+ statsd.histogram_percentile = (double)config_get_float(CONFIG_SECTION_STATSD, "histograms and timers percentile (percentThreshold)", statsd.histogram_percentile);
+ if(isless(statsd.histogram_percentile, 0) || isgreater(statsd.histogram_percentile, 100)) {
+ error("STATSD: invalid histograms and timers percentile %0.5f given", statsd.histogram_percentile);
+ statsd.histogram_percentile = 95.0;
+ }
+ {
+ char buffer[100 + 1];
+ snprintf(buffer, 100, "%0.1f%%", statsd.histogram_percentile);
+ statsd.histogram_percentile_str = strdupz(buffer);
+ }
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "add dimension for number of events received", 1)) {
+ statsd.gauges.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.counters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.meters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.sets.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.histograms.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ statsd.timers.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+ }
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on gauges (deleteGauges)", 0))
+ statsd.gauges.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on counters (deleteCounters)", 0))
+ statsd.counters.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on meters (deleteMeters)", 0))
+ statsd.meters.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on sets (deleteSets)", 0))
+ statsd.sets.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on histograms (deleteHistograms)", 0))
+ statsd.histograms.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+ if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on timers (deleteTimers)", 0))
+ statsd.timers.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
+#ifdef STATSD_MULTITHREADED
+ statsd.threads = (int)config_get_number(CONFIG_SECTION_STATSD, "threads", processors);
+ if(statsd.threads < 1) {
+ error("STATSD: Invalid number of threads %d, using %d", statsd.threads, processors);
+ statsd.threads = processors;
+ config_set_number(CONFIG_SECTION_STATSD, "collector threads", statsd.threads);
+ }
+#else
+ statsd.threads = 1;
+#endif
+
+ // read custom application definitions
+ {
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/statsd.d", netdata_configured_config_dir);
+ statsd_readdir(filename);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd setup
+
+ if(!statsd.enabled) return NULL;
+
+ statsd_listen_sockets_setup();
+ if(!statsd.sockets.opened) {
+ error("STATSD: No statsd sockets to listen to. statsd will be disabled.");
+ pthread_exit(NULL);
+ }
+
+ pthread_t threads[statsd.threads];
+ int i;
+
+ for(i = 0; i < statsd.threads ;i++) {
+ if(pthread_create(&threads[i], NULL, statsd_collector_thread, &i))
+ error("STATSD: failed to create child thread.");
+
+ else if(pthread_detach(threads[i]))
+ error("STATSD: cannot request detach of child thread.");
+ }
+
+ pthread_cleanup_push(statsd_main_cleanup, &threads);
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd monitoring charts
+
+ RRDSET *st_metrics = rrdset_create_localhost(
+ "netdata"
+ , "statsd_metrics"
+ , NULL
+ , "statsd"
+ , NULL
+ , "Metrics in the netdata statsd database"
+ , "metrics"
+ , 132000
+ , statsd.update_every
+ , RRDSET_TYPE_STACKED
+ );
+ RRDDIM *rd_metrics_gauge = rrddim_add(st_metrics, "gauges", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_metrics_counter = rrddim_add(st_metrics, "counters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_metrics_timer = rrddim_add(st_metrics, "timers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_metrics_meter = rrddim_add(st_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_metrics_histogram = rrddim_add(st_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_metrics_set = rrddim_add(st_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+ RRDSET *st_events = rrdset_create_localhost(
+ "netdata"
+ , "statsd_events"
+ , NULL
+ , "statsd"
+ , NULL
+ , "Events processed by the netdata statsd server"
+ , "events/s"
+ , 132001
+ , statsd.update_every
+ , RRDSET_TYPE_STACKED
+ );
+ RRDDIM *rd_events_gauge = rrddim_add(st_events, "gauges", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_events_counter = rrddim_add(st_events, "counters", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_events_timer = rrddim_add(st_events, "timers", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_events_meter = rrddim_add(st_events, "meters", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_events_histogram = rrddim_add(st_events, "histograms", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_events_set = rrddim_add(st_events, "sets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_events_unknown = rrddim_add(st_events, "unknown", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_events_errors = rrddim_add(st_events, "errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ RRDSET *st_reads = rrdset_create_localhost(
+ "netdata"
+ , "statsd_reads"
+ , NULL
+ , "statsd"
+ , NULL
+ , "Read operations made by the netdata statsd server"
+ , "reads/s"
+ , 132002
+ , statsd.update_every
+ , RRDSET_TYPE_STACKED
+ );
+ RRDDIM *rd_reads_tcp = rrddim_add(st_reads, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_reads_udp = rrddim_add(st_reads, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ RRDSET *st_bytes = rrdset_create_localhost(
+ "netdata"
+ , "statsd_bytes"
+ , NULL
+ , "statsd"
+ , NULL
+ , "Bytes read by the netdata statsd server"
+ , "kbps"
+ , 132003
+ , statsd.update_every
+ , RRDSET_TYPE_STACKED
+ );
+ RRDDIM *rd_bytes_tcp = rrddim_add(st_bytes, "tcp", NULL, 8, 1024, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_bytes_udp = rrddim_add(st_bytes, "udp", NULL, 8, 1024, RRD_ALGORITHM_INCREMENTAL);
+
+ RRDSET *st_packets = rrdset_create_localhost(
+ "netdata"
+ , "statsd_packets"
+ , NULL
+ , "statsd"
+ , NULL
+ , "Network packets processed by the netdata statsd server"
+ , "packets/s"
+ , 132004
+ , statsd.update_every
+ , RRDSET_TYPE_STACKED
+ );
+ RRDDIM *rd_packets_tcp = rrddim_add(st_packets, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ RRDDIM *rd_packets_udp = rrddim_add(st_packets, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ RRDSET *st_pcharts = rrdset_create_localhost(
+ "netdata"
+ , "private_charts"
+ , NULL
+ , "statsd"
+ , NULL
+ , "Private metric charts created by the netdata statsd server"
+ , "charts"
+ , 132010
+ , statsd.update_every
+ , RRDSET_TYPE_AREA
+ );
+ RRDDIM *rd_pcharts = rrddim_add(st_pcharts, "charts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // statsd thread to turn metrics into charts
+
+ usec_t step = statsd.update_every * USEC_PER_SEC;
+ heartbeat_t hb;
+ heartbeat_init(&hb);
+ for(;;) {
+ usec_t hb_dt = heartbeat_next(&hb, step);
+
+ if(unlikely(netdata_exit))
+ break;
+
+ statsd_flush_index_metrics(&statsd.gauges, statsd_flush_gauge);
+ statsd_flush_index_metrics(&statsd.counters, statsd_flush_counter);
+ statsd_flush_index_metrics(&statsd.meters, statsd_flush_meter);
+ statsd_flush_index_metrics(&statsd.timers, statsd_flush_timer);
+ statsd_flush_index_metrics(&statsd.histograms, statsd_flush_histogram);
+ statsd_flush_index_metrics(&statsd.sets, statsd_flush_set);
+
+ statsd_update_all_app_charts();
+
+ if(unlikely(netdata_exit))
+ break;
+
+ if(hb_dt) {
+ rrdset_next(st_metrics);
+ rrdset_next(st_events);
+ rrdset_next(st_reads);
+ rrdset_next(st_bytes);
+ rrdset_next(st_packets);
+ rrdset_next(st_pcharts);
+ }
+
+ rrddim_set_by_pointer(st_metrics, rd_metrics_gauge, (collected_number)statsd.gauges.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_counter, (collected_number)statsd.counters.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_timer, (collected_number)statsd.timers.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_meter, (collected_number)statsd.meters.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_histogram, (collected_number)statsd.histograms.metrics);
+ rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics);
+
+ rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events);
+ rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events);
+ rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events);
+ rrddim_set_by_pointer(st_events, rd_events_meter, (collected_number)statsd.meters.events);
+ rrddim_set_by_pointer(st_events, rd_events_histogram, (collected_number)statsd.histograms.events);
+ rrddim_set_by_pointer(st_events, rd_events_set, (collected_number)statsd.sets.events);
+ rrddim_set_by_pointer(st_events, rd_events_unknown, (collected_number)statsd.unknown_types);
+ rrddim_set_by_pointer(st_events, rd_events_errors, (collected_number)statsd.socket_errors);
+
+ rrddim_set_by_pointer(st_reads, rd_reads_tcp, (collected_number)statsd.tcp_socket_reads);
+ rrddim_set_by_pointer(st_reads, rd_reads_udp, (collected_number)statsd.udp_socket_reads);
+
+ rrddim_set_by_pointer(st_bytes, rd_bytes_tcp, (collected_number)statsd.tcp_bytes_read);
+ rrddim_set_by_pointer(st_bytes, rd_bytes_udp, (collected_number)statsd.udp_bytes_read);
+
+ rrddim_set_by_pointer(st_packets, rd_packets_tcp, (collected_number)statsd.tcp_packets_received);
+ rrddim_set_by_pointer(st_packets, rd_packets_udp, (collected_number)statsd.udp_packets_received);
+
+ rrddim_set_by_pointer(st_pcharts, rd_pcharts, (collected_number)statsd.private_charts);
+
+ if(unlikely(netdata_exit))
+ break;
+
+ rrdset_done(st_metrics);
+ rrdset_done(st_events);
+ rrdset_done(st_reads);
+ rrdset_done(st_bytes);
+ rrdset_done(st_packets);
+ rrdset_done(st_pcharts);
+
+ if(unlikely(netdata_exit))
+ break;
+ }
+
+ pthread_cleanup_pop(1);
+
+ pthread_exit(NULL);
+ return NULL;
+}
diff --git a/src/statsd.h b/src/statsd.h
new file mode 100644
index 000000000..17af098e8
--- /dev/null
+++ b/src/statsd.h
@@ -0,0 +1,9 @@
+#ifndef NETDATA_STATSD_H
+#define NETDATA_STATSD_H
+
+#define STATSD_LISTEN_PORT 8125
+#define STATSD_LISTEN_BACKLOG 4096
+
+extern void *statsd_main(void *ptr);
+
+#endif //NETDATA_STATSD_H
diff --git a/src/storage_number.h b/src/storage_number.h
index 74d24a322..34ed0d89c 100644
--- a/src/storage_number.h
+++ b/src/storage_number.h
@@ -29,6 +29,7 @@ typedef uint32_t storage_number;
// extract the flags
#define get_storage_number_flags(value) ((((storage_number)value) & (1 << 24)) | (((storage_number)value) & (2 << 24)) | (((storage_number)value) & (4 << 24)))
+#define SN_EMPTY_SLOT 0x00000000
// checks
#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0)
diff --git a/src/sys_fs_cgroup.c b/src/sys_fs_cgroup.c
index 8f31527de..0f9c8854a 100644
--- a/src/sys_fs_cgroup.c
+++ b/src/sys_fs_cgroup.c
@@ -147,11 +147,17 @@ void read_cgroup_plugin_configuration() {
enabled_cgroup_patterns = simple_pattern_create(
config_get("plugin:cgroups", "enable by default cgroups matching",
- " /system.slice/docker-*.scope "
- " /qemu.slice/*.scope " // #1949
+ // ----------------------------------------------------------------
+
+ " !*/init.scope " // ignore init.scope
+ " *.scope " // we need all *.scope for sure
+
+ // ----------------------------------------------------------------
+
+ " !*/vcpu* " // libvirtd adds these sub-cgroups
+ " !*/emulator " // libvirtd adds these sub-cgroups
" !*.mount "
" !*.partition "
- " !*.scope "
" !*.service "
" !*.slice "
" !*.swap "
@@ -171,12 +177,14 @@ void read_cgroup_plugin_configuration() {
enabled_cgroup_paths = simple_pattern_create(
config_get("plugin:cgroups", "search for cgroups in subpaths matching",
- " !*-qemu " // #345
+ " !*/init.scope " // ignore init.scope
+ " !*-qemu " // #345
" !/init.scope "
" !/system "
" !/systemd "
" !/user "
" !/user.slice "
+ " !/lxc/*/ns/* " // #2161
" * "
), SIMPLE_PATTERN_EXACT);
@@ -185,13 +193,13 @@ void read_cgroup_plugin_configuration() {
enabled_cgroup_renames = simple_pattern_create(
config_get("plugin:cgroups", "run script to rename cgroups matching",
- " /qemu.slice/*.scope " // #1949
+ " *.scope "
" *docker* "
" *lxc* "
+ " *qemu* "
" !/ "
" !*.mount "
" !*.partition "
- " !*.scope "
" !*.service "
" !*.slice "
" !*.swap "
@@ -893,20 +901,20 @@ static inline struct cgroup *cgroup_add(const char *id) {
static inline void cgroup_free(struct cgroup *cg) {
debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available");
- if(cg->st_cpu) rrdset_flag_set(cg->st_cpu, RRDSET_FLAG_OBSOLETE);
- if(cg->st_cpu_per_core) rrdset_flag_set(cg->st_cpu_per_core, RRDSET_FLAG_OBSOLETE);
- if(cg->st_mem) rrdset_flag_set(cg->st_mem, RRDSET_FLAG_OBSOLETE);
- if(cg->st_writeback) rrdset_flag_set(cg->st_writeback, RRDSET_FLAG_OBSOLETE);
- if(cg->st_mem_activity) rrdset_flag_set(cg->st_mem_activity, RRDSET_FLAG_OBSOLETE);
- if(cg->st_pgfaults) rrdset_flag_set(cg->st_pgfaults, RRDSET_FLAG_OBSOLETE);
- if(cg->st_mem_usage) rrdset_flag_set(cg->st_mem_usage, RRDSET_FLAG_OBSOLETE);
- if(cg->st_mem_failcnt) rrdset_flag_set(cg->st_mem_failcnt, RRDSET_FLAG_OBSOLETE);
- if(cg->st_io) rrdset_flag_set(cg->st_io, RRDSET_FLAG_OBSOLETE);
- if(cg->st_serviced_ops) rrdset_flag_set(cg->st_serviced_ops, RRDSET_FLAG_OBSOLETE);
- if(cg->st_throttle_io) rrdset_flag_set(cg->st_throttle_io, RRDSET_FLAG_OBSOLETE);
- if(cg->st_throttle_serviced_ops) rrdset_flag_set(cg->st_throttle_serviced_ops, RRDSET_FLAG_OBSOLETE);
- if(cg->st_queued_ops) rrdset_flag_set(cg->st_queued_ops, RRDSET_FLAG_OBSOLETE);
- if(cg->st_merged_ops) rrdset_flag_set(cg->st_merged_ops, RRDSET_FLAG_OBSOLETE);
+ if(cg->st_cpu) rrdset_is_obsolete(cg->st_cpu);
+ if(cg->st_cpu_per_core) rrdset_is_obsolete(cg->st_cpu_per_core);
+ if(cg->st_mem) rrdset_is_obsolete(cg->st_mem);
+ if(cg->st_writeback) rrdset_is_obsolete(cg->st_writeback);
+ if(cg->st_mem_activity) rrdset_is_obsolete(cg->st_mem_activity);
+ if(cg->st_pgfaults) rrdset_is_obsolete(cg->st_pgfaults);
+ if(cg->st_mem_usage) rrdset_is_obsolete(cg->st_mem_usage);
+ if(cg->st_mem_failcnt) rrdset_is_obsolete(cg->st_mem_failcnt);
+ if(cg->st_io) rrdset_is_obsolete(cg->st_io);
+ if(cg->st_serviced_ops) rrdset_is_obsolete(cg->st_serviced_ops);
+ if(cg->st_throttle_io) rrdset_is_obsolete(cg->st_throttle_io);
+ if(cg->st_throttle_serviced_ops) rrdset_is_obsolete(cg->st_throttle_serviced_ops);
+ if(cg->st_queued_ops) rrdset_is_obsolete(cg->st_queued_ops);
+ if(cg->st_merged_ops) rrdset_is_obsolete(cg->st_merged_ops);
freez(cg->cpuacct_usage.cpu_percpu);
diff --git a/src/unit_test.c b/src/unit_test.c
index 0866d215c..9b008138f 100644
--- a/src/unit_test.c
+++ b/src/unit_test.c
@@ -17,8 +17,8 @@ int check_storage_number(calculated_number n, int debug) {
if(dcdiff < 0) dcdiff = -dcdiff;
- size_t len = print_calculated_number(buffer, d);
- calculated_number p = str2l(buffer);
+ size_t len = (size_t)print_calculated_number(buffer, d);
+ calculated_number p = str2ld(buffer, NULL);
calculated_number pdiff = n - p;
calculated_number pcdiff = pdiff * 100.0 / n;
if(pcdiff < 0) pcdiff = -pcdiff;
@@ -229,6 +229,45 @@ int unit_test_storage()
return r;
}
+int unit_test_str2ld() {
+ char *values[] = {
+ "1.234567", "-35.6", "0.00123", "23842384234234.2", ".1", "1.2e-10",
+ "hello", "1wrong", "nan", "inf", NULL
+ };
+
+ int i;
+ for(i = 0; values[i] ; i++) {
+ char *e_mine = "hello", *e_sys = "world";
+ long double mine = str2ld(values[i], &e_mine);
+ long double sys = strtold(values[i], &e_sys);
+
+ if(isnan(mine)) {
+ if(!isnan(sys)) {
+ fprintf(stderr, "Value '%s' is parsed as %Lf, but system believes it is %Lf.\n", values[i], mine, sys);
+ return -1;
+ }
+ }
+ else if(isinf(mine)) {
+ if(!isinf(sys)) {
+ fprintf(stderr, "Value '%s' is parsed as %Lf, but system believes it is %Lf.\n", values[i], mine, sys);
+ return -1;
+ }
+ }
+ else if(mine != sys && abs(mine-sys) > 0.000001) {
+ fprintf(stderr, "Value '%s' is parsed as %Lf, but system believes it is %Lf, delta %Lf.\n", values[i], mine, sys, sys-mine);
+ return -1;
+ }
+
+ if(e_mine != e_sys) {
+ fprintf(stderr, "Value '%s' is parsed correctly, but endptr is not right\n", values[i]);
+ return -1;
+ }
+
+ fprintf(stderr, "str2ld() parsed value '%s' exactly the same way with strtold(), returned %Lf vs %Lf\n", values[i], mine, sys);
+ }
+
+ return 0;
+}
// --------------------------------------------------------------------------------------------------------------------
@@ -244,7 +283,7 @@ struct test {
int update_every;
unsigned long long multiplier;
unsigned long long divisor;
- int algorithm;
+ RRD_ALGORITHM algorithm;
unsigned long feed_entries;
unsigned long result_entries;
@@ -884,7 +923,7 @@ int run_test(struct test *test)
{
fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description);
- default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
+ default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
default_rrd_update_every = test->update_every;
char name[101];
@@ -916,7 +955,9 @@ int run_test(struct test *test)
(float)time_now / 1000000.0,
((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor,
(((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor) / (calculated_number)test->feed[c].microseconds * (calculated_number)1000000);
- rrdset_next_usec_unfiltered(st, test->feed[c].microseconds);
+
+ // rrdset_next_usec_unfiltered(st, test->feed[c].microseconds);
+ st->usec_since_last_update = test->feed[c].microseconds;
}
else {
fprintf(stderr, " > %s: feeding position %lu\n", test->name, c+1);
@@ -1091,7 +1132,7 @@ int unit_test(long delay, long shift)
snprintfz(name, 100, "unittest-%d-%ld-%ld", repeat, delay, shift);
//debug_flags = 0xffffffff;
- default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
+ default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
default_rrd_update_every = 1;
int do_abs = 1;
@@ -1125,7 +1166,8 @@ int unit_test(long delay, long shift)
fprintf(stderr, "\n\nLOOP = %lu, DELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", c, delay, i);
if(c) {
- rrdset_next_usec_unfiltered(st, delay);
+ // rrdset_next_usec_unfiltered(st, delay);
+ st->usec_since_last_update = delay;
}
if(do_abs) rrddim_set(st, "absolute", i);
if(do_inc) rrddim_set(st, "incremental", i);
diff --git a/src/unit_test.h b/src/unit_test.h
index 916ad71f2..3240b5f0e 100644
--- a/src/unit_test.h
+++ b/src/unit_test.h
@@ -4,5 +4,6 @@
extern int unit_test_storage(void);
extern int unit_test(long delay, long shift);
extern int run_all_mockup_tests(void);
+extern int unit_test_str2ld(void);
#endif /* NETDATA_UNIT_TEST_H */
diff --git a/src/web_api_v1.c b/src/web_api_v1.c
index 0acc43acb..3ffd8c324 100644
--- a/src/web_api_v1.c
+++ b/src/web_api_v1.c
@@ -208,6 +208,10 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,
inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) {
int format = ALLMETRICS_SHELL;
+ int help = 0, types = 0, names = backend_send_names; // prometheus options
+ const char *prometheus_server = w->client_ip;
+ uint32_t prometheus_options = backend_options;
+ const char *prometheus_prefix = backend_prefix;
while(url) {
char *value = mystrsep(&url, "?&");
@@ -222,11 +226,40 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
format = ALLMETRICS_SHELL;
else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS))
format = ALLMETRICS_PROMETHEUS;
+ else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS))
+ format = ALLMETRICS_PROMETHEUS_ALL_HOSTS;
else if(!strcmp(value, ALLMETRICS_FORMAT_JSON))
format = ALLMETRICS_JSON;
else
format = 0;
}
+ else if(!strcmp(name, "help")) {
+ if(!strcmp(value, "yes"))
+ help = 1;
+ else
+ help = 0;
+ }
+ else if(!strcmp(name, "types")) {
+ if(!strcmp(value, "yes"))
+ types = 1;
+ else
+ types = 0;
+ }
+ else if(!strcmp(name, "names")) {
+ if(!strcmp(value, "yes"))
+ names = 1;
+ else
+ names = 0;
+ }
+ else if(!strcmp(name, "server")) {
+ prometheus_server = value;
+ }
+ else if(!strcmp(name, "prefix")) {
+ prometheus_prefix = value;
+ }
+ else if(!strcmp(name, "data") || !strcmp(name, "source") || !strcmp(name, "data source") || !strcmp(name, "data-source") || !strcmp(name, "data_source") || !strcmp(name, "datasource")) {
+ prometheus_options = backend_parse_data_source(value, prometheus_options);
+ }
}
buffer_flush(w->response.data);
@@ -245,12 +278,17 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
case ALLMETRICS_PROMETHEUS:
w->response.data->contenttype = CT_PROMETHEUS;
- rrd_stats_api_v1_charts_allmetrics_prometheus(host, w->response.data);
+ rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_options, help, types, names);
+ return 200;
+
+ case ALLMETRICS_PROMETHEUS_ALL_HOSTS:
+ w->response.data->contenttype = CT_PROMETHEUS;
+ rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_options, help, types, names);
return 200;
default:
w->response.data->contenttype = CT_TEXT_PLAIN;
- buffer_strcat(w->response.data, "Which format? '" ALLMETRICS_FORMAT_SHELL "', '" ALLMETRICS_FORMAT_PROMETHEUS "' and '" ALLMETRICS_FORMAT_JSON "' are currently supported.");
+ buffer_strcat(w->response.data, "Which format? '" ALLMETRICS_FORMAT_SHELL "', '" ALLMETRICS_FORMAT_PROMETHEUS "', '" ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS "' and '" ALLMETRICS_FORMAT_JSON "' are currently supported.");
return 400;
}
}
diff --git a/src/web_client.c b/src/web_client.c
index b5b25899f..7da080705 100644
--- a/src/web_client.c
+++ b/src/web_client.c
@@ -57,13 +57,7 @@ struct web_client *web_client_create(int listener) {
w->mode = WEB_CLIENT_MODE_NORMAL;
{
- struct sockaddr *sadr;
- socklen_t addrlen;
-
- sadr = (struct sockaddr*) &w->clientaddr;
- addrlen = sizeof(w->clientaddr);
-
- w->ifd = accept4(listener, sadr, &addrlen, SOCK_NONBLOCK);
+ w->ifd = accept_socket(listener, SOCK_NONBLOCK, w->client_ip, sizeof(w->client_ip), w->client_port, sizeof(w->client_port));
if (w->ifd == -1) {
error("%llu: Cannot accept new incoming connection.", w->id);
freez(w);
@@ -71,33 +65,6 @@ struct web_client *web_client_create(int listener) {
}
w->ofd = w->ifd;
- if(getnameinfo(sadr, addrlen, w->client_ip, NI_MAXHOST, w->client_port, NI_MAXSERV, NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
- error("Cannot getnameinfo() on received client connection.");
- strncpyz(w->client_ip, "UNKNOWN", NI_MAXHOST);
- strncpyz(w->client_port, "UNKNOWN", NI_MAXSERV);
- }
- w->client_ip[NI_MAXHOST] = '\0';
- w->client_port[NI_MAXSERV] = '\0';
-
- switch(sadr->sa_family) {
- case AF_INET:
- debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
- break;
-
- case AF_INET6:
- if(strncmp(w->client_ip, "::ffff:", 7) == 0) {
- memmove(w->client_ip, &w->client_ip[7], strlen(&w->client_ip[7]) + 1);
- debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
- }
- else
- debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv6 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
- break;
-
- default:
- debug(D_WEB_CLIENT_ACCESS, "%llu: New UNKNOWN web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
- break;
- }
-
int flag = 1;
if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0)
error("%llu: failed to enable TCP_NODELAY on socket.", w->id);
@@ -388,8 +355,8 @@ int mysendfile(struct web_client *w, char *filename) {
return 404;
}
}
- if(fcntl(w->ifd, F_SETFL, O_NONBLOCK) < 0)
- error("%llu: Cannot set O_NONBLOCK on file '%s'.", w->id, webfilename);
+
+ sock_setnonblock(w->ifd);
// pick a Content-Type for the file
if(strstr(filename, ".html") != NULL) w->response.data->contenttype = CT_TEXT_HTML;
@@ -995,13 +962,22 @@ static inline void web_client_send_http_header(struct web_client *w) {
web_client_crock_socket(w);
- ssize_t bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0);
+ size_t count = 0;
+ ssize_t bytes;
+ while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
+ count++;
+
+ if(count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
+ error("Cannot send HTTP headers to web client.");
+ break;
+ }
+ }
+
if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) {
if(bytes > 0)
w->stats_sent_bytes += bytes;
- debug(D_WEB_CLIENT, "%llu: HTTP Header failed to be sent (I sent %zu bytes but the system sent %zd bytes). Closing web client."
- , w->id
+ error("HTTP headers failed to be sent (I sent %zu bytes but the system sent %zd bytes). Closing web client."
, buffer_strlen(w->response.header_output)
, bytes);
diff --git a/src/web_client.h b/src/web_client.h
index 70c5b1ff0..617917df0 100644
--- a/src/web_client.h
+++ b/src/web_client.h
@@ -83,7 +83,6 @@ struct web_client {
char cookie2[COOKIE_MAX+1];
char origin[ORIGIN_MAX+1];
- struct sockaddr_storage clientaddr;
struct response response;
size_t stats_received_bytes;
diff --git a/src/web_server.c b/src/web_server.c
index 593a82a57..491cd11aa 100644
--- a/src/web_server.c
+++ b/src/web_server.c
@@ -1,15 +1,14 @@
#include "common.h"
-int listen_backlog = LISTEN_BACKLOG;
-size_t listen_fds_count = 0;
-int listen_fds[MAX_LISTEN_FDS] = { [0 ... 99] = -1 };
-char *listen_fds_names[MAX_LISTEN_FDS] = { [0 ... 99] = NULL };
-int listen_port = LISTEN_PORT;
+static LISTEN_SOCKETS api_sockets = {
+ .config_section = CONFIG_SECTION_WEB,
+ .default_bind_to = "*",
+ .default_port = API_LISTEN_PORT,
+ .backlog = API_LISTEN_BACKLOG
+};
WEB_SERVER_MODE web_server_mode = WEB_SERVER_MODE_MULTI_THREADED;
-static int shown_server_socket_error = 0;
-
#ifdef NETDATA_INTERNAL_CHECKS
static void log_allocations(void)
{
@@ -47,42 +46,7 @@ static void log_allocations(void)
}
#endif /* NETDATA_INTERNAL_CHECKS */
-#ifndef HAVE_ACCEPT4
-int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) {
- int fd = accept(sock, addr, addrlen);
- int newflags = 0;
-
- if (fd < 0) return fd;
-
- if (flags & SOCK_NONBLOCK) {
- newflags |= O_NONBLOCK;
- flags &= ~SOCK_NONBLOCK;
- }
-
-#ifdef SOCK_CLOEXEC
-#ifdef O_CLOEXEC
- if (flags & SOCK_CLOEXEC) {
- newflags |= O_CLOEXEC;
- flags &= ~SOCK_CLOEXEC;
- }
-#endif
-#endif
-
- if (flags) {
- errno = -EINVAL;
- return -1;
- }
-
- if (fcntl(fd, F_SETFL, newflags) < 0) {
- int saved_errno = errno;
- close(fd);
- errno = saved_errno;
- return -1;
- }
-
- return fd;
-}
-#endif
+// --------------------------------------------------------------------------------------
WEB_SERVER_MODE web_server_mode_id(const char *mode) {
if(!strcmp(mode, "none"))
@@ -107,277 +71,15 @@ const char *web_server_mode_name(WEB_SERVER_MODE id) {
}
}
-int create_listen_socket4(const char *ip, int port, int listen_backlog) {
- int sock;
- int sockopt = 1;
-
- debug(D_LISTENER, "IPv4 creating new listening socket on ip '%s' port %d", ip, port);
-
- sock = socket(AF_INET, SOCK_STREAM, 0);
- if(sock < 0) {
- error("IPv4 socket() on ip '%s' port %d failed.", ip, port);
- shown_server_socket_error = 1;
- return -1;
- }
-
- /* avoid "address already in use" */
- if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt)) != 0)
- error("Cannot set SO_REUSEADDR on ip '%s' port's %d.", ip, port);
-
- struct sockaddr_in name;
- memset(&name, 0, sizeof(struct sockaddr_in));
- name.sin_family = AF_INET;
- name.sin_port = htons (port);
-
- int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr);
- if(ret != 1) {
- error("Failed to convert IP '%s' to a valid IPv4 address.", ip);
- shown_server_socket_error = 1;
- close(sock);
- return -1;
- }
-
- if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
- close(sock);
- error("IPv4 bind() on ip '%s' port %d failed.", ip, port);
- shown_server_socket_error = 1;
- return -1;
- }
-
- if(listen(sock, listen_backlog) < 0) {
- close(sock);
- error("IPv4 listen() on ip '%s' port %d failed.", ip, port);
- shown_server_socket_error = 1;
- return -1;
- }
-
- debug(D_LISTENER, "Listening on IPv4 ip '%s' port %d", ip, port);
- return sock;
-}
-
-int create_listen_socket6(const char *ip, int port, int listen_backlog) {
- int sock = -1;
- int sockopt = 1;
- int ipv6only = 1;
-
- debug(D_LISTENER, "IPv6 creating new listening socket on ip '%s' port %d", ip, port);
-
- sock = socket(AF_INET6, SOCK_STREAM, 0);
- if (sock < 0) {
- error("IPv6 socket() on ip '%s' port %d failed.", ip, port);
- shown_server_socket_error = 1;
- return -1;
- }
-
- /* avoid "address already in use" */
- if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt)) != 0)
- error("Cannot set SO_REUSEADDR on ip '%s' port's %d.", ip, port);
-
- /* IPv6 only */
- if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0)
- error("Cannot set IPV6_V6ONLY on ip '%s' port's %d.", ip, port);
-
- struct sockaddr_in6 name;
- memset(&name, 0, sizeof(struct sockaddr_in6));
- name.sin6_family = AF_INET6;
- name.sin6_port = htons ((uint16_t) port);
-
- int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr);
- if(ret != 1) {
- error("Failed to convert IP '%s' to a valid IPv6 address.", ip);
- shown_server_socket_error = 1;
- close(sock);
- return -1;
- }
-
- name.sin6_scope_id = 0;
-
- if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
- close(sock);
- error("IPv6 bind() on ip '%s' port %d failed.", ip, port);
- shown_server_socket_error = 1;
- return -1;
- }
-
- if (listen(sock, listen_backlog) < 0) {
- close(sock);
- error("IPv6 listen() on ip '%s' port %d failed.", ip, port);
- shown_server_socket_error = 1;
- return -1;
- }
-
- debug(D_LISTENER, "Listening on IPv6 ip '%s' port %d", ip, port);
- return sock;
-}
-
-static inline int add_listen_socket(int fd, const char *ip, int port) {
- if(listen_fds_count >= MAX_LISTEN_FDS) {
- error("Too many listening sockets. Failed to add listening socket at ip '%s' port %d", ip, port);
- shown_server_socket_error = 1;
- close(fd);
- return -1;
- }
-
- listen_fds[listen_fds_count] = fd;
-
- char buffer[100 + 1];
- snprintfz(buffer, 100, "[%s]:%d", ip, port);
- listen_fds_names[listen_fds_count] = strdupz(buffer);
-
- listen_fds_count++;
- return 0;
-}
-
-int is_listen_socket(int fd) {
- size_t i;
- for(i = 0; i < listen_fds_count ;i++)
- if(listen_fds[i] == fd) return 1;
-
- return 0;
-}
-
-static inline void close_listen_sockets(void) {
- size_t i;
- for(i = 0; i < listen_fds_count ;i++) {
- close(listen_fds[i]);
- listen_fds[i] = -1;
-
- freez(listen_fds_names[i]);
- listen_fds_names[i] = NULL;
- }
-
- listen_fds_count = 0;
-}
-
-static inline int bind_to_one(const char *definition, int default_port, int listen_backlog) {
- int added = 0;
- struct addrinfo hints;
- struct addrinfo *result = NULL, *rp = NULL;
-
- char buffer[strlen(definition) + 1];
- strcpy(buffer, definition);
-
- char buffer2[10 + 1];
- snprintfz(buffer2, 10, "%d", default_port);
-
- char *ip = buffer, *port = buffer2;
-
- char *e = ip;
- if(*e == '[') {
- e = ++ip;
- while(*e && *e != ']') e++;
- if(*e == ']') {
- *e = '\0';
- e++;
- }
- }
- else {
- while(*e && *e != ':') e++;
- }
-
- if(*e == ':') {
- port = e + 1;
- *e = '\0';
- }
-
- if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all"))
- ip = NULL;
- if(!*port)
- port = buffer2;
-
- memset(&hints, 0, sizeof(struct addrinfo));
- hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
- hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */
- hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */
- hints.ai_protocol = 0; /* Any protocol */
- hints.ai_canonname = NULL;
- hints.ai_addr = NULL;
- hints.ai_next = NULL;
-
- int r = getaddrinfo(ip, port, &hints, &result);
- if (r != 0) {
- error("getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r));
- return -1;
- }
-
- for (rp = result; rp != NULL; rp = rp->ai_next) {
- int fd = -1;
-
- char rip[INET_ADDRSTRLEN + INET6_ADDRSTRLEN] = "INVALID";
- int rport = default_port;
-
- switch (rp->ai_addr->sa_family) {
- case AF_INET: {
- struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr;
- inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN);
- rport = ntohs(sin->sin_port);
- fd = create_listen_socket4(rip, rport, listen_backlog);
- break;
- }
-
- case AF_INET6: {
- struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr;
- inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN);
- rport = ntohs(sin6->sin6_port);
- fd = create_listen_socket6(rip, rport, listen_backlog);
- break;
- }
- }
-
- if (fd == -1)
- error("Cannot bind to ip '%s', port %d", rip, rport);
- else {
- add_listen_socket(fd, rip, rport);
- added++;
- }
- }
-
- freeaddrinfo(result);
-
- return added;
-}
-
-int create_listen_sockets(void) {
- shown_server_socket_error = 0;
-
- listen_backlog = (int) config_get_number(CONFIG_SECTION_WEB, "listen backlog", LISTEN_BACKLOG);
-
- listen_port = (int) config_get_number(CONFIG_SECTION_WEB, "default port", LISTEN_PORT);
- if(listen_port < 1 || listen_port > 65535) {
- error("Invalid listen port %d given. Defaulting to %d.", listen_port, LISTEN_PORT);
- listen_port = (int) config_set_number(CONFIG_SECTION_WEB, "default port", LISTEN_PORT);
- }
- debug(D_OPTIONS, "Default listen port set to %d.", listen_port);
-
- char *s = config_get(CONFIG_SECTION_WEB, "bind to", "*");
- while(*s) {
- char *e = s;
-
- // skip separators, moving both s(tart) and e(nd)
- while(isspace(*e) || *e == ',') s = ++e;
-
- // move e(nd) to the first separator
- while(*e && !isspace(*e) && *e != ',') e++;
-
- // is there anything?
- if(!*s || s == e) break;
-
- char buf[e - s + 1];
- strncpyz(buf, s, e - s);
- bind_to_one(buf, listen_port, listen_backlog);
+// --------------------------------------------------------------------------------------
- s = e;
- }
+int api_listen_sockets_setup(void) {
+ int socks = listen_sockets_setup(&api_sockets);
- if(!listen_fds_count)
- fatal("Cannot listen on any socket. Exiting...");
- else if(shown_server_socket_error) {
- size_t i;
- for(i = 0; i < listen_fds_count ;i++)
- info("Listen socket %s opened.", listen_fds_names[i]);
- }
+ if(!socks)
+ fatal("LISTENER: Cannot listen on any API socket. Exiting...");
- return (int)listen_fds_count;
+ return socks;
}
// --------------------------------------------------------------------------------------
@@ -422,25 +124,25 @@ void *socket_listen_main_multi_threaded(void *ptr) {
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
error("Cannot set pthread cancel state to ENABLE.");
- if(!listen_fds_count)
+ if(!api_sockets.opened)
fatal("LISTENER: No sockets to listen to.");
- struct pollfd *fds = callocz(sizeof(struct pollfd), listen_fds_count);
+ struct pollfd *fds = callocz(sizeof(struct pollfd), api_sockets.opened);
size_t i;
- for(i = 0; i < listen_fds_count ;i++) {
- fds[i].fd = listen_fds[i];
+ for(i = 0; i < api_sockets.opened ;i++) {
+ fds[i].fd = api_sockets.fds[i];
fds[i].events = POLLIN;
fds[i].revents = 0;
- info("Listening on '%s'", (listen_fds_names[i])?listen_fds_names[i]:"UNKNOWN");
+ info("Listening on '%s'", (api_sockets.fds_names[i])?api_sockets.fds_names[i]:"UNKNOWN");
}
int timeout = 10 * 1000;
for(;;) {
// debug(D_WEB_CLIENT, "LISTENER: Waiting...");
- retval = poll(fds, listen_fds_count, timeout);
+ retval = poll(fds, api_sockets.opened, timeout);
if(unlikely(retval == -1)) {
error("LISTENER: poll() failed.");
@@ -453,7 +155,7 @@ void *socket_listen_main_multi_threaded(void *ptr) {
continue;
}
- for(i = 0 ; i < listen_fds_count ; i++) {
+ for(i = 0 ; i < api_sockets.opened ; i++) {
short int revents = fds[i].revents;
// check for new incoming connections
@@ -486,7 +188,7 @@ void *socket_listen_main_multi_threaded(void *ptr) {
}
debug(D_WEB_CLIENT, "LISTENER: exit!");
- close_listen_sockets();
+ listen_sockets_close(&api_sockets);
freez(fds);
@@ -555,7 +257,7 @@ void *socket_listen_main_single_threaded(void *ptr) {
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
error("Cannot set pthread cancel state to ENABLE.");
- if(!listen_fds_count)
+ if(!api_sockets.opened)
fatal("LISTENER: no listen sockets available.");
size_t i;
@@ -568,16 +270,16 @@ void *socket_listen_main_single_threaded(void *ptr) {
FD_ZERO (&efds);
int fdmax = 0;
- for(i = 0; i < listen_fds_count ; i++) {
- if (listen_fds[i] < 0 || listen_fds[i] >= FD_SETSIZE)
- fatal("LISTENER: Listen socket %d is not ready, or invalid.", listen_fds[i]);
+ for(i = 0; i < api_sockets.opened ; i++) {
+ if (api_sockets.fds[i] < 0 || api_sockets.fds[i] >= FD_SETSIZE)
+ fatal("LISTENER: Listen socket %d is not ready, or invalid.", api_sockets.fds[i]);
- info("Listening on '%s'", (listen_fds_names[i])?listen_fds_names[i]:"UNKNOWN");
+ info("Listening on '%s'", (api_sockets.fds_names[i])?api_sockets.fds_names[i]:"UNKNOWN");
- FD_SET(listen_fds[i], &ifds);
- FD_SET(listen_fds[i], &efds);
- if(fdmax < listen_fds[i])
- fdmax = listen_fds[i];
+ FD_SET(api_sockets.fds[i], &ifds);
+ FD_SET(api_sockets.fds[i], &efds);
+ if(fdmax < api_sockets.fds[i])
+ fdmax = api_sockets.fds[i];
}
for(;;) {
@@ -596,10 +298,10 @@ void *socket_listen_main_single_threaded(void *ptr) {
else if(likely(retval)) {
debug(D_WEB_CLIENT_ACCESS, "LISTENER: got something.");
- for(i = 0; i < listen_fds_count ; i++) {
- if (FD_ISSET(listen_fds[i], &rifds)) {
+ for(i = 0; i < api_sockets.opened ; i++) {
+ if (FD_ISSET(api_sockets.fds[i], &rifds)) {
debug(D_WEB_CLIENT_ACCESS, "LISTENER: new connection.");
- w = web_client_create(listen_fds[i]);
+ w = web_client_create(api_sockets.fds[i]);
if (single_threaded_link_client(w, &ifds, &ofds, &ifds, &fdmax) != 0) {
web_client_free(w);
}
@@ -658,7 +360,7 @@ void *socket_listen_main_single_threaded(void *ptr) {
}
debug(D_WEB_CLIENT, "LISTENER: exit!");
- close_listen_sockets();
+ listen_sockets_close(&api_sockets);
static_thread->enabled = 0;
pthread_exit(NULL);
diff --git a/src/web_server.h b/src/web_server.h
index 41dcfcf09..aa293695d 100644
--- a/src/web_server.h
+++ b/src/web_server.h
@@ -6,11 +6,12 @@
#define WEB_PATH_DATASOURCE "datasource"
#define WEB_PATH_GRAPH "graph"
-#define LISTEN_PORT 19999
-#define LISTEN_BACKLOG 100
+#ifndef API_LISTEN_PORT
+#define API_LISTEN_PORT 19999
+#endif
-#ifndef MAX_LISTEN_FDS
-#define MAX_LISTEN_FDS 100
+#ifndef API_LISTEN_BACKLOG
+#define API_LISTEN_BACKLOG 4096
#endif
typedef enum web_server_mode {
@@ -24,23 +25,8 @@ extern WEB_SERVER_MODE web_server_mode;
extern WEB_SERVER_MODE web_server_mode_id(const char *mode);
extern const char *web_server_mode_name(WEB_SERVER_MODE id);
-
extern void *socket_listen_main_multi_threaded(void *ptr);
extern void *socket_listen_main_single_threaded(void *ptr);
-extern int create_listen_sockets(void);
-extern int is_listen_socket(int fd);
-
-#ifndef HAVE_ACCEPT4
-extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags);
-
-#ifndef SOCK_NONBLOCK
-#define SOCK_NONBLOCK 00004000
-#endif /* #ifndef SOCK_NONBLOCK */
-
-#ifndef SOCK_CLOEXEC
-#define SOCK_CLOEXEC 02000000
-#endif /* #ifndef SOCK_CLOEXEC */
-
-#endif /* #ifndef HAVE_ACCEPT4 */
+extern int api_listen_sockets_setup(void);
#endif /* NETDATA_WEB_SERVER_H */
diff --git a/src/zfs_common.c b/src/zfs_common.c
new file mode 100644
index 000000000..7fa05b03e
--- /dev/null
+++ b/src/zfs_common.c
@@ -0,0 +1,677 @@
+#include "common.h"
+#include "zfs_common.h"
+
+extern struct arcstats arcstats;
+
+void generate_charts_arcstats(int update_every) {
+
+ // ARC reads
+ unsigned long long aread = arcstats.hits + arcstats.misses;
+
+ // Demand reads
+ unsigned long long dhit = arcstats.demand_data_hits + arcstats.demand_metadata_hits;
+ unsigned long long dmiss = arcstats.demand_data_misses + arcstats.demand_metadata_misses;
+ unsigned long long dread = dhit + dmiss;
+
+ // Prefetch reads
+ unsigned long long phit = arcstats.prefetch_data_hits + arcstats.prefetch_metadata_hits;
+ unsigned long long pmiss = arcstats.prefetch_data_misses + arcstats.prefetch_metadata_misses;
+ unsigned long long pread = phit + pmiss;
+
+ // Metadata reads
+ unsigned long long mhit = arcstats.prefetch_metadata_hits + arcstats.demand_metadata_hits;
+ unsigned long long mmiss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses;
+ unsigned long long mread = mhit + mmiss;
+
+ // l2 reads
+ unsigned long long l2hit = arcstats.l2_hits + arcstats.l2_misses;
+ unsigned long long l2miss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses;
+ unsigned long long l2read = l2hit + l2miss;
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_arc_size = NULL;
+ static RRDDIM *rd_arc_size = NULL;
+ static RRDDIM *rd_arc_target_size = NULL;
+ static RRDDIM *rd_arc_target_min_size = NULL;
+ static RRDDIM *rd_arc_target_max_size = NULL;
+
+ if (unlikely(!st_arc_size)) {
+ st_arc_size = rrdset_create_localhost(
+ "zfs"
+ , "arc_size"
+ , NULL
+ , ZFS_FAMILY_SIZE
+ , NULL
+ , "ZFS ARC Size"
+ , "MB"
+ , 2000
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_arc_size = rrddim_add(st_arc_size, "size", "arcsz", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_arc_target_size = rrddim_add(st_arc_size, "target", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_arc_target_min_size = rrddim_add(st_arc_size, "min", "min (hard limit)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_arc_target_max_size = rrddim_add(st_arc_size, "max", "max (high water)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+ else
+ rrdset_next(st_arc_size);
+
+ rrddim_set_by_pointer(st_arc_size, rd_arc_size, arcstats.size);
+ rrddim_set_by_pointer(st_arc_size, rd_arc_target_size, arcstats.c);
+ rrddim_set_by_pointer(st_arc_size, rd_arc_target_min_size, arcstats.c_min);
+ rrddim_set_by_pointer(st_arc_size, rd_arc_target_max_size, arcstats.c_max);
+ rrdset_done(st_arc_size);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(l2exist)) {
+ static RRDSET *st_l2_size = NULL;
+ static RRDDIM *rd_l2_size = NULL;
+ static RRDDIM *rd_l2_asize = NULL;
+
+ if (unlikely(!st_l2_size)) {
+ st_l2_size = rrdset_create_localhost(
+ "zfs"
+ , "l2_size"
+ , NULL
+ , ZFS_FAMILY_SIZE
+ , NULL
+ , "ZFS L2 ARC Size"
+ , "MB"
+ , 2000
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_l2_asize = rrddim_add(st_l2_size, "actual", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ rd_l2_size = rrddim_add(st_l2_size, "size", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
+ }
+ else
+ rrdset_next(st_l2_size);
+
+ rrddim_set_by_pointer(st_l2_size, rd_l2_size, arcstats.l2_size);
+ rrddim_set_by_pointer(st_l2_size, rd_l2_asize, arcstats.l2_asize);
+ rrdset_done(st_l2_size);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_reads = NULL;
+ static RRDDIM *rd_aread = NULL;
+ static RRDDIM *rd_dread = NULL;
+ static RRDDIM *rd_pread = NULL;
+ static RRDDIM *rd_mread = NULL;
+ static RRDDIM *rd_l2read = NULL;
+
+ if (unlikely(!st_reads)) {
+ st_reads = rrdset_create_localhost(
+ "zfs"
+ , "reads"
+ , NULL
+ , ZFS_FAMILY_ACCESSES
+ , NULL
+ , "ZFS Reads"
+ , "reads/s"
+ , 2010
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_aread = rrddim_add(st_reads, "areads", "arc", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_dread = rrddim_add(st_reads, "dreads", "demand", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_pread = rrddim_add(st_reads, "preads", "prefetch", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mread = rrddim_add(st_reads, "mreads", "metadata", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+
+ if(l2exist)
+ rd_l2read = rrddim_add(st_reads, "l2reads", "l2", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else
+ rrdset_next(st_reads);
+
+ rrddim_set_by_pointer(st_reads, rd_aread, aread);
+ rrddim_set_by_pointer(st_reads, rd_dread, dread);
+ rrddim_set_by_pointer(st_reads, rd_pread, pread);
+ rrddim_set_by_pointer(st_reads, rd_mread, mread);
+
+ if(l2exist)
+ rrddim_set_by_pointer(st_reads, rd_l2read, l2read);
+
+ rrdset_done(st_reads);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(l2exist)) {
+ static RRDSET *st_l2bytes = NULL;
+ static RRDDIM *rd_l2_read_bytes = NULL;
+ static RRDDIM *rd_l2_write_bytes = NULL;
+
+ if (unlikely(!st_l2bytes)) {
+ st_l2bytes = rrdset_create_localhost(
+ "zfs"
+ , "bytes"
+ , NULL
+ , ZFS_FAMILY_ACCESSES
+ , NULL
+ , "ZFS ARC L2 Read/Write Rate"
+ , "kilobytes/s"
+ , 2200
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_l2_read_bytes = rrddim_add(st_l2bytes, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ rd_l2_write_bytes = rrddim_add(st_l2bytes, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else
+ rrdset_next(st_l2bytes);
+
+ rrddim_set_by_pointer(st_l2bytes, rd_l2_read_bytes, arcstats.l2_read_bytes);
+ rrddim_set_by_pointer(st_l2bytes, rd_l2_write_bytes, arcstats.l2_write_bytes);
+ rrdset_done(st_l2bytes);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_ahits = NULL;
+ static RRDDIM *rd_ahits = NULL;
+ static RRDDIM *rd_amisses = NULL;
+
+ if (unlikely(!st_ahits)) {
+ st_ahits = rrdset_create_localhost(
+ "zfs"
+ , "hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS ARC Hits"
+ , "percentage"
+ , 2020
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_ahits = rrddim_add(st_ahits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_amisses = rrddim_add(st_ahits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_ahits);
+
+ rrddim_set_by_pointer(st_ahits, rd_ahits, arcstats.hits);
+ rrddim_set_by_pointer(st_ahits, rd_amisses, arcstats.misses);
+ rrdset_done(st_ahits);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_dhits = NULL;
+ static RRDDIM *rd_dhits = NULL;
+ static RRDDIM *rd_dmisses = NULL;
+
+ if (unlikely(!st_dhits)) {
+ st_dhits = rrdset_create_localhost(
+ "zfs"
+ , "dhits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Demand Hits"
+ , "percentage"
+ , 2030
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_dhits = rrddim_add(st_dhits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_dmisses = rrddim_add(st_dhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_dhits);
+
+ rrddim_set_by_pointer(st_dhits, rd_dhits, dhit);
+ rrddim_set_by_pointer(st_dhits, rd_dmisses, dmiss);
+ rrdset_done(st_dhits);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_phits = NULL;
+ static RRDDIM *rd_phits = NULL;
+ static RRDDIM *rd_pmisses = NULL;
+
+ if (unlikely(!st_phits)) {
+ st_phits = rrdset_create_localhost(
+ "zfs"
+ , "phits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Prefetch Hits"
+ , "percentage"
+ , 2040
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_phits = rrddim_add(st_phits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_pmisses = rrddim_add(st_phits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_phits);
+
+ rrddim_set_by_pointer(st_phits, rd_phits, phit);
+ rrddim_set_by_pointer(st_phits, rd_pmisses, pmiss);
+ rrdset_done(st_phits);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_mhits = NULL;
+ static RRDDIM *rd_mhits = NULL;
+ static RRDDIM *rd_mmisses = NULL;
+
+ if (unlikely(!st_mhits)) {
+ st_mhits = rrdset_create_localhost(
+ "zfs"
+ , "mhits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Metadata Hits"
+ , "percentage"
+ , 2050
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_mhits = rrddim_add(st_mhits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_mmisses = rrddim_add(st_mhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_mhits);
+
+ rrddim_set_by_pointer(st_mhits, rd_mhits, mhit);
+ rrddim_set_by_pointer(st_mhits, rd_mmisses, mmiss);
+ rrdset_done(st_mhits);
+ }
+
+ // --------------------------------------------------------------------
+
+ if(likely(l2exist)) {
+ static RRDSET *st_l2hits = NULL;
+ static RRDDIM *rd_l2hits = NULL;
+ static RRDDIM *rd_l2misses = NULL;
+
+ if (unlikely(!st_l2hits)) {
+ st_l2hits = rrdset_create_localhost(
+ "zfs"
+ , "l2hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS L2 Hits"
+ , "percentage"
+ , 2060
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_l2hits = rrddim_add(st_l2hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_l2misses = rrddim_add(st_l2hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_l2hits);
+
+ rrddim_set_by_pointer(st_l2hits, rd_l2hits, l2hit);
+ rrddim_set_by_pointer(st_l2hits, rd_l2misses, l2miss);
+ rrdset_done(st_l2hits);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_list_hits = NULL;
+ static RRDDIM *rd_mfu = NULL;
+ static RRDDIM *rd_mru = NULL;
+ static RRDDIM *rd_mfug = NULL;
+ static RRDDIM *rd_mrug = NULL;
+
+ if (unlikely(!st_list_hits)) {
+ st_list_hits = rrdset_create_localhost(
+ "zfs"
+ , "list_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS List Hits"
+ , "hits/s"
+ , 2100
+ , update_every
+ , RRDSET_TYPE_AREA
+ );
+
+ rd_mfu = rrddim_add(st_list_hits, "mfu", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mfug = rrddim_add(st_list_hits, "mfug", "mfu ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mru = rrddim_add(st_list_hits, "mru", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mrug = rrddim_add(st_list_hits, "mrug", "mru ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else
+ rrdset_next(st_list_hits);
+
+ rrddim_set_by_pointer(st_list_hits, rd_mfu, arcstats.mfu_hits);
+ rrddim_set_by_pointer(st_list_hits, rd_mru, arcstats.mru_hits);
+ rrddim_set_by_pointer(st_list_hits, rd_mfug, arcstats.mfu_ghost_hits);
+ rrddim_set_by_pointer(st_list_hits, rd_mrug, arcstats.mru_ghost_hits);
+ rrdset_done(st_list_hits);
+ }
+}
+
+void generate_charts_arc_summary(int update_every) {
+ unsigned long long arc_accesses_total = arcstats.hits + arcstats.misses;
+ unsigned long long real_hits = arcstats.mfu_hits + arcstats.mru_hits;
+ unsigned long long real_misses = arc_accesses_total - real_hits;
+
+ //unsigned long long anon_hits = arcstats.hits - (arcstats.mfu_hits + arcstats.mru_hits + arcstats.mfu_ghost_hits + arcstats.mru_ghost_hits);
+
+ unsigned long long arc_size = arcstats.size;
+ unsigned long long mru_size = arcstats.p;
+ //unsigned long long target_min_size = arcstats.c_min;
+ //unsigned long long target_max_size = arcstats.c_max;
+ unsigned long long target_size = arcstats.c;
+ //unsigned long long target_size_ratio = (target_max_size / target_min_size);
+
+ unsigned long long mfu_size;
+ if(arc_size > target_size)
+ mfu_size = arc_size - mru_size;
+ else
+ mfu_size = target_size - mru_size;
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_arc_size_breakdown = NULL;
+ static RRDDIM *rd_most_recent = NULL;
+ static RRDDIM *rd_most_frequent = NULL;
+
+ if (unlikely(!st_arc_size_breakdown)) {
+ st_arc_size_breakdown = rrdset_create_localhost(
+ "zfs"
+ , "arc_size_breakdown"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS ARC Size Breakdown"
+ , "percentage"
+ , 2020
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_most_recent = rrddim_add(st_arc_size_breakdown, "recent", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL);
+ rd_most_frequent = rrddim_add(st_arc_size_breakdown, "frequent", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL);
+ }
+ else
+ rrdset_next(st_arc_size_breakdown);
+
+ rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_recent, mru_size);
+ rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_frequent, mfu_size);
+ rrdset_done(st_arc_size_breakdown);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_memory = NULL;
+#ifndef __FreeBSD__
+ static RRDDIM *rd_direct = NULL;
+#endif
+ static RRDDIM *rd_throttled = NULL;
+#ifndef __FreeBSD__
+ static RRDDIM *rd_indirect = NULL;
+#endif
+
+ if (unlikely(!st_memory)) {
+ st_memory = rrdset_create_localhost(
+ "zfs"
+ , "memory_ops"
+ , NULL
+ , ZFS_FAMILY_OPERATIONS
+ , NULL
+ , "ZFS Memory Operations"
+ , "operations/s"
+ , 2023
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+#ifndef __FreeBSD__
+ rd_direct = rrddim_add(st_memory, "direct", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ rd_throttled = rrddim_add(st_memory, "throttled", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#ifndef __FreeBSD__
+ rd_indirect = rrddim_add(st_memory, "indirect", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+#endif
+ }
+ else
+ rrdset_next(st_memory);
+
+#ifndef __FreeBSD__
+ rrddim_set_by_pointer(st_memory, rd_direct, arcstats.memory_direct_count);
+#endif
+ rrddim_set_by_pointer(st_memory, rd_throttled, arcstats.memory_throttle_count);
+#ifndef __FreeBSD__
+ rrddim_set_by_pointer(st_memory, rd_indirect, arcstats.memory_indirect_count);
+#endif
+ rrdset_done(st_memory);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_important_ops = NULL;
+ static RRDDIM *rd_deleted = NULL;
+ static RRDDIM *rd_mutex_misses = NULL;
+ static RRDDIM *rd_evict_skips = NULL;
+ static RRDDIM *rd_hash_collisions = NULL;
+
+ if (unlikely(!st_important_ops)) {
+ st_important_ops = rrdset_create_localhost(
+ "zfs"
+ , "important_ops"
+ , NULL
+ , ZFS_FAMILY_OPERATIONS
+ , NULL
+ , "ZFS Important Operations"
+ , "operations/s"
+ , 2022
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_evict_skips = rrddim_add(st_important_ops, "eskip", "evict skip", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_deleted = rrddim_add(st_important_ops, "deleted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_mutex_misses = rrddim_add(st_important_ops, "mtxmis", "mutex miss", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ rd_hash_collisions = rrddim_add(st_important_ops, "hash_collisions", "hash collisions", 1, 1, RRD_ALGORITHM_INCREMENTAL);
+ }
+ else
+ rrdset_next(st_important_ops);
+
+ rrddim_set_by_pointer(st_important_ops, rd_deleted, arcstats.deleted);
+ rrddim_set_by_pointer(st_important_ops, rd_evict_skips, arcstats.evict_skip);
+ rrddim_set_by_pointer(st_important_ops, rd_mutex_misses, arcstats.mutex_miss);
+ rrddim_set_by_pointer(st_important_ops, rd_hash_collisions, arcstats.hash_collisions);
+ rrdset_done(st_important_ops);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_actual_hits = NULL;
+ static RRDDIM *rd_actual_hits = NULL;
+ static RRDDIM *rd_actual_misses = NULL;
+
+ if (unlikely(!st_actual_hits)) {
+ st_actual_hits = rrdset_create_localhost(
+ "zfs"
+ , "actual_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Actual Cache Hits"
+ , "percentage"
+ , 2019
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_actual_hits = rrddim_add(st_actual_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_actual_misses = rrddim_add(st_actual_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_actual_hits);
+
+ rrddim_set_by_pointer(st_actual_hits, rd_actual_hits, real_hits);
+ rrddim_set_by_pointer(st_actual_hits, rd_actual_misses, real_misses);
+ rrdset_done(st_actual_hits);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_demand_data_hits = NULL;
+ static RRDDIM *rd_demand_data_hits = NULL;
+ static RRDDIM *rd_demand_data_misses = NULL;
+
+ if (unlikely(!st_demand_data_hits)) {
+ st_demand_data_hits = rrdset_create_localhost(
+ "zfs"
+ , "demand_data_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Data Demand Efficiency"
+ , "percentage"
+ , 2031
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_demand_data_hits = rrddim_add(st_demand_data_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_demand_data_misses = rrddim_add(st_demand_data_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_demand_data_hits);
+
+ rrddim_set_by_pointer(st_demand_data_hits, rd_demand_data_hits, arcstats.demand_data_hits);
+ rrddim_set_by_pointer(st_demand_data_hits, rd_demand_data_misses, arcstats.demand_data_misses);
+ rrdset_done(st_demand_data_hits);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_prefetch_data_hits = NULL;
+ static RRDDIM *rd_prefetch_data_hits = NULL;
+ static RRDDIM *rd_prefetch_data_misses = NULL;
+
+ if (unlikely(!st_prefetch_data_hits)) {
+ st_prefetch_data_hits = rrdset_create_localhost(
+ "zfs"
+ , "prefetch_data_hits"
+ , NULL
+ , ZFS_FAMILY_EFFICIENCY
+ , NULL
+ , "ZFS Data Prefetch Efficiency"
+ , "percentage"
+ , 2032
+ , update_every
+ , RRDSET_TYPE_STACKED
+ );
+
+ rd_prefetch_data_hits = rrddim_add(st_prefetch_data_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ rd_prefetch_data_misses = rrddim_add(st_prefetch_data_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
+ }
+ else
+ rrdset_next(st_prefetch_data_hits);
+
+ rrddim_set_by_pointer(st_prefetch_data_hits, rd_prefetch_data_hits, arcstats.prefetch_data_hits);
+ rrddim_set_by_pointer(st_prefetch_data_hits, rd_prefetch_data_misses, arcstats.prefetch_data_misses);
+ rrdset_done(st_prefetch_data_hits);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_hash_elements = NULL;
+ static RRDDIM *rd_hash_elements_current = NULL;
+ static RRDDIM *rd_hash_elements_max = NULL;
+
+ if (unlikely(!st_hash_elements)) {
+ st_hash_elements = rrdset_create_localhost(
+ "zfs"
+ , "hash_elements"
+ , NULL
+ , ZFS_FAMILY_HASH
+ , NULL
+ , "ZFS ARC Hash Elements"
+ , "elements"
+ , 2300
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_hash_elements_current = rrddim_add(st_hash_elements, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_hash_elements_max = rrddim_add(st_hash_elements, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+ else
+ rrdset_next(st_hash_elements);
+
+ rrddim_set_by_pointer(st_hash_elements, rd_hash_elements_current, arcstats.hash_elements);
+ rrddim_set_by_pointer(st_hash_elements, rd_hash_elements_max, arcstats.hash_elements_max);
+ rrdset_done(st_hash_elements);
+ }
+
+ // --------------------------------------------------------------------
+
+ {
+ static RRDSET *st_hash_chains = NULL;
+ static RRDDIM *rd_hash_chains_current = NULL;
+ static RRDDIM *rd_hash_chains_max = NULL;
+
+ if (unlikely(!st_hash_chains)) {
+ st_hash_chains = rrdset_create_localhost(
+ "zfs"
+ , "hash_chains"
+ , NULL
+ , ZFS_FAMILY_HASH
+ , NULL
+ , "ZFS ARC Hash Chains"
+ , "chains"
+ , 2310
+ , update_every
+ , RRDSET_TYPE_LINE
+ );
+
+ rd_hash_chains_current = rrddim_add(st_hash_chains, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ rd_hash_chains_max = rrddim_add(st_hash_chains, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ }
+ else
+ rrdset_next(st_hash_chains);
+
+ rrddim_set_by_pointer(st_hash_chains, rd_hash_chains_current, arcstats.hash_chains);
+ rrddim_set_by_pointer(st_hash_chains, rd_hash_chains_max, arcstats.hash_chain_max);
+ rrdset_done(st_hash_chains);
+ }
+
+ // --------------------------------------------------------------------
+
+} \ No newline at end of file
diff --git a/src/zfs_common.h b/src/zfs_common.h
new file mode 100644
index 000000000..9d3aa7dfb
--- /dev/null
+++ b/src/zfs_common.h
@@ -0,0 +1,109 @@
+#ifndef NETDATA_ZFS_COMMON_H
+#define NETDATA_ZFS_COMMON_H
+
+#define ZFS_FAMILY_SIZE "size"
+#define ZFS_FAMILY_EFFICIENCY "efficiency"
+#define ZFS_FAMILY_ACCESSES "accesses"
+#define ZFS_FAMILY_OPERATIONS "operations"
+#define ZFS_FAMILY_HASH "hashes"
+
+struct arcstats {
+ unsigned long long hits;
+ unsigned long long misses;
+ unsigned long long demand_data_hits;
+ unsigned long long demand_data_misses;
+ unsigned long long demand_metadata_hits;
+ unsigned long long demand_metadata_misses;
+ unsigned long long prefetch_data_hits;
+ unsigned long long prefetch_data_misses;
+ unsigned long long prefetch_metadata_hits;
+ unsigned long long prefetch_metadata_misses;
+ unsigned long long mru_hits;
+ unsigned long long mru_ghost_hits;
+ unsigned long long mfu_hits;
+ unsigned long long mfu_ghost_hits;
+ unsigned long long deleted;
+ unsigned long long mutex_miss;
+ unsigned long long evict_skip;
+ unsigned long long evict_not_enough;
+ unsigned long long evict_l2_cached;
+ unsigned long long evict_l2_eligible;
+ unsigned long long evict_l2_ineligible;
+ unsigned long long evict_l2_skip;
+ unsigned long long hash_elements;
+ unsigned long long hash_elements_max;
+ unsigned long long hash_collisions;
+ unsigned long long hash_chains;
+ unsigned long long hash_chain_max;
+ unsigned long long p;
+ unsigned long long c;
+ unsigned long long c_min;
+ unsigned long long c_max;
+ unsigned long long size;
+ unsigned long long hdr_size;
+ unsigned long long data_size;
+ unsigned long long metadata_size;
+ unsigned long long other_size;
+ unsigned long long anon_size;
+ unsigned long long anon_evictable_data;
+ unsigned long long anon_evictable_metadata;
+ unsigned long long mru_size;
+ unsigned long long mru_evictable_data;
+ unsigned long long mru_evictable_metadata;
+ unsigned long long mru_ghost_size;
+ unsigned long long mru_ghost_evictable_data;
+ unsigned long long mru_ghost_evictable_metadata;
+ unsigned long long mfu_size;
+ unsigned long long mfu_evictable_data;
+ unsigned long long mfu_evictable_metadata;
+ unsigned long long mfu_ghost_size;
+ unsigned long long mfu_ghost_evictable_data;
+ unsigned long long mfu_ghost_evictable_metadata;
+ unsigned long long l2_hits;
+ unsigned long long l2_misses;
+ unsigned long long l2_feeds;
+ unsigned long long l2_rw_clash;
+ unsigned long long l2_read_bytes;
+ unsigned long long l2_write_bytes;
+ unsigned long long l2_writes_sent;
+ unsigned long long l2_writes_done;
+ unsigned long long l2_writes_error;
+ unsigned long long l2_writes_lock_retry;
+ unsigned long long l2_evict_lock_retry;
+ unsigned long long l2_evict_reading;
+ unsigned long long l2_evict_l1cached;
+ unsigned long long l2_free_on_write;
+ unsigned long long l2_cdata_free_on_write;
+ unsigned long long l2_abort_lowmem;
+ unsigned long long l2_cksum_bad;
+ unsigned long long l2_io_error;
+ unsigned long long l2_size;
+ unsigned long long l2_asize;
+ unsigned long long l2_hdr_size;
+ unsigned long long l2_compress_successes;
+ unsigned long long l2_compress_zeros;
+ unsigned long long l2_compress_failures;
+ unsigned long long memory_throttle_count;
+ unsigned long long duplicate_buffers;
+ unsigned long long duplicate_buffers_size;
+ unsigned long long duplicate_reads;
+ unsigned long long memory_direct_count;
+ unsigned long long memory_indirect_count;
+ unsigned long long arc_no_grow;
+ unsigned long long arc_tempreserve;
+ unsigned long long arc_loaned_bytes;
+ unsigned long long arc_prune;
+ unsigned long long arc_meta_used;
+ unsigned long long arc_meta_limit;
+ unsigned long long arc_meta_max;
+ unsigned long long arc_meta_min;
+ unsigned long long arc_need_free;
+ unsigned long long arc_sys_free;
+};
+
+int l2exist;
+
+void generate_charts_arcstats(int update_every);
+void generate_charts_arc_summary(int update_every);
+
+#endif //NETDATA_ZFS_COMMON_H