diff options
Diffstat (limited to 'src')
79 files changed, 9754 insertions, 3085 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 1c1dd3385..601d3204f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -44,6 +44,8 @@ netdata_SOURCES = \ appconfig.h \ avl.c \ avl.h \ + backend_prometheus.c \ + backend_prometheus.h \ backends.c \ backends.h \ clocks.c \ @@ -119,6 +121,10 @@ netdata_SOURCES = \ simple_pattern.h \ socket.c \ socket.h \ + statistical.c \ + statistical.h \ + statsd.c \ + statsd.h \ storage_number.c \ storage_number.h \ sys_devices_system_edac_mc.c \ @@ -146,6 +152,13 @@ netdata_SOURCES += \ plugin_freebsd.c \ plugin_freebsd.h \ freebsd_sysctl.c \ + freebsd_getmntinfo.c \ + freebsd_getifaddrs.c \ + freebsd_devstat.c \ + zfs_common.c \ + zfs_common.h \ + freebsd_kstat_zfs.c \ + freebsd_ipfw.c \ $(NULL) else if MACOS @@ -178,6 +191,9 @@ netdata_SOURCES += \ proc_net_softnet_stat.c \ proc_net_stat_conntrack.c \ proc_net_stat_synproxy.c \ + zfs_common.c \ + zfs_common.h \ + proc_spl_kstat_zfs.c \ proc_stat.c \ proc_sys_kernel_random_entropy_avail.c \ proc_vmstat.c \ diff --git a/src/Makefile.in b/src/Makefile.in index d7229d18a..3ce869b0d 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -1,9 +1,8 @@ -# Makefile.in generated by automake 1.11.3 from Makefile.am. +# Makefile.in generated by automake 1.14.1 from Makefile.am. # @configure_input@ -# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, -# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software -# Foundation, Inc. +# Copyright (C) 1994-2013 Free Software Foundation, Inc. + # This Makefile.in is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. @@ -17,6 +16,51 @@ VPATH = @srcdir@ +am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)' +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) pkgdatadir = $(datadir)/@PACKAGE@ pkgincludedir = $(includedir)/@PACKAGE@ pkglibdir = $(libdir)/@PACKAGE@ @@ -43,6 +87,13 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) @FREEBSD_TRUE@ plugin_freebsd.c \ @FREEBSD_TRUE@ plugin_freebsd.h \ @FREEBSD_TRUE@ freebsd_sysctl.c \ +@FREEBSD_TRUE@ freebsd_getmntinfo.c \ +@FREEBSD_TRUE@ freebsd_getifaddrs.c \ +@FREEBSD_TRUE@ freebsd_devstat.c \ +@FREEBSD_TRUE@ zfs_common.c \ +@FREEBSD_TRUE@ zfs_common.h \ +@FREEBSD_TRUE@ freebsd_kstat_zfs.c \ +@FREEBSD_TRUE@ freebsd_ipfw.c \ @FREEBSD_TRUE@ $(NULL) @FREEBSD_FALSE@@MACOS_TRUE@am__append_4 = \ @@ -74,6 +125,9 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ zfs_common.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ zfs_common.h \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_spl_kstat_zfs.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_stat.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_sys_kernel_random_entropy_avail.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_vmstat.c \ @@ -86,9 +140,9 @@ plugins_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) @FREEBSD_TRUE@ $(NULL) subdir = src -DIST_COMMON = $(dist_cache_DATA) $(dist_log_DATA) \ - $(dist_registry_DATA) $(dist_varlib_DATA) \ - $(srcdir)/Makefile.am $(srcdir)/Makefile.in +DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ + $(top_srcdir)/depcomp $(dist_cache_DATA) $(dist_log_DATA) \ + $(dist_registry_DATA) $(dist_varlib_DATA) ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/ax_c___atomic.m4 \ $(top_srcdir)/m4/ax_c__generic.m4 $(top_srcdir)/m4/ax_c_lto.m4 \ @@ -127,8 +181,9 @@ freeipmi_plugin_OBJECTS = $(am_freeipmi_plugin_OBJECTS) freeipmi_plugin_DEPENDENCIES = $(am__DEPENDENCIES_1) am__netdata_SOURCES_DIST = adaptive_resortable_list.c \ adaptive_resortable_list.h appconfig.c appconfig.h avl.c avl.h \ - backends.c backends.h clocks.c clocks.h common.c common.h \ - daemon.c daemon.h dictionary.c dictionary.h eval.c eval.h \ + backend_prometheus.c backend_prometheus.h backends.c \ + backends.h clocks.c clocks.h common.c common.h daemon.c \ + daemon.h dictionary.c dictionary.h eval.c eval.h \ global_statistics.c global_statistics.h health.c health.h \ health_config.c health_json.c health_log.c inlined.h locks.h \ log.c log.h main.c main.h plugin_checks.c plugin_checks.h \ @@ -144,25 +199,34 @@ am__netdata_SOURCES_DIST = adaptive_resortable_list.c \ rrdcalctemplate.c rrddim.c rrddimvar.c rrdfamily.c rrdhost.c \ rrdpush.c rrdpush.h rrdset.c rrdsetvar.c rrdvar.c \ simple_pattern.c simple_pattern.h socket.c socket.h \ - storage_number.c storage_number.h sys_devices_system_edac_mc.c \ + statistical.c statistical.h statsd.c statsd.h storage_number.c \ + storage_number.h sys_devices_system_edac_mc.c \ sys_devices_system_node.c sys_fs_cgroup.c unit_test.c \ unit_test.h url.c url.h web_api_old.c web_api_old.h \ web_api_v1.c web_api_v1.h web_buffer.c web_buffer.h \ web_buffer_svg.c web_buffer_svg.h web_client.c web_client.h \ web_server.c web_server.h plugin_freebsd.c plugin_freebsd.h \ - freebsd_sysctl.c plugin_macos.c plugin_macos.h macos_sysctl.c \ - macos_mach_smi.c macos_fw.c ipc.c ipc.h plugin_proc.c \ - plugin_proc.h plugin_proc_diskspace.c plugin_proc_diskspace.h \ + freebsd_sysctl.c freebsd_getmntinfo.c freebsd_getifaddrs.c \ + freebsd_devstat.c zfs_common.c zfs_common.h \ + freebsd_kstat_zfs.c freebsd_ipfw.c plugin_macos.c \ + plugin_macos.h macos_sysctl.c macos_mach_smi.c macos_fw.c \ + ipc.c ipc.h plugin_proc.c plugin_proc.h \ + plugin_proc_diskspace.c plugin_proc_diskspace.h \ proc_diskstats.c proc_interrupts.c proc_softirqs.c \ proc_loadavg.c proc_meminfo.c proc_net_dev.c \ proc_net_ip_vs_stats.c proc_net_netstat.c proc_net_rpc_nfs.c \ proc_net_rpc_nfsd.c proc_net_snmp.c proc_net_snmp6.c \ proc_net_softnet_stat.c proc_net_stat_conntrack.c \ - proc_net_stat_synproxy.c proc_stat.c \ + proc_net_stat_synproxy.c proc_spl_kstat_zfs.c proc_stat.c \ proc_sys_kernel_random_entropy_avail.c proc_vmstat.c \ proc_uptime.c sys_kernel_mm_ksm.c @FREEBSD_TRUE@am__objects_2 = plugin_freebsd.$(OBJEXT) \ -@FREEBSD_TRUE@ freebsd_sysctl.$(OBJEXT) +@FREEBSD_TRUE@ freebsd_sysctl.$(OBJEXT) \ +@FREEBSD_TRUE@ freebsd_getmntinfo.$(OBJEXT) \ +@FREEBSD_TRUE@ freebsd_getifaddrs.$(OBJEXT) \ +@FREEBSD_TRUE@ freebsd_devstat.$(OBJEXT) zfs_common.$(OBJEXT) \ +@FREEBSD_TRUE@ freebsd_kstat_zfs.$(OBJEXT) \ +@FREEBSD_TRUE@ freebsd_ipfw.$(OBJEXT) @FREEBSD_FALSE@@MACOS_TRUE@am__objects_3 = plugin_macos.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_TRUE@ macos_sysctl.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_TRUE@ macos_mach_smi.$(OBJEXT) \ @@ -185,15 +249,17 @@ am__netdata_SOURCES_DIST = adaptive_resortable_list.c \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ zfs_common.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_spl_kstat_zfs.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_stat.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_sys_kernel_random_entropy_avail.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_vmstat.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ proc_uptime.$(OBJEXT) \ @FREEBSD_FALSE@@MACOS_FALSE@ sys_kernel_mm_ksm.$(OBJEXT) am_netdata_OBJECTS = adaptive_resortable_list.$(OBJEXT) \ - appconfig.$(OBJEXT) avl.$(OBJEXT) backends.$(OBJEXT) \ - clocks.$(OBJEXT) common.$(OBJEXT) daemon.$(OBJEXT) \ - dictionary.$(OBJEXT) eval.$(OBJEXT) \ + appconfig.$(OBJEXT) avl.$(OBJEXT) backend_prometheus.$(OBJEXT) \ + backends.$(OBJEXT) clocks.$(OBJEXT) common.$(OBJEXT) \ + daemon.$(OBJEXT) dictionary.$(OBJEXT) eval.$(OBJEXT) \ global_statistics.$(OBJEXT) health.$(OBJEXT) \ health_config.$(OBJEXT) health_json.$(OBJEXT) \ health_log.$(OBJEXT) log.$(OBJEXT) main.$(OBJEXT) \ @@ -210,6 +276,7 @@ am_netdata_OBJECTS = adaptive_resortable_list.$(OBJEXT) \ rrddimvar.$(OBJEXT) rrdfamily.$(OBJEXT) rrdhost.$(OBJEXT) \ rrdpush.$(OBJEXT) rrdset.$(OBJEXT) rrdsetvar.$(OBJEXT) \ rrdvar.$(OBJEXT) simple_pattern.$(OBJEXT) socket.$(OBJEXT) \ + statistical.$(OBJEXT) statsd.$(OBJEXT) \ storage_number.$(OBJEXT) sys_devices_system_edac_mc.$(OBJEXT) \ sys_devices_system_node.$(OBJEXT) sys_fs_cgroup.$(OBJEXT) \ unit_test.$(OBJEXT) url.$(OBJEXT) web_api_old.$(OBJEXT) \ @@ -220,18 +287,43 @@ am_netdata_OBJECTS = adaptive_resortable_list.$(OBJEXT) \ netdata_OBJECTS = $(am_netdata_OBJECTS) netdata_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/depcomp am__depfiles_maybe = depfiles am__mv = mv -f COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = CCLD = $(CC) LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = SOURCES = $(apps_plugin_SOURCES) $(freeipmi_plugin_SOURCES) \ $(netdata_SOURCES) DIST_SOURCES = $(am__apps_plugin_SOURCES_DIST) \ $(freeipmi_plugin_SOURCES) $(am__netdata_SOURCES_DIST) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; am__vpath_adj = case $$p in \ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ @@ -261,11 +353,29 @@ am__uninstall_files_from_dir = { \ } DATA = $(dist_cache_DATA) $(dist_log_DATA) $(dist_registry_DATA) \ $(dist_varlib_DATA) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` ETAGS = etags CTAGS = ctags DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) ACLOCAL = @ACLOCAL@ AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ @@ -437,8 +547,9 @@ dist_registry_DATA = .keep dist_log_DATA = .keep netdata_SOURCES = adaptive_resortable_list.c \ adaptive_resortable_list.h appconfig.c appconfig.h avl.c avl.h \ - backends.c backends.h clocks.c clocks.h common.c common.h \ - daemon.c daemon.h dictionary.c dictionary.h eval.c eval.h \ + backend_prometheus.c backend_prometheus.h backends.c \ + backends.h clocks.c clocks.h common.c common.h daemon.c \ + daemon.h dictionary.c dictionary.h eval.c eval.h \ global_statistics.c global_statistics.h health.c health.h \ health_config.c health_json.c health_log.c inlined.h locks.h \ log.c log.h main.c main.h plugin_checks.c plugin_checks.h \ @@ -454,7 +565,8 @@ netdata_SOURCES = adaptive_resortable_list.c \ rrdcalctemplate.c rrddim.c rrddimvar.c rrdfamily.c rrdhost.c \ rrdpush.c rrdpush.h rrdset.c rrdsetvar.c rrdvar.c \ simple_pattern.c simple_pattern.h socket.c socket.h \ - storage_number.c storage_number.h sys_devices_system_edac_mc.c \ + statistical.c statistical.h statsd.c statsd.h storage_number.c \ + storage_number.h sys_devices_system_edac_mc.c \ sys_devices_system_node.c sys_fs_cgroup.c unit_test.c \ unit_test.h url.c url.h web_api_old.c web_api_old.h \ web_api_v1.c web_api_v1.h web_buffer.c web_buffer.h \ @@ -525,14 +637,18 @@ $(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) $(am__aclocal_m4_deps): install-pluginsPROGRAMS: $(plugins_PROGRAMS) @$(NORMAL_INSTALL) - test -z "$(pluginsdir)" || $(MKDIR_P) "$(DESTDIR)$(pluginsdir)" @list='$(plugins_PROGRAMS)'; test -n "$(pluginsdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pluginsdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pluginsdir)" || exit 1; \ + fi; \ for p in $$list; do echo "$$p $$p"; done | \ sed 's/$(EXEEXT)$$//' | \ - while read p p1; do if test -f $$p; \ - then echo "$$p"; echo "$$p"; else :; fi; \ + while read p p1; do if test -f $$p \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ done | \ - sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ sed 'N;N;N;s,\n, ,g' | \ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ @@ -553,7 +669,8 @@ uninstall-pluginsPROGRAMS: @list='$(plugins_PROGRAMS)'; test -n "$(pluginsdir)" || list=; \ files=`for p in $$list; do echo "$$p"; done | \ sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ - -e 's/$$/$(EXEEXT)/' `; \ + -e 's/$$/$(EXEEXT)/' \ + `; \ test -n "$$list" || exit 0; \ echo " ( cd '$(DESTDIR)$(pluginsdir)' && rm -f" $$files ")"; \ cd "$(DESTDIR)$(pluginsdir)" && rm -f $$files @@ -562,14 +679,18 @@ clean-pluginsPROGRAMS: -test -z "$(plugins_PROGRAMS)" || rm -f $(plugins_PROGRAMS) install-sbinPROGRAMS: $(sbin_PROGRAMS) @$(NORMAL_INSTALL) - test -z "$(sbindir)" || $(MKDIR_P) "$(DESTDIR)$(sbindir)" @list='$(sbin_PROGRAMS)'; test -n "$(sbindir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(sbindir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(sbindir)" || exit 1; \ + fi; \ for p in $$list; do echo "$$p $$p"; done | \ sed 's/$(EXEEXT)$$//' | \ - while read p p1; do if test -f $$p; \ - then echo "$$p"; echo "$$p"; else :; fi; \ + while read p p1; do if test -f $$p \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ done | \ - sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ sed 'N;N;N;s,\n, ,g' | \ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ @@ -590,22 +711,26 @@ uninstall-sbinPROGRAMS: @list='$(sbin_PROGRAMS)'; test -n "$(sbindir)" || list=; \ files=`for p in $$list; do echo "$$p"; done | \ sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ - -e 's/$$/$(EXEEXT)/' `; \ + -e 's/$$/$(EXEEXT)/' \ + `; \ test -n "$$list" || exit 0; \ echo " ( cd '$(DESTDIR)$(sbindir)' && rm -f" $$files ")"; \ cd "$(DESTDIR)$(sbindir)" && rm -f $$files clean-sbinPROGRAMS: -test -z "$(sbin_PROGRAMS)" || rm -f $(sbin_PROGRAMS) + apps.plugin$(EXEEXT): $(apps_plugin_OBJECTS) $(apps_plugin_DEPENDENCIES) $(EXTRA_apps_plugin_DEPENDENCIES) @rm -f apps.plugin$(EXEEXT) - $(LINK) $(apps_plugin_OBJECTS) $(apps_plugin_LDADD) $(LIBS) + $(AM_V_CCLD)$(LINK) $(apps_plugin_OBJECTS) $(apps_plugin_LDADD) $(LIBS) + freeipmi.plugin$(EXEEXT): $(freeipmi_plugin_OBJECTS) $(freeipmi_plugin_DEPENDENCIES) $(EXTRA_freeipmi_plugin_DEPENDENCIES) @rm -f freeipmi.plugin$(EXEEXT) - $(LINK) $(freeipmi_plugin_OBJECTS) $(freeipmi_plugin_LDADD) $(LIBS) + $(AM_V_CCLD)$(LINK) $(freeipmi_plugin_OBJECTS) $(freeipmi_plugin_LDADD) $(LIBS) + netdata$(EXEEXT): $(netdata_OBJECTS) $(netdata_DEPENDENCIES) $(EXTRA_netdata_DEPENDENCIES) @rm -f netdata$(EXEEXT) - $(LINK) $(netdata_OBJECTS) $(netdata_LDADD) $(LIBS) + $(AM_V_CCLD)$(LINK) $(netdata_OBJECTS) $(netdata_LDADD) $(LIBS) mostlyclean-compile: -rm -f *.$(OBJEXT) @@ -617,12 +742,18 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/appconfig.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/apps_plugin.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/avl.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/backend_prometheus.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/backends.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/clocks.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/daemon.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dictionary.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/eval.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_devstat.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_getifaddrs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_getmntinfo.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_ipfw.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_kstat_zfs.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_sysctl.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freeipmi_plugin.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/global_statistics.Po@am__quote@ @@ -662,6 +793,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_net_stat_synproxy.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_self_mountinfo.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_softirqs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_spl_kstat_zfs.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_stat.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_sys_kernel_random_entropy_avail.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_uptime.Po@am__quote@ @@ -690,6 +822,8 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rrdvar.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/simple_pattern.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/socket.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statistical.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statsd.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/storage_number.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_devices_system_edac_mc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_devices_system_node.Po@am__quote@ @@ -703,24 +837,28 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/web_buffer_svg.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/web_client.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/web_server.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/zfs_common.Po@am__quote@ .c.o: -@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< -@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po -@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(COMPILE) -c $< +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< .c.obj: -@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` -@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po -@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'` +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` install-dist_cacheDATA: $(dist_cache_DATA) @$(NORMAL_INSTALL) - test -z "$(cachedir)" || $(MKDIR_P) "$(DESTDIR)$(cachedir)" @list='$(dist_cache_DATA)'; test -n "$(cachedir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(cachedir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(cachedir)" || exit 1; \ + fi; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ echo "$$d$$p"; \ @@ -737,8 +875,11 @@ uninstall-dist_cacheDATA: dir='$(DESTDIR)$(cachedir)'; $(am__uninstall_files_from_dir) install-dist_logDATA: $(dist_log_DATA) @$(NORMAL_INSTALL) - test -z "$(logdir)" || $(MKDIR_P) "$(DESTDIR)$(logdir)" @list='$(dist_log_DATA)'; test -n "$(logdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(logdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(logdir)" || exit 1; \ + fi; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ echo "$$d$$p"; \ @@ -755,8 +896,11 @@ uninstall-dist_logDATA: dir='$(DESTDIR)$(logdir)'; $(am__uninstall_files_from_dir) install-dist_registryDATA: $(dist_registry_DATA) @$(NORMAL_INSTALL) - test -z "$(registrydir)" || $(MKDIR_P) "$(DESTDIR)$(registrydir)" @list='$(dist_registry_DATA)'; test -n "$(registrydir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(registrydir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(registrydir)" || exit 1; \ + fi; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ echo "$$d$$p"; \ @@ -773,8 +917,11 @@ uninstall-dist_registryDATA: dir='$(DESTDIR)$(registrydir)'; $(am__uninstall_files_from_dir) install-dist_varlibDATA: $(dist_varlib_DATA) @$(NORMAL_INSTALL) - test -z "$(varlibdir)" || $(MKDIR_P) "$(DESTDIR)$(varlibdir)" @list='$(dist_varlib_DATA)'; test -n "$(varlibdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(varlibdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(varlibdir)" || exit 1; \ + fi; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ echo "$$d$$p"; \ @@ -790,26 +937,15 @@ uninstall-dist_varlibDATA: files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ dir='$(DESTDIR)$(varlibdir)'; $(am__uninstall_files_from_dir) -ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) - list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ - unique=`for i in $$list; do \ - if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ - done | \ - $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ - END { if (nonempty) { for (i in files) print i; }; }'`; \ - mkid -fID $$unique -tags: TAGS - -TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ - $(TAGS_FILES) $(LISP) +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) set x; \ here=`pwd`; \ - list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ - unique=`for i in $$list; do \ - if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ - done | \ - $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ - END { if (nonempty) { for (i in files) print i; }; }'`; \ + $(am__define_uniq_tagged_files); \ shift; \ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ test -n "$$unique" || unique=$$empty_fix; \ @@ -821,15 +957,11 @@ TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ $$unique; \ fi; \ fi -ctags: CTAGS -CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ - $(TAGS_FILES) $(LISP) - list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ - unique=`for i in $$list; do \ - if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ - done | \ - $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ - END { if (nonempty) { for (i in files) print i; }; }'`; \ +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ test -z "$(CTAGS_ARGS)$$unique" \ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ $$unique @@ -838,6 +970,21 @@ GTAGS: here=`$(am__cd) $(top_builddir) && pwd` \ && $(am__cd) $(top_srcdir) \ && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files distclean-tags: -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags @@ -986,20 +1133,20 @@ uninstall-am: uninstall-dist_cacheDATA uninstall-dist_logDATA \ .MAKE: install-am install-strip -.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \ - clean-pluginsPROGRAMS clean-sbinPROGRAMS ctags distclean \ - distclean-compile distclean-generic distclean-tags distdir dvi \ - dvi-am html html-am info info-am install install-am \ - install-data install-data-am install-dist_cacheDATA \ - install-dist_logDATA install-dist_registryDATA \ - install-dist_varlibDATA install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-pdf \ - install-pdf-am install-pluginsPROGRAMS install-ps \ +.PHONY: CTAGS GTAGS TAGS all all-am check check-am clean clean-generic \ + clean-pluginsPROGRAMS clean-sbinPROGRAMS cscopelist-am ctags \ + ctags-am distclean distclean-compile distclean-generic \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am \ + install-dist_cacheDATA install-dist_logDATA \ + install-dist_registryDATA install-dist_varlibDATA install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-pluginsPROGRAMS install-ps \ install-ps-am install-sbinPROGRAMS install-strip installcheck \ installcheck-am installdirs maintainer-clean \ maintainer-clean-generic mostlyclean mostlyclean-compile \ - mostlyclean-generic pdf pdf-am ps ps-am tags uninstall \ + mostlyclean-generic pdf pdf-am ps ps-am tags tags-am uninstall \ uninstall-am uninstall-dist_cacheDATA uninstall-dist_logDATA \ uninstall-dist_registryDATA uninstall-dist_varlibDATA \ uninstall-pluginsPROGRAMS uninstall-sbinPROGRAMS diff --git a/src/appconfig.c b/src/appconfig.c index 71ff4b75e..91c4c5c54 100644 --- a/src/appconfig.c +++ b/src/appconfig.c @@ -135,6 +135,7 @@ static inline struct section *appconfig_section_create(struct config *root, cons struct section *co = callocz(1, sizeof(struct section)); co->name = strdupz(section); co->hash = simple_hash(co->name); + netdata_mutex_init(&co->mutex); avl_init_lock(&co->values_index, appconfig_option_compare); @@ -213,7 +214,8 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam if(!co_new) co_new = appconfig_section_create(root, section_new); config_section_wrlock(co_old); - config_section_wrlock(co_new); + if(co_old != co_new) + config_section_wrlock(co_new); cv_old = appconfig_option_index_find(co_old, name_old, 0); if(!cv_old) goto cleanup; @@ -250,7 +252,8 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam ret = 0; cleanup: - config_section_unlock(co_new); + if(co_old != co_new) + config_section_unlock(co_new); config_section_unlock(co_old); return ret; } @@ -294,6 +297,17 @@ long long appconfig_get_number(struct config *root, const char *section, const c return strtoll(s, NULL, 0); } +long double appconfig_get_float(struct config *root, const char *section, const char *name, long double value) +{ + char buffer[100], *s; + sprintf(buffer, "%0.5Lf", value); + + s = appconfig_get(root, section, name, buffer); + if(!s) return value; + + return str2ld(s, NULL); +} + int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value) { char *s; @@ -337,7 +351,7 @@ const char *appconfig_set_default(struct config *root, const char *section, cons { struct config_option *cv; - debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value); + debug(D_CONFIG, "request to set default config in section '%s', name '%s', value '%s'", section, name, value); struct section *co = appconfig_section_find(root, section); if(!co) return appconfig_set(root, section, name, value); @@ -393,6 +407,16 @@ long long appconfig_set_number(struct config *root, const char *section, const c return value; } +long double appconfig_set_float(struct config *root, const char *section, const char *name, long double value) +{ + char buffer[100]; + sprintf(buffer, "%0.5Lf", value); + + appconfig_set(root, section, name, buffer); + + return value; +} + int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value) { char *s; @@ -417,11 +441,11 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used) if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME; - debug(D_CONFIG, "Opening config file '%s'", filename); + debug(D_CONFIG, "CONFIG: opening config file '%s'", filename); FILE *fp = fopen(filename, "r"); if(!fp) { - error("Cannot open file '%s'", filename); + error("CONFIG: cannot open file '%s'", filename); return 0; } @@ -430,8 +454,8 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used) line++; s = trim(buffer); - if(!s) { - debug(D_CONFIG, "Ignoring line %d, it is empty.", line); + if(!s || *s == '#') { + debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename); continue; } @@ -449,14 +473,14 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used) if(!co) { // line outside a section - error("Ignoring line %d ('%s'), it is outside all sections.", line, s); + error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename); continue; } char *name = s; char *value = strchr(s, '='); if(!value) { - error("Ignoring line %d ('%s'), there is no = in it.", line, s); + error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename); continue; } *value = '\0'; @@ -465,12 +489,12 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used) name = trim(name); value = trim(value); - if(!name) { - error("Ignoring line %d, name is empty.", line); + if(!name || *name == '#') { + error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename); continue; } if(!value) { - debug(D_CONFIG, "Ignoring line %d, value is empty.", line); + debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', value is empty.", line, filename); continue; } @@ -479,12 +503,12 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used) if(!cv) cv = appconfig_value_create(co, name, value); else { if(((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) { - debug(D_CONFIG, "Line %d, overwriting '%s/%s'.", line, co->name, cv->name); + debug(D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name); freez(cv->value); cv->value = strdupz(value); } else - debug(D_CONFIG, "Ignoring line %d, '%s/%s' is already present and used.", line, co->name, cv->name); + debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.", line, filename, co->name, cv->name); } cv->flags |= CONFIG_VALUE_LOADED; } @@ -531,6 +555,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) for(co = root->sections; co ; co = co->next) { if(!strcmp(co->name, CONFIG_SECTION_GLOBAL) || !strcmp(co->name, CONFIG_SECTION_WEB) + || !strcmp(co->name, CONFIG_SECTION_STATSD) || !strcmp(co->name, CONFIG_SECTION_PLUGINS) || !strcmp(co->name, CONFIG_SECTION_REGISTRY) || !strcmp(co->name, CONFIG_SECTION_HEALTH) @@ -558,7 +583,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) if(only_changed && !changed) continue; if(!used) { - buffer_sprintf(wb, "\n# node '%s' is not used.", co->name); + buffer_sprintf(wb, "\n# section '%s' is not used.", co->name); } buffer_sprintf(wb, "\n[%s]\n", co->name); diff --git a/src/appconfig.h b/src/appconfig.h index 45cc8cfd5..b8c2ee80c 100644 --- a/src/appconfig.h +++ b/src/appconfig.h @@ -5,6 +5,7 @@ #define CONFIG_SECTION_GLOBAL "global" #define CONFIG_SECTION_WEB "web" +#define CONFIG_SECTION_STATSD "statsd" #define CONFIG_SECTION_PLUGINS "plugins" #define CONFIG_SECTION_REGISTRY "registry" #define CONFIG_SECTION_HEALTH "health" @@ -34,12 +35,14 @@ extern int appconfig_load(struct config *root, char *filename, int overwrite_use extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value); extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value); +extern long double appconfig_get_float(struct config *root, const char *section, const char *name, long double value); extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value); extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value); extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value); extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value); extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value); +extern long double appconfig_set_float(struct config *root, const char *section, const char *name, long double value); extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value); extern int appconfig_exists(struct config *root, const char *section, const char *name); @@ -53,12 +56,14 @@ extern void appconfig_generate(struct config *root, BUFFER *wb, int only_changed #define config_load(filename, overwrite_used) appconfig_load(&netdata_config, filename, overwrite_used) #define config_get(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value) #define config_get_number(section, name, value) appconfig_get_number(&netdata_config, section, name, value) +#define config_get_float(section, name, value) appconfig_get_float(&netdata_config, section, name, value) #define config_get_boolean(section, name, value) appconfig_get_boolean(&netdata_config, section, name, value) #define config_get_boolean_ondemand(section, name, value) appconfig_get_boolean_ondemand(&netdata_config, section, name, value) -#define config_set(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value) +#define config_set(section, name, default_value) appconfig_set(&netdata_config, section, name, default_value) #define config_set_default(section, name, value) appconfig_set_default(&netdata_config, section, name, value) #define config_set_number(section, name, value) appconfig_set_number(&netdata_config, section, name, value) +#define config_set_float(section, name, value) appconfig_set_float(&netdata_config, section, name, value) #define config_set_boolean(section, name, value) appconfig_set_boolean(&netdata_config, section, name, value) #define config_exists(section, name) appconfig_exists(&netdata_config, section, name) diff --git a/src/apps_plugin.c b/src/apps_plugin.c index b1bf06bee..ecb6aaeac 100644 --- a/src/apps_plugin.c +++ b/src/apps_plugin.c @@ -34,8 +34,7 @@ #define MAX_COMPARE_NAME 100 #define MAX_NAME 100 -#define MAX_CMDLINE 1024 - +#define MAX_CMDLINE 16384 // ---------------------------------------------------------------------------- // the rates we are going to send to netdata will have this detail a value of: @@ -109,11 +108,12 @@ static size_t // metric. // the total system time, as reported by /proc/stat +#if (ALL_PIDS_ARE_READ_INSTANTLY == 0) static kernel_uint_t global_utime = 0, global_stime = 0, global_gtime = 0; - +#endif // the normalization ratios, as calculated by normalize_utilization() double utime_fix_ratio = 1.0, @@ -127,7 +127,6 @@ double utime_fix_ratio = 1.0, cminflt_fix_ratio = 1.0, cmajflt_fix_ratio = 1.0; - // ---------------------------------------------------------------------------- // target // @@ -223,7 +222,7 @@ size_t struct pid_stat { int32_t pid; char comm[MAX_COMPARE_NAME + 1]; - char cmdline[MAX_CMDLINE + 1]; + char *cmdline; uint32_t log_thrown; @@ -438,7 +437,7 @@ static struct target *get_users_target(uid_t uid) { w->idhash = simple_hash(w->id); struct passwd *pw = getpwuid(uid); - if(!pw) + if(!pw || !pw->pw_name || !*pw->pw_name) snprintfz(w->name, MAX_NAME, "%u", uid); else snprintfz(w->name, MAX_NAME, "%s", pw->pw_name); @@ -471,7 +470,7 @@ struct target *get_groups_target(gid_t gid) w->idhash = simple_hash(w->id); struct group *gr = getgrgid(gid); - if(!gr) + if(!gr || !gr->gr_name || !*gr->gr_name) snprintfz(w->name, MAX_NAME, "%u", gid); else snprintfz(w->name, MAX_NAME, "%s", gr->gr_name); @@ -698,6 +697,7 @@ static inline void del_pid_entry(pid_t pid) { freez(p->statm_filename); freez(p->io_filename); freez(p->cmdline_filename); + freez(p->cmdline); freez(p); all_pids[pid] = NULL; @@ -768,7 +768,7 @@ static inline void assign_target_to_pid(struct pid_stat *p) { if(unlikely(( (!w->starts_with && !w->ends_with && w->comparehash == hash && !strcmp(w->compare, p->comm)) || (w->starts_with && !w->ends_with && !strncmp(w->compare, p->comm, w->comparelen)) || (!w->starts_with && w->ends_with && pclen >= w->comparelen && !strcmp(w->compare, &p->comm[pclen - w->comparelen])) - || (proc_pid_cmdline_is_needed && w->starts_with && w->ends_with && strstr(p->cmdline, w->compare)) + || (proc_pid_cmdline_is_needed && w->starts_with && w->ends_with && p->cmdline && strstr(p->cmdline, w->compare)) ))) { if(w->target) p->target = w->target; @@ -787,6 +787,7 @@ static inline void assign_target_to_pid(struct pid_stat *p) { // update pids from proc static inline int read_proc_pid_cmdline(struct pid_stat *p) { + static char cmdline[MAX_CMDLINE + 1]; #ifdef __FreeBSD__ size_t i, bytes = MAX_CMDLINE; @@ -796,7 +797,7 @@ static inline int read_proc_pid_cmdline(struct pid_stat *p) { mib[1] = KERN_PROC; mib[2] = KERN_PROC_ARGS; mib[3] = p->pid; - if (unlikely(sysctl(mib, 4, p->cmdline, &bytes, NULL, 0))) + if (unlikely(sysctl(mib, 4, cmdline, &bytes, NULL, 0))) goto cleanup; #else if(unlikely(!p->cmdline_filename)) { @@ -808,15 +809,17 @@ static inline int read_proc_pid_cmdline(struct pid_stat *p) { int fd = open(p->cmdline_filename, O_RDONLY, 0666); if(unlikely(fd == -1)) goto cleanup; - ssize_t i, bytes = read(fd, p->cmdline, MAX_CMDLINE); + ssize_t i, bytes = read(fd, cmdline, MAX_CMDLINE); close(fd); if(unlikely(bytes < 0)) goto cleanup; #endif - p->cmdline[bytes] = '\0'; + cmdline[bytes] = '\0'; for(i = 0; i < bytes ; i++) - if(unlikely(!p->cmdline[i])) p->cmdline[i] = ' '; + if(unlikely(!cmdline[i])) cmdline[i] = ' '; + + p->cmdline = strdupz(cmdline); if(unlikely(debug)) fprintf(stderr, "Read file '%s' contents: %s\n", p->cmdline_filename, p->cmdline); @@ -825,7 +828,7 @@ static inline int read_proc_pid_cmdline(struct pid_stat *p) { cleanup: // copy the command to the command line - strncpyz(p->cmdline, p->comm, MAX_CMDLINE); + p->cmdline = strdupz(p->comm); return 0; } @@ -1157,24 +1160,13 @@ cleanup: #endif } +#if (ALL_PIDS_ARE_READ_INSTANTLY == 0) static inline int read_proc_stat() { -#ifdef __FreeBSD__ - long cp_time[CPUSTATES]; - static kernel_uint_t utime_raw = 0, stime_raw = 0, ntime_raw = 0; - - if (unlikely(CPUSTATES != 5)) { - error("FREEBSD: There are %d CPU states (5 was expected)", CPUSTATES); - goto cleanup; - } - if (unlikely(GETSYSCTL_BY_NAME("kern.cp_time", cp_time))) goto cleanup; -#else static char filename[FILENAME_MAX + 1] = ""; static procfile *ff = NULL; static kernel_uint_t utime_raw = 0, stime_raw = 0, gtime_raw = 0, gntime_raw = 0, ntime_raw = 0; -#endif static usec_t collected_usec = 0, last_collected_usec = 0; -#ifndef __FreeBSD__ if(unlikely(!ff)) { snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix); ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT); @@ -1183,7 +1175,6 @@ static inline int read_proc_stat() { ff = procfile_readall(ff); if(unlikely(!ff)) goto cleanup; -#endif last_collected_usec = collected_usec; collected_usec = now_monotonic_usec(); @@ -1193,25 +1184,13 @@ static inline int read_proc_stat() { // temporary - it is added global_ntime; kernel_uint_t global_ntime = 0; -#ifdef __FreeBSD__ - incremental_rate(global_utime, utime_raw, cp_time[0], collected_usec, last_collected_usec); - incremental_rate(global_ntime, ntime_raw, cp_time[1], collected_usec, last_collected_usec); - incremental_rate(global_stime, stime_raw, cp_time[2], collected_usec, last_collected_usec); -#else incremental_rate(global_utime, utime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 1)), collected_usec, last_collected_usec); incremental_rate(global_ntime, ntime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 2)), collected_usec, last_collected_usec); incremental_rate(global_stime, stime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 3)), collected_usec, last_collected_usec); incremental_rate(global_gtime, gtime_raw, str2kernel_uint_t(procfile_lineword(ff, 0, 10)), collected_usec, last_collected_usec); -#endif global_utime += global_ntime; -#ifdef __FreeBSD__ - if(enable_guest_charts) { - enable_guest_charts = 0; - info("Guest charts aren't supported by FreeBSD"); - } -#else if(enable_guest_charts) { // temporary - it is added global_ntime; kernel_uint_t global_gntime = 0; @@ -1224,7 +1203,6 @@ static inline int read_proc_stat() { // remove guest time from user time global_utime -= (global_utime > global_gtime) ? global_gtime : global_utime; } -#endif if(unlikely(global_iterations_counter == 1)) { global_utime = 0; @@ -1240,7 +1218,11 @@ cleanup: global_gtime = 0; return 0; } - +#else +static inline int read_proc_stat() { + return 0; +} +#endif // ---------------------------------------------------------------------------- @@ -1751,7 +1733,6 @@ static inline int print_process_and_parents(struct pid_stat *p, usec_t time) { } static inline void print_process_tree(struct pid_stat *p, char *msg) { - log_date(stderr); fprintf(stderr, "%s: process %s (%d, %s) with parents:\n", msg, p->comm, p->pid, p->updated?"running":"exited"); print_process_and_parents(p, p->stat_collected_usec); } @@ -1860,7 +1841,6 @@ static inline void process_exited_processes() { continue; if(unlikely(debug)) { - log_date(stderr); fprintf(stderr, "Absorb %s (%d %s total resources: utime=" KERNEL_UINT_FORMAT " stime=" KERNEL_UINT_FORMAT " gtime=" KERNEL_UINT_FORMAT " minflt=" KERNEL_UINT_FORMAT " majflt=" KERNEL_UINT_FORMAT ")\n" , p->comm , p->pid @@ -2620,14 +2600,13 @@ static inline void send_END(void) { fprintf(stdout, "END\n"); } -static usec_t send_resource_usage_to_netdata() { +void send_resource_usage_to_netdata(usec_t dt) { static struct timeval last = { 0, 0 }; static struct rusage me_last; struct timeval now; struct rusage me; - usec_t usec; usec_t cpuuser; usec_t cpusyst; @@ -2635,10 +2614,6 @@ static usec_t send_resource_usage_to_netdata() { now_monotonic_timeval(&last); getrusage(RUSAGE_SELF, &me_last); - // the first time, give a zero to allow - // netdata calibrate to the current time - // usec = update_every * USEC_PER_SEC; - usec = 0ULL; cpuuser = 0; cpusyst = 0; } @@ -2646,7 +2621,6 @@ static usec_t send_resource_usage_to_netdata() { now_monotonic_timeval(&now); getrusage(RUSAGE_SELF, &me); - usec = dt_usec(&now, &last); cpuuser = me.ru_utime.tv_sec * USEC_PER_SEC + me.ru_utime.tv_usec; cpusyst = me.ru_stime.tv_sec * USEC_PER_SEC + me.ru_stime.tv_usec; @@ -2658,38 +2632,45 @@ static usec_t send_resource_usage_to_netdata() { if(unlikely(!created_charts)) { created_charts = 1; - fprintf(stdout - , "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' apps.plugin netdata.apps_cpu stacked 140000 %1$d\n" - "DIMENSION user '' incremental 1 1000\n" - "DIMENSION system '' incremental 1 1000\n" - "CHART netdata.apps_sizes '' 'Apps Plugin Files' 'files/s' apps.plugin netdata.apps_sizes line 140001 %1$d\n" - "DIMENSION calls '' incremental 1 1\n" - "DIMENSION files '' incremental 1 1\n" - "DIMENSION pids '' absolute 1 1\n" - "DIMENSION fds '' absolute 1 1\n" - "DIMENSION targets '' absolute 1 1\n" - "DIMENSION new_pids 'new pids' incremental 1 1\n" - "CHART netdata.apps_fix '' 'Apps Plugin Normalization Ratios' 'percentage' apps.plugin netdata.apps_fix line 140002 %1$d\n" - "DIMENSION utime '' absolute 1 %2$llu\n" - "DIMENSION stime '' absolute 1 %2$llu\n" - "DIMENSION gtime '' absolute 1 %2$llu\n" - "DIMENSION minflt '' absolute 1 %2$llu\n" - "DIMENSION majflt '' absolute 1 %2$llu\n" + fprintf(stdout, + "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' apps.plugin netdata.apps_cpu stacked 140000 %1$d\n" + "DIMENSION user '' incremental 1 1000\n" + "DIMENSION system '' incremental 1 1000\n" + "CHART netdata.apps_sizes '' 'Apps Plugin Files' 'files/s' apps.plugin netdata.apps_sizes line 140001 %1$d\n" + "DIMENSION calls '' incremental 1 1\n" + "DIMENSION files '' incremental 1 1\n" + "DIMENSION pids '' absolute 1 1\n" + "DIMENSION fds '' absolute 1 1\n" + "DIMENSION targets '' absolute 1 1\n" + "DIMENSION new_pids 'new pids' incremental 1 1\n" + , update_every + ); + +#if (ALL_PIDS_ARE_READ_INSTANTLY == 0) + fprintf(stdout, + "CHART netdata.apps_fix '' 'Apps Plugin Normalization Ratios' 'percentage' apps.plugin netdata.apps_fix line 140002 %1$d\n" + "DIMENSION utime '' absolute 1 %2$llu\n" + "DIMENSION stime '' absolute 1 %2$llu\n" + "DIMENSION gtime '' absolute 1 %2$llu\n" + "DIMENSION minflt '' absolute 1 %2$llu\n" + "DIMENSION majflt '' absolute 1 %2$llu\n" , update_every , RATES_DETAIL ); if(include_exited_childs) - fprintf(stdout - , "CHART netdata.apps_children_fix '' 'Apps Plugin Exited Children Normalization Ratios' 'percentage' apps.plugin netdata.apps_children_fix line 140003 %1$d\n" - "DIMENSION cutime '' absolute 1 %2$llu\n" - "DIMENSION cstime '' absolute 1 %2$llu\n" - "DIMENSION cgtime '' absolute 1 %2$llu\n" - "DIMENSION cminflt '' absolute 1 %2$llu\n" - "DIMENSION cmajflt '' absolute 1 %2$llu\n" + fprintf(stdout, + "CHART netdata.apps_children_fix '' 'Apps Plugin Exited Children Normalization Ratios' 'percentage' apps.plugin netdata.apps_children_fix line 140003 %1$d\n" + "DIMENSION cutime '' absolute 1 %2$llu\n" + "DIMENSION cstime '' absolute 1 %2$llu\n" + "DIMENSION cgtime '' absolute 1 %2$llu\n" + "DIMENSION cminflt '' absolute 1 %2$llu\n" + "DIMENSION cmajflt '' absolute 1 %2$llu\n" , update_every , RATES_DETAIL ); +#endif + } fprintf(stdout, @@ -2705,31 +2686,35 @@ static usec_t send_resource_usage_to_netdata() { "SET targets = %zu\n" "SET new_pids = %zu\n" "END\n" - "BEGIN netdata.apps_fix %llu\n" - "SET utime = %u\n" - "SET stime = %u\n" - "SET gtime = %u\n" - "SET minflt = %u\n" - "SET majflt = %u\n" - "END\n" - , usec + , dt , cpuuser , cpusyst - , usec + , dt , calls_counter , file_counter , all_pids_count , all_files_len , apps_groups_targets_count , targets_assignment_counter - , usec - , (unsigned int)(utime_fix_ratio * 100 * RATES_DETAIL) - , (unsigned int)(stime_fix_ratio * 100 * RATES_DETAIL) - , (unsigned int)(gtime_fix_ratio * 100 * RATES_DETAIL) - , (unsigned int)(minflt_fix_ratio * 100 * RATES_DETAIL) - , (unsigned int)(majflt_fix_ratio * 100 * RATES_DETAIL) ); +#if (ALL_PIDS_ARE_READ_INSTANTLY == 0) + fprintf(stdout, + "BEGIN netdata.apps_fix %llu\n" + "SET utime = %u\n" + "SET stime = %u\n" + "SET gtime = %u\n" + "SET minflt = %u\n" + "SET majflt = %u\n" + "END\n" + , dt + , (unsigned int)(utime_fix_ratio * 100 * RATES_DETAIL) + , (unsigned int)(stime_fix_ratio * 100 * RATES_DETAIL) + , (unsigned int)(gtime_fix_ratio * 100 * RATES_DETAIL) + , (unsigned int)(minflt_fix_ratio * 100 * RATES_DETAIL) + , (unsigned int)(majflt_fix_ratio * 100 * RATES_DETAIL) + ); + if(include_exited_childs) fprintf(stdout, "BEGIN netdata.apps_children_fix %llu\n" @@ -2739,17 +2724,17 @@ static usec_t send_resource_usage_to_netdata() { "SET cminflt = %u\n" "SET cmajflt = %u\n" "END\n" - , usec + , dt , (unsigned int)(cutime_fix_ratio * 100 * RATES_DETAIL) , (unsigned int)(cstime_fix_ratio * 100 * RATES_DETAIL) , (unsigned int)(cgtime_fix_ratio * 100 * RATES_DETAIL) , (unsigned int)(cminflt_fix_ratio * 100 * RATES_DETAIL) , (unsigned int)(cmajflt_fix_ratio * 100 * RATES_DETAIL) ); - - return usec; +#endif } +#if (ALL_PIDS_ARE_READ_INSTANTLY == 0) static void normalize_utilization(struct target *root) { struct target *w; @@ -2895,25 +2880,30 @@ static void normalize_utilization(struct target *root) { ); } } +#else // ALL_PIDS_ARE_READ_INSTANTLY == 1 +static void normalize_utilization(struct target *root) { + (void)root; +} +#endif // ALL_PIDS_ARE_READ_INSTANTLY -static void send_collected_data_to_netdata(struct target *root, const char *type, usec_t usec) { +static void send_collected_data_to_netdata(struct target *root, const char *type, usec_t dt) { struct target *w; - send_BEGIN(type, "cpu", usec); + send_BEGIN(type, "cpu", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, (kernel_uint_t)(w->utime * utime_fix_ratio) + (kernel_uint_t)(w->stime * stime_fix_ratio) + (kernel_uint_t)(w->gtime * gtime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cutime * cutime_fix_ratio) + (kernel_uint_t)(w->cstime * cstime_fix_ratio) + (kernel_uint_t)(w->cgtime * cgtime_fix_ratio)):0ULL)); } send_END(); - send_BEGIN(type, "cpu_user", usec); + send_BEGIN(type, "cpu_user", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, (kernel_uint_t)(w->utime * utime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cutime * cutime_fix_ratio)):0ULL)); } send_END(); - send_BEGIN(type, "cpu_system", usec); + send_BEGIN(type, "cpu_system", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, (kernel_uint_t)(w->stime * stime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cstime * cstime_fix_ratio)):0ULL)); @@ -2921,7 +2911,7 @@ static void send_collected_data_to_netdata(struct target *root, const char *type send_END(); if(show_guest_time) { - send_BEGIN(type, "cpu_guest", usec); + send_BEGIN(type, "cpu_guest", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, (kernel_uint_t)(w->gtime * gtime_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cgtime * cgtime_fix_ratio)):0ULL)); @@ -2929,42 +2919,42 @@ static void send_collected_data_to_netdata(struct target *root, const char *type send_END(); } - send_BEGIN(type, "threads", usec); + send_BEGIN(type, "threads", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, w->num_threads); } send_END(); - send_BEGIN(type, "processes", usec); + send_BEGIN(type, "processes", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, w->processes); } send_END(); - send_BEGIN(type, "mem", usec); + send_BEGIN(type, "mem", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, (w->statm_resident > w->statm_share)?(w->statm_resident - w->statm_share):0ULL); } send_END(); - send_BEGIN(type, "vmem", usec); + send_BEGIN(type, "vmem", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, w->statm_size); } send_END(); - send_BEGIN(type, "minor_faults", usec); + send_BEGIN(type, "minor_faults", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, (kernel_uint_t)(w->minflt * minflt_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cminflt * cminflt_fix_ratio)):0ULL)); } send_END(); - send_BEGIN(type, "major_faults", usec); + send_BEGIN(type, "major_faults", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, (kernel_uint_t)(w->majflt * majflt_fix_ratio) + (include_exited_childs?((kernel_uint_t)(w->cmajflt * cmajflt_fix_ratio)):0ULL)); @@ -2972,14 +2962,14 @@ static void send_collected_data_to_netdata(struct target *root, const char *type send_END(); #ifndef __FreeBSD__ - send_BEGIN(type, "lreads", usec); + send_BEGIN(type, "lreads", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, w->io_logical_bytes_read); } send_END(); - send_BEGIN(type, "lwrites", usec); + send_BEGIN(type, "lwrites", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, w->io_logical_bytes_written); @@ -2987,14 +2977,14 @@ static void send_collected_data_to_netdata(struct target *root, const char *type send_END(); #endif - send_BEGIN(type, "preads", usec); + send_BEGIN(type, "preads", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, w->io_storage_bytes_read); } send_END(); - send_BEGIN(type, "pwrites", usec); + send_BEGIN(type, "pwrites", dt); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) send_SET(w->name, w->io_storage_bytes_written); @@ -3002,21 +2992,21 @@ static void send_collected_data_to_netdata(struct target *root, const char *type send_END(); if(enable_file_charts) { - send_BEGIN(type, "files", usec); + send_BEGIN(type, "files", dt); for (w = root; w; w = w->next) { if (unlikely(w->exposed)) send_SET(w->name, w->openfiles); } send_END(); - send_BEGIN(type, "sockets", usec); + send_BEGIN(type, "sockets", dt); for (w = root; w; w = w->next) { if (unlikely(w->exposed)) send_SET(w->name, w->opensockets); } send_END(); - send_BEGIN(type, "pipes", usec); + send_BEGIN(type, "pipes", dt); for (w = root; w; w = w->next) { if (unlikely(w->exposed)) send_SET(w->name, w->openpipes); @@ -3472,8 +3462,9 @@ int main(int argc, char **argv) { static int profiling_count=0; profiling_count++; if(unlikely(profiling_count > 1000)) exit(0); + usec_t dt = update_every * USEC_PER_SEC; #else - heartbeat_next(&hb, step); + usec_t dt = heartbeat_next(&hb, step); #endif if(!collect_data_for_all_processes()) { @@ -3485,7 +3476,7 @@ int main(int argc, char **argv) { calculate_netdata_statistics(); normalize_utilization(apps_groups_root_target); - usec_t dt = send_resource_usage_to_netdata(); + send_resource_usage_to_netdata(dt); // this is smart enough to show only newly added apps, when needed send_charts_updates_to_netdata(apps_groups_root_target, "apps", "Apps"); diff --git a/src/backend_prometheus.c b/src/backend_prometheus.c new file mode 100644 index 000000000..88ec2c65b --- /dev/null +++ b/src/backend_prometheus.c @@ -0,0 +1,397 @@ +#include "common.h" + +// ---------------------------------------------------------------------------- +// PROMETHEUS +// /api/v1/allmetrics?format=prometheus + +static struct prometheus_server { + const char *server; + uint32_t hash; + time_t last_access; + struct prometheus_server *next; +} *prometheus_server_root = NULL; + +static inline time_t prometheus_server_last_access(const char *server, time_t now) { + uint32_t hash = simple_hash(server); + + struct prometheus_server *ps; + for(ps = prometheus_server_root; ps ;ps = ps->next) { + if (hash == ps->hash && !strcmp(server, ps->server)) { + time_t last = ps->last_access; + ps->last_access = now; + return last; + } + } + + ps = callocz(1, sizeof(struct prometheus_server)); + ps->server = strdupz(server); + ps->hash = hash; + ps->last_access = now; + ps->next = prometheus_server_root; + prometheus_server_root = ps; + + return 0; +} + +static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) { + size_t n; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(!isalnum(c)) *d = '_'; + else *d = c; + } + *d = '\0'; + + return n; +} + +static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable) { + size_t n; + + // make sure we can escape one character without overflowing the buffer + usable--; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(unlikely(c == '"' || c == '\\' || c == '\n')) { + *d++ = '\\'; + n++; + } + *d = c; + } + *d = '\0'; + + return n; +} + +static inline char *prometheus_units_copy(char *d, const char *s, size_t usable) { + const char *sorig = s; + char *ret = d; + size_t n; + + *d++ = '_'; + for(n = 1; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(!isalnum(c)) *d = '_'; + else *d = c; + } + + if(n == 2 && sorig[0] == '%') { + n = 0; + d = ret; + s = "_percent"; + for( ; *s && n < usable ; n++) *d++ = *s++; + } + else if(n > 3 && sorig[n-3] == '/' && sorig[n-2] == 's') { + n = n - 2; + d -= 2; + s = "_persec"; + for( ; *s && n < usable ; n++) *d++ = *s++; + } + + *d = '\0'; + + return ret; +} + + +#define PROMETHEUS_ELEMENT_MAX 256 +#define PROMETHEUS_LABELS_MAX 1024 + +static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, uint32_t options, time_t after, time_t before, int allhosts, int help, int types, int names) { + rrdhost_rdlock(host); + + char hostname[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX); + + char labels[PROMETHEUS_LABELS_MAX + 1] = ""; + if(allhosts) { + if(host->tags && *(host->tags)) + buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS); + + snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname); + } + else { + if(host->tags && *(host->tags)) + buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS); + } + + // for each chart + RRDSET *st; + rrdset_foreach_read(st, host) { + char chart[PROMETHEUS_ELEMENT_MAX + 1]; + char context[PROMETHEUS_ELEMENT_MAX + 1]; + char family[PROMETHEUS_ELEMENT_MAX + 1]; + char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; + + prometheus_label_copy(chart, (names && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + + if(likely(backends_can_send_rrdset(options, st))) { + rrdset_rdlock(st); + + int as_collected = ((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED); + int homogeneus = 1; + if(as_collected) { + if(rrdset_flag_check(st, RRDSET_FLAG_HOMEGENEOUS_CHECK)) + rrdset_update_heterogeneous_flag(st); + + if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) + homogeneus = 0; + } + else { + if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE) + prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX); + } + + if(unlikely(help)) + buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n" + , (homogeneus)?"homogeneus":"heterogeneous" + , (names && st->name) ? st->name : st->id + , st->context + , st->family + , st->units + ); + + // for each dimension + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if(rd->collections_counter) { + char dimension[PROMETHEUS_ELEMENT_MAX + 1]; + char *suffix = ""; + + if (as_collected) { + // we need as-collected / raw data + + const char *t = "gauge", *h = "gives"; + if(rd->algorithm == RRD_ALGORITHM_INCREMENTAL || + rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) { + t = "counter"; + h = "delta gives"; + suffix = "_total"; + } + + if(homogeneus) { + // all the dimensions of the chart, has the same algorithm, multiplier and divisor + // we add all dimensions as labels + + prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if(unlikely(help)) + buffer_sprintf(wb + , "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n" + , prefix + , context + , suffix + , (names && st->name) ? st->name : st->id + , st->context + , st->family + , (names && rd->name) ? rd->name : rd->id + , rd->multiplier + , rd->divisor + , h + , st->units + , t + ); + + if(unlikely(types)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n" + , prefix + , context + , suffix + , t + ); + + buffer_sprintf(wb + , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , suffix + , chart + , family + , dimension + , labels + , rd->last_collected_value + , timeval_msec(&rd->last_collected_time) + ); + } + else { + // the dimensions of the chart, do not have the same algorithm, multiplier or divisor + // we create a metric per dimension + + prometheus_name_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if(unlikely(help)) + buffer_sprintf(wb + , "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n" + , prefix + , context + , dimension + , suffix + , (names && st->name) ? st->name : st->id + , st->context + , st->family + , (names && rd->name) ? rd->name : rd->id + , rd->multiplier + , rd->divisor + , h + , st->units + , t + ); + + if(unlikely(types)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s_%s%s %s\n" + , prefix + , context + , dimension + , suffix + , t + ); + + buffer_sprintf(wb + , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , dimension + , suffix + , chart + , family + , labels + , rd->last_collected_value + , timeval_msec(&rd->last_collected_time) + ); + } + } + else { + // we need average or sum of the data + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t); + + if(!isnan(value) && !isinf(value)) { + + if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE) + suffix = "_average"; + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM) + suffix = "_sum"; + + prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if (unlikely(help)) + buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n" + , prefix + , context + , units + , suffix + , (names && rd->name) ? rd->name : rd->id + , st->units + , (unsigned long long)first_t + , (unsigned long long)last_t + ); + + if (unlikely(types)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n" + , prefix + , context + , units + , suffix + ); + + buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , units + , suffix + , chart + , family + , dimension + , labels + , value + , last_t * MSEC_PER_SEC + ); + } + } + } + } + + rrdset_unlock(st); + } + } + + rrdhost_unlock(host); +} + +static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, uint32_t options, const char *server, time_t now, int help) { + if(!server || !*server) server = "default"; + + time_t after = prometheus_server_last_access(server, now); + + int first_seen = 0; + if(!after) { + after = now - backend_update_every; + first_seen = 1; + } + + if(after > now) { + // oops! this should never happen + after = now - backend_update_every; + } + + if(help) { + int show_range = 1; + char *mode; + if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) { + mode = "as collected"; + show_range = 0; + } + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE) + mode = "average"; + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM) + mode = "sum"; + else + mode = "unknown"; + + buffer_sprintf(wb, "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s" + , host->hostname + , (first_seen)?"FIRST SEEN ":"" + , server + , mode + , (unsigned long)((first_seen)?0:(now - after)) + , (first_seen)?"never":"seconds ago" + ); + + if(show_range) + buffer_sprintf(wb, ", time range %lu to %lu", (unsigned long)after, (unsigned long)now); + + buffer_strcat(wb, "\n\n"); + } + + return after; +} + +void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names) { + time_t before = now_realtime_sec(); + + // we start at the point we had stopped before + time_t after = prometheus_preparation(host, wb, options, server, before, help); + + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 0, help, types, names); +} + +void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names) { + time_t before = now_realtime_sec(); + + // we start at the point we had stopped before + time_t after = prometheus_preparation(host, wb, options, server, before, help); + + rrd_rdlock(); + rrdhost_foreach_read(host) { + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 1, help, types, names); + } + rrd_unlock(); +} diff --git a/src/backend_prometheus.h b/src/backend_prometheus.h new file mode 100644 index 000000000..53dddb0d2 --- /dev/null +++ b/src/backend_prometheus.h @@ -0,0 +1,11 @@ +// +// Created by costa on 09/07/17. +// + +#ifndef NETDATA_BACKEND_PROMETHEUS_H +#define NETDATA_BACKEND_PROMETHEUS_H + +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names); +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names); + +#endif //NETDATA_BACKEND_PROMETHEUS_H diff --git a/src/backends.c b/src/backends.c index 3e385cab5..eed4ab409 100644 --- a/src/backends.c +++ b/src/backends.c @@ -22,47 +22,81 @@ // 5. repeats the above forever. // -#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001 -#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002 -#define BACKEND_SOURCE_DATA_SUM 0x00000004 - +const char *backend_prefix = "netdata"; +int backend_send_names = 1; +int backend_update_every = 10; +uint32_t backend_options = BACKEND_SOURCE_DATA_AVERAGE; // ---------------------------------------------------------------------------- // helper functions for backends +static inline size_t backend_name_copy(char *d, const char *s, size_t usable) { + size_t n; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + char c = *s; + + if(c != '.' && !isalnum(c)) *d = '_'; + else *d = c; + } + *d = '\0'; + + return n; +} + // calculate the SUM or AVERAGE of a dimension, for any timeframe // may return NAN if the database does not have any value in the give timeframe -static inline calculated_number backend_calculate_value_from_stored_data( +inline calculated_number backend_calculate_value_from_stored_data( RRDSET *st // the chart , RRDDIM *rd // the dimension , time_t after // the start timestamp , time_t before // the end timestamp , uint32_t options // BACKEND_SOURCE_* bitmap + , time_t *first_timestamp // the first point of the database used in this response + , time_t *last_timestamp // the timestamp that should be reported to backend ) { // find the edges of the rrd database for this chart time_t first_t = rrdset_first_entry_t(st); time_t last_t = rrdset_last_entry_t(st); + time_t update_every = st->update_every; - if(unlikely(before < first_t || after > last_t)) - // the chart has not been updated in the wanted timeframe - return NAN; + // step back a little, to make sure we have complete data collection + // for all metrics + after -= update_every * 2; + before -= update_every * 2; // align the time-frame - // for 'after' also skip the first value by adding st->update_every - after = after - after % st->update_every + st->update_every; - before = before - before % st->update_every; + after = after - (after % update_every); + before = before - (before % update_every); - if(unlikely(after < first_t)) - after = first_t; + // for before, loose another iteration + // the latest point will be reported the next time + before -= update_every; if(unlikely(after > before)) - // this can happen when st->update_every > before - after - before = after; + // this can happen when update_every > before - after + after = before; + + if(unlikely(after < first_t)) + after = first_t; if(unlikely(before > last_t)) before = last_t; + if(unlikely(before < first_t || after > last_t)) { + // the chart has not been updated in the wanted timeframe + debug(D_BACKEND, "BACKEND: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu", + st->rrdhost->hostname, st->id, rd->id, + (unsigned long)after, (unsigned long)before, + (unsigned long)first_t, (unsigned long)last_t + ); + return NAN; + } + + *first_timestamp = after; + *last_timestamp = before; + size_t counter = 0; calculated_number sum = 0; @@ -88,10 +122,15 @@ static inline calculated_number backend_calculate_value_from_stored_data( counter++; } - if(unlikely(!counter)) + if(unlikely(!counter)) { + debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu", + st->rrdhost->hostname, st->id, rd->id, + (unsigned long)after, (unsigned long)before + ); return NAN; + } - if(unlikely(options & BACKEND_SOURCE_DATA_SUM)) + if(unlikely((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)) return sum; return sum / (calculated_number)counter; @@ -113,7 +152,7 @@ static inline int discard_response(BUFFER *b, const char *backend) { } *d = '\0'; - info("Received %zu bytes from %s backend. Ignoring them. Sample: '%s'", buffer_strlen(b), backend, sample); + info("BACKEND: received %zu bytes from %s backend. Ignoring them. Sample: '%s'", buffer_strlen(b), backend, sample); buffer_flush(b); return 0; } @@ -138,13 +177,18 @@ static inline int format_dimension_collected_graphite_plaintext( (void)before; (void)options; + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); + buffer_sprintf( b , "%s.%s.%s.%s " COLLECTED_NUMBER_FORMAT " %u\n" , prefix , hostname - , st->id - , rd->id + , chart_name + , dimension_name , rd->last_collected_value , (uint32_t)rd->last_collected_time.tv_sec ); @@ -165,7 +209,13 @@ static inline int format_dimension_stored_graphite_plaintext( ) { (void)host; - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options); + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t); if(!isnan(value)) { @@ -174,10 +224,10 @@ static inline int format_dimension_stored_graphite_plaintext( , "%s.%s.%s.%s " CALCULATED_NUMBER_FORMAT " %u\n" , prefix , hostname - , st->id - , rd->id + , chart_name + , dimension_name , value - , (uint32_t) before + , (uint32_t) last_t ); return 1; @@ -209,15 +259,22 @@ static inline int format_dimension_collected_opentsdb_telnet( (void)before; (void)options; + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); + buffer_sprintf( b - , "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s\n" + , "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n" , prefix - , st->id - , rd->id + , chart_name + , dimension_name , (uint32_t)rd->last_collected_time.tv_sec , rd->last_collected_value , hostname + , (host->tags)?" ":"" + , (host->tags)?host->tags:"" ); return 1; @@ -236,19 +293,27 @@ static inline int format_dimension_stored_opentsdb_telnet( ) { (void)host; - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options); + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t); + + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); if(!isnan(value)) { buffer_sprintf( b - , "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s\n" + , "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n" , prefix - , st->id - , rd->id - , (uint32_t) before + , chart_name + , dimension_name + , (uint32_t) last_t , value , hostname + , (host->tags)?" ":"" + , (host->tags)?host->tags:"" ); return 1; @@ -329,7 +394,8 @@ static inline int format_dimension_stored_json_plaintext( ) { (void)host; - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options); + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t); if(!isnan(value)) { buffer_sprintf(b, "{" @@ -362,7 +428,7 @@ static inline int format_dimension_stored_json_plaintext( rd->name, value, - (uint32_t)before + (uint32_t) last_t ); return 1; @@ -378,6 +444,56 @@ static inline int process_json_response(BUFFER *b) { // ---------------------------------------------------------------------------- // the backend thread +static SIMPLE_PATTERN *charts_pattern = NULL; + +inline int backends_can_send_rrdset(uint32_t options, RRDSET *st) { + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE))) + return 0; + + if(unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) { + // we have not checked this chart + if(simple_pattern_matches(charts_pattern, st->id) || simple_pattern_matches(charts_pattern, st->name)) + rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND); + else { + rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE); + debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, st->rrdhost->hostname); + return 0; + } + } + + if(unlikely(!rrdset_is_available_for_backends(st))) { + debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.", st->id, st->rrdhost->hostname); + return 0; + } + + if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED))) { + debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, st->rrdhost->hostname, rrd_memory_mode_name(st->rrdhost->rrd_memory_mode)); + return 0; + } + + return 1; +} + +inline uint32_t backend_parse_data_source(const char *source, uint32_t mode) { + if(!strcmp(source, "raw") || !strcmp(source, "as collected") || !strcmp(source, "as-collected") || !strcmp(source, "as_collected") || !strcmp(source, "ascollected")) { + mode |= BACKEND_SOURCE_DATA_AS_COLLECTED; + mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AS_COLLECTED); + } + else if(!strcmp(source, "average")) { + mode |= BACKEND_SOURCE_DATA_AVERAGE; + mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AVERAGE); + } + else if(!strcmp(source, "sum") || !strcmp(source, "volume")) { + mode |= BACKEND_SOURCE_DATA_SUM; + mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_SUM); + } + else { + error("BACKEND: invalid data source method '%s'.", source); + } + + return mode; +} + void *backends_main(void *ptr) { int default_port = 0; int sock = -1; @@ -387,13 +503,13 @@ void *backends_main(void *ptr) { int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, uint32_t) = NULL; int (*backend_response_checker)(BUFFER *) = NULL; - info("BACKEND thread created with task id %d", gettid()); + info("BACKEND: thread created with task id %d", gettid()); if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) - error("Cannot set pthread cancel type to DEFERRED."); + error("BACKEND: cannot set pthread cancel type to DEFERRED."); if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) - error("Cannot set pthread cancel state to ENABLE."); + error("BACKEND: cannot set pthread cancel state to ENABLE."); // ------------------------------------------------------------------------ // collect configuration options @@ -402,45 +518,35 @@ void *backends_main(void *ptr) { .tv_sec = 0, .tv_usec = 0 }; - uint32_t options; int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0); const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average"); const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite"); const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost"); - const char *prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata"); + backend_prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata"); const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname); - int frequency = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", 10); + backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", backend_update_every); int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10); - long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", frequency * 2 * 1000); + long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", backend_update_every * 2 * 1000); + backend_send_names = config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", backend_send_names); + + charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), SIMPLE_PATTERN_EXACT); + // ------------------------------------------------------------------------ // validate configuration options // and prepare for sending data to our backend - if(!enabled || frequency < 1) - goto cleanup; - - if(!strcmp(source, "as collected")) { - options = BACKEND_SOURCE_DATA_AS_COLLECTED; - } - else if(!strcmp(source, "average")) { - options = BACKEND_SOURCE_DATA_AVERAGE; - } - else if(!strcmp(source, "sum") || !strcmp(source, "volume")) { - options = BACKEND_SOURCE_DATA_SUM; - } - else { - error("Invalid data source method '%s' for backend given. Disabling backed.", source); - goto cleanup; - } + backend_options = backend_parse_data_source(source, backend_options); if(timeoutms < 1) { - error("BACKED invalid timeout %ld ms given. Assuming %d ms.", timeoutms, frequency * 2 * 1000); - timeoutms = frequency * 2 * 1000; + error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, backend_update_every * 2 * 1000); + timeoutms = backend_update_every * 2 * 1000; } timeout.tv_sec = (timeoutms * 1000) / 1000000; timeout.tv_usec = (timeoutms * 1000) % 1000000; + if(!enabled || backend_update_every < 1) + goto cleanup; // ------------------------------------------------------------------------ // select the backend type @@ -450,7 +556,7 @@ void *backends_main(void *ptr) { default_port = 2003; backend_response_checker = process_graphite_response; - if(options == BACKEND_SOURCE_DATA_AS_COLLECTED) + if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) backend_request_formatter = format_dimension_collected_graphite_plaintext; else backend_request_formatter = format_dimension_stored_graphite_plaintext; @@ -461,7 +567,7 @@ void *backends_main(void *ptr) { default_port = 4242; backend_response_checker = process_opentsdb_response; - if(options == BACKEND_SOURCE_DATA_AS_COLLECTED) + if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) backend_request_formatter = format_dimension_collected_opentsdb_telnet; else backend_request_formatter = format_dimension_stored_opentsdb_telnet; @@ -472,19 +578,19 @@ void *backends_main(void *ptr) { default_port = 5448; backend_response_checker = process_json_response; - if (options == BACKEND_SOURCE_DATA_AS_COLLECTED) + if ((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) backend_request_formatter = format_dimension_collected_json_plaintext; else backend_request_formatter = format_dimension_stored_json_plaintext; } else { - error("Unknown backend type '%s'", type); + error("BACKEND: Unknown backend type '%s'", type); goto cleanup; } if(backend_request_formatter == NULL || backend_response_checker == NULL) { - error("backend is misconfigured - disabling it."); + error("BACKEND: backend is misconfigured - disabling it."); goto cleanup; } @@ -509,18 +615,18 @@ void *backends_main(void *ptr) { chart_backend_reconnects = 0, chart_backend_latency = 0; - RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, frequency, RRDSET_TYPE_LINE); + RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, backend_update_every, RRDSET_TYPE_LINE); rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, frequency, RRDSET_TYPE_AREA); + RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, backend_update_every, RRDSET_TYPE_AREA); rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); - RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, frequency, RRDSET_TYPE_LINE); + RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, backend_update_every, RRDSET_TYPE_LINE); rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); @@ -534,11 +640,11 @@ void *backends_main(void *ptr) { * * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html * - RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, frequency, RRDSET_TYPE_AREA); + RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, backend_update_every, RRDSET_TYPE_AREA); rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); */ - RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, frequency, RRDSET_TYPE_STACKED); + RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, backend_update_every, RRDSET_TYPE_STACKED); rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); @@ -546,9 +652,9 @@ void *backends_main(void *ptr) { // ------------------------------------------------------------------------ // prepare the backend main loop - info("BACKEND configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, frequency, hostname, prefix); + info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, backend_update_every, hostname, backend_prefix); - usec_t step_ut = frequency * USEC_PER_SEC; + usec_t step_ut = backend_update_every * USEC_PER_SEC; time_t after = now_realtime_sec(); int failures = 0; heartbeat_t hb; @@ -558,9 +664,10 @@ void *backends_main(void *ptr) { // ------------------------------------------------------------------------ // Wait for the next iteration point. + heartbeat_next(&hb, step_ut); time_t before = now_realtime_sec(); - + debug(D_BACKEND, "BACKEND: preparing buffer for timeframe %lu to %lu", (unsigned long)after, (unsigned long)before); // ------------------------------------------------------------------------ // add to the buffer the data we need to send to the backend @@ -568,33 +675,59 @@ void *backends_main(void *ptr) { int pthreadoldcancelstate; if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &pthreadoldcancelstate) != 0)) - error("Cannot set pthread cancel state to DISABLE."); + error("BACKEND: cannot set pthread cancel state to DISABLE."); + + size_t count_hosts = 0; + size_t count_charts_total = 0; + size_t count_dims_total = 0; rrd_rdlock(); RRDHOST *host; rrdhost_foreach_read(host) { - if(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE) - continue; - rrdhost_rdlock(host); + count_hosts++; + size_t count_charts = 0; + size_t count_dims = 0; + size_t count_dims_skipped = 0; + + const char *__hostname = (host == localhost)?hostname:host->hostname; + RRDSET *st; rrdset_foreach_read(st, host) { - rrdset_rdlock(st); + if(likely(backends_can_send_rrdset(backend_options, st))) { + rrdset_rdlock(st); + + count_charts++; + + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if (likely(rd->last_collected_time.tv_sec >= after)) { + chart_buffered_metrics += backend_request_formatter(b, backend_prefix, host, __hostname, st, rd, after, before, backend_options); + count_dims++; + } + else { + debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before); + count_dims_skipped++; + } + } - RRDDIM *rd; - rrddim_foreach_read(rd, st) { - if(rd->last_collected_time.tv_sec >= after) - chart_buffered_metrics += backend_request_formatter(b, prefix, host, (host == localhost)?hostname:host->hostname, st, rd, after, before, options); + rrdset_unlock(st); } - rrdset_unlock(st); } + + debug(D_BACKEND, "BACKEND: sending host '%s', metrics of %zu dimensions, of %zu charts. Skipped %zu dimensions.", __hostname, count_dims, count_charts, count_dims_skipped); + count_charts_total += count_charts; + count_dims_total += count_dims; + rrdhost_unlock(host); } rrd_unlock(); + debug(D_BACKEND, "BACKEND: buffer has %zu bytes, added metrics for %zu dimensions, of %zu charts, from %zu hosts", buffer_strlen(b), count_dims_total, count_charts_total, count_hosts); + if(unlikely(pthread_setcancelstate(pthreadoldcancelstate, NULL) != 0)) - error("Cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate); + error("BACKEND: cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate); // ------------------------------------------------------------------------ @@ -639,14 +772,14 @@ void *backends_main(void *ptr) { chart_receptions++; } else if(r == 0) { - error("Backend '%s' closed the socket", destination); + error("BACKEND: '%s' closed the socket", destination); close(sock); sock = -1; } else { // failed to receive data if(errno != EAGAIN && errno != EWOULDBLOCK) { - error("Cannot receive data from backend '%s'.", destination); + error("BACKEND: cannot receive data from backend '%s'.", destination); } } } @@ -698,7 +831,7 @@ void *backends_main(void *ptr) { } else { // oops! we couldn't send (all or some of the) data - error("Failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written); + error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written); chart_transmission_failures++; if(written != -1) @@ -713,7 +846,7 @@ void *backends_main(void *ptr) { } } else { - error("Failed to update database backend '%s'", destination); + error("BACKEND: failed to update database backend '%s'", destination); chart_transmission_failures++; // increment the counter we check for data loss @@ -723,7 +856,7 @@ void *backends_main(void *ptr) { if(failures > buffer_on_failures) { // too bad! we are going to lose data chart_lost_bytes += buffer_strlen(b); - error("Reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination); + error("BACKEND: reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination); buffer_flush(b); failures = 0; chart_data_lost_events++; @@ -781,7 +914,7 @@ cleanup: buffer_free(b); buffer_free(response); - info("BACKEND thread exiting"); + info("BACKEND: thread exiting"); static_thread->enabled = 0; pthread_exit(NULL); diff --git a/src/backends.h b/src/backends.h index 61122a1d0..e882f3db1 100644 --- a/src/backends.h +++ b/src/backends.h @@ -1,6 +1,30 @@ #ifndef NETDATA_BACKENDS_H #define NETDATA_BACKENDS_H 1 -void *backends_main(void *ptr); +#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001 +#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002 +#define BACKEND_SOURCE_DATA_SUM 0x00000004 + +#define BACKEND_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM) + +extern int backend_send_names; +extern int backend_update_every; +extern uint32_t backend_options; +extern const char *backend_prefix; + +extern void *backends_main(void *ptr); + +extern int backends_can_send_rrdset(uint32_t options, RRDSET *st); +extern uint32_t backend_parse_data_source(const char *source, uint32_t mode); + +extern calculated_number backend_calculate_value_from_stored_data( + RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , uint32_t options // BACKEND_SOURCE_* bitmap + , time_t *first_timestamp // the timestamp of the first point used in this response + , time_t *last_timestamp // the timestamp that should be reported to backend +); #endif /* NETDATA_BACKENDS_H */ diff --git a/src/clocks.c b/src/clocks.c index 879ebf911..8f2aa740e 100644 --- a/src/clocks.c +++ b/src/clocks.c @@ -115,8 +115,8 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) if(likely(*hb != 0ULL)) { usec_t dt = now - *hb; *hb = now; - if(unlikely(dt / tick > 1)) - error("heartbeat missed between %llu usec and %llu usec", *hb, now); + if(unlikely(dt >= tick + tick / 2)) + error("heartbeat missed %llu microseconds", dt - tick); return dt; } else { diff --git a/src/clocks.h b/src/clocks.h index 197b5431f..ca5715254 100644 --- a/src/clocks.h +++ b/src/clocks.h @@ -55,6 +55,8 @@ typedef usec_t heartbeat_t; #define USEC_PER_SEC 1000000ULL #define MSEC_PER_SEC 1000ULL +#define USEC_PER_MS 1000ULL + #ifndef HAVE_CLOCK_GETTIME /* Fallback function for POSIX.1-2001 clock_gettime() function. * diff --git a/src/common.c b/src/common.c index 88fcf85bc..aa75c198d 100644 --- a/src/common.c +++ b/src/common.c @@ -226,6 +226,13 @@ void json_escape_string(char *dst, const char *src, size_t size) { *d = '\0'; } +void json_fix_string(char *s) { + for( ; *s ;s++) { + if(unlikely(*s == '\\')) *s = '/'; + else if(unlikely(*s == '"')) *s = '\''; + } +} + int sleep_usec(usec_t usec) { #ifndef NETDATA_WITH_USLEEP @@ -895,9 +902,8 @@ char *mystrsep(char **ptr, char *s) { char *trim(char *s) { // skip leading spaces - // and 'comments' as well!? while (*s && isspace(*s)) s++; - if (!*s || *s == '#') return NULL; + if (!*s) return NULL; // skip tailing spaces // this way is way faster. Writes only one NUL char. @@ -913,105 +919,163 @@ char *trim(char *s) { return s; } -void *mymmap(const char *filename, size_t size, int flags, int ksm) { -#ifndef MADV_MERGEABLE - (void)ksm; -#endif - static int log_madvise_1 = 1; -#ifdef MADV_MERGEABLE - static int log_madvise_2 = 1, log_madvise_3 = 1; -#endif - void *mem = NULL; +inline char *trim_all(char *buffer) { + char *d = buffer, *s = buffer; + + // skip spaces + while(isspace(*s)) s++; + + while(*s) { + // copy the non-space part + while(*s && !isspace(*s)) *d++ = *s++; + + // add a space if we have to + if(*s && isspace(*s)) { + *d++ = ' '; + s++; + } + + // skip spaces + while(isspace(*s)) s++; + } + + *d = '\0'; + + if(d > buffer) { + d--; + if(isspace(*d)) *d = '\0'; + } + + if(!buffer[0]) return NULL; + return buffer; +} + +static int memory_file_open(const char *filename, size_t size) { + // info("memory_file_open('%s', %zu", filename, size); - errno = 0; int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664); if (fd != -1) { if (lseek(fd, size, SEEK_SET) == (off_t) size) { if (write(fd, "", 1) == 1) { if (ftruncate(fd, size)) error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size); + } + else error("Cannot write to file '%s' at position %zu.", filename, size); + } + else error("Cannot seek file '%s' to size %zu.", filename, size); + } + else error("Cannot create/open file '%s'.", filename); -#ifdef MADV_MERGEABLE - if (flags & MAP_SHARED || !enable_ksm || !ksm) { -#endif - mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0); - if (mem == MAP_FAILED) { - error("Cannot allocate SHARED memory for file '%s'.", filename); - mem = NULL; - } - else { + return fd; +} + +// mmap_shared is used for memory mode = map +static void *memory_file_mmap(const char *filename, size_t size, int flags) { + // info("memory_file_mmap('%s', %zu", filename, size); + static int log_madvise = 1; + + int fd = -1; + if(filename) { + fd = memory_file_open(filename, size); + if(fd == -1) return MAP_FAILED; + } + + void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0); + if (mem != MAP_FAILED) { #ifdef NETDATA_LOG_ALLOCATIONS - mmap_accounting(size); + mmap_accounting(size); #endif - int advise = MADV_SEQUENTIAL | MADV_DONTFORK; - if (flags & MAP_SHARED) advise |= MADV_WILLNEED; - - if (madvise(mem, size, advise) != 0 && log_madvise_1) { - error("Cannot advise the kernel about the memory usage of file '%s'.", filename); - log_madvise_1--; - } - } + int advise = MADV_SEQUENTIAL | MADV_DONTFORK; + if (flags & MAP_SHARED) advise |= MADV_WILLNEED; + + if (madvise(mem, size, advise) != 0 && log_madvise) { + error("Cannot advise the kernel about shared memory usage."); + log_madvise--; + } + } + + if(fd != -1) + close(fd); + + return mem; +} + #ifdef MADV_MERGEABLE - } - else { -/* - // test - load the file into memory - mem = calloc(1, size); - if(mem) { - if(lseek(fd, 0, SEEK_SET) == 0) { - if(read(fd, mem, size) != (ssize_t)size) - error("Cannot read from file '%s'", filename); - } - else - error("Cannot seek to beginning of file '%s'.", filename); - } -*/ - mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) { - error("Cannot allocate PRIVATE ANONYMOUS memory for KSM for file '%s'.", filename); - mem = NULL; - } - else { +static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) { + // info("memory_file_mmap_ksm('%s', %zu", filename, size); + static int log_madvise_2 = 1, log_madvise_3 = 1; + + int fd = -1; + if(filename) { + fd = memory_file_open(filename, size); + if(fd == -1) return MAP_FAILED; + } + + void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0); + if (mem != MAP_FAILED) { #ifdef NETDATA_LOG_ALLOCATIONS - mmap_accounting(size); -#endif - if (lseek(fd, 0, SEEK_SET) == 0) { - if (read(fd, mem, size) != (ssize_t) size) - error("Cannot read from file '%s'", filename); - } else - error("Cannot seek to beginning of file '%s'.", filename); - - // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE - if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) { - error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.", - filename); - log_madvise_2--; - } - - if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) { - error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.", - filename); - log_madvise_3--; - } - } - } + mmap_accounting(size); #endif + if(fd != -1) { + if (lseek(fd, 0, SEEK_SET) == 0) { + if (read(fd, mem, size) != (ssize_t) size) + error("Cannot read from file '%s'", filename); } - else - error("Cannot write to file '%s' at position %zu.", filename, size); + else error("Cannot seek to beginning of file '%s'.", filename); } - else - error("Cannot seek file '%s' to size %zu.", filename, size); - close(fd); + // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE + if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) { + error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.", filename); + log_madvise_2--; + } + + if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) { + error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.", filename); + log_madvise_3--; + } } + + if(fd != -1) + close(fd); + + return mem; +} +#else +static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) { + // info("memory_file_mmap_ksm FALLBACK ('%s', %zu", filename, size); + + if(filename) + return memory_file_mmap(filename, size, flags); + + // when KSM is not available and no filename is given (memory mode = ram), + // we just report failure + return MAP_FAILED; +} +#endif + +void *mymmap(const char *filename, size_t size, int flags, int ksm) { + void *mem = NULL; + + if (filename && (flags & MAP_SHARED || !enable_ksm || !ksm)) + // memory mode = map | save + // when KSM is not enabled + // MAP_SHARED is used for memory mode = map (no KSM possible) + mem = memory_file_mmap(filename, size, flags); + else - error("Cannot create/open file '%s'.", filename); + // memory mode = save | ram + // when KSM is enabled + // for memory mode = ram, the filename is NULL + mem = memory_file_mmap_ksm(filename, size, flags); + if(mem == MAP_FAILED) return NULL; + + errno = 0; return mem; } -int savememory(const char *filename, void *mem, size_t size) { +int memory_file_save(const char *filename, void *mem, size_t size) { char tmpfilename[FILENAME_MAX + 1]; snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid()); @@ -1228,3 +1292,47 @@ unsigned long end_tsc(void) { return (((unsigned long)d << 32) | (unsigned long)a) - tsc; } */ + +int recursively_delete_dir(const char *path, const char *reason) { + DIR *dir = opendir(path); + if(!dir) { + error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path); + return -1; + } + + int ret = 0; + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR + && ( + (de->d_name[0] == '.' && de->d_name[1] == '\0') + || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + )) + continue; + + char fullpath[FILENAME_MAX + 1]; + snprintfz(fullpath, FILENAME_MAX, "%s/%s", path, de->d_name); + + if(de->d_type == DT_DIR) { + int r = recursively_delete_dir(fullpath, reason); + if(r > 0) ret += r; + continue; + } + + info("Deleting %s file '%s'", reason?reason:"", fullpath); + if(unlikely(unlink(fullpath) == -1)) + error("Cannot delete %s file '%s'", reason?reason:"", fullpath); + else + ret++; + } + + info("Deleting empty directory '%s'", path); + if(unlikely(rmdir(path) == -1)) + error("Cannot delete empty directory '%s'", path); + else + ret++; + + closedir(dir); + + return ret; +} diff --git a/src/common.h b/src/common.h index b82c078fa..efeebf16f 100644 --- a/src/common.h +++ b/src/common.h @@ -40,6 +40,7 @@ #include <strings.h> #include <arpa/inet.h> #include <netinet/tcp.h> +#include <sys/ioctl.h> #ifdef HAVE_NETINET_IN_H #include <netinet/in.h> @@ -201,12 +202,14 @@ #define NETDATA_OS_TYPE "linux" #endif /* __FreeBSD__, __APPLE__*/ +#include "statistical.h" #include "socket.h" #include "eval.h" #include "health.h" #include "rrd.h" #include "plugin_tc.h" #include "plugins_d.h" +#include "statsd.h" #include "rrd2json.h" #include "rrd2json_api_old.h" #include "web_client.h" @@ -217,6 +220,7 @@ #include "unit_test.h" #include "ipc.h" #include "backends.h" +#include "backend_prometheus.h" #include "inlined.h" #include "adaptive_resortable_list.h" #include "rrdpush.h" @@ -238,7 +242,8 @@ extern void netdata_fix_chart_name(char *s); extern void strreverse(char* begin, char* end); extern char *mystrsep(char **ptr, char *s); -extern char *trim(char *s); +extern char *trim(char *s); // remove leading and trailing spaces; may return NULL +extern char *trim_all(char *buffer); // like trim(), but also remove duplicate spaces inside the string; may return NULL extern int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args); extern int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4); @@ -265,9 +270,10 @@ extern void freez(void *ptr); #endif extern void json_escape_string(char *dst, const char *src, size_t size); +extern void json_fix_string(char *s); extern void *mymmap(const char *filename, size_t size, int flags, int ksm); -extern int savememory(const char *filename, void *mem, size_t size); +extern int memory_file_save(const char *filename, void *mem, size_t size); extern int fd_is_valid(int fd); @@ -289,6 +295,8 @@ extern pid_t get_system_pid_max(void); extern unsigned int hz; extern void get_system_HZ(void); +extern int recursively_delete_dir(const char *path, const char *reason); + extern volatile sig_atomic_t netdata_exit; extern const char *os_type; diff --git a/src/daemon.c b/src/daemon.c index 42b04c401..bc02446e0 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -73,8 +73,9 @@ void create_needed_dir(const char *dir, uid_t uid, gid_t gid) error("Cannot create directory '%s'", dir); } -int become_user(const char *username, int pid_fd) -{ +int become_user(const char *username, int pid_fd) { + int am_i_root = (getuid() == 0)?1:0; + struct passwd *pw = getpwnam(username); if(!pw) { error("User %s is not present.", username); @@ -94,12 +95,12 @@ int become_user(const char *username, int pid_fd) int ngroups = (int)sysconf(_SC_NGROUPS_MAX); gid_t *supplementary_groups = NULL; - if(ngroups) { + if(ngroups > 0) { supplementary_groups = mallocz(sizeof(gid_t) * ngroups); if(getgrouplist(username, gid, supplementary_groups, &ngroups) == -1) { - error("Cannot get supplementary groups of user '%s'.", username); - freez(supplementary_groups); - supplementary_groups = NULL; + if(am_i_root) + error("Cannot get supplementary groups of user '%s'.", username); + ngroups = 0; } } @@ -109,14 +110,17 @@ int become_user(const char *username, int pid_fd) chown_open_file(stdaccess_fd, uid, gid); chown_open_file(pid_fd, uid, gid); - if(supplementary_groups && ngroups) { - if(setgroups(ngroups, supplementary_groups) == -1) - error("Cannot set supplementary groups for user '%s'", username); - - freez(supplementary_groups); + if(supplementary_groups && ngroups > 0) { + if(setgroups((size_t)ngroups, supplementary_groups) == -1) { + if(am_i_root) + error("Cannot set supplementary groups for user '%s'", username); + } ngroups = 0; } + if(supplementary_groups) + freez(supplementary_groups); + #ifdef __APPLE__ if(setregid(gid, gid) != 0) { #else @@ -155,22 +159,42 @@ int become_user(const char *username, int pid_fd) return(0); } +#ifndef OOM_SCORE_ADJ_MAX +#define OOM_SCORE_ADJ_MAX 1000 +#endif +#ifndef OOM_SCORE_ADJ_MIN +#define OOM_SCORE_ADJ_MIN -1000 +#endif + static void oom_score_adj(void) { - int score = (int)config_get_number(CONFIG_SECTION_GLOBAL, "OOM score", 1000); + char buf[10 + 1]; + snprintfz(buf, 10, "%d", OOM_SCORE_ADJ_MAX); + + // check the environment + char *s = getenv("OOMScoreAdjust"); + if(!s || !*s) s = buf; + + // check netdata.conf configuration + s = config_get(CONFIG_SECTION_GLOBAL, "OOM score", s); + if(!s || !*s) s = buf; + + if(!isdigit(*s) && *s != '-' && *s != '+') { + info("Out-Of-Memory score not changed due to setting: '%s'", s); + return; + } int done = 0; int fd = open("/proc/self/oom_score_adj", O_WRONLY); if(fd != -1) { - char buf[10 + 1]; - ssize_t len = snprintfz(buf, 10, "%d", score); + ssize_t len = strlen(s); if(len > 0 && write(fd, buf, (size_t)len) == len) done = 1; close(fd); } if(!done) - error("Cannot adjust my Out-Of-Memory score to %d.", score); + error("Cannot adjust my Out-Of-Memory score to '%s'.", s); else - debug(D_SYSTEM, "Adjusted my Out-Of-Memory score to %d.", score); + info("Adjusted my Out-Of-Memory score to '%s'.", s); } static void process_nice_level(void) { diff --git a/src/eval.c b/src/eval.c index 122959ce4..9248109b0 100644 --- a/src/eval.c +++ b/src/eval.c @@ -723,7 +723,7 @@ static inline int parse_variable(const char **string, char *buffer, size_t len) static inline int parse_constant(const char **string, calculated_number *number) { char *end = NULL; - calculated_number n = strtold(*string, &end); + calculated_number n = str2ld(*string, &end); if(unlikely(!end || *string == end)) { *number = 0; return 0; diff --git a/src/freebsd_devstat.c b/src/freebsd_devstat.c new file mode 100644 index 000000000..5b0687d5b --- /dev/null +++ b/src/freebsd_devstat.c @@ -0,0 +1,662 @@ +#include "common.h" + +#include <sys/devicestat.h> + +struct disk { + char *name; + uint32_t hash; + size_t len; + + // flags + int configured; + int enabled; + int updated; + + int do_io; + int do_ops; + int do_qops; + int do_util; + int do_iotime; + int do_await; + int do_avagsz; + int do_svctm; + + + // data for differential charts + + struct prev_dstat { + collected_number bytes_read; + collected_number bytes_write; + collected_number operations_read; + collected_number operations_write; + collected_number duration_read_ms; + collected_number duration_write_ms; + collected_number busy_time_ms; + } prev_dstat; + + // charts and dimensions + + RRDSET *st_io; + RRDDIM *rd_io_in; + RRDDIM *rd_io_out; + + RRDSET *st_ops; + RRDDIM *rd_ops_in; + RRDDIM *rd_ops_out; + + RRDSET *st_qops; + RRDDIM *rd_qops; + + RRDSET *st_util; + RRDDIM *rd_util; + + RRDSET *st_iotime; + RRDDIM *rd_iotime_in; + RRDDIM *rd_iotime_out; + + RRDSET *st_await; + RRDDIM *rd_await_in; + RRDDIM *rd_await_out; + + RRDSET *st_avagsz; + RRDDIM *rd_avagsz_in; + RRDDIM *rd_avagsz_out; + + RRDSET *st_svctm; + RRDDIM *rd_svctm; + + struct disk *next; +}; + +static struct disk *disks_root = NULL, *disks_last_used = NULL; + +static size_t disks_added = 0, disks_found = 0; + +static void disk_free(struct disk *dm) { + if (likely(dm->st_io)) + rrdset_is_obsolete(dm->st_io); + if (likely(dm->st_ops)) + rrdset_is_obsolete(dm->st_ops); + if (likely(dm->st_qops)) + rrdset_is_obsolete(dm->st_qops); + if (likely(dm->st_util)) + rrdset_is_obsolete(dm->st_util); + if (likely(dm->st_iotime)) + rrdset_is_obsolete(dm->st_iotime); + if (likely(dm->st_await)) + rrdset_is_obsolete(dm->st_await); + if (likely(dm->st_avagsz)) + rrdset_is_obsolete(dm->st_avagsz); + if (likely(dm->st_svctm)) + rrdset_is_obsolete(dm->st_svctm); + + disks_added--; + freez(dm->name); + freez(dm); +} + +static void disks_cleanup() { + if (likely(disks_found == disks_added)) return; + + struct disk *dm = disks_root, *last = NULL; + while(dm) { + if (unlikely(!dm->updated)) { + // info("Removing disk '%s', linked after '%s'", dm->name, last?last->name:"ROOT"); + + if (disks_last_used == dm) + disks_last_used = last; + + struct disk *t = dm; + + if (dm == disks_root || !last) + disks_root = dm = dm->next; + + else + last->next = dm = dm->next; + + t->next = NULL; + disk_free(t); + } + else { + last = dm; + dm->updated = 0; + dm = dm->next; + } + } +} + +static struct disk *get_disk(const char *name) { + struct disk *dm; + + uint32_t hash = simple_hash(name); + + // search it, from the last position to the end + for(dm = disks_last_used ; dm ; dm = dm->next) { + if (unlikely(hash == dm->hash && !strcmp(name, dm->name))) { + disks_last_used = dm->next; + return dm; + } + } + + // search it from the beginning to the last position we used + for(dm = disks_root ; dm != disks_last_used ; dm = dm->next) { + if (unlikely(hash == dm->hash && !strcmp(name, dm->name))) { + disks_last_used = dm->next; + return dm; + } + } + + // create a new one + dm = callocz(1, sizeof(struct disk)); + dm->name = strdupz(name); + dm->hash = simple_hash(dm->name); + dm->len = strlen(dm->name); + disks_added++; + + // link it to the end + if (disks_root) { + struct disk *e; + for(e = disks_root; e->next ; e = e->next) ; + e->next = dm; + } + else + disks_root = dm; + + return dm; +} + +// -------------------------------------------------------------------------------------------------------------------- +// kern.devstat + +int do_kern_devstat(int update_every, usec_t dt) { + +#define DELAULT_EXLUDED_DISKS "" +#define CONFIG_SECTION_KERN_DEVSTAT "plugin:freebsd:kern.devstat" +#define BINTIME_SCALE 5.42101086242752217003726400434970855712890625e-17 // this is 1000/2^64 + + static int enable_new_disks = -1; + static int enable_pass_devices = -1, do_system_io = -1, do_io = -1, do_ops = -1, do_qops = -1, do_util = -1, + do_iotime = -1, do_await = -1, do_avagsz = -1, do_svctm = -1; + static SIMPLE_PATTERN *excluded_disks = NULL; + + if (unlikely(enable_new_disks == -1)) { + enable_new_disks = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, + "enable new disks detected at runtime", CONFIG_BOOLEAN_AUTO); + + enable_pass_devices = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, + "performance metrics for pass devices", CONFIG_BOOLEAN_AUTO); + + do_system_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "total bandwidth for all disks", + CONFIG_BOOLEAN_YES); + + do_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "bandwidth for all disks", + CONFIG_BOOLEAN_AUTO); + do_ops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "operations for all disks", + CONFIG_BOOLEAN_AUTO); + do_qops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "queued operations for all disks", + CONFIG_BOOLEAN_AUTO); + do_util = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "utilization percentage for all disks", + CONFIG_BOOLEAN_AUTO); + do_iotime = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "i/o time for all disks", + CONFIG_BOOLEAN_AUTO); + do_await = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o time for all disks", + CONFIG_BOOLEAN_AUTO); + do_avagsz = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o bandwidth for all disks", + CONFIG_BOOLEAN_AUTO); + do_svctm = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average service time for all disks", + CONFIG_BOOLEAN_AUTO); + + excluded_disks = simple_pattern_create( + config_get(CONFIG_SECTION_KERN_DEVSTAT, "disable by default disks matching", DELAULT_EXLUDED_DISKS) + , SIMPLE_PATTERN_EXACT + ); + } + + if (likely(do_system_io || do_io || do_ops || do_qops || do_util || do_iotime || do_await || do_avagsz || do_svctm)) { + static int mib_numdevs[3] = {0, 0, 0}; + int numdevs; + int common_error = 0; + + if (unlikely(GETSYSCTL_SIMPLE("kern.devstat.numdevs", mib_numdevs, numdevs))) { + common_error = 1; + } else { + static int mib_devstat[3] = {0, 0, 0}; + static void *devstat_data = NULL; + static int old_numdevs = 0; + + if (unlikely(numdevs != old_numdevs)) { + devstat_data = reallocz(devstat_data, sizeof(long) + sizeof(struct devstat) * + numdevs); // there is generation number before devstat structures + old_numdevs = numdevs; + } + if (unlikely(GETSYSCTL_WSIZE("kern.devstat.all", mib_devstat, devstat_data, + sizeof(long) + sizeof(struct devstat) * numdevs))) { + common_error = 1; + } else { + struct devstat *dstat; + int i; + collected_number total_disk_kbytes_read = 0; + collected_number total_disk_kbytes_write = 0; + + disks_found = 0; + + dstat = devstat_data + sizeof(long); // skip generation number + + for (i = 0; i < numdevs; i++) { + if (likely(do_system_io)) { + if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { + total_disk_kbytes_read += dstat[i].bytes[DEVSTAT_READ] / KILO_FACTOR; + total_disk_kbytes_write += dstat[i].bytes[DEVSTAT_WRITE] / KILO_FACTOR; + } + } + + if (unlikely(!enable_pass_devices)) + if ((dstat[i].device_type & DEVSTAT_TYPE_PASS) == DEVSTAT_TYPE_PASS) + continue; + + if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { + char disk[DEVSTAT_NAME_LEN + MAX_INT_DIGITS + 1]; + struct cur_dstat { + collected_number duration_read_ms; + collected_number duration_write_ms; + collected_number busy_time_ms; + } cur_dstat; + + sprintf(disk, "%s%d", dstat[i].device_name, dstat[i].unit_number); + + struct disk *dm = get_disk(disk); + dm->updated = 1; + disks_found++; + + if(unlikely(!dm->configured)) { + char var_name[4096 + 1]; + + // this is the first time we see this disk + + // remember we configured it + dm->configured = 1; + + dm->enabled = enable_new_disks; + + if (likely(dm->enabled)) + dm->enabled = !simple_pattern_matches(excluded_disks, disk); + + snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_KERN_DEVSTAT, disk); + dm->enabled = config_get_boolean_ondemand(var_name, "enabled", dm->enabled); + + dm->do_io = config_get_boolean_ondemand(var_name, "bandwidth", do_io); + dm->do_ops = config_get_boolean_ondemand(var_name, "operations", do_ops); + dm->do_qops = config_get_boolean_ondemand(var_name, "queued operations", do_qops); + dm->do_util = config_get_boolean_ondemand(var_name, "utilization percentage", do_util); + dm->do_iotime = config_get_boolean_ondemand(var_name, "i/o time", do_iotime); + dm->do_await = config_get_boolean_ondemand(var_name, "average completed i/o time", + do_await); + dm->do_avagsz = config_get_boolean_ondemand(var_name, "average completed i/o bandwidth", + do_avagsz); + dm->do_svctm = config_get_boolean_ondemand(var_name, "average service time", do_svctm); + + // initialise data for differential charts + + dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ]; + dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE]; + dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ]; + dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE]; + dm->prev_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000 + + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + dm->prev_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000 + + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + dm->prev_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + + dstat[i].busy_time.frac * BINTIME_SCALE; + } + + cur_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000 + + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + cur_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000 + + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + cur_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE; + + // -------------------------------------------------------------------- + + if(dm->do_io == CONFIG_BOOLEAN_YES || (dm->do_io == CONFIG_BOOLEAN_AUTO && + (dstat[i].bytes[DEVSTAT_READ] || dstat[i].bytes[DEVSTAT_WRITE]))) { + if (unlikely(!dm->st_io)) { + dm->st_io = rrdset_create_localhost("disk", + disk, + NULL, + disk, + "disk.io", + "Disk I/O Bandwidth", + "kilobytes/s", + 2000, + update_every, + RRDSET_TYPE_AREA + ); + + dm->rd_io_in = rrddim_add(dm->st_io, "reads", NULL, 1, KILO_FACTOR, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_io_out = rrddim_add(dm->st_io, "writes", NULL, -1, KILO_FACTOR, + RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(dm->st_io); + + rrddim_set_by_pointer(dm->st_io, dm->rd_io_in, dstat[i].bytes[DEVSTAT_READ]); + rrddim_set_by_pointer(dm->st_io, dm->rd_io_out, dstat[i].bytes[DEVSTAT_WRITE]); + rrdset_done(dm->st_io); + } + + // -------------------------------------------------------------------- + + if(dm->do_ops == CONFIG_BOOLEAN_YES || (dm->do_ops == CONFIG_BOOLEAN_AUTO && + (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + if (unlikely(!dm->st_ops)) { + dm->st_ops = rrdset_create_localhost("disk_ops", + disk, + NULL, + disk, + "disk.ops", + "Disk Completed I/O Operations", + "operations/s", + 2001, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(dm->st_ops, RRDSET_FLAG_DETAIL); + + dm->rd_ops_in = rrddim_add(dm->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + dm->rd_ops_out = rrddim_add(dm->st_ops, "writes", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(dm->st_ops); + + rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_in, dstat[i].operations[DEVSTAT_READ]); + rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_out, dstat[i].operations[DEVSTAT_WRITE]); + rrdset_done(dm->st_ops); + } + + // -------------------------------------------------------------------- + + if(dm->do_qops == CONFIG_BOOLEAN_YES || (dm->do_qops == CONFIG_BOOLEAN_AUTO && + (dstat[i].start_count || dstat[i].end_count))) { + if (unlikely(!dm->st_qops)) { + dm->st_qops = rrdset_create_localhost("disk_qops", + disk, + NULL, + disk, + "disk.qops", + "Disk Current I/O Operations", + "operations", + 2002, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(dm->st_qops, RRDSET_FLAG_DETAIL); + + dm->rd_qops = rrddim_add(dm->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(dm->st_qops); + + rrddim_set_by_pointer(dm->st_qops, dm->rd_qops, dstat[i].start_count - dstat[i].end_count); + rrdset_done(dm->st_qops); + } + + // -------------------------------------------------------------------- + + if(dm->do_util == CONFIG_BOOLEAN_YES || (dm->do_util == CONFIG_BOOLEAN_AUTO && + cur_dstat.busy_time_ms)) { + if (unlikely(!dm->st_util)) { + dm->st_util = rrdset_create_localhost("disk_util", + disk, + NULL, + disk, + "disk.util", + "Disk Utilization Time", + "% of time working", + 2004, + update_every, + RRDSET_TYPE_AREA + ); + + rrdset_flag_set(dm->st_util, RRDSET_FLAG_DETAIL); + + dm->rd_util = rrddim_add(dm->st_util, "utilization", NULL, 1, 10, + RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(dm->st_util); + + rrddim_set_by_pointer(dm->st_util, dm->rd_util, cur_dstat.busy_time_ms); + rrdset_done(dm->st_util); + } + + // -------------------------------------------------------------------- + + if(dm->do_iotime == CONFIG_BOOLEAN_YES || (dm->do_iotime == CONFIG_BOOLEAN_AUTO && + (cur_dstat.duration_read_ms || cur_dstat.duration_write_ms))) { + if (unlikely(!dm->st_iotime)) { + dm->st_iotime = rrdset_create_localhost("disk_iotime", + disk, + NULL, + disk, + "disk.iotime", + "Disk Total I/O Time", + "milliseconds/s", + 2022, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(dm->st_iotime, RRDSET_FLAG_DETAIL); + + dm->rd_iotime_in = rrddim_add(dm->st_iotime, "reads", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + dm->rd_iotime_out = rrddim_add(dm->st_iotime, "writes", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(dm->st_iotime); + + rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_in, cur_dstat.duration_read_ms); + rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_out, cur_dstat.duration_write_ms); + rrdset_done(dm->st_iotime); + } + + // -------------------------------------------------------------------- + // calculate differential charts + // only if this is not the first time we run + + if (likely(dt)) { + + // -------------------------------------------------------------------- + + if(dm->do_await == CONFIG_BOOLEAN_YES || (dm->do_await == CONFIG_BOOLEAN_AUTO && + (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + if (unlikely(!dm->st_await)) { + dm->st_await = rrdset_create_localhost("disk_await", + disk, + NULL, + disk, + "disk.await", + "Average Completed I/O Operation Time", + "ms per operation", + 2005, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(dm->st_await, RRDSET_FLAG_DETAIL); + + dm->rd_await_in = rrddim_add(dm->st_await, "reads", NULL, 1, 1, + RRD_ALGORITHM_ABSOLUTE); + dm->rd_await_out = rrddim_add(dm->st_await, "writes", NULL, -1, 1, + RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(dm->st_await); + + rrddim_set_by_pointer(dm->st_await, dm->rd_await_in, + (dstat[i].operations[DEVSTAT_READ] - + dm->prev_dstat.operations_read) ? + (cur_dstat.duration_read_ms - dm->prev_dstat.duration_read_ms) / + (dstat[i].operations[DEVSTAT_READ] - + dm->prev_dstat.operations_read) : + 0); + rrddim_set_by_pointer(dm->st_await, dm->rd_await_out, + (dstat[i].operations[DEVSTAT_WRITE] - + dm->prev_dstat.operations_write) ? + (cur_dstat.duration_write_ms - dm->prev_dstat.duration_write_ms) / + (dstat[i].operations[DEVSTAT_WRITE] - + dm->prev_dstat.operations_write) : + 0); + rrdset_done(dm->st_await); + } + + // -------------------------------------------------------------------- + + if(dm->do_avagsz == CONFIG_BOOLEAN_YES || (dm->do_avagsz == CONFIG_BOOLEAN_AUTO && + (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + if (unlikely(!dm->st_avagsz)) { + dm->st_avagsz = rrdset_create_localhost("disk_avgsz", + disk, + NULL, + disk, + "disk.avgsz", + "Average Completed I/O Operation Bandwidth", + "kilobytes per operation", + 2006, + update_every, + RRDSET_TYPE_AREA + ); + + rrdset_flag_set(dm->st_avagsz, RRDSET_FLAG_DETAIL); + + dm->rd_avagsz_in = rrddim_add(dm->st_avagsz, "reads", NULL, 1, KILO_FACTOR, + RRD_ALGORITHM_ABSOLUTE); + dm->rd_avagsz_out = rrddim_add(dm->st_avagsz, "writes", NULL, -1, KILO_FACTOR, + RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(dm->st_avagsz); + + rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_in, + (dstat[i].operations[DEVSTAT_READ] - + dm->prev_dstat.operations_read) ? + (dstat[i].bytes[DEVSTAT_READ] - dm->prev_dstat.bytes_read) / + (dstat[i].operations[DEVSTAT_READ] - + dm->prev_dstat.operations_read) : + 0); + rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_out, + (dstat[i].operations[DEVSTAT_WRITE] - + dm->prev_dstat.operations_write) ? + (dstat[i].bytes[DEVSTAT_WRITE] - dm->prev_dstat.bytes_write) / + (dstat[i].operations[DEVSTAT_WRITE] - + dm->prev_dstat.operations_write) : + 0); + rrdset_done(dm->st_avagsz); + } + + // -------------------------------------------------------------------- + + if(dm->do_svctm == CONFIG_BOOLEAN_YES || (dm->do_svctm == CONFIG_BOOLEAN_AUTO && + (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { + if (unlikely(!dm->st_svctm)) { + dm->st_svctm = rrdset_create_localhost("disk_svctm", + disk, + NULL, + disk, + "disk.svctm", + "Average Service Time", + "ms per operation", + 2007, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(dm->st_svctm, RRDSET_FLAG_DETAIL); + + dm->rd_svctm = rrddim_add(dm->st_svctm, "svctm", NULL, 1, 1, + RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(dm->st_svctm); + + rrddim_set_by_pointer(dm->st_svctm, dm->rd_svctm, + ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) + + (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) ? + (cur_dstat.busy_time_ms - dm->prev_dstat.busy_time_ms) / + ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) + + (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) : + 0); + rrdset_done(dm->st_svctm); + } + + // -------------------------------------------------------------------- + + dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ]; + dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE]; + dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ]; + dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE]; + dm->prev_dstat.duration_read_ms = cur_dstat.duration_read_ms; + dm->prev_dstat.duration_write_ms = cur_dstat.duration_write_ms; + dm->prev_dstat.busy_time_ms = cur_dstat.busy_time_ms; + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_system_io)) { + static RRDSET *st = NULL; + static RRDDIM *rd_in = NULL, *rd_out = NULL; + + if (unlikely(!st)) { + st = rrdset_create_localhost("system", + "io", + NULL, + "disk", + NULL, + "Disk I/O", + "kilobytes/s", + 150, + update_every, + RRDSET_TYPE_AREA + ); + + rd_in = rrddim_add(st, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set_by_pointer(st, rd_in, total_disk_kbytes_read); + rrddim_set_by_pointer(st, rd_out, total_disk_kbytes_write); + rrdset_done(st); + } + } + } + if (unlikely(common_error)) { + do_system_io = 0; + error("DISABLED: system.io chart"); + do_io = 0; + error("DISABLED: disk.* charts"); + do_ops = 0; + error("DISABLED: disk_ops.* charts"); + do_qops = 0; + error("DISABLED: disk_qops.* charts"); + do_util = 0; + error("DISABLED: disk_util.* charts"); + do_iotime = 0; + error("DISABLED: disk_iotime.* charts"); + do_await = 0; + error("DISABLED: disk_await.* charts"); + do_avagsz = 0; + error("DISABLED: disk_avgsz.* charts"); + do_svctm = 0; + error("DISABLED: disk_svctm.* charts"); + error("DISABLED: kern.devstat module"); + return 1; + } + } else { + error("DISABLED: kern.devstat module"); + return 1; + } + + disks_cleanup(); + + return 0; +} diff --git a/src/freebsd_getifaddrs.c b/src/freebsd_getifaddrs.c new file mode 100644 index 000000000..7355fac9e --- /dev/null +++ b/src/freebsd_getifaddrs.c @@ -0,0 +1,494 @@ +#include "common.h" + +#include <ifaddrs.h> + +struct network_interface { + char *name; + uint32_t hash; + size_t len; + + // flags + int configured; + int enabled; + int updated; + + int do_bandwidth; + int do_packets; + int do_errors; + int do_drops; + int do_events; + + // charts and dimensions + + RRDSET *st_bandwidth; + RRDDIM *rd_bandwidth_in; + RRDDIM *rd_bandwidth_out; + + RRDSET *st_packets; + RRDDIM *rd_packets_in; + RRDDIM *rd_packets_out; + RRDDIM *rd_packets_m_in; + RRDDIM *rd_packets_m_out; + + RRDSET *st_errors; + RRDDIM *rd_errors_in; + RRDDIM *rd_errors_out; + + RRDSET *st_drops; + RRDDIM *rd_drops_in; + RRDDIM *rd_drops_out; + + RRDSET *st_events; + RRDDIM *rd_events_coll; + + struct network_interface *next; +}; + +static struct network_interface *network_interfaces_root = NULL, *network_interfaces_last_used = NULL; + +static size_t network_interfaces_added = 0, network_interfaces_found = 0; + +static void network_interface_free(struct network_interface *ifm) { + if (likely(ifm->st_bandwidth)) + rrdset_is_obsolete(ifm->st_bandwidth); + if (likely(ifm->st_packets)) + rrdset_is_obsolete(ifm->st_packets); + if (likely(ifm->st_errors)) + rrdset_is_obsolete(ifm->st_errors); + if (likely(ifm->st_drops)) + rrdset_is_obsolete(ifm->st_drops); + if (likely(ifm->st_events)) + rrdset_is_obsolete(ifm->st_events); + + network_interfaces_added--; + freez(ifm->name); + freez(ifm); +} + +static void network_interfaces_cleanup() { + if (likely(network_interfaces_found == network_interfaces_added)) return; + + struct network_interface *ifm = network_interfaces_root, *last = NULL; + while(ifm) { + if (unlikely(!ifm->updated)) { + // info("Removing network interface '%s', linked after '%s'", ifm->name, last?last->name:"ROOT"); + + if (network_interfaces_last_used == ifm) + network_interfaces_last_used = last; + + struct network_interface *t = ifm; + + if (ifm == network_interfaces_root || !last) + network_interfaces_root = ifm = ifm->next; + + else + last->next = ifm = ifm->next; + + t->next = NULL; + network_interface_free(t); + } + else { + last = ifm; + ifm->updated = 0; + ifm = ifm->next; + } + } +} + +static struct network_interface *get_network_interface(const char *name) { + struct network_interface *ifm; + + uint32_t hash = simple_hash(name); + + // search it, from the last position to the end + for(ifm = network_interfaces_last_used ; ifm ; ifm = ifm->next) { + if (unlikely(hash == ifm->hash && !strcmp(name, ifm->name))) { + network_interfaces_last_used = ifm->next; + return ifm; + } + } + + // search it from the beginning to the last position we used + for(ifm = network_interfaces_root ; ifm != network_interfaces_last_used ; ifm = ifm->next) { + if (unlikely(hash == ifm->hash && !strcmp(name, ifm->name))) { + network_interfaces_last_used = ifm->next; + return ifm; + } + } + + // create a new one + ifm = callocz(1, sizeof(struct network_interface)); + ifm->name = strdupz(name); + ifm->hash = simple_hash(ifm->name); + ifm->len = strlen(ifm->name); + network_interfaces_added++; + + // link it to the end + if (network_interfaces_root) { + struct network_interface *e; + for(e = network_interfaces_root; e->next ; e = e->next) ; + e->next = ifm; + } + else + network_interfaces_root = ifm; + + return ifm; +} + +// -------------------------------------------------------------------------------------------------------------------- +// getifaddrs + +int do_getifaddrs(int update_every, usec_t dt) { + (void)dt; + +#define DELAULT_EXLUDED_INTERFACES "lo*" +#define CONFIG_SECTION_GETIFADDRS "plugin:freebsd:getifaddrs" + + static int enable_new_interfaces = -1; + static int do_bandwidth_ipv4 = -1, do_bandwidth_ipv6 = -1, do_bandwidth = -1, do_packets = -1, + do_errors = -1, do_drops = -1, do_events = -1; + static SIMPLE_PATTERN *excluded_interfaces = NULL; + + if (unlikely(enable_new_interfaces == -1)) { + enable_new_interfaces = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, + "enable new interfaces detected at runtime", + CONFIG_BOOLEAN_AUTO); + + do_bandwidth_ipv4 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv4 interfaces", + CONFIG_BOOLEAN_AUTO); + do_bandwidth_ipv6 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv6 interfaces", + CONFIG_BOOLEAN_AUTO); + do_bandwidth = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "bandwidth for all interfaces", + CONFIG_BOOLEAN_AUTO); + do_packets = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "packets for all interfaces", + CONFIG_BOOLEAN_AUTO); + do_errors = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "errors for all interfaces", + CONFIG_BOOLEAN_AUTO); + do_drops = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "drops for all interfaces", + CONFIG_BOOLEAN_AUTO); + do_events = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "collisions for all interfaces", + CONFIG_BOOLEAN_AUTO); + + excluded_interfaces = simple_pattern_create( + config_get(CONFIG_SECTION_GETIFADDRS, "disable by default interfaces matching", + DELAULT_EXLUDED_INTERFACES) + , SIMPLE_PATTERN_EXACT + ); + } + + if (likely(do_bandwidth_ipv4 || do_bandwidth_ipv6 || do_bandwidth || do_packets || do_errors || + do_drops || do_events)) { + struct ifaddrs *ifap; + + if (unlikely(getifaddrs(&ifap))) { + error("FREEBSD: getifaddrs() failed"); + do_bandwidth_ipv4 = 0; + error("DISABLED: system.ipv4 chart"); + do_bandwidth_ipv6 = 0; + error("DISABLED: system.ipv6 chart"); + do_bandwidth = 0; + error("DISABLED: net.* charts"); + do_packets = 0; + error("DISABLED: net_packets.* charts"); + do_errors = 0; + error("DISABLED: net_errors.* charts"); + do_drops = 0; + error("DISABLED: net_drops.* charts"); + do_events = 0; + error("DISABLED: net_events.* charts"); + error("DISABLED: getifaddrs module"); + return 1; + } else { +#define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s) + struct ifaddrs *ifa; + struct iftot { + u_long ift_ibytes; + u_long ift_obytes; + } iftot = {0, 0}; + + // -------------------------------------------------------------------- + + if (likely(do_bandwidth_ipv4)) { + iftot.ift_ibytes = iftot.ift_obytes = 0; + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_INET) + continue; + iftot.ift_ibytes += IFA_DATA(ibytes); + iftot.ift_obytes += IFA_DATA(obytes); + } + + static RRDSET *st = NULL; + static RRDDIM *rd_in = NULL, *rd_out = NULL; + + if (unlikely(!st)) { + st = rrdset_create_localhost("system", + "ipv4", + NULL, + "network", + NULL, + "IPv4 Bandwidth", + "kilobits/s", + 500, + update_every, + RRDSET_TYPE_AREA + ); + + rd_in = rrddim_add(st, "InOctets", "received", 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st, "OutOctets", "sent", -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes); + rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_bandwidth_ipv6)) { + iftot.ift_ibytes = iftot.ift_obytes = 0; + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_INET6) + continue; + iftot.ift_ibytes += IFA_DATA(ibytes); + iftot.ift_obytes += IFA_DATA(obytes); + } + + static RRDSET *st = NULL; + static RRDDIM *rd_in = NULL, *rd_out = NULL; + + if (unlikely(!st)) { + st = rrdset_create_localhost("system", + "ipv6", + NULL, + "network", + NULL, + "IPv6 Bandwidth", + "kilobits/s", + 500, + update_every, + RRDSET_TYPE_AREA + ); + + rd_in = rrddim_add(st, "received", NULL, 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); + rd_out = rrddim_add(st, "sent", NULL, -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes); + rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + network_interfaces_found = 0; + + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_LINK) + continue; + + struct network_interface *ifm = get_network_interface(ifa->ifa_name); + ifm->updated = 1; + network_interfaces_found++; + + if (unlikely(!ifm->configured)) { + char var_name[4096 + 1]; + + // this is the first time we see this network interface + + // remember we configured it + ifm->configured = 1; + + ifm->enabled = enable_new_interfaces; + + if (likely(ifm->enabled)) + ifm->enabled = !simple_pattern_matches(excluded_interfaces, ifa->ifa_name); + + snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETIFADDRS, ifa->ifa_name); + ifm->enabled = config_get_boolean_ondemand(var_name, "enabled", ifm->enabled); + + if (unlikely(ifm->enabled == CONFIG_BOOLEAN_NO)) + continue; + + ifm->do_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", do_bandwidth); + ifm->do_packets = config_get_boolean_ondemand(var_name, "packets", do_packets); + ifm->do_errors = config_get_boolean_ondemand(var_name, "errors", do_errors); + ifm->do_drops = config_get_boolean_ondemand(var_name, "drops", do_drops); + ifm->do_events = config_get_boolean_ondemand(var_name, "events", do_events); + } + + if (unlikely(!ifm->enabled)) + continue; + + // -------------------------------------------------------------------- + + if (ifm->do_bandwidth == CONFIG_BOOLEAN_YES || (ifm->do_bandwidth == CONFIG_BOOLEAN_AUTO && + (IFA_DATA(ibytes) || IFA_DATA(obytes)))) { + if (unlikely(!ifm->st_bandwidth)) { + ifm->st_bandwidth = rrdset_create_localhost("net", + ifa->ifa_name, + NULL, + ifa->ifa_name, + "net.net", + "Bandwidth", + "kilobits/s", + 7000, + update_every, + RRDSET_TYPE_AREA + ); + + ifm->rd_bandwidth_in = rrddim_add(ifm->st_bandwidth, "received", NULL, 8, KILO_FACTOR, + RRD_ALGORITHM_INCREMENTAL); + ifm->rd_bandwidth_out = rrddim_add(ifm->st_bandwidth, "sent", NULL, -8, KILO_FACTOR, + RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(ifm->st_bandwidth); + + rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_in, IFA_DATA(ibytes)); + rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_out, IFA_DATA(obytes)); + rrdset_done(ifm->st_bandwidth); + } + + // -------------------------------------------------------------------- + + if (ifm->do_packets == CONFIG_BOOLEAN_YES || (ifm->do_packets == CONFIG_BOOLEAN_AUTO && + (IFA_DATA(ipackets) || IFA_DATA(opackets) || IFA_DATA(imcasts) || IFA_DATA(omcasts)))) { + if (unlikely(!ifm->st_packets)) { + ifm->st_packets = rrdset_create_localhost("net_packets", + ifa->ifa_name, + NULL, + ifa->ifa_name, + "net.packets", + "Packets", + "packets/s", + 7001, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(ifm->st_packets, RRDSET_FLAG_DETAIL); + + ifm->rd_packets_in = rrddim_add(ifm->st_packets, "received", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + ifm->rd_packets_out = rrddim_add(ifm->st_packets, "sent", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); + ifm->rd_packets_m_in = rrddim_add(ifm->st_packets, "multicast_received", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); + ifm->rd_packets_m_out = rrddim_add(ifm->st_packets, "multicast_sent", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(ifm->st_packets); + + rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_in, IFA_DATA(ipackets)); + rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_out, IFA_DATA(opackets)); + rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_in, IFA_DATA(imcasts)); + rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_out, IFA_DATA(omcasts)); + rrdset_done(ifm->st_packets); + } + + // -------------------------------------------------------------------- + + if (ifm->do_errors == CONFIG_BOOLEAN_YES || (ifm->do_errors == CONFIG_BOOLEAN_AUTO && + (IFA_DATA(ierrors) || IFA_DATA(oerrors)))) { + if (unlikely(!ifm->st_errors)) { + ifm->st_errors = rrdset_create_localhost("net_errors", + ifa->ifa_name, + NULL, + ifa->ifa_name, + "net.errors", + "Interface Errors", + "errors/s", + 7002, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(ifm->st_errors, RRDSET_FLAG_DETAIL); + + ifm->rd_errors_in = rrddim_add(ifm->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + ifm->rd_errors_out = rrddim_add(ifm->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(ifm->st_errors); + + rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_in, IFA_DATA(ierrors)); + rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_out, IFA_DATA(oerrors)); + rrdset_done(ifm->st_errors); + } + // -------------------------------------------------------------------- + + if (ifm->do_drops == CONFIG_BOOLEAN_YES || (ifm->do_drops == CONFIG_BOOLEAN_AUTO && + (IFA_DATA(iqdrops) + #if __FreeBSD__ >= 11 + || IFA_DATA(oqdrops) +#endif + ))) { + if (unlikely(!ifm->st_drops)) { + ifm->st_drops = rrdset_create_localhost("net_drops", + ifa->ifa_name, + NULL, + ifa->ifa_name, + "net.drops", + "Interface Drops", + "drops/s", + 7003, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(ifm->st_drops, RRDSET_FLAG_DETAIL); + + ifm->rd_drops_in = rrddim_add(ifm->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#if __FreeBSD__ >= 11 + ifm->rd_drops_out = rrddim_add(ifm->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); +#endif + } else + rrdset_next(ifm->st_drops); + + rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_in, IFA_DATA(iqdrops)); +#if __FreeBSD__ >= 11 + rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_out, IFA_DATA(oqdrops)); +#endif + rrdset_done(ifm->st_drops); + } + + // -------------------------------------------------------------------- + + if (ifm->do_events == CONFIG_BOOLEAN_YES || (ifm->do_events == CONFIG_BOOLEAN_AUTO && + IFA_DATA(collisions))) { + if (unlikely(!ifm->st_events)) { + ifm->st_events = rrdset_create_localhost("net_events", + ifa->ifa_name, + NULL, + ifa->ifa_name, + "net.events", + "Network Interface Events", + "events/s", + 7006, + update_every, + RRDSET_TYPE_LINE + ); + + rrdset_flag_set(ifm->st_events, RRDSET_FLAG_DETAIL); + + ifm->rd_events_coll = rrddim_add(ifm->st_events, "collisions", NULL, -1, 1, + RRD_ALGORITHM_INCREMENTAL); + } else + rrdset_next(ifm->st_events); + + rrddim_set_by_pointer(ifm->st_events, ifm->rd_events_coll, IFA_DATA(collisions)); + rrdset_done(ifm->st_events); + } + } + + freeifaddrs(ifap); + } + } else { + error("DISABLED: getifaddrs module"); + return 1; + } + + network_interfaces_cleanup(); + + return 0; +} diff --git a/src/freebsd_getmntinfo.c b/src/freebsd_getmntinfo.c new file mode 100644 index 000000000..e7ca56b57 --- /dev/null +++ b/src/freebsd_getmntinfo.c @@ -0,0 +1,293 @@ +#include "common.h" + +#include <sys/mount.h> + +struct mount_point { + char *name; + uint32_t hash; + size_t len; + + // flags + int configured; + int enabled; + int updated; + + int do_space; + int do_inodes; + + size_t collected; // the number of times this has been collected + + // charts and dimensions + + RRDSET *st_space; + RRDDIM *rd_space_used; + RRDDIM *rd_space_avail; + RRDDIM *rd_space_reserved; + + RRDSET *st_inodes; + RRDDIM *rd_inodes_used; + RRDDIM *rd_inodes_avail; + + struct mount_point *next; +}; + +static struct mount_point *mount_points_root = NULL, *mount_points_last_used = NULL; + +static size_t mount_points_added = 0, mount_points_found = 0; + +static void mount_point_free(struct mount_point *m) { + if (likely(m->st_space)) + rrdset_is_obsolete(m->st_space); + if (likely(m->st_inodes)) + rrdset_is_obsolete(m->st_inodes); + + mount_points_added--; + freez(m->name); + freez(m); +} + +static void mount_points_cleanup() { + if (likely(mount_points_found == mount_points_added)) return; + + struct mount_point *m = mount_points_root, *last = NULL; + while(m) { + if (unlikely(!m->updated)) { + // info("Removing mount point '%s', linked after '%s'", m->name, last?last->name:"ROOT"); + + if (mount_points_last_used == m) + mount_points_last_used = last; + + struct mount_point *t = m; + + if (m == mount_points_root || !last) + mount_points_root = m = m->next; + + else + last->next = m = m->next; + + t->next = NULL; + mount_point_free(t); + } + else { + last = m; + m->updated = 0; + m = m->next; + } + } +} + +static struct mount_point *get_mount_point(const char *name) { + struct mount_point *m; + + uint32_t hash = simple_hash(name); + + // search it, from the last position to the end + for(m = mount_points_last_used ; m ; m = m->next) { + if (unlikely(hash == m->hash && !strcmp(name, m->name))) { + mount_points_last_used = m->next; + return m; + } + } + + // search it from the beginning to the last position we used + for(m = mount_points_root ; m != mount_points_last_used ; m = m->next) { + if (unlikely(hash == m->hash && !strcmp(name, m->name))) { + mount_points_last_used = m->next; + return m; + } + } + + // create a new one + m = callocz(1, sizeof(struct mount_point)); + m->name = strdupz(name); + m->hash = simple_hash(m->name); + m->len = strlen(m->name); + mount_points_added++; + + // link it to the end + if (mount_points_root) { + struct mount_point *e; + for(e = mount_points_root; e->next ; e = e->next) ; + e->next = m; + } + else + mount_points_root = m; + + return m; +} + +// -------------------------------------------------------------------------------------------------------------------- +// getmntinfo + +int do_getmntinfo(int update_every, usec_t dt) { + (void)dt; + +#define DELAULT_EXLUDED_PATHS "/proc/*" +// taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes +#define DEFAULT_EXCLUDED_FILESYSTEMS "autofs procfs subfs devfs none" +#define CONFIG_SECTION_GETMNTINFO "plugin:freebsd:getmntinfo" + + static int enable_new_mount_points = -1; + static int do_space = -1, do_inodes = -1; + static SIMPLE_PATTERN *excluded_mountpoints = NULL; + static SIMPLE_PATTERN *excluded_filesystems = NULL; + + if (unlikely(enable_new_mount_points == -1)) { + enable_new_mount_points = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, + "enable new mount points detected at runtime", + CONFIG_BOOLEAN_AUTO); + + do_space = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "space usage for all disks", CONFIG_BOOLEAN_AUTO); + do_inodes = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "inodes usage for all disks", CONFIG_BOOLEAN_AUTO); + + excluded_mountpoints = simple_pattern_create( + config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on paths", + DELAULT_EXLUDED_PATHS), + SIMPLE_PATTERN_EXACT + ); + + excluded_filesystems = simple_pattern_create( + config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on filesystems", + DEFAULT_EXCLUDED_FILESYSTEMS), + SIMPLE_PATTERN_EXACT + ); + } + + if (likely(do_space || do_inodes)) { + struct statfs *mntbuf; + int mntsize; + + // there is no mount info in sysctl MIBs + if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) { + error("FREEBSD: getmntinfo() failed"); + do_space = 0; + error("DISABLED: disk_space.* charts"); + do_inodes = 0; + error("DISABLED: disk_inodes.* charts"); + error("DISABLED: getmntinfo module"); + return 1; + } else { + int i; + + mount_points_found = 0; + + for (i = 0; i < mntsize; i++) { + char title[4096 + 1]; + + struct mount_point *m = get_mount_point(mntbuf[i].f_mntonname); + m->updated = 1; + mount_points_found++; + + if (unlikely(!m->configured)) { + char var_name[4096 + 1]; + + // this is the first time we see this filesystem + + // remember we configured it + m->configured = 1; + + m->enabled = enable_new_mount_points; + + if (likely(m->enabled)) + m->enabled = !(simple_pattern_matches(excluded_mountpoints, mntbuf[i].f_mntonname) + || simple_pattern_matches(excluded_filesystems, mntbuf[i].f_fstypename)); + + snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETMNTINFO, mntbuf[i].f_mntonname); + m->enabled = config_get_boolean_ondemand(var_name, "enabled", m->enabled); + + if (unlikely(m->enabled == CONFIG_BOOLEAN_NO)) + continue; + + m->do_space = config_get_boolean_ondemand(var_name, "space usage", do_space); + m->do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", do_inodes); + } + + if (unlikely(!m->enabled)) + continue; + + if (unlikely(mntbuf[i].f_flags & MNT_RDONLY && !m->collected)) + continue; + + // -------------------------------------------------------------------------- + + int rendered = 0; + + if (m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_blocks > 2))) { + if (unlikely(!m->st_space)) { + snprintfz(title, 4096, "Disk Space Usage for %s [%s]", + mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); + m->st_space = rrdset_create_localhost("disk_space", + mntbuf[i].f_mntonname, + NULL, + mntbuf[i].f_mntonname, + "disk.space", + title, + "GB", + 2023, + update_every, + RRDSET_TYPE_STACKED + ); + + m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL, + mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); + m->rd_space_used = rrddim_add(m->st_space, "used", NULL, + mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); + m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root", + mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(m->st_space); + + rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number) mntbuf[i].f_bavail); + rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number) (mntbuf[i].f_blocks - + mntbuf[i].f_bfree)); + rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number) (mntbuf[i].f_bfree - + mntbuf[i].f_bavail)); + rrdset_done(m->st_space); + + rendered++; + } + + // -------------------------------------------------------------------------- + + if (m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_files > 1))) { + if (unlikely(!m->st_inodes)) { + snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", + mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); + m->st_inodes = rrdset_create_localhost("disk_inodes", + mntbuf[i].f_mntonname, + NULL, + mntbuf[i].f_mntonname, + "disk.inodes", + title, + "Inodes", + 2024, + update_every, + RRDSET_TYPE_STACKED + ); + + m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(m->st_inodes); + + rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number) mntbuf[i].f_ffree); + rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number) (mntbuf[i].f_files - + mntbuf[i].f_ffree)); + rrdset_done(m->st_inodes); + + rendered++; + } + + if (likely(rendered)) + m->collected++; + } + } + } else { + error("DISABLED: getmntinfo module"); + return 1; + } + + mount_points_cleanup(); + + return 0; +} diff --git a/src/freebsd_ipfw.c b/src/freebsd_ipfw.c new file mode 100644 index 000000000..b89650a04 --- /dev/null +++ b/src/freebsd_ipfw.c @@ -0,0 +1,360 @@ +#include "common.h" + +#include <netinet/ip_fw.h> + +#define FREE_MEM_THRESHOLD 10000 // number of unused chunks that trigger memory freeing + +#define COMMON_IPFW_ERROR() error("DISABLED: ipfw.packets chart"); \ + error("DISABLED: ipfw.bytes chart"); \ + error("DISABLED: ipfw.dyn_active chart"); \ + error("DISABLED: ipfw.dyn_expired chart"); \ + error("DISABLED: ipfw.mem chart"); + +// -------------------------------------------------------------------------------------------------------------------- +// ipfw + +int do_ipfw(int update_every, usec_t dt) { + (void)dt; +#if __FreeBSD__ >= 11 + + static int do_static = -1, do_dynamic = -1, do_mem = -1; + + if (unlikely(do_static == -1)) { + do_static = config_get_boolean("plugin:freebsd:ipfw", "counters for static rules", 1); + do_dynamic = config_get_boolean("plugin:freebsd:ipfw", "number of dynamic rules", 1); + do_mem = config_get_boolean("plugin:freebsd:ipfw", "allocated memory", 1); + } + + // variables for getting ipfw configuration + + int error; + static int ipfw_socket = -1; + static ipfw_cfg_lheader *cfg = NULL; + ip_fw3_opheader *op3 = NULL; + static socklen_t *optlen = NULL, cfg_size = 0; + + // variables for static rules handling + + ipfw_obj_ctlv *ctlv = NULL; + ipfw_obj_tlv *rbase = NULL; + int rcnt = 0; + + int n, seen; + struct ip_fw_rule *rule; + struct ip_fw_bcounter *cntr; + int c = 0; + + char rule_num_str[12]; + + // variables for dynamic rules handling + + caddr_t dynbase = NULL; + size_t dynsz = 0; + size_t readsz = sizeof(*cfg);; + int ttype = 0; + ipfw_obj_tlv *tlv; + ipfw_dyn_rule *dyn_rule; + uint16_t rulenum, prev_rulenum = IPFW_DEFAULT_RULE; + unsigned srn, static_rules_num = 0; + static size_t dyn_rules_num_size = 0; + + static struct dyn_rule_num { + uint16_t rule_num; + uint32_t active_rules; + uint32_t expired_rules; + } *dyn_rules_num = NULL; + + uint32_t *dyn_rules_counter; + + if (likely(do_static | do_dynamic | do_mem)) { + + // initialize the smallest ipfw_cfg_lheader possible + + if (unlikely((optlen == NULL) || (cfg == NULL))) { + optlen = reallocz(optlen, sizeof(socklen_t)); + *optlen = cfg_size = 32; + cfg = reallocz(cfg, *optlen); + } + + // get socket descriptor and initialize ipfw_cfg_lheader structure + + if (unlikely(ipfw_socket == -1)) + ipfw_socket = socket(AF_INET, SOCK_RAW, IPPROTO_RAW); + if (unlikely(ipfw_socket == -1)) { + error("FREEBSD: can't get socket for ipfw configuration"); + error("FREEBSD: run netdata as root to get access to ipfw data"); + COMMON_IPFW_ERROR(); + return 1; + } + + bzero(cfg, 32); + cfg->flags = IPFW_CFG_GET_STATIC | IPFW_CFG_GET_COUNTERS | IPFW_CFG_GET_STATES; + op3 = &cfg->opheader; + op3->opcode = IP_FW_XGET; + + // get ifpw configuration size than get configuration + + *optlen = cfg_size; + error = getsockopt(ipfw_socket, IPPROTO_IP, IP_FW3, op3, optlen); + if (error) + if (errno != ENOMEM) { + error("FREEBSD: ipfw socket reading error"); + COMMON_IPFW_ERROR(); + return 1; + } + if ((cfg->size > cfg_size) || ((cfg_size - cfg->size) > sizeof(struct dyn_rule_num) * FREE_MEM_THRESHOLD)) { + *optlen = cfg_size = cfg->size; + cfg = reallocz(cfg, *optlen); + bzero(cfg, 32); + cfg->flags = IPFW_CFG_GET_STATIC | IPFW_CFG_GET_COUNTERS | IPFW_CFG_GET_STATES; + op3 = &cfg->opheader; + op3->opcode = IP_FW_XGET; + error = getsockopt(ipfw_socket, IPPROTO_IP, IP_FW3, op3, optlen); + if (error) { + error("FREEBSD: ipfw socket reading error"); + COMMON_IPFW_ERROR(); + return 1; + } + } + + // go through static rules configuration structures + + ctlv = (ipfw_obj_ctlv *) (cfg + 1); + + if (cfg->flags & IPFW_CFG_GET_STATIC) { + /* We've requested static rules */ + if (ctlv->head.type == IPFW_TLV_TBLNAME_LIST) { + readsz += ctlv->head.length; + ctlv = (ipfw_obj_ctlv *) ((caddr_t) ctlv + + ctlv->head.length); + } + + if (ctlv->head.type == IPFW_TLV_RULE_LIST) { + rbase = (ipfw_obj_tlv *) (ctlv + 1); + rcnt = ctlv->count; + readsz += ctlv->head.length; + ctlv = (ipfw_obj_ctlv *) ((caddr_t) ctlv + ctlv->head.length); + } + } + + if ((cfg->flags & IPFW_CFG_GET_STATES) && (readsz != *optlen)) { + /* We may have some dynamic states */ + dynsz = *optlen - readsz; + /* Skip empty header */ + if (dynsz != sizeof(ipfw_obj_ctlv)) + dynbase = (caddr_t) ctlv; + else + dynsz = 0; + } + + // -------------------------------------------------------------------- + + if (likely(do_mem)) { + static RRDSET *st_mem = NULL; + static RRDDIM *rd_dyn_mem = NULL; + static RRDDIM *rd_stat_mem = NULL; + + if (unlikely(!st_mem)) { + st_mem = rrdset_create_localhost("ipfw", + "mem", + NULL, + "memory allocated", + NULL, + "Memory allocated by rules", + "bytes", + 3005, + update_every, + RRDSET_TYPE_STACKED + ); + rrdset_flag_set(st_mem, RRDSET_FLAG_DETAIL); + + rd_dyn_mem = rrddim_add(st_mem, "dynamic", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_stat_mem = rrddim_add(st_mem, "static", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } else + rrdset_next(st_mem); + + rrddim_set_by_pointer(st_mem, rd_dyn_mem, dynsz); + rrddim_set_by_pointer(st_mem, rd_stat_mem, *optlen - dynsz); + rrdset_done(st_mem); + } + + // -------------------------------------------------------------------- + + static RRDSET *st_packets = NULL, *st_bytes = NULL; + RRDDIM *rd_packets = NULL, *rd_bytes = NULL; + + if (likely(do_static || do_dynamic)) { + if (likely(do_static)) { + if (unlikely(!st_packets)) + st_packets = rrdset_create_localhost("ipfw", + "packets", + NULL, + "static rules", + NULL, + "Packets", + "packets/s", + 3001, + update_every, + RRDSET_TYPE_STACKED + ); + else + rrdset_next(st_packets); + + if (unlikely(!st_bytes)) + st_bytes = rrdset_create_localhost("ipfw", + "bytes", + NULL, + "static rules", + NULL, + "Bytes", + "bytes/s", + 3002, + update_every, + RRDSET_TYPE_STACKED + ); + else + rrdset_next(st_bytes); + } + + for (n = seen = 0; n < rcnt; n++, rbase = (ipfw_obj_tlv *) ((caddr_t) rbase + rbase->length)) { + cntr = (struct ip_fw_bcounter *) (rbase + 1); + rule = (struct ip_fw_rule *) ((caddr_t) cntr + cntr->size); + if (rule->rulenum != prev_rulenum) + static_rules_num++; + if (rule->rulenum > IPFW_DEFAULT_RULE) + break; + + if (likely(do_static)) { + sprintf(rule_num_str, "%d_%d", rule->rulenum, rule->id); + + rd_packets = rrddim_find(st_packets, rule_num_str); + if (unlikely(!rd_packets)) + rd_packets = rrddim_add(st_packets, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st_packets, rd_packets, cntr->pcnt); + + rd_bytes = rrddim_find(st_bytes, rule_num_str); + if (unlikely(!rd_bytes)) + rd_bytes = rrddim_add(st_bytes, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st_bytes, rd_bytes, cntr->bcnt); + } + + c += rbase->length; + seen++; + } + + if (likely(do_static)) { + rrdset_done(st_packets); + rrdset_done(st_bytes); + } + } + + // -------------------------------------------------------------------- + + // go through dynamic rules configuration structures + + if (likely(do_dynamic && (dynsz > 0))) { + if ((dyn_rules_num_size < sizeof(struct dyn_rule_num) * static_rules_num) || + ((dyn_rules_num_size - sizeof(struct dyn_rule_num) * static_rules_num) > + sizeof(struct dyn_rule_num) * FREE_MEM_THRESHOLD)) { + dyn_rules_num_size = sizeof(struct dyn_rule_num) * static_rules_num; + dyn_rules_num = reallocz(dyn_rules_num, dyn_rules_num_size); + } + bzero(dyn_rules_num, sizeof(struct dyn_rule_num) * static_rules_num); + dyn_rules_num->rule_num = IPFW_DEFAULT_RULE; + + if (dynsz > 0 && ctlv->head.type == IPFW_TLV_DYNSTATE_LIST) { + dynbase += sizeof(*ctlv); + dynsz -= sizeof(*ctlv); + ttype = IPFW_TLV_DYN_ENT; + } + + while (dynsz > 0) { + tlv = (ipfw_obj_tlv *) dynbase; + if (tlv->type != ttype) + break; + + dyn_rule = (ipfw_dyn_rule *) (tlv + 1); + bcopy(&dyn_rule->rule, &rulenum, sizeof(rulenum)); + + for (srn = 0; srn < (static_rules_num - 1); srn++) { + if (dyn_rule->expire > 0) + dyn_rules_counter = &dyn_rules_num[srn].active_rules; + else + dyn_rules_counter = &dyn_rules_num[srn].expired_rules; + if (dyn_rules_num[srn].rule_num == rulenum) { + (*dyn_rules_counter)++; + break; + } + if (dyn_rules_num[srn].rule_num == IPFW_DEFAULT_RULE) { + dyn_rules_num[srn].rule_num = rulenum; + dyn_rules_num[srn + 1].rule_num = IPFW_DEFAULT_RULE; + (*dyn_rules_counter)++; + break; + } + } + + dynsz -= tlv->length; + dynbase += tlv->length; + } + + // -------------------------------------------------------------------- + + static RRDSET *st_active = NULL, *st_expired = NULL; + RRDDIM *rd_active = NULL, *rd_expired = NULL; + + if (unlikely(!st_active)) + st_active = rrdset_create_localhost("ipfw", + "active", + NULL, + "dynamic_rules", + NULL, + "Active rules", + "rules", + 3003, + update_every, + RRDSET_TYPE_STACKED + ); + else + rrdset_next(st_active); + + if (unlikely(!st_expired)) + st_expired = rrdset_create_localhost("ipfw", + "expired", + NULL, + "dynamic_rules", + NULL, + "Expired rules", + "rules", + 3004, + update_every, + RRDSET_TYPE_STACKED + ); + else + rrdset_next(st_expired); + + for (srn = 0; (srn < (static_rules_num - 1)) && (dyn_rules_num[srn].rule_num != IPFW_DEFAULT_RULE); srn++) { + sprintf(rule_num_str, "%d", dyn_rules_num[srn].rule_num); + + rd_active = rrddim_find(st_active, rule_num_str); + if (unlikely(!rd_active)) + rd_active = rrddim_add(st_active, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_set_by_pointer(st_active, rd_active, dyn_rules_num[srn].active_rules); + + rd_expired = rrddim_find(st_expired, rule_num_str); + if (unlikely(!rd_expired)) + rd_expired = rrddim_add(st_expired, rule_num_str, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_set_by_pointer(st_expired, rd_expired, dyn_rules_num[srn].expired_rules); + } + + rrdset_done(st_active); + rrdset_done(st_expired); + } + } + + return 0; +#else + error("FREEBSD: ipfw charts supported for FreeBSD 11.0 and newer releases only"); + COMMON_IPFW_ERROR(); + return 1; +#endif +} diff --git a/src/freebsd_kstat_zfs.c b/src/freebsd_kstat_zfs.c new file mode 100644 index 000000000..17642994e --- /dev/null +++ b/src/freebsd_kstat_zfs.c @@ -0,0 +1,212 @@ +#include "common.h" +#include "zfs_common.h" + +struct arcstats arcstats = { 0 }; + +// kstat.zfs.misc.arcstats +int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt) { + (void)dt; + + unsigned long long l2_size; + size_t uint64_t_size = sizeof(uint64_t); + static struct mibs { + int hits[5]; + int misses[5]; + int demand_data_hits[5]; + int demand_data_misses[5]; + int demand_metadata_hits[5]; + int demand_metadata_misses[5]; + int prefetch_data_hits[5]; + int prefetch_data_misses[5]; + int prefetch_metadata_hits[5]; + int prefetch_metadata_misses[5]; + int mru_hits[5]; + int mru_ghost_hits[5]; + int mfu_hits[5]; + int mfu_ghost_hits[5]; + int deleted[5]; + int mutex_miss[5]; + int evict_skip[5]; + int evict_not_enough[5]; + int evict_l2_cached[5]; + int evict_l2_eligible[5]; + int evict_l2_ineligible[5]; + int evict_l2_skip[5]; + int hash_elements[5]; + int hash_elements_max[5]; + int hash_collisions[5]; + int hash_chains[5]; + int hash_chain_max[5]; + int p[5]; + int c[5]; + int c_min[5]; + int c_max[5]; + int size[5]; + int hdr_size[5]; + int data_size[5]; + int metadata_size[5]; + int other_size[5]; + int anon_size[5]; + int anon_evictable_data[5]; + int anon_evictable_metadata[5]; + int mru_size[5]; + int mru_evictable_data[5]; + int mru_evictable_metadata[5]; + int mru_ghost_size[5]; + int mru_ghost_evictable_data[5]; + int mru_ghost_evictable_metadata[5]; + int mfu_size[5]; + int mfu_evictable_data[5]; + int mfu_evictable_metadata[5]; + int mfu_ghost_size[5]; + int mfu_ghost_evictable_data[5]; + int mfu_ghost_evictable_metadata[5]; + int l2_hits[5]; + int l2_misses[5]; + int l2_feeds[5]; + int l2_rw_clash[5]; + int l2_read_bytes[5]; + int l2_write_bytes[5]; + int l2_writes_sent[5]; + int l2_writes_done[5]; + int l2_writes_error[5]; + int l2_writes_lock_retry[5]; + int l2_evict_lock_retry[5]; + int l2_evict_reading[5]; + int l2_evict_l1cached[5]; + int l2_free_on_write[5]; + int l2_cdata_free_on_write[5]; + int l2_abort_lowmem[5]; + int l2_cksum_bad[5]; + int l2_io_error[5]; + int l2_size[5]; + int l2_asize[5]; + int l2_hdr_size[5]; + int l2_compress_successes[5]; + int l2_compress_zeros[5]; + int l2_compress_failures[5]; + int memory_throttle_count[5]; + int duplicate_buffers[5]; + int duplicate_buffers_size[5]; + int duplicate_reads[5]; + int memory_direct_count[5]; + int memory_indirect_count[5]; + int arc_no_grow[5]; + int arc_tempreserve[5]; + int arc_loaned_bytes[5]; + int arc_prune[5]; + int arc_meta_used[5]; + int arc_meta_limit[5]; + int arc_meta_max[5]; + int arc_meta_min[5]; + int arc_need_free[5]; + int arc_sys_free[5]; + } mibs; + + l2exist = -1; + + if(unlikely(sysctlbyname("kstat.zfs.misc.arcstats.l2_size", &l2_size, &uint64_t_size, NULL, 0))) + return 0; + + if(likely(l2_size)) + l2exist = 1; + else + l2exist = 0; + + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hits", mibs.hits, arcstats.hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.misses", mibs.misses, arcstats.misses); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_hits", mibs.demand_data_hits, arcstats.demand_data_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_misses", mibs.demand_data_misses, arcstats.demand_data_misses); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_hits", mibs.demand_metadata_hits, arcstats.demand_metadata_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_misses", mibs.demand_metadata_misses, arcstats.demand_metadata_misses); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_hits", mibs.prefetch_data_hits, arcstats.prefetch_data_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_misses", mibs.prefetch_data_misses, arcstats.prefetch_data_misses); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_hits", mibs.prefetch_metadata_hits, arcstats.prefetch_metadata_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_misses", mibs.prefetch_metadata_misses, arcstats.prefetch_metadata_misses); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_hits", mibs.mru_hits, arcstats.mru_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_hits", mibs.mru_ghost_hits, arcstats.mru_ghost_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_hits", mibs.mfu_hits, arcstats.mfu_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_hits", mibs.mfu_ghost_hits, arcstats.mfu_ghost_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.deleted", mibs.deleted, arcstats.deleted); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mutex_miss", mibs.mutex_miss, arcstats.mutex_miss); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_skip", mibs.evict_skip, arcstats.evict_skip); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_not_enough", mibs.evict_not_enough, arcstats.evict_not_enough); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_cached", mibs.evict_l2_cached, arcstats.evict_l2_cached); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_eligible", mibs.evict_l2_eligible, arcstats.evict_l2_eligible); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_ineligible", mibs.evict_l2_ineligible, arcstats.evict_l2_ineligible); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_skip", mibs.evict_l2_skip, arcstats.evict_l2_skip); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements", mibs.hash_elements, arcstats.hash_elements); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements_max", mibs.hash_elements_max, arcstats.hash_elements_max); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_collisions", mibs.hash_collisions, arcstats.hash_collisions); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chains", mibs.hash_chains, arcstats.hash_chains); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chain_max", mibs.hash_chain_max, arcstats.hash_chain_max); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.p", mibs.p, arcstats.p); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c", mibs.c, arcstats.c); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_min", mibs.c_min, arcstats.c_min); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_max", mibs.c_max, arcstats.c_max); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.size", mibs.size, arcstats.size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hdr_size", mibs.hdr_size, arcstats.hdr_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.data_size", mibs.data_size, arcstats.data_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.metadata_size", mibs.metadata_size, arcstats.metadata_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.other_size", mibs.other_size, arcstats.other_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_size", mibs.anon_size, arcstats.anon_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_data", mibs.anon_evictable_data, arcstats.anon_evictable_data); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_metadata", mibs.anon_evictable_metadata, arcstats.anon_evictable_metadata); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_size", mibs.mru_size, arcstats.mru_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_data", mibs.mru_evictable_data, arcstats.mru_evictable_data); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_metadata", mibs.mru_evictable_metadata, arcstats.mru_evictable_metadata); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_size", mibs.mru_ghost_size, arcstats.mru_ghost_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_data", mibs.mru_ghost_evictable_data, arcstats.mru_ghost_evictable_data); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_metadata", mibs.mru_ghost_evictable_metadata, arcstats.mru_ghost_evictable_metadata); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_size", mibs.mfu_size, arcstats.mfu_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_data", mibs.mfu_evictable_data, arcstats.mfu_evictable_data); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_metadata", mibs.mfu_evictable_metadata, arcstats.mfu_evictable_metadata); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_size", mibs.mfu_ghost_size, arcstats.mfu_ghost_size); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_data", mibs.mfu_ghost_evictable_data, arcstats.mfu_ghost_evictable_data); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_metadata", mibs.mfu_ghost_evictable_metadata, arcstats.mfu_ghost_evictable_metadata); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hits", mibs.l2_hits, arcstats.l2_hits); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_misses", mibs.l2_misses, arcstats.l2_misses); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_feeds", mibs.l2_feeds, arcstats.l2_feeds); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_rw_clash", mibs.l2_rw_clash, arcstats.l2_rw_clash); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_read_bytes", mibs.l2_read_bytes, arcstats.l2_read_bytes); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_write_bytes", mibs.l2_write_bytes, arcstats.l2_write_bytes); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_sent", mibs.l2_writes_sent, arcstats.l2_writes_sent); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_done", mibs.l2_writes_done, arcstats.l2_writes_done); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_error", mibs.l2_writes_error, arcstats.l2_writes_error); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_lock_retry", mibs.l2_writes_lock_retry, arcstats.l2_writes_lock_retry); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_lock_retry", mibs.l2_evict_lock_retry, arcstats.l2_evict_lock_retry); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_reading", mibs.l2_evict_reading, arcstats.l2_evict_reading); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_l1cached", mibs.l2_evict_l1cached, arcstats.l2_evict_l1cached); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_free_on_write", mibs.l2_free_on_write, arcstats.l2_free_on_write); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cdata_free_on_write", mibs.l2_cdata_free_on_write, arcstats.l2_cdata_free_on_write); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_abort_lowmem", mibs.l2_abort_lowmem, arcstats.l2_abort_lowmem); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cksum_bad", mibs.l2_cksum_bad, arcstats.l2_cksum_bad); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_io_error", mibs.l2_io_error, arcstats.l2_io_error); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_size", mibs.l2_size, arcstats.l2_size); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_asize", mibs.l2_asize, arcstats.l2_asize); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hdr_size", mibs.l2_hdr_size, arcstats.l2_hdr_size); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_successes", mibs.l2_compress_successes, arcstats.l2_compress_successes); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_zeros", mibs.l2_compress_zeros, arcstats.l2_compress_zeros); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_failures", mibs.l2_compress_failures, arcstats.l2_compress_failures); + GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_throttle_count", mibs.memory_throttle_count, arcstats.memory_throttle_count); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers", mibs.duplicate_buffers, arcstats.duplicate_buffers); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers_size", mibs.duplicate_buffers_size, arcstats.duplicate_buffers_size); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_reads", mibs.duplicate_reads, arcstats.duplicate_reads); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_direct_count", mibs.memory_direct_count, arcstats.memory_direct_count); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_indirect_count", mibs.memory_indirect_count, arcstats.memory_indirect_count); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_no_grow", mibs.arc_no_grow, arcstats.arc_no_grow); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_tempreserve", mibs.arc_tempreserve, arcstats.arc_tempreserve); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_loaned_bytes", mibs.arc_loaned_bytes, arcstats.arc_loaned_bytes); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_prune", mibs.arc_prune, arcstats.arc_prune); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_used", mibs.arc_meta_used, arcstats.arc_meta_used); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_limit", mibs.arc_meta_limit, arcstats.arc_meta_limit); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_max", mibs.arc_meta_max, arcstats.arc_meta_max); + // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_min", mibs.arc_meta_min, arcstats.arc_meta_min); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_need_free", mibs.arc_need_free, arcstats.arc_need_free); + // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_sys_free", mibs.arc_sys_free, arcstats.arc_sys_free); + + generate_charts_arcstats(update_every); + generate_charts_arc_summary(update_every); + + return 0; +}
\ No newline at end of file diff --git a/src/freebsd_sysctl.c b/src/freebsd_sysctl.c index 965c1cbbf..d2f0eaa82 100644 --- a/src/freebsd_sysctl.c +++ b/src/freebsd_sysctl.c @@ -1,8 +1,6 @@ #include "common.h" #include <sys/vmmeter.h> -#include <sys/devicestat.h> -#include <sys/mount.h> #include <vm/vm_param.h> #define _KERNEL @@ -12,8 +10,6 @@ #undef _KERNEL #include <net/netisr.h> -#include <net/if.h> -#include <ifaddrs.h> #include <netinet/ip.h> #include <netinet/ip_var.h> @@ -29,12 +25,6 @@ // -------------------------------------------------------------------------------------------------------------------- // common definitions and variables -#define KILO_FACTOR 1024 -#define MEGA_FACTOR 1048576 // 1024 * 1024 -#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024 - -#define MAX_INT_DIGITS 10 // maximum number of digits for int - int system_pagesize = PAGE_SIZE; int number_of_cpus = 1; @@ -311,8 +301,10 @@ int do_kern_cp_times(int update_every, usec_t dt) { static int mib[2] = {0, 0}; long cp_time[CPUSTATES]; static long *pcpu_cp_time = NULL; + static int old_number_of_cpus = 0; - pcpu_cp_time = reallocz(pcpu_cp_time, sizeof(cp_time) * number_of_cpus); + if(unlikely(number_of_cpus != old_number_of_cpus)) + pcpu_cp_time = reallocz(pcpu_cp_time, sizeof(cp_time) * number_of_cpus); if (unlikely(GETSYSCTL_WSIZE("kern.cp_times", mib, pcpu_cp_time, sizeof(cp_time) * number_of_cpus))) { error("DISABLED: cpu.cpuXX charts"); error("DISABLED: kern.cp_times module"); @@ -331,12 +323,10 @@ int do_kern_cp_times(int update_every, usec_t dt) { RRDDIM *rd_interrupt; RRDDIM *rd_idle; } *all_cpu_charts = NULL; - static int old_number_of_cpus = 0; if(unlikely(number_of_cpus > old_number_of_cpus)) { all_cpu_charts = reallocz(all_cpu_charts, sizeof(struct cpu_chart) * number_of_cpus); memset(&all_cpu_charts[old_number_of_cpus], 0, sizeof(struct cpu_chart) * (number_of_cpus - old_number_of_cpus)); - old_number_of_cpus = number_of_cpus; } for (i = 0; i < number_of_cpus; i++) { @@ -375,6 +365,8 @@ int do_kern_cp_times(int update_every, usec_t dt) { rrdset_done(all_cpu_charts[i].st); } } + + old_number_of_cpus = number_of_cpus; } return 0; @@ -386,7 +378,7 @@ int do_kern_cp_times(int update_every, usec_t dt) { int do_hw_intcnt(int update_every, usec_t dt) { (void)dt; static int mib_hw_intrcnt[2] = {0, 0}; - size_t intrcnt_size = sizeof(mib_hw_intrcnt); + size_t intrcnt_size = 0; unsigned long i; if (unlikely(GETSYSCTL_SIZE("hw.intrcnt", mib_hw_intrcnt, intrcnt_size))) { @@ -396,11 +388,13 @@ int do_hw_intcnt(int update_every, usec_t dt) { return 1; } else { unsigned long nintr = 0; + static unsigned long old_nintr = 0; static unsigned long *intrcnt = NULL; unsigned long long totalintr = 0; nintr = intrcnt_size / sizeof(u_long); - intrcnt = reallocz(intrcnt, nintr * sizeof(u_long)); + if (unlikely(nintr != old_nintr)) + intrcnt = reallocz(intrcnt, nintr * sizeof(u_long)); if (unlikely(GETSYSCTL_WSIZE("hw.intrcnt", mib_hw_intrcnt, intrcnt, nintr * sizeof(u_long)))) { error("DISABLED: system.intr chart"); error("DISABLED: system.interrupts chart"); @@ -443,7 +437,8 @@ int do_hw_intcnt(int update_every, usec_t dt) { static char *intrnames = NULL; size = nintr * (MAXCOMLEN + 1); - intrnames = reallocz(intrnames, size); + if (unlikely(nintr != old_nintr)) + intrnames = reallocz(intrnames, size); if (unlikely(GETSYSCTL_WSIZE("hw.intrnames", mib_hw_intrnames, intrnames, size))) { error("DISABLED: system.intr chart"); error("DISABLED: system.interrupts chart"); @@ -484,6 +479,8 @@ int do_hw_intcnt(int update_every, usec_t dt) { rrdset_done(st_interrupts); } } + + old_nintr = nintr; } return 0; @@ -931,8 +928,12 @@ int do_kern_ipc_sem(int update_every, usec_t dt) { return 1; } else { static struct semid_kernel *ipc_sem_data = NULL; + static int old_semmni = 0; - ipc_sem_data = reallocz(ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni); + if (unlikely(ipc_sem.semmni != old_semmni)) { + ipc_sem_data = reallocz(ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni); + old_semmni = ipc_sem.semmni; + } if (unlikely(GETSYSCTL_WSIZE("kern.ipc.sema", mib_sema, ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni))) { error("DISABLED: system.ipc_semaphores chart"); error("DISABLED: system.ipc_semaphore_arrays chart"); @@ -1019,8 +1020,12 @@ int do_kern_ipc_shm(int update_every, usec_t dt) { return 1; } else { static struct shmid_kernel *ipc_shm_data = NULL; + static u_long old_shmmni = 0; - ipc_shm_data = reallocz(ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni); + if (unlikely(ipc_shm.shmmni != old_shmmni)) { + ipc_shm_data = reallocz(ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni); + old_shmmni = ipc_shm.shmmni; + } if (unlikely( GETSYSCTL_WSIZE("kern.ipc.shmsegs", mib_shmsegs, ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni))) { error("DISABLED: system.ipc_shared_mem_segs chart"); @@ -1111,8 +1116,12 @@ int do_kern_ipc_msq(int update_every, usec_t dt) { return 1; } else { static struct msqid_kernel *ipc_msq_data = NULL; + static int old_msgmni = 0; - ipc_msq_data = reallocz(ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni); + if (unlikely(ipc_msq.msgmni != old_msgmni)) { + ipc_msq_data = reallocz(ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni); + old_msgmni = ipc_msq.msgmni; + } if (unlikely( GETSYSCTL_WSIZE("kern.ipc.msqids", mib_msqids, ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni))) { error("DISABLED: system.ipc_msq_queues chart"); @@ -1259,7 +1268,7 @@ int do_net_isr(int update_every, usec_t dt) { static int mib_workstream[3] = {0, 0, 0}, mib_work[3] = {0, 0, 0}; int common_error = 0; - size_t netisr_workstream_size = sizeof(mib_workstream), netisr_work_size = sizeof(mib_work); + size_t netisr_workstream_size = 0, netisr_work_size = 0; unsigned long num_netisr_workstreams = 0, num_netisr_works = 0; static struct sysctl_netisr_workstream *netisr_workstream = NULL; static struct sysctl_netisr_work *netisr_work = NULL; @@ -1276,14 +1285,25 @@ int do_net_isr(int update_every, usec_t dt) { } else if (unlikely(GETSYSCTL_SIZE("net.isr.work", mib_work, netisr_work_size))) { common_error = 1; } else { + static size_t old_netisr_workstream_size = 0; + num_netisr_workstreams = netisr_workstream_size / sizeof(struct sysctl_netisr_workstream); - netisr_workstream = reallocz(netisr_workstream, num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream)); + if (unlikely(netisr_workstream_size != old_netisr_workstream_size)) { + netisr_workstream = reallocz(netisr_workstream, + num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream)); + old_netisr_workstream_size = netisr_workstream_size; + } if (unlikely(GETSYSCTL_WSIZE("net.isr.workstream", mib_workstream, netisr_workstream, num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream)))){ common_error = 1; } else { + static size_t old_netisr_work_size = 0; + num_netisr_works = netisr_work_size / sizeof(struct sysctl_netisr_work); - netisr_work = reallocz(netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work)); + if (unlikely(netisr_work_size != old_netisr_work_size)) { + netisr_work = reallocz(netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work)); + old_netisr_work_size = netisr_work_size; + } if (unlikely(GETSYSCTL_WSIZE("net.isr.work", mib_work, netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work)))){ common_error = 1; @@ -1301,8 +1321,12 @@ int do_net_isr(int update_every, usec_t dt) { } else { unsigned long i, n; int j; + static int old_number_of_cpus = 0; - netisr_stats = reallocz(netisr_stats, (number_of_cpus + 1) * sizeof(struct netisr_stats)); + if (unlikely(number_of_cpus != old_number_of_cpus)) { + netisr_stats = reallocz(netisr_stats, (number_of_cpus + 1) * sizeof(struct netisr_stats)); + old_number_of_cpus = number_of_cpus; + } memset(netisr_stats, 0, (number_of_cpus + 1) * sizeof(struct netisr_stats)); for (i = 0; i < num_netisr_workstreams; i++) { for (n = 0; n < num_netisr_works; n++) { @@ -2749,1216 +2773,3 @@ int do_net_inet6_icmp6_stats(int update_every, usec_t dt) { return 0; } - -// -------------------------------------------------------------------------------------------------------------------- -// getmntinfo - -int do_getmntinfo(int update_every, usec_t dt) { - (void)dt; - -#define DELAULT_EXLUDED_PATHS "/proc/*" -// taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes -#define DEFAULT_EXCLUDED_FILESYSTEMS "autofs procfs subfs devfs none" -#define CONFIG_SECTION_GETMNTINFO "plugin:freebsd:getmntinfo" - - static int do_space = -1, do_inodes = -1; - - if (unlikely(do_space == -1)) { - do_space = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "space usage for all disks", CONFIG_BOOLEAN_AUTO); - do_inodes = config_get_boolean_ondemand(CONFIG_SECTION_GETMNTINFO, "inodes usage for all disks", CONFIG_BOOLEAN_AUTO); - } - - if (likely(do_space || do_inodes)) { - struct statfs *mntbuf; - int mntsize; - - // there is no mount info in sysctl MIBs - if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) { - error("FREEBSD: getmntinfo() failed"); - do_space = 0; - error("DISABLED: disk_space.* charts"); - do_inodes = 0; - error("DISABLED: disk_inodes.* charts"); - error("DISABLED: getmntinfo module"); - return 1; - } else { - // Data to be stored in DICTIONARY mount_points. - // This DICTIONARY is used to lookup the settings of the mount point on each iteration. - struct mount_point_metadata { - int do_space; - int do_inodes; - - size_t collected; // the number of times this has been collected - - // charts and dimensions - - RRDSET *st_space; - RRDDIM *rd_space_used; - RRDDIM *rd_space_avail; - RRDDIM *rd_space_reserved; - - RRDSET *st_inodes; - RRDDIM *rd_inodes_used; - RRDDIM *rd_inodes_avail; - }; - static DICTIONARY *mount_points = NULL; - static SIMPLE_PATTERN *excluded_mountpoints = NULL; - static SIMPLE_PATTERN *excluded_filesystems = NULL; - int i; - - if(unlikely(!mount_points)) { - - excluded_mountpoints = simple_pattern_create( - config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on paths", - DELAULT_EXLUDED_PATHS), - SIMPLE_PATTERN_EXACT - ); - - excluded_filesystems = simple_pattern_create( - config_get(CONFIG_SECTION_GETMNTINFO, "exclude space metrics on filesystems", - DEFAULT_EXCLUDED_FILESYSTEMS), - SIMPLE_PATTERN_EXACT - ); - - mount_points = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); - } - - for (i = 0; i < mntsize; i++) { - - char title[4096 + 1]; - int def_space, def_inodes, iter_space, iter_inodes; - - struct mount_point_metadata *m = dictionary_get(mount_points, mntbuf[i].f_mntonname); - if(unlikely(!m)) { - char var_name[4096 + 1]; - snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETMNTINFO, mntbuf[i].f_mntonname); - - def_space = do_space; - def_inodes = do_space; - - if(unlikely(simple_pattern_matches(excluded_mountpoints, mntbuf[i].f_mntonname))) { - def_space = CONFIG_BOOLEAN_NO; - def_inodes = CONFIG_BOOLEAN_NO; - } - - if(unlikely(simple_pattern_matches(excluded_filesystems, mntbuf[i].f_fstypename))) { - def_space = CONFIG_BOOLEAN_NO; - def_inodes = CONFIG_BOOLEAN_NO; - } - - iter_space = config_get_boolean_ondemand(var_name, "space usage", def_space); - iter_inodes = config_get_boolean_ondemand(var_name, "inodes usage", def_inodes); - - struct mount_point_metadata mp = { - .do_space = iter_space, - .do_inodes = iter_inodes, - - .collected = 0, - - .st_space = NULL, - .rd_space_avail = NULL, - .rd_space_used = NULL, - .rd_space_reserved = NULL, - - .st_inodes = NULL, - .rd_inodes_avail = NULL, - .rd_inodes_used = NULL, - }; - - m = dictionary_set(mount_points, mntbuf[i].f_mntonname, &mp, sizeof(struct mount_point_metadata)); - } - - if(unlikely(m->do_space == CONFIG_BOOLEAN_NO && m->do_inodes == CONFIG_BOOLEAN_NO)) - continue; - - if(unlikely(mntbuf[i].f_flags & MNT_RDONLY && !m->collected)) - continue; - - // -------------------------------------------------------------------------- - - int rendered = 0; - - if (m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_blocks > 2))) { - if (unlikely(!m->st_space)) { - snprintfz(title, 4096, "Disk Space Usage for %s [%s]", - mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); - m->st_space = rrdset_create_localhost("disk_space", - mntbuf[i].f_mntonname, - NULL, - mntbuf[i].f_mntonname, - "disk.space", - title, - "GB", - 2023, - update_every, - RRDSET_TYPE_STACKED - ); - - m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL, - mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); - m->rd_space_used = rrddim_add(m->st_space, "used", NULL, - mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); - m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root", - mntbuf[i].f_bsize, GIGA_FACTOR, RRD_ALGORITHM_ABSOLUTE); - } else - rrdset_next(m->st_space); - - rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number) mntbuf[i].f_bavail); - rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number) (mntbuf[i].f_blocks - - mntbuf[i].f_bfree)); - rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number) (mntbuf[i].f_bfree - - mntbuf[i].f_bavail)); - rrdset_done(m->st_space); - - rendered++; - } - - // -------------------------------------------------------------------------- - - if (m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO && (mntbuf[i].f_files > 1))) { - if (unlikely(!m->st_inodes)) { - snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", - mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); - m->st_inodes = rrdset_create_localhost("disk_inodes", - mntbuf[i].f_mntonname, - NULL, - mntbuf[i].f_mntonname, - "disk.inodes", - title, - "Inodes", - 2024, - update_every, - RRDSET_TYPE_STACKED - ); - - m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - } else - rrdset_next(m->st_inodes); - - rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number) mntbuf[i].f_ffree); - rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number) (mntbuf[i].f_files - - mntbuf[i].f_ffree)); - rrdset_done(m->st_inodes); - - rendered++; - } - - if(likely(rendered)) - m->collected++; - } - } - } else { - error("DISABLED: getmntinfo module"); - return 1; - } - - return 0; -} - -// -------------------------------------------------------------------------------------------------------------------- -// getifaddrs - -int do_getifaddrs(int update_every, usec_t dt) { - (void)dt; - -#define DELAULT_EXLUDED_INTERFACES "lo*" -#define CONFIG_SECTION_GETIFADDRS "plugin:freebsd:getifaddrs" - - static int do_bandwidth_ipv4 = -1, do_bandwidth_ipv6 = -1, do_bandwidth = -1, do_packets = -1, - do_errors = -1, do_drops = -1, do_events = -1; - - if (unlikely(do_bandwidth_ipv4 == -1)) { - do_bandwidth_ipv4 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv4 interfaces", - CONFIG_BOOLEAN_AUTO); - do_bandwidth_ipv6 = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "total bandwidth for ipv6 interfaces", - CONFIG_BOOLEAN_AUTO); - do_bandwidth = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "bandwidth for all interfaces", - CONFIG_BOOLEAN_AUTO); - do_packets = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "packets for all interfaces", - CONFIG_BOOLEAN_AUTO); - do_errors = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "errors for all interfaces", - CONFIG_BOOLEAN_AUTO); - do_drops = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "drops for all interfaces", - CONFIG_BOOLEAN_AUTO); - do_events = config_get_boolean_ondemand(CONFIG_SECTION_GETIFADDRS, "collisions for all interfaces", - CONFIG_BOOLEAN_AUTO); - } - - if (likely(do_bandwidth_ipv4 || do_bandwidth_ipv6 || do_bandwidth || do_packets || do_errors || - do_drops || do_events)) { - struct ifaddrs *ifap; - - if (unlikely(getifaddrs(&ifap))) { - error("FREEBSD: getifaddrs() failed"); - do_bandwidth_ipv4 = 0; - error("DISABLED: system.ipv4 chart"); - do_bandwidth_ipv6 = 0; - error("DISABLED: system.ipv6 chart"); - do_bandwidth = 0; - error("DISABLED: net.* charts"); - do_packets = 0; - error("DISABLED: net_packets.* charts"); - do_errors = 0; - error("DISABLED: net_errors.* charts"); - do_drops = 0; - error("DISABLED: net_drops.* charts"); - do_events = 0; - error("DISABLED: net_events.* charts"); - error("DISABLED: getifaddrs module"); - return 1; - } else { - #define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s) - struct ifaddrs *ifa; - struct iftot { - u_long ift_ibytes; - u_long ift_obytes; - } iftot = {0, 0}; - - // -------------------------------------------------------------------- - - if (likely(do_bandwidth_ipv4)) { - iftot.ift_ibytes = iftot.ift_obytes = 0; - for (ifa = ifap; ifa; ifa = ifa->ifa_next) { - if (ifa->ifa_addr->sa_family != AF_INET) - continue; - iftot.ift_ibytes += IFA_DATA(ibytes); - iftot.ift_obytes += IFA_DATA(obytes); - } - - static RRDSET *st = NULL; - static RRDDIM *rd_in = NULL, *rd_out = NULL; - - if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "ipv4", - NULL, - "network", - NULL, - "IPv4 Bandwidth", - "kilobits/s", - 500, - update_every, - RRDSET_TYPE_AREA - ); - - rd_in = rrddim_add(st, "InOctets", "received", 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st, "OutOctets", "sent", -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(st); - - rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes); - rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes); - rrdset_done(st); - } - - // -------------------------------------------------------------------- - - if (likely(do_bandwidth_ipv6)) { - iftot.ift_ibytes = iftot.ift_obytes = 0; - for (ifa = ifap; ifa; ifa = ifa->ifa_next) { - if (ifa->ifa_addr->sa_family != AF_INET6) - continue; - iftot.ift_ibytes += IFA_DATA(ibytes); - iftot.ift_obytes += IFA_DATA(obytes); - } - - static RRDSET *st = NULL; - static RRDDIM *rd_in = NULL, *rd_out = NULL; - - if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "ipv6", - NULL, - "network", - NULL, - "IPv6 Bandwidth", - "kilobits/s", - 500, - update_every, - RRDSET_TYPE_AREA - ); - - rd_in = rrddim_add(st, "received", NULL, 8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st, "sent", NULL, -8, KILO_FACTOR, RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(st); - - rrddim_set_by_pointer(st, rd_in, iftot.ift_ibytes); - rrddim_set_by_pointer(st, rd_out, iftot.ift_obytes); - rrdset_done(st); - } - - // -------------------------------------------------------------------- - - // Data to be stored in DICTIONARY interfaces. - // This DICTIONARY is used to lookup the settings of the interfaces on each iteration. - struct interfaces_metadata { - int do_bandwidth; - int do_packets; - int do_errors; - int do_drops; - int do_events; - - // charts and dimensions - - RRDSET *st_bandwidth; - RRDDIM *rd_bandwidth_in; - RRDDIM *rd_bandwidth_out; - - RRDSET *st_packets; - RRDDIM *rd_packets_in; - RRDDIM *rd_packets_out; - RRDDIM *rd_packets_m_in; - RRDDIM *rd_packets_m_out; - - RRDSET *st_errors; - RRDDIM *rd_errors_in; - RRDDIM *rd_errors_out; - - RRDSET *st_drops; - RRDDIM *rd_drops_in; - RRDDIM *rd_drops_out; - - RRDSET *st_events; - RRDDIM *rd_events_coll; - }; - static DICTIONARY *interfaces = NULL; - static SIMPLE_PATTERN *excluded_interfaces = NULL; - - if(unlikely(!interfaces)) { - - excluded_interfaces = simple_pattern_create( - config_get(CONFIG_SECTION_GETIFADDRS, "disable by default interfaces matching", - DELAULT_EXLUDED_INTERFACES) - , SIMPLE_PATTERN_EXACT - ); - - interfaces = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); - } - - for (ifa = ifap; ifa; ifa = ifa->ifa_next) { - if (ifa->ifa_addr->sa_family != AF_LINK) - continue; - - int def_bandwidth, def_packets, def_errors, def_drops, def_events, - iter_bandwidth, iter_packets, iter_errors, iter_drops, iter_events; - - struct interfaces_metadata *ifm = dictionary_get(interfaces, ifa->ifa_name); - if(unlikely(!ifm)) { - char var_name[4096 + 1]; - snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_GETIFADDRS, ifa->ifa_name); - - def_bandwidth = do_bandwidth; - def_packets = do_packets; - def_errors = do_errors; - def_drops = do_drops; - def_events = do_events; - - if(unlikely(simple_pattern_matches(excluded_interfaces, ifa->ifa_name))) { - def_bandwidth = CONFIG_BOOLEAN_NO; - def_packets = CONFIG_BOOLEAN_NO; - def_errors = CONFIG_BOOLEAN_NO; - def_drops = CONFIG_BOOLEAN_NO; - def_events = CONFIG_BOOLEAN_NO; - } - - iter_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", def_bandwidth); - iter_packets = config_get_boolean_ondemand(var_name, "packets", def_packets); - iter_errors = config_get_boolean_ondemand(var_name, "errors", def_errors); - iter_drops = config_get_boolean_ondemand(var_name, "drops", def_drops); - iter_events = config_get_boolean_ondemand(var_name, "events", def_events); - - struct interfaces_metadata ifmp = { - .do_bandwidth = iter_bandwidth, - .do_packets = iter_packets, - .do_errors = iter_errors, - .do_drops = iter_drops, - .do_events = iter_events, - - .st_bandwidth = NULL, - .rd_bandwidth_in = NULL, - .rd_bandwidth_out = NULL, - - .st_packets = NULL, - .rd_packets_in = NULL, - .rd_packets_out = NULL, - .rd_packets_m_in = NULL, - .rd_packets_m_out = NULL, - - .st_errors = NULL, - .rd_errors_in = NULL, - .rd_errors_out = NULL, - - .st_drops = NULL, - .rd_drops_in = NULL, - .rd_drops_out = NULL, - - .st_events = NULL, - .rd_events_coll = NULL, - }; - - ifm = dictionary_set(interfaces, ifa->ifa_name, &ifmp, sizeof(struct interfaces_metadata)); - } - - // -------------------------------------------------------------------- - - if (ifm->do_bandwidth == CONFIG_BOOLEAN_YES || (ifm->do_bandwidth == CONFIG_BOOLEAN_AUTO && - (IFA_DATA(ibytes) || IFA_DATA(obytes)))) { - if (unlikely(!ifm->st_bandwidth)) { - ifm->st_bandwidth = rrdset_create_localhost("net", - ifa->ifa_name, - NULL, - ifa->ifa_name, - "net.net", - "Bandwidth", - "kilobits/s", - 7000, - update_every, - RRDSET_TYPE_AREA - ); - - ifm->rd_bandwidth_in = rrddim_add(ifm->st_bandwidth, "received", NULL, 8, KILO_FACTOR, - RRD_ALGORITHM_INCREMENTAL); - ifm->rd_bandwidth_out = rrddim_add(ifm->st_bandwidth, "sent", NULL, -8, KILO_FACTOR, - RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(ifm->st_bandwidth); - - rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_in, IFA_DATA(ibytes)); - rrddim_set_by_pointer(ifm->st_bandwidth, ifm->rd_bandwidth_out, IFA_DATA(obytes)); - rrdset_done(ifm->st_bandwidth); - } - - // -------------------------------------------------------------------- - - if (ifm->do_packets == CONFIG_BOOLEAN_YES || (ifm->do_packets == CONFIG_BOOLEAN_AUTO && - (IFA_DATA(ipackets) || IFA_DATA(opackets) || IFA_DATA(imcasts) || IFA_DATA(omcasts)))) { - if (unlikely(!ifm->st_packets)) { - ifm->st_packets = rrdset_create_localhost("net_packets", - ifa->ifa_name, - NULL, - ifa->ifa_name, - "net.packets", - "Packets", - "packets/s", - 7001, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(ifm->st_packets, RRDSET_FLAG_DETAIL); - - ifm->rd_packets_in = rrddim_add(ifm->st_packets, "received", NULL, 1, 1, - RRD_ALGORITHM_INCREMENTAL); - ifm->rd_packets_out = rrddim_add(ifm->st_packets, "sent", NULL, -1, 1, - RRD_ALGORITHM_INCREMENTAL); - ifm->rd_packets_m_in = rrddim_add(ifm->st_packets, "multicast_received", NULL, 1, 1, - RRD_ALGORITHM_INCREMENTAL); - ifm->rd_packets_m_out = rrddim_add(ifm->st_packets, "multicast_sent", NULL, -1, 1, - RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(ifm->st_packets); - - rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_in, IFA_DATA(ipackets)); - rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_out, IFA_DATA(opackets)); - rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_in, IFA_DATA(imcasts)); - rrddim_set_by_pointer(ifm->st_packets, ifm->rd_packets_m_out, IFA_DATA(omcasts)); - rrdset_done(ifm->st_packets); - } - - // -------------------------------------------------------------------- - - if (ifm->do_errors == CONFIG_BOOLEAN_YES || (ifm->do_errors == CONFIG_BOOLEAN_AUTO && - (IFA_DATA(ierrors) || IFA_DATA(oerrors)))) { - if (unlikely(!ifm->st_errors)) { - ifm->st_errors = rrdset_create_localhost("net_errors", - ifa->ifa_name, - NULL, - ifa->ifa_name, - "net.errors", - "Interface Errors", - "errors/s", - 7002, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(ifm->st_errors, RRDSET_FLAG_DETAIL); - - ifm->rd_errors_in = rrddim_add(ifm->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - ifm->rd_errors_out = rrddim_add(ifm->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(ifm->st_errors); - - rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_in, IFA_DATA(ierrors)); - rrddim_set_by_pointer(ifm->st_errors, ifm->rd_errors_out, IFA_DATA(oerrors)); - rrdset_done(ifm->st_errors); - } - // -------------------------------------------------------------------- - - if (ifm->do_drops == CONFIG_BOOLEAN_YES || (ifm->do_drops == CONFIG_BOOLEAN_AUTO && - (IFA_DATA(iqdrops) || IFA_DATA(oqdrops)))) { - if (unlikely(!ifm->st_drops)) { - ifm->st_drops = rrdset_create_localhost("net_drops", - ifa->ifa_name, - NULL, - ifa->ifa_name, - "net.drops", - "Interface Drops", - "drops/s", - 7003, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(ifm->st_drops, RRDSET_FLAG_DETAIL); - - ifm->rd_drops_in = rrddim_add(ifm->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); -#if __FreeBSD__ >= 11 - ifm->rd_drops_out = rrddim_add(ifm->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); -#endif - } else - rrdset_next(ifm->st_drops); - - rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_in, IFA_DATA(iqdrops)); -#if __FreeBSD__ >= 11 - rrddim_set_by_pointer(ifm->st_drops, ifm->rd_drops_out, IFA_DATA(oqdrops)); -#endif - rrdset_done(ifm->st_drops); - } - - // -------------------------------------------------------------------- - - if (ifm->do_events == CONFIG_BOOLEAN_YES || (ifm->do_events == CONFIG_BOOLEAN_AUTO && - IFA_DATA(collisions))) { - if (unlikely(!ifm->st_events)) { - ifm->st_events = rrdset_create_localhost("net_events", - ifa->ifa_name, - NULL, - ifa->ifa_name, - "net.events", - "Network Interface Events", - "events/s", - 7006, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(ifm->st_events, RRDSET_FLAG_DETAIL); - - ifm->rd_events_coll = rrddim_add(ifm->st_events, "collisions", NULL, -1, 1, - RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(ifm->st_events); - - rrddim_set_by_pointer(ifm->st_events, ifm->rd_events_coll, IFA_DATA(collisions)); - rrdset_done(ifm->st_events); - } - } - - freeifaddrs(ifap); - } - } else { - error("DISABLED: getifaddrs module"); - return 1; - } - - return 0; -} - -// -------------------------------------------------------------------------------------------------------------------- -// kern.devstat - -int do_kern_devstat(int update_every, usec_t dt) { - -#define DELAULT_EXLUDED_DISKS "" -#define CONFIG_SECTION_KERN_DEVSTAT "plugin:freebsd:kern.devstat" -#define BINTIME_SCALE 5.42101086242752217003726400434970855712890625e-17 // this is 1000/2^64 - - static int enable_pass_devices = -1, do_system_io = -1, do_io = -1, do_ops = -1, do_qops = -1, do_util = -1, - do_iotime = -1, do_await = -1, do_avagsz = -1, do_svctm = -1; - - if (unlikely(enable_pass_devices == -1)) { - enable_pass_devices = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, - "performance metrics for pass devices", CONFIG_BOOLEAN_AUTO); - - do_system_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "total bandwidth for all disks", - CONFIG_BOOLEAN_YES); - - do_io = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "bandwidth for all disks", - CONFIG_BOOLEAN_AUTO); - do_ops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "operations for all disks", - CONFIG_BOOLEAN_AUTO); - do_qops = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "queued operations for all disks", - CONFIG_BOOLEAN_AUTO); - do_util = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "utilization percentage for all disks", - CONFIG_BOOLEAN_AUTO); - do_iotime = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "i/o time for all disks", - CONFIG_BOOLEAN_AUTO); - do_await = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o time for all disks", - CONFIG_BOOLEAN_AUTO); - do_avagsz = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average completed i/o bandwidth for all disks", - CONFIG_BOOLEAN_AUTO); - do_svctm = config_get_boolean_ondemand(CONFIG_SECTION_KERN_DEVSTAT, "average service time for all disks", - CONFIG_BOOLEAN_AUTO); - } - - if (likely(do_system_io || do_io || do_ops || do_qops || do_util || do_iotime || do_await || do_avagsz || do_svctm)) { - static int mib_numdevs[3] = {0, 0, 0}; - int numdevs; - int common_error = 0; - - if (unlikely(GETSYSCTL_SIMPLE("kern.devstat.numdevs", mib_numdevs, numdevs))) { - common_error = 1; - } else { - static int mib_devstat[3] = {0, 0, 0}; - static void *devstat_data = NULL; - - devstat_data = reallocz(devstat_data, sizeof(long) + sizeof(struct devstat) * numdevs); // there is generation number before devstat structures - if (unlikely(GETSYSCTL_WSIZE("kern.devstat.all", mib_devstat, devstat_data, - sizeof(long) + sizeof(struct devstat) * numdevs))) { - common_error = 1; - } else { - struct devstat *dstat; - int i; - collected_number total_disk_kbytes_read = 0; - collected_number total_disk_kbytes_write = 0; - - // Data to be stored in DICTIONARY disks. - // This DICTIONARY is used to lookup the settings of the disks on each iteration. - struct disks_metadata { - int do_io; - int do_ops; - int do_qops; - int do_util; - int do_iotime; - int do_await; - int do_avagsz; - int do_svctm; - - - // data for differential charts - - struct prev_dstat { - collected_number bytes_read; - collected_number bytes_write; - collected_number operations_read; - collected_number operations_write; - collected_number duration_read_ms; - collected_number duration_write_ms; - collected_number busy_time_ms; - } prev_dstat; - - // charts and dimensions - - RRDSET *st_io; - RRDDIM *rd_io_in; - RRDDIM *rd_io_out; - - RRDSET *st_ops; - RRDDIM *rd_ops_in; - RRDDIM *rd_ops_out; - - RRDSET *st_qops; - RRDDIM *rd_qops; - - RRDSET *st_util; - RRDDIM *rd_util; - - RRDSET *st_iotime; - RRDDIM *rd_iotime_in; - RRDDIM *rd_iotime_out; - - RRDSET *st_await; - RRDDIM *rd_await_in; - RRDDIM *rd_await_out; - - RRDSET *st_avagsz; - RRDDIM *rd_avagsz_in; - RRDDIM *rd_avagsz_out; - - RRDSET *st_svctm; - RRDDIM *rd_svctm; - - }; - static DICTIONARY *disks = NULL; - static SIMPLE_PATTERN *excluded_disks = NULL; - - if(unlikely(!disks)) { - - excluded_disks = simple_pattern_create( - config_get(CONFIG_SECTION_KERN_DEVSTAT, "disable by default disks matching", - DELAULT_EXLUDED_DISKS) - , SIMPLE_PATTERN_EXACT - ); - - disks = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); - } - - dstat = devstat_data + sizeof(long); // skip generation number - - for (i = 0; i < numdevs; i++) { - if (likely(do_system_io)) { - if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { - total_disk_kbytes_read += dstat[i].bytes[DEVSTAT_READ] / KILO_FACTOR; - total_disk_kbytes_write += dstat[i].bytes[DEVSTAT_WRITE] / KILO_FACTOR; - } - } - - if (unlikely(!enable_pass_devices)) - if ((dstat[i].device_type & DEVSTAT_TYPE_PASS) == DEVSTAT_TYPE_PASS) - continue; - - if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { - char disk[DEVSTAT_NAME_LEN + MAX_INT_DIGITS + 1]; - int def_io, def_ops, def_qops, def_util, def_iotime, def_await, def_avagsz, def_svctm, - iter_io, iter_ops, iter_qops, iter_util, iter_iotime, iter_await, iter_avagsz, iter_svctm; - struct cur_dstat { - collected_number duration_read_ms; - collected_number duration_write_ms; - collected_number busy_time_ms; - } cur_dstat; - - sprintf(disk, "%s%d", dstat[i].device_name, dstat[i].unit_number); - - struct disks_metadata *dm = dictionary_get(disks, disk); - if(unlikely(!dm)) { - char var_name[4096 + 1]; - snprintfz(var_name, 4096, "%s:%s", CONFIG_SECTION_KERN_DEVSTAT, disk); - - def_io = do_io; - def_ops = do_ops; - def_qops = do_qops; - def_util = do_util; - def_iotime = do_iotime; - def_await = do_await; - def_avagsz = do_avagsz; - def_svctm = do_svctm; - - if(unlikely(simple_pattern_matches(excluded_disks, disk))) { - def_io = CONFIG_BOOLEAN_NO; - def_ops = CONFIG_BOOLEAN_NO; - def_qops = CONFIG_BOOLEAN_NO; - def_util = CONFIG_BOOLEAN_NO; - def_iotime = CONFIG_BOOLEAN_NO; - def_await = CONFIG_BOOLEAN_NO; - def_avagsz = CONFIG_BOOLEAN_NO; - def_svctm = CONFIG_BOOLEAN_NO; - } - - iter_io = config_get_boolean_ondemand(var_name, "bandwidth", def_io); - iter_ops = config_get_boolean_ondemand(var_name, "operations", def_ops); - iter_qops = config_get_boolean_ondemand(var_name, "queued operations", def_qops); - iter_util = config_get_boolean_ondemand(var_name, "utilization percentage", def_util); - iter_iotime = config_get_boolean_ondemand(var_name, "i/o time", def_iotime); - iter_await = config_get_boolean_ondemand(var_name, "average completed i/o time", def_await); - iter_avagsz = config_get_boolean_ondemand(var_name, "average completed i/o bandwidth", - def_avagsz); - iter_svctm = config_get_boolean_ondemand(var_name, "average service time", def_svctm); - - struct disks_metadata dmp = { - .do_io = iter_io, - .do_ops = iter_ops, - .do_qops = iter_qops, - .do_util = iter_util, - .do_iotime = iter_iotime, - .do_await = iter_await, - .do_avagsz = iter_avagsz, - .do_svctm = iter_svctm, - - .st_io = NULL, - .rd_io_in = NULL, - .rd_io_out = NULL, - - .st_ops = NULL, - .rd_ops_in = NULL, - .rd_ops_out = NULL, - - .st_qops = NULL, - .rd_qops = NULL, - - .st_util = NULL, - .rd_util = NULL, - - .st_iotime = NULL, - .rd_iotime_in = NULL, - .rd_iotime_out = NULL, - - .st_await = NULL, - .rd_await_in = NULL, - .rd_await_out = NULL, - - .st_avagsz = NULL, - .rd_avagsz_in = NULL, - .rd_avagsz_out = NULL, - - .st_svctm = NULL, - .rd_svctm = NULL, - }; - - // initialise data for differential charts - - dmp.prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ]; - dmp.prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE]; - dmp.prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ]; - dmp.prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE]; - dmp.prev_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000 - + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; - dmp.prev_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000 - + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; - dmp.prev_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 - + dstat[i].busy_time.frac * BINTIME_SCALE; - - dm = dictionary_set(disks, disk, &dmp, sizeof(struct disks_metadata)); - } - - cur_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000 - + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; - cur_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000 - + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; - cur_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE; - - // -------------------------------------------------------------------- - - if(dm->do_io == CONFIG_BOOLEAN_YES || (dm->do_io == CONFIG_BOOLEAN_AUTO && - (dstat[i].bytes[DEVSTAT_READ] || dstat[i].bytes[DEVSTAT_WRITE]))) { - if (unlikely(!dm->st_io)) { - dm->st_io = rrdset_create_localhost("disk", - disk, - NULL, - disk, - "disk.io", - "Disk I/O Bandwidth", - "kilobytes/s", - 2000, - update_every, - RRDSET_TYPE_AREA - ); - - dm->rd_io_in = rrddim_add(dm->st_io, "reads", NULL, 1, KILO_FACTOR, - RRD_ALGORITHM_INCREMENTAL); - dm->rd_io_out = rrddim_add(dm->st_io, "writes", NULL, -1, KILO_FACTOR, - RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(dm->st_io); - - rrddim_set_by_pointer(dm->st_io, dm->rd_io_in, dstat[i].bytes[DEVSTAT_READ]); - rrddim_set_by_pointer(dm->st_io, dm->rd_io_out, dstat[i].bytes[DEVSTAT_WRITE]); - rrdset_done(dm->st_io); - } - - // -------------------------------------------------------------------- - - if(dm->do_ops == CONFIG_BOOLEAN_YES || (dm->do_ops == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { - if (unlikely(!dm->st_ops)) { - dm->st_ops = rrdset_create_localhost("disk_ops", - disk, - NULL, - disk, - "disk.ops", - "Disk Completed I/O Operations", - "operations/s", - 2001, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(dm->st_ops, RRDSET_FLAG_DETAIL); - - dm->rd_ops_in = rrddim_add(dm->st_ops, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - dm->rd_ops_out = rrddim_add(dm->st_ops, "writes", NULL, -1, 1, - RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(dm->st_ops); - - rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_in, dstat[i].operations[DEVSTAT_READ]); - rrddim_set_by_pointer(dm->st_ops, dm->rd_ops_out, dstat[i].operations[DEVSTAT_WRITE]); - rrdset_done(dm->st_ops); - } - - // -------------------------------------------------------------------- - - if(dm->do_qops == CONFIG_BOOLEAN_YES || (dm->do_qops == CONFIG_BOOLEAN_AUTO && - (dstat[i].start_count || dstat[i].end_count))) { - if (unlikely(!dm->st_qops)) { - dm->st_qops = rrdset_create_localhost("disk_qops", - disk, - NULL, - disk, - "disk.qops", - "Disk Current I/O Operations", - "operations", - 2002, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(dm->st_qops, RRDSET_FLAG_DETAIL); - - dm->rd_qops = rrddim_add(dm->st_qops, "operations", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - } else - rrdset_next(dm->st_qops); - - rrddim_set_by_pointer(dm->st_qops, dm->rd_qops, dstat[i].start_count - dstat[i].end_count); - rrdset_done(dm->st_qops); - } - - // -------------------------------------------------------------------- - - if(dm->do_util == CONFIG_BOOLEAN_YES || (dm->do_util == CONFIG_BOOLEAN_AUTO && - cur_dstat.busy_time_ms)) { - if (unlikely(!dm->st_util)) { - dm->st_util = rrdset_create_localhost("disk_util", - disk, - NULL, - disk, - "disk.util", - "Disk Utilization Time", - "% of time working", - 2004, - update_every, - RRDSET_TYPE_AREA - ); - - rrdset_flag_set(dm->st_util, RRDSET_FLAG_DETAIL); - - dm->rd_util = rrddim_add(dm->st_util, "utilization", NULL, 1, 10, - RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(dm->st_util); - - rrddim_set_by_pointer(dm->st_util, dm->rd_util, cur_dstat.busy_time_ms); - rrdset_done(dm->st_util); - } - - // -------------------------------------------------------------------- - - if(dm->do_iotime == CONFIG_BOOLEAN_YES || (dm->do_iotime == CONFIG_BOOLEAN_AUTO && - (cur_dstat.duration_read_ms || cur_dstat.duration_write_ms))) { - if (unlikely(!dm->st_iotime)) { - dm->st_iotime = rrdset_create_localhost("disk_iotime", - disk, - NULL, - disk, - "disk.iotime", - "Disk Total I/O Time", - "milliseconds/s", - 2022, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(dm->st_iotime, RRDSET_FLAG_DETAIL); - - dm->rd_iotime_in = rrddim_add(dm->st_iotime, "reads", NULL, 1, 1, - RRD_ALGORITHM_INCREMENTAL); - dm->rd_iotime_out = rrddim_add(dm->st_iotime, "writes", NULL, -1, 1, - RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(dm->st_iotime); - - rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_in, cur_dstat.duration_read_ms); - rrddim_set_by_pointer(dm->st_iotime, dm->rd_iotime_out, cur_dstat.duration_write_ms); - rrdset_done(dm->st_iotime); - } - - // -------------------------------------------------------------------- - // calculate differential charts - // only if this is not the first time we run - - if (likely(dt)) { - - // -------------------------------------------------------------------- - - if(dm->do_await == CONFIG_BOOLEAN_YES || (dm->do_await == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { - if (unlikely(!dm->st_await)) { - dm->st_await = rrdset_create_localhost("disk_await", - disk, - NULL, - disk, - "disk.await", - "Average Completed I/O Operation Time", - "ms per operation", - 2005, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(dm->st_await, RRDSET_FLAG_DETAIL); - - dm->rd_await_in = rrddim_add(dm->st_await, "reads", NULL, 1, 1, - RRD_ALGORITHM_ABSOLUTE); - dm->rd_await_out = rrddim_add(dm->st_await, "writes", NULL, -1, 1, - RRD_ALGORITHM_ABSOLUTE); - } else - rrdset_next(dm->st_await); - - rrddim_set_by_pointer(dm->st_await, dm->rd_await_in, - (dstat[i].operations[DEVSTAT_READ] - - dm->prev_dstat.operations_read) ? - (cur_dstat.duration_read_ms - dm->prev_dstat.duration_read_ms) / - (dstat[i].operations[DEVSTAT_READ] - - dm->prev_dstat.operations_read) : - 0); - rrddim_set_by_pointer(dm->st_await, dm->rd_await_out, - (dstat[i].operations[DEVSTAT_WRITE] - - dm->prev_dstat.operations_write) ? - (cur_dstat.duration_write_ms - dm->prev_dstat.duration_write_ms) / - (dstat[i].operations[DEVSTAT_WRITE] - - dm->prev_dstat.operations_write) : - 0); - rrdset_done(dm->st_await); - } - - // -------------------------------------------------------------------- - - if(dm->do_avagsz == CONFIG_BOOLEAN_YES || (dm->do_avagsz == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { - if (unlikely(!dm->st_avagsz)) { - dm->st_avagsz = rrdset_create_localhost("disk_avgsz", - disk, - NULL, - disk, - "disk.avgsz", - "Average Completed I/O Operation Bandwidth", - "kilobytes per operation", - 2006, - update_every, - RRDSET_TYPE_AREA - ); - - rrdset_flag_set(dm->st_avagsz, RRDSET_FLAG_DETAIL); - - dm->rd_avagsz_in = rrddim_add(dm->st_avagsz, "reads", NULL, 1, KILO_FACTOR, - RRD_ALGORITHM_ABSOLUTE); - dm->rd_avagsz_out = rrddim_add(dm->st_avagsz, "writes", NULL, -1, KILO_FACTOR, - RRD_ALGORITHM_ABSOLUTE); - } else - rrdset_next(dm->st_avagsz); - - rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_in, - (dstat[i].operations[DEVSTAT_READ] - - dm->prev_dstat.operations_read) ? - (dstat[i].bytes[DEVSTAT_READ] - dm->prev_dstat.bytes_read) / - (dstat[i].operations[DEVSTAT_READ] - - dm->prev_dstat.operations_read) : - 0); - rrddim_set_by_pointer(dm->st_avagsz, dm->rd_avagsz_out, - (dstat[i].operations[DEVSTAT_WRITE] - - dm->prev_dstat.operations_write) ? - (dstat[i].bytes[DEVSTAT_WRITE] - dm->prev_dstat.bytes_write) / - (dstat[i].operations[DEVSTAT_WRITE] - - dm->prev_dstat.operations_write) : - 0); - rrdset_done(dm->st_avagsz); - } - - // -------------------------------------------------------------------- - - if(dm->do_svctm == CONFIG_BOOLEAN_YES || (dm->do_svctm == CONFIG_BOOLEAN_AUTO && - (dstat[i].operations[DEVSTAT_READ] || dstat[i].operations[DEVSTAT_WRITE]))) { - if (unlikely(!dm->st_svctm)) { - dm->st_svctm = rrdset_create_localhost("disk_svctm", - disk, - NULL, - disk, - "disk.svctm", - "Average Service Time", - "ms per operation", - 2007, - update_every, - RRDSET_TYPE_LINE - ); - - rrdset_flag_set(dm->st_svctm, RRDSET_FLAG_DETAIL); - - dm->rd_svctm = rrddim_add(dm->st_svctm, "svctm", NULL, 1, 1, - RRD_ALGORITHM_ABSOLUTE); - } else - rrdset_next(dm->st_svctm); - - rrddim_set_by_pointer(dm->st_svctm, dm->rd_svctm, - ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) + - (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) ? - (cur_dstat.busy_time_ms - dm->prev_dstat.busy_time_ms) / - ((dstat[i].operations[DEVSTAT_READ] - dm->prev_dstat.operations_read) + - (dstat[i].operations[DEVSTAT_WRITE] - dm->prev_dstat.operations_write)) : - 0); - rrdset_done(dm->st_svctm); - } - - // -------------------------------------------------------------------- - - dm->prev_dstat.bytes_read = dstat[i].bytes[DEVSTAT_READ]; - dm->prev_dstat.bytes_write = dstat[i].bytes[DEVSTAT_WRITE]; - dm->prev_dstat.operations_read = dstat[i].operations[DEVSTAT_READ]; - dm->prev_dstat.operations_write = dstat[i].operations[DEVSTAT_WRITE]; - dm->prev_dstat.duration_read_ms = cur_dstat.duration_read_ms; - dm->prev_dstat.duration_write_ms = cur_dstat.duration_write_ms; - dm->prev_dstat.busy_time_ms = cur_dstat.busy_time_ms; - } - } - } - - // -------------------------------------------------------------------- - - if (likely(do_system_io)) { - static RRDSET *st = NULL; - static RRDDIM *rd_in = NULL, *rd_out = NULL; - - if (unlikely(!st)) { - st = rrdset_create_localhost("system", - "io", - NULL, - "disk", - NULL, - "Disk I/O", - "kilobytes/s", - 150, - update_every, - RRDSET_TYPE_AREA - ); - - rd_in = rrddim_add(st, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_out = rrddim_add(st, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - } else - rrdset_next(st); - - rrddim_set_by_pointer(st, rd_in, total_disk_kbytes_read); - rrddim_set_by_pointer(st, rd_out, total_disk_kbytes_write); - rrdset_done(st); - } - } - } - if (unlikely(common_error)) { - do_system_io = 0; - error("DISABLED: system.io chart"); - do_io = 0; - error("DISABLED: disk.* charts"); - do_ops = 0; - error("DISABLED: disk_ops.* charts"); - do_qops = 0; - error("DISABLED: disk_qops.* charts"); - do_util = 0; - error("DISABLED: disk_util.* charts"); - do_iotime = 0; - error("DISABLED: disk_iotime.* charts"); - do_await = 0; - error("DISABLED: disk_await.* charts"); - do_avagsz = 0; - error("DISABLED: disk_avgsz.* charts"); - do_svctm = 0; - error("DISABLED: disk_svctm.* charts"); - error("DISABLED: kern.devstat module"); - return 1; - } - } else { - error("DISABLED: kern.devstat module"); - return 1; - } - - return 0; -} diff --git a/src/freeipmi_plugin.c b/src/freeipmi_plugin.c index 4459de7ca..146268a53 100644 --- a/src/freeipmi_plugin.c +++ b/src/freeipmi_plugin.c @@ -242,7 +242,7 @@ _get_sensor_type_string (int sensor_type) static int debug = 0; -static int netdata_update_every = 5; +static int netdata_update_every = 5; // this is the minimum update frequency static int netdata_priority = 90000; static int netdata_do_sel = 1; @@ -1403,7 +1403,7 @@ int ipmi_detect_speed_secs(struct ipmi_monitoring_ipmi_config *ipmi_config) { // we find the average in microseconds // and we round-up to the closest second - return (( total * 2 / checks / 1000000 ) + 1); + return (int)(( total * 2 / checks / 1000000 ) + 1); } int main (int argc, char **argv) { @@ -1426,15 +1426,14 @@ int main (int argc, char **argv) { int i, freq = 0; for(i = 1; i < argc ; i++) { - if(!freq) { + if(isdigit(*argv[i]) && !freq) { int n = atoi(argv[i]); - if(n > 0) { + if(n > 0 && freq < 86400) { freq = n; continue; } } - - if(strcmp("version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) { + else if(strcmp("version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0 || strcmp("-V", argv[i]) == 0) { printf("freeipmi.plugin %s\n", VERSION); exit(0); } @@ -1568,7 +1567,7 @@ int main (int argc, char **argv) { freq = ipmi_detect_speed_secs(&ipmi_config); if(debug) fprintf(stderr, "freeipmi.plugin: IPMI minimum update frequency was calculated to %d seconds.\n", freq); - if(netdata_update_every < freq) { + if(freq > netdata_update_every) { info("enforcing minimum data collection frequency, calculated to %d seconds.", freq); netdata_update_every = freq; } diff --git a/src/health.c b/src/health.c index 46b27db6f..cc470f81f 100644 --- a/src/health.c +++ b/src/health.c @@ -148,7 +148,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s' '%s' '%s'", exec, recipient, - host->hostname, + host->registry_hostname, ae->unique_id, ae->alarm_id, ae->alarm_event_id, @@ -356,9 +356,16 @@ void *health_main(void *ptr) { // detect if boottime and realtime have twice the difference // in which case we assume the system was just waken from hibernation - if(unlikely(now - last_now > 2 * (now_boottime - last_now_boottime))) + if(unlikely(now - last_now > 2 * (now_boottime - last_now_boottime))) { apply_hibernation_delay = 1; + info("Postponing alarm checks for %ld seconds, due to boottime discrepancy (realtime dt: %ld, boottime dt: %ld)." + , hibernation_delay + , (long)(now - last_now) + , (long)(now_boottime - last_now_boottime) + ); + } + last_now = now; last_now_boottime = now_boottime; @@ -374,11 +381,9 @@ void *health_main(void *ptr) { if(unlikely(apply_hibernation_delay)) { - info("Postponing alarm checks for %ld seconds, on host '%s', due to boottime discrepancy (realtime dt: %ld, boottime dt: %ld)." + info("Postponing alarm checks for %ld seconds, on host '%s'." , hibernation_delay , host->hostname - , (long)(now - last_now) - , (long)(now_boottime - last_now_boottime) ); host->health_delay_up_to = now + hibernation_delay; diff --git a/src/health_config.c b/src/health_config.c index ad954cbe1..b4655dc78 100644 --- a/src/health_config.c +++ b/src/health_config.c @@ -32,8 +32,8 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) { return 0; } - if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) { - error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name); + if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->calculation && !rc->warning && !rc->critical) { + error("Health configuration for alarm '%s.%s' is useless (no db lookup, no calculation, no warning and no critical expressions)", rc->chart?rc->chart:"NOCHART", rc->name); return 0; } @@ -136,7 +136,7 @@ static inline int health_parse_duration(char *string, int *result) { } char *e = NULL; - calculated_number n = strtold(string, &e); + calculated_number n = str2ld(string, &e); if(e && *e) { switch (*e) { case 'Y': @@ -241,10 +241,10 @@ static inline int health_parse_delay( if(!given_max) { if((*delay_max_duration) < (*delay_up_duration) * (*delay_multiplier)) - *delay_max_duration = (*delay_up_duration) * (*delay_multiplier); + *delay_max_duration = (int)((*delay_up_duration) * (*delay_multiplier)); if((*delay_max_duration) < (*delay_down_duration) * (*delay_multiplier)) - *delay_max_duration = (*delay_down_duration) * (*delay_multiplier); + *delay_max_duration = (int)((*delay_down_duration) * (*delay_multiplier)); } return 1; @@ -381,37 +381,6 @@ static inline int health_parse_db_lookup( return 1; } -static inline char *trim_all_spaces(char *buffer) { - char *d = buffer, *s = buffer; - - // skip spaces - while(isspace(*s)) s++; - - while(*s) { - // copy the non-space part - while(*s && !isspace(*s)) *d++ = *s++; - - // add a space if we have to - if(*s && isspace(*s)) { - *d++ = ' '; - s++; - } - - // skip spaces - while(isspace(*s)) s++; - } - - *d = '\0'; - - if(d > buffer) { - d--; - if(isspace(*d)) *d = '\0'; - } - - if(!buffer[0]) return NULL; - return buffer; -} - static inline char *health_source_file(size_t line, const char *path, const char *filename) { char buffer[FILENAME_MAX + 1]; snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename); @@ -485,7 +454,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { int stop_appending = !s; line++; s = trim(buffer); - if(!s) continue; + if(!s || *s == '#') continue; append = strlen(s); if(!stop_appending && s[append - 1] == '\\') { @@ -509,8 +478,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { s++; char *value = s; - key = trim_all_spaces(key); - value = trim_all_spaces(value); + key = trim_all(key); + value = trim_all(value); if(!key) { error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename); @@ -593,7 +562,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { } else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { char *e; - rc->green = strtold(value, &e); + rc->green = str2ld(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rc->name, key, e); @@ -601,7 +570,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { } else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) { char *e; - rc->red = strtold(value, &e); + rc->red = str2ld(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rc->name, key, e); @@ -717,7 +686,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { } else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { char *e; - rt->green = strtold(value, &e); + rt->green = str2ld(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rt->name, key, e); @@ -725,7 +694,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) { } else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) { char *e; - rt->red = strtold(value, &e); + rt->red = str2ld(value, &e); if(e && *e) { error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rt->name, key, e); diff --git a/src/inlined.h b/src/inlined.h index 0dc11c950..f1812ba1c 100644 --- a/src/inlined.h +++ b/src/inlined.h @@ -123,6 +123,103 @@ static inline unsigned long long str2ull(const char *s) { return n; } +static inline long long str2ll(const char *s, char **endptr) { + int negative = 0; + + if(unlikely(*s == '-')) { + s++; + negative = 1; + } + else if(unlikely(*s == '+')) + s++; + + long long n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + + if(unlikely(endptr)) + *endptr = (char *)s; + + if(unlikely(negative)) + return -n; + else + return n; +} + +static inline long double str2ld(const char *s, char **endptr) { + int negative = 0; + const char *start = s; + unsigned long long integer_part = 0; + unsigned long decimal_part = 0; + size_t decimal_digits = 0; + + switch(*s) { + case '-': + s++; + negative = 1; + break; + + case '+': + s++; + break; + + case 'n': + if(s[1] == 'a' && s[2] == 'n') { + if(endptr) *endptr = (char *)&s[3]; + return NAN; + } + break; + + case 'i': + if(s[1] == 'n' && s[2] == 'f') { + if(endptr) *endptr = (char *)&s[3]; + return INFINITY; + } + break; + + default: + break; + } + + while (*s >= '0' && *s <= '9') { + integer_part = (integer_part * 10) + (*s - '0'); + s++; + } + + if(unlikely(*s == '.')) { + decimal_part = 0; + s++; + + while (*s >= '0' && *s <= '9') { + decimal_part = (decimal_part * 10) + (*s - '0'); + s++; + decimal_digits++; + } + } + + if(unlikely(*s == 'e' || *s == 'E')) + return strtold(start, endptr); + + if(unlikely(endptr)) + *endptr = (char *)s; + + if(unlikely(negative)) { + if(unlikely(decimal_digits)) + return -((long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits)); + else + return -((long double)integer_part); + } + else { + if(unlikely(decimal_digits)) + return (long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits); + else + return (long double)integer_part; + } +} + #ifdef NETDATA_STRCMP_OVERRIDE #ifdef strcmp #undef strcmp @@ -23,6 +23,37 @@ void syslog_init(void) { } } +#define LOG_DATE_LENGTH 26 + +static inline void log_date(char *buffer, size_t len) { + if(unlikely(!buffer || !len)) + return; + + time_t t; + struct tm *tmp, tmbuf; + + t = now_realtime_sec(); + tmp = localtime_r(&t, &tmbuf); + + if (tmp == NULL) { + buffer[0] = '\0'; + return; + } + + if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0)) + buffer[0] = '\0'; + + buffer[len - 1] = '\0'; +} + +static netdata_mutex_t log_mutex = NETDATA_MUTEX_INITIALIZER; +static inline void log_lock() { + netdata_mutex_lock(&log_mutex); +} +static inline void log_unlock() { + netdata_mutex_unlock(&log_mutex); +} + int open_log_file(int fd, FILE **fp, const char *filename, int *enabled_syslog) { int f; @@ -136,8 +167,10 @@ int error_log_limit(int reset) { if(reset) { if(prevented) { - log_date(stderr); - fprintf(stderr, "%s: Resetting logging for process '%s' (prevented %lu logs in the last %ld seconds).\n" + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stderr, "%s: %s Resetting logging for process '%s' (prevented %lu logs in the last %ld seconds).\n" + , date , program_name , program_name , prevented @@ -155,8 +188,10 @@ int error_log_limit(int reset) { if(now - start > error_log_throttle_period) { if(prevented) { - log_date(stderr); - fprintf(stderr, "%s: Resuming logging from process '%s' (prevented %lu logs in the last %ld seconds).\n" + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stderr, "%s: %s Resuming logging from process '%s' (prevented %lu logs in the last %ld seconds).\n" + , date , program_name , program_name , prevented @@ -175,8 +210,10 @@ int error_log_limit(int reset) { if(counter > error_log_errors_per_period) { if(!prevented) { - log_date(stderr); - fprintf(stderr, "%s: Too many logs (%lu logs in %ld seconds, threshold is set to %lu logs in %ld seconds). Preventing more logs from process '%s' for %ld seconds.\n" + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stderr, "%s: %s Too many logs (%lu logs in %ld seconds, threshold is set to %lu logs in %ld seconds). Preventing more logs from process '%s' for %ld seconds.\n" + , date , program_name , counter , now - start @@ -200,37 +237,16 @@ int error_log_limit(int reset) { } // ---------------------------------------------------------------------------- -// print the date - -// FIXME -// this should print the date in a buffer the way it -// is now, logs from multiple threads may be multiplexed - -void log_date(FILE *out) -{ - char outstr[26]; - time_t t; - struct tm *tmp, tmbuf; - - t = now_realtime_sec(); - tmp = localtime_r(&t, &tmbuf); - - if (tmp == NULL) return; - if (unlikely(strftime(outstr, sizeof(outstr), "%Y-%m-%d %H:%M:%S", tmp) == 0)) return; - - fprintf(out, "%s: ", outstr); -} - -// ---------------------------------------------------------------------------- // debug log -void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) -{ +void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { va_list args; - log_date(stdout); + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + va_start( args, fmt ); - printf("%s: DEBUG (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function); + printf("%s: %s DEBUG (%04lu@%-10.10s:%-15.15s): ", date, program_name, line, file, function); vprintf(fmt, args); va_end( args ); putchar('\n'); @@ -254,21 +270,26 @@ void info_int( const char *file, const char *function, const unsigned long line, // prevent logging too much if(error_log_limit(0)) return; - log_date(stderr); + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_INFO, fmt, args ); + va_end( args ); + } + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + + log_lock(); va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "%s: INFO : (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function); - else fprintf(stderr, "%s: INFO : ", program_name); + if(debug_flags) fprintf(stderr, "%s: %s INFO : (%04lu@%-10.10s:%-15.15s): ", date, program_name, line, file, function); + else fprintf(stderr, "%s: %s INFO : ", date, program_name); vfprintf( stderr, fmt, args ); va_end( args ); fputc('\n', stderr); - if(error_log_syslog) { - va_start( args, fmt ); - vsyslog(LOG_INFO, fmt, args ); - va_end( args ); - } + log_unlock(); } // ---------------------------------------------------------------------------- @@ -296,56 +317,67 @@ static const char *strerror_result_string(const char *a, const char *b) { (void) #error "cannot detect the format of function strerror_r()" #endif -void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) -{ +void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { + // save a copy of errno - just in case this function generates a new error + int __errno = errno; + va_list args; // prevent logging too much if(error_log_limit(0)) return; - log_date(stderr); + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_ERR, fmt, args ); + va_end( args ); + } + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + + log_lock(); va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "%s: %s: (%04lu@%-10.10s:%-15.15s): ", program_name, prefix, line, file, function); - else fprintf(stderr, "%s: %s: ", program_name, prefix); + if(debug_flags) fprintf(stderr, "%s: %s %s: (%04lu@%-10.10s:%-15.15s): ", date, program_name, prefix, line, file, function); + else fprintf(stderr, "%s: %s %s: ", date, program_name, prefix); vfprintf( stderr, fmt, args ); va_end( args ); - if(errno) { + if(__errno) { char buf[1024]; - fprintf(stderr, " (errno %d, %s)\n", errno, strerror_result(strerror_r(errno, buf, 1023), buf)); + fprintf(stderr, " (errno %d, %s)\n", __errno, strerror_result(strerror_r(__errno, buf, 1023), buf)); errno = 0; } else fputc('\n', stderr); + log_unlock(); +} + +void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { + va_list args; + if(error_log_syslog) { va_start( args, fmt ); - vsyslog(LOG_ERR, fmt, args ); + vsyslog(LOG_CRIT, fmt, args ); va_end( args ); } -} -void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) -{ - va_list args; + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); - log_date(stderr); + log_lock(); va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "%s: FATAL: (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function); - else fprintf(stderr, "%s: FATAL: ", program_name); + if(debug_flags) fprintf(stderr, "%s: %s FATAL: (%04lu@%-10.10s:%-15.15s): ", date, program_name, line, file, function); + else fprintf(stderr, "%s: %s FATAL: ", date, program_name); vfprintf( stderr, fmt, args ); va_end( args ); perror(" # "); fputc('\n', stderr); - if(error_log_syslog) { - va_start( args, fmt ); - vsyslog(LOG_CRIT, fmt, args ); - va_end( args ); - } + log_unlock(); netdata_cleanup_and_exit(1); } @@ -353,23 +385,29 @@ void fatal_int( const char *file, const char *function, const unsigned long line // ---------------------------------------------------------------------------- // access log -void log_access( const char *fmt, ... ) -{ +void log_access( const char *fmt, ... ) { va_list args; + if(access_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_INFO, fmt, args ); + va_end( args ); + } + if(stdaccess) { - log_date(stdaccess); + static netdata_mutex_t access_mutex = NETDATA_MUTEX_INITIALIZER; + + netdata_mutex_lock(&access_mutex); + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stdaccess, "%s: ", date); va_start( args, fmt ); vfprintf( stdaccess, fmt, args ); va_end( args ); fputc('\n', stdaccess); - } - if(access_log_syslog) { - va_start( args, fmt ); - vsyslog(LOG_INFO, fmt, args ); - va_end( args ); + netdata_mutex_unlock(&access_mutex); } } - @@ -28,6 +28,10 @@ #define D_CONNECT_TO 0x0000000001000000 #define D_RRDHOST 0x0000000002000000 #define D_LOCKS 0x0000000004000000 +#define D_BACKEND 0x0000000008000000 +#define D_STATSD 0x0000000010000000 +#define D_POLLFD 0x0000000020000000 +#define D_STREAM 0x0000000040000000 #define D_SYSTEM 0x8000000000000000 //#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS) @@ -56,16 +60,22 @@ extern int error_log_limit(int reset); extern void open_all_log_files(); extern void reopen_all_log_files(); +static inline void debug_dummy(void) {} + #define error_log_limit_reset() do { error_log_throttle_period = error_log_throttle_period_backup; error_log_limit(1); } while(0) #define error_log_limit_unlimited() do { error_log_throttle_period = 0; } while(0) +#ifdef NETDATA_INTERNAL_CHECKS #define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0) +#else +#define debug(type, args...) debug_dummy() +#endif + #define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args) #define infoerr(args...) error_int("INFO", __FILE__, __FUNCTION__, __LINE__, ##args) #define error(args...) error_int("ERROR", __FILE__, __FUNCTION__, __LINE__, ##args) #define fatal(args...) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args) -extern void log_date(FILE *out); extern void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5); extern void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5); extern void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(5, 6); diff --git a/src/main.c b/src/main.c index a72585e28..bf5d787ad 100644 --- a/src/main.c +++ b/src/main.c @@ -9,8 +9,8 @@ void netdata_cleanup_and_exit(int ret) { debug(D_EXIT, "Called: netdata_cleanup_and_exit()"); - // save the database - rrdhost_save_all(); + // cleanup the database + rrdhost_cleanup_all(); // unlink the pid if(pidfile[0]) { @@ -56,6 +56,7 @@ struct netdata_static_thread static_threads[] = { {"web", NULL, NULL, 1, NULL, NULL, socket_listen_main_multi_threaded}, {"web-single-threaded", NULL, NULL, 0, NULL, NULL, socket_listen_main_single_threaded}, {"push-metrics", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread}, + {"statsd", NULL, NULL, 1, NULL, NULL, statsd_main}, {NULL, NULL, NULL, 0, NULL, NULL, NULL} }; @@ -66,14 +67,16 @@ void web_server_threading_selection(void) { int single_threaded = (web_server_mode == WEB_SERVER_MODE_SINGLE_THREADED); int i; - for(i = 0; static_threads[i].name ; i++) { - if(static_threads[i].start_routine == socket_listen_main_multi_threaded) + for (i = 0; static_threads[i].name; i++) { + if (static_threads[i].start_routine == socket_listen_main_multi_threaded) static_threads[i].enabled = multi_threaded; - if(static_threads[i].start_routine == socket_listen_main_single_threaded) + if (static_threads[i].start_routine == socket_listen_main_single_threaded) static_threads[i].enabled = single_threaded; } +} +void web_server_config_options(void) { web_client_timeout = (int) config_get_number(CONFIG_SECTION_WEB, "disconnect idle clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS); respect_web_browser_do_not_track_policy = config_get_boolean(CONFIG_SECTION_WEB, "respect do not track policy", respect_web_browser_do_not_track_policy); @@ -299,6 +302,8 @@ void help(int exitcode) { " -W stacksize=N Set the stacksize (in bytes).\n\n" " -W debug_flags=N Set runtime tracing to debug.log.\n\n" " -W unittest Run internal unittests and exit.\n\n" + " -W set section option value\n" + " set netdata.conf option from the command line.\n\n" " -W simple-pattern pattern string\n" " Check if string matches pattern and exit.\n\n" ); @@ -405,6 +410,9 @@ static void backwards_compatible_config() { config_move(CONFIG_SECTION_GLOBAL, "web files group", CONFIG_SECTION_WEB, "web files group"); + + config_move(CONFIG_SECTION_BACKEND, "opentsdb host tags", + CONFIG_SECTION_BACKEND, "host tags"); } static void get_netdata_configured_variables() { @@ -420,8 +428,6 @@ static void get_netdata_configured_variables() { netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", buf); debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname); - netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", CONFIG_DIR); - // ------------------------------------------------------------------------ // get default database size @@ -502,7 +508,9 @@ void set_global_environment() { // avoid flood calls to stat(/etc/localtime) // http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux - setenv("TZ", ":/etc/localtime", 0); + const char *tz = getenv("TZ"); + if(!tz || !*tz) + setenv("TZ", config_get(CONFIG_SECTION_GLOBAL, "TZ environment variable", ":/etc/localtime"), 0); // set the path we need char path[1024 + 1], *p = getenv("PATH"); @@ -625,10 +633,12 @@ int main(int argc, char **argv) { { char* stacksize_string = "stacksize="; char* debug_flags_string = "debug_flags="; + if(strcmp(optarg, "unittest") == 0) { - default_rrd_update_every = 1; - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; - if(!config_loaded) config_load(NULL, 0); + if(unit_test_str2ld()) exit(1); + //default_rrd_update_every = 1; + //default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + //if(!config_loaded) config_load(NULL, 0); get_netdata_configured_variables(); default_rrd_update_every = 1; default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; @@ -691,9 +701,71 @@ int main(int argc, char **argv) { config_set(CONFIG_SECTION_GLOBAL, "debug flags", optarg); debug_flags = strtoull(optarg, NULL, 0); } + else if(strcmp(optarg, "set") == 0) { + if(optind + 3 > argc) { + fprintf(stderr, "%s", "\nUSAGE: -W set 'section' 'key' 'value'\n\n" + " Overwrites settings of netdata.conf.\n" + "\n" + " These options interact with: -c netdata.conf\n" + " If -c netdata.conf is given on the command line,\n" + " before -W set... the user may overwrite command\n" + " line parameters at netdata.conf\n" + " If -c netdata.conf is given after (or missing)\n" + " -W set... the user cannot overwrite the command line\n" + " parameters." + "\n" + ); + exit(1); + } + const char *section = argv[optind]; + const char *key = argv[optind + 1]; + const char *value = argv[optind + 2]; + optind += 3; + + // set this one as the default + // only if it is not already set in the config file + // so the caller can use -c netdata.conf before or + // after this parameter to prevent or allow overwriting + // variables at netdata.conf + config_set_default(section, key, value); + + // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value); + } + else if(strcmp(optarg, "get") == 0) { + if(optind + 3 > argc) { + fprintf(stderr, "%s", "\nUSAGE: -W get 'section' 'key' 'value'\n\n" + " Prints settings of netdata.conf.\n" + "\n" + " These options interact with: -c netdata.conf\n" + " -c netdata.conf has to be given before -W get.\n" + "\n" + ); + exit(1); + } + + if(!config_loaded) { + fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n"); + config_load(NULL, 0); + } + + backwards_compatible_config(); + get_netdata_configured_variables(); + + const char *section = argv[optind]; + const char *key = argv[optind + 1]; + const char *def = argv[optind + 2]; + const char *value = config_get(section, key, def); + printf("%s\n", value); + exit(0); + } + else { + fprintf(stderr, "Unknown -W parameter '%s'\n", optarg); + help(1); + } } break; default: /* ? */ + fprintf(stderr, "Unknown parameter '%c'\n", opt); help(1); break; } @@ -875,8 +947,10 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // create the listening sockets + web_server_threading_selection(); + if(web_server_mode != WEB_SERVER_MODE_NONE) - create_listen_sockets(); + api_listen_sockets_setup(); } // initialize the log files @@ -928,7 +1002,7 @@ int main(int argc, char **argv) { // ------------------------------------------------------------------------ // spawn the threads - web_server_threading_selection(); + web_server_config_options(); for (i = 0; static_threads[i].name != NULL ; i++) { struct netdata_static_thread *st = &static_threads[i]; diff --git a/src/plugin_freebsd.c b/src/plugin_freebsd.c index 31ab6e0c4..020fdb41c 100644 --- a/src/plugin_freebsd.c +++ b/src/plugin_freebsd.c @@ -53,6 +53,12 @@ static struct freebsd_module { // network interfaces metrics { .name = "getifaddrs", .dim = "getifaddrs", .enabled = 1, .func = do_getifaddrs }, + // ZFS metrics + { .name = "kstat.zfs.misc.arcstats", .dim = "arcstats", .enabled = 1, .func = do_kstat_zfs_misc_arcstats }, + + // ipfw metrics + { .name = "ipfw", .dim = "ipfw", .enabled = 1, .func = do_ipfw }, + // the terminator of this array { .name = NULL, .dim = NULL, .enabled = 0, .func = NULL } }; diff --git a/src/plugin_freebsd.h b/src/plugin_freebsd.h index 166c64338..541bf852f 100644 --- a/src/plugin_freebsd.h +++ b/src/plugin_freebsd.h @@ -3,6 +3,12 @@ #include <sys/sysctl.h> +#define KILO_FACTOR 1024 +#define MEGA_FACTOR 1048576 // 1024 * 1024 +#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024 + +#define MAX_INT_DIGITS 10 // maximum number of digits for int + void *freebsd_main(void *ptr); extern int freebsd_plugin_init(); @@ -35,6 +41,8 @@ extern int do_net_inet6_icmp6_stats(int update_every, usec_t dt); extern int do_getifaddrs(int update_every, usec_t dt); extern int do_getmntinfo(int update_every, usec_t dt); extern int do_kern_devstat(int update_every, usec_t dt); +extern int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt); +extern int do_ipfw(int update_every, usec_t dt); #define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int)) diff --git a/src/plugin_idlejitter.c b/src/plugin_idlejitter.c index 2ed78160c..89f490233 100644 --- a/src/plugin_idlejitter.c +++ b/src/plugin_idlejitter.c @@ -13,41 +13,71 @@ void *cpuidlejitter_main(void *ptr) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("Cannot set pthread cancel state to ENABLE."); - int sleep_ms = (int) config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS); - if(sleep_ms <= 0) { + usec_t sleep_ut = config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS) * USEC_PER_MS; + if(sleep_ut <= 0) { config_set_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS); - sleep_ms = CPU_IDLEJITTER_SLEEP_TIME_MS; - } - - RRDSET *st = rrdset_find_localhost("system.idlejitter"); - if(!st) { - st = rrdset_create_localhost("system", "idlejitter", NULL, "processes", NULL, "CPU Idle Jitter" - , "microseconds lost/s", 9999, localhost->rrd_update_every, RRDSET_TYPE_LINE); - rrddim_add(st, "jitter", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + sleep_ut = CPU_IDLEJITTER_SLEEP_TIME_MS * USEC_PER_MS; } + RRDSET *st = rrdset_create_localhost( + "system" + , "idlejitter" + , NULL + , "processes" + , NULL + , "CPU Idle Jitter" + , "microseconds lost/s" + , 9999 + , localhost->rrd_update_every + , RRDSET_TYPE_AREA + ); + RRDDIM *rd_min = rrddim_add(st, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_avg = rrddim_add(st, "average", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + usec_t update_every_ut = localhost->rrd_update_every * USEC_PER_SEC; struct timeval before, after; unsigned long long counter; for(counter = 0; 1 ;counter++) { - usec_t usec = 0, susec = 0; + int iterations = 0; + usec_t error_total = 0, + error_min = 0, + error_max = 0, + elapsed = 0; if(netdata_exit) break; - while(susec < (localhost->rrd_update_every * USEC_PER_SEC)) { - + while(elapsed < update_every_ut) { now_monotonic_timeval(&before); - sleep_usec(sleep_ms * 1000); + sleep_usec(sleep_ut); now_monotonic_timeval(&after); - // calculate the time it took for a full loop - usec = dt_usec(&after, &before); - susec += usec; + usec_t dt = dt_usec(&after, &before); + elapsed += dt; + + usec_t error = dt - sleep_ut; + error_total += error; + + if(unlikely(!iterations)) + error_min = error; + else if(error < error_min) + error_min = error; + + if(error > error_max) + error_max = error; + + iterations++; } - usec -= (sleep_ms * 1000); - if(counter) rrdset_next(st); - rrddim_set(st, "jitter", usec); - rrdset_done(st); + if(netdata_exit) break; + + if(iterations) { + if (likely(counter)) rrdset_next(st); + rrddim_set_by_pointer(st, rd_min, error_min); + rrddim_set_by_pointer(st, rd_max, error_max); + rrddim_set_by_pointer(st, rd_avg, error_total / iterations); + rrdset_done(st); + } } info("IDLEJITTER thread exiting"); diff --git a/src/plugin_proc.c b/src/plugin_proc.c index 2ca77491d..e64f57398 100644 --- a/src/plugin_proc.c +++ b/src/plugin_proc.c @@ -32,7 +32,7 @@ static struct proc_module { // network metrics { .name = "/proc/net/dev", .dim = "netdev", .func = do_proc_net_dev }, - { .name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat }, + { .name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat }, // this has to be before /proc/net/snmp, because there is a shared metric { .name = "/proc/net/snmp", .dim = "snmp", .func = do_proc_net_snmp }, { .name = "/proc/net/snmp6", .dim = "snmp6", .func = do_proc_net_snmp6 }, { .name = "/proc/net/softnet_stat", .dim = "softnet", .func = do_proc_net_softnet_stat }, @@ -49,6 +49,9 @@ static struct proc_module { { .name = "/proc/net/rpc/nfsd", .dim = "nfsd", .func = do_proc_net_rpc_nfsd }, { .name = "/proc/net/rpc/nfs", .dim = "nfs", .func = do_proc_net_rpc_nfs }, + // ZFS metrics + { .name = "/proc/spl/kstat/zfs/arcstats", .dim = "zfs_arcstats", .func = do_proc_spl_kstat_zfs_arcstats }, + // IPC metrics { .name = "ipc", .dim = "ipc", .func = do_ipc }, diff --git a/src/plugin_proc.h b/src/plugin_proc.h index 5dee7853c..688b23de9 100644 --- a/src/plugin_proc.h +++ b/src/plugin_proc.h @@ -25,7 +25,11 @@ extern int do_proc_net_softnet_stat(int update_every, usec_t dt); extern int do_proc_uptime(int update_every, usec_t dt); extern int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt); extern int do_proc_sys_devices_system_node(int update_every, usec_t dt); +extern int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt); extern int get_numa_node_count(void); +// metrics that need to be shared among data collectors +extern unsigned long long tcpext_TCPSynRetrans; + #endif /* NETDATA_PLUGIN_PROC_H */ diff --git a/src/plugin_proc_diskspace.c b/src/plugin_proc_diskspace.c index 37133e044..750086a2c 100644 --- a/src/plugin_proc_diskspace.c +++ b/src/plugin_proc_diskspace.c @@ -45,7 +45,7 @@ struct mount_point_metadata { static DICTIONARY *dict_mountpoints = NULL; -#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); st = NULL; } } while(st) +#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); st = NULL; } } while(st) int mount_point_cleanup(void *entry, void *data) { (void)data; @@ -125,6 +125,33 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { def_inodes = CONFIG_BOOLEAN_NO; } + // check if the mount point is a directory #2407 + { + struct stat bs; + if(stat(mi->mount_point, &bs) == -1) { + error("DISKSPACE: Cannot stat() mount point '%s' (disk '%s', filesystem '%s', root '%s')." + , mi->mount_point + , disk + , mi->filesystem?mi->filesystem:"" + , mi->root?mi->root:"" + ); + def_space = CONFIG_BOOLEAN_NO; + def_inodes = CONFIG_BOOLEAN_NO; + } + else { + if((bs.st_mode & S_IFMT) != S_IFDIR) { + error("DISKSPACE: Mount point '%s' (disk '%s', filesystem '%s', root '%s') is not a directory." + , mi->mount_point + , disk + , mi->filesystem?mi->filesystem:"" + , mi->root?mi->root:"" + ); + def_space = CONFIG_BOOLEAN_NO; + def_inodes = CONFIG_BOOLEAN_NO; + } + } + } + do_space = config_get_boolean_ondemand(var_name, "space usage", def_space); do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", def_inodes); @@ -161,7 +188,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { struct statvfs buff_statvfs; if (statvfs(mi->mount_point, &buff_statvfs) < 0) { if(!m->shown_error) { - error("Failed statvfs() for '%s' (disk '%s', filesystem '%s', root '%s')" + error("DISKSPACE: failed to statvfs() mount point '%s' (disk '%s', filesystem '%s', root '%s')" , mi->mount_point , disk , mi->filesystem?mi->filesystem:"" @@ -188,7 +215,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { #ifdef NETDATA_INTERNAL_CHECKS if(unlikely(btotal != bavail + breserved_root + bused)) - error("Disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused); + error("DISKSPACE: disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused); #endif // -------------------------------------------------------------------------- @@ -201,7 +228,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { #ifdef NETDATA_INTERNAL_CHECKS if(unlikely(btotal != bavail + breserved_root + bused)) - error("Disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused); + error("DISKSPACE: disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused); #endif // -------------------------------------------------------------------------- @@ -294,10 +321,10 @@ void *proc_diskspace_main(void *ptr) { info("DISKSPACE thread created with task id %d", gettid()); if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) - error("Cannot set pthread cancel type to DEFERRED."); + error("DISKSPACE: Cannot set pthread cancel type to DEFERRED."); if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) - error("Cannot set pthread cancel state to ENABLE."); + error("DISKSPACE: Cannot set pthread cancel state to ENABLE."); int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1); @@ -334,7 +361,7 @@ void *proc_diskspace_main(void *ptr) { struct mountinfo *mi; for(mi = disk_mountinfo_root; mi; mi = mi->next) { - if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND | MOUNTINFO_IS_SAME_DEV | MOUNTINFO_NO_STAT | MOUNTINFO_NO_SIZE))) + if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND))) continue; do_disk_space_stats(mi, update_every); diff --git a/src/plugin_tc.c b/src/plugin_tc.c index 7dcfedb33..6bf5782a9 100644 --- a/src/plugin_tc.c +++ b/src/plugin_tc.c @@ -713,7 +713,7 @@ static inline void tc_device_free_all() tc_device_free(tc_device_root); } -#define MAX_WORDS 20 +#define PLUGINSD_MAX_WORDS 20 static inline int tc_space(char c) { switch(c) { @@ -779,7 +779,7 @@ void *tc_main(void *ptr) { RRDSET *stcpu = NULL, *sttime = NULL; char buffer[TC_LINE_MAX+1] = ""; - char *words[MAX_WORDS] = { NULL }; + char *words[PLUGINSD_MAX_WORDS] = { NULL }; uint32_t BEGIN_HASH = simple_hash("BEGIN"); uint32_t END_HASH = simple_hash("END"); @@ -822,7 +822,7 @@ void *tc_main(void *ptr) { buffer[TC_LINE_MAX] = '\0'; // debug(D_TC_LOOP, "TC: read '%s'", buffer); - tc_split_words(buffer, words, MAX_WORDS); + tc_split_words(buffer, words, PLUGINSD_MAX_WORDS); if(unlikely(!words[0] || !*words[0])) { // debug(D_TC_LOOP, "empty line"); diff --git a/src/plugins_d.c b/src/plugins_d.c index 7fa19eaf0..9eb102770 100644 --- a/src/plugins_d.c +++ b/src/plugins_d.c @@ -2,8 +2,6 @@ struct plugind *pluginsd_root = NULL; -#define MAX_WORDS 20 - static inline int pluginsd_space(char c) { switch(c) { case ' ': @@ -18,7 +16,8 @@ static inline int pluginsd_space(char c) { } } -static int pluginsd_split_words(char *str, char **words, int max_words) { +// split a text into words, respecting quotes +inline int pluginsd_split_words(char *str, char **words, int max_words) { char *s = str, quote = 0; int i = 0, j; @@ -95,14 +94,13 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int char line[PLUGINSD_LINE_MAX + 1]; - char *words[MAX_WORDS] = { NULL }; - /* uint32_t HOST_HASH = simple_hash("HOST"); */ - uint32_t BEGIN_HASH = simple_hash("BEGIN"); - uint32_t END_HASH = simple_hash("END"); - uint32_t FLUSH_HASH = simple_hash("FLUSH"); - uint32_t CHART_HASH = simple_hash("CHART"); - uint32_t DIMENSION_HASH = simple_hash("DIMENSION"); - uint32_t DISABLE_HASH = simple_hash("DISABLE"); + char *words[PLUGINSD_MAX_WORDS] = { NULL }; + uint32_t BEGIN_HASH = simple_hash(PLUGINSD_KEYWORD_BEGIN); + uint32_t END_HASH = simple_hash(PLUGINSD_KEYWORD_END); + uint32_t FLUSH_HASH = simple_hash(PLUGINSD_KEYWORD_FLUSH); + uint32_t CHART_HASH = simple_hash(PLUGINSD_KEYWORD_CHART); + uint32_t DIMENSION_HASH = simple_hash(PLUGINSD_KEYWORD_DIMENSION); + uint32_t DISABLE_HASH = simple_hash(PLUGINSD_KEYWORD_DISABLE); RRDSET *st = NULL; uint32_t hash; @@ -130,7 +128,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int // debug(D_PLUGINSD, "PLUGINSD: %s: %s", cd->filename, line); - int w = pluginsd_split_words(line, words, MAX_WORDS); + int w = pluginsd_split_words(line, words, PLUGINSD_MAX_WORDS); char *s = words[0]; if(unlikely(!s || !*s || !w)) { // debug(D_PLUGINSD, "PLUGINSD: empty line"); @@ -159,9 +157,18 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) debug(D_PLUGINSD, "PLUGINSD: '%s' is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value?value:"<nothing>"); - if(value) rrddim_set(st, dimension, strtoll(value, NULL, 0)); + if(value) { + RRDDIM *rd = rrddim_find(st, dimension); + if(unlikely(!rd)) { + error("PLUGINSD: '%s' is requesting a SET to dimension with id '%s' on stats '%s' (%s) on host '%s', which does not exist. Disabling it.", cd->fullfilename, dimension, st->name, st->id, st->rrdhost->hostname); + enabled = 0; + break; + } + else + rrddim_set_by_pointer(st, rd, strtoll(value, NULL, 0)); + } } - else if(likely(hash == BEGIN_HASH && !strcmp(s, "BEGIN"))) { + else if(likely(hash == BEGIN_HASH && !strcmp(s, PLUGINSD_KEYWORD_BEGIN))) { char *id = words[1]; char *microseconds_txt = words[2]; @@ -191,7 +198,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int else rrdset_next(st); } } - else if(likely(hash == END_HASH && !strcmp(s, "END"))) { + else if(likely(hash == END_HASH && !strcmp(s, PLUGINSD_KEYWORD_END))) { if(unlikely(!st)) { error("PLUGINSD: '%s' is requesting an END, without a BEGIN on host '%s'. Disabling it.", cd->fullfilename, host->hostname); enabled = 0; @@ -205,28 +212,11 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int count++; } -/* else if(likely(hash == HOST_HASH && !strcmp(s, "HOST"))) { - char *guid = words[1]; - char *hostname = words[2]; - - if(unlikely(!guid || !*guid)) { - error("PLUGINSD: '%s' is requesting HOST with guid '%s' and hostname '%s', without a guid. Disabling it.", cd->fullfilename, guid?guid:"", hostname?hostname:""); - enabled = 0; - break; - } - if(unlikely(!hostname || !*hostname)) { - error("PLUGINSD: '%s' is requesting HOST with guid '%s' and hostname '%s', without a hostname. Disabling it.", cd->fullfilename, guid?guid:"", hostname?hostname:""); - enabled = 0; - break; - } - - host = rrdhost_find_or_create(hostname, guid); - } */ - else if(likely(hash == FLUSH_HASH && !strcmp(s, "FLUSH"))) { + else if(likely(hash == FLUSH_HASH && !strcmp(s, PLUGINSD_KEYWORD_FLUSH))) { debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename); st = NULL; } - else if(likely(hash == CHART_HASH && !strcmp(s, "CHART"))) { + else if(likely(hash == CHART_HASH && !strcmp(s, PLUGINSD_KEYWORD_CHART))) { int noname = 0; st = NULL; @@ -247,6 +237,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int char *chart = words[7]; char *priority_s = words[8]; char *update_every_s = words[9]; + char *options = words[10]; if(unlikely(!type || !*type || !id || !*id)) { error("PLUGINSD: '%s' is requesting a CHART, without a type.id, on host '%s'. Disabling it.", cd->fullfilename, host->hostname); @@ -284,8 +275,25 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int cd->update_every = update_every; } else debug(D_PLUGINSD, "PLUGINSD: Chart '%s' already exists. Not adding it again.", st->id); + + if(options && *options) { + if(strstr(options, "obsolete")) + rrdset_is_obsolete(st); + else + rrdset_isnot_obsolete(st); + + if(strstr(options, "detail")) + rrdset_flag_set(st, RRDSET_FLAG_DETAIL); + else + rrdset_flag_clear(st, RRDSET_FLAG_DETAIL); + + if(strstr(options, "store_first")) + rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST); + else + rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST); + } } - else if(likely(hash == DIMENSION_HASH && !strcmp(s, "DIMENSION"))) { + else if(likely(hash == DIMENSION_HASH && !strcmp(s, PLUGINSD_KEYWORD_DIMENSION))) { char *id = words[1]; char *name = words[2]; char *algorithm = words[3]; @@ -326,21 +334,16 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int , options?options:"" ); - RRDDIM *rd = rrddim_find(st, id); - if(unlikely(!rd)) { - rd = rrddim_add(st, id, name, multiplier, divisor, rrd_algorithm_id(algorithm)); - rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN); - rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); - if(options && *options) { - if(strstr(options, "hidden") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN); - if(strstr(options, "noreset") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); - if(strstr(options, "nooverflow") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); - } + RRDDIM *rd = rrddim_add(st, id, name, multiplier, divisor, rrd_algorithm_id(algorithm)); + rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN); + rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); + if(options && *options) { + if(strstr(options, "hidden") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN); + if(strstr(options, "noreset") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); + if(strstr(options, "nooverflow") != NULL) rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS); } - else if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_PLUGINSD, "PLUGINSD: dimension %s/%s already exists. Not adding it again.", st->id, id); } - else if(unlikely(hash == DISABLE_HASH && !strcmp(s, "DISABLE"))) { + else if(unlikely(hash == DISABLE_HASH && !strcmp(s, PLUGINSD_KEYWORD_DISABLE))) { info("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename); enabled = 0; break; diff --git a/src/plugins_d.h b/src/plugins_d.h index d34c4030c..595a515c4 100644 --- a/src/plugins_d.h +++ b/src/plugins_d.h @@ -4,7 +4,16 @@ #define PLUGINSD_FILE_SUFFIX ".plugin" #define PLUGINSD_FILE_SUFFIX_LEN strlen(PLUGINSD_FILE_SUFFIX) #define PLUGINSD_CMD_MAX (FILENAME_MAX*2) + +#define PLUGINSD_KEYWORD_CHART "CHART" +#define PLUGINSD_KEYWORD_DIMENSION "DIMENSION" +#define PLUGINSD_KEYWORD_BEGIN "BEGIN" +#define PLUGINSD_KEYWORD_END "END" +#define PLUGINSD_KEYWORD_FLUSH "FLUSH" +#define PLUGINSD_KEYWORD_DISABLE "DISABLE" + #define PLUGINSD_LINE_MAX 1024 +#define PLUGINSD_MAX_WORDS 20 struct plugind { char id[CONFIG_MAX_NAME+1]; // config node id @@ -35,5 +44,6 @@ extern struct plugind *pluginsd_root; extern void *pluginsd_main(void *ptr); extern size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations); +extern int pluginsd_split_words(char *str, char **words, int max_words); #endif /* NETDATA_PLUGINS_D_H */ diff --git a/src/proc_diskstats.c b/src/proc_diskstats.c index a1b4072dd..d3fed5a6d 100644 --- a/src/proc_diskstats.c +++ b/src/proc_diskstats.c @@ -10,7 +10,8 @@ #define DELAULT_EXLUDED_DISKS "loop* ram*" static struct disk { - char *disk; // the name of the disk (sda, sdb, etc) + char *disk; // the name of the disk (sda, sdb, etc, after being looked up) + char *device; // the device of the disk (before being looked up) unsigned long major; unsigned long minor; int sector_size; @@ -44,12 +45,75 @@ static struct disk { struct disk *next; } *disk_root = NULL; -#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); st = NULL; } } while(st) +#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); st = NULL; } } while(st) + +static char *path_to_get_hw_sector_size = NULL; +static char *path_to_get_hw_sector_size_partitions = NULL; +static char *path_to_find_block_device = NULL; +static char *path_to_device_mapper = NULL; + +static inline char *get_disk_name(unsigned long major, unsigned long minor, char *disk) { + static int enabled = 1; + + if(!enabled) goto cleanup; + + char filename[FILENAME_MAX + 1]; + char link[FILENAME_MAX + 1]; + + DIR *dir = opendir(path_to_device_mapper); + if (!dir) { + error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot open directory '%s'. Disabling device-mapper support.", disk, major, minor, path_to_device_mapper); + enabled = 0; + goto cleanup; + } + + struct dirent *de = NULL; + while ((de = readdir(dir))) { + if(de->d_type != DT_LNK) continue; + + snprintfz(filename, FILENAME_MAX, "%s/%s", path_to_device_mapper, de->d_name); + ssize_t len = readlink(filename, link, FILENAME_MAX); + if(len <= 0) { + error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot read link '%s'.", disk, major, minor, filename); + continue; + } + + link[len] = '\0'; + if(link[0] != '/') + snprintfz(filename, FILENAME_MAX, "%s/%s", path_to_device_mapper, link); + else + strncpyz(filename, link, FILENAME_MAX); + + struct stat sb; + if(stat(filename, &sb) == -1) { + error("DEVICE-MAPPER ('%s', %lu:%lu): Cannot stat() file '%s'.", disk, major, minor, filename); + continue; + } + + if((sb.st_mode & S_IFMT) != S_IFBLK) { + // info("DEVICE-MAPPER ('%s', %lu:%lu): file '%s' is not a block device.", disk, major, minor, filename); + continue; + } + + if(major(sb.st_rdev) != major || minor(sb.st_rdev) != minor) { + // info("DEVICE-MAPPER ('%s', %lu:%lu): filename '%s' does not match %lu:%lu.", disk, major, minor, filename, (unsigned long)major(sb.st_rdev), (unsigned long)minor(sb.st_rdev)); + continue; + } + + // info("DEVICE-MAPPER ('%s', %lu:%lu): filename '%s' matches.", disk, major, minor, filename); + + strncpy(link, de->d_name, FILENAME_MAX); + netdata_fix_chart_name(link); + disk = link; + break; + } + closedir(dir); + +cleanup: + return strdupz(disk); +} static struct disk *get_disk(unsigned long major, unsigned long minor, char *disk) { - static char path_to_get_hw_sector_size[FILENAME_MAX + 1] = ""; - static char path_to_get_hw_sector_size_partitions[FILENAME_MAX + 1] = ""; - static char path_find_block_device[FILENAME_MAX + 1] = ""; static struct mountinfo *disk_mountinfo_root = NULL; struct disk *d; @@ -66,7 +130,8 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis // create a new disk structure d = (struct disk *)callocz(1, sizeof(struct disk)); - d->disk = strdupz(disk); + d->disk = get_disk_name(major, minor, disk); + d->device = strdupz(disk); d->major = major; d->minor = minor; d->type = DISK_TYPE_PHYSICAL; // Default type. Changed later if not correct. @@ -83,27 +148,17 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis last->next = d; } - // ------------------------------------------------------------------------ - // find the type of the device - - char buffer[FILENAME_MAX + 1]; - - // get the default path for finding info about the block device - if(unlikely(!path_find_block_device[0])) { - snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/%s"); - snprintfz(path_find_block_device, FILENAME_MAX, "%s", config_get(CONFIG_SECTION_DISKSTATS, "path to get block device infos", buffer)); - } - // find if it is a partition // by checking if /sys/dev/block/MAJOR:MINOR/partition is readable. - snprintfz(buffer, FILENAME_MAX, path_find_block_device, major, minor, "partition"); + char buffer[FILENAME_MAX + 1]; + snprintfz(buffer, FILENAME_MAX, path_to_find_block_device, major, minor, "partition"); if(likely(access(buffer, R_OK) == 0)) { d->type = DISK_TYPE_PARTITION; } else { // find if it is a container // by checking if /sys/dev/block/MAJOR:MINOR/slaves has entries - snprintfz(buffer, FILENAME_MAX, path_find_block_device, major, minor, "slaves/"); + snprintfz(buffer, FILENAME_MAX, path_to_find_block_device, major, minor, "slaves/"); DIR *dirp = opendir(buffer); if(likely(dirp != NULL)) { struct dirent *dp; @@ -143,18 +198,9 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis // ------------------------------------------------------------------------ // find the disk sector size - if(unlikely(!path_to_get_hw_sector_size[0])) { - snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/queue/hw_sector_size"); - snprintfz(path_to_get_hw_sector_size, FILENAME_MAX, "%s", config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size", buffer)); - } - if(unlikely(!path_to_get_hw_sector_size_partitions[0])) { - snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/subsystem/%s/../queue/hw_sector_size"); - snprintfz(path_to_get_hw_sector_size_partitions, FILENAME_MAX, "%s", config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size for partitions", buffer)); - } - { char tf[FILENAME_MAX + 1], *t; - strncpyz(tf, d->disk, FILENAME_MAX); + strncpyz(tf, d->device, FILENAME_MAX); // replace all / with ! for(t = tf; *t ;t++) @@ -173,15 +219,15 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis if(likely(tmp)) { d->sector_size = str2i(tmp); if(unlikely(d->sector_size <= 0)) { - error("Invalid sector size %d for device %s in %s. Assuming 512.", d->sector_size, d->disk, buffer); + error("Invalid sector size %d for device %s in %s. Assuming 512.", d->sector_size, d->device, buffer); d->sector_size = 512; } } - else error("Cannot read data for sector size for device %s from %s. Assuming 512.", d->disk, buffer); + else error("Cannot read data for sector size for device %s from %s. Assuming 512.", d->device, buffer); fclose(fpss); } - else error("Cannot read sector size for device %s from %s. Assuming 512.", d->disk, buffer); + else error("Cannot read sector size for device %s from %s. Assuming 512.", d->device, buffer); } return d; @@ -230,6 +276,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { globals_initialized = 0; if(unlikely(!globals_initialized)) { + globals_initialized = 1; + global_enable_new_disks_detected_at_runtime = config_get_boolean(CONFIG_SECTION_DISKSTATS, "enable new disks detected at runtime", global_enable_new_disks_detected_at_runtime); global_enable_performance_for_physical_disks = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "performance metrics for physical disks", global_enable_performance_for_physical_disks); global_enable_performance_for_virtual_disks = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "performance metrics for virtual disks", global_enable_performance_for_virtual_disks); @@ -243,7 +291,19 @@ int do_proc_diskstats(int update_every, usec_t dt) { global_do_util = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "utilization percentage for all disks", global_do_util); global_do_backlog = config_get_boolean_ondemand(CONFIG_SECTION_DISKSTATS, "backlog for all disks", global_do_backlog); - globals_initialized = 1; + char buffer[FILENAME_MAX + 1]; + + snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/%s"); + path_to_find_block_device = config_get(CONFIG_SECTION_DISKSTATS, "path to get block device infos", buffer); + + snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/queue/hw_sector_size"); + path_to_get_hw_sector_size = config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size", buffer); + + snprintfz(buffer, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/dev/block/%lu:%lu/subsystem/%s/../queue/hw_sector_size"); + path_to_get_hw_sector_size_partitions = config_get(CONFIG_SECTION_DISKSTATS, "path to get h/w sector size for partitions", buffer); + + snprintfz(buffer, FILENAME_MAX, "%s/dev/mapper", netdata_configured_host_prefix); + path_to_device_mapper = config_get(CONFIG_SECTION_DISKSTATS, "path to device mapper", buffer); } // -------------------------------------------------------------------------- @@ -339,7 +399,7 @@ int do_proc_diskstats(int update_every, usec_t dt) { // Set its family based on mount point char *family = d->mount_point; - if(!family) family = disk; + if(!family) family = d->disk; // -------------------------------------------------------------------------- @@ -359,11 +419,11 @@ int do_proc_diskstats(int update_every, usec_t dt) { int def_enable = global_enable_new_disks_detected_at_runtime; - if(def_enable != CONFIG_BOOLEAN_NO && simple_pattern_matches(excluded_disks, disk)) + if(def_enable != CONFIG_BOOLEAN_NO && (simple_pattern_matches(excluded_disks, d->device) || simple_pattern_matches(excluded_disks, d->disk))) def_enable = CONFIG_BOOLEAN_NO; char var_name[4096 + 1]; - snprintfz(var_name, 4096, "plugin:proc:/proc/diskstats:%s", disk); + snprintfz(var_name, 4096, "plugin:proc:/proc/diskstats:%s", d->disk); def_enable = config_get_boolean_ondemand(var_name, "enable", def_enable); if(unlikely(def_enable == CONFIG_BOOLEAN_NO)) { @@ -449,8 +509,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_io)) { d->st_io = rrdset_create_localhost( RRD_TYPE_DISK - , disk - , NULL + , d->device + , d->disk , family , "disk.io" , "Disk I/O Bandwidth" @@ -478,8 +538,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_ops)) { d->st_ops = rrdset_create_localhost( "disk_ops" - , disk - , NULL + , d->device + , d->disk , family , "disk.ops" , "Disk Completed I/O Operations" @@ -509,8 +569,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_qops)) { d->st_qops = rrdset_create_localhost( "disk_qops" - , disk - , NULL + , d->device + , d->disk , family , "disk.qops" , "Disk Current I/O Operations" @@ -538,8 +598,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_backlog)) { d->st_backlog = rrdset_create_localhost( "disk_backlog" - , disk - , NULL + , d->device + , d->disk , family , "disk.backlog" , "Disk Backlog" @@ -567,8 +627,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_util)) { d->st_util = rrdset_create_localhost( "disk_util" - , disk - , NULL + , d->device + , d->disk , family , "disk.util" , "Disk Utilization Time" @@ -596,8 +656,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_mops)) { d->st_mops = rrdset_create_localhost( "disk_mops" - , disk - , NULL + , d->device + , d->disk , family , "disk.mops" , "Disk Merged Operations" @@ -627,8 +687,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_iotime)) { d->st_iotime = rrdset_create_localhost( "disk_iotime" - , disk - , NULL + , d->device + , d->disk , family , "disk.iotime" , "Disk Total I/O Time" @@ -661,8 +721,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_await)) { d->st_await = rrdset_create_localhost( "disk_await" - , disk - , NULL + , d->device + , d->disk , family , "disk.await" , "Average Completed I/O Operation Time" @@ -690,8 +750,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_avgsz)) { d->st_avgsz = rrdset_create_localhost( "disk_avgsz" - , disk - , NULL + , d->device + , d->disk , family , "disk.avgsz" , "Average Completed I/O Operation Bandwidth" @@ -719,8 +779,8 @@ int do_proc_diskstats(int update_every, usec_t dt) { if(unlikely(!d->st_svctm)) { d->st_svctm = rrdset_create_localhost( "disk_svctm" - , disk - , NULL + , d->device + , d->disk , family , "disk.svctm" , "Average Service Time" @@ -769,6 +829,7 @@ int do_proc_diskstats(int update_every, usec_t dt) { } freez(t->disk); + freez(t->device); freez(t->mount_point); freez(t); } diff --git a/src/proc_loadavg.c b/src/proc_loadavg.c index e7863f114..a48801b37 100644 --- a/src/proc_loadavg.c +++ b/src/proc_loadavg.c @@ -68,9 +68,10 @@ int do_proc_loadavg(int update_every, usec_t dt) { rrddim_set(load_chart, "load5", (collected_number) (load5 * 1000)); rrddim_set(load_chart, "load15", (collected_number) (load15 * 1000)); rrdset_done(load_chart); - } - next_loadavg_dt = load_chart->update_every * USEC_PER_SEC; + next_loadavg_dt = load_chart->update_every * USEC_PER_SEC; + } + else next_loadavg_dt = MIN_LOADAVG_UPDATE_EVERY * USEC_PER_SEC; } else next_loadavg_dt -= dt; diff --git a/src/proc_net_dev.c b/src/proc_net_dev.c index 1b00758d5..ee7588990 100644 --- a/src/proc_net_dev.c +++ b/src/proc_net_dev.c @@ -73,13 +73,13 @@ static struct netdev *netdev_root = NULL, *netdev_last_used = NULL; static size_t netdev_added = 0, netdev_found = 0; static void netdev_free(struct netdev *d) { - if(d->st_bandwidth) rrdset_flag_set(d->st_bandwidth, RRDSET_FLAG_OBSOLETE); - if(d->st_packets) rrdset_flag_set(d->st_packets, RRDSET_FLAG_OBSOLETE); - if(d->st_errors) rrdset_flag_set(d->st_errors, RRDSET_FLAG_OBSOLETE); - if(d->st_drops) rrdset_flag_set(d->st_drops, RRDSET_FLAG_OBSOLETE); - if(d->st_fifo) rrdset_flag_set(d->st_fifo, RRDSET_FLAG_OBSOLETE); - if(d->st_compressed) rrdset_flag_set(d->st_compressed, RRDSET_FLAG_OBSOLETE); - if(d->st_events) rrdset_flag_set(d->st_events, RRDSET_FLAG_OBSOLETE); + if(d->st_bandwidth) rrdset_is_obsolete(d->st_bandwidth); + if(d->st_packets) rrdset_is_obsolete(d->st_packets); + if(d->st_errors) rrdset_is_obsolete(d->st_errors); + if(d->st_drops) rrdset_is_obsolete(d->st_drops); + if(d->st_fifo) rrdset_is_obsolete(d->st_fifo); + if(d->st_compressed) rrdset_is_obsolete(d->st_compressed); + if(d->st_events) rrdset_is_obsolete(d->st_events); netdev_added--; freez(d->name); diff --git a/src/proc_net_netstat.c b/src/proc_net_netstat.c index 2677a6c17..322e51d13 100644 --- a/src/proc_net_netstat.c +++ b/src/proc_net_netstat.c @@ -1,5 +1,7 @@ #include "common.h" +unsigned long long tcpext_TCPSynRetrans; + static void parse_line_pair(procfile *ff, ARL_BASE *base, size_t header_line, size_t values_line) { size_t hwords = procfile_linewords(ff, header_line); size_t vwords = procfile_linewords(ff, values_line); @@ -94,6 +96,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) { // IPv4 TCP memory pressures static unsigned long long tcpext_TCPMemoryPressures = 0; + // shared: tcpext_TCPSynRetrans + + if(unlikely(!arl_ipext)) { hash_ipext = simple_hash("IpExt"); hash_tcpext = simple_hash("TcpExt"); @@ -191,6 +196,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) { if(do_tcpext_memory != CONFIG_BOOLEAN_NO) { arl_expect(arl_tcpext, "TCPMemoryPressures", &tcpext_TCPMemoryPressures); } + + // shared metrics + arl_expect(arl_tcpext, "TCPSynRetrans", &tcpext_TCPSynRetrans); } if(unlikely(!ff)) { diff --git a/src/proc_net_snmp.c b/src/proc_net_snmp.c index ba7b40013..7c0fd9b4a 100644 --- a/src/proc_net_snmp.c +++ b/src/proc_net_snmp.c @@ -645,24 +645,35 @@ int do_proc_net_snmp(int update_every, usec_t dt) { if(do_tcp_handshake) { st = rrdset_find_localhost(RRD_TYPE_NET_SNMP ".tcphandshake"); if(!st) { - st = rrdset_create_localhost(RRD_TYPE_NET_SNMP, "tcphandshake", NULL, "tcp", NULL - , "IPv4 TCP Handshake Issues", "events/s", 2900, update_every - , RRDSET_TYPE_LINE); + st = rrdset_create_localhost( + RRD_TYPE_NET_SNMP + , "tcphandshake" + , NULL + , "tcp" + , NULL + , "IPv4 TCP Handshake Issues" + , "events/s" + , 2900 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(st, RRDSET_FLAG_DETAIL); - rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "OutRsts", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st, "EstabResets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st, "OutRsts", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st, "AttemptFails", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st, "TCPSynRetrans", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "EstabResets", *tcp_EstabResets); - rrddim_set(st, "OutRsts", *tcp_OutRsts); - rrddim_set(st, "ActiveOpens", *tcp_ActiveOpens); - rrddim_set(st, "PassiveOpens", *tcp_PassiveOpens); - rrddim_set(st, "AttemptFails", *tcp_AttemptFails); + rrddim_set(st, "EstabResets", *tcp_EstabResets); + rrddim_set(st, "OutRsts", *tcp_OutRsts); + rrddim_set(st, "ActiveOpens", *tcp_ActiveOpens); + rrddim_set(st, "PassiveOpens", *tcp_PassiveOpens); + rrddim_set(st, "AttemptFails", *tcp_AttemptFails); + rrddim_set(st, "TCPSynRetrans", tcpext_TCPSynRetrans); rrdset_done(st); } } diff --git a/src/proc_net_snmp6.c b/src/proc_net_snmp6.c index 8c4581c1b..aa9ab2209 100644 --- a/src/proc_net_snmp6.c +++ b/src/proc_net_snmp6.c @@ -311,8 +311,8 @@ int do_proc_net_snmp6(int update_every, usec_t dt) { rrddim_set(st, "sent", Ip6OutRequests); rrddim_set(st, "received", Ip6InReceives); - rrddim_set(st, "forwarded", Ip6InDelivers); - rrddim_set(st, "delivers", Ip6OutForwDatagrams); + rrddim_set(st, "forwarded", Ip6OutForwDatagrams); + rrddim_set(st, "delivers", Ip6InDelivers); rrdset_done(st); } diff --git a/src/proc_net_softnet_stat.c b/src/proc_net_softnet_stat.c index 40946a7a5..b03a43c52 100644 --- a/src/proc_net_softnet_stat.c +++ b/src/proc_net_softnet_stat.c @@ -79,7 +79,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) { st = rrdset_find_bytype_localhost("system", "softnet_stat"); if(unlikely(!st)) { - st = rrdset_create_localhost("system", "softnet_stat", NULL, "softnet_stat", NULL, "System softnet_stat" + st = rrdset_create_localhost("system", "softnet_stat", NULL, "softnet_stat", "system.softnet_stat", "System softnet_stat" , "events/s", 955, update_every, RRDSET_TYPE_LINE); for(w = 0; w < allocated_columns ;w++) if(unlikely(softnet_column_name(w))) @@ -103,7 +103,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) { char title[100+1]; snprintfz(title, 100, "CPU%zu softnet_stat", l); - st = rrdset_create_localhost("cpu", id, NULL, "softnet_stat", NULL, title, "events/s", 4101 + l + st = rrdset_create_localhost("cpu", id, NULL, "softnet_stat", "cpu.softnet_stat", title, "events/s", 4101 + l , update_every, RRDSET_TYPE_LINE); for(w = 0; w < allocated_columns ;w++) if(unlikely(softnet_column_name(w))) diff --git a/src/proc_spl_kstat_zfs.c b/src/proc_spl_kstat_zfs.c new file mode 100644 index 000000000..dee7a6b3d --- /dev/null +++ b/src/proc_spl_kstat_zfs.c @@ -0,0 +1,153 @@ +#include "common.h" +#include "zfs_common.h" + +#define ZFS_PROC_ARCSTATS "/proc/spl/kstat/zfs/arcstats" + +struct arcstats arcstats = { 0 }; + +int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt) { + (void)dt; + + static procfile *ff = NULL; + static ARL_BASE *arl_base = NULL; + + l2exist = -1; + + if(unlikely(!arl_base)) { + arl_base = arl_create("arcstats", NULL, 60); + + arl_expect(arl_base, "hits", &arcstats.hits); + arl_expect(arl_base, "misses", &arcstats.misses); + arl_expect(arl_base, "demand_data_hits", &arcstats.demand_data_hits); + arl_expect(arl_base, "demand_data_misses", &arcstats.demand_data_misses); + arl_expect(arl_base, "demand_metadata_hits", &arcstats.demand_metadata_hits); + arl_expect(arl_base, "demand_metadata_misses", &arcstats.demand_metadata_misses); + arl_expect(arl_base, "prefetch_data_hits", &arcstats.prefetch_data_hits); + arl_expect(arl_base, "prefetch_data_misses", &arcstats.prefetch_data_misses); + arl_expect(arl_base, "prefetch_metadata_hits", &arcstats.prefetch_metadata_hits); + arl_expect(arl_base, "prefetch_metadata_misses", &arcstats.prefetch_metadata_misses); + arl_expect(arl_base, "mru_hits", &arcstats.mru_hits); + arl_expect(arl_base, "mru_ghost_hits", &arcstats.mru_ghost_hits); + arl_expect(arl_base, "mfu_hits", &arcstats.mfu_hits); + arl_expect(arl_base, "mfu_ghost_hits", &arcstats.mfu_ghost_hits); + arl_expect(arl_base, "deleted", &arcstats.deleted); + arl_expect(arl_base, "mutex_miss", &arcstats.mutex_miss); + arl_expect(arl_base, "evict_skip", &arcstats.evict_skip); + arl_expect(arl_base, "evict_not_enough", &arcstats.evict_not_enough); + arl_expect(arl_base, "evict_l2_cached", &arcstats.evict_l2_cached); + arl_expect(arl_base, "evict_l2_eligible", &arcstats.evict_l2_eligible); + arl_expect(arl_base, "evict_l2_ineligible", &arcstats.evict_l2_ineligible); + arl_expect(arl_base, "evict_l2_skip", &arcstats.evict_l2_skip); + arl_expect(arl_base, "hash_elements", &arcstats.hash_elements); + arl_expect(arl_base, "hash_elements_max", &arcstats.hash_elements_max); + arl_expect(arl_base, "hash_collisions", &arcstats.hash_collisions); + arl_expect(arl_base, "hash_chains", &arcstats.hash_chains); + arl_expect(arl_base, "hash_chain_max", &arcstats.hash_chain_max); + arl_expect(arl_base, "p", &arcstats.p); + arl_expect(arl_base, "c", &arcstats.c); + arl_expect(arl_base, "c_min", &arcstats.c_min); + arl_expect(arl_base, "c_max", &arcstats.c_max); + arl_expect(arl_base, "size", &arcstats.size); + arl_expect(arl_base, "hdr_size", &arcstats.hdr_size); + arl_expect(arl_base, "data_size", &arcstats.data_size); + arl_expect(arl_base, "metadata_size", &arcstats.metadata_size); + arl_expect(arl_base, "other_size", &arcstats.other_size); + arl_expect(arl_base, "anon_size", &arcstats.anon_size); + arl_expect(arl_base, "anon_evictable_data", &arcstats.anon_evictable_data); + arl_expect(arl_base, "anon_evictable_metadata", &arcstats.anon_evictable_metadata); + arl_expect(arl_base, "mru_size", &arcstats.mru_size); + arl_expect(arl_base, "mru_evictable_data", &arcstats.mru_evictable_data); + arl_expect(arl_base, "mru_evictable_metadata", &arcstats.mru_evictable_metadata); + arl_expect(arl_base, "mru_ghost_size", &arcstats.mru_ghost_size); + arl_expect(arl_base, "mru_ghost_evictable_data", &arcstats.mru_ghost_evictable_data); + arl_expect(arl_base, "mru_ghost_evictable_metadata", &arcstats.mru_ghost_evictable_metadata); + arl_expect(arl_base, "mfu_size", &arcstats.mfu_size); + arl_expect(arl_base, "mfu_evictable_data", &arcstats.mfu_evictable_data); + arl_expect(arl_base, "mfu_evictable_metadata", &arcstats.mfu_evictable_metadata); + arl_expect(arl_base, "mfu_ghost_size", &arcstats.mfu_ghost_size); + arl_expect(arl_base, "mfu_ghost_evictable_data", &arcstats.mfu_ghost_evictable_data); + arl_expect(arl_base, "mfu_ghost_evictable_metadata", &arcstats.mfu_ghost_evictable_metadata); + arl_expect(arl_base, "l2_hits", &arcstats.l2_hits); + arl_expect(arl_base, "l2_misses", &arcstats.l2_misses); + arl_expect(arl_base, "l2_feeds", &arcstats.l2_feeds); + arl_expect(arl_base, "l2_rw_clash", &arcstats.l2_rw_clash); + arl_expect(arl_base, "l2_read_bytes", &arcstats.l2_read_bytes); + arl_expect(arl_base, "l2_write_bytes", &arcstats.l2_write_bytes); + arl_expect(arl_base, "l2_writes_sent", &arcstats.l2_writes_sent); + arl_expect(arl_base, "l2_writes_done", &arcstats.l2_writes_done); + arl_expect(arl_base, "l2_writes_error", &arcstats.l2_writes_error); + arl_expect(arl_base, "l2_writes_lock_retry", &arcstats.l2_writes_lock_retry); + arl_expect(arl_base, "l2_evict_lock_retry", &arcstats.l2_evict_lock_retry); + arl_expect(arl_base, "l2_evict_reading", &arcstats.l2_evict_reading); + arl_expect(arl_base, "l2_evict_l1cached", &arcstats.l2_evict_l1cached); + arl_expect(arl_base, "l2_free_on_write", &arcstats.l2_free_on_write); + arl_expect(arl_base, "l2_cdata_free_on_write", &arcstats.l2_cdata_free_on_write); + arl_expect(arl_base, "l2_abort_lowmem", &arcstats.l2_abort_lowmem); + arl_expect(arl_base, "l2_cksum_bad", &arcstats.l2_cksum_bad); + arl_expect(arl_base, "l2_io_error", &arcstats.l2_io_error); + arl_expect(arl_base, "l2_size", &arcstats.l2_size); + arl_expect(arl_base, "l2_asize", &arcstats.l2_asize); + arl_expect(arl_base, "l2_hdr_size", &arcstats.l2_hdr_size); + arl_expect(arl_base, "l2_compress_successes", &arcstats.l2_compress_successes); + arl_expect(arl_base, "l2_compress_zeros", &arcstats.l2_compress_zeros); + arl_expect(arl_base, "l2_compress_failures", &arcstats.l2_compress_failures); + arl_expect(arl_base, "memory_throttle_count", &arcstats.memory_throttle_count); + arl_expect(arl_base, "duplicate_buffers", &arcstats.duplicate_buffers); + arl_expect(arl_base, "duplicate_buffers_size", &arcstats.duplicate_buffers_size); + arl_expect(arl_base, "duplicate_reads", &arcstats.duplicate_reads); + arl_expect(arl_base, "memory_direct_count", &arcstats.memory_direct_count); + arl_expect(arl_base, "memory_indirect_count", &arcstats.memory_indirect_count); + arl_expect(arl_base, "arc_no_grow", &arcstats.arc_no_grow); + arl_expect(arl_base, "arc_tempreserve", &arcstats.arc_tempreserve); + arl_expect(arl_base, "arc_loaned_bytes", &arcstats.arc_loaned_bytes); + arl_expect(arl_base, "arc_prune", &arcstats.arc_prune); + arl_expect(arl_base, "arc_meta_used", &arcstats.arc_meta_used); + arl_expect(arl_base, "arc_meta_limit", &arcstats.arc_meta_limit); + arl_expect(arl_base, "arc_meta_max", &arcstats.arc_meta_max); + arl_expect(arl_base, "arc_meta_min", &arcstats.arc_meta_min); + arl_expect(arl_base, "arc_need_free", &arcstats.arc_need_free); + arl_expect(arl_base, "arc_sys_free", &arcstats.arc_sys_free); + } + + if(unlikely(!ff)) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, ZFS_PROC_ARCSTATS); + ff = procfile_open(config_get("plugin:proc:" ZFS_PROC_ARCSTATS, "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) + return 1; + } + + ff = procfile_readall(ff); + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time + + size_t lines = procfile_lines(ff), l; + + arl_begin(arl_base); + + for(l = 0; l < lines ;l++) { + size_t words = procfile_linewords(ff, l); + if(unlikely(words < 3)) { + if(unlikely(words)) error("Cannot read " ZFS_PROC_ARCSTATS " line %zu. Expected 3 params, read %zu.", l, words); + continue; + } + + const char *key = procfile_lineword(ff, l, 0); + const char *value = procfile_lineword(ff, l, 2); + + if(unlikely(l2exist == -1)) { + if(key[0] == 'l' && key[1] == '2' && key[2] == '_') + l2exist = 1; + } + + if(unlikely(arl_check(arl_base, key, value))) break; + } + + if(unlikely(l2exist == -1)) + l2exist = 0; + + generate_charts_arcstats(update_every); + generate_charts_arc_summary(update_every); + + return 0; +} diff --git a/src/proc_vmstat.c b/src/proc_vmstat.c index 847487363..a2416313a 100644 --- a/src/proc_vmstat.c +++ b/src/proc_vmstat.c @@ -169,11 +169,11 @@ int do_proc_vmstat(int update_every, usec_t dt) { // The following stats depend on CONFIG_NUMA_BALANCING in the // kernel. - rrddim_add(st_numa, "pte updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st_numa, "huge pte updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st_numa, "hint faults", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st_numa, "hint faults local", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rrddim_add(st_numa, "pages migrated", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st_numa, "pte_updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st_numa, "huge_pte_updates", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st_numa, "hint_faults", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st_numa, "hint_faults_local", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(st_numa, "pages_migrated", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st_numa); @@ -182,11 +182,11 @@ int do_proc_vmstat(int update_every, usec_t dt) { rrddim_set(st_numa, "interleave", numa_interleave); rrddim_set(st_numa, "other", numa_other); - rrddim_set(st_numa, "pte updates", numa_pte_updates); - rrddim_set(st_numa, "huge pte updates", numa_huge_pte_updates); - rrddim_set(st_numa, "hint faults", numa_hint_faults); - rrddim_set(st_numa, "hint faults local", numa_hint_faults_local); - rrddim_set(st_numa, "pages migrated", numa_pages_migrated); + rrddim_set(st_numa, "pte_updates", numa_pte_updates); + rrddim_set(st_numa, "huge_pte_updates", numa_huge_pte_updates); + rrddim_set(st_numa, "hint_faults", numa_hint_faults); + rrddim_set(st_numa, "hint_faults_local", numa_hint_faults_local); + rrddim_set(st_numa, "pages_migrated", numa_pages_migrated); rrdset_done(st_numa); } diff --git a/src/registry.c b/src/registry.c index ed9be9848..76e3fa4d2 100644 --- a/src/registry.c +++ b/src/registry.c @@ -45,7 +45,7 @@ static inline void registry_json_header(RRDHOST *host, struct web_client *w, con buffer_flush(w->response.data); w->response.data->contenttype = CT_APPLICATION_JSON; buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"", - action, status, (host == localhost)?registry.hostname:host->hostname, host->machine_guid); + action, status, host->registry_hostname, host->machine_guid); } static inline void registry_json_footer(struct web_client *w) { diff --git a/src/registry.h b/src/registry.h index 2c4592b92..9aa241564 100644 --- a/src/registry.h +++ b/src/registry.h @@ -70,6 +70,8 @@ extern int registry_request_hello_json(RRDHOST *host, struct web_client *w); extern void registry_statistics(void); extern char *registry_get_this_machine_guid(void); +extern char *registry_get_this_machine_hostname(void); + extern int regenerate_guid(const char *guid, char *result); #endif /* NETDATA_REGISTRY_H */ diff --git a/src/registry_init.c b/src/registry_init.c index 2a41d36ee..654f66d12 100644 --- a/src/registry_init.c +++ b/src/registry_init.c @@ -34,7 +34,7 @@ int registry_init(void) { registry.persons_expiration = config_get_number(CONFIG_SECTION_REGISTRY, "registry expire idle persons days", 365) * 86400; registry.registry_domain = config_get(CONFIG_SECTION_REGISTRY, "registry domain", ""); registry.registry_to_announce = config_get(CONFIG_SECTION_REGISTRY, "registry to announce", "https://registry.my-netdata.io"); - registry.hostname = config_get(CONFIG_SECTION_REGISTRY, "registry hostname", config_get(CONFIG_SECTION_GLOBAL, "hostname", "localhost")); + registry.hostname = config_get(CONFIG_SECTION_REGISTRY, "registry hostname", netdata_configured_hostname); registry.verify_cookies_redirects = config_get_boolean(CONFIG_SECTION_REGISTRY, "verify browser cookies support", 1); setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1); diff --git a/src/registry_internals.c b/src/registry_internals.c index 9ec91ba40..fd3c295ce 100644 --- a/src/registry_internals.c +++ b/src/registry_internals.c @@ -274,6 +274,10 @@ static inline int is_machine_guid_blacklisted(const char *guid) { return 0; } +char *registry_get_this_machine_hostname(void) { + return registry.hostname; +} + char *registry_get_this_machine_guid(void) { static char guid[GUID_LEN + 1] = ""; @@ -31,19 +31,28 @@ inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) { return RRD_MEMORY_MODE_NONE_NAME; case RRD_MEMORY_MODE_SAVE: - default: return RRD_MEMORY_MODE_SAVE_NAME; + + case RRD_MEMORY_MODE_ALLOC: + return RRD_MEMORY_MODE_ALLOC_NAME; } + + return RRD_MEMORY_MODE_SAVE_NAME; } RRD_MEMORY_MODE rrd_memory_mode_id(const char *name) { if(unlikely(!strcmp(name, RRD_MEMORY_MODE_RAM_NAME))) return RRD_MEMORY_MODE_RAM; + else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_MAP_NAME))) return RRD_MEMORY_MODE_MAP; + else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_NONE_NAME))) return RRD_MEMORY_MODE_NONE; + else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_ALLOC_NAME))) + return RRD_MEMORY_MODE_ALLOC; + return RRD_MEMORY_MODE_SAVE; } @@ -5,7 +5,7 @@ #define UPDATE_EVERY_MAX 3600 #define RRD_DEFAULT_HISTORY_ENTRIES 3600 -#define RRD_HISTORY_ENTRIES_MAX (86400*10) +#define RRD_HISTORY_ENTRIES_MAX (86400*365) extern int default_rrd_update_every; extern int default_rrd_history_entries; @@ -42,13 +42,15 @@ typedef enum rrd_memory_mode { RRD_MEMORY_MODE_NONE = 0, RRD_MEMORY_MODE_RAM = 1, RRD_MEMORY_MODE_MAP = 2, - RRD_MEMORY_MODE_SAVE = 3 + RRD_MEMORY_MODE_SAVE = 3, + RRD_MEMORY_MODE_ALLOC = 4 } RRD_MEMORY_MODE; #define RRD_MEMORY_MODE_NONE_NAME "none" #define RRD_MEMORY_MODE_RAM_NAME "ram" #define RRD_MEMORY_MODE_MAP_NAME "map" #define RRD_MEMORY_MODE_SAVE_NAME "save" +#define RRD_MEMORY_MODE_ALLOC_NAME "alloc" extern RRD_MEMORY_MODE default_rrd_memory_mode; @@ -101,9 +103,15 @@ typedef enum rrddim_flags { RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = 1 << 1 // do not offer RESET or OVERFLOW info to callers } RRDDIM_FLAGS; +#ifdef HAVE_C___ATOMIC +#define rrddim_flag_check(rd, flag) (__atomic_load_n(&((rd)->flags), __ATOMIC_SEQ_CST) & flag) +#define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), flag, __ATOMIC_SEQ_CST) +#define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~flag, __ATOMIC_SEQ_CST) +#else #define rrddim_flag_check(rd, flag) ((rd)->flags & flag) #define rrddim_flag_set(rd, flag) (rd)->flags |= flag #define rrddim_flag_clear(rd, flag) (rd)->flags &= ~flag +#endif // ---------------------------------------------------------------------------- @@ -210,16 +218,28 @@ typedef struct rrddim RRDDIM; // and may lead to missing information. typedef enum rrdset_flags { - RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart - RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another - // (the master data set should be the one that has the same family and is not detail) - RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart - RRDSET_FLAG_OBSOLETE = 1 << 3 // this is marked by the collector/module as obsolete + RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart + RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another + // (the master data set should be the one that has the same family and is not detail) + RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart + RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete + RRDSET_FLAG_BACKEND_SEND = 1 << 4, // if set, this chart should be sent to backends + RRDSET_FLAG_BACKEND_IGNORE = 1 << 5, // if set, this chart should not be sent to backends + RRDSET_FLAG_EXPOSED_UPSTREAM = 1 << 6, // if set, we have sent this chart to netdata master (streaming) + RRDSET_FLAG_STORE_FIRST = 1 << 7, // if set, do not eliminate the first collection during interpolation + RRDSET_FLAG_HETEROGENEOUS = 1 << 8, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) + RRDSET_FLAG_HOMEGENEOUS_CHECK= 1 << 9 // if set, the chart should be checked to determine if the dimensions as homogeneous } RRDSET_FLAGS; +#ifdef HAVE_C___ATOMIC +#define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_SEQ_CST) & flag) +#define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_SEQ_CST) +#define rrdset_flag_clear(st, flag) __atomic_and_fetch(&((st)->flags), ~flag, __ATOMIC_SEQ_CST) +#else #define rrdset_flag_check(st, flag) ((st)->flags & flag) #define rrdset_flag_set(st, flag) (st)->flags |= flag #define rrdset_flag_clear(st, flag) (st)->flags &= ~flag +#endif struct rrdset { // ------------------------------------------------------------------------ @@ -279,7 +299,8 @@ struct rrdset { size_t counter_done; // the number of times rrdset_done() has been called time_t last_accessed_time; // the last time this RRDSET has been accessed - size_t unused[9]; + time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream + size_t unused[8]; uint32_t hash; // a simple hash on the id, to speed up searching // we first compare hashes, and only if the hashes are equal we do string comparisons @@ -347,14 +368,28 @@ typedef struct rrdset RRDSET; // and may lead to missing information. typedef enum rrdhost_flags { - RRDHOST_ORPHAN = 1 << 0, // this host is orphan - RRDHOST_DELETE_OBSOLETE_FILES = 1 << 1, // delete files of obsolete charts - RRDHOST_DELETE_ORPHAN_FILES = 1 << 2 // delete the entire host when orphan + RRDHOST_ORPHAN = 1 << 0, // this host is orphan (not receiving data) + RRDHOST_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts + RRDHOST_DELETE_ORPHAN_HOST = 1 << 2 // delete the entire host when orphan } RRDHOST_FLAGS; +#ifdef HAVE_C___ATOMIC +#define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & flag) +#define rrdhost_flag_set(host, flag) __atomic_or_fetch(&((host)->flags), flag, __ATOMIC_SEQ_CST) +#define rrdhost_flag_clear(host, flag) __atomic_and_fetch(&((host)->flags), ~flag, __ATOMIC_SEQ_CST) +#else #define rrdhost_flag_check(host, flag) ((host)->flags & flag) #define rrdhost_flag_set(host, flag) (host)->flags |= flag #define rrdhost_flag_clear(host, flag) (host)->flags &= ~flag +#endif + +#ifdef NETDATA_INTERNAL_CHECKS +#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \ + debug_int(__FILE__, __FUNCTION__, __LINE__, "%s: " fmt, st->name, ##args); } while(0) +#else +#define rrdset_debug(st, fmt, args...) debug_dummy() +#endif + // ---------------------------------------------------------------------------- // RRD HOST @@ -368,10 +403,13 @@ struct rrdhost { char *hostname; // the hostname of this host uint32_t hash_hostname; // the hostname hash + char *registry_hostname; // the registry hostname for this host + char machine_guid[GUID_LEN + 1]; // the unique ID of this host uint32_t hash_machine_guid; // the hash of the unique ID - char *os; // the O/S type of the host + const char *os; // the O/S type of the host + const char *tags; // tags for this host uint32_t flags; // flags about this RRDHOST @@ -488,8 +526,10 @@ extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash); extern RRDHOST *rrdhost_find_or_create( const char *hostname + , const char *registry_hostname , const char *guid , const char *os + , const char *tags , int update_every , long history , RRD_MEMORY_MODE mode @@ -528,7 +568,7 @@ extern void __rrd_check_wrlock(const char *file, const char *function, const uns extern void rrdset_set_name(RRDSET *st, const char *name); -extern RRDSET *rrdset_create(RRDHOST *host +extern RRDSET *rrdset_create_custom(RRDHOST *host , const char *type , const char *id , const char *name @@ -538,18 +578,27 @@ extern RRDSET *rrdset_create(RRDHOST *host , const char *units , long priority , int update_every - , RRDSET_TYPE chart_type); + , RRDSET_TYPE chart_type + , RRD_MEMORY_MODE memory_mode + , long history_entries); + +#define rrdset_create(host, type, id, name, family, context, title, units, priority, update_every, chart_type) \ + rrdset_create_custom(host, type, id, name, family, context, title, units, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries) -#define rrdset_create_localhost(type, id, name, family, context, title, units, priority, update_every, chart_type) rrdset_create(localhost, type, id, name, family, context, title, units, priority, update_every, chart_type) +#define rrdset_create_localhost(type, id, name, family, context, title, units, priority, update_every, chart_type) \ + rrdset_create(localhost, type, id, name, family, context, title, units, priority, update_every, chart_type) extern void rrdhost_free_all(void); extern void rrdhost_save_all(void); +extern void rrdhost_cleanup_all(void); -extern void rrdhost_cleanup_orphan(RRDHOST *protected); +extern void rrdhost_cleanup_orphan_hosts(RRDHOST *protected); extern void rrdhost_free(RRDHOST *host); extern void rrdhost_save(RRDHOST *host); extern void rrdhost_delete(RRDHOST *host); +extern void rrdset_update_heterogeneous_flag(RRDSET *st); + extern RRDSET *rrdset_find(RRDHOST *host, const char *id); #define rrdset_find_localhost(id) rrdset_find(localhost, id) @@ -565,8 +614,12 @@ extern void rrdset_next_usec(RRDSET *st, usec_t microseconds); extern void rrdset_done(RRDSET *st); +extern void rrdset_is_obsolete(RRDSET *st); +extern void rrdset_isnot_obsolete(RRDSET *st); + // checks if the RRDSET should be offered to viewers -#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions) +#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE) +#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions) // get the total duration in seconds of the round robin database #define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every )) @@ -604,9 +657,14 @@ extern void rrdset_done(RRDSET *st); // ---------------------------------------------------------------------------- // RRD DIMENSION functions -extern RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm); +extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode); +#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode) + +extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name); +extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm); +extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier); +extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor); -extern void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name); extern RRDDIM *rrddim_find(RRDSET *st, const char *id); extern int rrddim_hide(RRDSET *st, const char *id); @@ -647,7 +705,7 @@ extern void rrdset_reset(RRDSET *st); extern void rrdset_save(RRDSET *st); extern void rrdset_delete(RRDSET *st); -extern void rrdhost_cleanup_obsolete(RRDHOST *host); +extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host); #endif /* NETDATA_RRD_INTERNALS */ diff --git a/src/rrd2json.c b/src/rrd2json.c index 4d853930c..98080139c 100644 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -11,7 +11,7 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions "\t\t\t\"type\": \"%s\",\n" "\t\t\t\"family\": \"%s\",\n" "\t\t\t\"context\": \"%s\",\n" - "\t\t\t\"title\": \"%s\",\n" + "\t\t\t\"title\": \"%s (%s)\",\n" "\t\t\t\"priority\": %ld,\n" "\t\t\t\"enabled\": %s,\n" "\t\t\t\"units\": \"%s\",\n" @@ -27,7 +27,7 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions , st->type , st->family , st->context - , st->title + , st->title, st->name , st->priority , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?"true":"false" , st->units @@ -167,79 +167,6 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) { } // ---------------------------------------------------------------------------- -// PROMETHEUS -// /api/v1/allmetrics?format=prometheus - -static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) { - size_t n; - - for(n = 0; *s && n < usable ; d++, s++, n++) { - register char c = *s; - - if(unlikely(!isalnum(c))) *d = '_'; - else *d = c; - } - *d = '\0'; - - return n; -} - -#define PROMETHEUS_ELEMENT_MAX 256 - -void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb) { - rrdhost_rdlock(host); - - char hostname[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_name_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX); - - // for each chart - RRDSET *st; - rrdset_foreach_read(st, host) { - char chart[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_name_copy(chart, st->id, PROMETHEUS_ELEMENT_MAX); - - buffer_strcat(wb, "\n"); - if(rrdset_is_available_for_viewers(st)) { - rrdset_rdlock(st); - - // for each dimension - RRDDIM *rd; - rrddim_foreach_read(rd, st) { - if(rd->collections_counter) { - char dimension[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_name_copy(dimension, rd->id, PROMETHEUS_ELEMENT_MAX); - - // buffer_sprintf(wb, "# HELP %s.%s %s\n", st->id, rd->id, st->units); - - switch(rd->algorithm) { - case RRD_ALGORITHM_INCREMENTAL: - case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: - buffer_sprintf(wb, "# TYPE %s_%s counter\n", chart, dimension); - break; - - default: - buffer_sprintf(wb, "# TYPE %s_%s gauge\n", chart, dimension); - break; - } - - // calculated_number n = (calculated_number)rd->last_collected_value * (calculated_number)(abs(rd->multiplier)) / (calculated_number)(abs(rd->divisor)); - // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n, timeval_msec(&rd->last_collected_time)); - - buffer_sprintf(wb, "%s_%s{instance=\"%s\"} " COLLECTED_NUMBER_FORMAT " %llu\n", - chart, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time) - ); - - } - } - - rrdset_unlock(st); - } - } - - rrdhost_unlock(host); -} - -// ---------------------------------------------------------------------------- // BASH // /api/v1/allmetrics?format=bash @@ -267,7 +194,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) { rrdset_foreach_read(st, host) { calculated_number total = 0.0; char chart[SHELL_ELEMENT_MAX + 1]; - shell_name_copy(chart, st->id, SHELL_ELEMENT_MAX); + shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX); buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name); if(rrdset_is_available_for_viewers(st)) { @@ -278,7 +205,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) { rrddim_foreach_read(rd, st) { if(rd->collections_counter) { char dimension[SHELL_ELEMENT_MAX + 1]; - shell_name_copy(dimension, rd->id, SHELL_ELEMENT_MAX); + shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX); calculated_number n = rd->last_stored_value; @@ -306,7 +233,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) { if(!rc->rrdset) continue; char chart[SHELL_ELEMENT_MAX + 1]; - shell_name_copy(chart, rc->rrdset->id, SHELL_ELEMENT_MAX); + shell_name_copy(chart, rc->rrdset->name?rc->rrdset->name:rc->rrdset->id, SHELL_ELEMENT_MAX); char alarm[SHELL_ELEMENT_MAX + 1]; shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX); diff --git a/src/rrd2json.h b/src/rrd2json.h index f2f03c640..a3b9c9509 100644 --- a/src/rrd2json.h +++ b/src/rrd2json.h @@ -31,13 +31,15 @@ #define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma" #define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray" -#define ALLMETRICS_FORMAT_SHELL "shell" -#define ALLMETRICS_FORMAT_PROMETHEUS "prometheus" -#define ALLMETRICS_FORMAT_JSON "json" +#define ALLMETRICS_FORMAT_SHELL "shell" +#define ALLMETRICS_FORMAT_PROMETHEUS "prometheus" +#define ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS "prometheus_all_hosts" +#define ALLMETRICS_FORMAT_JSON "json" -#define ALLMETRICS_SHELL 1 -#define ALLMETRICS_PROMETHEUS 2 -#define ALLMETRICS_JSON 3 +#define ALLMETRICS_SHELL 1 +#define ALLMETRICS_PROMETHEUS 2 +#define ALLMETRICS_JSON 3 +#define ALLMETRICS_PROMETHEUS_ALL_HOSTS 4 #define GROUP_UNDEFINED 0 #define GROUP_AVERAGE 1 @@ -65,13 +67,13 @@ extern void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb); extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb); extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb); -extern void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb); extern int rrdset2anything_api_v1(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points - , long long after, long long before, int group_method, uint32_t options - , time_t *latest_timestamp); + , long long after, long long before, int group_method, uint32_t options + , time_t *latest_timestamp); + extern int rrdset2value_api_v1(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points - , long long after, long long before, int group_method, uint32_t options - , time_t *db_before, time_t *db_after, int *value_is_null); + , long long after, long long before, int group_method, uint32_t options + , time_t *db_before, time_t *db_after, int *value_is_null); #endif /* NETDATA_RRD2JSON_H */ diff --git a/src/rrd2json_api_old.c b/src/rrd2json_api_old.c index 6710f31cf..003b8626d 100644 --- a/src/rrd2json_api_old.c +++ b/src/rrd2json_api_old.c @@ -14,7 +14,7 @@ unsigned long rrdset_info2json_api_old(RRDSET *st, char *options, BUFFER *wb) { "\t\t\t\"type\": \"%s\",\n" "\t\t\t\"family\": \"%s\",\n" "\t\t\t\"context\": \"%s\",\n" - "\t\t\t\"title\": \"%s\",\n" + "\t\t\t\"title\": \"%s (%s)\",\n" "\t\t\t\"priority\": %ld,\n" "\t\t\t\"enabled\": %d,\n" "\t\t\t\"units\": \"%s\",\n" @@ -37,7 +37,7 @@ unsigned long rrdset_info2json_api_old(RRDSET *st, char *options, BUFFER *wb) { , st->type , st->family , st->context - , st->title + , st->title, st->name , st->priority , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?1:0 , st->units diff --git a/src/rrdcalc.c b/src/rrdcalc.c index 1f1845409..bb90a4c6d 100644 --- a/src/rrdcalc.c +++ b/src/rrdcalc.c @@ -219,7 +219,7 @@ inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, ui for(rc = host->alarms; rc ; rc = rc->next) { if (unlikely(rc->chart && rc->hash == hash_name && rc->hash_chart == hash_chart && !strcmp(name, rc->name) && !strcmp(chart, rc->chart))) { debug(D_HEALTH, "Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname); - error("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname); + info("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname); return 1; } } diff --git a/src/rrdcalctemplate.c b/src/rrdcalctemplate.c index 2c5e2bd16..b5d2c7d61 100644 --- a/src/rrdcalctemplate.c +++ b/src/rrdcalctemplate.c @@ -12,7 +12,7 @@ void rrdcalctemplate_link_matching(RRDSET *st) { && (!rt->family_pattern || simple_pattern_matches(rt->family_pattern, st->family))) { RRDCALC *rc = rrdcalc_create(st->rrdhost, rt, st->id); if(unlikely(!rc)) - error("Health tried to create alarm from template '%s', but it failed", rt->name); + info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, st->rrdhost->hostname); #ifdef NETDATA_INTERNAL_CHECKS else if(rc->rrdset != st) diff --git a/src/rrddim.c b/src/rrddim.c index 54a17522f..e75aa3fd2 100644 --- a/src/rrddim.c +++ b/src/rrddim.c @@ -35,9 +35,9 @@ inline RRDDIM *rrddim_find(RRDSET *st, const char *id) { // ---------------------------------------------------------------------------- // RRDDIM rename a dimension -inline void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) { - if(unlikely(!strcmp(rd->name, name))) - return; +inline int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) { + if(unlikely(!name || !*name || !strcmp(rd->name, name))) + return 0; debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", st->name, rd->name, st->name, name); @@ -45,18 +45,57 @@ inline void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) { snprintfz(varname, CONFIG_MAX_NAME, "dim %s name", rd->id); rd->name = config_set_default(st->config_section, varname, name); rd->hash_name = simple_hash(rd->name); - rrddimvar_rename_all(rd); + rd->exposed = 0; + return 1; +} + +inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm) { + if(unlikely(rd->algorithm == algorithm)) + return 0; + + debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", st->id, rd->name, rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm)); + rd->algorithm = algorithm; + rd->exposed = 0; + rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK); + return 1; +} + +inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier) { + if(unlikely(rd->multiplier == multiplier)) + return 0; + + debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->multiplier, multiplier); + rd->multiplier = multiplier; + rd->exposed = 0; + rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK); + return 1; } +inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) { + if(unlikely(rd->divisor == divisor)) + return 0; + + debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, st->id, rd->name, rd->divisor, divisor); + rd->divisor = divisor; + rd->exposed = 0; + rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK); + return 1; +} // ---------------------------------------------------------------------------- // RRDDIM create a dimension -RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm) { +RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) { RRDDIM *rd = rrddim_find(st, id); if(unlikely(rd)) { debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>"); + + rrddim_set_name(st, rd, name); + rrddim_set_algorithm(st, rd, algorithm); + rrddim_set_multiplier(st, rd, multiplier); + rrddim_set_divisor(st, rd, divisor); + return rd; } @@ -71,8 +110,14 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe rrdset_strncpyz_name(filename, id, FILENAME_MAX); snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename); - if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) { - rd = (RRDDIM *)mymmap(fullfilename, size, ((st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1); + if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_RAM) { + rd = (RRDDIM *)mymmap( + (memory_mode == RRD_MEMORY_MODE_RAM)?NULL:fullfilename + , size + , ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE) + , 1 + ); + if(likely(rd)) { // we have a file mapped for rd @@ -83,56 +128,64 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe rd->variables = NULL; rd->next = NULL; rd->rrdset = NULL; + rd->exposed = 0; struct timeval now; now_realtime_timeval(&now); - if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) { - errno = 0; - info("Initializing file %s.", fullfilename); + if(memory_mode == RRD_MEMORY_MODE_RAM) { memset(rd, 0, size); } - else if(rd->memsize != size) { - errno = 0; - error("File %s does not have the desired size. Clearing it.", fullfilename); - memset(rd, 0, size); - } - else if(rd->multiplier != multiplier) { - errno = 0; - error("File %s does not have the same multiplier. Clearing it.", fullfilename); - memset(rd, 0, size); - } - else if(rd->divisor != divisor) { - errno = 0; - error("File %s does not have the same divisor. Clearing it.", fullfilename); - memset(rd, 0, size); - } - else if(rd->update_every != st->update_every) { - errno = 0; - error("File %s does not have the same refresh frequency. Clearing it.", fullfilename); - memset(rd, 0, size); - } - else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) { - errno = 0; - error("File %s is too old. Clearing it.", fullfilename); - memset(rd, 0, size); + else { + int reset = 0; + + if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) { + info("Initializing file %s.", fullfilename); + memset(rd, 0, size); + reset = 1; + } + else if(rd->memsize != size) { + error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd->memsize); + memset(rd, 0, size); + reset = 1; + } + else if(rd->update_every != st->update_every) { + error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd->update_every); + memset(rd, 0, size); + reset = 1; + } + else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) { + error("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd->last_collected_time) / USEC_PER_SEC, rd->entries * rd->update_every); + memset(rd, 0, size); + reset = 1; + } + + if(!reset) { + if(rd->algorithm != algorithm) { + info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.", + fullfilename, algorithm, rrd_algorithm_name(algorithm), rd->algorithm, rrd_algorithm_name(rd->algorithm)); + } + + if(rd->multiplier != multiplier) { + info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT ". Previous values may be wrong.", fullfilename, multiplier, rd->multiplier); + } + + if(rd->divisor != divisor) { + info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT ". Previous values may be wrong.", fullfilename, divisor, rd->divisor); + } + } } - if(rd->algorithm && rd->algorithm != algorithm) - error("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong." - , fullfilename, algorithm, rrd_algorithm_name(algorithm), rd->algorithm, - rrd_algorithm_name(rd->algorithm)); - // make sure we have the right memory mode // even if we cleared the memory - rd->rrd_memory_mode = st->rrd_memory_mode; + rd->rrd_memory_mode = memory_mode; } } if(unlikely(!rd)) { // if we didn't manage to get a mmap'd dimension, just create one rd = callocz(1, size); - rd->rrd_memory_mode = (st->rrd_memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_RAM; + rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC; } rd->memsize = size; @@ -161,8 +214,11 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe rd->entries = st->entries; rd->update_every = st->update_every; - // prevent incremental calculation spikes - rd->collections_counter = 0; + if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)) + rd->collections_counter = 1; + else + rd->collections_counter = 0; + rd->updated = 0; rd->flags = 0x00000000; @@ -173,7 +229,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe rd->collected_volume = 0; rd->stored_volume = 0; rd->last_stored_value = 0; - rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS); + rd->values[st->current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS); rd->last_collected_time.tv_sec = 0; rd->last_collected_time.tv_usec = 0; rd->rrdset = st; @@ -184,6 +240,23 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe st->dimensions = rd; else { RRDDIM *td = st->dimensions; + + if(td->algorithm != rd->algorithm || abs(td->multiplier) != abs(rd->multiplier) || abs(td->divisor) != abs(rd->divisor)) { + if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { + #ifdef NETDATA_INTERNAL_CHECKS + info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", + rd->name, + st->name, + st->rrdhost->hostname, + rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(td->algorithm), + rd->multiplier, td->multiplier, + rd->divisor, td->divisor + ); + #endif + rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS); + } + } + for(; td->next; td = td->next) ; td->next = rd; } @@ -202,7 +275,6 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe return(rd); } - // ---------------------------------------------------------------------------- // RRDDIM remove / free a dimension @@ -233,19 +305,16 @@ void rrddim_free(RRDSET *st, RRDDIM *rd) switch(rd->rrd_memory_mode) { case RRD_MEMORY_MODE_SAVE: - debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename); - savememory(rd->cache_filename, rd, rd->memsize); - // continue to map mode - no break; - case RRD_MEMORY_MODE_MAP: + case RRD_MEMORY_MODE_RAM: debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name); freez((void *)rd->id); freez(rd->cache_filename); munmap(rd, rd->memsize); break; + case RRD_MEMORY_MODE_ALLOC: case RRD_MEMORY_MODE_NONE: - case RRD_MEMORY_MODE_RAM: debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name); freez((void *)rd->id); freez(rd->cache_filename); @@ -263,7 +332,7 @@ int rrddim_hide(RRDSET *st, const char *id) { RRDDIM *rd = rrddim_find(st, id); if(unlikely(!rd)) { - error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id); + error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, st->rrdhost->hostname); return 1; } @@ -276,7 +345,7 @@ int rrddim_unhide(RRDSET *st, const char *id) { RRDDIM *rd = rrddim_find(st, id); if(unlikely(!rd)) { - error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id); + error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, st->rrdhost->hostname); return 1; } @@ -305,7 +374,7 @@ inline collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) { RRDDIM *rd = rrddim_find(st, id); if(unlikely(!rd)) { - error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id); + error("Cannot find dimension with id '%s' on stats '%s' (%s) on host '%s'.", id, st->name, st->id, st->rrdhost->hostname); return 0; } diff --git a/src/rrdhost.c b/src/rrdhost.c index a2310330d..ff8aa5617 100644 --- a/src/rrdhost.c +++ b/src/rrdhost.c @@ -58,15 +58,32 @@ RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash) { // ---------------------------------------------------------------------------- // RRDHOST - internal helpers +static inline void rrdhost_init_tags(RRDHOST *host, const char *tags) { + if(host->tags && tags && !strcmp(host->tags, tags)) + return; + + void *old = (void *)host->tags; + host->tags = (tags && *tags)?strdupz(tags):NULL; + freez(old); +} + static inline void rrdhost_init_hostname(RRDHOST *host, const char *hostname) { - freez(host->hostname); - host->hostname = strdupz(hostname); + if(host->hostname && hostname && !strcmp(host->hostname, hostname)) + return; + + void *old = host->hostname; + host->hostname = strdupz(hostname?hostname:"localhost"); host->hash_hostname = simple_hash(host->hostname); + freez(old); } static inline void rrdhost_init_os(RRDHOST *host, const char *os) { - freez(host->os); + if(host->os && os && !strcmp(host->os, os)) + return; + + void *old = (void *)host->os; host->os = strdupz(os?os:"unknown"); + freez(old); } static inline void rrdhost_init_machine_guid(RRDHOST *host, const char *machine_guid) { @@ -80,16 +97,18 @@ static inline void rrdhost_init_machine_guid(RRDHOST *host, const char *machine_ // RRDHOST - add a host RRDHOST *rrdhost_create(const char *hostname, - const char *guid, - const char *os, - int update_every, - long entries, - RRD_MEMORY_MODE memory_mode, - int health_enabled, - int rrdpush_enabled, - char *rrdpush_destination, - char *rrdpush_api_key, - int is_localhost + const char *registry_hostname, + const char *guid, + const char *os, + const char *tags, + int update_every, + long entries, + RRD_MEMORY_MODE memory_mode, + int health_enabled, + int rrdpush_enabled, + char *rrdpush_destination, + char *rrdpush_api_key, + int is_localhost ) { debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid); @@ -115,6 +134,8 @@ RRDHOST *rrdhost_create(const char *hostname, rrdhost_init_hostname(host, hostname); rrdhost_init_machine_guid(host, guid); rrdhost_init_os(host, os); + rrdhost_init_tags(host, tags); + host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname); avl_init_lock(&(host->rrdset_root_index), rrdset_compare); avl_init_lock(&(host->rrdset_root_index_name), rrdset_compare_name); @@ -122,10 +143,10 @@ RRDHOST *rrdhost_create(const char *hostname, avl_init_lock(&(host->variables_root_index), rrdvar_compare); if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1)) - rrdhost_flag_set(host, RRDHOST_DELETE_OBSOLETE_FILES); + rrdhost_flag_set(host, RRDHOST_DELETE_OBSOLETE_CHARTS); if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost) - rrdhost_flag_set(host, RRDHOST_DELETE_ORPHAN_FILES); + rrdhost_flag_set(host, RRDHOST_DELETE_ORPHAN_HOST); // ------------------------------------------------------------------------ @@ -229,8 +250,9 @@ RRDHOST *rrdhost_create(const char *hostname, host = NULL; } else { - info("Host '%s' with guid '%s' initialized" + info("Host '%s' (at registry as '%s') with guid '%s' initialized" ", os %s" + ", tags '%s'" ", update every %d" ", memory mode %s" ", history entries %ld" @@ -243,8 +265,10 @@ RRDHOST *rrdhost_create(const char *hostname, ", alarms default handler '%s'" ", alarms default recipient '%s'" , host->hostname + , host->registry_hostname , host->machine_guid , host->os + , (host->tags)?host->tags:"" , host->rrd_update_every , rrd_memory_mode_name(host->rrd_memory_mode) , host->rrd_history_entries @@ -267,8 +291,10 @@ RRDHOST *rrdhost_create(const char *hostname, RRDHOST *rrdhost_find_or_create( const char *hostname + , const char *registry_hostname , const char *guid , const char *os + , const char *tags , int update_every , long history , RRD_MEMORY_MODE mode @@ -284,8 +310,10 @@ RRDHOST *rrdhost_find_or_create( if(!host) { host = rrdhost_create( hostname + , registry_hostname , guid , os + , tags , update_every , history , mode @@ -307,22 +335,25 @@ RRDHOST *rrdhost_find_or_create( } if(host->rrd_update_every != update_every) - error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds.", host->hostname, host->rrd_update_every, update_every); + error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every); - if(host->rrd_history_entries != history) - error("Host '%s' has history of %ld entries, but the wanted one is %ld entries.", host->hostname, host->rrd_history_entries, history); + if(host->rrd_history_entries < history) + error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history); if(host->rrd_memory_mode != mode) - error("Host '%s' has memory mode '%s', but the wanted one is '%s'.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); + error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); + + // update host tags + rrdhost_init_tags(host, tags); } rrd_unlock(); - rrdhost_cleanup_orphan(host); + rrdhost_cleanup_orphan_hosts(host); return host; } -static inline int rrdhost_should_be_deleted(RRDHOST *host, RRDHOST *protected, time_t now) { +static inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) { if(host != protected && host != localhost && !host->connected_senders @@ -333,7 +364,7 @@ static inline int rrdhost_should_be_deleted(RRDHOST *host, RRDHOST *protected, t return 0; } -void rrdhost_cleanup_orphan(RRDHOST *protected) { +void rrdhost_cleanup_orphan_hosts(RRDHOST *protected) { time_t now = now_realtime_sec(); rrd_wrlock(); @@ -342,10 +373,10 @@ void rrdhost_cleanup_orphan(RRDHOST *protected) { restart_after_removal: rrdhost_foreach_write(host) { - if(rrdhost_should_be_deleted(host, protected, now)) { + if(rrdhost_should_be_removed(host, protected, now)) { info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid); - if(rrdset_flag_check(host, RRDHOST_ORPHAN)) + if(rrdset_flag_check(host, RRDHOST_DELETE_ORPHAN_HOST) && rrdset_flag_check(host, RRDHOST_ORPHAN)) rrdhost_delete(host); else rrdhost_save(host); @@ -372,8 +403,10 @@ void rrd_init(char *hostname) { rrd_wrlock(); localhost = rrdhost_create( hostname + , registry_get_this_machine_hostname() , registry_get_this_machine_guid() , os_type + , config_get(CONFIG_SECTION_BACKEND, "host tags", "") , default_rrd_update_every , default_rrd_history_entries , default_rrd_memory_mode @@ -473,7 +506,8 @@ void rrdhost_free(RRDHOST *host) { // ------------------------------------------------------------------------ // free it - freez(host->os); + freez((void *)host->tags); + freez((void *)host->os); freez(host->cache_dir); freez(host->varlib_dir); freez(host->rrdpush_api_key); @@ -482,6 +516,7 @@ void rrdhost_free(RRDHOST *host) { freez(host->health_default_recipient); freez(host->health_log_filename); freez(host->hostname); + freez(host->registry_hostname); rrdhost_unlock(host); netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock); netdata_rwlock_destroy(&host->rrdhost_rwlock); @@ -497,12 +532,12 @@ void rrdhost_free_all(void) { } // ---------------------------------------------------------------------------- -// RRDHOST - save +// RRDHOST - save host files void rrdhost_save(RRDHOST *host) { if(!host) return; - info("Saving database of host '%s'...", host->hostname); + info("Saving/Closing database of host '%s'...", host->hostname); RRDSET *st; @@ -520,7 +555,7 @@ void rrdhost_save(RRDHOST *host) { } // ---------------------------------------------------------------------------- -// RRDHOST - delete files +// RRDHOST - delete host files void rrdhost_delete(RRDHOST *host) { if(!host) return; @@ -539,9 +574,43 @@ void rrdhost_delete(RRDHOST *host) { rrdset_unlock(st); } + recursively_delete_dir(host->cache_dir, "left over host"); + rrdhost_unlock(host); } +// ---------------------------------------------------------------------------- +// RRDHOST - cleanup host files + +void rrdhost_cleanup(RRDHOST *host) { + if(!host) return; + + info("Cleaning up database of host '%s'...", host->hostname); + + RRDSET *st; + + // we get a write lock + // to ensure only one thread is saving the database + rrdhost_wrlock(host); + + rrdset_foreach_write(st, host) { + rrdset_rdlock(st); + + if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS)) + rrdset_delete(st); + else + rrdset_save(st); + + rrdset_unlock(st); + } + + rrdhost_unlock(host); +} + + +// ---------------------------------------------------------------------------- +// RRDHOST - save all hosts to disk + void rrdhost_save_all(void) { info("Saving database [%zu hosts(s)]...", rrd_hosts_available); @@ -554,7 +623,30 @@ void rrdhost_save_all(void) { rrd_unlock(); } -void rrdhost_cleanup_obsolete(RRDHOST *host) { +// ---------------------------------------------------------------------------- +// RRDHOST - save or delete all hosts from disk + +void rrdhost_cleanup_all(void) { + info("Cleaning up database [%zu hosts(s)]...", rrd_hosts_available); + + rrd_rdlock(); + + RRDHOST *host; + rrdhost_foreach_read(host) { + if(host != localhost && rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS) && !host->connected_senders) + rrdhost_delete(host); + else + rrdhost_cleanup(host); + } + + rrd_unlock(); +} + + +// ---------------------------------------------------------------------------- +// RRDHOST - save or delete all the host charts from disk + +void rrdhost_cleanup_obsolete_charts(RRDHOST *host) { time_t now = now_realtime_sec(); RRDSET *st; @@ -569,7 +661,7 @@ restart_after_removal: rrdset_rdlock(st); - if(rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_FILES)) + if(rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_CHARTS)) rrdset_delete(st); else rrdset_save(st); diff --git a/src/rrdpush.c b/src/rrdpush.c index 72e6d8a73..6def90fe5 100644 --- a/src/rrdpush.c +++ b/src/rrdpush.c @@ -57,13 +57,16 @@ int rrdpush_init() { // to its current clock, we send for this many // iterations a BEGIN line without microseconds // this is for the first iterations of each chart -static unsigned int remote_clock_resync_iterations = 60; +unsigned int remote_clock_resync_iterations = 60; #define rrdpush_lock(host) netdata_mutex_lock(&((host)->rrdpush_mutex)) #define rrdpush_unlock(host) netdata_mutex_unlock(&((host)->rrdpush_mutex)) // checks if the current chart definition has been sent static inline int need_to_send_chart_definition(RRDSET *st) { + if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_EXPOSED_UPSTREAM)))) + return 1; + RRDDIM *rd; rrddim_foreach_read(rd, st) if(!rd->exposed) @@ -74,7 +77,9 @@ static inline int need_to_send_chart_definition(RRDSET *st) { // sends the current chart definition static inline void send_chart_definition(RRDSET *st) { - buffer_sprintf(st->rrdhost->rrdpush_buffer, "CHART '%s' '%s' '%s' '%s' '%s' '%s' '%s' %ld %d\n" + rrdset_flag_set(st, RRDSET_FLAG_EXPOSED_UPSTREAM); + + buffer_sprintf(st->rrdhost->rrdpush_buffer, "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s\"\n" , st->id , st->name , st->title @@ -84,11 +89,14 @@ static inline void send_chart_definition(RRDSET *st) { , rrdset_type_name(st->chart_type) , st->priority , st->update_every + , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":"" + , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":"" + , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":"" ); RRDDIM *rd; rrddim_foreach_read(rd, st) { - buffer_sprintf(st->rrdhost->rrdpush_buffer, "DIMENSION '%s' '%s' '%s' " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " '%s %s'\n" + buffer_sprintf(st->rrdhost->rrdpush_buffer, "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s\"\n" , rd->id , rd->name , rrd_algorithm_name(rd->algorithm) @@ -99,11 +107,13 @@ static inline void send_chart_definition(RRDSET *st) { ); rd->exposed = 1; } + + st->upstream_resync_time = st->last_collected_time.tv_sec + (remote_clock_resync_iterations * st->update_every); } // sends the current chart dimensions static inline void send_chart_metrics(RRDSET *st) { - buffer_sprintf(st->rrdhost->rrdpush_buffer, "BEGIN %s %llu\n", st->id, (st->counter_done > remote_clock_resync_iterations)?st->usec_since_last_update:0); + buffer_sprintf(st->rrdhost->rrdpush_buffer, "BEGIN %s %llu\n", st->id, (st->upstream_resync_time > st->last_collected_time.tv_sec)?st->usec_since_last_update:0); RRDDIM *rd; rrddim_foreach_read(rd, st) { @@ -117,7 +127,17 @@ static inline void send_chart_metrics(RRDSET *st) { buffer_strcat(st->rrdhost->rrdpush_buffer, "END\n"); } -void rrdpush_sender_thread_spawn(RRDHOST *host); +static void rrdpush_sender_thread_spawn(RRDHOST *host); + +void rrdset_push_chart_definition(RRDSET *st) { + RRDHOST *host = st->rrdhost; + + rrdset_rdlock(st); + rrdpush_lock(host); + send_chart_definition(st); + rrdpush_unlock(host); + rrdset_unlock(st); +} void rrdset_done_push(RRDSET *st) { RRDHOST *host = st->rrdhost; @@ -167,9 +187,7 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) { RRDSET *st; rrdset_foreach_read(st, host) { - // make it re-align the current time - // on the remote host - st->counter_done = 0; + st->upstream_resync_time = 0; rrdset_rdlock(st); @@ -219,8 +237,6 @@ static void rrdpush_sender_thread_cleanup_locked_all(RRDHOST *host) { host->rrdpush_buffer = NULL; host->rrdpush_spawn = 0; - - rrdhost_flag_set(host, RRDHOST_ORPHAN); } void rrdpush_sender_thread_stop(RRDHOST *host) { @@ -274,6 +290,7 @@ void *rrdpush_sender_thread(void *ptr) { .tv_usec = 0 }; + time_t last_sent_t = 0; struct pollfd fds[2], *ifd, *ofd; nfds_t fdmax; @@ -281,8 +298,16 @@ void *rrdpush_sender_thread(void *ptr) { ofd = &fds[1]; for(; host->rrdpush_enabled && !netdata_exit ;) { + debug(D_STREAM, "STREAM: Checking if we need to timeout the connection..."); + if(host->rrdpush_socket != -1 && now_monotonic_sec() - last_sent_t > timeout) { + error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, timeout, sent_connection); + close(host->rrdpush_socket); + host->rrdpush_socket = -1; + } if(unlikely(host->rrdpush_socket == -1)) { + debug(D_STREAM, "STREAM: Attempting to connect..."); + // stop appending data into rrdpush_buffer // they will be lost, so there is no point to do it host->rrdpush_connected = 0; @@ -298,16 +323,19 @@ void *rrdpush_sender_thread(void *ptr) { info("STREAM %s [send to %s]: initializing communication...", host->hostname, connected_to); - char http[1000 + 1]; - snprintfz(http, 1000, - "STREAM key=%s&hostname=%s&machine_guid=%s&os=%s&update_every=%d HTTP/1.1\r\n" + #define HTTP_HEADER_SIZE 8192 + char http[HTTP_HEADER_SIZE + 1]; + snprintfz(http, HTTP_HEADER_SIZE, + "STREAM key=%s&hostname=%s®istry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&tags=%s HTTP/1.1\r\n" "User-Agent: netdata-push-service/%s\r\n" "Accept: */*\r\n\r\n" , host->rrdpush_api_key , host->hostname + , host->registry_hostname , host->machine_guid - , host->os , default_rrd_update_every + , host->os + , (host->tags)?host->tags:"" , program_version ); @@ -321,7 +349,7 @@ void *rrdpush_sender_thread(void *ptr) { info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, connected_to); - if(recv_timeout(host->rrdpush_socket, http, 1000, 0, timeout) == -1) { + if(recv_timeout(host->rrdpush_socket, http, HTTP_HEADER_SIZE, 0, timeout) == -1) { close(host->rrdpush_socket); host->rrdpush_socket = -1; error("STREAM %s [send to %s]: failed to initialize communication", host->hostname, connected_to); @@ -338,15 +366,21 @@ void *rrdpush_sender_thread(void *ptr) { } info("STREAM %s [send to %s]: established communication - sending metrics...", host->hostname, connected_to); + last_sent_t = now_monotonic_sec(); - if(fcntl(host->rrdpush_socket, F_SETFL, O_NONBLOCK) < 0) + if(sock_setnonblock(host->rrdpush_socket) < 0) error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, connected_to); + if(sock_enlarge_out(host->rrdpush_socket) < 0) + error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, connected_to); + rrdpush_sender_thread_data_flush(host); sent_connection = 0; // allow appending data into rrdpush_buffer host->rrdpush_connected = 1; + + debug(D_STREAM, "Connected..."); } ifd->fd = host->rrdpush_pipe[PIPE_READ]; @@ -356,82 +390,113 @@ void *rrdpush_sender_thread(void *ptr) { ofd->fd = host->rrdpush_socket; ofd->revents = 0; if(begin < buffer_strlen(host->rrdpush_buffer)) { + debug(D_STREAM, "STREAM: Requesting data output on streaming socket..."); ofd->events = POLLOUT; fdmax = 2; } else { + debug(D_STREAM, "STREAM: Not requesting data output on streaming socket (nothing to send now)..."); ofd->events = 0; fdmax = 1; } + debug(D_STREAM, "STREAM: Waiting for poll() events (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer)); if(netdata_exit) break; - int retval = poll(fds, fdmax, timeout * 1000); + int retval = poll(fds, fdmax, 1000); if(netdata_exit) break; if(unlikely(retval == -1)) { - if(errno == EAGAIN || errno == EINTR) + debug(D_STREAM, "STREAM: poll() failed (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer)); + + if(errno == EAGAIN || errno == EINTR) { + debug(D_STREAM, "STREAM: poll() failed with EAGAIN or EINTR..."); continue; + } error("STREAM %s [send to %s]: failed to poll().", host->hostname, connected_to); close(host->rrdpush_socket); host->rrdpush_socket = -1; break; } - else if(unlikely(!retval)) { - // timeout - continue; - } - - if(ifd->revents & POLLIN) { - char buffer[1000 + 1]; - if(read(host->rrdpush_pipe[PIPE_READ], buffer, 1000) == -1) - error("STREAM %s [send to %s]: cannot read from internal pipe.", host->hostname, connected_to); - } - - if(ofd->revents & POLLOUT && begin < buffer_strlen(host->rrdpush_buffer)) { + else if(likely(retval)) { + if (ifd->revents & POLLIN) { + debug(D_STREAM, "STREAM: Data added to send buffer (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer)); - // BEGIN RRDPUSH LOCKED SESSION - - // during this session, data collectors - // will not be able to append data to our buffer - // but the socket is in non-blocking mode - // so, we will not block at send() - - if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) - error("STREAM %s [send]: cannot set pthread cancel state to DISABLE.", host->hostname); - - rrdpush_lock(host); + char buffer[1000 + 1]; + if (read(host->rrdpush_pipe[PIPE_READ], buffer, 1000) == -1) + error("STREAM %s [send to %s]: cannot read from internal pipe.", host->hostname, connected_to); + } - ssize_t ret = send(host->rrdpush_socket, &host->rrdpush_buffer->buffer[begin], buffer_strlen(host->rrdpush_buffer) - begin, MSG_DONTWAIT); - if(ret == -1) { - if(errno != EAGAIN && errno != EINTR) { - error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection); + if (ofd->revents & POLLOUT && begin < buffer_strlen(host->rrdpush_buffer)) { + debug(D_STREAM, "STREAM: Sending data (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_buffer)); + + // BEGIN RRDPUSH LOCKED SESSION + + // during this session, data collectors + // will not be able to append data to our buffer + // but the socket is in non-blocking mode + // so, we will not block at send() + + if (pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) + error("STREAM %s [send]: cannot set pthread cancel state to DISABLE.", host->hostname); + + debug(D_STREAM, "STREAM: Getting exclusive lock on host..."); + rrdpush_lock(host); + + debug(D_STREAM, "STREAM: Sending data, starting from %zu, size %zu...", begin, buffer_strlen(host->rrdpush_buffer)); + ssize_t ret = send(host->rrdpush_socket, &host->rrdpush_buffer->buffer[begin], buffer_strlen(host->rrdpush_buffer) - begin, MSG_DONTWAIT); + if (unlikely(ret == -1)) { + if (errno != EAGAIN && errno != EINTR && errno != EWOULDBLOCK) { + debug(D_STREAM, "STREAM: Send failed - closing socket..."); + error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_connection); + close(host->rrdpush_socket); + host->rrdpush_socket = -1; + } + else { + debug(D_STREAM, "STREAM: Send failed - will retry..."); + } + } + else if(likely(ret > 0)) { + sent_connection += ret; + sent_bytes += ret; + begin += ret; + + if (begin == buffer_strlen(host->rrdpush_buffer)) { + // we send it all + + debug(D_STREAM, "STREAM: Sent %zd bytes (the whole buffer)...", ret); + buffer_flush(host->rrdpush_buffer); + begin = 0; + } + else { + debug(D_STREAM, "STREAM: Sent %zd bytes (part of the data buffer)...", ret); + } + + last_sent_t = now_monotonic_sec(); + } + else { + debug(D_STREAM, "STREAM: send() returned %zd - closing the socket...", ret); + error("STREAM %s [send to %s]: failed to send metrics (send() returned %zd) - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, ret, sent_connection); close(host->rrdpush_socket); host->rrdpush_socket = -1; } - } - else { - sent_connection += ret; - sent_bytes += ret; - begin += ret; - if(begin == buffer_strlen(host->rrdpush_buffer)) { - // we send it all - - buffer_flush(host->rrdpush_buffer); - begin = 0; - } - } - rrdpush_unlock(host); + debug(D_STREAM, "STREAM: Releasing exclusive lock on host..."); + rrdpush_unlock(host); - if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) - error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname); + if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("STREAM %s [send]: cannot set pthread cancel state to ENABLE.", host->hostname); - // END RRDPUSH LOCKED SESSION + // END RRDPUSH LOCKED SESSION + } + } + else { + debug(D_STREAM, "STREAM: poll() timed out."); } // protection from overflow - if(host->rrdpush_buffer->len > max_size) { + if(buffer_strlen(host->rrdpush_buffer) > max_size) { + debug(D_STREAM, "STREAM: Buffer is too big (%zu bytes), bigger than the max (%zu) - flushing it...", buffer_strlen(host->rrdpush_buffer), max_size); errno = 0; error("STREAM %s [send to %s]: too many data pending - buffer is %zu bytes long, %zu unsent - we have sent %zu bytes in total, %zu on this connection. Closing connection to flush the data.", host->hostname, connected_to, host->rrdpush_buffer->len, host->rrdpush_buffer->len - begin, sent_bytes, sent_connection); if(host->rrdpush_socket != -1) { @@ -464,7 +529,7 @@ cleanup: // ---------------------------------------------------------------------------- // rrdpush receiver thread -int rrdpush_receive(int fd, const char *key, const char *hostname, const char *machine_guid, const char *os, int update_every, char *client_ip, char *client_port) { +static int rrdpush_receive(int fd, const char *key, const char *hostname, const char *registry_hostname, const char *machine_guid, const char *os, const char *tags, int update_every, char *client_ip, char *client_port) { RRDHOST *host; int history = default_rrd_history_entries; RRD_MEMORY_MODE mode = default_rrd_memory_mode; @@ -499,13 +564,18 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m rrdpush_api_key = appconfig_get(&stream_config, key, "default proxy api key", rrdpush_api_key); rrdpush_api_key = appconfig_get(&stream_config, machine_guid, "proxy api key", rrdpush_api_key); + tags = appconfig_set_default(&stream_config, machine_guid, "host tags", (tags)?tags:""); + if(tags && !*tags) tags = NULL; + if(!strcmp(machine_guid, "localhost")) host = localhost; else host = rrdhost_find_or_create( hostname + , registry_hostname , machine_guid , os + , tags , update_every , history , mode @@ -522,7 +592,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m } #ifdef NETDATA_INTERNAL_CHECKS - info("STREAM %s [receive from [%s]:%s]: client willing to stream metrics for host '%s' with machine_guid '%s': update every = %d, history = %ld, memory mode = %s, health %s" + info("STREAM %s [receive from [%s]:%s]: client willing to stream metrics for host '%s' with machine_guid '%s': update every = %d, history = %ld, memory mode = %s, health %s, tags '%s'" , hostname , client_ip , client_port @@ -532,6 +602,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m , host->rrd_history_entries , rrd_memory_mode_name(host->rrd_memory_mode) , (health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto") + , host->tags ); #endif // NETDATA_INTERNAL_CHECKS @@ -560,7 +631,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m } // remove the non-blocking flag from the socket - if(fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) & ~O_NONBLOCK) == -1) + if(sock_delnonblock(fd) < 0) error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", host->hostname, client_ip, client_port, fd); // convert the socket to a FILE * @@ -572,7 +643,11 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m } rrdhost_wrlock(host); + if(host->connected_senders > 0) + info("STREAM %s [receive from [%s]:%s]: multiple streaming connections for the same host detected. If multiple netdata are pushing metrics for the same charts, at the same time, the result is unexpected.", host->hostname, client_ip, client_port); + host->connected_senders++; + rrdhost_flag_clear(host, RRDHOST_ORPHAN); if(health_enabled != CONFIG_BOOLEAN_NO) host->health_delay_up_to = now_realtime_sec() + alarms_delay; rrdhost_unlock(host); @@ -586,6 +661,7 @@ int rrdpush_receive(int fd, const char *key, const char *hostname, const char *m host->senders_disconnected_time = now_realtime_sec(); host->connected_senders--; if(!host->connected_senders) { + rrdhost_flag_set(host, RRDHOST_ORPHAN); if(health_enabled == CONFIG_BOOLEAN_AUTO) host->health_enabled = 0; } @@ -603,14 +679,16 @@ struct rrdpush_thread { int fd; char *key; char *hostname; + char *registry_hostname; char *machine_guid; char *os; + char *tags; char *client_ip; char *client_port; int update_every; }; -void *rrdpush_receiver_thread(void *ptr) { +static void *rrdpush_receiver_thread(void *ptr) { struct rrdpush_thread *rpt = (struct rrdpush_thread *)ptr; if (pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) @@ -621,13 +699,15 @@ void *rrdpush_receiver_thread(void *ptr) { info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid()); - rrdpush_receive(rpt->fd, rpt->key, rpt->hostname, rpt->machine_guid, rpt->os, rpt->update_every, rpt->client_ip, rpt->client_port); + rrdpush_receive(rpt->fd, rpt->key, rpt->hostname, rpt->registry_hostname, rpt->machine_guid, rpt->os, rpt->tags, rpt->update_every, rpt->client_ip, rpt->client_port); info("STREAM %s [receive from [%s]:%s]: receive thread ended (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid()); freez(rpt->key); freez(rpt->hostname); + freez(rpt->registry_hostname); freez(rpt->machine_guid); freez(rpt->os); + freez(rpt->tags); freez(rpt->client_ip); freez(rpt->client_port); freez(rpt); @@ -636,7 +716,7 @@ void *rrdpush_receiver_thread(void *ptr) { return NULL; } -void rrdpush_sender_thread_spawn(RRDHOST *host) { +static void rrdpush_sender_thread_spawn(RRDHOST *host) { rrdhost_wrlock(host); if(!host->rrdpush_spawn) { @@ -646,7 +726,6 @@ void rrdpush_sender_thread_spawn(RRDHOST *host) { else if(pthread_detach(host->rrdpush_thread)) error("STREAM %s [send]: cannot request detach newly created thread.", host->hostname); - rrdhost_flag_clear(host, RRDHOST_ORPHAN); host->rrdpush_spawn = 1; } @@ -658,7 +737,7 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url info("STREAM [receive from [%s]:%s]: new client connection.", w->client_ip, w->client_port); - char *key = NULL, *hostname = NULL, *machine_guid = NULL, *os = "unknown"; + char *key = NULL, *hostname = NULL, *registry_hostname = NULL, *machine_guid = NULL, *os = "unknown", *tags = NULL; int update_every = default_rrd_update_every; char buf[GUID_LEN + 1]; @@ -674,12 +753,18 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url key = value; else if(!strcmp(name, "hostname")) hostname = value; + else if(!strcmp(name, "registry_hostname")) + registry_hostname = value; else if(!strcmp(name, "machine_guid")) machine_guid = value; else if(!strcmp(name, "update_every")) update_every = (int)strtoul(value, NULL, 0); else if(!strcmp(name, "os")) os = value; + else if(!strcmp(name, "tags")) + tags = value; + else + info("STREAM [receive from [%s]:%s]: request has parameter '%s' = '%s', which is not used.", w->client_ip, w->client_port, key, value); } if(!key || !*key) { @@ -704,21 +789,21 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url } if(regenerate_guid(key, buf) == -1) { - error("STREAM [receive from [%s]:%s]: API key '%s' is not valid GUID. Forbidding access.", w->client_ip, w->client_port, key); + error("STREAM [receive from [%s]:%s]: API key '%s' is not valid GUID (use the command uuidgen to generate one). Forbidding access.", w->client_ip, w->client_port, key); buffer_flush(w->response.data); buffer_sprintf(w->response.data, "Your API key is invalid."); return 401; } if(regenerate_guid(machine_guid, buf) == -1) { - error("STREAM [receive from [%s]:%s]: machine GUID '%s' is not GUID. Forbidding access.", w->client_ip, w->client_port, key); + error("STREAM [receive from [%s]:%s]: machine GUID '%s' is not GUID. Forbidding access.", w->client_ip, w->client_port, machine_guid); buffer_flush(w->response.data); buffer_sprintf(w->response.data, "Your machine GUID is invalid."); return 404; } if(!appconfig_get_boolean(&stream_config, key, "enabled", 0)) { - error("STREAM [receive from [%s]:%s]: API key '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, machine_guid); + error("STREAM [receive from [%s]:%s]: API key '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, key); buffer_flush(w->response.data); buffer_sprintf(w->response.data, "Your API key is not permitted access."); return 401; @@ -732,14 +817,16 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url } struct rrdpush_thread *rpt = mallocz(sizeof(struct rrdpush_thread)); - rpt->fd = w->ifd; - rpt->key = strdupz(key); - rpt->hostname = strdupz(hostname); - rpt->machine_guid = strdupz(machine_guid); - rpt->os = strdupz(os); - rpt->client_ip = strdupz(w->client_ip); - rpt->client_port = strdupz(w->client_port); - rpt->update_every = update_every; + rpt->fd = w->ifd; + rpt->key = strdupz(key); + rpt->hostname = strdupz(hostname); + rpt->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname); + rpt->machine_guid = strdupz(machine_guid); + rpt->os = strdupz(os); + rpt->tags = (tags)?strdupz(tags):NULL; + rpt->client_ip = strdupz(w->client_ip); + rpt->client_port = strdupz(w->client_port); + rpt->update_every = update_every; pthread_t thread; debug(D_SYSTEM, "STREAM [receive from [%s]:%s]: starting receiving thread.", w->client_ip, w->client_port); diff --git a/src/rrdpush.h b/src/rrdpush.h index dddbe758b..c3c7f4a54 100644 --- a/src/rrdpush.h +++ b/src/rrdpush.h @@ -4,9 +4,11 @@ extern int default_rrdpush_enabled; extern char *default_rrdpush_destination; extern char *default_rrdpush_api_key; +extern unsigned int remote_clock_resync_iterations; extern int rrdpush_init(); extern void rrdset_done_push(RRDSET *st); +extern void rrdset_push_chart_definition(RRDSET *st); extern void *rrdpush_sender_thread(void *ptr); extern int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url); diff --git a/src/rrdset.c b/src/rrdset.c index c847b9690..caa427ff6 100644 --- a/src/rrdset.c +++ b/src/rrdset.c @@ -168,6 +168,58 @@ void rrdset_set_name(RRDSET *st, const char *name) { error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name); } +inline void rrdset_is_obsolete(RRDSET *st) { + if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) { + rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); + rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM); + + // the chart will not get more updates (data collection) + // so, we have to push its definition now + if(unlikely(st->rrdhost->rrdpush_enabled)) + rrdset_push_chart_definition(st); + } +} + +inline void rrdset_isnot_obsolete(RRDSET *st) { + if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) { + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM); + + // the chart will be pushed upstream automatically + // due to data collection + } +} + +inline void rrdset_update_heterogeneous_flag(RRDSET *st) { + RRDDIM *rd; + + rrdset_flag_clear(st, RRDSET_FLAG_HOMEGENEOUS_CHECK); + + RRD_ALGORITHM algorithm = st->dimensions->algorithm; + collected_number multiplier = abs(st->dimensions->multiplier); + collected_number divisor = abs(st->dimensions->divisor); + + rrddim_foreach_read(rd, st) { + if(algorithm != rd->algorithm || multiplier != abs(rd->multiplier) || divisor != abs(rd->divisor)) { + if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { + #ifdef NETDATA_INTERNAL_CHECKS + info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", + rd->name, + st->name, + st->rrdhost->hostname, + rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm), + rd->multiplier, multiplier, + rd->divisor, divisor + ); + #endif + rrdset_flag_set(st, RRDSET_FLAG_HETEROGENEOUS); + } + return; + } + } + + rrdset_flag_clear(st, RRDSET_FLAG_HETEROGENEOUS); +} // ---------------------------------------------------------------------------- // RRDSET - reset a chart @@ -188,7 +240,7 @@ void rrdset_reset(RRDSET *st) { rd->last_collected_time.tv_sec = 0; rd->last_collected_time.tv_usec = 0; rd->collections_counter = 0; - memset(rd->values, 0, rd->entries * sizeof(storage_number)); + // memset(rd->values, 0, rd->entries * sizeof(storage_number)); } } @@ -199,7 +251,7 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) { if(unlikely(entries < 5)) entries = 5; if(unlikely(entries > RRD_HISTORY_ENTRIES_MAX)) entries = RRD_HISTORY_ENTRIES_MAX; - if(unlikely(mode == RRD_MEMORY_MODE_NONE || mode == RRD_MEMORY_MODE_RAM)) + if(unlikely(mode == RRD_MEMORY_MODE_NONE || mode == RRD_MEMORY_MODE_ALLOC)) return entries; long page = (size_t)sysconf(_SC_PAGESIZE); @@ -215,14 +267,18 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) { return entries; } -static inline void last_collected_time_align(struct timeval *tv, int update_every) { - tv->tv_sec -= tv->tv_sec % update_every; - tv->tv_usec = 500000; +static inline void last_collected_time_align(RRDSET *st) { + st->last_collected_time.tv_sec -= st->last_collected_time.tv_sec % st->update_every; + + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))) + st->last_collected_time.tv_usec = 0; + else + st->last_collected_time.tv_usec = 500000; } -static inline void last_updated_time_align(struct timeval *tv, int update_every) { - tv->tv_sec -= tv->tv_sec % update_every; - tv->tv_usec = 0; +static inline void last_updated_time_align(RRDSET *st) { + st->last_updated.tv_sec -= st->last_updated.tv_sec % st->update_every; + st->last_updated.tv_usec = 0; } // ---------------------------------------------------------------------------- @@ -279,30 +335,36 @@ void rrdset_free(RRDSET *st) { // free directly allocated members freez(st->config_section); - if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) { - debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name); - munmap(st, st->memsize); + switch(st->rrd_memory_mode) { + case RRD_MEMORY_MODE_SAVE: + case RRD_MEMORY_MODE_MAP: + case RRD_MEMORY_MODE_RAM: + debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name); + munmap(st, st->memsize); + break; + + case RRD_MEMORY_MODE_ALLOC: + case RRD_MEMORY_MODE_NONE: + freez(st); + break; } - else - freez(st); } void rrdset_save(RRDSET *st) { - RRDDIM *rd; - rrdset_check_rdlock(st); // info("Saving chart '%s' ('%s')", st->id, st->name); if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) { debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename); - savememory(st->cache_filename, st, st->memsize); + memory_file_save(st->cache_filename, st, st->memsize); } + RRDDIM *rd; rrddim_foreach_read(rd, st) { if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) { debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename); - savememory(rd->cache_filename, rd, rd->memsize); + memory_file_save(rd->cache_filename, rd, rd->memsize); } } } @@ -312,19 +374,23 @@ void rrdset_delete(RRDSET *st) { rrdset_check_rdlock(st); - // info("Deleting chart '%s' ('%s')", st->id, st->name); + info("Deleting chart '%s' ('%s') from disk...", st->id, st->name); - if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) { - debug(D_RRD_STATS, "Deleting stats '%s' to '%s'.", st->name, st->cache_filename); - unlink(st->cache_filename); + if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) { + info("Deleting chart header file '%s'.", st->cache_filename); + if(unlikely(unlink(st->cache_filename) == -1)) + error("Cannot delete chart header file '%s'", st->cache_filename); } rrddim_foreach_read(rd, st) { - if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) { - debug(D_RRD_STATS, "Deleting dimension '%s' to '%s'.", rd->name, rd->cache_filename); - unlink(rd->cache_filename); + if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || rd->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) { + info("Deleting dimension file '%s'.", rd->cache_filename); + if(unlikely(unlink(rd->cache_filename) == -1)) + error("Cannot delete dimension file '%s'", rd->cache_filename); } } + + recursively_delete_dir(st->cache_dir, "left-over chart"); } // ---------------------------------------------------------------------------- @@ -333,7 +399,7 @@ void rrdset_delete(RRDSET *st) { static inline RRDSET *rrdset_find_on_create(RRDHOST *host, const char *fullid) { RRDSET *st = rrdset_find(host, fullid); if(unlikely(st)) { - rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + rrdset_isnot_obsolete(st); debug(D_RRD_CALLS, "RRDSET '%s', already exists.", fullid); return st; } @@ -341,7 +407,7 @@ static inline RRDSET *rrdset_find_on_create(RRDHOST *host, const char *fullid) { return NULL; } -RRDSET *rrdset_create( +RRDSET *rrdset_create_custom( RRDHOST *host , const char *type , const char *id @@ -353,6 +419,8 @@ RRDSET *rrdset_create( , long priority , int update_every , RRDSET_TYPE chart_type + , RRD_MEMORY_MODE memory_mode + , long history_entries ) { if(!type || !type[0]) { fatal("Cannot create rrd stats without a type."); @@ -395,11 +463,11 @@ RRDSET *rrdset_create( // ------------------------------------------------------------------------ // get the options from the config, we need to create it - long rentries = config_get_number(config_section, "history", host->rrd_history_entries); - long entries = align_entries_to_pagesize(host->rrd_memory_mode, rentries); + long rentries = config_get_number(config_section, "history", history_entries); + long entries = align_entries_to_pagesize(memory_mode, rentries); if(entries != rentries) entries = config_set_number(config_section, "history", entries); - if(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE && entries != rentries) + if(memory_mode == RRD_MEMORY_MODE_NONE && entries != rentries) entries = config_set_number(config_section, "history", 10); int enabled = config_get_boolean(config_section, "enabled", 1); @@ -416,8 +484,14 @@ RRDSET *rrdset_create( debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id); snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir); - if(host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) { - st = (RRDSET *) mymmap(fullfilename, size, ((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0); + if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_RAM) { + st = (RRDSET *) mymmap( + (memory_mode == RRD_MEMORY_MODE_RAM)?NULL:fullfilename + , size + , ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE) + , 0 + ); + if(st) { memset(&st->avl, 0, sizeof(avl)); memset(&st->avlname, 0, sizeof(avl)); @@ -426,64 +500,68 @@ RRDSET *rrdset_create( memset(&st->rrdset_rwlock, 0, sizeof(netdata_rwlock_t)); st->name = NULL; + st->config_section = NULL; st->type = NULL; st->family = NULL; - st->context = NULL; st->title = NULL; st->units = NULL; + st->context = NULL; + st->cache_dir = NULL; st->dimensions = NULL; + st->rrdfamily = NULL; + st->rrdhost = NULL; st->next = NULL; st->variables = NULL; st->alarms = NULL; st->flags = 0x00000000; - if(strcmp(st->magic, RRDSET_MAGIC) != 0) { - errno = 0; - info("Initializing file %s.", fullfilename); - memset(st, 0, size); - } - else if(strcmp(st->id, fullid) != 0) { - errno = 0; - error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid); - // munmap(st, size); - // st = NULL; + if(memory_mode == RRD_MEMORY_MODE_RAM) { memset(st, 0, size); } - else if(st->memsize != size || st->entries != entries) { - errno = 0; - error("File %s does not have the desired size. Clearing it.", fullfilename); - memset(st, 0, size); - } - else if(st->update_every != update_every) { - errno = 0; - error("File %s does not have the desired update frequency. Clearing it.", fullfilename); - memset(st, 0, size); - } - else if((now - st->last_updated.tv_sec) > update_every * entries) { - errno = 0; - error("File %s is too old. Clearing it.", fullfilename); - memset(st, 0, size); - } - else if(st->last_updated.tv_sec > now + update_every) { - errno = 0; - error("File %s refers to the future. Clearing it.", fullfilename); - memset(st, 0, size); - } - - // make sure the database is aligned - if(st->last_updated.tv_sec) - last_updated_time_align(&st->last_updated, update_every); + else { + if(strcmp(st->magic, RRDSET_MAGIC) != 0) { + info("Initializing file %s.", fullfilename); + memset(st, 0, size); + } + else if(strcmp(st->id, fullid) != 0) { + error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid); + // munmap(st, size); + // st = NULL; + memset(st, 0, size); + } + else if(st->memsize != size || st->entries != entries) { + error("File %s does not have the desired size. Clearing it.", fullfilename); + memset(st, 0, size); + } + else if(st->update_every != update_every) { + error("File %s does not have the desired update frequency. Clearing it.", fullfilename); + memset(st, 0, size); + } + else if((now - st->last_updated.tv_sec) > update_every * entries) { + error("File %s is too old. Clearing it.", fullfilename); + memset(st, 0, size); + } + else if(st->last_updated.tv_sec > now + update_every) { + error("File %s refers to the future. Clearing it.", fullfilename); + memset(st, 0, size); + } + // make sure the database is aligned + if(st->last_updated.tv_sec) { + st->update_every = update_every; + last_updated_time_align(st); + } + } // make sure we have the right memory mode // even if we cleared the memory - st->rrd_memory_mode = host->rrd_memory_mode; + st->rrd_memory_mode = memory_mode; } } if(unlikely(!st)) { st = callocz(1, size); - st->rrd_memory_mode = (host->rrd_memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_RAM; + st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC; } st->config_section = strdup(config_section); @@ -519,6 +597,7 @@ RRDSET *rrdset_create( rrdset_flag_clear(st, RRDSET_FLAG_DETAIL); rrdset_flag_clear(st, RRDSET_FLAG_DEBUG); rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM); // if(!strcmp(st->id, "disk_util.dm-0")) { // st->debug = 1; @@ -536,6 +615,9 @@ RRDSET *rrdset_create( st->gap_when_lost_iterations_above = (int) ( config_get_number(st->config_section, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS) + 2); + st->last_accessed_time = 0; + st->upstream_resync_time = 0; + avl_init_lock(&st->dimensions_index, rrddim_compare); avl_init_lock(&st->variables_root_index, rrdvar_compare); @@ -544,13 +626,8 @@ RRDSET *rrdset_create( if(name && *name) rrdset_set_name(st, name); else rrdset_set_name(st, id); - { - char varvalue[CONFIG_MAX_VALUE + 1]; - char varvalue2[CONFIG_MAX_VALUE + 1]; - snprintfz(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name); - json_escape_string(varvalue2, varvalue, sizeof(varvalue2)); - st->title = config_get(st->config_section, "title", varvalue2); - } + st->title = config_get(st->config_section, "title", title); + json_fix_string(st->title); st->rrdfamily = rrdfamily_create(host, st->family); @@ -571,7 +648,7 @@ RRDSET *rrdset_create( rrdsetcalc_link_matching(st); rrdcalctemplate_link_matching(st); - rrdhost_cleanup_obsolete(host); + rrdhost_cleanup_obsolete_charts(host); rrdhost_unlock(host); @@ -583,16 +660,10 @@ RRDSET *rrdset_create( // RRDSET - data collection iteration control inline void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds) { - - if(unlikely(!st->last_collected_time.tv_sec)) { - // the first entry - microseconds = st->update_every * USEC_PER_SEC; - } - else if(unlikely(!microseconds)) { - // no dt given by the plugin - struct timeval now; - now_realtime_timeval(&now); - microseconds = dt_usec(&now, &st->last_collected_time); + if(unlikely(!st->last_collected_time.tv_sec || !microseconds || (st->counter % remote_clock_resync_iterations) == 0)) { + // call the full next_usec() function + rrdset_next_usec(st, microseconds); + return; } st->usec_since_last_update = microseconds; @@ -612,52 +683,34 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) { } else { // microseconds has the time since the last collection -//#ifdef NETDATA_INTERNAL_CHECKS -// usec_t now_usec = timeval_usec(&now); -// usec_t last_usec = timeval_usec(&st->last_collected_time); -//#endif susec_t since_last_usec = dt_usec_signed(&now, &st->last_collected_time); if(unlikely(since_last_usec < 0)) { // oops! the database is in the future - error("Database for chart '%s' on host '%s' is %lld microseconds in the future. Adjusting it to current time.", st->id, st->rrdhost->hostname, -since_last_usec); + info("RRD database for chart '%s' on host '%s' is %0.5Lf secs in the future. Adjusting it to current time.", st->id, st->rrdhost->hostname, (long double)-since_last_usec / USEC_PER_SEC); st->last_collected_time.tv_sec = now.tv_sec - st->update_every; st->last_collected_time.tv_usec = now.tv_usec; - last_collected_time_align(&st->last_collected_time, st->update_every); + last_collected_time_align(st); st->last_updated.tv_sec = now.tv_sec - st->update_every; st->last_updated.tv_usec = now.tv_usec; - last_updated_time_align(&st->last_updated, st->update_every); + last_updated_time_align(st); microseconds = st->update_every * USEC_PER_SEC; - since_last_usec = st->update_every * USEC_PER_SEC; - } - - // verify the microseconds given is good - if(unlikely(microseconds > (usec_t)since_last_usec)) { - debug(D_RRD_CALLS, "dt %llu usec given is too big - it leads %llu usec to the future, for chart '%s' (%s).", microseconds, microseconds - (usec_t)since_last_usec, st->name, st->id); - -//#ifdef NETDATA_INTERNAL_CHECKS -// if(unlikely(last_usec + microseconds > now_usec + 1000)) -// error("dt %llu usec given is too big - it leads %llu usec to the future, for chart '%s' (%s).", microseconds, microseconds - (usec_t)since_last_usec, st->name, st->id); -//#endif - - microseconds = (usec_t)since_last_usec; } - else if(unlikely(microseconds < (usec_t)since_last_usec * 0.8)) { - debug(D_RRD_CALLS, "dt %llu usec given is too small - expected %llu usec up to -20%%, for chart '%s' (%s).", microseconds, (usec_t)since_last_usec, st->name, st->id); + else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 10 * USEC_PER_SEC))) { + // oops! the database is too far behind + info("RRD database for chart '%s' on host '%s' is %0.5Lf secs in the past. Adjusting it to current time.", st->id, st->rrdhost->hostname, (long double)since_last_usec / USEC_PER_SEC); -//#ifdef NETDATA_INTERNAL_CHECKS -// error("dt %llu usec given is too small - expected %llu usec up to -20%%, for chart '%s' (%s).", microseconds, (usec_t)since_last_usec, st->name, st->id); -//#endif microseconds = (usec_t)since_last_usec; } } - debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds); - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds); + #ifdef NETDATA_INTERNAL_CHECKS + debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds); + rrdset_debug(st, "NEXT: %llu microseconds", microseconds); + #endif st->usec_since_last_update = microseconds; } @@ -666,9 +719,17 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) { // ---------------------------------------------------------------------------- // RRDSET - process the collected values for all dimensions of a chart -static inline void rrdset_init_last_collected_time(RRDSET *st) { +static inline usec_t rrdset_init_last_collected_time(RRDSET *st) { now_realtime_timeval(&st->last_collected_time); - last_collected_time_align(&st->last_collected_time, st->update_every); + last_collected_time_align(st); + + usec_t last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec; + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "initialized last collected time to %0.3Lf", (long double)last_collect_ut / USEC_PER_SEC); + #endif + + return last_collect_ut; } static inline usec_t rrdset_update_last_collected_time(RRDSET *st) { @@ -676,17 +737,42 @@ static inline usec_t rrdset_update_last_collected_time(RRDSET *st) { usec_t ut = last_collect_ut + st->usec_since_last_update; st->last_collected_time.tv_sec = (time_t) (ut / USEC_PER_SEC); st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC); + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "updated last collected time to %0.3Lf", (long double)last_collect_ut / USEC_PER_SEC); + #endif + return last_collect_ut; } -static inline void rrdset_init_last_updated_time(RRDSET *st) { +static inline usec_t rrdset_init_last_updated_time(RRDSET *st) { // copy the last collected time to last updated time st->last_updated.tv_sec = st->last_collected_time.tv_sec; st->last_updated.tv_usec = st->last_collected_time.tv_usec; - last_updated_time_align(&st->last_updated, st->update_every); + + if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)) + st->last_updated.tv_sec -= st->update_every; + + last_updated_time_align(st); + + usec_t last_updated_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec; + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "initialized last updated time to %0.3Lf", (long double)last_updated_ut / USEC_PER_SEC); + #endif + + return last_updated_ut; } static inline void rrdset_done_push_exclusive(RRDSET *st) { +// usec_t update_every_ut = st->update_every * USEC_PER_SEC; // st->update_every in microseconds +// +// if(unlikely(st->usec_since_last_update > update_every_ut * remote_clock_resync_iterations)) { +// error("Chart '%s' was last collected %llu usec before. Resetting it.", st->id, st->usec_since_last_update); +// rrdset_reset(st); +// st->usec_since_last_update = update_every_ut; +// } + if(unlikely(!st->last_collected_time.tv_sec)) { // it is the first entry // set the last_collected_time to now @@ -705,6 +791,235 @@ static inline void rrdset_done_push_exclusive(RRDSET *st) { rrdset_unlock(st); } + +static inline size_t rrdset_done_interpolate( + RRDSET *st + , usec_t update_every_ut + , usec_t last_stored_ut + , usec_t next_store_ut + , usec_t last_collect_ut + , usec_t now_collect_ut + , char store_this_entry + , uint32_t storage_flags +) { + RRDDIM *rd; + + size_t stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done() + + usec_t first_ut = last_stored_ut, last_ut = 0; + ssize_t iterations = (ssize_t)((now_collect_ut - last_stored_ut) / (update_every_ut)); + if((now_collect_ut % (update_every_ut)) == 0) iterations++; + + size_t counter = st->counter; + long current_entry = st->current_entry; + + for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) { + + #ifdef NETDATA_INTERNAL_CHECKS + if(iterations < 0) { error("INTERNAL CHECK: %s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); } + rrdset_debug(st, "last_stored_ut = %0.3Lf (last updated time)", (long double)last_stored_ut/USEC_PER_SEC); + rrdset_debug(st, "next_store_ut = %0.3Lf (next interpolation point)", (long double)next_store_ut/USEC_PER_SEC); + #endif + + last_ut = next_store_ut; + + rrddim_foreach_read(rd, st) { + calculated_number new_value; + + switch(rd->algorithm) { + case RRD_ALGORITHM_INCREMENTAL: + new_value = (calculated_number) + ( rd->calculated_value + * (calculated_number)(next_store_ut - last_collect_ut) + / (calculated_number)(now_collect_ut - last_collect_ut) + ); + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: CALC2 INC " + CALCULATED_NUMBER_FORMAT " = " + CALCULATED_NUMBER_FORMAT + " * (%llu - %llu)" + " / (%llu - %llu)" + , rd->name + , new_value + , rd->calculated_value + , next_store_ut, last_collect_ut + , now_collect_ut, last_collect_ut + ); + #endif + + rd->calculated_value -= new_value; + new_value += rd->last_calculated_value; + rd->last_calculated_value = 0; + new_value /= (calculated_number)st->update_every; + + if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) { + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING", + rd->name + , (calculated_number)(next_store_ut - last_stored_ut) + ); + #endif + + new_value = new_value * (calculated_number)(st->update_every * USEC_PER_SEC) / (calculated_number)(next_store_ut - last_stored_ut); + } + break; + + case RRD_ALGORITHM_ABSOLUTE: + case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: + case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: + default: + if(iterations == 1) { + // this is the last iteration + // do not interpolate + // just show the calculated value + + new_value = rd->calculated_value; + } + else { + // we have missed an update + // interpolate in the middle values + + new_value = (calculated_number) + ( ( (rd->calculated_value - rd->last_calculated_value) + * (calculated_number)(next_store_ut - last_collect_ut) + / (calculated_number)(now_collect_ut - last_collect_ut) + ) + + rd->last_calculated_value + ); + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: CALC2 DEF " + CALCULATED_NUMBER_FORMAT " = (((" + "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")" + " * %llu" + " / %llu) + " CALCULATED_NUMBER_FORMAT + , rd->name + , new_value + , rd->calculated_value, rd->last_calculated_value + , (next_store_ut - first_ut) + , (now_collect_ut - first_ut), rd->last_calculated_value + ); + #endif + } + break; + } + + if(unlikely(!store_this_entry)) { + rd->values[current_entry] = SN_EMPTY_SLOT; //pack_storage_number(0, SN_NOT_EXISTS); + continue; + } + + if(likely(rd->updated && rd->collections_counter > 1 && iterations < st->gap_when_lost_iterations_above)) { + rd->values[current_entry] = pack_storage_number(new_value, storage_flags ); + rd->last_stored_value = new_value; + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: STORE[%ld] " + CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT + , rd->name + , current_entry + , unpack_storage_number(rd->values[current_entry]), new_value + ); + #endif + + } + else { + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING " + , rd->name + , current_entry + ); + #endif + + rd->values[current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS); + rd->last_stored_value = NAN; + } + + stored_entries++; + + #ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) { + calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; + calculated_number t2 = unpack_storage_number(rd->values[current_entry]); + + calculated_number accuracy = accuracy_loss(t1, t2); + debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" + , st->id, rd->name + , current_entry + , t2 + , get_storage_number_flags(rd->values[current_entry]) + , t1 + , accuracy + , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" + ); + + rd->collected_volume += t1; + rd->stored_volume += t2; + + accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume); + debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s" + , st->id, rd->name + , current_entry + , rd->stored_volume + , rd->collected_volume + , accuracy + , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" + ); + } + #endif + } + // reset the storage flags for the next point, if any; + storage_flags = SN_EXISTS; + + counter++; + current_entry = ((current_entry + 1) >= st->entries) ? 0 : current_entry + 1; + last_stored_ut = next_store_ut; + } + + st->counter = counter; + st->current_entry = current_entry; + + if(likely(last_ut)) { + st->last_updated.tv_sec = (time_t) (last_ut / USEC_PER_SEC); + st->last_updated.tv_usec = 0; + } + + return stored_entries; +} + +static inline void rrdset_done_fill_the_gap(RRDSET *st) { + usec_t update_every_ut = st->update_every * USEC_PER_SEC; + usec_t now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec; + + long c = 0, entries = st->entries; + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + usec_t next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC; + long current_entry = st->current_entry; + + for(c = 0; c < entries && next_store_ut <= now_collect_ut ; next_store_ut += update_every_ut, c++) { + rd->values[current_entry] = SN_EMPTY_SLOT; + current_entry = ((current_entry + 1) >= entries) ? 0 : current_entry + 1; + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING (FILLED THE GAP)", rd->name, current_entry); + #endif + } + } + + if(c > 0) { + c--; + st->last_updated.tv_sec += c * st->update_every; + + st->current_entry += c; + if(st->current_entry >= st->entries) + st->current_entry -= st->entries; + } +} + void rrdset_done(RRDSET *st) { if(unlikely(netdata_exit)) return; @@ -726,9 +1041,6 @@ void rrdset_done(RRDSET *st) { store_this_entry = 1, // boolean: 1 = store this entry, 0 = don't store this entry first_entry = 0; // boolean: 1 = this is the first entry seen for this chart, 0 = all other entries - unsigned int - stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done() - usec_t last_collect_ut, // the timestamp in microseconds, of the last collected value now_collect_ut, // the timestamp in microseconds, of this collected value (this is NOW) @@ -742,42 +1054,33 @@ void rrdset_done(RRDSET *st) { // a read lock is OK here rrdset_rdlock(st); -/* - // enable the chart, if it was disabled - if(unlikely(rrd_delete_unupdated_dimensions) && !st->enabled) - st->enabled = 1; -*/ - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) { error("Chart '%s' has the OBSOLETE flag set, but it is collected.", st->id); - rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + rrdset_isnot_obsolete(st); } // check if the chart has a long time to be updated if(unlikely(st->usec_since_last_update > st->entries * update_every_ut)) { - info("%s: took too long to be updated (%0.3Lf secs). Resetting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); + info("host '%s', chart %s: took too long to be updated (%0.3Lf secs). Resetting it.", st->rrdhost->hostname, st->name, (long double)st->usec_since_last_update / USEC_PER_SEC); rrdset_reset(st); st->usec_since_last_update = update_every_ut; + store_this_entry = 0; first_entry = 1; } - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update); + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "microseconds since last update: %llu", st->usec_since_last_update); + #endif // set last_collected_time if(unlikely(!st->last_collected_time.tv_sec)) { // it is the first entry // set the last_collected_time to now - rrdset_init_last_collected_time(st); - - last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec - update_every_ut; + last_collect_ut = rrdset_init_last_collected_time(st) - update_every_ut; // the first entry should not be stored store_this_entry = 0; first_entry = 1; - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: has not set last_collected_time. Setting it now. Will not store the next entry.", st->name); } else { // it is not the first entry @@ -795,9 +1098,6 @@ void rrdset_done(RRDSET *st) { // the first entry should not be stored store_this_entry = 0; first_entry = 1; - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: initializing last_updated to last_collected_time - %llu microseconds. Will not store the next entry.", st->name, st->usec_since_last_update); } // check if we will re-write the entire data set @@ -817,27 +1117,48 @@ void rrdset_done(RRDSET *st) { // last_stored_ut = the last time we added a value to the storage // now_collect_ut = the time the current value has been collected // next_store_ut = the time of the next interpolation point - last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec; now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec; + last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec; next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) { - debug(D_RRD_STATS, "%s: last_collect_ut = %0.3Lf (last collection time)", st->name, (long double)last_collect_ut/1000000.0); - debug(D_RRD_STATS, "%s: now_collect_ut = %0.3Lf (current collection time)", st->name, (long double)now_collect_ut/1000000.0); - debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/1000000.0); - debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/1000000.0); - } - if(unlikely(!st->counter_done)) { - store_this_entry = 0; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name); + // if we have not collected metrics this session (st->counter_done == 0) + // and we have collected metrics for this chart in the past (st->counter != 0) + // fill the gap (the chart has been just loaded from disk) + if(unlikely(st->counter)) { + rrdset_done_fill_the_gap(st); + last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec; + next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC; + } + + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))) { + store_this_entry = 1; + last_collect_ut = next_store_ut - update_every_ut; + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "Fixed first entry."); + #endif + } + else { + store_this_entry = 0; + + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "Will not store the next entry."); + #endif + } } st->counter_done++; if(unlikely(st->rrdhost->rrdpush_enabled)) rrdset_done_push(st); + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "last_collect_ut = %0.3Lf (last collection time)", (long double)last_collect_ut/USEC_PER_SEC); + rrdset_debug(st, "now_collect_ut = %0.3Lf (current collection time)", (long double)now_collect_ut/USEC_PER_SEC); + rrdset_debug(st, "last_stored_ut = %0.3Lf (last updated time)", (long double)last_stored_ut/USEC_PER_SEC); + rrdset_debug(st, "next_store_ut = %0.3Lf (next interpolation point)", (long double)next_store_ut/USEC_PER_SEC); + #endif + // calculate totals and count the dimensions int dimensions = 0; st->collected_total = 0; @@ -859,18 +1180,19 @@ void rrdset_done(RRDSET *st) { continue; } - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: START " - " last_collected_value = " COLLECTED_NUMBER_FORMAT - " collected_value = " COLLECTED_NUMBER_FORMAT - " last_calculated_value = " CALCULATED_NUMBER_FORMAT - " calculated_value = " CALCULATED_NUMBER_FORMAT - , st->id, rd->name + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: START " + " last_collected_value = " COLLECTED_NUMBER_FORMAT + " collected_value = " COLLECTED_NUMBER_FORMAT + " last_calculated_value = " CALCULATED_NUMBER_FORMAT + " calculated_value = " CALCULATED_NUMBER_FORMAT + , rd->name , rd->last_collected_value , rd->collected_value , rd->last_calculated_value , rd->calculated_value - ); + ); + #endif switch(rd->algorithm) { case RRD_ALGORITHM_ABSOLUTE: @@ -878,18 +1200,20 @@ void rrdset_done(RRDSET *st) { * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN " + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: CALC ABS/ABS-NO-IN " CALCULATED_NUMBER_FORMAT " = " COLLECTED_NUMBER_FORMAT " * " CALCULATED_NUMBER_FORMAT " / " CALCULATED_NUMBER_FORMAT - , st->id, rd->name + , rd->name , rd->calculated_value , rd->collected_value , (calculated_number)rd->multiplier , (calculated_number)rd->divisor - ); + ); + #endif + break; case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: @@ -903,16 +1227,18 @@ void rrdset_done(RRDSET *st) { * (calculated_number)rd->collected_value / (calculated_number)st->collected_total; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW " + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: CALC PCENT-ROW " CALCULATED_NUMBER_FORMAT " = 100" - " * " COLLECTED_NUMBER_FORMAT + " * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT - , st->id, rd->name + , rd->name , rd->calculated_value , rd->collected_value , st->collected_total - ); + ); + #endif + break; case RRD_ALGORITHM_INCREMENTAL: @@ -940,19 +1266,21 @@ void rrdset_done(RRDSET *st) { * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: CALC INC PRE " + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: CALC INC PRE " CALCULATED_NUMBER_FORMAT " = (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" " * " CALCULATED_NUMBER_FORMAT " / " CALCULATED_NUMBER_FORMAT - , st->id, rd->name + , rd->name , rd->calculated_value , rd->collected_value, rd->last_collected_value , (calculated_number)rd->multiplier , (calculated_number)rd->divisor - ); + ); + #endif + break; case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: @@ -967,7 +1295,8 @@ void rrdset_done(RRDSET *st) { debug(D_RRD_STATS, "%s.%s: RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT , st->name, rd->name , rd->last_collected_value - , rd->collected_value); + , rd->collected_value + ); if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS))) storage_flags = SN_EXISTS_RESET; @@ -985,16 +1314,18 @@ void rrdset_done(RRDSET *st) { * (calculated_number)(rd->collected_value - rd->last_collected_value) / (calculated_number)(st->collected_total - st->last_collected_total); - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF " + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: CALC PCENT-DIFF " CALCULATED_NUMBER_FORMAT " = 100" - " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" - " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" - , st->id, rd->name + " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" + " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")" + , rd->name , rd->calculated_value , rd->collected_value, rd->last_collected_value , st->collected_total, st->last_collected_total - ); + ); + #endif + break; default: @@ -1002,203 +1333,53 @@ void rrdset_done(RRDSET *st) { // it gets noticed when we add new types rd->calculated_value = 0; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: CALC " + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: CALC " CALCULATED_NUMBER_FORMAT " = 0" - , st->id, rd->name + , rd->name , rd->calculated_value - ); + ); + #endif + break; } - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: PHASE2 " + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: PHASE2 " " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " CALCULATED_NUMBER_FORMAT - , st->id, rd->name + , rd->name , rd->last_collected_value , rd->collected_value , rd->last_calculated_value , rd->calculated_value - ); + ); + #endif } // at this point we have all the calculated values ready // it is now time to interpolate values on a second boundary +#ifdef NETDATA_INTERNAL_CHECKS if(unlikely(now_collect_ut < next_store_ut)) { // this is collected in the same interpolation point - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: THIS IS IN THE SAME INTERPOLATION POINT", st->name); - -//#ifdef NETDATA_INTERNAL_CHECKS -// info("%s is collected in the same interpolation point: short by %llu microseconds", st->name, next_store_ut - now_collect_ut); -//#endif - } - - usec_t first_ut = last_stored_ut; - long long iterations = (now_collect_ut - last_stored_ut) / (update_every_ut); - if((now_collect_ut % (update_every_ut)) == 0) iterations++; - - for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) { -//#ifdef NETDATA_INTERNAL_CHECKS -// if(iterations < 0) { error("%s: iterations calculation wrapped! first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu", st->name, first_ut, last_stored_ut, next_store_ut, now_collect_ut); } -//#endif - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) { - debug(D_RRD_STATS, "%s: last_stored_ut = %0.3Lf (last updated time)", st->name, (long double)last_stored_ut/1000000.0); - debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/1000000.0); - } - - st->last_updated.tv_sec = (time_t) (next_store_ut / USEC_PER_SEC); - st->last_updated.tv_usec = 0; - - rrddim_foreach_read(rd, st) { - calculated_number new_value; - - switch(rd->algorithm) { - case RRD_ALGORITHM_INCREMENTAL: - new_value = (calculated_number) - ( rd->calculated_value - * (calculated_number)(next_store_ut - last_collect_ut) - / (calculated_number)(now_collect_ut - last_collect_ut) - ); - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: CALC2 INC " - CALCULATED_NUMBER_FORMAT " = " - CALCULATED_NUMBER_FORMAT - " * %llu" - " / %llu" - , st->id, rd->name - , new_value - , rd->calculated_value - , (next_store_ut - last_stored_ut) - , (now_collect_ut - last_stored_ut) - ); - - rd->calculated_value -= new_value; - new_value += rd->last_calculated_value; - rd->last_calculated_value = 0; - new_value /= (calculated_number)st->update_every; - - if(unlikely(next_store_ut - last_stored_ut < update_every_ut)) { - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: COLLECTION POINT IS SHORT " CALCULATED_NUMBER_FORMAT " - EXTRAPOLATING", - st->id, rd->name - , (calculated_number)(next_store_ut - last_stored_ut) - ); - new_value = new_value * (calculated_number)(st->update_every * 1000000) / (calculated_number)(next_store_ut - last_stored_ut); - } - break; - - case RRD_ALGORITHM_ABSOLUTE: - case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: - case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: - default: - if(iterations == 1) { - // this is the last iteration - // do not interpolate - // just show the calculated value - - new_value = rd->calculated_value; - } - else { - // we have missed an update - // interpolate in the middle values - - new_value = (calculated_number) - ( ( (rd->calculated_value - rd->last_calculated_value) - * (calculated_number)(next_store_ut - last_collect_ut) - / (calculated_number)(now_collect_ut - last_collect_ut) - ) - + rd->last_calculated_value - ); - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: CALC2 DEF " - CALCULATED_NUMBER_FORMAT " = (((" - "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")" - " * %llu" - " / %llu) + " CALCULATED_NUMBER_FORMAT - , st->id, rd->name - , new_value - , rd->calculated_value, rd->last_calculated_value - , (next_store_ut - first_ut) - , (now_collect_ut - first_ut), rd->last_calculated_value - ); - } - break; - } - - if(unlikely(!store_this_entry)) { - rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS); - continue; - } - - if(likely(rd->updated && rd->collections_counter > 1 && iterations < st->gap_when_lost_iterations_above)) { - rd->values[st->current_entry] = pack_storage_number(new_value, storage_flags ); - rd->last_stored_value = new_value; - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: STORE[%ld] " - CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT - , st->id, rd->name - , st->current_entry - , unpack_storage_number(rd->values[st->current_entry]), new_value - ); - } - else { - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING " - , st->id, rd->name - , st->current_entry - ); - rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS); - rd->last_stored_value = NAN; - } - - stored_entries++; - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) { - calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; - calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]); - calculated_number accuracy = accuracy_loss(t1, t2); - debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" - , st->id, rd->name - , st->current_entry - , t2 - , get_storage_number_flags(rd->values[st->current_entry]) - , t1 - , accuracy - , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" - ); - - rd->collected_volume += t1; - rd->stored_volume += t2; - accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume); - debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s" - , st->id, rd->name - , st->current_entry - , rd->stored_volume - , rd->collected_volume - , accuracy - , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" - ); - - } - } - // reset the storage flags for the next point, if any; - storage_flags = SN_EXISTS; - - st->counter++; - st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1; - last_stored_ut = next_store_ut; + rrdset_debug(st, "THIS IS IN THE SAME INTERPOLATION POINT"); + info("INTERNAL CHECK: host '%s', chart '%s' is collected in the same interpolation point: short by %llu microseconds", st->rrdhost->hostname, st->name, next_store_ut - now_collect_ut); } +#endif + + rrdset_done_interpolate(st + , update_every_ut + , last_stored_ut + , next_store_ut + , last_collect_ut + , now_collect_ut + , store_this_entry + , storage_flags + ); st->last_collected_total = st->collected_total; @@ -1206,29 +1387,35 @@ void rrdset_done(RRDSET *st) { if(unlikely(!rd->updated)) continue; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_collected_value, rd->collected_value); + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rd->name, rd->last_collected_value, rd->collected_value); + #endif rd->last_collected_value = rd->collected_value; switch(rd->algorithm) { case RRD_ALGORITHM_INCREMENTAL: if(unlikely(!first_entry)) { - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value); + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value + rd->calculated_value, rd->calculated_value); + #endif + rd->last_calculated_value += rd->calculated_value; } else { - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: THIS IS THE FIRST POINT", st->name); + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "THIS IS THE FIRST POINT"); + #endif } break; case RRD_ALGORITHM_ABSOLUTE: case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", st->id, rd->name, rd->last_calculated_value, rd->calculated_value); + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: setting last_calculated_value (old: " CALCULATED_NUMBER_FORMAT ") to last_calculated_value (new: " CALCULATED_NUMBER_FORMAT ")", rd->name, rd->last_calculated_value, rd->calculated_value); + #endif + rd->last_calculated_value = rd->calculated_value; break; } @@ -1237,18 +1424,20 @@ void rrdset_done(RRDSET *st) { rd->collected_value = 0; rd->updated = 0; - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s/%s: END " + #ifdef NETDATA_INTERNAL_CHECKS + rrdset_debug(st, "%s: END " " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT " last_calculated_value = " CALCULATED_NUMBER_FORMAT " calculated_value = " CALCULATED_NUMBER_FORMAT - , st->id, rd->name + , rd->name , rd->last_collected_value , rd->collected_value , rd->last_calculated_value , rd->calculated_value - ); + ); + #endif + } // ALL DONE ABOUT THE DATA UPDATE diff --git a/src/socket.c b/src/socket.c index 400c1ef4e..2b3821190 100644 --- a/src/socket.c +++ b/src/socket.c @@ -1,44 +1,267 @@ #include "common.h" -// connect_to() -// -// definition format: -// -// [PROTOCOL:]IP[%INTERFACE][:PORT] -// -// PROTOCOL = tcp or udp -// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6) -// INTERFACE = for IPv6 only, the network interface to use -// PORT = port number or service name +// -------------------------------------------------------------------------------------------------------------------- +// various library calls -int connect_to(const char *definition, int default_port, struct timeval *timeout) { +#ifdef __gnu_linux__ +#define LARGE_SOCK_SIZE 33554431 // don't ask why - I found it at brubeck source - I guess it is just a large number +#else +#define LARGE_SOCK_SIZE 4096 +#endif + +int sock_setnonblock(int fd) { + int flags; + + flags = fcntl(fd, F_GETFL); + flags |= O_NONBLOCK; + + int ret = fcntl(fd, F_SETFL, flags); + if(ret < 0) + error("Failed to set O_NONBLOCK on socket %d", fd); + + return ret; +} + +int sock_delnonblock(int fd) { + int flags; + + flags = fcntl(fd, F_GETFL); + flags &= ~O_NONBLOCK; + + int ret = fcntl(fd, F_SETFL, flags); + if(ret < 0) + error("Failed to remove O_NONBLOCK on socket %d", fd); + + return ret; +} + +int sock_setreuse(int fd, int reuse) { + int ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + + if(ret == -1) + error("Failed to set SO_REUSEADDR on socket %d", fd); + + return ret; +} + +int sock_setreuse_port(int fd, int reuse) { + int ret = -1; +#ifdef SO_REUSEPORT + ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &reuse, sizeof(reuse)); + if(ret == -1) + error("failed to set SO_REUSEPORT on socket %d", fd); +#endif + + return ret; +} + +int sock_enlarge_in(int fd) { + int ret, bs = LARGE_SOCK_SIZE; + + ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs)); + + if(ret == -1) + error("Failed to set SO_RCVBUF on socket %d", fd); + + return ret; +} + +int sock_enlarge_out(int fd) { + int ret, bs = LARGE_SOCK_SIZE; + ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs)); + + if(ret == -1) + error("Failed to set SO_SNDBUF on socket %d", fd); + + return ret; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// listening sockets + +int create_listen_socket4(int socktype, const char *ip, int port, int listen_backlog) { + int sock; + + debug(D_LISTENER, "LISTENER: IPv4 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); + + sock = socket(AF_INET, socktype, 0); + if(sock < 0) { + error("LISTENER: IPv4 socket() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + sock_setreuse(sock, 1); + sock_setreuse_port(sock, 1); + sock_setnonblock(sock); + sock_enlarge_in(sock); + + struct sockaddr_in name; + memset(&name, 0, sizeof(struct sockaddr_in)); + name.sin_family = AF_INET; + name.sin_port = htons (port); + + int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr); + if(ret != 1) { + error("LISTENER: Failed to convert IP '%s' to a valid IPv4 address.", ip); + close(sock); + return -1; + } + + if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { + close(sock); + error("LISTENER: IPv4 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + if(socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) { + close(sock); + error("LISTENER: IPv4 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + debug(D_LISTENER, "LISTENER: Listening on IPv4 ip '%s' port %d, socktype %d", ip, port, socktype); + return sock; +} + +int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int port, int listen_backlog) { + int sock; + int ipv6only = 1; + + debug(D_LISTENER, "LISTENER: IPv6 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); + + sock = socket(AF_INET6, socktype, 0); + if (sock < 0) { + error("LISTENER: IPv6 socket() on ip '%s' port %d, socktype %d, failed.", ip, port, socktype); + return -1; + } + + sock_setreuse(sock, 1); + sock_setreuse_port(sock, 1); + sock_setnonblock(sock); + sock_enlarge_in(sock); + + /* IPv6 only */ + if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0) + error("LISTENER: Cannot set IPV6_V6ONLY on ip '%s' port %d, socktype %d.", ip, port, socktype); + + struct sockaddr_in6 name; + memset(&name, 0, sizeof(struct sockaddr_in6)); + name.sin6_family = AF_INET6; + name.sin6_port = htons ((uint16_t) port); + name.sin6_scope_id = scope_id; + + int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr); + if(ret != 1) { + error("LISTENER: Failed to convert IP '%s' to a valid IPv6 address.", ip); + close(sock); + return -1; + } + + name.sin6_scope_id = scope_id; + + if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { + close(sock); + error("LISTENER: IPv6 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + if (socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) { + close(sock); + error("LISTENER: IPv6 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + debug(D_LISTENER, "LISTENER: Listening on IPv6 ip '%s' port %d, socktype %d", ip, port, socktype); + return sock; +} + +static inline int listen_sockets_add(LISTEN_SOCKETS *sockets, int fd, int socktype, const char *protocol, const char *ip, int port) { + if(sockets->opened >= MAX_LISTEN_FDS) { + error("LISTENER: Too many listening sockets. Failed to add listening %s socket at ip '%s' port %d, protocol %s, socktype %d", protocol, ip, port, protocol, socktype); + close(fd); + return -1; + } + + sockets->fds[sockets->opened] = fd; + + char buffer[100 + 1]; + snprintfz(buffer, 100, "%s:[%s]:%d", protocol, ip, port); + sockets->fds_names[sockets->opened] = strdupz(buffer); + sockets->fds_types[sockets->opened] = socktype; + + sockets->opened++; + return 0; +} + +int listen_sockets_check_is_member(LISTEN_SOCKETS *sockets, int fd) { + size_t i; + for(i = 0; i < sockets->opened ;i++) + if(sockets->fds[i] == fd) return 1; + + return 0; +} + +static inline void listen_sockets_init(LISTEN_SOCKETS *sockets) { + size_t i; + for(i = 0; i < MAX_LISTEN_FDS ;i++) { + sockets->fds[i] = -1; + sockets->fds_names[i] = NULL; + sockets->fds_types[i] = -1; + } + + sockets->opened = 0; + sockets->failed = 0; +} + +void listen_sockets_close(LISTEN_SOCKETS *sockets) { + size_t i; + for(i = 0; i < sockets->opened ;i++) { + close(sockets->fds[i]); + sockets->fds[i] = -1; + + freez(sockets->fds_names[i]); + sockets->fds_names[i] = NULL; + + sockets->fds_types[i] = -1; + } + + sockets->opened = 0; + sockets->failed = 0; +} + +static inline int bind_to_one(LISTEN_SOCKETS *sockets, const char *definition, int default_port, int listen_backlog) { + int added = 0; struct addrinfo hints; - struct addrinfo *ai_head = NULL, *ai = NULL; + struct addrinfo *result = NULL, *rp = NULL; char buffer[strlen(definition) + 1]; strcpy(buffer, definition); - char default_service[10 + 1]; - snprintfz(default_service, 10, "%d", default_port); + char buffer2[10 + 1]; + snprintfz(buffer2, 10, "%d", default_port); + + char *ip = buffer, *port = buffer2, *interface = "";; - char *host = buffer, *service = default_service, *interface = ""; int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; - uint32_t scope_id = 0; + const char *protocol_str = "tcp"; - if(strncmp(host, "tcp:", 4) == 0) { - host += 4; + if(strncmp(ip, "tcp:", 4) == 0) { + ip += 4; protocol = IPPROTO_TCP; socktype = SOCK_STREAM; + protocol_str = "tcp"; } - else if(strncmp(host, "udp:", 4) == 0) { - host += 4; + else if(strncmp(ip, "udp:", 4) == 0) { + ip += 4; protocol = IPPROTO_UDP; socktype = SOCK_DGRAM; + protocol_str = "udp"; } - char *e = host; + char *e = ip; if(*e == '[') { - e = ++host; + e = ++ip; while(*e && *e != ']') e++; if(*e == ']') { *e = '\0'; @@ -57,26 +280,142 @@ int connect_to(const char *definition, int default_port, struct timeval *timeout } if(*e == ':') { + port = e + 1; *e = '\0'; - e++; - service = e; } - debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP); + uint32_t scope_id = 0; + if(*interface) { + scope_id = if_nametoindex(interface); + if(!scope_id) + error("LISTENER: Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + } - if(!*host) { - error("Definition '%s' does not specify a host.", definition); + if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all")) + ip = NULL; + + if(!*port) + port = buffer2; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = socktype; + hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ + hints.ai_protocol = protocol; + hints.ai_canonname = NULL; + hints.ai_addr = NULL; + hints.ai_next = NULL; + + int r = getaddrinfo(ip, port, &hints, &result); + if (r != 0) { + error("LISTENER: getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r)); return -1; } - if(*interface) { - scope_id = if_nametoindex(interface); - if(!scope_id) - error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + for (rp = result; rp != NULL; rp = rp->ai_next) { + int fd = -1; + + char rip[INET_ADDRSTRLEN + INET6_ADDRSTRLEN] = "INVALID"; + int rport = default_port; + + switch (rp->ai_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr; + inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN); + rport = ntohs(sin->sin_port); + // info("Attempting to listen on IPv4 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); + fd = create_listen_socket4(socktype, rip, rport, listen_backlog); + break; + } + + case AF_INET6: { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr; + inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN); + rport = ntohs(sin6->sin6_port); + // info("Attempting to listen on IPv6 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); + fd = create_listen_socket6(socktype, scope_id, rip, rport, listen_backlog); + break; + } + + default: + debug(D_LISTENER, "LISTENER: Unknown socket family %d", rp->ai_addr->sa_family); + break; + } + + if (fd == -1) { + error("LISTENER: Cannot bind to ip '%s', port %d", rip, rport); + sockets->failed++; + } + else { + listen_sockets_add(sockets, fd, socktype, protocol_str, rip, rport); + added++; + } } - if(!*service) - service = default_service; + freeaddrinfo(result); + + return added; +} + +int listen_sockets_setup(LISTEN_SOCKETS *sockets) { + listen_sockets_init(sockets); + + sockets->backlog = (int) config_get_number(sockets->config_section, "listen backlog", sockets->backlog); + + int old_port = sockets->default_port; + sockets->default_port = (int) config_get_number(sockets->config_section, "default port", sockets->default_port); + if(sockets->default_port < 1 || sockets->default_port > 65535) { + error("LISTENER: Invalid listen port %d given. Defaulting to %d.", sockets->default_port, old_port); + sockets->default_port = (int) config_set_number(sockets->config_section, "default port", old_port); + } + debug(D_OPTIONS, "LISTENER: Default listen port set to %d.", sockets->default_port); + + char *s = config_get(sockets->config_section, "bind to", sockets->default_bind_to); + while(*s) { + char *e = s; + + // skip separators, moving both s(tart) and e(nd) + while(isspace(*e) || *e == ',') s = ++e; + + // move e(nd) to the first separator + while(*e && !isspace(*e) && *e != ',') e++; + + // is there anything? + if(!*s || s == e) break; + + char buf[e - s + 1]; + strncpyz(buf, s, e - s); + bind_to_one(sockets, buf, sockets->default_port, sockets->backlog); + + s = e; + } + + if(sockets->failed) { + size_t i; + for(i = 0; i < sockets->opened ;i++) + info("LISTENER: Listen socket %s opened successfully.", sockets->fds_names[i]); + } + + return (int)sockets->opened; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// connect to another host/port + +// _connect_to() +// protocol IPPROTO_TCP, IPPROTO_UDP +// socktype SOCK_STREAM, SOCK_DGRAM +// host the destination hostname or IP address (IPv4 or IPv6) to connect to +// if it resolves to many IPs, all are tried (IPv4 and IPv6) +// scope_id the if_index id of the interface to use for connecting (0 = any) +// (used only under IPv6) +// service the service name or port to connect to +// timeout the timeout for establishing a connection + +static inline int _connect_to(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout) { + struct addrinfo hints; + struct addrinfo *ai_head = NULL, *ai = NULL; memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; /* Allow IPv4 or IPv6 */ @@ -103,52 +442,52 @@ int connect_to(const char *definition, int default_port, struct timeval *timeout char servBfr[NI_MAXSERV + 1]; getnameinfo(ai->ai_addr, - ai->ai_addrlen, - hostBfr, - sizeof(hostBfr), - servBfr, - sizeof(servBfr), - NI_NUMERICHOST | NI_NUMERICSERV); + ai->ai_addrlen, + hostBfr, + sizeof(hostBfr), + servBfr, + sizeof(servBfr), + NI_NUMERICHOST | NI_NUMERICSERV); debug(D_CONNECT_TO, "Address info: host = '%s', service = '%s', ai_flags = 0x%02X, ai_family = %d (PF_INET = %d, PF_INET6 = %d), ai_socktype = %d (SOCK_STREAM = %d, SOCK_DGRAM = %d), ai_protocol = %d (IPPROTO_TCP = %d, IPPROTO_UDP = %d), ai_addrlen = %lu (sockaddr_in = %lu, sockaddr_in6 = %lu)", - hostBfr, - servBfr, - (unsigned int)ai->ai_flags, - ai->ai_family, - PF_INET, - PF_INET6, - ai->ai_socktype, - SOCK_STREAM, - SOCK_DGRAM, - ai->ai_protocol, - IPPROTO_TCP, - IPPROTO_UDP, - (unsigned long)ai->ai_addrlen, - (unsigned long)sizeof(struct sockaddr_in), - (unsigned long)sizeof(struct sockaddr_in6)); + hostBfr, + servBfr, + (unsigned int)ai->ai_flags, + ai->ai_family, + PF_INET, + PF_INET6, + ai->ai_socktype, + SOCK_STREAM, + SOCK_DGRAM, + ai->ai_protocol, + IPPROTO_TCP, + IPPROTO_UDP, + (unsigned long)ai->ai_addrlen, + (unsigned long)sizeof(struct sockaddr_in), + (unsigned long)sizeof(struct sockaddr_in6)); switch (ai->ai_addr->sa_family) { case PF_INET: { struct sockaddr_in *pSadrIn = (struct sockaddr_in *)ai->ai_addr; debug(D_CONNECT_TO, "ai_addr = sin_family: %d (AF_INET = %d, AF_INET6 = %d), sin_addr: '%s', sin_port: '%s'", - pSadrIn->sin_family, - AF_INET, - AF_INET6, - hostBfr, - servBfr); + pSadrIn->sin_family, + AF_INET, + AF_INET6, + hostBfr, + servBfr); break; } case PF_INET6: { struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr; debug(D_CONNECT_TO,"ai_addr = sin6_family: %d (AF_INET = %d, AF_INET6 = %d), sin6_addr: '%s', sin6_port: '%s', sin6_flowinfo: %u, sin6_scope_id: %u", - pSadrIn6->sin6_family, - AF_INET, - AF_INET6, - hostBfr, - servBfr, - pSadrIn6->sin6_flowinfo, - pSadrIn6->sin6_scope_id); + pSadrIn6->sin6_family, + AF_INET, + AF_INET6, + hostBfr, + servBfr, + pSadrIn6->sin6_flowinfo, + pSadrIn6->sin6_scope_id); break; } @@ -180,6 +519,85 @@ int connect_to(const char *definition, int default_port, struct timeval *timeout return fd; } +// connect_to() +// +// definition format: +// +// [PROTOCOL:]IP[%INTERFACE][:PORT] +// +// PROTOCOL = tcp or udp +// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6) +// INTERFACE = for IPv6 only, the network interface to use +// PORT = port number or service name + +int connect_to(const char *definition, int default_port, struct timeval *timeout) { + char buffer[strlen(definition) + 1]; + strcpy(buffer, definition); + + char default_service[10 + 1]; + snprintfz(default_service, 10, "%d", default_port); + + char *host = buffer, *service = default_service, *interface = ""; + int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; + uint32_t scope_id = 0; + + if(strncmp(host, "tcp:", 4) == 0) { + host += 4; + protocol = IPPROTO_TCP; + socktype = SOCK_STREAM; + } + else if(strncmp(host, "udp:", 4) == 0) { + host += 4; + protocol = IPPROTO_UDP; + socktype = SOCK_DGRAM; + } + + char *e = host; + if(*e == '[') { + e = ++host; + while(*e && *e != ']') e++; + if(*e == ']') { + *e = '\0'; + e++; + } + } + else { + while(*e && *e != ':' && *e != '%') e++; + } + + if(*e == '%') { + *e = '\0'; + e++; + interface = e; + while(*e && *e != ':') e++; + } + + if(*e == ':') { + *e = '\0'; + e++; + service = e; + } + + debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP); + + if(!*host) { + error("Definition '%s' does not specify a host.", definition); + return -1; + } + + if(*interface) { + scope_id = if_nametoindex(interface); + if(!scope_id) + error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + } + + if(!*service) + service = default_service; + + + return _connect_to(protocol, socktype, host, scope_id, service, timeout); +} + int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) { int sock = -1; @@ -213,6 +631,10 @@ int connect_to_one_of(const char *destination, int default_port, struct timeval return sock; } + +// -------------------------------------------------------------------------------------------------------------------- +// helpers to send/receive data in one call, in blocking mode, with a timeout + ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) { for(;;) { struct pollfd fd = { @@ -274,3 +696,455 @@ ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) return send(sockfd, buf, len, flags); } + + +// -------------------------------------------------------------------------------------------------------------------- +// accept4() replacement for systems that do not have one + +#ifndef HAVE_ACCEPT4 +int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) { + int fd = accept(sock, addr, addrlen); + int newflags = 0; + + if (fd < 0) return fd; + + if (flags & SOCK_NONBLOCK) { + newflags |= O_NONBLOCK; + flags &= ~SOCK_NONBLOCK; + } + +#ifdef SOCK_CLOEXEC +#ifdef O_CLOEXEC + if (flags & SOCK_CLOEXEC) { + newflags |= O_CLOEXEC; + flags &= ~SOCK_CLOEXEC; + } +#endif +#endif + + if (flags) { + errno = -EINVAL; + return -1; + } + + if (fcntl(fd, F_SETFL, newflags) < 0) { + int saved_errno = errno; + close(fd); + errno = saved_errno; + return -1; + } + + return fd; +} +#endif + + +// -------------------------------------------------------------------------------------------------------------------- +// accept_socket() - accept a socket and store client IP and port + +int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize) { + struct sockaddr_storage sadr; + socklen_t addrlen = sizeof(sadr); + + int nfd = accept4(fd, (struct sockaddr *)&sadr, &addrlen, flags); + if (nfd >= 0) { + if (getnameinfo((struct sockaddr *)&sadr, addrlen, client_ip, (socklen_t)ipsize, client_port, (socklen_t)portsize, NI_NUMERICHOST | NI_NUMERICSERV) != 0) { + error("LISTENER: cannot getnameinfo() on received client connection."); + strncpyz(client_ip, "UNKNOWN", ipsize - 1); + strncpyz(client_port, "UNKNOWN", portsize - 1); + } + + client_ip[ipsize - 1] = '\0'; + client_port[portsize - 1] = '\0'; + + switch (((struct sockaddr *)&sadr)->sa_family) { + case AF_INET: + debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); + break; + + case AF_INET6: + if (strncmp(client_ip, "::ffff:", 7) == 0) { + memmove(client_ip, &client_ip[7], strlen(&client_ip[7]) + 1); + debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); + } else + debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd); + break; + + default: + debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd); + break; + } + } + + return nfd; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// poll() based listener +// this should be the fastest possible listener for up to 100 sockets +// above 100, an epoll() interface is needed on Linux + +#define POLL_FDS_INCREASE_STEP 10 + +#define POLLINFO_FLAG_SERVER_SOCKET 0x00000001 +#define POLLINFO_FLAG_CLIENT_SOCKET 0x00000002 + +struct pollinfo { + size_t slot; + char *client; + struct pollinfo *next; + uint32_t flags; + int socktype; + + void *data; +}; + +struct poll { + size_t slots; + size_t used; + size_t min; + size_t max; + struct pollfd *fds; + struct pollinfo *inf; + struct pollinfo *first_free; + + void *(*add_callback)(int fd, short int *events); + void (*del_callback)(int fd, void *data); + int (*rcv_callback)(int fd, int socktype, void *data, short int *events); + int (*snd_callback)(int fd, int socktype, void *data, short int *events); +}; + +static inline struct pollinfo *poll_add_fd(struct poll *p, int fd, int socktype, short int events, uint32_t flags) { + debug(D_POLLFD, "POLLFD: ADD: request to add fd %d, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", fd, p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + if(unlikely(fd < 0)) return NULL; + + if(unlikely(!p->first_free)) { + size_t new_slots = p->slots + POLL_FDS_INCREASE_STEP; + debug(D_POLLFD, "POLLFD: ADD: increasing size (current = %zu, new = %zu, used = %zu, min = %zu, max = %zu)", p->slots, new_slots, p->used, p->min, p->max); + + p->fds = reallocz(p->fds, sizeof(struct pollfd) * new_slots); + p->inf = reallocz(p->inf, sizeof(struct pollinfo) * new_slots); + + ssize_t i; + for(i = new_slots - 1; i >= (ssize_t)p->slots ; i--) { + debug(D_POLLFD, "POLLFD: ADD: resetting new slot %zd", i); + p->fds[i].fd = -1; + p->fds[i].events = 0; + p->fds[i].revents = 0; + + p->inf[i].slot = (size_t)i; + p->inf[i].flags = 0; + p->inf[i].socktype = -1; + p->inf[i].client = NULL; + p->inf[i].data = NULL; + p->inf[i].next = p->first_free; + p->first_free = &p->inf[i]; + } + + p->slots = new_slots; + } + + struct pollinfo *pi = p->first_free; + p->first_free = p->first_free->next; + + debug(D_POLLFD, "POLLFD: ADD: selected slot %zu, next free is %zd", pi->slot, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + struct pollfd *pf = &p->fds[pi->slot]; + pf->fd = fd; + pf->events = events; + pf->revents = 0; + + pi->socktype = socktype; + pi->flags = flags; + pi->next = NULL; + + p->used++; + if(unlikely(pi->slot > p->max)) + p->max = pi->slot; + + if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) { + pi->data = p->add_callback(fd, &pf->events); + } + + if(pi->flags & POLLINFO_FLAG_SERVER_SOCKET) { + p->min = pi->slot; + } + + debug(D_POLLFD, "POLLFD: ADD: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + return pi; +} + +static inline void poll_close_fd(struct poll *p, struct pollinfo *pi) { + struct pollfd *pf = &p->fds[pi->slot]; + debug(D_POLLFD, "POLLFD: DEL: request to clear slot %zu (fd %d), old next free was %zd", pi->slot, pf->fd, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + if(unlikely(pf->fd == -1)) return; + + if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) { + p->del_callback(pf->fd, pi->data); + } + + close(pf->fd); + pf->fd = -1; + pf->events = 0; + pf->revents = 0; + + pi->socktype = -1; + pi->flags = 0; + pi->data = NULL; + + freez(pi->client); + pi->client = NULL; + + pi->next = p->first_free; + p->first_free = pi; + + p->used--; + if(p->max == pi->slot) { + p->max = p->min; + ssize_t i; + for(i = (ssize_t)pi->slot; i > (ssize_t)p->min ;i--) { + if (unlikely(p->fds[i].fd != -1)) { + p->max = (size_t)i; + break; + } + } + } + + debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); +} + +static void *add_callback_default(int fd, short int *events) { + (void)fd; + (void)events; + + return NULL; +} +static void del_callback_default(int fd, void *data) { + (void)fd; + (void)data; + + if(data) + error("POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak"); +} + +static int rcv_callback_default(int fd, int socktype, void *data, short int *events) { + (void)socktype; + (void)data; + (void)events; + + char buffer[1024 + 1]; + + ssize_t rc; + do { + rc = recv(fd, buffer, 1024, MSG_DONTWAIT); + if (rc < 0) { + // read failed + if (errno != EWOULDBLOCK && errno != EAGAIN) { + error("POLLFD: recv() failed."); + return -1; + } + } else if (rc) { + // data received + info("POLLFD: internal error: discarding %zd bytes received on socket %d", rc, fd); + } + } while (rc != -1); + + return 0; +} + +static int snd_callback_default(int fd, int socktype, void *data, short int *events) { + (void)socktype; + (void)data; + (void)events; + + *events &= ~POLLOUT; + + info("POLLFD: internal error: nothing to send on socket %d", fd); + return 0; +} + +void poll_events_cleanup(void *data) { + struct poll *p = (struct poll *)data; + + size_t i; + for(i = 0 ; i <= p->max ; i++) { + struct pollinfo *pi = &p->inf[i]; + poll_close_fd(p, pi); + } + + freez(p->fds); + freez(p->inf); +} + +void poll_events(LISTEN_SOCKETS *sockets + , void *(*add_callback)(int fd, short int *events) + , void (*del_callback)(int fd, void *data) + , int (*rcv_callback)(int fd, int socktype, void *data, short int *events) + , int (*snd_callback)(int fd, int socktype, void *data, short int *events) + , void *data +) { + int retval; + + struct poll p = { + .slots = 0, + .used = 0, + .max = 0, + .fds = NULL, + .inf = NULL, + .first_free = NULL, + + .add_callback = add_callback?add_callback:add_callback_default, + .del_callback = del_callback?del_callback:del_callback_default, + .rcv_callback = rcv_callback?rcv_callback:rcv_callback_default, + .snd_callback = snd_callback?snd_callback:snd_callback_default + }; + + size_t i; + for(i = 0; i < sockets->opened ;i++) { + struct pollinfo *pi = poll_add_fd(&p, sockets->fds[i], sockets->fds_types[i], POLLIN, POLLINFO_FLAG_SERVER_SOCKET); + pi->data = data; + info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN"); + } + + int timeout = -1; // wait forever + + pthread_cleanup_push(poll_events_cleanup, &p); + + for(;;) { + if(unlikely(netdata_exit)) break; + + debug(D_POLLFD, "POLLFD: LISTENER: Waiting on %zu sockets...", p.max + 1); + retval = poll(p.fds, p.max + 1, timeout); + + if(unlikely(retval == -1)) { + error("POLLFD: LISTENER: poll() failed."); + continue; + } + else if(unlikely(!retval)) { + debug(D_POLLFD, "POLLFD: LISTENER: poll() timeout."); + continue; + } + + if(unlikely(netdata_exit)) break; + + for(i = 0 ; i <= p.max ; i++) { + struct pollfd *pf = &p.fds[i]; + struct pollinfo *pi = &p.inf[i]; + int fd = pf->fd; + short int revents = pf->revents; + pf->revents = 0; + + if(unlikely(fd == -1)) { + debug(D_POLLFD, "POLLFD: LISTENER: ignoring slot %zu, it does not have an fd", i); + continue; + } + + debug(D_POLLFD, "POLLFD: LISTENER: processing events for slot %zu (events = %d, revents = %d)", i, pf->events, revents); + + if(revents & POLLIN || revents & POLLPRI) { + // receiving data + + if(likely(pi->flags & POLLINFO_FLAG_SERVER_SOCKET)) { + // new connection + // debug(D_POLLFD, "POLLFD: LISTENER: accepting connections from slot %zu (fd %d)", i, fd); + + switch(pi->socktype) { + case SOCK_STREAM: { + // a TCP socket + // we accept the connection + + int nfd; + do { + char client_ip[NI_MAXHOST + 1]; + char client_port[NI_MAXSERV + 1]; + + debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", i, fd); + nfd = accept_socket(fd, SOCK_NONBLOCK, client_ip, NI_MAXHOST + 1, client_port, NI_MAXSERV + 1); + if (nfd < 0) { + // accept failed + + debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", i, fd); + + if(errno != EWOULDBLOCK && errno != EAGAIN) + error("POLLFD: LISTENER: accept() failed."); + + break; + } + else { + // accept ok + info("POLLFD: LISTENER: client '[%s]:%s' connected to '%s'", client_ip, client_port, sockets->fds_names[i]); + poll_add_fd(&p, nfd, SOCK_STREAM, POLLIN, POLLINFO_FLAG_CLIENT_SOCKET); + + // it may have realloced them, so refresh our pointers + pf = &p.fds[i]; + pi = &p.inf[i]; + } + } while (nfd != -1); + break; + } + + case SOCK_DGRAM: { + // a UDP socket + // we read data from the server socket + + debug(D_POLLFD, "POLLFD: LISTENER: reading data from UDP slot %zu (fd %d)", i, fd); + + p.rcv_callback(fd, pi->socktype, pi->data, &pf->events); + break; + } + + default: { + error("POLLFD: LISTENER: Unknown socktype %d on slot %zu", pi->socktype, pi->slot); + break; + } + } + } + + if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) { + // read data from client TCP socket + debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", i, fd); + + if (p.rcv_callback(fd, pi->socktype, pi->data, &pf->events) == -1) { + poll_close_fd(&p, pi); + continue; + } + } + } + + if(unlikely(revents & POLLOUT)) { + // sending data + debug(D_POLLFD, "POLLFD: LISTENER: sending data to socket on slot %zu (fd %d)", i, fd); + + if (p.snd_callback(fd, pi->socktype, pi->data, &pf->events) == -1) { + poll_close_fd(&p, pi); + continue; + } + } + + if(unlikely(revents & POLLERR)) { + error("POLLFD: LISTENER: processing POLLERR events for slot %zu (events = %d, revents = %d)", i, pf->events, revents); + poll_close_fd(&p, pi); + continue; + } + + if(unlikely(revents & POLLHUP)) { + error("POLLFD: LISTENER: processing POLLHUP events for slot %zu (events = %d, revents = %d)", i, pf->events, pf->revents); + poll_close_fd(&p, pi); + continue; + } + + if(unlikely(revents & POLLNVAL)) { + error("POLLFD: LISTENER: processing POLLNVAP events for slot %zu (events = %d, revents = %d)", i, pf->events, revents); + poll_close_fd(&p, pi); + continue; + } + } + } + + pthread_cleanup_pop(1); + debug(D_POLLFD, "POLLFD: LISTENER: cleanup completed"); +} diff --git a/src/socket.h b/src/socket.h index 89c154a61..bb95347ab 100644 --- a/src/socket.h +++ b/src/socket.h @@ -1,14 +1,61 @@ -// -// Created by costa on 24/12/2016. -// - #ifndef NETDATA_SOCKET_H #define NETDATA_SOCKET_H +#ifndef MAX_LISTEN_FDS +#define MAX_LISTEN_FDS 50 +#endif + +typedef struct listen_sockets { + const char *config_section; // the netdata configuration section to read settings from + const char *default_bind_to; // the default bind to configuration string + int default_port; // the default port to use + int backlog; // the default listen backlog to use + + size_t opened; // the number of sockets opened + size_t failed; // the number of sockets attempted to open, but failed + int fds[MAX_LISTEN_FDS]; // the open sockets + char *fds_names[MAX_LISTEN_FDS]; // descriptions for the open sockets + int fds_types[MAX_LISTEN_FDS]; // the socktype for the open sockets +} LISTEN_SOCKETS; + +extern int listen_sockets_setup(LISTEN_SOCKETS *sockets); +extern void listen_sockets_close(LISTEN_SOCKETS *sockets); + extern int connect_to(const char *definition, int default_port, struct timeval *timeout); extern int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size); extern ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); extern ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); +extern int sock_setnonblock(int fd); +extern int sock_delnonblock(int fd); +extern int sock_setreuse(int fd, int reuse); +extern int sock_setreuse_port(int fd, int reuse); +extern int sock_enlarge_in(int fd); +extern int sock_enlarge_out(int fd); + +extern int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize); + +#ifndef HAVE_ACCEPT4 +extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags); + +#ifndef SOCK_NONBLOCK +#define SOCK_NONBLOCK 00004000 +#endif /* #ifndef SOCK_NONBLOCK */ + +#ifndef SOCK_CLOEXEC +#define SOCK_CLOEXEC 02000000 +#endif /* #ifndef SOCK_CLOEXEC */ + +#endif /* #ifndef HAVE_ACCEPT4 */ + + +extern void poll_events(LISTEN_SOCKETS *sockets + , void *(*add_callback)(int fd, short int *events) + , void (*del_callback)(int fd, void *data) + , int (*rcv_callback)(int fd, int socktype, void *data, short int *events) + , int (*snd_callback)(int fd, int socktype, void *data, short int *events) + , void *data +); + #endif //NETDATA_SOCKET_H diff --git a/src/statistical.c b/src/statistical.c new file mode 100644 index 000000000..807bc25ea --- /dev/null +++ b/src/statistical.c @@ -0,0 +1,459 @@ +#include "common.h" + +// -------------------------------------------------------------------------------------------------------------------- + +inline long double sum_and_count(long double *series, size_t entries, size_t *count) { + if(unlikely(entries == 0)) { + if(likely(count)) + *count = 0; + + return NAN; + } + + if(unlikely(entries == 1)) { + if(likely(count)) + *count = (isnan(series[0])?0:1); + + return series[0]; + } + + size_t i, c = 0; + long double sum = 0; + + for(i = 0; i < entries ; i++) { + long double value = series[i]; + if(unlikely(isnan(value) || isinf(value))) continue; + c++; + sum += value; + } + + if(likely(count)) + *count = c; + + if(unlikely(c == 0)) + return NAN; + + return sum; +} + +inline long double sum(long double *series, size_t entries) { + return sum_and_count(series, entries, NULL); +} + +inline long double average(long double *series, size_t entries) { + size_t count = 0; + long double sum = sum_and_count(series, entries, &count); + + if(unlikely(count == 0)) + return NAN; + + return sum / (long double)count; +} + +// -------------------------------------------------------------------------------------------------------------------- + +long double moving_average(long double *series, size_t entries, size_t period) { + if(unlikely(period <= 0)) + return 0.0; + + size_t i, count; + long double sum = 0, avg = 0; + long double p[period]; + + for(count = 0; count < period ; count++) + p[count] = 0.0; + + for(i = 0, count = 0; i < entries; i++) { + long double value = series[i]; + if(unlikely(isnan(value) || isinf(value))) continue; + + if(unlikely(count < period)) { + sum += value; + avg = (count == period - 1) ? sum / (long double)period : 0; + } + else { + sum = sum - p[count % period] + value; + avg = sum / (long double)period; + } + + p[count % period] = value; + count++; + } + + return avg; +} + +// -------------------------------------------------------------------------------------------------------------------- + +static int qsort_compare(const void *a, const void *b) { + long double *p1 = (long double *)a, *p2 = (long double *)b; + long double n1 = *p1, n2 = *p2; + + if(unlikely(isnan(n1) || isnan(n2))) { + if(isnan(n1) && !isnan(n2)) return -1; + if(!isnan(n1) && isnan(n2)) return 1; + return 0; + } + if(unlikely(isinf(n1) || isinf(n2))) { + if(!isinf(n1) && isinf(n2)) return -1; + if(isinf(n1) && !isinf(n2)) return 1; + return 0; + } + + if(unlikely(n1 < n2)) return -1; + if(unlikely(n1 > n2)) return 1; + return 0; +} + +inline void sort_series(long double *series, size_t entries) { + qsort(series, entries, sizeof(long double), qsort_compare); +} + +inline long double *copy_series(long double *series, size_t entries) { + long double *copy = mallocz(sizeof(long double) * entries); + memcpy(copy, series, sizeof(long double) * entries); + return copy; +} + +long double median_on_sorted_series(long double *series, size_t entries) { + if(unlikely(entries == 0)) + return NAN; + + if(unlikely(entries == 1)) + return series[0]; + + if(unlikely(entries == 2)) + return (series[0] + series[1]) / 2; + + long double avg; + if(entries % 2 == 0) { + size_t m = entries / 2; + avg = (series[m] + series[m + 1]) / 2; + } + else { + avg = series[entries / 2]; + } + + return avg; +} + +long double median(long double *series, size_t entries) { + if(unlikely(entries == 0)) + return NAN; + + if(unlikely(entries == 1)) + return series[0]; + + if(unlikely(entries == 2)) + return (series[0] + series[1]) / 2; + + long double *copy = copy_series(series, entries); + sort_series(copy, entries); + + long double avg = median_on_sorted_series(copy, entries); + + freez(copy); + return avg; +} + +// -------------------------------------------------------------------------------------------------------------------- + +long double moving_median(long double *series, size_t entries, size_t period) { + if(entries <= period) + return median(series, entries); + + long double *data = copy_series(series, entries); + + size_t i; + for(i = period; i < entries; i++) { + data[i - period] = median(&series[i - period], period); + } + + long double avg = median(data, entries - period); + freez(data); + return avg; +} + +// -------------------------------------------------------------------------------------------------------------------- + +// http://stackoverflow.com/a/15150143/4525767 +long double running_median_estimate(long double *series, size_t entries) { + long double median = 0.0f; + long double average = 0.0f; + size_t i; + + for(i = 0; i < entries ; i++) { + long double value = series[i]; + if(unlikely(isnan(value) || isinf(value))) continue; + + average += ( value - average ) * 0.1f; // rough running average. + median += copysignl( average * 0.01, value - median ); + } + + return median; +} + +// -------------------------------------------------------------------------------------------------------------------- + +long double standard_deviation(long double *series, size_t entries) { + if(unlikely(entries < 1)) + return NAN; + + if(unlikely(entries == 1)) + return series[0]; + + size_t i, count = 0; + long double sum = 0; + + for(i = 0; i < entries ; i++) { + long double value = series[i]; + if(unlikely(isnan(value) || isinf(value))) continue; + + count++; + sum += value; + } + + if(unlikely(count == 0)) + return NAN; + + if(unlikely(count == 1)) + return sum; + + long double average = sum / (long double)count; + + for(i = 0, count = 0, sum = 0; i < entries ; i++) { + long double value = series[i]; + if(unlikely(isnan(value) || isinf(value))) continue; + + count++; + sum += powl(value - average, 2); + } + + if(unlikely(count == 0)) + return NAN; + + if(unlikely(count == 1)) + return average; + + long double variance = sum / (long double)(count - 1); // remove -1 to have a population stddev + + long double stddev = sqrtl(variance); + return stddev; +} + +// -------------------------------------------------------------------------------------------------------------------- + +long double single_exponential_smoothing(long double *series, size_t entries, long double alpha) { + size_t i, count = 0; + long double level = 0, sum = 0; + + if(unlikely(isnan(alpha))) + alpha = 0.3; + + for(i = 0; i < entries ; i++) { + long double value = series[i]; + if(unlikely(isnan(value) || isinf(value))) continue; + count++; + + sum += value; + + long double last_level = level; + level = alpha * value + (1.0 - alpha) * last_level; + } + + return level; +} + +// -------------------------------------------------------------------------------------------------------------------- + +// http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/ +long double double_exponential_smoothing(long double *series, size_t entries, long double alpha, long double beta, long double *forecast) { + size_t i, count = 0; + long double level = series[0], trend, sum; + + if(unlikely(isnan(alpha))) + alpha = 0.3; + + if(unlikely(isnan(beta))) + beta = 0.05; + + if(likely(entries > 1)) + trend = series[1] - series[0]; + else + trend = 0; + + sum = series[0]; + + for(i = 1; i < entries ; i++) { + long double value = series[i]; + if(unlikely(isnan(value) || isinf(value))) continue; + count++; + + sum += value; + + long double last_level = level; + + level = alpha * value + (1.0 - alpha) * (level + trend); + trend = beta * (level - last_level) + (1.0 - beta) * trend; + } + + if(forecast) + *forecast = level + trend; + + return level; +} + +// -------------------------------------------------------------------------------------------------------------------- + +/* + * Based on th R implementation + * + * a: level component + * b: trend component + * s: seasonal component + * + * Additive: + * + * Yhat[t+h] = a[t] + h * b[t] + s[t + 1 + (h - 1) mod p], + * a[t] = α (Y[t] - s[t-p]) + (1-α) (a[t-1] + b[t-1]) + * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1] + * s[t] = γ (Y[t] - a[t]) + (1-γ) s[t-p] + * + * Multiplicative: + * + * Yhat[t+h] = (a[t] + h * b[t]) * s[t + 1 + (h - 1) mod p], + * a[t] = α (Y[t] / s[t-p]) + (1-α) (a[t-1] + b[t-1]) + * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1] + * s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p] + */ +static int __HoltWinters( + long double *series, + int entries, // start_time + h + + long double alpha, // alpha parameter of Holt-Winters Filter. + long double beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing. + long double gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted. + + int *seasonal, + int *period, + long double *a, // Start value for level (a[0]). + long double *b, // Start value for trend (b[0]). + long double *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0]) + + /* return values */ + long double *SSE, // The final sum of squared errors achieved in optimizing + long double *level, // Estimated values for the level component (size entries - t + 2) + long double *trend, // Estimated values for the trend component (size entries - t + 2) + long double *season // Estimated values for the seasonal component (size entries - t + 2) +) +{ + if(unlikely(entries < 4)) + return 0; + + int start_time = 2; + + long double res = 0, xhat = 0, stmp = 0; + int i, i0, s0; + + /* copy start values to the beginning of the vectors */ + level[0] = *a; + if(beta > 0) trend[0] = *b; + if(gamma > 0) memcpy(season, s, *period * sizeof(long double)); + + for(i = start_time - 1; i < entries; i++) { + /* indices for period i */ + i0 = i - start_time + 2; + s0 = i0 + *period - 1; + + /* forecast *for* period i */ + xhat = level[i0 - 1] + (beta > 0 ? trend[i0 - 1] : 0); + stmp = gamma > 0 ? season[s0 - *period] : (*seasonal != 1); + if (*seasonal == 1) + xhat += stmp; + else + xhat *= stmp; + + /* Sum of Squared Errors */ + res = series[i] - xhat; + *SSE += res * res; + + /* estimate of level *in* period i */ + if (*seasonal == 1) + level[i0] = alpha * (series[i] - stmp) + + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]); + else + level[i0] = alpha * (series[i] / stmp) + + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]); + + /* estimate of trend *in* period i */ + if (beta > 0) + trend[i0] = beta * (level[i0] - level[i0 - 1]) + + (1 - beta) * trend[i0 - 1]; + + /* estimate of seasonal component *in* period i */ + if (gamma > 0) { + if (*seasonal == 1) + season[s0] = gamma * (series[i] - level[i0]) + + (1 - gamma) * stmp; + else + season[s0] = gamma * (series[i] / level[i0]) + + (1 - gamma) * stmp; + } + } + + return 1; +} + +long double holtwinters(long double *series, size_t entries, long double alpha, long double beta, long double gamma, long double *forecast) { + if(unlikely(isnan(alpha))) + alpha = 0.3; + + if(unlikely(isnan(beta))) + beta = 0.05; + + if(unlikely(isnan(gamma))) + gamma = 0; + + int seasonal = 0; + int period = 0; + long double a0 = series[0]; + long double b0 = 0; + long double s[] = {}; + + long double errors = 0.0; + size_t nb_computations = entries; + long double *estimated_level = callocz(nb_computations, sizeof(long double)); + long double *estimated_trend = callocz(nb_computations, sizeof(long double)); + long double *estimated_season = callocz(nb_computations, sizeof(long double)); + + int ret = __HoltWinters( + series, + (int)entries, + alpha, + beta, + gamma, + &seasonal, + &period, + &a0, + &b0, + s, + &errors, + estimated_level, + estimated_trend, + estimated_season + ); + + long double value = estimated_level[nb_computations - 1]; + + if(forecast) + *forecast = 0.0; + + freez(estimated_level); + freez(estimated_trend); + freez(estimated_season); + + if(!ret) + return 0.0; + + return value; +} diff --git a/src/statistical.h b/src/statistical.h new file mode 100644 index 000000000..844e579bb --- /dev/null +++ b/src/statistical.h @@ -0,0 +1,19 @@ +#ifndef NETDATA_STATISTICAL_H +#define NETDATA_STATISTICAL_H + +extern long double average(long double *series, size_t entries); +extern long double moving_average(long double *series, size_t entries, size_t period); +extern long double median(long double *series, size_t entries); +extern long double moving_median(long double *series, size_t entries, size_t period); +extern long double running_median_estimate(long double *series, size_t entries); +extern long double standard_deviation(long double *series, size_t entries); +extern long double single_exponential_smoothing(long double *series, size_t entries, long double alpha); +extern long double double_exponential_smoothing(long double *series, size_t entries, long double alpha, long double beta, long double *forecast); +extern long double holtwinters(long double *series, size_t entries, long double alpha, long double beta, long double gamma, long double *forecast); +extern long double sum_and_count(long double *series, size_t entries, size_t *count); +extern long double sum(long double *series, size_t entries); +extern long double median_on_sorted_series(long double *series, size_t entries); +extern long double *copy_series(long double *series, size_t entries); +extern void sort_series(long double *series, size_t entries); + +#endif //NETDATA_STATISTICAL_H diff --git a/src/statsd.c b/src/statsd.c new file mode 100644 index 000000000..4dd04757b --- /dev/null +++ b/src/statsd.c @@ -0,0 +1,2041 @@ +#include "common.h" + +#define STATSD_CHART_PREFIX "statsd" +#define STATSD_CHART_PRIORITY 90000 + +// -------------------------------------------------------------------------------------- + +// #define STATSD_MULTITHREADED 1 + +#ifdef STATSD_MULTITHREADED +// DO NOT ENABLE MULTITHREADING - IT IS NOT WELL TESTED +#define STATSD_AVL_TREE avl_tree_lock +#define STATSD_AVL_INSERT avl_insert_lock +#define STATSD_AVL_SEARCH avl_search_lock +#define STATSD_AVL_INDEX_INIT { .avl_tree = { NULL, statsd_metric_compare }, .rwlock = AVL_LOCK_INITIALIZER } +#define STATSD_FIRST_PTR_MUTEX netdata_mutex_t first_mutex +#define STATSD_FIRST_PTR_MUTEX_INIT .first_mutex = NETDATA_MUTEX_INITIALIZER +#define STATSD_FIRST_PTR_MUTEX_LOCK(index) netdata_mutex_lock(&((index)->first_mutex)) +#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index) netdata_mutex_unlock(&((index)->first_mutex)) +#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DEFAULT +#else +#define STATSD_AVL_TREE avl_tree +#define STATSD_AVL_INSERT avl_insert +#define STATSD_AVL_SEARCH avl_search +#define STATSD_AVL_INDEX_INIT { .root = NULL, .compar = statsd_metric_compare } +#define STATSD_FIRST_PTR_MUTEX +#define STATSD_FIRST_PTR_MUTEX_INIT +#define STATSD_FIRST_PTR_MUTEX_LOCK(index) +#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index) +#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_SINGLE_THREADED +#endif + +#define STATSD_DECIMAL_DETAIL 1000 // floating point values get multiplied by this, with the same divisor + +// -------------------------------------------------------------------------------------------------------------------- +// data specific to each metric type + +typedef struct statsd_metric_gauge { + long double value; +} STATSD_METRIC_GAUGE; + +typedef struct statsd_metric_counter { // counter and meter + long long value; +} STATSD_METRIC_COUNTER; + +typedef struct statsd_histogram_extensions { + netdata_mutex_t mutex; + + // average is stored in metric->last + collected_number last_min; + collected_number last_max; + collected_number last_percentile; + collected_number last_median; + collected_number last_stddev; + collected_number last_sum; + + RRDDIM *rd_min; + RRDDIM *rd_max; + RRDDIM *rd_percentile; + RRDDIM *rd_median; + RRDDIM *rd_stddev; + RRDDIM *rd_sum; + + size_t size; + size_t used; + long double *values; // dynamic array of values collected +} STATSD_METRIC_HISTOGRAM_EXTENSIONS; + +typedef struct statsd_metric_histogram { // histogram and timer + STATSD_METRIC_HISTOGRAM_EXTENSIONS *ext; +} STATSD_METRIC_HISTOGRAM; + +typedef struct statsd_metric_set { + DICTIONARY *dict; + size_t unique; +} STATSD_METRIC_SET; + + +// -------------------------------------------------------------------------------------------------------------------- +// this is a metric - for all types of metrics + +typedef enum statsd_metric_options { + STATSD_METRIC_OPTION_NONE = 0x00000000, // no options set + STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED = 0x00000001, // do not update the chart dimension, when this metric is not collected + STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED = 0x00000002, // render a private chart for this metric + STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED = 0x00000004, // the metric has been checked if it should get private chart or not + STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT = 0x00000008, // show the count of events for this private chart + STATSD_METRIC_OPTION_CHECKED_IN_APPS = 0x00000010, // set when this metric has been checked agains apps +} STATS_METRIC_OPTIONS; + +typedef enum statsd_metric_type { + STATSD_METRIC_TYPE_GAUGE, + STATSD_METRIC_TYPE_COUNTER, + STATSD_METRIC_TYPE_METER, + STATSD_METRIC_TYPE_TIMER, + STATSD_METRIC_TYPE_HISTOGRAM, + STATSD_METRIC_TYPE_SET +} STATSD_METRIC_TYPE; + + +typedef struct statsd_metric { + avl avl; // indexing + + const char *name; // the name of the metric + uint32_t hash; // hash of the name + + STATSD_METRIC_TYPE type; + + // metadata about data collection + collected_number events; // the number of times this metric has been collected (never resets) + size_t count; // the number of times this metric has been collected since the last flush + + // the actual collected data + union { + STATSD_METRIC_GAUGE gauge; + STATSD_METRIC_COUNTER counter; + STATSD_METRIC_HISTOGRAM histogram; + STATSD_METRIC_SET set; + }; + + // chart related members + STATS_METRIC_OPTIONS options; // STATSD_METRIC_OPTION_* (bitfield) + char reset; // set to 1 to reset this metric to zero + collected_number last; // the last value sent to netdata + RRDSET *st; // the chart of this metric + RRDDIM *rd_value; // the dimension of this metric value + RRDDIM *rd_count; // the dimension for the number of events received + + // linking, used for walking through all metrics + struct statsd_metric *next; +} STATSD_METRIC; + + +// -------------------------------------------------------------------------------------------------------------------- +// each type of metric has its own index + +typedef struct statsd_index { + char *name; // the name of the index of metrics + size_t events; // the number of events processed for this index + size_t metrics; // the number of metrics in this index + + STATSD_AVL_TREE index; // the AVL tree + + STATSD_METRIC *first; // the linked list of metrics (new metrics are added in front) + STATSD_FIRST_PTR_MUTEX; // when mutli-threading is enabled, a lock to protect the linked list + + STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index +} STATSD_INDEX; + +static int statsd_metric_compare(void* a, void* b); + +// -------------------------------------------------------------------------------------------------------------------- +// synthetic charts + +typedef enum statsd_app_chart_dimension_value_type { + STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS, + STATSD_APP_CHART_DIM_VALUE_TYPE_LAST, + STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE, + STATSD_APP_CHART_DIM_VALUE_TYPE_SUM, + STATSD_APP_CHART_DIM_VALUE_TYPE_MIN, + STATSD_APP_CHART_DIM_VALUE_TYPE_MAX, + STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE, + STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN, + STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV +} STATSD_APP_CHART_DIM_VALUE_TYPE; + +typedef struct statsd_app_chart_dimension { + const char *name; + const char *metric; + uint32_t metric_hash; + collected_number multiplier; + collected_number divisor; + STATSD_APP_CHART_DIM_VALUE_TYPE value_type; + + RRDDIM *rd; + collected_number *value_ptr; + RRD_ALGORITHM algorithm; + + struct statsd_app_chart_dimension *next; +} STATSD_APP_CHART_DIM; + +typedef struct statsd_app_chart { + const char *source; + const char *id; + const char *name; + const char *title; + const char *family; + const char *context; + const char *units; + long priority; + RRDSET_TYPE chart_type; + STATSD_APP_CHART_DIM *dimensions; + size_t dimensions_count; + size_t dimensions_linked_count; + + RRDSET *st; + struct statsd_app_chart *next; +} STATSD_APP_CHART; + +typedef struct statsd_app { + const char *name; + SIMPLE_PATTERN *metrics; + STATS_METRIC_OPTIONS default_options; + RRD_MEMORY_MODE rrd_memory_mode; + long rrd_history_entries; + + const char *source; + STATSD_APP_CHART *charts; + struct statsd_app *next; +} STATSD_APP; + +// -------------------------------------------------------------------------------------------------------------------- +// global statsd data + +static struct statsd { + STATSD_INDEX gauges; + STATSD_INDEX counters; + STATSD_INDEX timers; + STATSD_INDEX histograms; + STATSD_INDEX meters; + STATSD_INDEX sets; + size_t unknown_types; + size_t socket_errors; + size_t tcp_socket_reads; + size_t tcp_packets_received; + size_t tcp_bytes_read; + size_t udp_socket_reads; + size_t udp_packets_received; + size_t udp_bytes_read; + + int enabled; + int update_every; + SIMPLE_PATTERN *charts_for; + + size_t private_charts; + size_t max_private_charts; + size_t max_private_charts_hard; + RRD_MEMORY_MODE private_charts_memory_mode; + long private_charts_rrd_history_entries; + + STATSD_APP *apps; + size_t recvmmsg_size; + size_t histogram_increase_step; + double histogram_percentile; + char *histogram_percentile_str; + int threads; + LISTEN_SOCKETS sockets; +} statsd = { + .enabled = 1, + .max_private_charts = 200, + .max_private_charts_hard = 1000, + .recvmmsg_size = 10, + + .gauges = { + .name = "gauge", + .events = 0, + .metrics = 0, + .index = STATSD_AVL_INDEX_INIT, + .default_options = STATSD_METRIC_OPTION_NONE, + .first = NULL, + STATSD_FIRST_PTR_MUTEX_INIT + }, + .counters = { + .name = "counter", + .events = 0, + .metrics = 0, + .index = STATSD_AVL_INDEX_INIT, + .default_options = STATSD_METRIC_OPTION_NONE, + .first = NULL, + STATSD_FIRST_PTR_MUTEX_INIT + }, + .timers = { + .name = "timer", + .events = 0, + .metrics = 0, + .index = STATSD_AVL_INDEX_INIT, + .default_options = STATSD_METRIC_OPTION_NONE, + .first = NULL, + STATSD_FIRST_PTR_MUTEX_INIT + }, + .histograms = { + .name = "histogram", + .events = 0, + .metrics = 0, + .index = STATSD_AVL_INDEX_INIT, + .default_options = STATSD_METRIC_OPTION_NONE, + .first = NULL, + STATSD_FIRST_PTR_MUTEX_INIT + }, + .meters = { + .name = "meter", + .events = 0, + .metrics = 0, + .index = STATSD_AVL_INDEX_INIT, + .default_options = STATSD_METRIC_OPTION_NONE, + .first = NULL, + STATSD_FIRST_PTR_MUTEX_INIT + }, + .sets = { + .name = "set", + .events = 0, + .metrics = 0, + .index = STATSD_AVL_INDEX_INIT, + .default_options = STATSD_METRIC_OPTION_NONE, + .first = NULL, + STATSD_FIRST_PTR_MUTEX_INIT + }, + + .apps = NULL, + .histogram_percentile = 95.0, + .histogram_increase_step = 10, + .threads = 0, + .sockets = { + .config_section = CONFIG_SECTION_STATSD, + .default_bind_to = "udp:localhost tcp:localhost", + .default_port = STATSD_LISTEN_PORT, + .backlog = STATSD_LISTEN_BACKLOG + }, +}; + + +// -------------------------------------------------------------------------------------------------------------------- +// statsd index management - add/find metrics + +static int statsd_metric_compare(void* a, void* b) { + if(((STATSD_METRIC *)a)->hash < ((STATSD_METRIC *)b)->hash) return -1; + else if(((STATSD_METRIC *)a)->hash > ((STATSD_METRIC *)b)->hash) return 1; + else return strcmp(((STATSD_METRIC *)a)->name, ((STATSD_METRIC *)b)->name); +} + +static inline STATSD_METRIC *stasd_metric_index_find(STATSD_INDEX *index, const char *name, uint32_t hash) { + STATSD_METRIC tmp; + tmp.name = name; + tmp.hash = (hash)?hash:simple_hash(tmp.name); + + return (STATSD_METRIC *)STATSD_AVL_SEARCH(&index->index, (avl *)&tmp); +} + +static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, const char *name, STATSD_METRIC_TYPE type) { + debug(D_STATSD, "searching for metric '%s' under '%s'", name, index->name); + + uint32_t hash = simple_hash(name); + + STATSD_METRIC *m = stasd_metric_index_find(index, name, hash); + if(unlikely(!m)) { + debug(D_STATSD, "Creating new %s metric '%s'", index->name, name); + + m = (STATSD_METRIC *)callocz(sizeof(STATSD_METRIC), 1); + m->name = strdupz(name); + m->hash = hash; + m->type = type; + m->options = index->default_options; + + if(type == STATSD_METRIC_TYPE_HISTOGRAM || type == STATSD_METRIC_TYPE_TIMER) { + m->histogram.ext = callocz(sizeof(STATSD_METRIC_HISTOGRAM_EXTENSIONS), 1); + netdata_mutex_init(&m->histogram.ext->mutex); + } + STATSD_METRIC *n = (STATSD_METRIC *)STATSD_AVL_INSERT(&index->index, (avl *)m); + if(unlikely(n != m)) { + freez((void *)m->histogram.ext); + freez((void *)m->name); + freez((void *)m); + m = n; + } + else { + STATSD_FIRST_PTR_MUTEX_LOCK(index); + index->metrics++; + m->next = index->first; + index->first = m; + STATSD_FIRST_PTR_MUTEX_UNLOCK(index); + } + } + + index->events++; + return m; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// statsd parsing numbers + +static inline long double statsd_parse_float(const char *v, long double def) { + long double value; + + if(likely(v && *v)) { + char *e = NULL; + value = str2ld(v, &e); + if(unlikely(e && *e)) + error("STATSD: excess data '%s' after value '%s'", e, v); + } + else + value = def; + + return value; +} + +static inline long long statsd_parse_int(const char *v, long long def) { + long long value; + + if(likely(v && *v)) { + char *e = NULL; + value = str2ll(v, &e); + if(unlikely(e && *e)) + error("STATSD: excess data '%s' after value '%s'", e, v); + } + else + value = def; + + return value; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// statsd processors per metric type + +static inline void statsd_reset_metric(STATSD_METRIC *m) { + m->reset = 0; + m->count = 0; +} + +static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, const char *sampling) { + if(unlikely(!value || !*value)) { + error("STATSD: metric '%s' of type gauge, with empty value is ignored.", m->name); + return; + } + + if(unlikely(m->reset)) { + // no need to reset anything specific for gauges + statsd_reset_metric(m); + } + + if(unlikely(*value == '+' || *value == '-')) + m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0); + else + m->gauge.value = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0); + + m->events++; + m->count++; +} + +static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, const char *sampling) { + // we accept empty values for counters + + if(unlikely(m->reset)) statsd_reset_metric(m); + + m->counter.value += roundl((long double)statsd_parse_int(value, 1) / statsd_parse_float(sampling, 1.0)); + + m->events++; + m->count++; +} + +static inline void statsd_process_meter(STATSD_METRIC *m, const char *value, const char *sampling) { + // this is the same with the counter + statsd_process_counter(m, value, sampling); +} + +static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value, const char *sampling) { + if(unlikely(!value || !*value)) { + error("STATSD: metric '%s' of type histogram, with empty value is ignored.", m->name); + return; + } + + if(unlikely(m->reset)) { + m->histogram.ext->used = 0; + statsd_reset_metric(m); + } + + if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) { + netdata_mutex_lock(&m->histogram.ext->mutex); + m->histogram.ext->size += statsd.histogram_increase_step; + m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(long double) * m->histogram.ext->size); + netdata_mutex_unlock(&m->histogram.ext->mutex); + } + + m->histogram.ext->values[m->histogram.ext->used++] = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0); + + m->events++; + m->count++; +} + +static inline void statsd_process_timer(STATSD_METRIC *m, const char *value, const char *sampling) { + if(unlikely(!value || !*value)) { + error("STATSD: metric of type set, with empty value is ignored."); + return; + } + + // timers are a use case of histogram + statsd_process_histogram(m, value, sampling); +} + +static inline void statsd_process_set(STATSD_METRIC *m, const char *value) { + if(unlikely(!value || !*value)) { + error("STATSD: metric of type set, with empty value is ignored."); + return; + } + + if(unlikely(m->reset)) { + if(likely(m->set.dict)) { + dictionary_destroy(m->set.dict); + m->set.dict = NULL; + } + statsd_reset_metric(m); + } + + if(unlikely(!m->set.dict)) { + m->set.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE); + m->set.unique = 0; + } + + void *t = dictionary_get(m->set.dict, value); + if(unlikely(!t)) { + dictionary_set(m->set.dict, value, NULL, 1); + m->set.unique++; + } + + m->events++; + m->count++; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// statsd parsing + +static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling) { + debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', rate '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)"); + + if(unlikely(!name || !*name)) return; + if(unlikely(!type || !*type)) type = "m"; + + char t0 = type[0], t1 = type[1]; + + if(unlikely(t0 == 'g' && t1 == '\0')) { + statsd_process_gauge( + statsd_find_or_add_metric(&statsd.gauges, name, STATSD_METRIC_TYPE_GAUGE), + value, sampling); + } + else if(unlikely((t0 == 'c' || t0 == 'C') && t1 == '\0')) { + // etsy/statsd uses 'c' + // brubeck uses 'C' + statsd_process_counter( + statsd_find_or_add_metric(&statsd.counters, name, STATSD_METRIC_TYPE_COUNTER), + value, sampling); + } + else if(unlikely(t0 == 'm' && t1 == '\0')) { + statsd_process_meter( + statsd_find_or_add_metric(&statsd.meters, name, STATSD_METRIC_TYPE_METER), + value, sampling); + } + else if(unlikely(t0 == 'h' && t1 == '\0')) { + statsd_process_histogram( + statsd_find_or_add_metric(&statsd.histograms, name, STATSD_METRIC_TYPE_HISTOGRAM), + value, sampling); + } + else if(unlikely(t0 == 's' && t1 == '\0')) { + statsd_process_set( + statsd_find_or_add_metric(&statsd.sets, name, STATSD_METRIC_TYPE_SET), + value); + } + else if(unlikely(t0 == 'm' && t1 == 's' && type[2] == '\0')) { + statsd_process_timer( + statsd_find_or_add_metric(&statsd.timers, name, STATSD_METRIC_TYPE_TIMER), + value, sampling); + } + else { + statsd.unknown_types++; + error("STATSD: metric '%s' with value '%s' is sent with unknown metric type '%s'", name, value?value:"", type); + } +} + +static inline const char *statsd_parse_skip_up_to(const char *s, char d1, char d2) { + char c; + + for(c = *s; c && c != d1 && c != d2 && c != '\r' && c != '\n'; c = *++s) ; + + return s; +} + +const char *statsd_parse_skip_spaces(const char *s) { + char c; + + for(c = *s; c && ( c == ' ' || c == '\t' || c == '\r' || c == '\n' ); c = *++s) ; + + return s; +} + +static inline const char *statsd_parse_field_trim(const char *start, char *end) { + if(unlikely(!start)) { + start = end; + return start; + } + + while(start <= end && (*start == ' ' || *start == '\t')) + start++; + + *end = '\0'; + end--; + while(end >= start && (*end == ' ' || *end == '\t')) + *end-- = '\0'; + + return start; +} + +static inline size_t statsd_process(char *buffer, size_t size, int require_newlines) { + buffer[size] = '\0'; + debug(D_STATSD, "RECEIVED: %zu bytes: '%s'", size, buffer); + + const char *s = buffer; + while(*s) { + const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL; + char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL; + + s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '|'); + if(name == name_end) { + s = statsd_parse_skip_spaces(s); + continue; + } + + if(likely(*s == ':')) + s = value_end = (char *) statsd_parse_skip_up_to(value = ++s, '|', '|'); + + if(likely(*s == '|')) + s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@'); + + if(likely(*s == '|' || *s == '@')) { + s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '\r', '\n'); + if(*sampling == '@') sampling++; + } + + // skip everything until the end of the line + while(*s && *s != '\n') s++; + + if(unlikely(require_newlines && *s != '\n' && s > buffer)) { + // move the remaining data to the beginning + size -= (name - buffer); + memmove(buffer, name, size); + return size; + } + else + s = statsd_parse_skip_spaces(s); + + statsd_process_metric( + statsd_parse_field_trim(name, name_end) + , statsd_parse_field_trim(value, value_end) + , statsd_parse_field_trim(type, type_end) + , statsd_parse_field_trim(sampling, sampling_end) + ); + } + + return 0; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// statsd pollfd interface + +#define STATSD_TCP_BUFFER_SIZE 65536 // minimize tcp reads +#define STATSD_UDP_BUFFER_SIZE 9000 // this should be up to MTU + +typedef enum { + STATSD_SOCKET_DATA_TYPE_TCP, + STATSD_SOCKET_DATA_TYPE_UDP +} STATSD_SOCKET_DATA_TYPE; + +struct statsd_tcp { + STATSD_SOCKET_DATA_TYPE type; + size_t size; + size_t len; + char buffer[]; +}; + +#ifdef HAVE_RECVMMSG +struct statsd_udp { + STATSD_SOCKET_DATA_TYPE type; + size_t size; + struct iovec *iovecs; + struct mmsghdr *msgs; +}; +#else +struct statsd_udp { + STATSD_SOCKET_DATA_TYPE type; + char buffer[STATSD_UDP_BUFFER_SIZE]; +}; +#endif + +// new TCP client connected +static void *statsd_add_callback(int fd, short int *events) { + (void)fd; + *events = POLLIN; + + struct statsd_tcp *data = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1); + data->type = STATSD_SOCKET_DATA_TYPE_TCP; + data->size = STATSD_TCP_BUFFER_SIZE - 1; + + return data; +} + +// TCP client disconnected +static void statsd_del_callback(int fd, void *data) { + (void)fd; + + if(data) { + struct statsd_tcp *t = data; + if(t->type == STATSD_SOCKET_DATA_TYPE_TCP) { + if(t->len != 0) { + statsd.socket_errors++; + error("STATSD: client is probably sending unterminated metrics. Closed socket left with '%s'. Trying to process it.", t->buffer); + statsd_process(t->buffer, t->len, 0); + } + } + else + error("STATSD: internal error: received socket data type is %d, but expected %d", (int)t->type, (int)STATSD_SOCKET_DATA_TYPE_TCP); + + freez(data); + } + + return; +} + +// Receive data +static int statsd_rcv_callback(int fd, int socktype, void *data, short int *events) { + *events = POLLIN; + + switch(socktype) { + case SOCK_STREAM: { + struct statsd_tcp *d = (struct statsd_tcp *)data; + if(unlikely(!d)) { + error("STATSD: internal error: expected TCP data pointer is NULL"); + statsd.socket_errors++; + return -1; + } + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_TCP)) { + error("STATSD: internal error: socket data type should be %d, but it is %d", (int)STATSD_SOCKET_DATA_TYPE_TCP, (int)d->type); + statsd.socket_errors++; + return -1; + } +#endif + + int ret = 0; + ssize_t rc; + do { + rc = recv(fd, &d->buffer[d->len], d->size - d->len, MSG_DONTWAIT); + if (rc < 0) { + // read failed + if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) { + error("STATSD: recv() on TCP socket %d failed.", fd); + statsd.socket_errors++; + ret = -1; + } + } + else if (!rc) { + // connection closed + debug(D_STATSD, "STATSD: client disconnected."); + ret = -1; + } + else { + // data received + d->len += rc; + statsd.tcp_socket_reads++; + statsd.tcp_bytes_read += rc; + } + + if(likely(d->len > 0)) { + statsd.tcp_packets_received++; + d->len = statsd_process(d->buffer, d->len, 1); + } + + if(unlikely(ret == -1)) + return -1; + + } while (rc != -1); + break; + } + + case SOCK_DGRAM: { + struct statsd_udp *d = (struct statsd_udp *)data; + if(unlikely(!d)) { + error("STATSD: internal error: expected UDP data pointer is NULL"); + statsd.socket_errors++; + return -1; + } + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) { + error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_UDP); + statsd.socket_errors++; + return -1; + } +#endif + +#ifdef HAVE_RECVMMSG + ssize_t rc; + do { + rc = recvmmsg(fd, d->msgs, (unsigned int)d->size, MSG_DONTWAIT, NULL); + if (rc < 0) { + // read failed + if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) { + error("STATSD: recvmmsg() on UDP socket %d failed.", fd); + statsd.socket_errors++; + return -1; + } + } else if (rc) { + // data received + statsd.udp_socket_reads++; + statsd.udp_packets_received += rc; + + size_t i; + for (i = 0; i < (size_t)rc; ++i) { + size_t len = (size_t)d->msgs[i].msg_len; + statsd.udp_bytes_read += len; + statsd_process(d->msgs[i].msg_hdr.msg_iov->iov_base, len, 0); + } + } + } while (rc != -1); + +#else // !HAVE_RECVMMSG + ssize_t rc; + do { + rc = recv(fd, d->buffer, STATSD_UDP_BUFFER_SIZE - 1, MSG_DONTWAIT); + if (rc < 0) { + // read failed + if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) { + error("STATSD: recv() on UDP socket %d failed.", fd); + statsd.socket_errors++; + return -1; + } + } else if (rc) { + // data received + statsd.udp_socket_reads++; + statsd.udp_packets_received++; + statsd.udp_bytes_read += rc; + statsd_process(d->buffer, (size_t) rc, 0); + } + } while (rc != -1); +#endif + + break; + } + + default: { + error("STATSD: internal error: unknown socktype %d on socket %d", socktype, fd); + statsd.socket_errors++; + return -1; + } + } + + return 0; +} + +static int statsd_snd_callback(int fd, int socktype, void *data, short int *events) { + (void)fd; + (void)socktype; + (void)data; + (void)events; + + error("STATSD: snd_callback() called, but we never requested to send data to statsd clients."); + return -1; +} + +// -------------------------------------------------------------------------------------------------------------------- +// statsd child thread to collect metrics from network + +void statsd_collector_thread_cleanup(void *data) { + struct statsd_udp *d = data; + +#ifdef HAVE_RECVMMSG + size_t i; + for (i = 0; i < d->size; i++) + freez(d->iovecs[i].iov_base); + + freez(d->iovecs); + freez(d->msgs); +#endif + + freez(d); +} + +void *statsd_collector_thread(void *ptr) { + int id = *((int *)ptr); + + info("STATSD collector thread No %d created with task id %d", id + 1, gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + struct statsd_udp *d = callocz(sizeof(struct statsd_udp), 1); + pthread_cleanup_push(statsd_collector_thread_cleanup, d); + +#ifdef HAVE_RECVMMSG + d->type = STATSD_SOCKET_DATA_TYPE_UDP; + d->size = statsd.recvmmsg_size; + d->iovecs = callocz(sizeof(struct iovec), d->size); + d->msgs = callocz(sizeof(struct mmsghdr), d->size); + + size_t i; + for (i = 0; i < d->size; i++) { + d->iovecs[i].iov_base = mallocz(STATSD_UDP_BUFFER_SIZE); + d->iovecs[i].iov_len = STATSD_UDP_BUFFER_SIZE - 1; + d->msgs[i].msg_hdr.msg_iov = &d->iovecs[i]; + d->msgs[i].msg_hdr.msg_iovlen = 1; + } +#endif + + poll_events(&statsd.sockets + , statsd_add_callback + , statsd_del_callback + , statsd_rcv_callback + , statsd_snd_callback + , (void *)d + ); + + pthread_cleanup_pop(1); + + debug(D_WEB_CLIENT, "STATSD: exit!"); + pthread_exit(NULL); + return NULL; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// statsd applications configuration files parsing + +#define STATSD_CONF_LINE_MAX 8192 + +int statsd_readfile(const char *path, const char *filename) { + debug(D_STATSD, "STATSD configuration reading file '%s/%s'", path, filename); + + char buffer[STATSD_CONF_LINE_MAX + 1]; + + FILE *fp = NULL; + snprintfz(buffer, STATSD_CONF_LINE_MAX, "%s/%s", path, filename); + fp = fopen(buffer, "r"); + if(!fp) { + error("STATSD: cannot open file '%s'.", buffer); + return -1; + } + + STATSD_APP *app = NULL; + STATSD_APP_CHART *chart = NULL; + + size_t line = 0; + char *s; + while(fgets(buffer, STATSD_CONF_LINE_MAX, fp) != NULL) { + buffer[STATSD_CONF_LINE_MAX] = '\0'; + line++; + + s = trim(buffer); + if (!s || *s == '#') { + debug(D_STATSD, "STATSD: ignoring line %zu of file '%s/%s', it is empty.", line, path, filename); + continue; + } + debug(D_STATSD, "STATSD: processing line %zu of file '%s/%s': %s", line, path, filename, buffer); + + int len = (int) strlen(s); + if (*s == '[' && s[len - 1] == ']') { + // new section + s[len - 1] = '\0'; + s++; + + if (!strcmp(s, "app")) { + // a new app + app = callocz(sizeof(STATSD_APP), 1); + app->name = strdupz("unnamed"); + app->rrd_memory_mode = localhost->rrd_memory_mode; + app->rrd_history_entries = localhost->rrd_history_entries; + + app->next = statsd.apps; + statsd.apps = app; + chart = NULL; + } + else if(app) { + // a new chart + chart = callocz(sizeof(STATSD_APP_CHART), 1); + netdata_fix_chart_id(s); + chart->id = strdupz(s); + chart->name = strdupz(s); + chart->title = strdupz("Statsd chart"); + chart->context = strdupz(s); + chart->family = strdupz("overview"); + chart->units = strdupz("value"); + chart->priority = STATSD_CHART_PRIORITY; + chart->chart_type = RRDSET_TYPE_LINE; + + chart->next = app->charts; + app->charts = chart; + } + else + error("STATSD: ignoring line %zu ('%s') of file '%s/%s', [app] is not defined.", line, s, path, filename); + + continue; + } + + if(!app) { + error("STATSD: ignoring line %zu ('%s') of file '%s/%s', it is outside all sections.", line, s, path, filename); + continue; + } + + char *name = s; + char *value = strchr(s, '='); + if(!value) { + error("STATSD: ignoring line %zu ('%s') of file '%s/%s', there is no = in it.", line, s, path, filename); + continue; + } + *value = '\0'; + value++; + + name = trim(name); + value = trim(value); + + if(!name || *name == '#') { + error("STATSD: ignoring line %zu of file '%s/%s', name is empty.", line, path, filename); + continue; + } + if(!value) { + debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s/%s', value is empty.", line, path, filename); + continue; + } + + if(!chart) { + if(!strcmp(name, "name")) { + freez((void *)app->name); + netdata_fix_chart_name(value); + app->name = strdupz(value); + } + else if (!strcmp(name, "metrics")) { + simple_pattern_free(app->metrics); + app->metrics = simple_pattern_create(value, SIMPLE_PATTERN_EXACT); + } + else if (!strcmp(name, "private charts")) { + if (!strcmp(value, "yes") || !strcmp(value, "on")) + app->default_options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; + else + app->default_options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; + } + else if (!strcmp(name, "gaps when not collected")) { + if (!strcmp(value, "yes") || !strcmp(value, "on")) + app->default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + } + else if (!strcmp(name, "memory mode")) { + app->rrd_memory_mode = rrd_memory_mode_id(value); + } + else if (!strcmp(name, "history")) { + app->rrd_history_entries = atol(value); + if (app->rrd_history_entries < 5) + app->rrd_history_entries = 5; + } + else { + error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [app] section.", line, name, path, filename); + continue; + } + } + else { + if(!strcmp(name, "name")) { + freez((void *)chart->name); + netdata_fix_chart_id(value); + chart->name = strdupz(value); + } + else if(!strcmp(name, "title")) { + freez((void *)chart->title); + chart->title = strdupz(value); + } + else if (!strcmp(name, "family")) { + freez((void *)chart->family); + chart->family = strdupz(value); + } + else if (!strcmp(name, "context")) { + freez((void *)chart->context); + netdata_fix_chart_id(value); + chart->context = strdupz(value); + } + else if (!strcmp(name, "units")) { + freez((void *)chart->units); + chart->units = strdupz(value); + } + else if (!strcmp(name, "priority")) { + chart->priority = atol(value); + } + else if (!strcmp(name, "type")) { + chart->chart_type = rrdset_type_id(value); + } + else if (!strcmp(name, "dimension")) { + // metric [name [type [multiplier [divisor]]]] + char *words[5]; + pluginsd_split_words(value, words, 5); + + char *metric_name = words[0]; + char *dim_name = words[1]; + char *type = words[2]; + char *multipler = words[3]; + char *divisor = words[4]; + + STATSD_APP_CHART_DIM *dim = callocz(sizeof(STATSD_APP_CHART_DIM), 1); + + dim->metric = strdupz(metric_name); + dim->metric_hash = simple_hash(dim->metric); + + dim->name = strdupz((dim_name && *dim_name)?dim_name:metric_name); + dim->multiplier = (multipler && *multipler)?str2l(multipler):1; + dim->divisor = (divisor && *divisor)?str2l(divisor):1; + + if(!type || !*type) type = "last"; + if(!strcmp(type, "events")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS; + else if(!strcmp(type, "last")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST; + else if(!strcmp(type, "min")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MIN; + else if(!strcmp(type, "max")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MAX; + else if(!strcmp(type, "sum")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_SUM; + else if(!strcmp(type, "average")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE; + else if(!strcmp(type, "median")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN; + else if(!strcmp(type, "stddev")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV; + else if(!strcmp(type, "percentile")) dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE; + else { + error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename); + dim->value_type = STATSD_APP_CHART_DIM_VALUE_TYPE_LAST; + } + + if(!dim->multiplier) { + error("STATSD: invalid multiplier value '%s' at line %zu of file '%s/%s'. Using 1.", multipler, line, path, filename); + dim->multiplier = 1; + } + if(!dim->divisor) { + error("STATSD: invalid divisor value '%s' at line %zu of file '%s/%s'. Using 1.", divisor, line, path, filename); + dim->divisor = 1; + } + + // append it to the list of dimension + STATSD_APP_CHART_DIM *tdim; + for(tdim = chart->dimensions; tdim && tdim->next ; tdim = tdim->next) ; + if(!tdim) { + dim->next = chart->dimensions; + chart->dimensions = dim; + } + else { + dim->next = tdim->next; + tdim->next = dim; + } + chart->dimensions_count++; + + debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT, + dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor); + } + else { + error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id); + continue; + } + } + } + + fclose(fp); + return 0; +} + +static void statsd_readdir(const char *path) { + size_t pathlen = strlen(path); + + debug(D_STATSD, "STATSD configuration reading directory '%s'", path); + + DIR *dir = opendir(path); + if (!dir) { + error("STATSD configuration cannot open directory '%s'.", path); + return; + } + + struct dirent *de = NULL; + while ((de = readdir(dir))) { + size_t len = strlen(de->d_name); + + if(de->d_type == DT_DIR + && ( + (de->d_name[0] == '.' && de->d_name[1] == '\0') + || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + )) { + debug(D_STATSD, "STATSD: ignoring directory '%s'", de->d_name); + continue; + } + + else if(de->d_type == DT_DIR) { + char *s = mallocz(pathlen + strlen(de->d_name) + 2); + strcpy(s, path); + strcat(s, "/"); + strcat(s, de->d_name); + statsd_readdir(s); + freez(s); + continue; + } + + else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) && + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { + statsd_readfile(path, de->d_name); + } + + else debug(D_STATSD, "STATSD: ignoring file '%s'", de->d_name); + } + + closedir(dir); +} + +// -------------------------------------------------------------------------------------------------------------------- +// send metrics to netdata - in private charts - called from the main thread + +// extract chart type and chart id from metric name +static inline void statsd_get_metric_type_and_id(STATSD_METRIC *m, char *type, char *id, const char *defid, size_t len) { + char *s; + + snprintfz(type, len, "%s_%s_%s", STATSD_CHART_PREFIX, defid, m->name); + for(s = type; *s ;s++) + if(unlikely(*s == '.')) break; + + if(*s == '.') { + *s++ = '\0'; + strncpyz(id, s, len); + } + else { + strncpyz(id, defid, len); + } + + netdata_fix_chart_id(type); + netdata_fix_chart_id(id); +} + +static inline RRDSET *statsd_private_rrdset_create( + STATSD_METRIC *m + , const char *type + , const char *id + , const char *name + , const char *family + , const char *context + , const char *title + , const char *units + , long priority + , int update_every + , RRDSET_TYPE chart_type +) { + RRD_MEMORY_MODE memory_mode = statsd.private_charts_memory_mode; + long history = statsd.private_charts_rrd_history_entries; + + if(unlikely(statsd.private_charts >= statsd.max_private_charts)) { + debug(D_STATSD, "STATSD: metric '%s' will be charted with memory mode = none, because the maximum number of charts has been reached.", m->name); + info("STATSD: metric '%s' will be charted with memory mode = none, because the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts); + memory_mode = RRD_MEMORY_MODE_NONE; + history = 5; + } + + statsd.private_charts++; + RRDSET *st = rrdset_create_custom( + localhost + , type + , id + , name + , family + , context + , title + , units + , priority + , update_every + , chart_type + , memory_mode + , history + ); + rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST); + // rrdset_flag_set(st, RRDSET_FLAG_DEBUG); + return st; +} + +static inline void statsd_private_chart_gauge(STATSD_METRIC *m) { + debug(D_STATSD, "updating private chart for gauge metric '%s'", m->name); + + if(unlikely(!m->st)) { + char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1]; + statsd_get_metric_type_and_id(m, type, id, "gauge", RRD_ID_LENGTH_MAX); + + m->st = statsd_private_rrdset_create( + m + , type + , id + , NULL // name + , "gauges" // family (submenu) + , m->name // context + , m->name // title + , "value" // units + , STATSD_CHART_PRIORITY + , statsd.update_every + , RRDSET_TYPE_LINE + ); + + m->rd_value = rrddim_add(m->st, "gauge", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + + if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT) + m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(m->st); + + rrddim_set_by_pointer(m->st, m->rd_value, m->last); + + if(m->rd_count) + rrddim_set_by_pointer(m->st, m->rd_count, m->events); + + rrdset_done(m->st); +} + +static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) { + debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name); + + if(unlikely(!m->st)) { + char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1]; + statsd_get_metric_type_and_id(m, type, id, dim, RRD_ID_LENGTH_MAX); + + m->st = statsd_private_rrdset_create( + m + , type + , id + , NULL // name + , family // family (submenu) + , m->name // context + , m->name // title + , "events/s" // units + , STATSD_CHART_PRIORITY + , statsd.update_every + , RRDSET_TYPE_AREA + ); + + m->rd_value = rrddim_add(m->st, dim, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT) + m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(m->st); + + rrddim_set_by_pointer(m->st, m->rd_value, m->last); + + if(m->rd_count) + rrddim_set_by_pointer(m->st, m->rd_count, m->events); + + rrdset_done(m->st); +} + +static inline void statsd_private_chart_set(STATSD_METRIC *m) { + debug(D_STATSD, "updating private chart for set metric '%s'", m->name); + + if(unlikely(!m->st)) { + char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1]; + statsd_get_metric_type_and_id(m, type, id, "set", RRD_ID_LENGTH_MAX); + + m->st = statsd_private_rrdset_create( + m + , type + , id + , NULL // name + , "sets" // family (submenu) + , m->name // context + , m->name // title + , "entries" // units + , STATSD_CHART_PRIORITY + , statsd.update_every + , RRDSET_TYPE_LINE + ); + + m->rd_value = rrddim_add(m->st, "set", "set size", 1, 1, RRD_ALGORITHM_ABSOLUTE); + + if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT) + m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(m->st); + + rrddim_set_by_pointer(m->st, m->rd_value, m->last); + + if(m->rd_count) + rrddim_set_by_pointer(m->st, m->rd_count, m->events); + + rrdset_done(m->st); +} + +static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) { + debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name); + + if(unlikely(!m->st)) { + char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1]; + statsd_get_metric_type_and_id(m, type, id, dim, RRD_ID_LENGTH_MAX); + + m->st = statsd_private_rrdset_create( + m + , type + , id + , NULL // name + , family // family (submenu) + , m->name // context + , m->name // title + , units // units + , STATSD_CHART_PRIORITY + , statsd.update_every + , RRDSET_TYPE_AREA + ); + + m->histogram.ext->rd_min = rrddim_add(m->st, "min", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + m->histogram.ext->rd_max = rrddim_add(m->st, "max", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + m->rd_value = rrddim_add(m->st, "average", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + m->histogram.ext->rd_percentile = rrddim_add(m->st, statsd.histogram_percentile_str, NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + m->histogram.ext->rd_median = rrddim_add(m->st, "median", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + m->histogram.ext->rd_stddev = rrddim_add(m->st, "stddev", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + m->histogram.ext->rd_sum = rrddim_add(m->st, "sum", NULL, 1, STATSD_DECIMAL_DETAIL, RRD_ALGORITHM_ABSOLUTE); + + if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT) + m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else rrdset_next(m->st); + + rrddim_set_by_pointer(m->st, m->histogram.ext->rd_min, m->histogram.ext->last_min); + rrddim_set_by_pointer(m->st, m->histogram.ext->rd_max, m->histogram.ext->last_max); + rrddim_set_by_pointer(m->st, m->histogram.ext->rd_percentile, m->histogram.ext->last_percentile); + rrddim_set_by_pointer(m->st, m->histogram.ext->rd_median, m->histogram.ext->last_median); + rrddim_set_by_pointer(m->st, m->histogram.ext->rd_stddev, m->histogram.ext->last_stddev); + rrddim_set_by_pointer(m->st, m->histogram.ext->rd_sum, m->histogram.ext->last_sum); + rrddim_set_by_pointer(m->st, m->rd_value, m->last); + + if(m->rd_count) + rrddim_set_by_pointer(m->st, m->rd_count, m->events); + + rrdset_done(m->st); +} + +// -------------------------------------------------------------------------------------------------------------------- +// statsd flush metrics + +static inline void statsd_flush_gauge(STATSD_METRIC *m) { + debug(D_STATSD, "flushing gauge metric '%s'", m->name); + + int updated = 0; + if(m->count && !m->reset) { + m->last = (collected_number) (m->gauge.value * STATSD_DECIMAL_DETAIL); + + m->reset = 1; + updated = 1; + } + + if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) + statsd_private_chart_gauge(m); +} + +static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *dim, const char *family) { + debug(D_STATSD, "flushing %s metric '%s'", dim, m->name); + + int updated = 0; + if(m->count && !m->reset) { + m->last = m->counter.value; + + m->reset = 1; + updated = 1; + } + + if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) + statsd_private_chart_counter_or_meter(m, dim, family); +} + +static inline void statsd_flush_counter(STATSD_METRIC *m) { + statsd_flush_counter_or_meter(m, "counter", "counters"); +} + +static inline void statsd_flush_meter(STATSD_METRIC *m) { + statsd_flush_counter_or_meter(m, "meter", "meters"); +} + +static inline void statsd_flush_set(STATSD_METRIC *m) { + debug(D_STATSD, "flushing set metric '%s'", m->name); + + int updated = 0; + if(m->count && !m->reset) { + m->last = (collected_number)m->set.unique; + + m->reset = 1; + updated = 1; + } + + if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) + statsd_private_chart_set(m); +} + +static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) { + debug(D_STATSD, "flushing %s metric '%s'", dim, m->name); + + netdata_mutex_lock(&m->histogram.ext->mutex); + + int updated = 0; + if(m->count && !m->reset && m->histogram.ext->used > 0) { + size_t len = m->histogram.ext->used; + long double *series = m->histogram.ext->values; + sort_series(series, len); + + m->histogram.ext->last_min = (collected_number)roundl(series[0] * STATSD_DECIMAL_DETAIL); + m->histogram.ext->last_max = (collected_number)roundl(series[len - 1] * STATSD_DECIMAL_DETAIL); + m->last = (collected_number)roundl(average(series, len) * STATSD_DECIMAL_DETAIL); + m->histogram.ext->last_median = (collected_number)roundl(median_on_sorted_series(series, len) * STATSD_DECIMAL_DETAIL); + m->histogram.ext->last_stddev = (collected_number)roundl(standard_deviation(series, len) * STATSD_DECIMAL_DETAIL); + m->histogram.ext->last_sum = (collected_number)roundl(sum(series, len) * STATSD_DECIMAL_DETAIL); + + size_t pct_len = (size_t)floor((double)len * statsd.histogram_percentile / 100.0); + if(pct_len < 1) + m->histogram.ext->last_percentile = (collected_number)(series[0] * STATSD_DECIMAL_DETAIL); + else + m->histogram.ext->last_percentile = (collected_number)roundl(average(series, pct_len) * STATSD_DECIMAL_DETAIL); + + debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT, + dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum); + + m->reset = 1; + updated = 1; + } + + + if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))) + statsd_private_chart_timer_or_histogram(m, dim, family, units); + + netdata_mutex_unlock(&m->histogram.ext->mutex); +} + +static inline void statsd_flush_timer(STATSD_METRIC *m) { + statsd_flush_timer_or_histogram(m, "timer", "timers", "milliseconds"); +} + +static inline void statsd_flush_histogram(STATSD_METRIC *m) { + statsd_flush_timer_or_histogram(m, "histogram", "histograms", "value"); +} + +static inline RRD_ALGORITHM statsd_algorithm_for_metric(STATSD_METRIC *m) { + switch(m->type) { + default: + case STATSD_METRIC_TYPE_GAUGE: + case STATSD_METRIC_TYPE_SET: + case STATSD_METRIC_TYPE_TIMER: + case STATSD_METRIC_TYPE_HISTOGRAM: + return RRD_ALGORITHM_ABSOLUTE; + + case STATSD_METRIC_TYPE_METER: + case STATSD_METRIC_TYPE_COUNTER: + return RRD_ALGORITHM_INCREMENTAL; + } +} + +static inline void check_if_metric_is_for_app(STATSD_INDEX *index, STATSD_METRIC *m) { + (void)index; + + STATSD_APP *app; + for(app = statsd.apps; app ;app = app->next) { + if(unlikely(simple_pattern_matches(app->metrics, m->name))) { + debug(D_STATSD, "metric '%s' matches app '%s'", m->name, app->name); + + // the metric should get the options from the app + + if(app->default_options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED) + m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; + else + m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; + + if(app->default_options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED) + m->options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + else + m->options &= ~STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + + m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED; + + // check if there is a chart in this app, willing to get this metric + STATSD_APP_CHART *chart; + for(chart = app->charts; chart; chart = chart->next) { + STATSD_APP_CHART_DIM *dim; + for(dim = chart->dimensions; dim ; dim = dim->next) { + if(!dim->value_ptr && dim->metric_hash == m->hash && !strcmp(dim->metric, m->name)) { + // we have a match - this metric should be linked to this dimension + + if(dim->value_type == STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS) { + dim->value_ptr = &m->events; + dim->algorithm = RRD_ALGORITHM_INCREMENTAL; + } + else if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) { + dim->algorithm = RRD_ALGORITHM_ABSOLUTE; + dim->divisor *= STATSD_DECIMAL_DETAIL; + + switch(dim->value_type) { + case STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS: + // will never match - added to avoid warning + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_LAST: + case STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE: + dim->value_ptr = &m->last; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_SUM: + dim->value_ptr = &m->histogram.ext->last_sum; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_MIN: + dim->value_ptr = &m->histogram.ext->last_min; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_MAX: + dim->value_ptr = &m->histogram.ext->last_max; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_MEDIAN: + dim->value_ptr = &m->histogram.ext->last_median; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE: + dim->value_ptr = &m->histogram.ext->last_percentile; + break; + + case STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV: + dim->value_ptr = &m->histogram.ext->last_stddev; + break; + } + } + else { + if (dim->value_type != STATSD_APP_CHART_DIM_VALUE_TYPE_LAST) + error("STATSD: unsupported value type for dimension '%s' of chart '%s' of app '%s' on metric '%s'", dim->name, chart->id, app->name, m->name); + + dim->value_ptr = &m->last; + dim->algorithm = statsd_algorithm_for_metric(m); + + if(m->type == STATSD_METRIC_TYPE_GAUGE) + dim->divisor *= STATSD_DECIMAL_DETAIL; + } + + if(unlikely(chart->st && dim->rd)) { + rrddim_set_algorithm(chart->st, dim->rd, dim->algorithm); + rrddim_set_multiplier(chart->st, dim->rd, dim->multiplier); + rrddim_set_divisor(chart->st, dim->rd, dim->divisor); + } + + chart->dimensions_linked_count++; + debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm)); + } + } + } + } + } +} + +static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *chart) { + debug(D_STATSD, "updating chart '%s' for app '%s'", chart->id, app->name); + + if(!chart->st) { + chart->st = rrdset_create_custom( + localhost + , app->name + , chart->id + , chart->name + , chart->family + , chart->context + , chart->title + , chart->units + , chart->priority + , statsd.update_every + , chart->chart_type + , app->rrd_memory_mode + , app->rrd_history_entries + ); + + rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST); + // rrdset_flag_set(chart->st, RRDSET_FLAG_DEBUG); + } + else rrdset_next(chart->st); + + STATSD_APP_CHART_DIM *dim; + for(dim = chart->dimensions; dim ;dim = dim->next) { + if(unlikely(!dim->rd)) + dim->rd = rrddim_add(chart->st, dim->name, NULL, dim->multiplier, dim->divisor, dim->algorithm); + + if(unlikely(dim->value_ptr)) { + debug(D_STATSD, "updating dimension '%s' (%s) of chart '%s' (%s) for app '%s' with value " COLLECTED_NUMBER_FORMAT, dim->name, dim->rd->id, chart->id, chart->st->id, app->name, *dim->value_ptr); + rrddim_set_by_pointer(chart->st, dim->rd, *dim->value_ptr); + } + } + + rrdset_done(chart->st); + debug(D_STATSD, "completed update of chart '%s' for app '%s'", chart->id, app->name); +} + +static inline void statsd_update_all_app_charts(void) { + // debug(D_STATSD, "updating app charts"); + + STATSD_APP *app; + for(app = statsd.apps; app ;app = app->next) { + // debug(D_STATSD, "updating charts for app '%s'", app->name); + + STATSD_APP_CHART *chart; + for(chart = app->charts; chart ;chart = chart->next) { + if(unlikely(chart->dimensions_linked_count)) { + statsd_update_app_chart(app, chart); + } + } + } + + // debug(D_STATSD, "completed update of app charts"); +} + +static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_metric)(STATSD_METRIC *)) { + STATSD_METRIC *m; + for(m = index->first; m ; m = m->next) { + if(unlikely(!(m->options & STATSD_METRIC_OPTION_CHECKED_IN_APPS))) { + check_if_metric_is_for_app(index, m); + m->options |= STATSD_METRIC_OPTION_CHECKED_IN_APPS; + } + + if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) { + if(statsd.private_charts >= statsd.max_private_charts_hard) { + debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name); + info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts); + m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; + } + else { + if (simple_pattern_matches(statsd.charts_for, m->name)) { + debug(D_STATSD, "STATSD: metric '%s' will be charted.", m->name); + m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; + } else { + debug(D_STATSD, "STATSD: metric '%s' will not be charted.", m->name); + m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED; + } + } + + m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED; + } + + flush_metric(m); + } +} + + +// -------------------------------------------------------------------------------------- +// statsd main thread + +int statsd_listen_sockets_setup(void) { + return listen_sockets_setup(&statsd.sockets); +} + +void statsd_main_cleanup(void *data) { + pthread_t *threads = data; + + int i; + for(i = 0; i < statsd.threads ;i++) + pthread_cancel(threads[i]); + + listen_sockets_close(&statsd.sockets); +} + +void *statsd_main(void *ptr) { + (void)ptr; + + info("STATSD main thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + // ---------------------------------------------------------------------------------------------------------------- + // statsd configuration + + statsd.enabled = config_get_boolean(CONFIG_SECTION_STATSD, "enabled", statsd.enabled); + + statsd.update_every = default_rrd_update_every; + statsd.update_every = (int)config_get_number(CONFIG_SECTION_STATSD, "update every (flushInterval)", statsd.update_every); + if(statsd.update_every < default_rrd_update_every) { + error("STATSD: minimum flush interval %d given, but the minimum is the update every of netdata. Using %d", statsd.update_every, default_rrd_update_every); + statsd.update_every = default_rrd_update_every; + } + +#ifdef HAVE_RECVMMSG + statsd.recvmmsg_size = (size_t)config_get_number(CONFIG_SECTION_STATSD, "udp messages to process at once", (long long)statsd.recvmmsg_size); +#endif + + statsd.charts_for = simple_pattern_create(config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), SIMPLE_PATTERN_EXACT); + statsd.max_private_charts = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts allowed", (long long)statsd.max_private_charts); + statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts * 5); + statsd.private_charts_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_STATSD, "private charts memory mode", rrd_memory_mode_name(default_rrd_memory_mode))); + statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries); + + statsd.histogram_percentile = (double)config_get_float(CONFIG_SECTION_STATSD, "histograms and timers percentile (percentThreshold)", statsd.histogram_percentile); + if(isless(statsd.histogram_percentile, 0) || isgreater(statsd.histogram_percentile, 100)) { + error("STATSD: invalid histograms and timers percentile %0.5f given", statsd.histogram_percentile); + statsd.histogram_percentile = 95.0; + } + { + char buffer[100 + 1]; + snprintf(buffer, 100, "%0.1f%%", statsd.histogram_percentile); + statsd.histogram_percentile_str = strdupz(buffer); + } + + if(config_get_boolean(CONFIG_SECTION_STATSD, "add dimension for number of events received", 1)) { + statsd.gauges.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT; + statsd.counters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT; + statsd.meters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT; + statsd.sets.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT; + statsd.histograms.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT; + statsd.timers.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT; + } + + if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on gauges (deleteGauges)", 0)) + statsd.gauges.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + + if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on counters (deleteCounters)", 0)) + statsd.counters.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + + if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on meters (deleteMeters)", 0)) + statsd.meters.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + + if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on sets (deleteSets)", 0)) + statsd.sets.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + + if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on histograms (deleteHistograms)", 0)) + statsd.histograms.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + + if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on timers (deleteTimers)", 0)) + statsd.timers.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED; + +#ifdef STATSD_MULTITHREADED + statsd.threads = (int)config_get_number(CONFIG_SECTION_STATSD, "threads", processors); + if(statsd.threads < 1) { + error("STATSD: Invalid number of threads %d, using %d", statsd.threads, processors); + statsd.threads = processors; + config_set_number(CONFIG_SECTION_STATSD, "collector threads", statsd.threads); + } +#else + statsd.threads = 1; +#endif + + // read custom application definitions + { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/statsd.d", netdata_configured_config_dir); + statsd_readdir(filename); + } + + // ---------------------------------------------------------------------------------------------------------------- + // statsd setup + + if(!statsd.enabled) return NULL; + + statsd_listen_sockets_setup(); + if(!statsd.sockets.opened) { + error("STATSD: No statsd sockets to listen to. statsd will be disabled."); + pthread_exit(NULL); + } + + pthread_t threads[statsd.threads]; + int i; + + for(i = 0; i < statsd.threads ;i++) { + if(pthread_create(&threads[i], NULL, statsd_collector_thread, &i)) + error("STATSD: failed to create child thread."); + + else if(pthread_detach(threads[i])) + error("STATSD: cannot request detach of child thread."); + } + + pthread_cleanup_push(statsd_main_cleanup, &threads); + + // ---------------------------------------------------------------------------------------------------------------- + // statsd monitoring charts + + RRDSET *st_metrics = rrdset_create_localhost( + "netdata" + , "statsd_metrics" + , NULL + , "statsd" + , NULL + , "Metrics in the netdata statsd database" + , "metrics" + , 132000 + , statsd.update_every + , RRDSET_TYPE_STACKED + ); + RRDDIM *rd_metrics_gauge = rrddim_add(st_metrics, "gauges", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_metrics_counter = rrddim_add(st_metrics, "counters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_metrics_timer = rrddim_add(st_metrics, "timers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_metrics_meter = rrddim_add(st_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_metrics_histogram = rrddim_add(st_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + RRDDIM *rd_metrics_set = rrddim_add(st_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + RRDSET *st_events = rrdset_create_localhost( + "netdata" + , "statsd_events" + , NULL + , "statsd" + , NULL + , "Events processed by the netdata statsd server" + , "events/s" + , 132001 + , statsd.update_every + , RRDSET_TYPE_STACKED + ); + RRDDIM *rd_events_gauge = rrddim_add(st_events, "gauges", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_events_counter = rrddim_add(st_events, "counters", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_events_timer = rrddim_add(st_events, "timers", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_events_meter = rrddim_add(st_events, "meters", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_events_histogram = rrddim_add(st_events, "histograms", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_events_set = rrddim_add(st_events, "sets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_events_unknown = rrddim_add(st_events, "unknown", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_events_errors = rrddim_add(st_events, "errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + RRDSET *st_reads = rrdset_create_localhost( + "netdata" + , "statsd_reads" + , NULL + , "statsd" + , NULL + , "Read operations made by the netdata statsd server" + , "reads/s" + , 132002 + , statsd.update_every + , RRDSET_TYPE_STACKED + ); + RRDDIM *rd_reads_tcp = rrddim_add(st_reads, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_reads_udp = rrddim_add(st_reads, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + RRDSET *st_bytes = rrdset_create_localhost( + "netdata" + , "statsd_bytes" + , NULL + , "statsd" + , NULL + , "Bytes read by the netdata statsd server" + , "kbps" + , 132003 + , statsd.update_every + , RRDSET_TYPE_STACKED + ); + RRDDIM *rd_bytes_tcp = rrddim_add(st_bytes, "tcp", NULL, 8, 1024, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_bytes_udp = rrddim_add(st_bytes, "udp", NULL, 8, 1024, RRD_ALGORITHM_INCREMENTAL); + + RRDSET *st_packets = rrdset_create_localhost( + "netdata" + , "statsd_packets" + , NULL + , "statsd" + , NULL + , "Network packets processed by the netdata statsd server" + , "packets/s" + , 132004 + , statsd.update_every + , RRDSET_TYPE_STACKED + ); + RRDDIM *rd_packets_tcp = rrddim_add(st_packets, "tcp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + RRDDIM *rd_packets_udp = rrddim_add(st_packets, "udp", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + + RRDSET *st_pcharts = rrdset_create_localhost( + "netdata" + , "private_charts" + , NULL + , "statsd" + , NULL + , "Private metric charts created by the netdata statsd server" + , "charts" + , 132010 + , statsd.update_every + , RRDSET_TYPE_AREA + ); + RRDDIM *rd_pcharts = rrddim_add(st_pcharts, "charts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + + // ---------------------------------------------------------------------------------------------------------------- + // statsd thread to turn metrics into charts + + usec_t step = statsd.update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); + for(;;) { + usec_t hb_dt = heartbeat_next(&hb, step); + + if(unlikely(netdata_exit)) + break; + + statsd_flush_index_metrics(&statsd.gauges, statsd_flush_gauge); + statsd_flush_index_metrics(&statsd.counters, statsd_flush_counter); + statsd_flush_index_metrics(&statsd.meters, statsd_flush_meter); + statsd_flush_index_metrics(&statsd.timers, statsd_flush_timer); + statsd_flush_index_metrics(&statsd.histograms, statsd_flush_histogram); + statsd_flush_index_metrics(&statsd.sets, statsd_flush_set); + + statsd_update_all_app_charts(); + + if(unlikely(netdata_exit)) + break; + + if(hb_dt) { + rrdset_next(st_metrics); + rrdset_next(st_events); + rrdset_next(st_reads); + rrdset_next(st_bytes); + rrdset_next(st_packets); + rrdset_next(st_pcharts); + } + + rrddim_set_by_pointer(st_metrics, rd_metrics_gauge, (collected_number)statsd.gauges.metrics); + rrddim_set_by_pointer(st_metrics, rd_metrics_counter, (collected_number)statsd.counters.metrics); + rrddim_set_by_pointer(st_metrics, rd_metrics_timer, (collected_number)statsd.timers.metrics); + rrddim_set_by_pointer(st_metrics, rd_metrics_meter, (collected_number)statsd.meters.metrics); + rrddim_set_by_pointer(st_metrics, rd_metrics_histogram, (collected_number)statsd.histograms.metrics); + rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics); + + rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events); + rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events); + rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events); + rrddim_set_by_pointer(st_events, rd_events_meter, (collected_number)statsd.meters.events); + rrddim_set_by_pointer(st_events, rd_events_histogram, (collected_number)statsd.histograms.events); + rrddim_set_by_pointer(st_events, rd_events_set, (collected_number)statsd.sets.events); + rrddim_set_by_pointer(st_events, rd_events_unknown, (collected_number)statsd.unknown_types); + rrddim_set_by_pointer(st_events, rd_events_errors, (collected_number)statsd.socket_errors); + + rrddim_set_by_pointer(st_reads, rd_reads_tcp, (collected_number)statsd.tcp_socket_reads); + rrddim_set_by_pointer(st_reads, rd_reads_udp, (collected_number)statsd.udp_socket_reads); + + rrddim_set_by_pointer(st_bytes, rd_bytes_tcp, (collected_number)statsd.tcp_bytes_read); + rrddim_set_by_pointer(st_bytes, rd_bytes_udp, (collected_number)statsd.udp_bytes_read); + + rrddim_set_by_pointer(st_packets, rd_packets_tcp, (collected_number)statsd.tcp_packets_received); + rrddim_set_by_pointer(st_packets, rd_packets_udp, (collected_number)statsd.udp_packets_received); + + rrddim_set_by_pointer(st_pcharts, rd_pcharts, (collected_number)statsd.private_charts); + + if(unlikely(netdata_exit)) + break; + + rrdset_done(st_metrics); + rrdset_done(st_events); + rrdset_done(st_reads); + rrdset_done(st_bytes); + rrdset_done(st_packets); + rrdset_done(st_pcharts); + + if(unlikely(netdata_exit)) + break; + } + + pthread_cleanup_pop(1); + + pthread_exit(NULL); + return NULL; +} diff --git a/src/statsd.h b/src/statsd.h new file mode 100644 index 000000000..17af098e8 --- /dev/null +++ b/src/statsd.h @@ -0,0 +1,9 @@ +#ifndef NETDATA_STATSD_H +#define NETDATA_STATSD_H + +#define STATSD_LISTEN_PORT 8125 +#define STATSD_LISTEN_BACKLOG 4096 + +extern void *statsd_main(void *ptr); + +#endif //NETDATA_STATSD_H diff --git a/src/storage_number.h b/src/storage_number.h index 74d24a322..34ed0d89c 100644 --- a/src/storage_number.h +++ b/src/storage_number.h @@ -29,6 +29,7 @@ typedef uint32_t storage_number; // extract the flags #define get_storage_number_flags(value) ((((storage_number)value) & (1 << 24)) | (((storage_number)value) & (2 << 24)) | (((storage_number)value) & (4 << 24))) +#define SN_EMPTY_SLOT 0x00000000 // checks #define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0) diff --git a/src/sys_fs_cgroup.c b/src/sys_fs_cgroup.c index 8f31527de..0f9c8854a 100644 --- a/src/sys_fs_cgroup.c +++ b/src/sys_fs_cgroup.c @@ -147,11 +147,17 @@ void read_cgroup_plugin_configuration() { enabled_cgroup_patterns = simple_pattern_create( config_get("plugin:cgroups", "enable by default cgroups matching", - " /system.slice/docker-*.scope " - " /qemu.slice/*.scope " // #1949 + // ---------------------------------------------------------------- + + " !*/init.scope " // ignore init.scope + " *.scope " // we need all *.scope for sure + + // ---------------------------------------------------------------- + + " !*/vcpu* " // libvirtd adds these sub-cgroups + " !*/emulator " // libvirtd adds these sub-cgroups " !*.mount " " !*.partition " - " !*.scope " " !*.service " " !*.slice " " !*.swap " @@ -171,12 +177,14 @@ void read_cgroup_plugin_configuration() { enabled_cgroup_paths = simple_pattern_create( config_get("plugin:cgroups", "search for cgroups in subpaths matching", - " !*-qemu " // #345 + " !*/init.scope " // ignore init.scope + " !*-qemu " // #345 " !/init.scope " " !/system " " !/systemd " " !/user " " !/user.slice " + " !/lxc/*/ns/* " // #2161 " * " ), SIMPLE_PATTERN_EXACT); @@ -185,13 +193,13 @@ void read_cgroup_plugin_configuration() { enabled_cgroup_renames = simple_pattern_create( config_get("plugin:cgroups", "run script to rename cgroups matching", - " /qemu.slice/*.scope " // #1949 + " *.scope " " *docker* " " *lxc* " + " *qemu* " " !/ " " !*.mount " " !*.partition " - " !*.scope " " !*.service " " !*.slice " " !*.swap " @@ -893,20 +901,20 @@ static inline struct cgroup *cgroup_add(const char *id) { static inline void cgroup_free(struct cgroup *cg) { debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available"); - if(cg->st_cpu) rrdset_flag_set(cg->st_cpu, RRDSET_FLAG_OBSOLETE); - if(cg->st_cpu_per_core) rrdset_flag_set(cg->st_cpu_per_core, RRDSET_FLAG_OBSOLETE); - if(cg->st_mem) rrdset_flag_set(cg->st_mem, RRDSET_FLAG_OBSOLETE); - if(cg->st_writeback) rrdset_flag_set(cg->st_writeback, RRDSET_FLAG_OBSOLETE); - if(cg->st_mem_activity) rrdset_flag_set(cg->st_mem_activity, RRDSET_FLAG_OBSOLETE); - if(cg->st_pgfaults) rrdset_flag_set(cg->st_pgfaults, RRDSET_FLAG_OBSOLETE); - if(cg->st_mem_usage) rrdset_flag_set(cg->st_mem_usage, RRDSET_FLAG_OBSOLETE); - if(cg->st_mem_failcnt) rrdset_flag_set(cg->st_mem_failcnt, RRDSET_FLAG_OBSOLETE); - if(cg->st_io) rrdset_flag_set(cg->st_io, RRDSET_FLAG_OBSOLETE); - if(cg->st_serviced_ops) rrdset_flag_set(cg->st_serviced_ops, RRDSET_FLAG_OBSOLETE); - if(cg->st_throttle_io) rrdset_flag_set(cg->st_throttle_io, RRDSET_FLAG_OBSOLETE); - if(cg->st_throttle_serviced_ops) rrdset_flag_set(cg->st_throttle_serviced_ops, RRDSET_FLAG_OBSOLETE); - if(cg->st_queued_ops) rrdset_flag_set(cg->st_queued_ops, RRDSET_FLAG_OBSOLETE); - if(cg->st_merged_ops) rrdset_flag_set(cg->st_merged_ops, RRDSET_FLAG_OBSOLETE); + if(cg->st_cpu) rrdset_is_obsolete(cg->st_cpu); + if(cg->st_cpu_per_core) rrdset_is_obsolete(cg->st_cpu_per_core); + if(cg->st_mem) rrdset_is_obsolete(cg->st_mem); + if(cg->st_writeback) rrdset_is_obsolete(cg->st_writeback); + if(cg->st_mem_activity) rrdset_is_obsolete(cg->st_mem_activity); + if(cg->st_pgfaults) rrdset_is_obsolete(cg->st_pgfaults); + if(cg->st_mem_usage) rrdset_is_obsolete(cg->st_mem_usage); + if(cg->st_mem_failcnt) rrdset_is_obsolete(cg->st_mem_failcnt); + if(cg->st_io) rrdset_is_obsolete(cg->st_io); + if(cg->st_serviced_ops) rrdset_is_obsolete(cg->st_serviced_ops); + if(cg->st_throttle_io) rrdset_is_obsolete(cg->st_throttle_io); + if(cg->st_throttle_serviced_ops) rrdset_is_obsolete(cg->st_throttle_serviced_ops); + if(cg->st_queued_ops) rrdset_is_obsolete(cg->st_queued_ops); + if(cg->st_merged_ops) rrdset_is_obsolete(cg->st_merged_ops); freez(cg->cpuacct_usage.cpu_percpu); diff --git a/src/unit_test.c b/src/unit_test.c index 0866d215c..9b008138f 100644 --- a/src/unit_test.c +++ b/src/unit_test.c @@ -17,8 +17,8 @@ int check_storage_number(calculated_number n, int debug) { if(dcdiff < 0) dcdiff = -dcdiff; - size_t len = print_calculated_number(buffer, d); - calculated_number p = str2l(buffer); + size_t len = (size_t)print_calculated_number(buffer, d); + calculated_number p = str2ld(buffer, NULL); calculated_number pdiff = n - p; calculated_number pcdiff = pdiff * 100.0 / n; if(pcdiff < 0) pcdiff = -pcdiff; @@ -229,6 +229,45 @@ int unit_test_storage() return r; } +int unit_test_str2ld() { + char *values[] = { + "1.234567", "-35.6", "0.00123", "23842384234234.2", ".1", "1.2e-10", + "hello", "1wrong", "nan", "inf", NULL + }; + + int i; + for(i = 0; values[i] ; i++) { + char *e_mine = "hello", *e_sys = "world"; + long double mine = str2ld(values[i], &e_mine); + long double sys = strtold(values[i], &e_sys); + + if(isnan(mine)) { + if(!isnan(sys)) { + fprintf(stderr, "Value '%s' is parsed as %Lf, but system believes it is %Lf.\n", values[i], mine, sys); + return -1; + } + } + else if(isinf(mine)) { + if(!isinf(sys)) { + fprintf(stderr, "Value '%s' is parsed as %Lf, but system believes it is %Lf.\n", values[i], mine, sys); + return -1; + } + } + else if(mine != sys && abs(mine-sys) > 0.000001) { + fprintf(stderr, "Value '%s' is parsed as %Lf, but system believes it is %Lf, delta %Lf.\n", values[i], mine, sys, sys-mine); + return -1; + } + + if(e_mine != e_sys) { + fprintf(stderr, "Value '%s' is parsed correctly, but endptr is not right\n", values[i]); + return -1; + } + + fprintf(stderr, "str2ld() parsed value '%s' exactly the same way with strtold(), returned %Lf vs %Lf\n", values[i], mine, sys); + } + + return 0; +} // -------------------------------------------------------------------------------------------------------------------- @@ -244,7 +283,7 @@ struct test { int update_every; unsigned long long multiplier; unsigned long long divisor; - int algorithm; + RRD_ALGORITHM algorithm; unsigned long feed_entries; unsigned long result_entries; @@ -884,7 +923,7 @@ int run_test(struct test *test) { fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description); - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC; default_rrd_update_every = test->update_every; char name[101]; @@ -916,7 +955,9 @@ int run_test(struct test *test) (float)time_now / 1000000.0, ((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor, (((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor) / (calculated_number)test->feed[c].microseconds * (calculated_number)1000000); - rrdset_next_usec_unfiltered(st, test->feed[c].microseconds); + + // rrdset_next_usec_unfiltered(st, test->feed[c].microseconds); + st->usec_since_last_update = test->feed[c].microseconds; } else { fprintf(stderr, " > %s: feeding position %lu\n", test->name, c+1); @@ -1091,7 +1132,7 @@ int unit_test(long delay, long shift) snprintfz(name, 100, "unittest-%d-%ld-%ld", repeat, delay, shift); //debug_flags = 0xffffffff; - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC; default_rrd_update_every = 1; int do_abs = 1; @@ -1125,7 +1166,8 @@ int unit_test(long delay, long shift) fprintf(stderr, "\n\nLOOP = %lu, DELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", c, delay, i); if(c) { - rrdset_next_usec_unfiltered(st, delay); + // rrdset_next_usec_unfiltered(st, delay); + st->usec_since_last_update = delay; } if(do_abs) rrddim_set(st, "absolute", i); if(do_inc) rrddim_set(st, "incremental", i); diff --git a/src/unit_test.h b/src/unit_test.h index 916ad71f2..3240b5f0e 100644 --- a/src/unit_test.h +++ b/src/unit_test.h @@ -4,5 +4,6 @@ extern int unit_test_storage(void); extern int unit_test(long delay, long shift); extern int run_all_mockup_tests(void); +extern int unit_test_str2ld(void); #endif /* NETDATA_UNIT_TEST_H */ diff --git a/src/web_api_v1.c b/src/web_api_v1.c index 0acc43acb..3ffd8c324 100644 --- a/src/web_api_v1.c +++ b/src/web_api_v1.c @@ -208,6 +208,10 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) { int format = ALLMETRICS_SHELL; + int help = 0, types = 0, names = backend_send_names; // prometheus options + const char *prometheus_server = w->client_ip; + uint32_t prometheus_options = backend_options; + const char *prometheus_prefix = backend_prefix; while(url) { char *value = mystrsep(&url, "?&"); @@ -222,11 +226,40 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client format = ALLMETRICS_SHELL; else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS)) format = ALLMETRICS_PROMETHEUS; + else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS)) + format = ALLMETRICS_PROMETHEUS_ALL_HOSTS; else if(!strcmp(value, ALLMETRICS_FORMAT_JSON)) format = ALLMETRICS_JSON; else format = 0; } + else if(!strcmp(name, "help")) { + if(!strcmp(value, "yes")) + help = 1; + else + help = 0; + } + else if(!strcmp(name, "types")) { + if(!strcmp(value, "yes")) + types = 1; + else + types = 0; + } + else if(!strcmp(name, "names")) { + if(!strcmp(value, "yes")) + names = 1; + else + names = 0; + } + else if(!strcmp(name, "server")) { + prometheus_server = value; + } + else if(!strcmp(name, "prefix")) { + prometheus_prefix = value; + } + else if(!strcmp(name, "data") || !strcmp(name, "source") || !strcmp(name, "data source") || !strcmp(name, "data-source") || !strcmp(name, "data_source") || !strcmp(name, "datasource")) { + prometheus_options = backend_parse_data_source(value, prometheus_options); + } } buffer_flush(w->response.data); @@ -245,12 +278,17 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client case ALLMETRICS_PROMETHEUS: w->response.data->contenttype = CT_PROMETHEUS; - rrd_stats_api_v1_charts_allmetrics_prometheus(host, w->response.data); + rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_options, help, types, names); + return 200; + + case ALLMETRICS_PROMETHEUS_ALL_HOSTS: + w->response.data->contenttype = CT_PROMETHEUS; + rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_options, help, types, names); return 200; default: w->response.data->contenttype = CT_TEXT_PLAIN; - buffer_strcat(w->response.data, "Which format? '" ALLMETRICS_FORMAT_SHELL "', '" ALLMETRICS_FORMAT_PROMETHEUS "' and '" ALLMETRICS_FORMAT_JSON "' are currently supported."); + buffer_strcat(w->response.data, "Which format? '" ALLMETRICS_FORMAT_SHELL "', '" ALLMETRICS_FORMAT_PROMETHEUS "', '" ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS "' and '" ALLMETRICS_FORMAT_JSON "' are currently supported."); return 400; } } diff --git a/src/web_client.c b/src/web_client.c index b5b25899f..7da080705 100644 --- a/src/web_client.c +++ b/src/web_client.c @@ -57,13 +57,7 @@ struct web_client *web_client_create(int listener) { w->mode = WEB_CLIENT_MODE_NORMAL; { - struct sockaddr *sadr; - socklen_t addrlen; - - sadr = (struct sockaddr*) &w->clientaddr; - addrlen = sizeof(w->clientaddr); - - w->ifd = accept4(listener, sadr, &addrlen, SOCK_NONBLOCK); + w->ifd = accept_socket(listener, SOCK_NONBLOCK, w->client_ip, sizeof(w->client_ip), w->client_port, sizeof(w->client_port)); if (w->ifd == -1) { error("%llu: Cannot accept new incoming connection.", w->id); freez(w); @@ -71,33 +65,6 @@ struct web_client *web_client_create(int listener) { } w->ofd = w->ifd; - if(getnameinfo(sadr, addrlen, w->client_ip, NI_MAXHOST, w->client_port, NI_MAXSERV, NI_NUMERICHOST | NI_NUMERICSERV) != 0) { - error("Cannot getnameinfo() on received client connection."); - strncpyz(w->client_ip, "UNKNOWN", NI_MAXHOST); - strncpyz(w->client_port, "UNKNOWN", NI_MAXSERV); - } - w->client_ip[NI_MAXHOST] = '\0'; - w->client_port[NI_MAXSERV] = '\0'; - - switch(sadr->sa_family) { - case AF_INET: - debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); - break; - - case AF_INET6: - if(strncmp(w->client_ip, "::ffff:", 7) == 0) { - memmove(w->client_ip, &w->client_ip[7], strlen(&w->client_ip[7]) + 1); - debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); - } - else - debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv6 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); - break; - - default: - debug(D_WEB_CLIENT_ACCESS, "%llu: New UNKNOWN web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd); - break; - } - int flag = 1; if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0) error("%llu: failed to enable TCP_NODELAY on socket.", w->id); @@ -388,8 +355,8 @@ int mysendfile(struct web_client *w, char *filename) { return 404; } } - if(fcntl(w->ifd, F_SETFL, O_NONBLOCK) < 0) - error("%llu: Cannot set O_NONBLOCK on file '%s'.", w->id, webfilename); + + sock_setnonblock(w->ifd); // pick a Content-Type for the file if(strstr(filename, ".html") != NULL) w->response.data->contenttype = CT_TEXT_HTML; @@ -995,13 +962,22 @@ static inline void web_client_send_http_header(struct web_client *w) { web_client_crock_socket(w); - ssize_t bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0); + size_t count = 0; + ssize_t bytes; + while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) { + count++; + + if(count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) { + error("Cannot send HTTP headers to web client."); + break; + } + } + if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) { if(bytes > 0) w->stats_sent_bytes += bytes; - debug(D_WEB_CLIENT, "%llu: HTTP Header failed to be sent (I sent %zu bytes but the system sent %zd bytes). Closing web client." - , w->id + error("HTTP headers failed to be sent (I sent %zu bytes but the system sent %zd bytes). Closing web client." , buffer_strlen(w->response.header_output) , bytes); diff --git a/src/web_client.h b/src/web_client.h index 70c5b1ff0..617917df0 100644 --- a/src/web_client.h +++ b/src/web_client.h @@ -83,7 +83,6 @@ struct web_client { char cookie2[COOKIE_MAX+1]; char origin[ORIGIN_MAX+1]; - struct sockaddr_storage clientaddr; struct response response; size_t stats_received_bytes; diff --git a/src/web_server.c b/src/web_server.c index 593a82a57..491cd11aa 100644 --- a/src/web_server.c +++ b/src/web_server.c @@ -1,15 +1,14 @@ #include "common.h" -int listen_backlog = LISTEN_BACKLOG; -size_t listen_fds_count = 0; -int listen_fds[MAX_LISTEN_FDS] = { [0 ... 99] = -1 }; -char *listen_fds_names[MAX_LISTEN_FDS] = { [0 ... 99] = NULL }; -int listen_port = LISTEN_PORT; +static LISTEN_SOCKETS api_sockets = { + .config_section = CONFIG_SECTION_WEB, + .default_bind_to = "*", + .default_port = API_LISTEN_PORT, + .backlog = API_LISTEN_BACKLOG +}; WEB_SERVER_MODE web_server_mode = WEB_SERVER_MODE_MULTI_THREADED; -static int shown_server_socket_error = 0; - #ifdef NETDATA_INTERNAL_CHECKS static void log_allocations(void) { @@ -47,42 +46,7 @@ static void log_allocations(void) } #endif /* NETDATA_INTERNAL_CHECKS */ -#ifndef HAVE_ACCEPT4 -int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) { - int fd = accept(sock, addr, addrlen); - int newflags = 0; - - if (fd < 0) return fd; - - if (flags & SOCK_NONBLOCK) { - newflags |= O_NONBLOCK; - flags &= ~SOCK_NONBLOCK; - } - -#ifdef SOCK_CLOEXEC -#ifdef O_CLOEXEC - if (flags & SOCK_CLOEXEC) { - newflags |= O_CLOEXEC; - flags &= ~SOCK_CLOEXEC; - } -#endif -#endif - - if (flags) { - errno = -EINVAL; - return -1; - } - - if (fcntl(fd, F_SETFL, newflags) < 0) { - int saved_errno = errno; - close(fd); - errno = saved_errno; - return -1; - } - - return fd; -} -#endif +// -------------------------------------------------------------------------------------- WEB_SERVER_MODE web_server_mode_id(const char *mode) { if(!strcmp(mode, "none")) @@ -107,277 +71,15 @@ const char *web_server_mode_name(WEB_SERVER_MODE id) { } } -int create_listen_socket4(const char *ip, int port, int listen_backlog) { - int sock; - int sockopt = 1; - - debug(D_LISTENER, "IPv4 creating new listening socket on ip '%s' port %d", ip, port); - - sock = socket(AF_INET, SOCK_STREAM, 0); - if(sock < 0) { - error("IPv4 socket() on ip '%s' port %d failed.", ip, port); - shown_server_socket_error = 1; - return -1; - } - - /* avoid "address already in use" */ - if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt)) != 0) - error("Cannot set SO_REUSEADDR on ip '%s' port's %d.", ip, port); - - struct sockaddr_in name; - memset(&name, 0, sizeof(struct sockaddr_in)); - name.sin_family = AF_INET; - name.sin_port = htons (port); - - int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr); - if(ret != 1) { - error("Failed to convert IP '%s' to a valid IPv4 address.", ip); - shown_server_socket_error = 1; - close(sock); - return -1; - } - - if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { - close(sock); - error("IPv4 bind() on ip '%s' port %d failed.", ip, port); - shown_server_socket_error = 1; - return -1; - } - - if(listen(sock, listen_backlog) < 0) { - close(sock); - error("IPv4 listen() on ip '%s' port %d failed.", ip, port); - shown_server_socket_error = 1; - return -1; - } - - debug(D_LISTENER, "Listening on IPv4 ip '%s' port %d", ip, port); - return sock; -} - -int create_listen_socket6(const char *ip, int port, int listen_backlog) { - int sock = -1; - int sockopt = 1; - int ipv6only = 1; - - debug(D_LISTENER, "IPv6 creating new listening socket on ip '%s' port %d", ip, port); - - sock = socket(AF_INET6, SOCK_STREAM, 0); - if (sock < 0) { - error("IPv6 socket() on ip '%s' port %d failed.", ip, port); - shown_server_socket_error = 1; - return -1; - } - - /* avoid "address already in use" */ - if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt)) != 0) - error("Cannot set SO_REUSEADDR on ip '%s' port's %d.", ip, port); - - /* IPv6 only */ - if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0) - error("Cannot set IPV6_V6ONLY on ip '%s' port's %d.", ip, port); - - struct sockaddr_in6 name; - memset(&name, 0, sizeof(struct sockaddr_in6)); - name.sin6_family = AF_INET6; - name.sin6_port = htons ((uint16_t) port); - - int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr); - if(ret != 1) { - error("Failed to convert IP '%s' to a valid IPv6 address.", ip); - shown_server_socket_error = 1; - close(sock); - return -1; - } - - name.sin6_scope_id = 0; - - if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { - close(sock); - error("IPv6 bind() on ip '%s' port %d failed.", ip, port); - shown_server_socket_error = 1; - return -1; - } - - if (listen(sock, listen_backlog) < 0) { - close(sock); - error("IPv6 listen() on ip '%s' port %d failed.", ip, port); - shown_server_socket_error = 1; - return -1; - } - - debug(D_LISTENER, "Listening on IPv6 ip '%s' port %d", ip, port); - return sock; -} - -static inline int add_listen_socket(int fd, const char *ip, int port) { - if(listen_fds_count >= MAX_LISTEN_FDS) { - error("Too many listening sockets. Failed to add listening socket at ip '%s' port %d", ip, port); - shown_server_socket_error = 1; - close(fd); - return -1; - } - - listen_fds[listen_fds_count] = fd; - - char buffer[100 + 1]; - snprintfz(buffer, 100, "[%s]:%d", ip, port); - listen_fds_names[listen_fds_count] = strdupz(buffer); - - listen_fds_count++; - return 0; -} - -int is_listen_socket(int fd) { - size_t i; - for(i = 0; i < listen_fds_count ;i++) - if(listen_fds[i] == fd) return 1; - - return 0; -} - -static inline void close_listen_sockets(void) { - size_t i; - for(i = 0; i < listen_fds_count ;i++) { - close(listen_fds[i]); - listen_fds[i] = -1; - - freez(listen_fds_names[i]); - listen_fds_names[i] = NULL; - } - - listen_fds_count = 0; -} - -static inline int bind_to_one(const char *definition, int default_port, int listen_backlog) { - int added = 0; - struct addrinfo hints; - struct addrinfo *result = NULL, *rp = NULL; - - char buffer[strlen(definition) + 1]; - strcpy(buffer, definition); - - char buffer2[10 + 1]; - snprintfz(buffer2, 10, "%d", default_port); - - char *ip = buffer, *port = buffer2; - - char *e = ip; - if(*e == '[') { - e = ++ip; - while(*e && *e != ']') e++; - if(*e == ']') { - *e = '\0'; - e++; - } - } - else { - while(*e && *e != ':') e++; - } - - if(*e == ':') { - port = e + 1; - *e = '\0'; - } - - if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all")) - ip = NULL; - if(!*port) - port = buffer2; - - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ - hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */ - hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ - hints.ai_protocol = 0; /* Any protocol */ - hints.ai_canonname = NULL; - hints.ai_addr = NULL; - hints.ai_next = NULL; - - int r = getaddrinfo(ip, port, &hints, &result); - if (r != 0) { - error("getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r)); - return -1; - } - - for (rp = result; rp != NULL; rp = rp->ai_next) { - int fd = -1; - - char rip[INET_ADDRSTRLEN + INET6_ADDRSTRLEN] = "INVALID"; - int rport = default_port; - - switch (rp->ai_addr->sa_family) { - case AF_INET: { - struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr; - inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN); - rport = ntohs(sin->sin_port); - fd = create_listen_socket4(rip, rport, listen_backlog); - break; - } - - case AF_INET6: { - struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr; - inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN); - rport = ntohs(sin6->sin6_port); - fd = create_listen_socket6(rip, rport, listen_backlog); - break; - } - } - - if (fd == -1) - error("Cannot bind to ip '%s', port %d", rip, rport); - else { - add_listen_socket(fd, rip, rport); - added++; - } - } - - freeaddrinfo(result); - - return added; -} - -int create_listen_sockets(void) { - shown_server_socket_error = 0; - - listen_backlog = (int) config_get_number(CONFIG_SECTION_WEB, "listen backlog", LISTEN_BACKLOG); - - listen_port = (int) config_get_number(CONFIG_SECTION_WEB, "default port", LISTEN_PORT); - if(listen_port < 1 || listen_port > 65535) { - error("Invalid listen port %d given. Defaulting to %d.", listen_port, LISTEN_PORT); - listen_port = (int) config_set_number(CONFIG_SECTION_WEB, "default port", LISTEN_PORT); - } - debug(D_OPTIONS, "Default listen port set to %d.", listen_port); - - char *s = config_get(CONFIG_SECTION_WEB, "bind to", "*"); - while(*s) { - char *e = s; - - // skip separators, moving both s(tart) and e(nd) - while(isspace(*e) || *e == ',') s = ++e; - - // move e(nd) to the first separator - while(*e && !isspace(*e) && *e != ',') e++; - - // is there anything? - if(!*s || s == e) break; - - char buf[e - s + 1]; - strncpyz(buf, s, e - s); - bind_to_one(buf, listen_port, listen_backlog); +// -------------------------------------------------------------------------------------- - s = e; - } +int api_listen_sockets_setup(void) { + int socks = listen_sockets_setup(&api_sockets); - if(!listen_fds_count) - fatal("Cannot listen on any socket. Exiting..."); - else if(shown_server_socket_error) { - size_t i; - for(i = 0; i < listen_fds_count ;i++) - info("Listen socket %s opened.", listen_fds_names[i]); - } + if(!socks) + fatal("LISTENER: Cannot listen on any API socket. Exiting..."); - return (int)listen_fds_count; + return socks; } // -------------------------------------------------------------------------------------- @@ -422,25 +124,25 @@ void *socket_listen_main_multi_threaded(void *ptr) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("Cannot set pthread cancel state to ENABLE."); - if(!listen_fds_count) + if(!api_sockets.opened) fatal("LISTENER: No sockets to listen to."); - struct pollfd *fds = callocz(sizeof(struct pollfd), listen_fds_count); + struct pollfd *fds = callocz(sizeof(struct pollfd), api_sockets.opened); size_t i; - for(i = 0; i < listen_fds_count ;i++) { - fds[i].fd = listen_fds[i]; + for(i = 0; i < api_sockets.opened ;i++) { + fds[i].fd = api_sockets.fds[i]; fds[i].events = POLLIN; fds[i].revents = 0; - info("Listening on '%s'", (listen_fds_names[i])?listen_fds_names[i]:"UNKNOWN"); + info("Listening on '%s'", (api_sockets.fds_names[i])?api_sockets.fds_names[i]:"UNKNOWN"); } int timeout = 10 * 1000; for(;;) { // debug(D_WEB_CLIENT, "LISTENER: Waiting..."); - retval = poll(fds, listen_fds_count, timeout); + retval = poll(fds, api_sockets.opened, timeout); if(unlikely(retval == -1)) { error("LISTENER: poll() failed."); @@ -453,7 +155,7 @@ void *socket_listen_main_multi_threaded(void *ptr) { continue; } - for(i = 0 ; i < listen_fds_count ; i++) { + for(i = 0 ; i < api_sockets.opened ; i++) { short int revents = fds[i].revents; // check for new incoming connections @@ -486,7 +188,7 @@ void *socket_listen_main_multi_threaded(void *ptr) { } debug(D_WEB_CLIENT, "LISTENER: exit!"); - close_listen_sockets(); + listen_sockets_close(&api_sockets); freez(fds); @@ -555,7 +257,7 @@ void *socket_listen_main_single_threaded(void *ptr) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("Cannot set pthread cancel state to ENABLE."); - if(!listen_fds_count) + if(!api_sockets.opened) fatal("LISTENER: no listen sockets available."); size_t i; @@ -568,16 +270,16 @@ void *socket_listen_main_single_threaded(void *ptr) { FD_ZERO (&efds); int fdmax = 0; - for(i = 0; i < listen_fds_count ; i++) { - if (listen_fds[i] < 0 || listen_fds[i] >= FD_SETSIZE) - fatal("LISTENER: Listen socket %d is not ready, or invalid.", listen_fds[i]); + for(i = 0; i < api_sockets.opened ; i++) { + if (api_sockets.fds[i] < 0 || api_sockets.fds[i] >= FD_SETSIZE) + fatal("LISTENER: Listen socket %d is not ready, or invalid.", api_sockets.fds[i]); - info("Listening on '%s'", (listen_fds_names[i])?listen_fds_names[i]:"UNKNOWN"); + info("Listening on '%s'", (api_sockets.fds_names[i])?api_sockets.fds_names[i]:"UNKNOWN"); - FD_SET(listen_fds[i], &ifds); - FD_SET(listen_fds[i], &efds); - if(fdmax < listen_fds[i]) - fdmax = listen_fds[i]; + FD_SET(api_sockets.fds[i], &ifds); + FD_SET(api_sockets.fds[i], &efds); + if(fdmax < api_sockets.fds[i]) + fdmax = api_sockets.fds[i]; } for(;;) { @@ -596,10 +298,10 @@ void *socket_listen_main_single_threaded(void *ptr) { else if(likely(retval)) { debug(D_WEB_CLIENT_ACCESS, "LISTENER: got something."); - for(i = 0; i < listen_fds_count ; i++) { - if (FD_ISSET(listen_fds[i], &rifds)) { + for(i = 0; i < api_sockets.opened ; i++) { + if (FD_ISSET(api_sockets.fds[i], &rifds)) { debug(D_WEB_CLIENT_ACCESS, "LISTENER: new connection."); - w = web_client_create(listen_fds[i]); + w = web_client_create(api_sockets.fds[i]); if (single_threaded_link_client(w, &ifds, &ofds, &ifds, &fdmax) != 0) { web_client_free(w); } @@ -658,7 +360,7 @@ void *socket_listen_main_single_threaded(void *ptr) { } debug(D_WEB_CLIENT, "LISTENER: exit!"); - close_listen_sockets(); + listen_sockets_close(&api_sockets); static_thread->enabled = 0; pthread_exit(NULL); diff --git a/src/web_server.h b/src/web_server.h index 41dcfcf09..aa293695d 100644 --- a/src/web_server.h +++ b/src/web_server.h @@ -6,11 +6,12 @@ #define WEB_PATH_DATASOURCE "datasource" #define WEB_PATH_GRAPH "graph" -#define LISTEN_PORT 19999 -#define LISTEN_BACKLOG 100 +#ifndef API_LISTEN_PORT +#define API_LISTEN_PORT 19999 +#endif -#ifndef MAX_LISTEN_FDS -#define MAX_LISTEN_FDS 100 +#ifndef API_LISTEN_BACKLOG +#define API_LISTEN_BACKLOG 4096 #endif typedef enum web_server_mode { @@ -24,23 +25,8 @@ extern WEB_SERVER_MODE web_server_mode; extern WEB_SERVER_MODE web_server_mode_id(const char *mode); extern const char *web_server_mode_name(WEB_SERVER_MODE id); - extern void *socket_listen_main_multi_threaded(void *ptr); extern void *socket_listen_main_single_threaded(void *ptr); -extern int create_listen_sockets(void); -extern int is_listen_socket(int fd); - -#ifndef HAVE_ACCEPT4 -extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags); - -#ifndef SOCK_NONBLOCK -#define SOCK_NONBLOCK 00004000 -#endif /* #ifndef SOCK_NONBLOCK */ - -#ifndef SOCK_CLOEXEC -#define SOCK_CLOEXEC 02000000 -#endif /* #ifndef SOCK_CLOEXEC */ - -#endif /* #ifndef HAVE_ACCEPT4 */ +extern int api_listen_sockets_setup(void); #endif /* NETDATA_WEB_SERVER_H */ diff --git a/src/zfs_common.c b/src/zfs_common.c new file mode 100644 index 000000000..7fa05b03e --- /dev/null +++ b/src/zfs_common.c @@ -0,0 +1,677 @@ +#include "common.h" +#include "zfs_common.h" + +extern struct arcstats arcstats; + +void generate_charts_arcstats(int update_every) { + + // ARC reads + unsigned long long aread = arcstats.hits + arcstats.misses; + + // Demand reads + unsigned long long dhit = arcstats.demand_data_hits + arcstats.demand_metadata_hits; + unsigned long long dmiss = arcstats.demand_data_misses + arcstats.demand_metadata_misses; + unsigned long long dread = dhit + dmiss; + + // Prefetch reads + unsigned long long phit = arcstats.prefetch_data_hits + arcstats.prefetch_metadata_hits; + unsigned long long pmiss = arcstats.prefetch_data_misses + arcstats.prefetch_metadata_misses; + unsigned long long pread = phit + pmiss; + + // Metadata reads + unsigned long long mhit = arcstats.prefetch_metadata_hits + arcstats.demand_metadata_hits; + unsigned long long mmiss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses; + unsigned long long mread = mhit + mmiss; + + // l2 reads + unsigned long long l2hit = arcstats.l2_hits + arcstats.l2_misses; + unsigned long long l2miss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses; + unsigned long long l2read = l2hit + l2miss; + + // -------------------------------------------------------------------- + + { + static RRDSET *st_arc_size = NULL; + static RRDDIM *rd_arc_size = NULL; + static RRDDIM *rd_arc_target_size = NULL; + static RRDDIM *rd_arc_target_min_size = NULL; + static RRDDIM *rd_arc_target_max_size = NULL; + + if (unlikely(!st_arc_size)) { + st_arc_size = rrdset_create_localhost( + "zfs" + , "arc_size" + , NULL + , ZFS_FAMILY_SIZE + , NULL + , "ZFS ARC Size" + , "MB" + , 2000 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_arc_size = rrddim_add(st_arc_size, "size", "arcsz", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_arc_target_size = rrddim_add(st_arc_size, "target", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_arc_target_min_size = rrddim_add(st_arc_size, "min", "min (hard limit)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_arc_target_max_size = rrddim_add(st_arc_size, "max", "max (high water)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_arc_size); + + rrddim_set_by_pointer(st_arc_size, rd_arc_size, arcstats.size); + rrddim_set_by_pointer(st_arc_size, rd_arc_target_size, arcstats.c); + rrddim_set_by_pointer(st_arc_size, rd_arc_target_min_size, arcstats.c_min); + rrddim_set_by_pointer(st_arc_size, rd_arc_target_max_size, arcstats.c_max); + rrdset_done(st_arc_size); + } + + // -------------------------------------------------------------------- + + if(likely(l2exist)) { + static RRDSET *st_l2_size = NULL; + static RRDDIM *rd_l2_size = NULL; + static RRDDIM *rd_l2_asize = NULL; + + if (unlikely(!st_l2_size)) { + st_l2_size = rrdset_create_localhost( + "zfs" + , "l2_size" + , NULL + , ZFS_FAMILY_SIZE + , NULL + , "ZFS L2 ARC Size" + , "MB" + , 2000 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_l2_asize = rrddim_add(st_l2_size, "actual", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rd_l2_size = rrddim_add(st_l2_size, "size", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_l2_size); + + rrddim_set_by_pointer(st_l2_size, rd_l2_size, arcstats.l2_size); + rrddim_set_by_pointer(st_l2_size, rd_l2_asize, arcstats.l2_asize); + rrdset_done(st_l2_size); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_reads = NULL; + static RRDDIM *rd_aread = NULL; + static RRDDIM *rd_dread = NULL; + static RRDDIM *rd_pread = NULL; + static RRDDIM *rd_mread = NULL; + static RRDDIM *rd_l2read = NULL; + + if (unlikely(!st_reads)) { + st_reads = rrdset_create_localhost( + "zfs" + , "reads" + , NULL + , ZFS_FAMILY_ACCESSES + , NULL + , "ZFS Reads" + , "reads/s" + , 2010 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_aread = rrddim_add(st_reads, "areads", "arc", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_dread = rrddim_add(st_reads, "dreads", "demand", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_pread = rrddim_add(st_reads, "preads", "prefetch", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_mread = rrddim_add(st_reads, "mreads", "metadata", 1, 1, RRD_ALGORITHM_INCREMENTAL); + + if(l2exist) + rd_l2read = rrddim_add(st_reads, "l2reads", "l2", 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_reads); + + rrddim_set_by_pointer(st_reads, rd_aread, aread); + rrddim_set_by_pointer(st_reads, rd_dread, dread); + rrddim_set_by_pointer(st_reads, rd_pread, pread); + rrddim_set_by_pointer(st_reads, rd_mread, mread); + + if(l2exist) + rrddim_set_by_pointer(st_reads, rd_l2read, l2read); + + rrdset_done(st_reads); + } + + // -------------------------------------------------------------------- + + if(likely(l2exist)) { + static RRDSET *st_l2bytes = NULL; + static RRDDIM *rd_l2_read_bytes = NULL; + static RRDDIM *rd_l2_write_bytes = NULL; + + if (unlikely(!st_l2bytes)) { + st_l2bytes = rrdset_create_localhost( + "zfs" + , "bytes" + , NULL + , ZFS_FAMILY_ACCESSES + , NULL + , "ZFS ARC L2 Read/Write Rate" + , "kilobytes/s" + , 2200 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_l2_read_bytes = rrddim_add(st_l2bytes, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); + rd_l2_write_bytes = rrddim_add(st_l2bytes, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_l2bytes); + + rrddim_set_by_pointer(st_l2bytes, rd_l2_read_bytes, arcstats.l2_read_bytes); + rrddim_set_by_pointer(st_l2bytes, rd_l2_write_bytes, arcstats.l2_write_bytes); + rrdset_done(st_l2bytes); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_ahits = NULL; + static RRDDIM *rd_ahits = NULL; + static RRDDIM *rd_amisses = NULL; + + if (unlikely(!st_ahits)) { + st_ahits = rrdset_create_localhost( + "zfs" + , "hits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS ARC Hits" + , "percentage" + , 2020 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_ahits = rrddim_add(st_ahits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_amisses = rrddim_add(st_ahits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_ahits); + + rrddim_set_by_pointer(st_ahits, rd_ahits, arcstats.hits); + rrddim_set_by_pointer(st_ahits, rd_amisses, arcstats.misses); + rrdset_done(st_ahits); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_dhits = NULL; + static RRDDIM *rd_dhits = NULL; + static RRDDIM *rd_dmisses = NULL; + + if (unlikely(!st_dhits)) { + st_dhits = rrdset_create_localhost( + "zfs" + , "dhits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS Demand Hits" + , "percentage" + , 2030 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_dhits = rrddim_add(st_dhits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_dmisses = rrddim_add(st_dhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_dhits); + + rrddim_set_by_pointer(st_dhits, rd_dhits, dhit); + rrddim_set_by_pointer(st_dhits, rd_dmisses, dmiss); + rrdset_done(st_dhits); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_phits = NULL; + static RRDDIM *rd_phits = NULL; + static RRDDIM *rd_pmisses = NULL; + + if (unlikely(!st_phits)) { + st_phits = rrdset_create_localhost( + "zfs" + , "phits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS Prefetch Hits" + , "percentage" + , 2040 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_phits = rrddim_add(st_phits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_pmisses = rrddim_add(st_phits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_phits); + + rrddim_set_by_pointer(st_phits, rd_phits, phit); + rrddim_set_by_pointer(st_phits, rd_pmisses, pmiss); + rrdset_done(st_phits); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_mhits = NULL; + static RRDDIM *rd_mhits = NULL; + static RRDDIM *rd_mmisses = NULL; + + if (unlikely(!st_mhits)) { + st_mhits = rrdset_create_localhost( + "zfs" + , "mhits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS Metadata Hits" + , "percentage" + , 2050 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_mhits = rrddim_add(st_mhits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_mmisses = rrddim_add(st_mhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_mhits); + + rrddim_set_by_pointer(st_mhits, rd_mhits, mhit); + rrddim_set_by_pointer(st_mhits, rd_mmisses, mmiss); + rrdset_done(st_mhits); + } + + // -------------------------------------------------------------------- + + if(likely(l2exist)) { + static RRDSET *st_l2hits = NULL; + static RRDDIM *rd_l2hits = NULL; + static RRDDIM *rd_l2misses = NULL; + + if (unlikely(!st_l2hits)) { + st_l2hits = rrdset_create_localhost( + "zfs" + , "l2hits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS L2 Hits" + , "percentage" + , 2060 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_l2hits = rrddim_add(st_l2hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_l2misses = rrddim_add(st_l2hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_l2hits); + + rrddim_set_by_pointer(st_l2hits, rd_l2hits, l2hit); + rrddim_set_by_pointer(st_l2hits, rd_l2misses, l2miss); + rrdset_done(st_l2hits); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_list_hits = NULL; + static RRDDIM *rd_mfu = NULL; + static RRDDIM *rd_mru = NULL; + static RRDDIM *rd_mfug = NULL; + static RRDDIM *rd_mrug = NULL; + + if (unlikely(!st_list_hits)) { + st_list_hits = rrdset_create_localhost( + "zfs" + , "list_hits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS List Hits" + , "hits/s" + , 2100 + , update_every + , RRDSET_TYPE_AREA + ); + + rd_mfu = rrddim_add(st_list_hits, "mfu", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_mfug = rrddim_add(st_list_hits, "mfug", "mfu ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_mru = rrddim_add(st_list_hits, "mru", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_mrug = rrddim_add(st_list_hits, "mrug", "mru ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_list_hits); + + rrddim_set_by_pointer(st_list_hits, rd_mfu, arcstats.mfu_hits); + rrddim_set_by_pointer(st_list_hits, rd_mru, arcstats.mru_hits); + rrddim_set_by_pointer(st_list_hits, rd_mfug, arcstats.mfu_ghost_hits); + rrddim_set_by_pointer(st_list_hits, rd_mrug, arcstats.mru_ghost_hits); + rrdset_done(st_list_hits); + } +} + +void generate_charts_arc_summary(int update_every) { + unsigned long long arc_accesses_total = arcstats.hits + arcstats.misses; + unsigned long long real_hits = arcstats.mfu_hits + arcstats.mru_hits; + unsigned long long real_misses = arc_accesses_total - real_hits; + + //unsigned long long anon_hits = arcstats.hits - (arcstats.mfu_hits + arcstats.mru_hits + arcstats.mfu_ghost_hits + arcstats.mru_ghost_hits); + + unsigned long long arc_size = arcstats.size; + unsigned long long mru_size = arcstats.p; + //unsigned long long target_min_size = arcstats.c_min; + //unsigned long long target_max_size = arcstats.c_max; + unsigned long long target_size = arcstats.c; + //unsigned long long target_size_ratio = (target_max_size / target_min_size); + + unsigned long long mfu_size; + if(arc_size > target_size) + mfu_size = arc_size - mru_size; + else + mfu_size = target_size - mru_size; + + // -------------------------------------------------------------------- + + { + static RRDSET *st_arc_size_breakdown = NULL; + static RRDDIM *rd_most_recent = NULL; + static RRDDIM *rd_most_frequent = NULL; + + if (unlikely(!st_arc_size_breakdown)) { + st_arc_size_breakdown = rrdset_create_localhost( + "zfs" + , "arc_size_breakdown" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS ARC Size Breakdown" + , "percentage" + , 2020 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_most_recent = rrddim_add(st_arc_size_breakdown, "recent", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL); + rd_most_frequent = rrddim_add(st_arc_size_breakdown, "frequent", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL); + } + else + rrdset_next(st_arc_size_breakdown); + + rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_recent, mru_size); + rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_frequent, mfu_size); + rrdset_done(st_arc_size_breakdown); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_memory = NULL; +#ifndef __FreeBSD__ + static RRDDIM *rd_direct = NULL; +#endif + static RRDDIM *rd_throttled = NULL; +#ifndef __FreeBSD__ + static RRDDIM *rd_indirect = NULL; +#endif + + if (unlikely(!st_memory)) { + st_memory = rrdset_create_localhost( + "zfs" + , "memory_ops" + , NULL + , ZFS_FAMILY_OPERATIONS + , NULL + , "ZFS Memory Operations" + , "operations/s" + , 2023 + , update_every + , RRDSET_TYPE_LINE + ); + +#ifndef __FreeBSD__ + rd_direct = rrddim_add(st_memory, "direct", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#endif + rd_throttled = rrddim_add(st_memory, "throttled", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#ifndef __FreeBSD__ + rd_indirect = rrddim_add(st_memory, "indirect", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); +#endif + } + else + rrdset_next(st_memory); + +#ifndef __FreeBSD__ + rrddim_set_by_pointer(st_memory, rd_direct, arcstats.memory_direct_count); +#endif + rrddim_set_by_pointer(st_memory, rd_throttled, arcstats.memory_throttle_count); +#ifndef __FreeBSD__ + rrddim_set_by_pointer(st_memory, rd_indirect, arcstats.memory_indirect_count); +#endif + rrdset_done(st_memory); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_important_ops = NULL; + static RRDDIM *rd_deleted = NULL; + static RRDDIM *rd_mutex_misses = NULL; + static RRDDIM *rd_evict_skips = NULL; + static RRDDIM *rd_hash_collisions = NULL; + + if (unlikely(!st_important_ops)) { + st_important_ops = rrdset_create_localhost( + "zfs" + , "important_ops" + , NULL + , ZFS_FAMILY_OPERATIONS + , NULL + , "ZFS Important Operations" + , "operations/s" + , 2022 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_evict_skips = rrddim_add(st_important_ops, "eskip", "evict skip", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_deleted = rrddim_add(st_important_ops, "deleted", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_mutex_misses = rrddim_add(st_important_ops, "mtxmis", "mutex miss", 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_hash_collisions = rrddim_add(st_important_ops, "hash_collisions", "hash collisions", 1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_important_ops); + + rrddim_set_by_pointer(st_important_ops, rd_deleted, arcstats.deleted); + rrddim_set_by_pointer(st_important_ops, rd_evict_skips, arcstats.evict_skip); + rrddim_set_by_pointer(st_important_ops, rd_mutex_misses, arcstats.mutex_miss); + rrddim_set_by_pointer(st_important_ops, rd_hash_collisions, arcstats.hash_collisions); + rrdset_done(st_important_ops); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_actual_hits = NULL; + static RRDDIM *rd_actual_hits = NULL; + static RRDDIM *rd_actual_misses = NULL; + + if (unlikely(!st_actual_hits)) { + st_actual_hits = rrdset_create_localhost( + "zfs" + , "actual_hits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS Actual Cache Hits" + , "percentage" + , 2019 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_actual_hits = rrddim_add(st_actual_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_actual_misses = rrddim_add(st_actual_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_actual_hits); + + rrddim_set_by_pointer(st_actual_hits, rd_actual_hits, real_hits); + rrddim_set_by_pointer(st_actual_hits, rd_actual_misses, real_misses); + rrdset_done(st_actual_hits); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_demand_data_hits = NULL; + static RRDDIM *rd_demand_data_hits = NULL; + static RRDDIM *rd_demand_data_misses = NULL; + + if (unlikely(!st_demand_data_hits)) { + st_demand_data_hits = rrdset_create_localhost( + "zfs" + , "demand_data_hits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS Data Demand Efficiency" + , "percentage" + , 2031 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_demand_data_hits = rrddim_add(st_demand_data_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_demand_data_misses = rrddim_add(st_demand_data_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_demand_data_hits); + + rrddim_set_by_pointer(st_demand_data_hits, rd_demand_data_hits, arcstats.demand_data_hits); + rrddim_set_by_pointer(st_demand_data_hits, rd_demand_data_misses, arcstats.demand_data_misses); + rrdset_done(st_demand_data_hits); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_prefetch_data_hits = NULL; + static RRDDIM *rd_prefetch_data_hits = NULL; + static RRDDIM *rd_prefetch_data_misses = NULL; + + if (unlikely(!st_prefetch_data_hits)) { + st_prefetch_data_hits = rrdset_create_localhost( + "zfs" + , "prefetch_data_hits" + , NULL + , ZFS_FAMILY_EFFICIENCY + , NULL + , "ZFS Data Prefetch Efficiency" + , "percentage" + , 2032 + , update_every + , RRDSET_TYPE_STACKED + ); + + rd_prefetch_data_hits = rrddim_add(st_prefetch_data_hits, "hits", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + rd_prefetch_data_misses = rrddim_add(st_prefetch_data_hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL); + } + else + rrdset_next(st_prefetch_data_hits); + + rrddim_set_by_pointer(st_prefetch_data_hits, rd_prefetch_data_hits, arcstats.prefetch_data_hits); + rrddim_set_by_pointer(st_prefetch_data_hits, rd_prefetch_data_misses, arcstats.prefetch_data_misses); + rrdset_done(st_prefetch_data_hits); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_hash_elements = NULL; + static RRDDIM *rd_hash_elements_current = NULL; + static RRDDIM *rd_hash_elements_max = NULL; + + if (unlikely(!st_hash_elements)) { + st_hash_elements = rrdset_create_localhost( + "zfs" + , "hash_elements" + , NULL + , ZFS_FAMILY_HASH + , NULL + , "ZFS ARC Hash Elements" + , "elements" + , 2300 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_hash_elements_current = rrddim_add(st_hash_elements, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_hash_elements_max = rrddim_add(st_hash_elements, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_hash_elements); + + rrddim_set_by_pointer(st_hash_elements, rd_hash_elements_current, arcstats.hash_elements); + rrddim_set_by_pointer(st_hash_elements, rd_hash_elements_max, arcstats.hash_elements_max); + rrdset_done(st_hash_elements); + } + + // -------------------------------------------------------------------- + + { + static RRDSET *st_hash_chains = NULL; + static RRDDIM *rd_hash_chains_current = NULL; + static RRDDIM *rd_hash_chains_max = NULL; + + if (unlikely(!st_hash_chains)) { + st_hash_chains = rrdset_create_localhost( + "zfs" + , "hash_chains" + , NULL + , ZFS_FAMILY_HASH + , NULL + , "ZFS ARC Hash Chains" + , "chains" + , 2310 + , update_every + , RRDSET_TYPE_LINE + ); + + rd_hash_chains_current = rrddim_add(st_hash_chains, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_hash_chains_max = rrddim_add(st_hash_chains, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_hash_chains); + + rrddim_set_by_pointer(st_hash_chains, rd_hash_chains_current, arcstats.hash_chains); + rrddim_set_by_pointer(st_hash_chains, rd_hash_chains_max, arcstats.hash_chain_max); + rrdset_done(st_hash_chains); + } + + // -------------------------------------------------------------------- + +}
\ No newline at end of file diff --git a/src/zfs_common.h b/src/zfs_common.h new file mode 100644 index 000000000..9d3aa7dfb --- /dev/null +++ b/src/zfs_common.h @@ -0,0 +1,109 @@ +#ifndef NETDATA_ZFS_COMMON_H +#define NETDATA_ZFS_COMMON_H + +#define ZFS_FAMILY_SIZE "size" +#define ZFS_FAMILY_EFFICIENCY "efficiency" +#define ZFS_FAMILY_ACCESSES "accesses" +#define ZFS_FAMILY_OPERATIONS "operations" +#define ZFS_FAMILY_HASH "hashes" + +struct arcstats { + unsigned long long hits; + unsigned long long misses; + unsigned long long demand_data_hits; + unsigned long long demand_data_misses; + unsigned long long demand_metadata_hits; + unsigned long long demand_metadata_misses; + unsigned long long prefetch_data_hits; + unsigned long long prefetch_data_misses; + unsigned long long prefetch_metadata_hits; + unsigned long long prefetch_metadata_misses; + unsigned long long mru_hits; + unsigned long long mru_ghost_hits; + unsigned long long mfu_hits; + unsigned long long mfu_ghost_hits; + unsigned long long deleted; + unsigned long long mutex_miss; + unsigned long long evict_skip; + unsigned long long evict_not_enough; + unsigned long long evict_l2_cached; + unsigned long long evict_l2_eligible; + unsigned long long evict_l2_ineligible; + unsigned long long evict_l2_skip; + unsigned long long hash_elements; + unsigned long long hash_elements_max; + unsigned long long hash_collisions; + unsigned long long hash_chains; + unsigned long long hash_chain_max; + unsigned long long p; + unsigned long long c; + unsigned long long c_min; + unsigned long long c_max; + unsigned long long size; + unsigned long long hdr_size; + unsigned long long data_size; + unsigned long long metadata_size; + unsigned long long other_size; + unsigned long long anon_size; + unsigned long long anon_evictable_data; + unsigned long long anon_evictable_metadata; + unsigned long long mru_size; + unsigned long long mru_evictable_data; + unsigned long long mru_evictable_metadata; + unsigned long long mru_ghost_size; + unsigned long long mru_ghost_evictable_data; + unsigned long long mru_ghost_evictable_metadata; + unsigned long long mfu_size; + unsigned long long mfu_evictable_data; + unsigned long long mfu_evictable_metadata; + unsigned long long mfu_ghost_size; + unsigned long long mfu_ghost_evictable_data; + unsigned long long mfu_ghost_evictable_metadata; + unsigned long long l2_hits; + unsigned long long l2_misses; + unsigned long long l2_feeds; + unsigned long long l2_rw_clash; + unsigned long long l2_read_bytes; + unsigned long long l2_write_bytes; + unsigned long long l2_writes_sent; + unsigned long long l2_writes_done; + unsigned long long l2_writes_error; + unsigned long long l2_writes_lock_retry; + unsigned long long l2_evict_lock_retry; + unsigned long long l2_evict_reading; + unsigned long long l2_evict_l1cached; + unsigned long long l2_free_on_write; + unsigned long long l2_cdata_free_on_write; + unsigned long long l2_abort_lowmem; + unsigned long long l2_cksum_bad; + unsigned long long l2_io_error; + unsigned long long l2_size; + unsigned long long l2_asize; + unsigned long long l2_hdr_size; + unsigned long long l2_compress_successes; + unsigned long long l2_compress_zeros; + unsigned long long l2_compress_failures; + unsigned long long memory_throttle_count; + unsigned long long duplicate_buffers; + unsigned long long duplicate_buffers_size; + unsigned long long duplicate_reads; + unsigned long long memory_direct_count; + unsigned long long memory_indirect_count; + unsigned long long arc_no_grow; + unsigned long long arc_tempreserve; + unsigned long long arc_loaned_bytes; + unsigned long long arc_prune; + unsigned long long arc_meta_used; + unsigned long long arc_meta_limit; + unsigned long long arc_meta_max; + unsigned long long arc_meta_min; + unsigned long long arc_need_free; + unsigned long long arc_sys_free; +}; + +int l2exist; + +void generate_charts_arcstats(int update_every); +void generate_charts_arc_summary(int update_every); + +#endif //NETDATA_ZFS_COMMON_H |