diff options
Diffstat (limited to 'src')
101 files changed, 15014 insertions, 6476 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 86b9a9fe4..d8274d255 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -24,26 +24,75 @@ dist_cache_DATA = .keep dist_varlib_DATA = .keep dist_registry_DATA = .keep dist_log_DATA = .keep +if !MACOS plugins_PROGRAMS = apps.plugin +endif netdata_SOURCES = \ appconfig.c appconfig.h \ + adaptive_resortable_list.c adaptive_resortable_list.h \ avl.c avl.h \ + backends.c backends.h \ + clocks.c clocks.h \ common.c common.h \ daemon.c daemon.h \ dictionary.c dictionary.h \ eval.c eval.h \ global_statistics.c global_statistics.h \ health.c health.h \ + inlined.h \ log.c log.h \ main.c main.h \ plugin_checks.c plugin_checks.h \ plugin_idlejitter.c plugin_idlejitter.h \ plugin_nfacct.c plugin_nfacct.h \ - plugin_proc.c plugin_proc.h \ plugin_tc.c plugin_tc.h \ plugins_d.c plugins_d.h \ popen.c popen.h \ + socket.c socket.h \ + simple_pattern.c simple_pattern.h \ + sys_fs_cgroup.c \ + sys_devices_system_edac_mc.c \ + sys_devices_system_node.c \ + procfile.c procfile.h \ + proc_self_mountinfo.c proc_self_mountinfo.h \ + registry.c registry.h \ + registry_internals.c registry_internals.h \ + registry_url.c registry_url.h \ + registry_person.c registry_person.h \ + registry_machine.c registry_machine.h \ + registry_init.c \ + registry_db.c \ + registry_log.c \ + rrd.c rrd.h \ + rrd2json.c rrd2json.h \ + storage_number.c storage_number.h \ + unit_test.c unit_test.h \ + url.c url.h \ + web_buffer.c web_buffer.h \ + web_buffer_svg.c web_buffer_svg.h \ + web_client.c web_client.h \ + web_server.c web_server.h \ + $(NULL) + +if FREEBSD +netdata_SOURCES += \ + plugin_freebsd.c plugin_freebsd.h \ + freebsd_sysctl.c \ + $(NULL) +else +if MACOS +netdata_SOURCES += \ + plugin_macos.c plugin_macos.h \ + macos_sysctl.c \ + macos_mach_smi.c \ + macos_fw.c \ + $(NULL) +else +netdata_SOURCES += \ + ipc.c ipc.h \ + plugin_proc.c plugin_proc.h \ + plugin_proc_diskspace.c plugin_proc_diskspace.h \ proc_diskstats.c \ proc_interrupts.c \ proc_softirqs.c \ @@ -60,23 +109,14 @@ netdata_SOURCES = \ proc_net_stat_conntrack.c \ proc_net_stat_synproxy.c \ proc_stat.c \ - proc_self_mountinfo.c proc_self_mountinfo.h \ proc_sys_kernel_random_entropy_avail.c \ proc_vmstat.c \ + proc_uptime.c \ sys_kernel_mm_ksm.c \ - sys_fs_cgroup.c \ - procfile.c procfile.h \ - registry.c registry.h \ - rrd.c rrd.h \ - rrd2json.c rrd2json.h \ - storage_number.c storage_number.h \ - unit_test.c unit_test.h \ - url.c url.h \ - web_buffer.c web_buffer.h \ - web_buffer_svg.c web_buffer_svg.h \ - web_client.c web_client.h \ - web_server.c web_server.h \ $(NULL) +endif +endif + netdata_LDADD = \ $(OPTIONAL_MATH_LIBS) \ $(OPTIONAL_NFACCT_LIBS) \ @@ -87,7 +127,9 @@ netdata_LDADD = \ apps_plugin_SOURCES = \ apps_plugin.c \ avl.c avl.h \ + clocks.c clocks.h \ common.c common.h \ + inlined.h \ log.c log.h \ procfile.c procfile.h \ web_buffer.c web_buffer.h \ diff --git a/src/Makefile.in b/src/Makefile.in index 6645f40de..f94646363 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -1,7 +1,7 @@ -# Makefile.in generated by automake 1.14.1 from Makefile.am. +# Makefile.in generated by automake 1.15 from Makefile.am. # @configure_input@ -# Copyright (C) 1994-2013 Free Software Foundation, Inc. +# Copyright (C) 1994-2014 Free Software Foundation, Inc. # This Makefile.in is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -16,7 +16,17 @@ VPATH = @srcdir@ -am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)' +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} am__make_running_with_option = \ case $${target_option-} in \ ?) ;; \ @@ -80,21 +90,60 @@ POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ sbin_PROGRAMS = netdata$(EXEEXT) -plugins_PROGRAMS = apps.plugin$(EXEEXT) +@MACOS_FALSE@plugins_PROGRAMS = apps.plugin$(EXEEXT) +@FREEBSD_TRUE@am__append_1 = \ +@FREEBSD_TRUE@ plugin_freebsd.c plugin_freebsd.h \ +@FREEBSD_TRUE@ freebsd_sysctl.c \ +@FREEBSD_TRUE@ $(NULL) + +@FREEBSD_FALSE@@MACOS_TRUE@am__append_2 = \ +@FREEBSD_FALSE@@MACOS_TRUE@ plugin_macos.c plugin_macos.h \ +@FREEBSD_FALSE@@MACOS_TRUE@ macos_sysctl.c \ +@FREEBSD_FALSE@@MACOS_TRUE@ macos_mach_smi.c \ +@FREEBSD_FALSE@@MACOS_TRUE@ macos_fw.c \ +@FREEBSD_FALSE@@MACOS_TRUE@ $(NULL) + +@FREEBSD_FALSE@@MACOS_FALSE@am__append_3 = \ +@FREEBSD_FALSE@@MACOS_FALSE@ ipc.c ipc.h \ +@FREEBSD_FALSE@@MACOS_FALSE@ plugin_proc.c plugin_proc.h \ +@FREEBSD_FALSE@@MACOS_FALSE@ plugin_proc_diskspace.c plugin_proc_diskspace.h \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_diskstats.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_interrupts.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_softirqs.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_loadavg.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_meminfo.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_dev.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_ip_vs_stats.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_netstat.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_rpc_nfs.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_rpc_nfsd.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp6.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_stat.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_sys_kernel_random_entropy_avail.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_vmstat.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_uptime.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ sys_kernel_mm_ksm.c \ +@FREEBSD_FALSE@@MACOS_FALSE@ $(NULL) + subdir = src -DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ - $(top_srcdir)/depcomp $(dist_cache_DATA) $(dist_log_DATA) \ - $(dist_registry_DATA) $(dist_varlib_DATA) ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/ax_c___atomic.m4 \ $(top_srcdir)/m4/ax_c__generic.m4 \ $(top_srcdir)/m4/ax_c_mallinfo.m4 \ $(top_srcdir)/m4/ax_c_mallopt.m4 \ $(top_srcdir)/m4/ax_check_compile_flag.m4 \ + $(top_srcdir)/m4/ax_gcc_func_attribute.m4 \ $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/jemalloc.m4 \ $(top_srcdir)/m4/tcmalloc.m4 $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(dist_cache_DATA) \ + $(dist_log_DATA) $(dist_registry_DATA) $(dist_varlib_DATA) \ + $(am__DIST_COMMON) mkinstalldirs = $(install_sh) -d CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = @@ -104,33 +153,91 @@ am__installdirs = "$(DESTDIR)$(pluginsdir)" "$(DESTDIR)$(sbindir)" \ "$(DESTDIR)$(registrydir)" "$(DESTDIR)$(varlibdir)" PROGRAMS = $(plugins_PROGRAMS) $(sbin_PROGRAMS) am_apps_plugin_OBJECTS = apps_plugin.$(OBJEXT) avl.$(OBJEXT) \ - common.$(OBJEXT) log.$(OBJEXT) procfile.$(OBJEXT) \ - web_buffer.$(OBJEXT) + clocks.$(OBJEXT) common.$(OBJEXT) log.$(OBJEXT) \ + procfile.$(OBJEXT) web_buffer.$(OBJEXT) apps_plugin_OBJECTS = $(am_apps_plugin_OBJECTS) apps_plugin_LDADD = $(LDADD) -am_netdata_OBJECTS = appconfig.$(OBJEXT) avl.$(OBJEXT) \ - common.$(OBJEXT) daemon.$(OBJEXT) dictionary.$(OBJEXT) \ - eval.$(OBJEXT) global_statistics.$(OBJEXT) health.$(OBJEXT) \ - log.$(OBJEXT) main.$(OBJEXT) plugin_checks.$(OBJEXT) \ +am__netdata_SOURCES_DIST = appconfig.c appconfig.h \ + adaptive_resortable_list.c adaptive_resortable_list.h avl.c \ + avl.h backends.c backends.h clocks.c clocks.h common.c \ + common.h daemon.c daemon.h dictionary.c dictionary.h eval.c \ + eval.h global_statistics.c global_statistics.h health.c \ + health.h inlined.h log.c log.h main.c main.h plugin_checks.c \ + plugin_checks.h plugin_idlejitter.c plugin_idlejitter.h \ + plugin_nfacct.c plugin_nfacct.h plugin_tc.c plugin_tc.h \ + plugins_d.c plugins_d.h popen.c popen.h socket.c socket.h \ + simple_pattern.c simple_pattern.h sys_fs_cgroup.c \ + sys_devices_system_edac_mc.c sys_devices_system_node.c \ + procfile.c procfile.h proc_self_mountinfo.c \ + proc_self_mountinfo.h registry.c registry.h \ + registry_internals.c registry_internals.h registry_url.c \ + registry_url.h registry_person.c registry_person.h \ + registry_machine.c registry_machine.h registry_init.c \ + registry_db.c registry_log.c rrd.c rrd.h rrd2json.c rrd2json.h \ + storage_number.c storage_number.h unit_test.c unit_test.h \ + url.c url.h web_buffer.c web_buffer.h web_buffer_svg.c \ + web_buffer_svg.h web_client.c web_client.h web_server.c \ + web_server.h plugin_freebsd.c plugin_freebsd.h \ + freebsd_sysctl.c plugin_macos.c plugin_macos.h macos_sysctl.c \ + macos_mach_smi.c macos_fw.c ipc.c ipc.h plugin_proc.c \ + plugin_proc.h plugin_proc_diskspace.c plugin_proc_diskspace.h \ + proc_diskstats.c proc_interrupts.c proc_softirqs.c \ + proc_loadavg.c proc_meminfo.c proc_net_dev.c \ + proc_net_ip_vs_stats.c proc_net_netstat.c proc_net_rpc_nfs.c \ + proc_net_rpc_nfsd.c proc_net_snmp.c proc_net_snmp6.c \ + proc_net_softnet_stat.c proc_net_stat_conntrack.c \ + proc_net_stat_synproxy.c proc_stat.c \ + proc_sys_kernel_random_entropy_avail.c proc_vmstat.c \ + proc_uptime.c sys_kernel_mm_ksm.c +@FREEBSD_TRUE@am__objects_1 = plugin_freebsd.$(OBJEXT) \ +@FREEBSD_TRUE@ freebsd_sysctl.$(OBJEXT) +@FREEBSD_FALSE@@MACOS_TRUE@am__objects_2 = plugin_macos.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_TRUE@ macos_sysctl.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_TRUE@ macos_mach_smi.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_TRUE@ macos_fw.$(OBJEXT) +@FREEBSD_FALSE@@MACOS_FALSE@am__objects_3 = ipc.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ plugin_proc.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ plugin_proc_diskspace.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_diskstats.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_interrupts.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_softirqs.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_loadavg.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_meminfo.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_dev.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_ip_vs_stats.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_netstat.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_rpc_nfs.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_rpc_nfsd.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_snmp6.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_softnet_stat.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_conntrack.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_net_stat_synproxy.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_stat.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_sys_kernel_random_entropy_avail.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_vmstat.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ proc_uptime.$(OBJEXT) \ +@FREEBSD_FALSE@@MACOS_FALSE@ sys_kernel_mm_ksm.$(OBJEXT) +am_netdata_OBJECTS = appconfig.$(OBJEXT) \ + adaptive_resortable_list.$(OBJEXT) avl.$(OBJEXT) \ + backends.$(OBJEXT) clocks.$(OBJEXT) common.$(OBJEXT) \ + daemon.$(OBJEXT) dictionary.$(OBJEXT) eval.$(OBJEXT) \ + global_statistics.$(OBJEXT) health.$(OBJEXT) log.$(OBJEXT) \ + main.$(OBJEXT) plugin_checks.$(OBJEXT) \ plugin_idlejitter.$(OBJEXT) plugin_nfacct.$(OBJEXT) \ - plugin_proc.$(OBJEXT) plugin_tc.$(OBJEXT) plugins_d.$(OBJEXT) \ - popen.$(OBJEXT) proc_diskstats.$(OBJEXT) \ - proc_interrupts.$(OBJEXT) proc_softirqs.$(OBJEXT) \ - proc_loadavg.$(OBJEXT) proc_meminfo.$(OBJEXT) \ - proc_net_dev.$(OBJEXT) proc_net_ip_vs_stats.$(OBJEXT) \ - proc_net_netstat.$(OBJEXT) proc_net_rpc_nfs.$(OBJEXT) \ - proc_net_rpc_nfsd.$(OBJEXT) proc_net_snmp.$(OBJEXT) \ - proc_net_snmp6.$(OBJEXT) proc_net_softnet_stat.$(OBJEXT) \ - proc_net_stat_conntrack.$(OBJEXT) \ - proc_net_stat_synproxy.$(OBJEXT) proc_stat.$(OBJEXT) \ - proc_self_mountinfo.$(OBJEXT) \ - proc_sys_kernel_random_entropy_avail.$(OBJEXT) \ - proc_vmstat.$(OBJEXT) sys_kernel_mm_ksm.$(OBJEXT) \ - sys_fs_cgroup.$(OBJEXT) procfile.$(OBJEXT) registry.$(OBJEXT) \ - rrd.$(OBJEXT) rrd2json.$(OBJEXT) storage_number.$(OBJEXT) \ - unit_test.$(OBJEXT) url.$(OBJEXT) web_buffer.$(OBJEXT) \ - web_buffer_svg.$(OBJEXT) web_client.$(OBJEXT) \ - web_server.$(OBJEXT) + plugin_tc.$(OBJEXT) plugins_d.$(OBJEXT) popen.$(OBJEXT) \ + socket.$(OBJEXT) simple_pattern.$(OBJEXT) \ + sys_fs_cgroup.$(OBJEXT) sys_devices_system_edac_mc.$(OBJEXT) \ + sys_devices_system_node.$(OBJEXT) procfile.$(OBJEXT) \ + proc_self_mountinfo.$(OBJEXT) registry.$(OBJEXT) \ + registry_internals.$(OBJEXT) registry_url.$(OBJEXT) \ + registry_person.$(OBJEXT) registry_machine.$(OBJEXT) \ + registry_init.$(OBJEXT) registry_db.$(OBJEXT) \ + registry_log.$(OBJEXT) rrd.$(OBJEXT) rrd2json.$(OBJEXT) \ + storage_number.$(OBJEXT) unit_test.$(OBJEXT) url.$(OBJEXT) \ + web_buffer.$(OBJEXT) web_buffer_svg.$(OBJEXT) \ + web_client.$(OBJEXT) web_server.$(OBJEXT) $(am__objects_1) \ + $(am__objects_2) $(am__objects_3) netdata_OBJECTS = $(am_netdata_OBJECTS) am__DEPENDENCIES_1 = netdata_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ @@ -164,7 +271,7 @@ am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) am__v_CCLD_0 = @echo " CCLD " $@; am__v_CCLD_1 = SOURCES = $(apps_plugin_SOURCES) $(netdata_SOURCES) -DIST_SOURCES = $(apps_plugin_SOURCES) $(netdata_SOURCES) +DIST_SOURCES = $(apps_plugin_SOURCES) $(am__netdata_SOURCES_DIST) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ n|no|NO) false;; \ @@ -218,6 +325,7 @@ am__define_uniq_tagged_files = \ done | $(am__uniquify_input)` ETAGS = etags CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) ACLOCAL = @ACLOCAL@ AMTAR = @AMTAR@ @@ -381,58 +489,27 @@ dist_cache_DATA = .keep dist_varlib_DATA = .keep dist_registry_DATA = .keep dist_log_DATA = .keep -netdata_SOURCES = \ - appconfig.c appconfig.h \ - avl.c avl.h \ - common.c common.h \ - daemon.c daemon.h \ - dictionary.c dictionary.h \ - eval.c eval.h \ - global_statistics.c global_statistics.h \ - health.c health.h \ - log.c log.h \ - main.c main.h \ - plugin_checks.c plugin_checks.h \ - plugin_idlejitter.c plugin_idlejitter.h \ - plugin_nfacct.c plugin_nfacct.h \ - plugin_proc.c plugin_proc.h \ - plugin_tc.c plugin_tc.h \ - plugins_d.c plugins_d.h \ - popen.c popen.h \ - proc_diskstats.c \ - proc_interrupts.c \ - proc_softirqs.c \ - proc_loadavg.c \ - proc_meminfo.c \ - proc_net_dev.c \ - proc_net_ip_vs_stats.c \ - proc_net_netstat.c \ - proc_net_rpc_nfs.c \ - proc_net_rpc_nfsd.c \ - proc_net_snmp.c \ - proc_net_snmp6.c \ - proc_net_softnet_stat.c \ - proc_net_stat_conntrack.c \ - proc_net_stat_synproxy.c \ - proc_stat.c \ - proc_self_mountinfo.c proc_self_mountinfo.h \ - proc_sys_kernel_random_entropy_avail.c \ - proc_vmstat.c \ - sys_kernel_mm_ksm.c \ - sys_fs_cgroup.c \ - procfile.c procfile.h \ - registry.c registry.h \ - rrd.c rrd.h \ - rrd2json.c rrd2json.h \ - storage_number.c storage_number.h \ - unit_test.c unit_test.h \ - url.c url.h \ - web_buffer.c web_buffer.h \ - web_buffer_svg.c web_buffer_svg.h \ - web_client.c web_client.h \ - web_server.c web_server.h \ - $(NULL) - +netdata_SOURCES = appconfig.c appconfig.h adaptive_resortable_list.c \ + adaptive_resortable_list.h avl.c avl.h backends.c backends.h \ + clocks.c clocks.h common.c common.h daemon.c daemon.h \ + dictionary.c dictionary.h eval.c eval.h global_statistics.c \ + global_statistics.h health.c health.h inlined.h log.c log.h \ + main.c main.h plugin_checks.c plugin_checks.h \ + plugin_idlejitter.c plugin_idlejitter.h plugin_nfacct.c \ + plugin_nfacct.h plugin_tc.c plugin_tc.h plugins_d.c \ + plugins_d.h popen.c popen.h socket.c socket.h simple_pattern.c \ + simple_pattern.h sys_fs_cgroup.c sys_devices_system_edac_mc.c \ + sys_devices_system_node.c procfile.c procfile.h \ + proc_self_mountinfo.c proc_self_mountinfo.h registry.c \ + registry.h registry_internals.c registry_internals.h \ + registry_url.c registry_url.h registry_person.c \ + registry_person.h registry_machine.c registry_machine.h \ + registry_init.c registry_db.c registry_log.c rrd.c rrd.h \ + rrd2json.c rrd2json.h storage_number.c storage_number.h \ + unit_test.c unit_test.h url.c url.h web_buffer.c web_buffer.h \ + web_buffer_svg.c web_buffer_svg.h web_client.c web_client.h \ + web_server.c web_server.h $(NULL) $(am__append_1) \ + $(am__append_2) $(am__append_3) netdata_LDADD = \ $(OPTIONAL_MATH_LIBS) \ $(OPTIONAL_NFACCT_LIBS) \ @@ -443,7 +520,9 @@ netdata_LDADD = \ apps_plugin_SOURCES = \ apps_plugin.c \ avl.c avl.h \ + clocks.c clocks.h \ common.c common.h \ + inlined.h \ log.c log.h \ procfile.c procfile.h \ web_buffer.c web_buffer.h \ @@ -465,7 +544,6 @@ $(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__confi echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu src/Makefile'; \ $(am__cd) $(top_srcdir) && \ $(AUTOMAKE) --gnu src/Makefile -.PRECIOUS: Makefile Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status @case '$?' in \ *config.status*) \ @@ -582,21 +660,32 @@ mostlyclean-compile: distclean-compile: -rm -f *.tab.c +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/adaptive_resortable_list.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/appconfig.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/apps_plugin.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/avl.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/backends.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/clocks.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/daemon.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dictionary.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/eval.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/freebsd_sysctl.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/global_statistics.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/health.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ipc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/macos_fw.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/macos_mach_smi.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/macos_sysctl.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_checks.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_freebsd.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_idlejitter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_macos.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_nfacct.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_proc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_proc_diskspace.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_tc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugins_d.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/popen.Po@am__quote@ @@ -618,12 +707,24 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_softirqs.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_stat.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_sys_kernel_random_entropy_avail.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_uptime.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proc_vmstat.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/procfile.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry_db.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry_init.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry_internals.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry_log.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry_machine.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry_person.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/registry_url.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rrd.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/rrd2json.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/simple_pattern.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/socket.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/storage_number.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_devices_system_edac_mc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_devices_system_node.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_fs_cgroup.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sys_kernel_mm_ksm.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unit_test.Po@am__quote@ @@ -946,6 +1047,8 @@ uninstall-am: uninstall-dist_cacheDATA uninstall-dist_logDATA \ uninstall-dist_registryDATA uninstall-dist_varlibDATA \ uninstall-pluginsPROGRAMS uninstall-sbinPROGRAMS +.PRECIOUS: Makefile + install-data-hook: if [ `id -u` == 0 ]; then \ diff --git a/src/adaptive_resortable_list.c b/src/adaptive_resortable_list.c new file mode 100644 index 000000000..a37c396fa --- /dev/null +++ b/src/adaptive_resortable_list.c @@ -0,0 +1,222 @@ +#include "adaptive_resortable_list.h" + +// the default processor() of the ARL +// can be overwritten at arl_create() +static inline void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst) { + (void)name; + (void)hash; + + register unsigned long long *d = dst; + *d = str2ull(value); + // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, *d); +} + +// create a new ARL +ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks) { + ARL_BASE *base = callocz(1, sizeof(ARL_BASE)); + + base->name = strdupz(name); + + if(!processor) + base->processor = arl_callback_str2ull; + else + base->processor = processor; + + base->rechecks = rechecks; + + return base; +} + +void arl_free(ARL_BASE *arl_base) { + if(unlikely(!arl_base)) + return; + + while(arl_base->head) { + ARL_ENTRY *e = arl_base->head; + arl_base->head = e->next; + + freez(e->name); +#ifdef NETDATA_INTERNAL_CHECKS + memset(e, 0, sizeof(ARL_ENTRY)); +#endif + freez(e); + } + + freez(arl_base->name); + +#ifdef NETDATA_INTERNAL_CHECKS + memset(arl_base, 0, sizeof(ARL_BASE)); +#endif + + freez(arl_base); +} + +void arl_begin(ARL_BASE *base) { + ARL_ENTRY *e; + +#ifdef NETDATA_INTERNAL_CHECKS + if(likely(base->iteration > 10)) { + // do these checks after the ARL has been sorted + + if(unlikely(base->relinkings > (base->expected + base->allocated))) + info("ARL '%s' has %zu relinkings with %zu expected and %zu allocated entries. Is the source changing so fast?" + , base->name, base->relinkings, base->expected, base->allocated); + + if(unlikely(base->slow > base->fast)) + info("ARL '%s' has %zu fast searches and %zu slow searches. Is the source really changing so fast?" + , base->name, base->fast, base->slow); + + if(unlikely(base->iteration % 60 == 0)) { + info("ARL '%s' statistics: iteration %zu, expected %zu, wanted %zu, allocated %zu, fred %zu, relinkings %zu, found %zu, added %zu, fast %zu, slow %zu" + , base->name + , base->iteration + , base->expected + , base->wanted + , base->allocated + , base->fred + , base->relinkings + , base->found + , base->added + , base->fast + , base->slow + ); + // for(e = base->head; e; e = e->next) fprintf(stderr, "%s ", e->name); + // fprintf(stderr, "\n"); + } + } +#endif + + if(unlikely(base->added || base->iteration % base->rechecks) == 1) { + base->added = 0; + base->wanted = 0; + for(e = base->head; e ; e = e->next) { + if(e->flags & ARL_ENTRY_FLAG_FOUND) { + + // remove the found flag + e->flags &= ~ARL_ENTRY_FLAG_FOUND; + + // count it in wanted + if(e->flags & ARL_ENTRY_FLAG_EXPECTED) + base->wanted++; + } + else if(e->flags & ARL_ENTRY_FLAG_DYNAMIC) { + // we can remove this entry + // it is not found, and it was created because + // it was found in the source file + if(e->next) e->next->prev = e->prev; + if(e->prev) e->prev->next = e->next; + if(base->head == e) base->head = e->next; + freez(e->name); + freez(e); + + base->fred++; + } + } + } + + base->iteration++; + base->next_keyword = base->head; + base->found = 0; +} + +// register an expected keyword to the ARL +// together with its destination ( i.e. the output of the processor() ) +ARL_ENTRY *arl_expect(ARL_BASE *base, const char *keyword, void *dst) { + ARL_ENTRY *e = callocz(1, sizeof(ARL_ENTRY)); + e->name = strdupz(keyword); + e->hash = simple_hash(e->name); + e->dst = dst; + e->flags = ARL_ENTRY_FLAG_EXPECTED; + e->prev = NULL; + e->next = base->head; + + if(base->head) base->head->prev = e; + else base->next_keyword = e; + + base->head = e; + base->expected++; + base->allocated++; + + base->wanted = base->expected; + + return e; +} + +int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const char *value) { + ARL_ENTRY *e; + + uint32_t hash = simple_hash(s); + + // find if it already exists in the data + for(e = base->head; e ; e = e->next) + if(e->hash == hash && !strcmp(e->name, s)) + break; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(e == base->next_keyword)) + fatal("Internal Error: e == base->last"); +#endif + + if(e) { + // found it in the keywords + + base->relinkings++; + + // run the processor for it + if(unlikely(e->dst)) { + base->processor(e->name, hash, value, e->dst); + base->found++; + } + + // unlink it - we will relink it below + if(e->next) e->next->prev = e->prev; + if(e->prev) e->prev->next = e->next; + + // make sure the head is properly linked + if(base->head == e) + base->head = e->next; + } + else { + // not found + + // create it + e = callocz(1, sizeof(ARL_ENTRY)); + e->name = strdupz(s); + e->hash = hash; + e->flags = ARL_ENTRY_FLAG_DYNAMIC; + + base->allocated++; + base->added++; + } + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(base->iteration % 60 == 0 && e->flags & ARL_ENTRY_FLAG_FOUND)) + info("ARL '%s': entry '%s' is already found. Did you forget to call arl_begin()?", base->name, s); +#endif + + e->flags |= ARL_ENTRY_FLAG_FOUND; + + // link it here + e->next = base->next_keyword; + if(base->next_keyword) { + e->prev = base->next_keyword->prev; + base->next_keyword->prev = e; + + if(e->prev) + e->prev->next = e; + + if(base->head == base->next_keyword) + base->head = e; + } + else + e->prev = NULL; + + base->next_keyword = e->next; + if(unlikely(!base->next_keyword)) + base->next_keyword = base->head; + + if(unlikely(base->found == base->wanted)) + return 1; + + return 0; +} diff --git a/src/adaptive_resortable_list.h b/src/adaptive_resortable_list.h new file mode 100644 index 000000000..609cd0c43 --- /dev/null +++ b/src/adaptive_resortable_list.h @@ -0,0 +1,162 @@ +#ifndef NETDATA_ADAPTIVE_RESORTABLE_LIST_H +#define NETDATA_ADAPTIVE_RESORTABLE_LIST_H + +/* + * ADAPTIVE RE-SORTABLE LIST + * This structure allows netdata to read a file of NAME VALUE lines + * in the fastest possible way. + * + * It maintains a linked list of all NAME (keywords), sorted in the + * same order as found in the source data file. + * The linked list is kept sorted at all times - the source file + * may change at any time, the list will adapt. + * + * The caller: + * + * 1. calls arl_create() to create a list + * + * 2. calls arl_expect() to register the expected keyword + * + * Then: + * + * 3. calls arl_begin() to initiate a data collection iteration. + * This is to be called just ONCE every time the source is re-scanned. + * + * 4. calls arl_check() for each line read from the file. + * + * Finally: + * + * 5. calls arl_free() to destroy this and free all memory. + * + * The program will call the processor() function, given to + * arl_create(), for each expected keyword found. + * The default processor() expects dst to be an unsigned long long *. + * + * LIMITATIONS + * DO NOT USE THIS IF THE A NAME/KEYWORD MAY APPEAR MORE THAN + * ONCE IN THE SOURCE DATA SET. + */ + +#include "common.h" + +#define ARL_ENTRY_FLAG_FOUND 0x01 // the entry has been found in the source data +#define ARL_ENTRY_FLAG_EXPECTED 0x02 // the entry is expected by the program +#define ARL_ENTRY_FLAG_DYNAMIC 0x04 // the entry was dynamically allocated, from source data + +typedef struct arl_entry { + char *name; // the keywords + uint32_t hash; // the hash of the keyword + + void *dst; // the dst to pass to the processor + + uint8_t flags; // ARL_ENTRY_FLAG_* + + // double linked list for fast re-linkings + struct arl_entry *prev, *next; +} ARL_ENTRY; + +typedef struct arl_base { + char *name; + + size_t iteration; // incremented on each iteration (arl_begin()) + size_t found; // the number of expected keywords found in this iteration + size_t expected; // the number of expected keywords + size_t wanted; // the number of wanted keywords + // i.e. the number of keywords found and expected + + size_t relinkings; // the number of relinkings we have made so far + + size_t allocated; // the number of keywords allocated + size_t fred; // the number of keywords cleaned up + + size_t rechecks; // the number of iterations between re-checks of the + // wanted number of keywords + // this is only needed in cases where the source + // is having less lines over time. + + size_t added; // it is non-zero if new keywords have been added + // this is only needed to detect new lines have + // been added to the file, over time. + +#ifdef NETDATA_INTERNAL_CHECKS + size_t fast; // the number of times we have taken the fast path + size_t slow; // the number of times we have taken the slow path +#endif + + // the processor to do the job + void (*processor)(const char *name, uint32_t hash, const char *value, void *dst); + + // the linked list of the keywords + ARL_ENTRY *head; + + // since we keep the list of keywords sorted (as found in the source data) + // this is next keyword that we expect to find in the source data. + ARL_ENTRY *next_keyword; +} ARL_BASE; + +// create a new ARL +extern ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks); + +// free an ARL +extern void arl_free(ARL_BASE *arl_base); + +// register an expected keyword to the ARL +// together with its destination ( i.e. the output of the processor() ) +extern ARL_ENTRY *arl_expect(ARL_BASE *base, const char *keyword, void *dst); + +// an internal call to complete the check() call +extern int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const char *value); + +// begin an ARL iteration +extern void arl_begin(ARL_BASE *base); + +// check a keyword against the ARL +// this is to be called for each keyword read from source data +// s = the keyword, as collected +// src = the src data to be passed to the processor +// it is defined in the header file in order to be inlined +static inline int arl_check(ARL_BASE *base, const char *keyword, const char *value) { + ARL_ENTRY *e = base->next_keyword; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely((base->fast + base->slow) % (base->expected + base->allocated) == 0 && (base->fast + base->slow) > (base->expected + base->allocated) * base->iteration)) + info("ARL '%s': Did you forget to call arl_begin()?", base->name); +#endif + + // it should be the first entry (pointed by base->next_keyword) + if(likely(!strcmp(keyword, e->name))) { + // it is + +#ifdef NETDATA_INTERNAL_CHECKS + base->fast++; +#endif + + e->flags |= ARL_ENTRY_FLAG_FOUND; + + // execute the processor + if(unlikely(e->dst)) { + base->processor(e->name, e->hash, value, e->dst); + base->found++; + } + + // be prepared for the next iteration + base->next_keyword = e->next; + if(unlikely(!base->next_keyword)) + base->next_keyword = base->head; + + // stop if we collected all the values for this iteration + if(unlikely(base->found == base->wanted)) + return 1; + + return 0; + } + +#ifdef NETDATA_INTERNAL_CHECKS + base->slow++; +#endif + + // we read from source, a not-expected keyword + return arl_find_or_create_and_relink(base, keyword, value); +} + +#endif //NETDATA_ADAPTIVE_RESORTABLE_LIST_H diff --git a/src/appconfig.c b/src/appconfig.c index 947407484..81ab01be2 100644 --- a/src/appconfig.c +++ b/src/appconfig.c @@ -72,8 +72,8 @@ static int config_value_compare(void* a, void* b) { else return strcmp(((struct config_value *)a)->name, ((struct config_value *)b)->name); } -#define config_value_index_add(co, cv) avl_insert_lock(&((co)->values_index), (avl *)(cv)) -#define config_value_index_del(co, cv) avl_remove_lock(&((co)->values_index), (avl *)(cv)) +#define config_value_index_add(co, cv) (struct config_value *)avl_insert_lock(&((co)->values_index), (avl *)(cv)) +#define config_value_index_del(co, cv) (struct config_value *)avl_remove_lock(&((co)->values_index), (avl *)(cv)) static struct config_value *config_value_index_find(struct config *co, const char *name, uint32_t hash) { struct config_value tmp; @@ -98,8 +98,8 @@ avl_tree_lock config_root_index = { AVL_LOCK_INITIALIZER }; -#define config_index_add(cfg) avl_insert_lock(&config_root_index, (avl *)(cfg)) -#define config_index_del(cfg) avl_remove_lock(&config_root_index, (avl *)(cfg)) +#define config_index_add(cfg) (struct config *)avl_insert_lock(&config_root_index, (avl *)(cfg)) +#define config_index_del(cfg) (struct config *)avl_remove_lock(&config_root_index, (avl *)(cfg)) static struct config *config_index_find(const char *name, uint32_t hash) { struct config tmp; @@ -127,7 +127,8 @@ static inline struct config *config_section_create(const char *section) avl_init_lock(&co->values_index, config_value_compare); - config_index_add(co); + if(unlikely(config_index_add(co) != co)) + error("INTERNAL ERROR: indexing of section '%s', already exists.", co->name); config_global_write_lock(); struct config *co2 = config_root; @@ -154,7 +155,8 @@ static inline struct config_value *config_value_create(struct config *co, const cv->hash = simple_hash(cv->name); cv->value = strdupz(value); - config_value_index_add(co, cv); + if(unlikely(config_value_index_add(co, cv) != cv)) + error("INTERNAL ERROR: indexing of config '%s' in section '%s': already exists.", cv->name, co->name); config_section_write_lock(co); struct config_value *cv2 = co->values; @@ -198,14 +200,15 @@ int config_rename(const char *section, const char *old, const char *new) { cv2 = config_value_index_find(co, new, 0); if(cv2) goto cleanup; - config_value_index_del(co, cv); + if(unlikely(config_value_index_del(co, cv) != cv)) + error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted tge wrong config entry.", cv->name, co->name); freez(cv->name); cv->name = strdupz(new); - cv->hash = simple_hash(cv->name); + if(unlikely(config_value_index_add(co, cv) != cv)) + error("INTERNAL ERROR: indexing of config '%s' in section '%s', already exists.", cv->name, co->name); - config_value_index_add(co, cv); config_section_unlock(co); return 0; @@ -490,9 +493,10 @@ void generate_config(BUFFER *wb, int only_changed) config_global_write_lock(); for(co = config_root; co ; co = co->next) { if(!strcmp(co->name, "global") || - !strcmp(co->name, "plugins") || + !strcmp(co->name, "plugins") || !strcmp(co->name, "registry") || - !strcmp(co->name, "health")) + !strcmp(co->name, "health") || + !strcmp(co->name, "backend")) pri = 0; else if(!strncmp(co->name, "plugin:", 7)) pri = 1; else pri = 2; diff --git a/src/apps_plugin.c b/src/apps_plugin.c index f22a575ba..0a72190aa 100644 --- a/src/apps_plugin.c +++ b/src/apps_plugin.c @@ -13,6 +13,8 @@ // etc. #define RATES_DETAIL 10000ULL +#define MAX_SPARE_FDS 1 + int debug = 0; int update_every = 1; @@ -30,6 +32,8 @@ int show_guest_time_old = 0; int enable_guest_charts = 0; int enable_file_charts = 1; +int enable_users_charts = 1; +int enable_groups_charts = 1; // ---------------------------------------------------------------------------- @@ -84,7 +88,9 @@ struct target { unsigned long long io_storage_bytes_written; // unsigned long long io_cancelled_write_bytes; - int *fds; + int *target_fds; + int target_fds_size; + unsigned long long openfiles; unsigned long long openpipes; unsigned long long opensockets; @@ -119,7 +125,7 @@ long apps_groups_targets = 0; struct target *users_root_target = NULL; struct target *groups_root_target = NULL; -struct target *get_users_target(uid_t uid) +static struct target *get_users_target(uid_t uid) { struct target *w; for(w = users_root_target ; w ; w = w->next) @@ -187,10 +193,11 @@ struct target *get_groups_target(gid_t gid) // find or create a new target // there are targets that are just aggregated to other target (the second argument) -struct target *get_apps_groups_target(const char *id, struct target *target) { - int tdebug = 0, thidden = 0, ends_with = 0; +static struct target *get_apps_groups_target(const char *id, struct target *target, const char *name) { + int tdebug = 0, thidden = target?target->hidden:0, ends_with = 0; const char *nid = id; + // extract the options while(nid[0] == '-' || nid[0] == '+' || nid[0] == '*') { if(nid[0] == '-') thidden = 1; if(nid[0] == '+') tdebug = 1; @@ -199,6 +206,7 @@ struct target *get_apps_groups_target(const char *id, struct target *target) { } uint32_t hash = simple_hash(id); + // find if it already exists struct target *w, *last = apps_groups_root_target; for(w = apps_groups_root_target ; w ; w = w->next) { if(w->idhash == hash && strncmp(nid, w->id, MAX_NAME) == 0) @@ -207,11 +215,37 @@ struct target *get_apps_groups_target(const char *id, struct target *target) { last = w; } + // find an existing target + if(unlikely(!target)) { + while(*name == '-') { + if(*name == '-') thidden = 1; + name++; + } + for(target = apps_groups_root_target ; target ; target = target->next) { + if(!target->target && strcmp(name, target->name) == 0) + break; + } + if(unlikely(debug)) { + if(unlikely(target)) + fprintf(stderr, "apps.plugin: REUSING TARGET NAME '%s' on ID '%s'\n", target->name, target->id); + else + fprintf(stderr, "apps.plugin: NEW TARGET NAME '%s' on ID '%s'\n", name, id); + } + } + + if(target && target->target) + fatal("Internal Error: request to link process '%s' to target '%s' which is linked to target '%s'", id, target->id, target->target->id); + w = callocz(sizeof(struct target), 1); strncpyz(w->id, nid, MAX_NAME); w->idhash = simple_hash(w->id); - strncpyz(w->name, nid, MAX_NAME); + if(unlikely(!target)) + // copy the name + strncpyz(w->name, name, MAX_NAME); + else + // copy the id + strncpyz(w->name, nid, MAX_NAME); strncpyz(w->compare, nid, MAX_COMPARE_NAME); size_t len = strlen(w->compare); @@ -239,7 +273,7 @@ struct target *get_apps_groups_target(const char *id, struct target *target) { fprintf(stderr, "apps.plugin: ADDING TARGET ID '%s', process name '%s' (%s), aggregated on target '%s', options: %s %s\n" , w->id , w->compare, (w->starts_with && w->ends_with)?"substring":((w->starts_with)?"prefix":((w->ends_with)?"suffix":"exact")) - , w->target?w->target->id:w->id + , w->target?w->target->name:w->name , (w->hidden)?"hidden":"-" , (w->debug)?"debug":"-" ); @@ -248,11 +282,11 @@ struct target *get_apps_groups_target(const char *id, struct target *target) { } // read the apps_groups.conf file -int read_apps_groups_conf(const char *name) +static int read_apps_groups_conf(const char *file) { char filename[FILENAME_MAX + 1]; - snprintfz(filename, FILENAME_MAX, "%s/apps_%s.conf", config_dir, name); + snprintfz(filename, FILENAME_MAX, "%s/apps_%s.conf", config_dir, file); if(unlikely(debug)) fprintf(stderr, "apps.plugin: process groups file: '%s'\n", filename); @@ -272,59 +306,45 @@ int read_apps_groups_conf(const char *name) for(line = 0; line < lines ;line++) { unsigned long word, words = procfile_linewords(ff, line); - struct target *w = NULL; + if(!words) continue; + + char *name = procfile_lineword(ff, line, 0); + if(!name || !*name) continue; - char *t = procfile_lineword(ff, line, 0); - if(!t || !*t) continue; + // find a possibly existing target + struct target *w = NULL; + // loop through all words, skipping the first one (the name) for(word = 0; word < words ;word++) { char *s = procfile_lineword(ff, line, word); if(!s || !*s) continue; if(*s == '#') break; - if(t == s) continue; + // is this the first word? skip it + if(s == name) continue; - struct target *n = get_apps_groups_target(s, w); + // add this target + struct target *n = get_apps_groups_target(s, w, name); if(!n) { error("Cannot create target '%s' (line %lu, word %lu)", s, line, word); continue; } - if(!w) w = n; - } - - if(w) { - int tdebug = 0, thidden = 0; - - while(t[0] == '-' || t[0] == '+') { - if(t[0] == '-') thidden = 1; - if(t[0] == '+') tdebug = 1; - t++; - } - - strncpyz(w->name, t, MAX_NAME); - w->hidden = thidden; - w->debug = tdebug; - - if(unlikely(debug)) - fprintf(stderr, "apps.plugin: AGGREGATION TARGET NAME '%s' on ID '%s', process name '%s' (%s), aggregated on target '%s', options: %s %s\n" - , w->name - , w->id - , w->compare, (w->starts_with && w->ends_with)?"substring":((w->starts_with)?"prefix":((w->ends_with)?"suffix":"exact")) - , w->target?w->target->id:w->id - , (w->hidden)?"hidden":"-" - , (w->debug)?"debug":"-" - ); + // just some optimization + // to avoid searching for a target for each process + if(!w) w = n->target?n->target:n; } } procfile_close(ff); - apps_groups_default_target = get_apps_groups_target("p+!o@w#e$i^r&7*5(-i)l-o_", NULL); // match nothing + apps_groups_default_target = get_apps_groups_target("p+!o@w#e$i^r&7*5(-i)l-o_", NULL, "other"); // match nothing if(!apps_groups_default_target) - error("Cannot create default target"); - else - strncpyz(apps_groups_default_target->name, "other", MAX_NAME); + fatal("Cannot create default target"); + + // allow the user to override group 'other' + if(apps_groups_default_target->target) + apps_groups_default_target = apps_groups_default_target->target; return 0; } @@ -467,15 +487,15 @@ struct pid_stat { long all_pids_count = 0; -struct pid_stat *get_pid_entry(pid_t pid) { +static inline struct pid_stat *get_pid_entry(pid_t pid) { if(all_pids[pid]) { all_pids[pid]->new_entry = 0; return all_pids[pid]; } all_pids[pid] = callocz(sizeof(struct pid_stat), 1); - all_pids[pid]->fds = callocz(sizeof(int), 100); - all_pids[pid]->fds_size = 100; + all_pids[pid]->fds = callocz(sizeof(int), MAX_SPARE_FDS); + all_pids[pid]->fds_size = MAX_SPARE_FDS; if(root_of_pids) root_of_pids->prev = all_pids[pid]; all_pids[pid]->next = root_of_pids; @@ -489,7 +509,7 @@ struct pid_stat *get_pid_entry(pid_t pid) { return all_pids[pid]; } -void del_pid_entry(pid_t pid) { +static inline void del_pid_entry(pid_t pid) { if(!all_pids[pid]) { error("attempted to free pid %d that is not allocated.", pid); return; @@ -517,7 +537,7 @@ void del_pid_entry(pid_t pid) { // ---------------------------------------------------------------------------- // update pids from proc -int read_proc_pid_cmdline(struct pid_stat *p) { +static inline int read_proc_pid_cmdline(struct pid_stat *p) { if(unlikely(!p->cmdline_filename)) { char filename[FILENAME_MAX + 1]; @@ -548,7 +568,7 @@ cleanup: return 0; } -int read_proc_pid_ownership(struct pid_stat *p) { +static inline int read_proc_pid_ownership(struct pid_stat *p) { if(unlikely(!p->stat_filename)) { error("pid %d does not have a stat_filename", p->pid); return 0; @@ -569,7 +589,7 @@ int read_proc_pid_ownership(struct pid_stat *p) { return 1; } -int read_proc_pid_stat(struct pid_stat *p) { +static inline int read_proc_pid_stat(struct pid_stat *p) { static procfile *ff = NULL; if(unlikely(!p->stat_filename)) { @@ -590,89 +610,89 @@ int read_proc_pid_stat(struct pid_stat *p) { if(unlikely(!ff)) goto cleanup; p->last_stat_collected_usec = p->stat_collected_usec; - p->stat_collected_usec = time_usec(); + p->stat_collected_usec = now_realtime_usec(); file_counter++; - // p->pid = atol(procfile_lineword(ff, 0, 0+i)); + // p->pid = str2ul(procfile_lineword(ff, 0, 0+i)); strncpyz(p->comm, procfile_lineword(ff, 0, 1), MAX_COMPARE_NAME); // p->state = *(procfile_lineword(ff, 0, 2)); - p->ppid = (int32_t) atol(procfile_lineword(ff, 0, 3)); - // p->pgrp = atol(procfile_lineword(ff, 0, 4)); - // p->session = atol(procfile_lineword(ff, 0, 5)); - // p->tty_nr = atol(procfile_lineword(ff, 0, 6)); - // p->tpgid = atol(procfile_lineword(ff, 0, 7)); - // p->flags = strtoull(procfile_lineword(ff, 0, 8), NULL, 10); + p->ppid = (int32_t)str2ul(procfile_lineword(ff, 0, 3)); + // p->pgrp = str2ul(procfile_lineword(ff, 0, 4)); + // p->session = str2ul(procfile_lineword(ff, 0, 5)); + // p->tty_nr = str2ul(procfile_lineword(ff, 0, 6)); + // p->tpgid = str2ul(procfile_lineword(ff, 0, 7)); + // p->flags = str2ull(procfile_lineword(ff, 0, 8)); unsigned long long last; last = p->minflt_raw; - p->minflt_raw = strtoull(procfile_lineword(ff, 0, 9), NULL, 10); - p->minflt = (p->minflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->minflt_raw = str2ull(procfile_lineword(ff, 0, 9)); + p->minflt = (p->minflt_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->cminflt_raw; - p->cminflt_raw = strtoull(procfile_lineword(ff, 0, 10), NULL, 10); - p->cminflt = (p->cminflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->cminflt_raw = str2ull(procfile_lineword(ff, 0, 10)); + p->cminflt = (p->cminflt_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->majflt_raw; - p->majflt_raw = strtoull(procfile_lineword(ff, 0, 11), NULL, 10); - p->majflt = (p->majflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->majflt_raw = str2ull(procfile_lineword(ff, 0, 11)); + p->majflt = (p->majflt_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->cmajflt_raw; - p->cmajflt_raw = strtoull(procfile_lineword(ff, 0, 12), NULL, 10); - p->cmajflt = (p->cmajflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->cmajflt_raw = str2ull(procfile_lineword(ff, 0, 12)); + p->cmajflt = (p->cmajflt_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->utime_raw; - p->utime_raw = strtoull(procfile_lineword(ff, 0, 13), NULL, 10); - p->utime = (p->utime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->utime_raw = str2ull(procfile_lineword(ff, 0, 13)); + p->utime = (p->utime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->stime_raw; - p->stime_raw = strtoull(procfile_lineword(ff, 0, 14), NULL, 10); - p->stime = (p->stime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->stime_raw = str2ull(procfile_lineword(ff, 0, 14)); + p->stime = (p->stime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->cutime_raw; - p->cutime_raw = strtoull(procfile_lineword(ff, 0, 15), NULL, 10); - p->cutime = (p->cutime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->cutime_raw = str2ull(procfile_lineword(ff, 0, 15)); + p->cutime = (p->cutime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->cstime_raw; - p->cstime_raw = strtoull(procfile_lineword(ff, 0, 16), NULL, 10); - p->cstime = (p->cstime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); - - // p->priority = strtoull(procfile_lineword(ff, 0, 17), NULL, 10); - // p->nice = strtoull(procfile_lineword(ff, 0, 18), NULL, 10); - p->num_threads = (int32_t) atol(procfile_lineword(ff, 0, 19)); - // p->itrealvalue = strtoull(procfile_lineword(ff, 0, 20), NULL, 10); - // p->starttime = strtoull(procfile_lineword(ff, 0, 21), NULL, 10); - // p->vsize = strtoull(procfile_lineword(ff, 0, 22), NULL, 10); - // p->rss = strtoull(procfile_lineword(ff, 0, 23), NULL, 10); - // p->rsslim = strtoull(procfile_lineword(ff, 0, 24), NULL, 10); - // p->starcode = strtoull(procfile_lineword(ff, 0, 25), NULL, 10); - // p->endcode = strtoull(procfile_lineword(ff, 0, 26), NULL, 10); - // p->startstack = strtoull(procfile_lineword(ff, 0, 27), NULL, 10); - // p->kstkesp = strtoull(procfile_lineword(ff, 0, 28), NULL, 10); - // p->kstkeip = strtoull(procfile_lineword(ff, 0, 29), NULL, 10); - // p->signal = strtoull(procfile_lineword(ff, 0, 30), NULL, 10); - // p->blocked = strtoull(procfile_lineword(ff, 0, 31), NULL, 10); - // p->sigignore = strtoull(procfile_lineword(ff, 0, 32), NULL, 10); - // p->sigcatch = strtoull(procfile_lineword(ff, 0, 33), NULL, 10); - // p->wchan = strtoull(procfile_lineword(ff, 0, 34), NULL, 10); - // p->nswap = strtoull(procfile_lineword(ff, 0, 35), NULL, 10); - // p->cnswap = strtoull(procfile_lineword(ff, 0, 36), NULL, 10); - // p->exit_signal = atol(procfile_lineword(ff, 0, 37)); - // p->processor = atol(procfile_lineword(ff, 0, 38)); - // p->rt_priority = strtoul(procfile_lineword(ff, 0, 39), NULL, 10); - // p->policy = strtoul(procfile_lineword(ff, 0, 40), NULL, 10); - // p->delayacct_blkio_ticks = strtoull(procfile_lineword(ff, 0, 41), NULL, 10); + p->cstime_raw = str2ull(procfile_lineword(ff, 0, 16)); + p->cstime = (p->cstime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + + // p->priority = str2ull(procfile_lineword(ff, 0, 17)); + // p->nice = str2ull(procfile_lineword(ff, 0, 18)); + p->num_threads = (int32_t)str2ul(procfile_lineword(ff, 0, 19)); + // p->itrealvalue = str2ull(procfile_lineword(ff, 0, 20)); + // p->starttime = str2ull(procfile_lineword(ff, 0, 21)); + // p->vsize = str2ull(procfile_lineword(ff, 0, 22)); + // p->rss = str2ull(procfile_lineword(ff, 0, 23)); + // p->rsslim = str2ull(procfile_lineword(ff, 0, 24)); + // p->starcode = str2ull(procfile_lineword(ff, 0, 25)); + // p->endcode = str2ull(procfile_lineword(ff, 0, 26)); + // p->startstack = str2ull(procfile_lineword(ff, 0, 27)); + // p->kstkesp = str2ull(procfile_lineword(ff, 0, 28)); + // p->kstkeip = str2ull(procfile_lineword(ff, 0, 29)); + // p->signal = str2ull(procfile_lineword(ff, 0, 30)); + // p->blocked = str2ull(procfile_lineword(ff, 0, 31)); + // p->sigignore = str2ull(procfile_lineword(ff, 0, 32)); + // p->sigcatch = str2ull(procfile_lineword(ff, 0, 33)); + // p->wchan = str2ull(procfile_lineword(ff, 0, 34)); + // p->nswap = str2ull(procfile_lineword(ff, 0, 35)); + // p->cnswap = str2ull(procfile_lineword(ff, 0, 36)); + // p->exit_signal = str2ul(procfile_lineword(ff, 0, 37)); + // p->processor = str2ul(procfile_lineword(ff, 0, 38)); + // p->rt_priority = str2ul(procfile_lineword(ff, 0, 39)); + // p->policy = str2ul(procfile_lineword(ff, 0, 40)); + // p->delayacct_blkio_ticks = str2ull(procfile_lineword(ff, 0, 41)); if(enable_guest_charts) { last = p->gtime_raw; - p->gtime_raw = strtoull(procfile_lineword(ff, 0, 42), NULL, 10); - p->gtime = (p->gtime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->gtime_raw = str2ull(procfile_lineword(ff, 0, 42)); + p->gtime = (p->gtime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); last = p->cgtime_raw; - p->cgtime_raw = strtoull(procfile_lineword(ff, 0, 43), NULL, 10); - p->cgtime = (p->cgtime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); + p->cgtime_raw = str2ull(procfile_lineword(ff, 0, 43)); + p->cgtime = (p->cgtime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec); if (show_guest_time || p->gtime || p->cgtime) { p->utime -= (p->utime >= p->gtime) ? p->gtime : p->utime; @@ -715,7 +735,7 @@ cleanup: return 0; } -int read_proc_pid_statm(struct pid_stat *p) { +static inline int read_proc_pid_statm(struct pid_stat *p) { static procfile *ff = NULL; if(unlikely(!p->statm_filename)) { @@ -732,13 +752,13 @@ int read_proc_pid_statm(struct pid_stat *p) { file_counter++; - p->statm_size = strtoull(procfile_lineword(ff, 0, 0), NULL, 10); - p->statm_resident = strtoull(procfile_lineword(ff, 0, 1), NULL, 10); - p->statm_share = strtoull(procfile_lineword(ff, 0, 2), NULL, 10); - // p->statm_text = strtoull(procfile_lineword(ff, 0, 3), NULL, 10); - // p->statm_lib = strtoull(procfile_lineword(ff, 0, 4), NULL, 10); - // p->statm_data = strtoull(procfile_lineword(ff, 0, 5), NULL, 10); - // p->statm_dirty = strtoull(procfile_lineword(ff, 0, 6), NULL, 10); + p->statm_size = str2ull(procfile_lineword(ff, 0, 0)); + p->statm_resident = str2ull(procfile_lineword(ff, 0, 1)); + p->statm_share = str2ull(procfile_lineword(ff, 0, 2)); + // p->statm_text = str2ull(procfile_lineword(ff, 0, 3)); + // p->statm_lib = str2ull(procfile_lineword(ff, 0, 4)); + // p->statm_data = str2ull(procfile_lineword(ff, 0, 5)); + // p->statm_dirty = str2ull(procfile_lineword(ff, 0, 6)); return 1; @@ -753,7 +773,7 @@ cleanup: return 0; } -int read_proc_pid_io(struct pid_stat *p) { +static inline int read_proc_pid_io(struct pid_stat *p) { static procfile *ff = NULL; if(unlikely(!p->io_filename)) { @@ -772,37 +792,37 @@ int read_proc_pid_io(struct pid_stat *p) { file_counter++; p->last_io_collected_usec = p->io_collected_usec; - p->io_collected_usec = time_usec(); + p->io_collected_usec = now_realtime_usec(); unsigned long long last; last = p->io_logical_bytes_read_raw; - p->io_logical_bytes_read_raw = strtoull(procfile_lineword(ff, 0, 1), NULL, 10); - p->io_logical_bytes_read = (p->io_logical_bytes_read_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); + p->io_logical_bytes_read_raw = str2ull(procfile_lineword(ff, 0, 1)); + p->io_logical_bytes_read = (p->io_logical_bytes_read_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); last = p->io_logical_bytes_written_raw; - p->io_logical_bytes_written_raw = strtoull(procfile_lineword(ff, 1, 1), NULL, 10); - p->io_logical_bytes_written = (p->io_logical_bytes_written_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); + p->io_logical_bytes_written_raw = str2ull(procfile_lineword(ff, 1, 1)); + p->io_logical_bytes_written = (p->io_logical_bytes_written_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); // last = p->io_read_calls_raw; - // p->io_read_calls_raw = strtoull(procfile_lineword(ff, 2, 1), NULL, 10); - // p->io_read_calls = (p->io_read_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); + // p->io_read_calls_raw = str2ull(procfile_lineword(ff, 2, 1)); + // p->io_read_calls = (p->io_read_calls_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); // last = p->io_write_calls_raw; - // p->io_write_calls_raw = strtoull(procfile_lineword(ff, 3, 1), NULL, 10); - // p->io_write_calls = (p->io_write_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); + // p->io_write_calls_raw = str2ull(procfile_lineword(ff, 3, 1)); + // p->io_write_calls = (p->io_write_calls_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); last = p->io_storage_bytes_read_raw; - p->io_storage_bytes_read_raw = strtoull(procfile_lineword(ff, 4, 1), NULL, 10); - p->io_storage_bytes_read = (p->io_storage_bytes_read_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); + p->io_storage_bytes_read_raw = str2ull(procfile_lineword(ff, 4, 1)); + p->io_storage_bytes_read = (p->io_storage_bytes_read_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); last = p->io_storage_bytes_written_raw; - p->io_storage_bytes_written_raw = strtoull(procfile_lineword(ff, 5, 1), NULL, 10); - p->io_storage_bytes_written = (p->io_storage_bytes_written_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); + p->io_storage_bytes_written_raw = str2ull(procfile_lineword(ff, 5, 1)); + p->io_storage_bytes_written = (p->io_storage_bytes_written_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); // last = p->io_cancelled_write_bytes_raw; - // p->io_cancelled_write_bytes_raw = strtoull(procfile_lineword(ff, 6, 1), NULL, 10); - // p->io_cancelled_write_bytes = (p->io_cancelled_write_bytes_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); + // p->io_cancelled_write_bytes_raw = str2ull(procfile_lineword(ff, 6, 1)); + // p->io_cancelled_write_bytes = (p->io_cancelled_write_bytes_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec); if(unlikely(global_iterations_counter == 1)) { p->io_logical_bytes_read = 0; @@ -831,10 +851,11 @@ unsigned long long global_utime = 0; unsigned long long global_stime = 0; unsigned long long global_gtime = 0; -int read_proc_stat() { +static inline int read_proc_stat() { static char filename[FILENAME_MAX + 1] = ""; static procfile *ff = NULL; - static unsigned long long utime_raw = 0, stime_raw = 0, gtime_raw = 0, gntime_raw = 0, ntime_raw = 0, collected_usec = 0, last_collected_usec = 0; + static unsigned long long utime_raw = 0, stime_raw = 0, gtime_raw = 0, gntime_raw = 0, ntime_raw = 0; + static usec_t collected_usec = 0, last_collected_usec = 0; if(unlikely(!ff)) { snprintfz(filename, FILENAME_MAX, "%s/proc/stat", global_host_prefix); @@ -846,34 +867,34 @@ int read_proc_stat() { if(unlikely(!ff)) goto cleanup; last_collected_usec = collected_usec; - collected_usec = time_usec(); + collected_usec = now_realtime_usec(); file_counter++; unsigned long long last; last = utime_raw; - utime_raw = strtoull(procfile_lineword(ff, 0, 1), NULL, 10); - global_utime = (utime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec); + utime_raw = str2ull(procfile_lineword(ff, 0, 1)); + global_utime = (utime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (collected_usec - last_collected_usec); // nice time, on user time last = ntime_raw; - ntime_raw = strtoull(procfile_lineword(ff, 0, 2), NULL, 10); - global_utime += (ntime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec); + ntime_raw = str2ull(procfile_lineword(ff, 0, 2)); + global_utime += (ntime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (collected_usec - last_collected_usec); last = stime_raw; - stime_raw = strtoull(procfile_lineword(ff, 0, 3), NULL, 10); - global_stime = (stime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec); + stime_raw = str2ull(procfile_lineword(ff, 0, 3)); + global_stime = (stime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (collected_usec - last_collected_usec); last = gtime_raw; - gtime_raw = strtoull(procfile_lineword(ff, 0, 10), NULL, 10); - global_gtime = (gtime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec); + gtime_raw = str2ull(procfile_lineword(ff, 0, 10)); + global_gtime = (gtime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (collected_usec - last_collected_usec); if(enable_guest_charts) { // guest nice time, on guest time last = gntime_raw; - gntime_raw = strtoull(procfile_lineword(ff, 0, 11), NULL, 10); - global_gtime += (gntime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec); + gntime_raw = str2ull(procfile_lineword(ff, 0, 11)); + global_gtime += (gntime_raw - last) * (USEC_PER_SEC * RATES_DETAIL) / (collected_usec - last_collected_usec); // remove guest time from user time global_utime -= (global_utime > global_gtime) ? global_gtime : global_utime; @@ -966,7 +987,7 @@ static struct file_descriptor *file_descriptor_find(const char *name, uint32_t h #define FILETYPE_TIMERFD 7 #define FILETYPE_SIGNALFD 8 -void file_descriptor_not_used(int id) +static inline void file_descriptor_not_used(int id) { if(id > 0 && id < all_files_size) { @@ -987,7 +1008,9 @@ void file_descriptor_not_used(int id) if(unlikely(debug)) fprintf(stderr, "apps.plugin: >> slot %d is empty.\n", id); - file_descriptor_remove(&all_files[id]); + if(unlikely(file_descriptor_remove(&all_files[id]) != (void *)&all_files[id])) + error("INTERNAL ERROR: removal of unused fd from index, removed a different fd"); + #ifdef NETDATA_INTERNAL_CHECKS all_files[id].magic = 0x00000000; #endif /* NETDATA_INTERNAL_CHECKS */ @@ -1000,69 +1023,60 @@ void file_descriptor_not_used(int id) else error("Request to decrease counter of fd %d, which is outside the array size (1 to %d)", id, all_files_size); } -int file_descriptor_find_or_add(const char *name) -{ - static int last_pos = 0; - uint32_t hash = simple_hash(name); +static inline void all_files_grow() { + void *old = all_files; + int i; + // there is no empty slot if(unlikely(debug)) - fprintf(stderr, "apps.plugin: adding or finding name '%s' with hash %u\n", name, hash); + fprintf(stderr, "apps.plugin: extending fd array to %d entries\n", all_files_size + FILE_DESCRIPTORS_INCREASE_STEP); - struct file_descriptor *fd = file_descriptor_find(name, hash); - if(fd) { - // found - if(unlikely(debug)) - fprintf(stderr, "apps.plugin: >> found on slot %d\n", fd->pos); - - fd->count++; - return fd->pos; - } - // not found + all_files = reallocz(all_files, (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP) * sizeof(struct file_descriptor)); - // check we have enough memory to add it - if(!all_files || all_files_len == all_files_size) { - void *old = all_files; - int i; + // if the address changed, we have to rebuild the index + // since all pointers are now invalid - // there is no empty slot + if(unlikely(old && old != (void *)all_files)) { if(unlikely(debug)) - fprintf(stderr, "apps.plugin: extending fd array to %d entries\n", all_files_size + FILE_DESCRIPTORS_INCREASE_STEP); + fprintf(stderr, "apps.plugin: >> re-indexing.\n"); - all_files = reallocz(all_files, (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP) * sizeof(struct file_descriptor)); - - // if the address changed, we have to rebuild the index - // since all pointers are now invalid - if(old && old != (void *)all_files) { - if(unlikely(debug)) - fprintf(stderr, "apps.plugin: >> re-indexing.\n"); + all_files_index.root = NULL; + for(i = 0; i < all_files_size; i++) { + if(!all_files[i].count) continue; + if(unlikely(file_descriptor_add(&all_files[i]) != (void *)&all_files[i])) + error("INTERNAL ERROR: duplicate indexing of fd during realloc."); + } - all_files_index.root = NULL; - for(i = 0; i < all_files_size; i++) { - if(!all_files[i].count) continue; - file_descriptor_add(&all_files[i]); - } + if(unlikely(debug)) + fprintf(stderr, "apps.plugin: >> re-indexing done.\n"); + } - if(unlikely(debug)) - fprintf(stderr, "apps.plugin: >> re-indexing done.\n"); - } + // initialize the newly added entries - for(i = all_files_size; i < (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP); i++) { - all_files[i].count = 0; - all_files[i].name = NULL; + for(i = all_files_size; i < (all_files_size + FILE_DESCRIPTORS_INCREASE_STEP); i++) { + all_files[i].count = 0; + all_files[i].name = NULL; #ifdef NETDATA_INTERNAL_CHECKS - all_files[i].magic = 0x00000000; + all_files[i].magic = 0x00000000; #endif /* NETDATA_INTERNAL_CHECKS */ - all_files[i].pos = i; - } - - if(!all_files_size) all_files_len = 1; - all_files_size += FILE_DESCRIPTORS_INCREASE_STEP; + all_files[i].pos = i; } + if(unlikely(!all_files_size)) all_files_len = 1; + all_files_size += FILE_DESCRIPTORS_INCREASE_STEP; +} + +static inline int file_descriptor_set_on_empty_slot(const char *name, uint32_t hash, int type) { + // check we have enough memory to add it + if(!all_files || all_files_len == all_files_size) + all_files_grow(); + if(unlikely(debug)) fprintf(stderr, "apps.plugin: >> searching for empty slot.\n"); // search for an empty slot + + static int last_pos = 0; int i, c; for(i = 0, c = last_pos ; i < all_files_size ; i++, c++) { if(c >= all_files_size) c = 0; @@ -1080,23 +1094,58 @@ int file_descriptor_find_or_add(const char *name) if(unlikely(debug)) fprintf(stderr, "apps.plugin: >> %s fd position %d for %s (last name: %s)\n", all_files[c].name?"re-using":"using", c, name, all_files[c].name); - if(all_files[c].name) freez((void *)all_files[c].name); + freez((void *)all_files[c].name); all_files[c].name = NULL; last_pos = c; break; } } + + all_files_len++; + if(i == all_files_size) { fatal("We should find an empty slot, but there isn't any"); exit(1); } + // else we have an empty slot in 'c' if(unlikely(debug)) fprintf(stderr, "apps.plugin: >> updating slot %d.\n", c); - all_files_len++; + all_files[c].name = strdupz(name); + all_files[c].hash = hash; + all_files[c].type = type; + all_files[c].pos = c; + all_files[c].count = 1; +#ifdef NETDATA_INTERNAL_CHECKS + all_files[c].magic = 0x0BADCAFE; +#endif /* NETDATA_INTERNAL_CHECKS */ + if(unlikely(file_descriptor_add(&all_files[c]) != (void *)&all_files[c])) + error("INTERNAL ERROR: duplicate indexing of fd."); - // else we have an empty slot in 'c' + if(unlikely(debug)) + fprintf(stderr, "apps.plugin: using fd position %d (name: %s)\n", c, all_files[c].name); + + return c; +} + +static inline int file_descriptor_find_or_add(const char *name) +{ + uint32_t hash = simple_hash(name); + + if(unlikely(debug)) + fprintf(stderr, "apps.plugin: adding or finding name '%s' with hash %u\n", name, hash); + + struct file_descriptor *fd = file_descriptor_find(name, hash); + if(fd) { + // found + if(unlikely(debug)) + fprintf(stderr, "apps.plugin: >> found on slot %d\n", fd->pos); + + fd->count++; + return fd->pos; + } + // not found int type; if(name[0] == '/') type = FILETYPE_FILE; @@ -1120,23 +1169,10 @@ int file_descriptor_find_or_add(const char *name) type = FILETYPE_OTHER; } - all_files[c].name = strdupz(name); - all_files[c].hash = hash; - all_files[c].type = type; - all_files[c].pos = c; - all_files[c].count = 1; -#ifdef NETDATA_INTERNAL_CHECKS - all_files[c].magic = 0x0BADCAFE; -#endif /* NETDATA_INTERNAL_CHECKS */ - file_descriptor_add(&all_files[c]); - - if(unlikely(debug)) - fprintf(stderr, "apps.plugin: using fd position %d (name: %s)\n", c, all_files[c].name); - - return c; + return file_descriptor_set_on_empty_slot(name, hash, type); } -int read_pid_file_descriptors(struct pid_stat *p) { +static inline int read_pid_file_descriptors(struct pid_stat *p) { char dirname[FILENAME_MAX+1]; snprintfz(dirname, FILENAME_MAX, "%s/proc/%d/fd", global_host_prefix, p->pid); @@ -1156,22 +1192,18 @@ int read_pid_file_descriptors(struct pid_stat *p) { continue; // check if the fds array is small - int fdid = atoi(de->d_name); + int fdid = (int)str2l(de->d_name); if(fdid < 0) continue; if(fdid >= p->fds_size) { // it is small, extend it if(unlikely(debug)) - fprintf(stderr, "apps.plugin: extending fd memory slots for %s from %d to %d\n", p->comm, p->fds_size, fdid + 100); + fprintf(stderr, "apps.plugin: extending fd memory slots for %s from %d to %d\n", p->comm, p->fds_size, fdid + MAX_SPARE_FDS); - p->fds = reallocz(p->fds, (fdid + 100) * sizeof(int)); - if(!p->fds) { - fatal("Cannot re-allocate fds for %s", p->comm); - break; - } + p->fds = reallocz(p->fds, (fdid + MAX_SPARE_FDS) * sizeof(int)); // and initialize it - for(c = p->fds_size ; c < (fdid + 100) ; c++) p->fds[c] = 0; - p->fds_size = fdid + 100; + for(c = p->fds_size ; c < (fdid + MAX_SPARE_FDS) ; c++) p->fds[c] = 0; + p->fds_size = fdid + MAX_SPARE_FDS; } if(p->fds[fdid] == 0) { @@ -1214,7 +1246,7 @@ int read_pid_file_descriptors(struct pid_stat *p) { // ---------------------------------------------------------------------------- -int print_process_and_parents(struct pid_stat *p, unsigned long long time) { +static inline int print_process_and_parents(struct pid_stat *p, unsigned long long time) { char *prefix = "\\_ "; int indent = 0; @@ -1253,13 +1285,13 @@ int print_process_and_parents(struct pid_stat *p, unsigned long long time) { return indent + 1; } -void print_process_tree(struct pid_stat *p, char *msg) { +static inline void print_process_tree(struct pid_stat *p, char *msg) { log_date(stderr); fprintf(stderr, "%s: process %s (%d, %s) with parents:\n", msg, p->comm, p->pid, p->updated?"running":"exited"); print_process_and_parents(p, p->stat_collected_usec); } -void find_lost_child_debug(struct pid_stat *pe, unsigned long long lost, int type) { +static inline void find_lost_child_debug(struct pid_stat *pe, unsigned long long lost, int type) { int found = 0; struct pid_stat *p = NULL; @@ -1329,7 +1361,7 @@ void find_lost_child_debug(struct pid_stat *pe, unsigned long long lost, int typ } } -unsigned long long remove_exited_child_from_parent(unsigned long long *field, unsigned long long *pfield) { +static inline unsigned long long remove_exited_child_from_parent(unsigned long long *field, unsigned long long *pfield) { unsigned long long absorbed = 0; if(*field > *pfield) { @@ -1346,7 +1378,7 @@ unsigned long long remove_exited_child_from_parent(unsigned long long *field, un return absorbed; } -void process_exited_processes() { +static inline void process_exited_processes() { struct pid_stat *p; for(p = root_of_pids; p ; p = p->next) { @@ -1439,11 +1471,11 @@ void process_exited_processes() { ); } - p->utime_raw = utime * (p->stat_collected_usec - p->last_stat_collected_usec) / (1000000ULL * RATES_DETAIL); - p->stime_raw = stime * (p->stat_collected_usec - p->last_stat_collected_usec) / (1000000ULL * RATES_DETAIL); - p->gtime_raw = gtime * (p->stat_collected_usec - p->last_stat_collected_usec) / (1000000ULL * RATES_DETAIL); - p->minflt_raw = minflt * (p->stat_collected_usec - p->last_stat_collected_usec) / (1000000ULL * RATES_DETAIL); - p->majflt_raw = majflt * (p->stat_collected_usec - p->last_stat_collected_usec) / (1000000ULL * RATES_DETAIL); + p->utime_raw = utime * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL); + p->stime_raw = stime * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL); + p->gtime_raw = gtime * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL); + p->minflt_raw = minflt * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL); + p->majflt_raw = majflt * (p->stat_collected_usec - p->last_stat_collected_usec) / (USEC_PER_SEC * RATES_DETAIL); p->cutime_raw = p->cstime_raw = p->cgtime_raw = p->cminflt_raw = p->cmajflt_raw = 0; if(unlikely(debug)) @@ -1459,7 +1491,7 @@ void process_exited_processes() { } } -void link_all_processes_to_their_parents(void) { +static inline void link_all_processes_to_their_parents(void) { struct pid_stat *p, *pp; // link all children to their parents @@ -1562,15 +1594,15 @@ static inline int managed_log(struct pid_stat *p, uint32_t log, int status) { return status; } -void collect_data_for_pid(pid_t pid) { +static inline int collect_data_for_pid(pid_t pid) { if(unlikely(pid <= 0 || pid > pid_max)) { error("Invalid pid %d read (expected 1 to %d). Ignoring process.", pid, pid_max); - return; + return 0; } struct pid_stat *p = get_pid_entry(pid); - if(unlikely(!p || p->read)) return; - p->read = 1; + if(unlikely(!p || p->read)) return 0; + p->read = 1; // fprintf(stderr, "Reading process %d (%s), sortlist %d\n", p->pid, p->comm, p->sortlist); @@ -1579,7 +1611,7 @@ void collect_data_for_pid(pid_t pid) { if(unlikely(!managed_log(p, PID_LOG_STAT, read_proc_pid_stat(p)))) // there is no reason to proceed if we cannot get its status - return; + return 0; read_proc_pid_ownership(p); @@ -1599,7 +1631,7 @@ void collect_data_for_pid(pid_t pid) { if(unlikely(!managed_log(p, PID_LOG_STATM, read_proc_pid_statm(p)))) // there is no reason to proceed if we cannot get its memory status - return; + return 0; // -------------------------------------------------------------------- // link it @@ -1658,9 +1690,11 @@ void collect_data_for_pid(pid_t pid) { p->updated = 1; p->keep = 0; p->keeploops = 0; + + return 1; } -int collect_data_for_all_processes_from_proc(void) { +static int collect_data_for_all_processes_from_proc(void) { struct pid_stat *p = NULL; if(all_pids_count) { @@ -1688,7 +1722,7 @@ int collect_data_for_all_processes_from_proc(void) { } if(include_exited_childs) { - qsort((void *)all_pids_sortlist, all_pids_count, sizeof(pid_t), compar_pid); + qsort((void *)all_pids_sortlist, (size_t)all_pids_count, sizeof(pid_t), compar_pid); for(slc = 0; slc < all_pids_count; slc++) collect_data_for_pid(all_pids_sortlist[slc]); } @@ -1714,6 +1748,9 @@ int collect_data_for_all_processes_from_proc(void) { } closedir(dir); + if(!all_pids_count) + return 0; + // normally this is done // however we may have processes exited while we collected values // so let's find the exited ones @@ -1743,7 +1780,7 @@ int collect_data_for_all_processes_from_proc(void) { // 9. find the unique file count for each target // check: update_apps_groups_statistics() -void cleanup_exited_pids(void) { +static void cleanup_exited_pids(void) { int c; struct pid_stat *p = NULL; @@ -1771,7 +1808,7 @@ void cleanup_exited_pids(void) { } } -void apply_apps_groups_targets_inheritance(void) { +static void apply_apps_groups_targets_inheritance(void) { struct pid_stat *p = NULL; // children that do not have a target @@ -1881,16 +1918,13 @@ void apply_apps_groups_targets_inheritance(void) { fprintf(stderr, "apps.plugin: apply_apps_groups_targets_inheritance() made %d loops on the process tree\n", loops); } -long zero_all_targets(struct target *root) { +static long zero_all_targets(struct target *root) { struct target *w; long count = 0; for (w = root; w ; w = w->next) { count++; - if(w->fds) freez(w->fds); - w->fds = NULL; - w->minflt = 0; w->majflt = 0; w->utime = 0; @@ -1920,132 +1954,168 @@ long zero_all_targets(struct target *root) { w->io_storage_bytes_read = 0; w->io_storage_bytes_written = 0; // w->io_cancelled_write_bytes = 0; + + // zero file counters + if(w->target_fds) { + memset(w->target_fds, 0, sizeof(int) * w->target_fds_size); + w->openfiles = 0; + w->openpipes = 0; + w->opensockets = 0; + w->openinotifies = 0; + w->openeventfds = 0; + w->opentimerfds = 0; + w->opensignalfds = 0; + w->openeventpolls = 0; + w->openother = 0; + } } return count; } -void aggregate_pid_on_target(struct target *w, struct pid_stat *p, struct target *o) { - (void)o; +static inline void reallocate_target_fds(struct target *w) { + if(unlikely(!w)) + return; - if(unlikely(!w->fds)) - w->fds = callocz(sizeof(int), (size_t) all_files_size); - - if(likely(p->updated)) { - w->cutime += p->cutime; - w->cstime += p->cstime; - w->cgtime += p->cgtime; - w->cminflt += p->cminflt; - w->cmajflt += p->cmajflt; - - w->utime += p->utime; - w->stime += p->stime; - w->gtime += p->gtime; - w->minflt += p->minflt; - w->majflt += p->majflt; - - // w->rss += p->rss; - - w->statm_size += p->statm_size; - w->statm_resident += p->statm_resident; - w->statm_share += p->statm_share; - // w->statm_text += p->statm_text; - // w->statm_lib += p->statm_lib; - // w->statm_data += p->statm_data; - // w->statm_dirty += p->statm_dirty; - - w->io_logical_bytes_read += p->io_logical_bytes_read; - w->io_logical_bytes_written += p->io_logical_bytes_written; - // w->io_read_calls += p->io_read_calls; - // w->io_write_calls += p->io_write_calls; - w->io_storage_bytes_read += p->io_storage_bytes_read; - w->io_storage_bytes_written += p->io_storage_bytes_written; - // w->io_cancelled_write_bytes += p->io_cancelled_write_bytes; - - w->processes++; - w->num_threads += p->num_threads; - - if(likely(w->fds)) { - int c; - for(c = 0; c < p->fds_size ;c++) { - if(p->fds[c] == 0) continue; - - if(likely(p->fds[c] < all_files_size)) { - if(w->fds) w->fds[p->fds[c]]++; - } - else - error("Invalid fd number %d", p->fds[c]); - } - } + if(unlikely(!w->target_fds || w->target_fds_size < all_files_size)) { + w->target_fds = reallocz(w->target_fds, sizeof(int) * all_files_size); + memset(&w->target_fds[w->target_fds_size], 0, sizeof(int) * (all_files_size - w->target_fds_size)); + w->target_fds_size = all_files_size; + } +} + +static inline void aggregate_fd_on_target(int fd, struct target *w) { + if(unlikely(!w)) + return; - if(unlikely(debug || w->debug)) - fprintf(stderr, "apps.plugin: \taggregating '%s' pid %d on target '%s' utime=%llu, stime=%llu, gtime=%llu, cutime=%llu, cstime=%llu, cgtime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->comm, p->pid, w->name, p->utime, p->stime, p->gtime, p->cutime, p->cstime, p->cgtime, p->minflt, p->majflt, p->cminflt, p->cmajflt); + if(unlikely(w->target_fds[fd])) { + // it is already aggregated + // just increase its usage counter + w->target_fds[fd]++; + return; + } + + // increase its usage counter + // so that we will not add it again + w->target_fds[fd]++; + + switch(all_files[fd].type) { + case FILETYPE_FILE: + w->openfiles++; + break; + + case FILETYPE_PIPE: + w->openpipes++; + break; + + case FILETYPE_SOCKET: + w->opensockets++; + break; + + case FILETYPE_INOTIFY: + w->openinotifies++; + break; + + case FILETYPE_EVENTFD: + w->openeventfds++; + break; + + case FILETYPE_TIMERFD: + w->opentimerfds++; + break; + + case FILETYPE_SIGNALFD: + w->opensignalfds++; + break; + + case FILETYPE_EVENTPOLL: + w->openeventpolls++; + break; + + default: + w->openother++; + break; } } -void count_targets_fds(struct target *root) { - int c; - struct target *w; +static inline void aggregate_pid_fds_on_targets(struct pid_stat *p) { - for (w = root; w ; w = w->next) { - if(!w->fds) continue; - - w->openfiles = 0; - w->openpipes = 0; - w->opensockets = 0; - w->openinotifies = 0; - w->openeventfds = 0; - w->opentimerfds = 0; - w->opensignalfds = 0; - w->openeventpolls = 0; - w->openother = 0; - - for(c = 1; c < all_files_size ;c++) { - if(w->fds[c] > 0) - switch(all_files[c].type) { - case FILETYPE_FILE: - w->openfiles++; - break; - - case FILETYPE_PIPE: - w->openpipes++; - break; - - case FILETYPE_SOCKET: - w->opensockets++; - break; - - case FILETYPE_INOTIFY: - w->openinotifies++; - break; - - case FILETYPE_EVENTFD: - w->openeventfds++; - break; - - case FILETYPE_TIMERFD: - w->opentimerfds++; - break; - - case FILETYPE_SIGNALFD: - w->opensignalfds++; - break; - - case FILETYPE_EVENTPOLL: - w->openeventpolls++; - break; - - default: - w->openother++; - } - } + if(unlikely(!p->updated)) { + // the process is not running + return; + } + + struct target *w = p->target, *u = p->user_target, *g = p->group_target; + + reallocate_target_fds(w); + reallocate_target_fds(u); + reallocate_target_fds(g); + + int c, size = p->fds_size, *fds = p->fds; + for(c = 0; c < size ;c++) { + int fd = fds[c]; + + if(likely(fd <= 0 || fd >= all_files_size)) + continue; + + aggregate_fd_on_target(fd, w); + aggregate_fd_on_target(fd, u); + aggregate_fd_on_target(fd, g); + } +} + +static inline void aggregate_pid_on_target(struct target *w, struct pid_stat *p, struct target *o) { + (void)o; - freez(w->fds); - w->fds = NULL; + if(unlikely(!p->updated)) { + // the process is not running + return; + } + + if(unlikely(!w)) { + error("pid %d %s was left without a target!", p->pid, p->comm); + return; } + + w->cutime += p->cutime; + w->cstime += p->cstime; + w->cgtime += p->cgtime; + w->cminflt += p->cminflt; + w->cmajflt += p->cmajflt; + + w->utime += p->utime; + w->stime += p->stime; + w->gtime += p->gtime; + w->minflt += p->minflt; + w->majflt += p->majflt; + + // w->rss += p->rss; + + w->statm_size += p->statm_size; + w->statm_resident += p->statm_resident; + w->statm_share += p->statm_share; + // w->statm_text += p->statm_text; + // w->statm_lib += p->statm_lib; + // w->statm_data += p->statm_data; + // w->statm_dirty += p->statm_dirty; + + w->io_logical_bytes_read += p->io_logical_bytes_read; + w->io_logical_bytes_written += p->io_logical_bytes_written; + // w->io_read_calls += p->io_read_calls; + // w->io_write_calls += p->io_write_calls; + w->io_storage_bytes_read += p->io_storage_bytes_read; + w->io_storage_bytes_written += p->io_storage_bytes_written; + // w->io_cancelled_write_bytes += p->io_cancelled_write_bytes; + + w->processes++; + w->num_threads += p->num_threads; + + if(unlikely(debug || w->debug)) + fprintf(stderr, "apps.plugin: \taggregating '%s' pid %d on target '%s' utime=%llu, stime=%llu, gtime=%llu, cutime=%llu, cstime=%llu, cgtime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->comm, p->pid, w->name, p->utime, p->stime, p->gtime, p->cutime, p->cstime, p->cgtime, p->minflt, p->majflt, p->cminflt, p->cmajflt); } -void calculate_netdata_statistics(void) { +static void calculate_netdata_statistics(void) { + apply_apps_groups_targets_inheritance(); zero_all_targets(users_root_target); @@ -2056,19 +2126,18 @@ void calculate_netdata_statistics(void) { struct pid_stat *p = NULL; struct target *w = NULL, *o = NULL; - // concentrate everything on the apps_groups_targets + // concentrate everything on the targets for(p = root_of_pids; p ; p = p->next) { // -------------------------------------------------------------------- - // apps_groups targets - if(likely(p->target)) - aggregate_pid_on_target(p->target, p, NULL); - else - error("pid %d %s was left without a target!", p->pid, p->comm); + // apps_groups target + + aggregate_pid_on_target(p->target, p, NULL); // -------------------------------------------------------------------- - // user targets + // user target + o = p->user_target; if(likely(p->user_target && p->user_target->uid == p->uid)) w = p->user_target; @@ -2079,14 +2148,12 @@ void calculate_netdata_statistics(void) { w = p->user_target = get_users_target(p->uid); } - if(likely(w)) - aggregate_pid_on_target(w, p, o); - else - error("pid %d %s was left without a user target!", p->pid, p->comm); + aggregate_pid_on_target(w, p, o); // -------------------------------------------------------------------- - // group targets + // user group target + o = p->group_target; if(likely(p->group_target && p->group_target->gid == p->gid)) w = p->group_target; @@ -2097,16 +2164,15 @@ void calculate_netdata_statistics(void) { w = p->group_target = get_groups_target(p->gid); } - if(likely(w)) - aggregate_pid_on_target(w, p, o); - else - error("pid %d %s was left without a group target!", p->pid, p->comm); + aggregate_pid_on_target(w, p, o); - } - count_targets_fds(apps_groups_root_target); - count_targets_fds(users_root_target); - count_targets_fds(groups_root_target); + // -------------------------------------------------------------------- + // aggregate all file descriptors + + if(enable_file_charts) + aggregate_pid_fds_on_targets(p); + } cleanup_exited_pids(); } @@ -2114,72 +2180,58 @@ void calculate_netdata_statistics(void) { // ---------------------------------------------------------------------------- // update chart dimensions -BUFFER *output = NULL; int print_calculated_number(char *str, calculated_number value) { (void)str; (void)value; return 0; } static inline void send_BEGIN(const char *type, const char *id, unsigned long long usec) { - // fprintf(stdout, "BEGIN %s.%s %llu\n", type, id, usec); - buffer_strcat(output, "BEGIN "); - buffer_strcat(output, type); - buffer_strcat(output, "."); - buffer_strcat(output, id); - buffer_strcat(output, " "); - buffer_print_llu(output, usec); - buffer_strcat(output, "\n"); + fprintf(stdout, "BEGIN %s.%s %llu\n", type, id, usec); } static inline void send_SET(const char *name, unsigned long long value) { - // fprintf(stdout, "SET %s = %llu\n", name, value); - buffer_strcat(output, "SET "); - buffer_strcat(output, name); - buffer_strcat(output, " = "); - buffer_print_llu(output, value); - buffer_strcat(output, "\n"); + fprintf(stdout, "SET %s = %llu\n", name, value); } static inline void send_END(void) { - // fprintf(stdout, "END\n"); - buffer_strcat(output, "END\n"); + fprintf(stdout, "END\n"); } double utime_fix_ratio = 1.0, stime_fix_ratio = 1.0, gtime_fix_ratio = 1.0, cutime_fix_ratio = 1.0, cstime_fix_ratio = 1.0, cgtime_fix_ratio = 1.0; double minflt_fix_ratio = 1.0, majflt_fix_ratio = 1.0, cminflt_fix_ratio = 1.0, cmajflt_fix_ratio = 1.0; -unsigned long long send_resource_usage_to_netdata() { +static usec_t send_resource_usage_to_netdata() { static struct timeval last = { 0, 0 }; static struct rusage me_last; struct timeval now; struct rusage me; - unsigned long long usec; - unsigned long long cpuuser; - unsigned long long cpusyst; + usec_t usec; + usec_t cpuuser; + usec_t cpusyst; if(!last.tv_sec) { - gettimeofday(&last, NULL); + now_realtime_timeval(&last); getrusage(RUSAGE_SELF, &me_last); // the first time, give a zero to allow // netdata calibrate to the current time - // usec = update_every * 1000000ULL; + // usec = update_every * USEC_PER_SEC; usec = 0ULL; cpuuser = 0; cpusyst = 0; } else { - gettimeofday(&now, NULL); + now_realtime_timeval(&now); getrusage(RUSAGE_SELF, &me); - usec = usec_dt(&now, &last); - cpuuser = me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec; - cpusyst = me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec; + usec = dt_usec(&now, &last); + cpuuser = me.ru_utime.tv_sec * USEC_PER_SEC + me.ru_utime.tv_usec; + cpusyst = me.ru_stime.tv_sec * USEC_PER_SEC + me.ru_stime.tv_usec; memmove(&last, &now, sizeof(struct timeval)); memmove(&me_last, &me, sizeof(struct rusage)); } - buffer_sprintf(output, + fprintf(stdout, "BEGIN netdata.apps_cpu %llu\n" "SET user = %llu\n" "SET system = %llu\n" @@ -2214,7 +2266,7 @@ unsigned long long send_resource_usage_to_netdata() { ); if(include_exited_childs) - buffer_sprintf(output, + fprintf(stdout, "BEGIN netdata.apps_children_fix %llu\n" "SET cutime = %llu\n" "SET cstime = %llu\n" @@ -2233,7 +2285,7 @@ unsigned long long send_resource_usage_to_netdata() { return usec; } -void normalize_data(struct target *root) { +static void normalize_data(struct target *root) { struct target *w; // childs processing introduces spikes @@ -2379,7 +2431,7 @@ void normalize_data(struct target *root) { } } -void send_collected_data_to_netdata(struct target *root, const char *type, unsigned long long usec) { +static void send_collected_data_to_netdata(struct target *root, const char *type, usec_t usec) { struct target *w; send_BEGIN(type, "cpu", usec); @@ -2510,7 +2562,7 @@ void send_collected_data_to_netdata(struct target *root, const char *type, unsig // ---------------------------------------------------------------------------- // generate the charts -void send_charts_updates_to_netdata(struct target *root, const char *type, const char *title) +static void send_charts_updates_to_netdata(struct target *root, const char *type, const char *title) { struct target *w; int newly_added = 0; @@ -2530,112 +2582,112 @@ void send_charts_updates_to_netdata(struct target *root, const char *type, const // we have something new to show // update the charts - buffer_sprintf(output, "CHART %s.cpu '' '%s CPU Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu stacked 20001 %d\n", type, title, (processors * 100), processors, (processors>1)?"s":"", type, update_every); + fprintf(stdout, "CHART %s.cpu '' '%s CPU Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu stacked 20001 %d\n", type, title, (processors * 100), processors, (processors>1)?"s":"", type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu %s\n", w->name, hz * RATES_DETAIL / 100, w->hidden ? "hidden" : ""); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu %s\n", w->name, hz * RATES_DETAIL / 100, w->hidden ? "hidden" : ""); } - buffer_sprintf(output, "CHART %s.mem '' '%s Real Memory (w/o shared)' 'MB' mem %s.mem stacked 20003 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.mem '' '%s Real Memory (w/o shared)' 'MB' mem %s.mem stacked 20003 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute %ld %ld\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L); + fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L); } - buffer_sprintf(output, "CHART %s.vmem '' '%s Virtual Memory Size' 'MB' mem %s.vmem stacked 20004 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.vmem '' '%s Virtual Memory Size' 'MB' mem %s.vmem stacked 20004 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute %ld %ld\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L); + fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L); } - buffer_sprintf(output, "CHART %s.threads '' '%s Threads' 'threads' processes %s.threads stacked 20005 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.threads '' '%s Threads' 'threads' processes %s.threads stacked 20005 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 1\n", w->name); + fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name); } - buffer_sprintf(output, "CHART %s.processes '' '%s Processes' 'processes' processes %s.processes stacked 20004 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.processes '' '%s Processes' 'processes' processes %s.processes stacked 20004 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 1\n", w->name); + fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name); } - buffer_sprintf(output, "CHART %s.cpu_user '' '%s CPU User Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu_user stacked 20020 %d\n", type, title, (processors * 100), processors, (processors>1)?"s":"", type, update_every); + fprintf(stdout, "CHART %s.cpu_user '' '%s CPU User Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu_user stacked 20020 %d\n", type, title, (processors * 100), processors, (processors>1)?"s":"", type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, hz * RATES_DETAIL / 100LLU); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, hz * RATES_DETAIL / 100LLU); } - buffer_sprintf(output, "CHART %s.cpu_system '' '%s CPU System Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu_system stacked 20021 %d\n", type, title, (processors * 100), processors, (processors>1)?"s":"", type, update_every); + fprintf(stdout, "CHART %s.cpu_system '' '%s CPU System Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu_system stacked 20021 %d\n", type, title, (processors * 100), processors, (processors>1)?"s":"", type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, hz * RATES_DETAIL / 100LLU); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, hz * RATES_DETAIL / 100LLU); } if(show_guest_time) { - buffer_sprintf(output, "CHART %s.cpu_guest '' '%s CPU Guest Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu_system stacked 20022 %d\n", type, title, (processors * 100), processors, (processors > 1) ? "s" : "", type, update_every); + fprintf(stdout, "CHART %s.cpu_guest '' '%s CPU Guest Time (%d%% = %d core%s)' 'cpu time %%' cpu %s.cpu_system stacked 20022 %d\n", type, title, (processors * 100), processors, (processors > 1) ? "s" : "", type, update_every); for (w = root; w; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, hz * RATES_DETAIL / 100LLU); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, hz * RATES_DETAIL / 100LLU); } } - buffer_sprintf(output, "CHART %s.major_faults '' '%s Major Page Faults (swap read)' 'page faults/s' swap %s.major_faults stacked 20010 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.major_faults '' '%s Major Page Faults (swap read)' 'page faults/s' swap %s.major_faults stacked 20010 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL); } - buffer_sprintf(output, "CHART %s.minor_faults '' '%s Minor Page Faults' 'page faults/s' mem %s.minor_faults stacked 20011 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.minor_faults '' '%s Minor Page Faults' 'page faults/s' mem %s.minor_faults stacked 20011 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, RATES_DETAIL); } - buffer_sprintf(output, "CHART %s.lreads '' '%s Disk Logical Reads' 'kilobytes/s' disk %s.lreads stacked 20042 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.lreads '' '%s Disk Logical Reads' 'kilobytes/s' disk %s.lreads stacked 20042 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); } - buffer_sprintf(output, "CHART %s.lwrites '' '%s I/O Logical Writes' 'kilobytes/s' disk %s.lwrites stacked 20042 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.lwrites '' '%s I/O Logical Writes' 'kilobytes/s' disk %s.lwrites stacked 20042 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); } - buffer_sprintf(output, "CHART %s.preads '' '%s Disk Reads' 'kilobytes/s' disk %s.preads stacked 20002 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.preads '' '%s Disk Reads' 'kilobytes/s' disk %s.preads stacked 20002 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); } - buffer_sprintf(output, "CHART %s.pwrites '' '%s Disk Writes' 'kilobytes/s' disk %s.pwrites stacked 20002 %d\n", type, title, type, update_every); + fprintf(stdout, "CHART %s.pwrites '' '%s Disk Writes' 'kilobytes/s' disk %s.pwrites stacked 20002 %d\n", type, title, type, update_every); for (w = root; w ; w = w->next) { if(unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); + fprintf(stdout, "DIMENSION %s '' absolute 1 %llu\n", w->name, 1024LLU * RATES_DETAIL); } if(enable_file_charts) { - buffer_sprintf(output, "CHART %s.files '' '%s Open Files' 'open files' disk %s.files stacked 20050 %d\n", type, + fprintf(stdout, "CHART %s.files '' '%s Open Files' 'open files' disk %s.files stacked 20050 %d\n", type, title, type, update_every); for (w = root; w; w = w->next) { if (unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 1\n", w->name); + fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name); } - buffer_sprintf(output, "CHART %s.sockets '' '%s Open Sockets' 'open sockets' net %s.sockets stacked 20051 %d\n", + fprintf(stdout, "CHART %s.sockets '' '%s Open Sockets' 'open sockets' net %s.sockets stacked 20051 %d\n", type, title, type, update_every); for (w = root; w; w = w->next) { if (unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 1\n", w->name); + fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name); } - buffer_sprintf(output, "CHART %s.pipes '' '%s Pipes' 'open pipes' processes %s.pipes stacked 20053 %d\n", type, + fprintf(stdout, "CHART %s.pipes '' '%s Pipes' 'open pipes' processes %s.pipes stacked 20053 %d\n", type, title, type, update_every); for (w = root; w; w = w->next) { if (unlikely(w->exposed)) - buffer_sprintf(output, "DIMENSION %s '' absolute 1 1\n", w->name); + fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name); } } } @@ -2644,20 +2696,25 @@ void send_charts_updates_to_netdata(struct target *root, const char *type, const // ---------------------------------------------------------------------------- // parse command line arguments -void parse_args(int argc, char **argv) +static void parse_args(int argc, char **argv) { int i, freq = 0; char *name = NULL; for(i = 1; i < argc; i++) { if(!freq) { - int n = atoi(argv[i]); + int n = (int)str2l(argv[i]); if(n > 0) { freq = n; continue; } } + if(strcmp("version", argv[i]) == 0 || strcmp("-v", argv[i]) == 0) { + printf("apps.plugin %s\n", VERSION); + exit(0); + } + if(strcmp("debug", argv[i]) == 0) { debug = 1; // debug_flags = 0xffffffff; @@ -2694,35 +2751,52 @@ void parse_args(int argc, char **argv) continue; } + if(strcmp("no-users", argv[i]) == 0 || strcmp("without-users", argv[i]) == 0) { + enable_users_charts = 0; + continue; + } + + if(strcmp("no-groups", argv[i]) == 0 || strcmp("without-groups", argv[i]) == 0) { + enable_groups_charts = 0; + continue; + } + if(strcmp("-h", argv[i]) == 0 || strcmp("--help", argv[i]) == 0) { fprintf(stderr, - "apps.plugin\n" - "(C) 2016 Costa Tsaousis" - "GPL v3+\n" - "This program is a data collector plugin for netdata.\n" "\n" - "Valid command line options:\n" + " netdata apps.plugin %s\n" + " Copyright (C) 2016-2017 Costa Tsaousis <costa@tsaousis.gr>\n" + " Released under GNU Public License v3 or later.\n" + " All rights reserved.\n" + "\n" + " This program is a data collector plugin for netdata.\n" + "\n" + " Valid command line options:\n" "\n" - "SECONDS set the data collection frequency\n" + " SECONDS set the data collection frequency\n" "\n" - "debug enable debugging (lot of output)\n" + " debug enable debugging (lot of output)\n" "\n" - "with-childs\n" - "without-childs enable / disable aggregating exited\n" - " children resources into parents\n" - " (default is enabled)\n" + " with-childs\n" + " without-childs enable / disable aggregating exited\n" + " children resources into parents\n" + " (default is enabled)\n" "\n" - "with-guest\n" - "without-guest enable / disable reporting guest charts\n" - " (default is disabled)\n" + " with-guest\n" + " without-guest enable / disable reporting guest charts\n" + " (default is disabled)\n" "\n" - "with-files\n" - "without-files enable / disable reporting files, sockets, pipes\n" - " (default is enabled)\n" + " with-files\n" + " without-files enable / disable reporting files, sockets, pipes\n" + " (default is enabled)\n" "\n" - "NAME read apps_NAME.conf instead of\n" - " apps_groups.conf\n" - " (default NAME=groups)\n" + " NAME read apps_NAME.conf instead of\n" + " apps_groups.conf\n" + " (default NAME=groups)\n" + "\n" + " version print program version and exit\n" + "\n" + , VERSION ); exit(1); } @@ -2740,7 +2814,7 @@ void parse_args(int argc, char **argv) if(!name) name = "groups"; if(read_apps_groups_conf(name)) { - error("Cannot read process groups %s", name); + error("Cannot read process groups '%s/apps_%s.conf'. There are no internal defaults. Failing.", config_dir, name); exit(1); } } @@ -2786,8 +2860,7 @@ int main(int argc, char **argv) procfile_adaptive_initial_allocation = 1; - time_t started_t = time(NULL); - time_t current_t; + time_t started_t = now_realtime_sec(); get_system_HZ(); get_system_pid_max(); get_system_cpus(); @@ -2797,8 +2870,7 @@ int main(int argc, char **argv) all_pids_sortlist = callocz(sizeof(pid_t), (size_t)pid_max); all_pids = callocz(sizeof(struct pid_stat *), (size_t) pid_max); - output = buffer_create(1024); - buffer_sprintf(output, + fprintf(stdout, "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' apps.plugin netdata.apps_cpu stacked 140000 %1$d\n" "DIMENSION user '' incremental 1 1000\n" "DIMENSION system '' incremental 1 1000\n" @@ -2818,7 +2890,7 @@ int main(int argc, char **argv) ); if(include_exited_childs) - buffer_sprintf(output, + fprintf(stdout, "CHART netdata.apps_children_fix '' 'Apps Plugin Exited Children Normalization Ratios' 'percentage' apps.plugin netdata.apps_children_fix line 140003 %1$d\n" "DIMENSION cutime '' absolute 1 %2$llu\n" "DIMENSION cstime '' absolute 1 %2$llu\n" @@ -2829,22 +2901,23 @@ int main(int argc, char **argv) , RATES_DETAIL ); -#ifndef PROFILING_MODE - unsigned long long sunext = (time(NULL) - (time(NULL) % update_every) + update_every) * 1000000ULL; - unsigned long long sunow; -#endif /* PROFILING_MODE */ - + usec_t step = update_every * USEC_PER_SEC; global_iterations_counter = 1; for(;1; global_iterations_counter++) { -#ifndef PROFILING_MODE - // delay until it is our time to run - while((sunow = time_usec()) < sunext) - sleep_usec(sunext - sunow); - - // find the next time we need to run - while(time_usec() > sunext) - sunext += update_every * 1000000ULL; -#endif /* PROFILING_MODE */ + usec_t now = now_realtime_usec(); + usec_t next = now - (now % step) + step; + +#ifdef NETDATA_PROFILING +#warning "compiling for profiling" + static int profiling_count=0; + profiling_count++; + if(unlikely(profiling_count > 1000)) exit(0); +#else + while(now < next) { + sleep_usec(next - now); + now = now_realtime_usec(); + } +#endif if(!collect_data_for_all_processes_from_proc()) { error("Cannot collect /proc data for running processes. Disabling apps.plugin..."); @@ -2855,36 +2928,35 @@ int main(int argc, char **argv) calculate_netdata_statistics(); normalize_data(apps_groups_root_target); - unsigned long long dt = send_resource_usage_to_netdata(); + usec_t dt = send_resource_usage_to_netdata(); // this is smart enough to show only newly added apps, when needed send_charts_updates_to_netdata(apps_groups_root_target, "apps", "Apps"); - send_charts_updates_to_netdata(users_root_target, "users", "Users"); - send_charts_updates_to_netdata(groups_root_target, "groups", "User Groups"); + + if(likely(enable_users_charts)) + send_charts_updates_to_netdata(users_root_target, "users", "Users"); + + if(likely(enable_groups_charts)) + send_charts_updates_to_netdata(groups_root_target, "groups", "User Groups"); send_collected_data_to_netdata(apps_groups_root_target, "apps", dt); - send_collected_data_to_netdata(users_root_target, "users", dt); - send_collected_data_to_netdata(groups_root_target, "groups", dt); - show_guest_time_old = show_guest_time; + if(likely(enable_users_charts)) + send_collected_data_to_netdata(users_root_target, "users", dt); - //if(puts(buffer_tostring(output)) == EOF) - if(write(STDOUT_FILENO, buffer_tostring(output), buffer_strlen(output)) == -1) - fatal("Cannot send chart values to netdata."); + if(likely(enable_groups_charts)) + send_collected_data_to_netdata(groups_root_target, "groups", dt); - // fflush(stdout); - buffer_flush(output); + fflush(stdout); + + show_guest_time_old = show_guest_time; if(unlikely(debug)) fprintf(stderr, "apps.plugin: done Loop No %llu\n", global_iterations_counter); - current_t = time(NULL); + time_t current_t = now_realtime_sec(); -#ifndef PROFILING_MODE // restart check (14400 seconds) if(current_t - started_t > 14400) exit(0); -#else - if(current_t - started_t > 10) exit(0); -#endif /* PROFILING_MODE */ } } @@ -283,18 +283,30 @@ avl *avl_remove(avl_tree *tree, avl *item) { // --------------------------- // traversing -void avl_walker(avl *node, void (*callback)(void *)) { - if(node->avl_link[0]) - avl_walker(node->avl_link[0], callback); +int avl_walker(avl *node, int (*callback)(void *entry, void *data), void *data) { + int total = 0, ret = 0; - callback(node); + if(node->avl_link[0]) { + ret = avl_walker(node->avl_link[0], callback, data); + if(ret < 0) return ret; + total += ret; + } + + ret = callback(node, data); + if(ret < 0) return ret; + total += ret; - if(node->avl_link[1]) - avl_walker(node->avl_link[1], callback); + if(node->avl_link[1]) { + ret = avl_walker(node->avl_link[1], callback, data); + if (ret < 0) return ret; + total += ret; + } + + return total; } -void avl_traverse(avl_tree *t, void (*callback)(void *)) { - avl_walker(t->root, callback); +int avl_traverse(avl_tree *t, int (*callback)(void *entry, void *data), void *data) { + return avl_walker(t->root, callback, data); } // --------------------------- @@ -372,10 +384,12 @@ avl *avl_insert_lock(avl_tree_lock *t, avl *a) { return ret; } -void avl_traverse_lock(avl_tree_lock *t, void (*callback)(void *)) { +int avl_traverse_lock(avl_tree_lock *t, int (*callback)(void *entry, void *data), void *data) { + int ret; avl_read_lock(t); - avl_traverse(&t->avl_tree, callback); + ret = avl_traverse(&t->avl_tree, callback, data); avl_unlock(t); + return ret; } void avl_init(avl_tree *t, int (*compar)(void *a, void *b)) { @@ -56,16 +56,16 @@ typedef struct avl_tree_lock { * a is linked directly to the tree, so it has to * be properly allocated by the caller. */ -avl *avl_insert_lock(avl_tree_lock *t, avl *a); -avl *avl_insert(avl_tree *t, avl *a); +avl *avl_insert_lock(avl_tree_lock *t, avl *a) NEVERNULL WARNUNUSED; +avl *avl_insert(avl_tree *t, avl *a) NEVERNULL WARNUNUSED; /* Remove an element a from the AVL tree t * returns a pointer to the removed element * or NULL if an element equal to a is not found * (equal as returned by t->compar()) */ -avl *avl_remove_lock(avl_tree_lock *t, avl *a); -avl *avl_remove(avl_tree *t, avl *a); +avl *avl_remove_lock(avl_tree_lock *t, avl *a) WARNUNUSED; +avl *avl_remove(avl_tree *t, avl *a) WARNUNUSED; /* Find the element into the tree that equal to a * (equal as returned by t->compar()) @@ -80,7 +80,7 @@ void avl_init_lock(avl_tree_lock *t, int (*compar)(void *a, void *b)); void avl_init(avl_tree *t, int (*compar)(void *a, void *b)); -void avl_traverse_lock(avl_tree_lock *t, void (*callback)(void *)); -void avl_traverse(avl_tree *t, void (*callback)(void *)); +int avl_traverse_lock(avl_tree_lock *t, int (*callback)(void *entry, void *data), void *data); +int avl_traverse(avl_tree *t, int (*callback)(void *entry, void *data), void *data); #endif /* avl.h */ diff --git a/src/backends.c b/src/backends.c new file mode 100644 index 000000000..1272d0473 --- /dev/null +++ b/src/backends.c @@ -0,0 +1,549 @@ +#include "common.h" + +#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001 +#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002 +#define BACKEND_SOURCE_DATA_SUM 0x00000004 + +static inline calculated_number backend_calculate_value_from_stored_data(RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) { + time_t first_t = rrdset_first_entry_t(st); + time_t last_t = rrdset_last_entry_t(st); + + if(unlikely(before - after < st->update_every && after != after - after % st->update_every)) + // when st->update_every is bigger than the frequency we send data to backend + // skip the iterations that are not aligned to the database + return NAN; + + // align the time-frame + // for 'after' also skip the first value by adding st->update_every + after = after - after % st->update_every + st->update_every; + before = before - before % st->update_every; + + if(unlikely(after < first_t)) + after = first_t; + + if(unlikely(after > before)) + // this can happen when the st->update_every > before - after + before = after; + + if(unlikely(before > last_t)) + before = last_t; + + size_t counter = 0; + calculated_number sum = 0; + + long start_at_slot = rrdset_time2slot(st, before), + stop_at_slot = rrdset_time2slot(st, after), + slot, stop_now = 0; + + for(slot = start_at_slot; !stop_now ; slot--) { + if(unlikely(slot < 0)) slot = st->entries - 1; + if(unlikely(slot == stop_at_slot)) stop_now = 1; + + storage_number n = rd->values[slot]; + if(unlikely(!does_storage_number_exist(n))) continue; + + calculated_number value = unpack_storage_number(n); + sum += value; + counter++; + } + + if(unlikely(!counter)) + return NAN; + + if(unlikely(options & BACKEND_SOURCE_DATA_SUM)) + return sum; + + return sum / (calculated_number)counter; +} + +static inline int format_dimension_collected_graphite_plaintext(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) { + (void)host; + (void)after; + (void)before; + (void)options; + buffer_sprintf(b, "%s.%s.%s.%s " COLLECTED_NUMBER_FORMAT " %u\n", prefix, hostname, st->id, rd->id, rd->last_collected_value, (uint32_t)rd->last_collected_time.tv_sec); + return 1; +} + +static inline int format_dimension_stored_graphite_plaintext(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) { + (void)host; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options); + if(!isnan(value)) { + buffer_sprintf(b, "%s.%s.%s.%s " CALCULATED_NUMBER_FORMAT " %u\n", prefix, hostname, st->id, rd->id, value, (uint32_t) before); + return 1; + } + return 0; +} + +static inline int format_dimension_collected_opentsdb_telnet(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) { + (void)host; + (void)after; + (void)before; + (void)options; + buffer_sprintf(b, "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s\n", prefix, st->id, rd->id, (uint32_t)rd->last_collected_time.tv_sec, rd->last_collected_value, hostname); + return 1; +} + +static inline int format_dimension_stored_opentsdb_telnet(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) { + (void)host; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options); + if(!isnan(value)) { + buffer_sprintf(b, "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s\n", prefix, st->id, rd->id, (uint32_t) before, value, hostname); + return 1; + } + return 0; +} + +static inline int process_graphite_response(BUFFER *b) { + char sample[1024]; + const char *s = buffer_tostring(b); + char *d = sample, *e = &sample[sizeof(sample) - 1]; + + for(; *s && d < e ;s++) { + char c = *s; + if(unlikely(!isprint(c))) c = ' '; + *d++ = c; + } + *d = '\0'; + + info("Received %zu bytes from graphite backend. Ignoring them. Sample: '%s'", buffer_strlen(b), sample); + buffer_flush(b); + return 0; +} + +static inline int process_opentsdb_response(BUFFER *b) { + char sample[1024]; + const char *s = buffer_tostring(b); + char *d = sample, *e = &sample[sizeof(sample) - 1]; + + for(; *s && d < e ;s++) { + char c = *s; + if(unlikely(!isprint(c))) c = ' '; + *d++ = c; + } + *d = '\0'; + + info("Received %zu bytes from opentsdb backend. Ignoring them. Sample: '%s'", buffer_strlen(b), sample); + buffer_flush(b); + return 0; +} + +void *backends_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + + BUFFER *b = buffer_create(1), *response = buffer_create(1); + int (*backend_request_formatter)(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) = NULL; + int (*backend_response_checker)(BUFFER *b) = NULL; + + info("BACKEND thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + // ------------------------------------------------------------------------ + // collect configuration options + + struct timeval timeout = { + .tv_sec = 0, + .tv_usec = 0 + }; + int default_port = 0; + int sock = -1; + uint32_t options; + int enabled = config_get_boolean("backend", "enabled", 0); + const char *source = config_get("backend", "data source", "average"); + const char *type = config_get("backend", "type", "graphite"); + const char *destination = config_get("backend", "destination", "localhost"); + const char *prefix = config_get("backend", "prefix", "netdata"); + const char *hostname = config_get("backend", "hostname", localhost.hostname); + int frequency = (int)config_get_number("backend", "update every", 10); + int buffer_on_failures = (int)config_get_number("backend", "buffer on failures", 10); + long timeoutms = config_get_number("backend", "timeout ms", frequency * 2 * 1000); + + // ------------------------------------------------------------------------ + // validate configuration options + // and prepare for sending data to our backend + if(!enabled || frequency < 1) + goto cleanup; + + if(!strcmp(source, "as collected")) { + options = BACKEND_SOURCE_DATA_AS_COLLECTED; + } + else if(!strcmp(source, "average")) { + options = BACKEND_SOURCE_DATA_AVERAGE; + } + else if(!strcmp(source, "sum") || !strcmp(source, "volume")) { + options = BACKEND_SOURCE_DATA_SUM; + } + else { + error("Invalid data source method '%s' for backend given. Disabling backed.", source); + goto cleanup; + } + + if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) { + default_port = 2003; + if(options == BACKEND_SOURCE_DATA_AS_COLLECTED) + backend_request_formatter = format_dimension_collected_graphite_plaintext; + else + backend_request_formatter = format_dimension_stored_graphite_plaintext; + + backend_response_checker = process_graphite_response; + } + else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) { + default_port = 4242; + if(options == BACKEND_SOURCE_DATA_AS_COLLECTED) + backend_request_formatter = format_dimension_collected_opentsdb_telnet; + else + backend_request_formatter = format_dimension_stored_opentsdb_telnet; + + backend_response_checker = process_opentsdb_response; + } + else { + error("Unknown backend type '%s'", type); + goto cleanup; + } + + if(backend_request_formatter == NULL || backend_response_checker == NULL) { + error("backend is misconfigured - disabling it."); + goto cleanup; + } + + if(timeoutms < 1) { + error("BACKED invalid timeout %ld ms given. Assuming %d ms.", timeoutms, frequency * 2 * 1000); + timeoutms = frequency * 2 * 1000; + } + timeout.tv_sec = (timeoutms * 1000) / 1000000; + timeout.tv_usec = (timeoutms * 1000) % 1000000; + + // ------------------------------------------------------------------------ + // prepare the charts for monitoring the backend + + struct rusage thread; + + collected_number + chart_buffered_metrics = 0, + chart_lost_metrics = 0, + chart_sent_metrics = 0, + chart_buffered_bytes = 0, + chart_received_bytes = 0, + chart_sent_bytes = 0, + chart_receptions = 0, + chart_transmission_successes = 0, + chart_transmission_failures = 0, + chart_data_lost_events = 0, + chart_lost_bytes = 0, + chart_backend_reconnects = 0, + chart_backend_latency = 0; + + RRDSET *chart_metrics = rrdset_find("netdata.backend_metrics"); + if(!chart_metrics) { + chart_metrics = rrdset_create("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, frequency, RRDSET_TYPE_LINE); + rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + + RRDSET *chart_bytes = rrdset_find("netdata.backend_bytes"); + if(!chart_bytes) { + chart_bytes = rrdset_create("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, frequency, RRDSET_TYPE_AREA); + rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + + RRDSET *chart_ops = rrdset_find("netdata.backend_ops"); + if(!chart_ops) { + chart_ops = rrdset_create("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, frequency, RRDSET_TYPE_LINE); + rrddim_add(chart_ops, "write", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(chart_ops, "discard", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(chart_ops, "failure", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(chart_ops, "read", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + + /* + * this is misleading - we can only measure the time we need to send data + * this time is not related to the time required for the data to travel to + * the backend database and the time that server needed to process them + * + * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html + * + RRDSET *chart_latency = rrdset_find("netdata.backend_latency"); + if(!chart_latency) { + chart_latency = rrdset_create("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, frequency, RRDSET_TYPE_AREA); + rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + */ + + RRDSET *chart_rusage = rrdset_find("netdata.backend_thread_cpu"); + if(!chart_rusage) { + chart_rusage = rrdset_create("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, frequency, RRDSET_TYPE_STACKED); + rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); + } + + // ------------------------------------------------------------------------ + // prepare the backend main loop + + info("BACKEND configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, frequency, hostname, prefix); + + usec_t step_ut = frequency * USEC_PER_SEC; + usec_t random_ut = now_realtime_usec() % (step_ut / 2); + time_t before = (time_t)((now_realtime_usec() - step_ut) / USEC_PER_SEC); + time_t after = before; + int failures = 0; + + for(;;) { + // ------------------------------------------------------------------------ + // wait for the next iteration point + + usec_t now_ut = now_realtime_usec(); + usec_t next_ut = now_ut - (now_ut % step_ut) + step_ut; + before = (time_t)(next_ut / USEC_PER_SEC); + + // add a little delay (1/4 of the step) plus some randomness + next_ut += (step_ut / 4) + random_ut; + + while(now_ut < next_ut) { + sleep_usec(next_ut - now_ut); + now_ut = now_realtime_usec(); + } + + // ------------------------------------------------------------------------ + // add to the buffer the data we need to send to the backend + + RRDSET *st; + int pthreadoldcancelstate; + + if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &pthreadoldcancelstate) != 0)) + error("Cannot set pthread cancel state to DISABLE."); + + rrdhost_rdlock(&localhost); + for(st = localhost.rrdset_root; st ;st = st->next) { + pthread_rwlock_rdlock(&st->rwlock); + + RRDDIM *rd; + for(rd = st->dimensions; rd ;rd = rd->next) { + if(rd->last_collected_time.tv_sec >= after) + chart_buffered_metrics += backend_request_formatter(b, prefix, &localhost, hostname, st, rd, after, before, options); + } + + pthread_rwlock_unlock(&st->rwlock); + } + rrdhost_unlock(&localhost); + + if(unlikely(pthread_setcancelstate(pthreadoldcancelstate, NULL) != 0)) + error("Cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate); + + // ------------------------------------------------------------------------ + + chart_buffered_bytes = (collected_number)buffer_strlen(b); + + // reset the monitoring chart counters + chart_received_bytes = + chart_sent_bytes = + chart_sent_metrics = + chart_lost_metrics = + chart_transmission_successes = + chart_transmission_failures = + chart_data_lost_events = + chart_lost_bytes = + chart_backend_reconnects = + chart_backend_latency = 0; + + if(unlikely(netdata_exit)) break; + + //fprintf(stderr, "\nBACKEND BEGIN:\n%s\nBACKEND END\n", buffer_tostring(b)); // FIXME + //fprintf(stderr, "after = %lu, before = %lu\n", after, before); + + // ------------------------------------------------------------------------ + // if we are connected, receive a response, without blocking + + if(likely(sock != -1)) { + errno = 0; + + // loop through to collect all data + while(sock != -1 && errno != EWOULDBLOCK) { + buffer_need_bytes(response, 4096); + + ssize_t r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT); + if(likely(r > 0)) { + // we received some data + response->len += r; + chart_received_bytes += r; + chart_receptions++; + } + else if(r == 0) { + error("Backend '%s' closed the socket", destination); + close(sock); + sock = -1; + } + else { + // failed to receive data + if(errno != EAGAIN && errno != EWOULDBLOCK) { + error("Cannot receive data from backend '%s'.", destination); + } + } + } + + // if we received data, process them + if(buffer_strlen(response)) + backend_response_checker(response); + } + + // ------------------------------------------------------------------------ + // if we are not connected, connect to a backend server + + if(unlikely(sock == -1)) { + usec_t start_ut = now_realtime_usec(); + const char *s = destination; + while(*s) { + const char *e = s; + + // skip separators, moving both s(tart) and e(nd) + while(isspace(*e) || *e == ',') s = ++e; + + // move e(nd) to the first separator + while(*e && !isspace(*e) && *e != ',') e++; + + // is there anything? + if(!*s || s == e) break; + + char buf[e - s + 1]; + strncpyz(buf, s, e - s); + chart_backend_reconnects++; + sock = connect_to(buf, default_port, &timeout); + if(sock != -1) break; + s = e; + } + chart_backend_latency += now_realtime_usec() - start_ut; + } + + if(unlikely(netdata_exit)) break; + + // ------------------------------------------------------------------------ + // if we are connected, send our buffer to the backend server + + if(likely(sock != -1)) { + size_t len = buffer_strlen(b); + usec_t start_ut = now_realtime_usec(); + int flags = 0; +#ifdef MSG_NOSIGNAL + flags += MSG_NOSIGNAL; +#endif + + ssize_t written = send(sock, buffer_tostring(b), len, flags); + chart_backend_latency += now_realtime_usec() - start_ut; + if(written != -1 && (size_t)written == len) { + // we sent the data successfully + chart_transmission_successes++; + chart_sent_bytes += written; + chart_sent_metrics = chart_buffered_metrics; + + // reset the failures count + failures = 0; + + // empty the buffer + buffer_flush(b); + } + else { + // oops! we couldn't send (all or some of the) data + error("Failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written); + chart_transmission_failures++; + + if(written != -1) + chart_sent_bytes += written; + + // increment the counter we check for data loss + failures++; + + // close the socket - we will re-open it next time + close(sock); + sock = -1; + } + + // either the buffer is empty + // or is holding the data we couldn't send + // so, make sure the next iteration will continue + // from where we are now + after = before; + } + else { + error("Failed to update database backend '%s'", destination); + chart_transmission_failures++; + + // increment the counter we check for data loss + failures++; + } + + if(failures > buffer_on_failures) { + // too bad! we are going to lose data + chart_lost_bytes += buffer_strlen(b); + error("Reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination); + buffer_flush(b); + failures = 0; + chart_data_lost_events++; + chart_lost_metrics = chart_buffered_metrics; + } + + if(unlikely(netdata_exit)) break; + + // ------------------------------------------------------------------------ + // update the monitoring charts + + if(chart_ops->counter_done) rrdset_next(chart_ops); + rrddim_set(chart_ops, "read", chart_receptions); + rrddim_set(chart_ops, "write", chart_transmission_successes); + rrddim_set(chart_ops, "discard", chart_data_lost_events); + rrddim_set(chart_ops, "failure", chart_transmission_failures); + rrddim_set(chart_ops, "reconnect", chart_backend_reconnects); + rrdset_done(chart_ops); + + if(chart_metrics->counter_done) rrdset_next(chart_metrics); + rrddim_set(chart_metrics, "buffered", chart_buffered_metrics); + rrddim_set(chart_metrics, "lost", chart_lost_metrics); + rrddim_set(chart_metrics, "sent", chart_sent_metrics); + rrdset_done(chart_metrics); + + if(chart_bytes->counter_done) rrdset_next(chart_bytes); + rrddim_set(chart_bytes, "buffered", chart_buffered_bytes); + rrddim_set(chart_bytes, "lost", chart_lost_bytes); + rrddim_set(chart_bytes, "sent", chart_sent_bytes); + rrddim_set(chart_bytes, "received", chart_received_bytes); + rrdset_done(chart_bytes); + + /* + if(chart_latency->counter_done) rrdset_next(chart_latency); + rrddim_set(chart_latency, "latency", chart_backend_latency); + rrdset_done(chart_latency); + */ + + getrusage(RUSAGE_THREAD, &thread); + if(chart_rusage->counter_done) rrdset_next(chart_rusage); + rrddim_set(chart_rusage, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set(chart_rusage, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrdset_done(chart_rusage); + + if(likely(buffer_strlen(b) == 0)) + chart_buffered_metrics = 0; + + if(unlikely(netdata_exit)) break; + } + +cleanup: + if(sock != -1) + close(sock); + + buffer_free(b); + buffer_free(response); + + info("BACKEND thread exiting"); + + static_thread->enabled = 0; + pthread_exit(NULL); + return NULL; +} diff --git a/src/backends.h b/src/backends.h new file mode 100644 index 000000000..61122a1d0 --- /dev/null +++ b/src/backends.h @@ -0,0 +1,6 @@ +#ifndef NETDATA_BACKENDS_H +#define NETDATA_BACKENDS_H 1 + +void *backends_main(void *ptr); + +#endif /* NETDATA_BACKENDS_H */ diff --git a/src/clocks.c b/src/clocks.c new file mode 100644 index 000000000..c90a07c2f --- /dev/null +++ b/src/clocks.c @@ -0,0 +1,73 @@ +#include "common.h" + +#ifndef HAVE_CLOCK_GETTIME +inline int clock_gettime(clockid_t clk_id, struct timespec *ts) { + struct timeval tv; + if(unlikely(gettimeofday(&tv, NULL) == -1)) + return -1; + ts->tv_sec = tv.tv_sec; + ts->tv_nsec = tv.tv_usec * NSEC_PER_USEC; + return 0; +} +#endif + +inline time_t now_realtime_sec(void) { + struct timespec ts; + if(unlikely(clock_gettime(CLOCK_REALTIME, &ts) == -1)) + return 0; + return ts.tv_sec; +} + +inline int now_realtime_timeval(struct timeval *tv) { + struct timespec ts; + if(unlikely(clock_gettime(CLOCK_REALTIME, &ts) == -1)) + return -1; + tv->tv_sec = ts.tv_sec; + tv->tv_usec = ts.tv_nsec / NSEC_PER_USEC; + return 0; +} + +inline usec_t now_realtime_usec(void) { + struct timespec ts; + if(unlikely(clock_gettime(CLOCK_REALTIME, &ts) == -1)) + return 0; + return (usec_t)ts.tv_sec * USEC_PER_SEC + ts.tv_nsec / NSEC_PER_USEC; +} + +inline time_t now_monotonic_sec(void) { + struct timespec ts; + if(unlikely(clock_gettime(CLOCK_MONOTONIC, &ts) == -1)) + return 0; + return ts.tv_sec; +} + +inline usec_t now_monotonic_usec(void) { + struct timespec ts; + if(unlikely(clock_gettime(CLOCK_MONOTONIC, &ts) == -1)) + return 0; + return (usec_t)ts.tv_sec * USEC_PER_SEC + ts.tv_nsec / NSEC_PER_USEC; +} + +inline time_t now_boottime_sec(void) { + struct timespec ts; + if(unlikely(clock_gettime(CLOCK_BOOTTIME, &ts) == -1)) + return 0; + return ts.tv_sec; +} + +inline usec_t now_boottime_usec(void) { + struct timespec ts; + if(unlikely(clock_gettime(CLOCK_BOOTTIME, &ts) == -1)) + return 0; + return (usec_t)ts.tv_sec * USEC_PER_SEC + ts.tv_nsec / NSEC_PER_USEC; +} + +inline usec_t timeval_usec(struct timeval *tv) { + return (usec_t)tv->tv_sec * USEC_PER_SEC + tv->tv_usec; +} + +inline usec_t dt_usec(struct timeval *now, struct timeval *old) { + usec_t ts1 = timeval_usec(now); + usec_t ts2 = timeval_usec(old); + return (ts1 > ts2) ? (ts1 - ts2) : (ts2 - ts1); +} diff --git a/src/clocks.h b/src/clocks.h new file mode 100644 index 000000000..c1b8e7017 --- /dev/null +++ b/src/clocks.h @@ -0,0 +1,92 @@ +#ifndef NETDATA_CLOCKS_H +#define NETDATA_CLOCKS_H 1 + +#ifndef HAVE_STRUCT_TIMESPEC +struct timespec { + time_t tv_sec; /* seconds */ + long tv_nsec; /* nanoseconds */ +}; +#endif + +#ifndef HAVE_CLOCKID_T +typedef int clockid_t; +#endif + +#ifndef HAVE_CLOCK_GETTIME +int clock_gettime(clockid_t clk_id, struct timespec *ts); +#endif + +/* Linux value is as good as any other */ +#ifndef CLOCK_REALTIME +#define CLOCK_REALTIME 0 +#endif + +#ifndef CLOCK_MONOTONIC +/* fallback to CLOCK_REALTIME if not available */ +#define CLOCK_MONOTONIC CLOCK_REALTIME +#endif + +#ifndef CLOCK_BOOTTIME +/* fallback to CLOCK_MONOTONIC if not available */ +#define CLOCK_BOOTTIME CLOCK_MONOTONIC +#else +#ifdef HAVE_CLOCK_GETTIME +#define CLOCK_BOOTTIME_IS_AVAILABLE 1 // required for /proc/uptime +#endif +#endif + +typedef unsigned long long usec_t; + +#define NSEC_PER_SEC 1000000000ULL +#define NSEC_PER_MSEC 1000000ULL +#define NSEC_PER_USEC 1000ULL +#define USEC_PER_SEC 1000000ULL + +#ifndef HAVE_CLOCK_GETTIME +/* Fallback function for POSIX.1-2001 clock_gettime() function. + * + * We use a realtime clock from gettimeofday(), this will + * make systems without clock_gettime() support sensitive + * to time jumps or hibernation/suspend side effects. + */ +extern int clock_gettime(clockid_t clk_id, struct timespec *ts); +#endif + +/* Fills struct timeval with time since EPOCH from real-time clock (i.e. wall-clock). + * - Hibernation/suspend time is included + * - adjtime()/NTP adjustments affect this clock + * Return 0 on succes, -1 else with errno set appropriately. + */ +extern int now_realtime_timeval(struct timeval *tv); + +/* Returns time since EPOCH from real-time clock (i.e. wall-clock). + * - Hibernation/suspend time is included + * - adjtime()/NTP adjustments affect this clock + */ +extern time_t now_realtime_sec(void); +extern usec_t now_realtime_usec(void); + +/* Returns time from monotonic clock if available, real-time clock else. + * If monotonic clock is available: + * - hibernation/suspend time is not included + * - adjtime()/NTP adjusments affect this clock + * If monotonic clock is not available, this fallbacks to now_realtime(). + */ +extern time_t now_monotonic_sec(void); +extern usec_t now_monotonic_usec(void); + +/* Returns time from boottime clock if available, + * monotonic clock else if available, real-time clock else. + * If boottime clock is available: + * - hibernation/suspend time is included + * - adjtime()/NTP adjusments affect this clock + * If boottime clock is not available, this fallbacks to now_monotonic(). + * If monotonic clock is not available, this fallbacks to now_realtime(). + */ +extern time_t now_boottime_sec(void); +extern usec_t now_boottime_usec(void); + +extern usec_t timeval_usec(struct timeval *ts); +extern usec_t dt_usec(struct timeval *now, struct timeval *old); + +#endif /* NETDATA_CLOCKS_H */ diff --git a/src/common.c b/src/common.c index e1925ff5e..42f3d8d15 100644 --- a/src/common.c +++ b/src/common.c @@ -1,5 +1,13 @@ #include "common.h" +#ifdef __APPLE__ +#define INHERIT_NONE 0 +#endif /* __APPLE__ */ +#if defined(__FreeBSD__) || defined(__APPLE__) +# define O_NOATIME 0 +# define MADV_DONTFORK INHERIT_NONE +#endif /* __FreeBSD__ || __APPLE__*/ + char *global_host_prefix = ""; int enable_ksm = 1; @@ -192,27 +200,22 @@ void freez(void *ptr) { free(ptr); } -// ---------------------------------------------------------------------------- -// time functions - -inline unsigned long long timeval_usec(struct timeval *tv) { - return tv->tv_sec * 1000000ULL + tv->tv_usec; -} +void json_escape_string(char *dst, const char *src, size_t size) { + const char *t; + char *d = dst, *e = &dst[size - 1]; -// time(NULL) in nanoseconds -inline unsigned long long time_usec(void) { - struct timeval now; - gettimeofday(&now, NULL); - return timeval_usec(&now); -} + for(t = src; *t && d < e ;t++) { + if(unlikely(*t == '\\' || *t == '"')) { + if(unlikely(d + 1 >= e)) break; + *d++ = '\\'; + } + *d++ = *t; + } -inline unsigned long long usec_dt(struct timeval *now, struct timeval *old) { - unsigned long long tv1 = timeval_usec(now); - unsigned long long tv2 = timeval_usec(old); - return (tv1 > tv2) ? (tv1 - tv2) : (tv2 - tv1); + *d = '\0'; } -int sleep_usec(unsigned long long usec) { +int sleep_usec(usec_t usec) { #ifndef NETDATA_WITH_USLEEP // we expect microseconds (1.000.000 per second) @@ -224,7 +227,7 @@ int sleep_usec(unsigned long long usec) { while (nanosleep(&req, &rem) == -1) { if (likely(errno == EINTR)) { - info("nanosleep() interrupted (while sleeping for %llu microseconds).", usec); + debug(D_SYSTEM, "nanosleep() interrupted (while sleeping for %llu microseconds).", usec); req.tv_sec = rem.tv_sec; req.tv_nsec = rem.tv_nsec; } else { @@ -804,7 +807,7 @@ uint32_t simple_hash(const char *name) } */ - +/* // http://isthe.com/chongo/tech/comp/fnv/#FNV-1a uint32_t simple_hash(const char *name) { unsigned char *s = (unsigned char *) name; @@ -839,6 +842,7 @@ uint32_t simple_uhash(const char *name) { } return hval; } +*/ /* // http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx @@ -864,11 +868,9 @@ uint32_t simple_hash(const char *name) { */ void strreverse(char *begin, char *end) { - char aux; - while (end > begin) { // clearer code. - aux = *end; + char aux = *end; *end-- = *begin; *begin++ = aux; } @@ -905,11 +907,10 @@ void *mymmap(const char *filename, size_t size, int flags, int ksm) { #ifdef MADV_MERGEABLE static int log_madvise_2 = 1, log_madvise_3 = 1; #endif - int fd; void *mem = NULL; errno = 0; - fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664); + int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664); if (fd != -1) { if (lseek(fd, size, SEEK_SET) == (off_t) size) { if (write(fd, "", 1) == 1) { @@ -1028,7 +1029,15 @@ int fd_is_valid(int fd) { } pid_t gettid(void) { +#ifdef __FreeBSD__ + return (pid_t)pthread_getthreadid_np(); +#elif defined(__APPLE__) + uint64_t curthreadid; + pthread_threadid_np(NULL, &curthreadid); + return (pid_t)curthreadid; +#else return (pid_t)syscall(SYS_gettid); +#endif /* __FreeBSD__, __APPLE__*/ } char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) { @@ -1090,14 +1099,24 @@ int snprintfz(char *dst, size_t n, const char *fmt, ...) { int processors = 1; long get_system_cpus(void) { - procfile *ff = NULL; - processors = 1; + #ifdef __APPLE__ + int32_t tmp_processors; + + if (unlikely(GETSYSCTL("hw.logicalcpu", tmp_processors))) { + error("Assuming system has %d processors.", processors); + } else { + processors = tmp_processors; + } + + return processors; + #else + char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s/proc/stat", global_host_prefix); - ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); + procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); if(!ff) { error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors); return processors; @@ -1121,17 +1140,24 @@ long get_system_cpus(void) { procfile_close(ff); - info("System has %d processors.", processors); + debug(D_SYSTEM, "System has %d processors.", processors); return processors; + + #endif /* __APPLE__ */ } pid_t pid_max = 32768; pid_t get_system_pid_max(void) { - procfile *ff = NULL; + #ifdef __APPLE__ + // As we currently do not know a solution to query pid_max from the os + // we use the number defined in bsd/sys/proc_internal.h in XNU sources + pid_max = 99999; + return pid_max; + #else char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", global_host_prefix); - ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); + procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); if(!ff) { error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max); return pid_max; @@ -1143,7 +1169,7 @@ pid_t get_system_pid_max(void) { return pid_max; } - pid_max = (pid_t)atoi(procfile_lineword(ff, 0, 0)); + pid_max = (pid_t)str2i(procfile_lineword(ff, 0, 0)); if(!pid_max) { procfile_close(ff); pid_max = 32768; @@ -1152,8 +1178,10 @@ pid_t get_system_pid_max(void) { } procfile_close(ff); - info("System supports %d pids.", pid_max); + debug(D_SYSTEM, "System supports %d pids.", pid_max); return pid_max; + + #endif /* __APPLE__ */ } unsigned int hz; @@ -1161,25 +1189,23 @@ void get_system_HZ(void) { long ticks; if ((ticks = sysconf(_SC_CLK_TCK)) == -1) { - perror("sysconf"); + error("Cannot get system clock ticks"); } hz = (unsigned int) ticks; } -int read_single_number_file(const char *filename, unsigned long long *result) { - char buffer[1024 + 1]; - - int fd = open(filename, O_RDONLY, 0666); - if(unlikely(fd == -1)) return 1; - - ssize_t r = read(fd, buffer, 1024); - if(unlikely(r == -1)) { - close(fd); - return 2; - } - - close(fd); - *result = strtoull(buffer, NULL, 0); - return 0; +/* +// poor man cycle counting +static unsigned long tsc; +void begin_tsc(void) { + unsigned long a, d; + asm volatile ("cpuid\nrdtsc" : "=a" (a), "=d" (d) : "0" (0) : "ebx", "ecx"); + tsc = ((unsigned long)d << 32) | (unsigned long)a; } +unsigned long end_tsc(void) { + unsigned long a, d; + asm volatile ("rdtscp" : "=a" (a), "=d" (d) : : "ecx"); + return (((unsigned long)d << 32) | (unsigned long)a) - tsc; +} +*/ diff --git a/src/common.h b/src/common.h index 9ffa8c8bc..e38e95b48 100644 --- a/src/common.h +++ b/src/common.h @@ -5,6 +5,9 @@ #include <config.h> #endif +// ---------------------------------------------------------------------------- +// system include files for all netdata C programs + /* select the memory allocator, based on autoconf findings */ #if defined(ENABLE_JEMALLOC) @@ -20,26 +23,32 @@ #else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ +#if !(defined(__FreeBSD__) || defined(__APPLE__)) #include <malloc.h> +#endif /* __FreeBSD__ || __APPLE__ */ #endif #include <pthread.h> #include <errno.h> - #include <stdio.h> #include <stdlib.h> #include <stdarg.h> #include <stddef.h> - #include <ctype.h> #include <string.h> #include <strings.h> - #include <arpa/inet.h> -#include <netinet/in.h> #include <netinet/tcp.h> +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#ifdef HAVE_RESOLV_H +#include <resolv.h> +#endif + #include <dirent.h> #include <fcntl.h> #include <getopt.h> @@ -47,12 +56,21 @@ #include <pwd.h> #include <locale.h> +#ifdef HAVE_NETDB_H #include <netdb.h> +#endif + +#include <net/if.h> + #include <poll.h> #include <signal.h> #include <syslog.h> #include <sys/mman.h> + +#if !(defined(__FreeBSD__) || defined(__APPLE__)) #include <sys/prctl.h> +#endif /* __FreeBSD__ || __APPLE__*/ + #include <sys/resource.h> #include <sys/socket.h> #include <sys/stat.h> @@ -65,6 +83,18 @@ #include <unistd.h> #include <uuid/uuid.h> +// #1408 +#ifdef MAJOR_IN_MKDEV +#include <sys/mkdev.h> +#endif +#ifdef MAJOR_IN_SYSMACROS +#include <sys/sysmacros.h> +#endif + +/* +#include <mntent.h> +*/ + #ifdef STORAGE_WITH_MATH #include <math.h> #endif @@ -79,6 +109,9 @@ #include <zlib.h> #endif +// ---------------------------------------------------------------------------- +// netdata common definitions + #if (SIZEOF_VOID_P == 8) #define ENVIRONMENT64 #elif (SIZEOF_VOID_P == 4) @@ -91,7 +124,49 @@ #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) #endif // __GNUC__ +#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL +#define NEVERNULL __attribute__((returns_nonnull)) +#else +#define NEVERNULL +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC +#define MALLOCLIKE __attribute__((malloc)) +#else +#define MALLOCLIKE +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_FORMAT +#define PRINTFLIKE(f, a) __attribute__ ((format(__printf__, f, a))) +#else +#define PRINTFLIKE(f, a) +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN +#define NORETURN __attribute__ ((noreturn)) +#else +#define NORETURN +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT +#define WARNUNUSED __attribute__ ((warn_unused_result)) +#else +#define WARNUNUSED +#endif + +#ifdef abs +#undef abs +#endif +#define abs(x) ((x < 0)? -x : x) + +#define GUID_LEN 36 + +// ---------------------------------------------------------------------------- +// netdata include files + +#include "simple_pattern.h" #include "avl.h" +#include "clocks.h" #include "log.h" #include "global_statistics.h" #include "storage_number.h" @@ -107,47 +182,44 @@ #include "plugin_checks.h" #include "plugin_idlejitter.h" #include "plugin_nfacct.h" + +#if defined(__FreeBSD__) +#include "plugin_freebsd.h" +#elif defined(__APPLE__) +#include "plugin_macos.h" +#else #include "plugin_proc.h" +#include "plugin_proc_diskspace.h" +#endif /* __FreeBSD__, __APPLE__*/ + #include "plugin_tc.h" #include "plugins_d.h" - +#include "socket.h" #include "eval.h" #include "health.h" - #include "rrd.h" #include "rrd2json.h" - #include "web_client.h" #include "web_server.h" - #include "registry.h" #include "daemon.h" #include "main.h" #include "unit_test.h" - -#ifdef abs -#undef abs -#endif -#define abs(x) ((x < 0)? -x : x) - -extern unsigned long long usec_dt(struct timeval *now, struct timeval *old); -extern unsigned long long timeval_usec(struct timeval *tv); - -// #define usec_dt(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec))) +#include "ipc.h" +#include "backends.h" +#include "inlined.h" +#include "adaptive_resortable_list.h" extern void netdata_fix_chart_id(char *s); extern void netdata_fix_chart_name(char *s); -extern uint32_t simple_hash(const char *name); -extern uint32_t simple_uhash(const char *name); - extern void strreverse(char* begin, char* end); extern char *mystrsep(char **ptr, char *s); extern char *trim(char *s); extern char *strncpyz(char *dst, const char *src, size_t n); extern int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args); -extern int snprintfz(char *dst, size_t n, const char *fmt, ...) __attribute__ (( format (printf, 3, 4))); +extern int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4); // memory allocation functions that handle failures #ifdef NETDATA_LOG_ALLOCATIONS @@ -163,13 +235,15 @@ extern void *mallocz_int(const char *file, const char *function, const unsigned extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size); extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr); #else -extern char *strdupz(const char *s); -extern void *callocz(size_t nmemb, size_t size); -extern void *mallocz(size_t size); -extern void *reallocz(void *ptr, size_t size); +extern char *strdupz(const char *s) MALLOCLIKE NEVERNULL; +extern void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL; +extern void *mallocz(size_t size) MALLOCLIKE NEVERNULL; +extern void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL; extern void freez(void *ptr); #endif +extern void json_escape_string(char *dst, const char *src, size_t size); + extern void *mymmap(const char *filename, size_t size, int flags, int ksm); extern int savememory(const char *filename, void *mem, size_t size); @@ -180,8 +254,7 @@ extern int enable_ksm; extern pid_t gettid(void); -extern unsigned long long time_usec(void); -extern int sleep_usec(unsigned long long usec); +extern int sleep_usec(usec_t usec); extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len); @@ -203,6 +276,4 @@ extern void get_system_HZ(void); #endif #endif -extern int read_single_number_file(const char *filename, unsigned long long *result); - #endif /* NETDATA_COMMON_H */ diff --git a/src/daemon.c b/src/daemon.c index 1c34405d8..4fd8ca5e5 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -58,6 +58,21 @@ static void chown_open_file(int fd, uid_t uid, gid_t gid) { } } +void create_needed_dir(const char *dir, uid_t uid, gid_t gid) +{ + // attempt to create the directory + if(mkdir(dir, 0755) == 0) { + // we created it + + // chown it to match the required user + if(chown(dir, uid, gid) == -1) + error("Cannot chown directory '%s' to %u:%u", dir, (unsigned int)uid, (unsigned int)gid); + } + else if(errno != EEXIST) + // log an error only if the directory does not exist + error("Cannot create directory '%s'", dir); +} + int become_user(const char *username, int pid_fd) { struct passwd *pw = getpwnam(username); @@ -69,6 +84,14 @@ int become_user(const char *username, int pid_fd) uid_t uid = pw->pw_uid; gid_t gid = pw->pw_gid; + create_needed_dir(CACHE_DIR, uid, gid); + create_needed_dir(VARLIB_DIR, uid, gid); + + if(pidfile[0]) { + if(chown(pidfile, uid, gid) == -1) + error("Cannot chown '%s' to %u:%u", pidfile, (unsigned int)uid, (unsigned int)gid); + } + int ngroups = (int)sysconf(_SC_NGROUPS_MAX); gid_t *supplementary_groups = NULL; if(ngroups) { @@ -91,16 +114,23 @@ int become_user(const char *username, int pid_fd) error("Cannot set supplementary groups for user '%s'", username); freez(supplementary_groups); - supplementary_groups = NULL; ngroups = 0; } +#ifdef __APPLE__ + if(setregid(gid, gid) != 0) { +#else if(setresgid(gid, gid, gid) != 0) { +#endif /* __APPLE__ */ error("Cannot switch to user's %s group (gid: %u).", username, gid); return -1; } +#ifdef __APPLE__ + if(setreuid(uid, uid) != 0) { +#else if(setresuid(uid, uid, uid) != 0) { +#endif /* __APPLE__ */ error("Cannot switch to user %s (uid: %u).", username, uid); return -1; } @@ -138,7 +168,7 @@ void oom_score_adj(int score) { if(!done) error("Cannot adjust my Out-Of-Memory score to %d.", score); else - info("Adjusted my Out-Of-Memory score to %d.", score); + debug(D_SYSTEM, "Adjusted my Out-Of-Memory score to %d.", score); } int sched_setscheduler_idle(void) { @@ -151,7 +181,7 @@ int sched_setscheduler_idle(void) { if(i != 0) error("Cannot adjust my scheduling priority to IDLE."); else - info("Adjusted my scheduling priority to IDLE."); + debug(D_SYSTEM, "Adjusted my scheduling priority to IDLE."); return i; #else @@ -206,7 +236,7 @@ int become_daemon(int dont_fork, const char *user) } // Set new file permissions - umask(0002); + umask(0007); // adjust my Out-Of-Memory score oom_score_adj(1000); @@ -214,20 +244,22 @@ int become_daemon(int dont_fork, const char *user) // never become a problem if(sched_setscheduler_idle() != 0) { if(nice(19) == -1) error("Cannot lower my CPU priority."); - else info("Set my nice value to 19."); + else debug(D_SYSTEM, "Set my nice value to 19."); } if(user && *user) { if(become_user(user, pidfd) != 0) { error("Cannot become user '%s'. Continuing as we are.", user); } - else info("Successfully became user '%s'.", user); + else debug(D_SYSTEM, "Successfully became user '%s'.", user); + } + else { + create_needed_dir(CACHE_DIR, getuid(), getgid()); + create_needed_dir(VARLIB_DIR, getuid(), getgid()); } - if(pidfd != -1) { + if(pidfd != -1) close(pidfd); - pidfd = -1; - } return(0); } diff --git a/src/dictionary.c b/src/dictionary.c index 91d3b45f1..fb9efeedb 100644 --- a/src/dictionary.c +++ b/src/dictionary.c @@ -59,9 +59,6 @@ static int name_value_compare(void* a, void* b) { else return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name); } -#define dictionary_name_value_index_add_nolock(dict, nv) do { NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(dict); avl_insert(&((dict)->values_index), (avl *)(nv)); } while(0) -#define dictionary_name_value_index_del_nolock(dict, nv) do { NETDATA_DICTIONARY_STATS_DELETES_PLUS1(dict); avl_remove(&(dict->values_index), (avl *)(nv)); } while(0) - static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *dict, const char *name, uint32_t hash) { NAME_VALUE tmp; tmp.hash = (hash)?hash:simple_hash(name); @@ -95,7 +92,10 @@ static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const c } // index it - dictionary_name_value_index_add_nolock(dict, nv); + NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(dict); + if(unlikely(avl_insert(&((dict)->values_index), (avl *)(nv)) != (avl *)nv)) + error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary."); + NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(dict); return nv; @@ -104,7 +104,9 @@ static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const c static void dictionary_name_value_destroy_nolock(DICTIONARY *dict, NAME_VALUE *nv) { debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name); - dictionary_name_value_index_del_nolock(dict, nv); + NETDATA_DICTIONARY_STATS_DELETES_PLUS1(dict); + if(unlikely(avl_remove(&(dict->values_index), (avl *)(nv)) != (avl *)nv)) + error("dictionary: INTERNAL ERROR: dictionary invalid removal of node."); NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(dict); diff --git a/src/eval.c b/src/eval.c index 397b43bd7..122959ce4 100644 --- a/src/eval.c +++ b/src/eval.c @@ -59,20 +59,6 @@ static inline void print_parsed_as_constant(BUFFER *out, calculated_number n); // ---------------------------------------------------------------------------- // evaluation of expressions -static inline calculated_number eval_check_number(calculated_number n, int *error) { - if(unlikely(isnan(n))) { - *error = EVAL_ERROR_VALUE_IS_NAN; - return 0; - } - - if(unlikely(isinf(n))) { - *error = EVAL_ERROR_VALUE_IS_INFINITE; - return 0; - } - - return n; -} - static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) { static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0; calculated_number n; @@ -116,7 +102,7 @@ static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABL } if(unlikely(v->hash == now_hash && !strcmp(v->name, "now"))) { - n = time(NULL); + n = now_realtime_sec(); buffer_strcat(exp->error_msg, "[ $now = "); print_parsed_as_constant(exp->error_msg, n); buffer_strcat(exp->error_msg, " ] "); @@ -187,7 +173,7 @@ static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABL } *error = EVAL_ERROR_UNKNOWN_VARIABLE; - buffer_sprintf(exp->error_msg, "unknown variable '%s'", v->name); + buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", v->name); return 0; } @@ -213,7 +199,6 @@ static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, break; } - // return eval_check_number(n, error); return n; } @@ -362,7 +347,6 @@ static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, i calculated_number n = operators[op->operator].eval(exp, op, error); - // return eval_check_number(n, error); return n; } @@ -1047,9 +1031,7 @@ static inline EVAL_NODE *parse_rest_of_expression(const char **string, int *erro // high level function to parse an expression or a sub-expression static inline EVAL_NODE *parse_full_expression(const char **string, int *error) { - EVAL_NODE *op1 = NULL; - - op1 = parse_one_full_operand(string, error); + EVAL_NODE *op1 = parse_one_full_operand(string, error); if(!op1) { *error = EVAL_ERROR_MISSING_OPERAND; return NULL; @@ -1067,8 +1049,19 @@ int expression_evaluate(EVAL_EXPRESSION *exp) { buffer_reset(exp->error_msg); exp->result = eval_node(exp, (EVAL_NODE *)exp->nodes, &exp->error); - if(exp->error == EVAL_ERROR_OK) - exp->result = eval_check_number(exp->result, &exp->error); + if(unlikely(isnan(exp->result))) { + if(exp->error == EVAL_ERROR_OK) + exp->error = EVAL_ERROR_VALUE_IS_NAN; + } + else if(unlikely(isinf(exp->result))) { + if(exp->error == EVAL_ERROR_OK) + exp->error = EVAL_ERROR_VALUE_IS_INFINITE; + } + else if(unlikely(exp->error == EVAL_ERROR_UNKNOWN_VARIABLE)) { + // although there is an unknown variable + // the expression was evaluated successfully + exp->error = EVAL_ERROR_OK; + } if(exp->error != EVAL_ERROR_OK) { exp->result = NAN; @@ -1086,7 +1079,6 @@ int expression_evaluate(EVAL_EXPRESSION *exp) { EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error) { const char *s = string; int err = EVAL_ERROR_OK; - unsigned long pos = 0; EVAL_NODE *op = parse_full_expression(&s, &err); @@ -1102,7 +1094,7 @@ EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, in if (error) *error = err; if(!op) { - pos = s - string + 1; + unsigned long pos = s - string + 1; error("failed to parse expression '%s': %s at character %lu (i.e.: '%s').", string, expression_strerror(err), pos, s); return NULL; } diff --git a/src/freebsd_sysctl.c b/src/freebsd_sysctl.c new file mode 100644 index 000000000..9400089db --- /dev/null +++ b/src/freebsd_sysctl.c @@ -0,0 +1,2207 @@ +#include "common.h" + +// NEEDED BY: struct vmtotal, struct vmmeter +#include <sys/vmmeter.h> +// NEEDED BY: struct devstat +#include <sys/devicestat.h> +// NEEDED BY: struct xswdev +#include <vm/vm_param.h> +// NEEDED BY: struct semid_kernel, struct shmid_kernel, struct msqid_kernel +#define _KERNEL +#include <sys/sem.h> +#include <sys/shm.h> +#include <sys/msg.h> +#undef _KERNEL +// NEEDED BY: struct sysctl_netisr_workstream, struct sysctl_netisr_work +#include <net/netisr.h> +// NEEDED BY: struct ifaddrs, getifaddrs() +#include <net/if.h> +#include <ifaddrs.h> +// NEEDED BY do_tcp... +#include <netinet/tcp_var.h> +#include <netinet/tcp_fsm.h> +// NEEDED BY do_udp..., do_ip... +#include <netinet/ip_var.h> +// NEEDED BY do_udp... +#include <netinet/udp.h> +#include <netinet/udp_var.h> +// NEEDED BY do_icmp... +#include <netinet/ip.h> +#include <netinet/ip_icmp.h> +#include <netinet/icmp_var.h> +// NEEDED BY do_ip6... +#include <netinet6/ip6_var.h> +// NEEDED BY do_icmp6... +#include <netinet/icmp6.h> +// NEEDED BY do_space, do_inodes +#include <sys/mount.h> +// NEEDED BY do_uptime +#include <time.h> + +#define KILO_FACTOR 1024 +#define MEGA_FACTOR 1048576 // 1024 * 1024 +#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024 + +// NEEDED BY: do_disk_io +#define RRD_TYPE_DISK "disk" + +// FreeBSD calculates load averages once every 5 seconds +#define MIN_LOADAVG_UPDATE_EVERY 5 + +// NEEDED BY: do_bandwidth +#define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s) + +int do_freebsd_sysctl(int update_every, usec_t dt) { + (void)dt; + + static int do_cpu = -1, do_cpu_cores = -1, do_interrupts = -1, do_context = -1, do_forks = -1, do_processes = -1, + do_loadavg = -1, do_all_processes = -1, do_disk_io = -1, do_swap = -1, do_ram = -1, do_swapio = -1, + do_pgfaults = -1, do_committed = -1, do_ipc_semaphores = -1, do_ipc_shared_mem = -1, do_ipc_msg_queues = -1, + do_dev_intr = -1, do_soft_intr = -1, do_netisr = -1, do_netisr_per_core = -1, do_bandwidth = -1, + do_tcp_sockets = -1, do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, + do_ecn = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, + do_udp_packets = -1, do_udp_errors = -1, do_icmp_packets = -1, do_icmpmsg = -1, + do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1, + do_ip6_packets = -1, do_ip6_fragsout = -1, do_ip6_fragsin = -1, do_ip6_errors = -1, + do_icmp6 = -1, do_icmp6_redir = -1, do_icmp6_errors = -1, do_icmp6_echos = -1, do_icmp6_router = -1, + do_icmp6_neighbor = -1, do_icmp6_types = -1, do_space = -1, do_inodes = -1, do_uptime = -1; + + if (unlikely(do_cpu == -1)) { + do_cpu = config_get_boolean("plugin:freebsd:sysctl", "cpu utilization", 1); + do_cpu_cores = config_get_boolean("plugin:freebsd:sysctl", "per cpu core utilization", 1); + do_interrupts = config_get_boolean("plugin:freebsd:sysctl", "cpu interrupts", 1); + do_dev_intr = config_get_boolean("plugin:freebsd:sysctl", "device interrupts", 1); + do_soft_intr = config_get_boolean("plugin:freebsd:sysctl", "software interrupts", 1); + do_context = config_get_boolean("plugin:freebsd:sysctl", "context switches", 1); + do_forks = config_get_boolean("plugin:freebsd:sysctl", "processes started", 1); + do_processes = config_get_boolean("plugin:freebsd:sysctl", "processes running", 1); + do_loadavg = config_get_boolean("plugin:freebsd:sysctl", "enable load average", 1); + do_all_processes = config_get_boolean("plugin:freebsd:sysctl", "enable total processes", 1); + do_disk_io = config_get_boolean("plugin:freebsd:sysctl", "stats for all disks", 1); + do_swap = config_get_boolean("plugin:freebsd:sysctl", "system swap", 1); + do_ram = config_get_boolean("plugin:freebsd:sysctl", "system ram", 1); + do_swapio = config_get_boolean("plugin:freebsd:sysctl", "swap i/o", 1); + do_pgfaults = config_get_boolean("plugin:freebsd:sysctl", "memory page faults", 1); + do_committed = config_get_boolean("plugin:freebsd:sysctl", "committed memory", 1); + do_ipc_semaphores = config_get_boolean("plugin:freebsd:sysctl", "ipc semaphores", 1); + do_ipc_shared_mem = config_get_boolean("plugin:freebsd:sysctl", "ipc shared memory", 1); + do_ipc_msg_queues = config_get_boolean("plugin:freebsd:sysctl", "ipc message queues", 1); + do_netisr = config_get_boolean("plugin:freebsd:sysctl", "netisr", 1); + do_netisr_per_core = config_get_boolean("plugin:freebsd:sysctl", "netisr per core", 1); + do_bandwidth = config_get_boolean("plugin:freebsd:sysctl", "bandwidth", 1); + do_tcp_sockets = config_get_boolean("plugin:freebsd:sysctl", "ipv4 TCP connections", 1); + do_tcp_packets = config_get_boolean("plugin:freebsd:sysctl", "ipv4 TCP packets", 1); + do_tcp_errors = config_get_boolean("plugin:freebsd:sysctl", "ipv4 TCP errors", 1); + do_tcp_handshake = config_get_boolean("plugin:freebsd:sysctl", "ipv4 TCP handshake issues", 1); + do_ecn = config_get_boolean_ondemand("plugin:freebsd:sysctl", "ECN packets", CONFIG_ONDEMAND_ONDEMAND); + do_tcpext_syscookies = config_get_boolean_ondemand("plugin:freebsd:sysctl", "TCP SYN cookies", CONFIG_ONDEMAND_ONDEMAND); + do_tcpext_ofo = config_get_boolean_ondemand("plugin:freebsd:sysctl", "TCP out-of-order queue", CONFIG_ONDEMAND_ONDEMAND); + do_tcpext_connaborts = config_get_boolean_ondemand("plugin:freebsd:sysctl", "TCP connection aborts", CONFIG_ONDEMAND_ONDEMAND); + do_udp_packets = config_get_boolean("plugin:freebsd:sysctl", "ipv4 UDP packets", 1); + do_udp_errors = config_get_boolean("plugin:freebsd:sysctl", "ipv4 UDP errors", 1); + do_icmp_packets = config_get_boolean("plugin:freebsd:sysctl", "ipv4 ICMP packets", 1); + do_icmpmsg = config_get_boolean("plugin:freebsd:sysctl", "ipv4 ICMP messages", 1); + do_ip_packets = config_get_boolean("plugin:freebsd:sysctl", "ipv4 packets", 1); + do_ip_fragsout = config_get_boolean("plugin:freebsd:sysctl", "ipv4 fragments sent", 1); + do_ip_fragsin = config_get_boolean("plugin:freebsd:sysctl", "ipv4 fragments assembly", 1); + do_ip_errors = config_get_boolean("plugin:freebsd:sysctl", "ipv4 errors", 1); + do_ip6_packets = config_get_boolean_ondemand("plugin:freebsd:sysctl", "ipv6 packets", CONFIG_ONDEMAND_ONDEMAND); + do_ip6_fragsout = config_get_boolean_ondemand("plugin:freebsd:sysctl", "ipv6 fragments sent", CONFIG_ONDEMAND_ONDEMAND); + do_ip6_fragsin = config_get_boolean_ondemand("plugin:freebsd:sysctl", "ipv6 fragments assembly", CONFIG_ONDEMAND_ONDEMAND); + do_ip6_errors = config_get_boolean_ondemand("plugin:freebsd:sysctl", "ipv6 errors", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6 = config_get_boolean_ondemand("plugin:freebsd:sysctl", "icmp", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_redir = config_get_boolean_ondemand("plugin:freebsd:sysctl", "icmp redirects", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_errors = config_get_boolean_ondemand("plugin:freebsd:sysctl", "icmp errors", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_echos = config_get_boolean_ondemand("plugin:freebsd:sysctl", "icmp echos", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_router = config_get_boolean_ondemand("plugin:freebsd:sysctl", "icmp router", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_neighbor = config_get_boolean_ondemand("plugin:freebsd:sysctl", "icmp neighbor", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_types = config_get_boolean_ondemand("plugin:freebsd:sysctl", "icmp types", CONFIG_ONDEMAND_ONDEMAND); + do_space = config_get_boolean("plugin:freebsd:sysctl", "space usage for all disks", 1); + do_inodes = config_get_boolean("plugin:freebsd:sysctl", "inodes usage for all disks", 1); + do_uptime = config_get_boolean("plugin:macos:sysctl", "system uptime", 1); + } + + RRDSET *st; + RRDDIM *rd; + + int system_pagesize = getpagesize(); // wouldn't it be better to get value directly from hw.pagesize? + int i, n; + void *p; + int common_error = 0; + size_t size; + char title[4096 + 1]; + + // NEEDED BY: do_loadavg + static usec_t last_loadavg_usec = 0; + struct loadavg sysload; + + // NEEDED BY: do_cpu, do_cpu_cores + long cp_time[CPUSTATES]; + + // NEEDED BY: du_cpu_cores, do_netisr, do_netisr_per_core + int ncpus; + + // NEEDED BY: do_cpu_cores + static long *pcpu_cp_time = NULL; + char cpuid[8]; // no more than 4 digits expected + + // NEEDED BY: do_all_processes, do_processes + struct vmtotal vmtotal_data; + + // NEEDED BY: do_context, do_forks + u_int u_int_data; + + // NEEDED BY: do_interrupts + size_t intrcnt_size; + unsigned long nintr = 0; + static unsigned long *intrcnt = NULL; + static char *intrnames = NULL; + unsigned long long totalintr = 0; + + // NEEDED BY: do_disk_io + #define BINTIME_SCALE 5.42101086242752217003726400434970855712890625e-17 // this is 1000/2^64 + int numdevs; + static void *devstat_data = NULL; + struct devstat *dstat; + char disk[DEVSTAT_NAME_LEN + 10 + 1]; // 10 - maximum number of digits for int + struct cur_dstat { + collected_number duration_read_ms; + collected_number duration_write_ms; + collected_number busy_time_ms; + } cur_dstat; + struct prev_dstat { + collected_number bytes_read; + collected_number bytes_write; + collected_number operations_read; + collected_number operations_write; + collected_number duration_read_ms; + collected_number duration_write_ms; + collected_number busy_time_ms; + } prev_dstat; + + // NEEDED BY: do_swap + size_t mibsize; + int mib[3]; // CTL_MAXNAME = 24 maximum mib components (sysctl.h) + struct xswdev xsw; + struct total_xsw { + collected_number bytes_used; + collected_number bytes_total; + } total_xsw = {0, 0}; + + // NEEDED BY: do_swapio, do_ram + struct vmmeter vmmeter_data; + + // NEEDED BY: do_ram + int vfs_bufspace_count; + + // NEEDED BY: do_ipc_semaphores + struct ipc_sem { + int semmni; + collected_number sets; + collected_number semaphores; + } ipc_sem = {0, 0, 0}; + static struct semid_kernel *ipc_sem_data = NULL; + + // NEEDED BY: do_ipc_shared_mem + struct ipc_shm { + u_long shmmni; + collected_number segs; + collected_number segsize; + } ipc_shm = {0, 0, 0}; + static struct shmid_kernel *ipc_shm_data = NULL; + + // NEEDED BY: do_ipc_msg_queues + struct ipc_msq { + int msgmni; + collected_number queues; + collected_number messages; + collected_number usedsize; + collected_number allocsize; + } ipc_msq = {0, 0, 0, 0, 0}; + static struct msqid_kernel *ipc_msq_data = NULL; + + // NEEDED BY: do_netisr, do_netisr_per_core + size_t netisr_workstream_size; + size_t netisr_work_size; + unsigned long num_netisr_workstreams = 0, num_netisr_works = 0; + static struct sysctl_netisr_workstream *netisr_workstream = NULL; + static struct sysctl_netisr_work *netisr_work = NULL; + static struct netisr_stats { + collected_number dispatched; + collected_number hybrid_dispatched; + collected_number qdrops; + collected_number queued; + } *netisr_stats = NULL; + char netstat_cpuid[21]; // no more than 4 digits expected + + // NEEDED BY: do_bandwidth + struct ifaddrs *ifa, *ifap; + struct iftot { + u_long ift_ibytes; + u_long ift_obytes; + } iftot = {0, 0}; + + // NEEDED BY: do_tcp... + struct tcpstat tcpstat; + uint64_t tcps_states[TCP_NSTATES]; + + // NEEDED BY: do_udp... + struct udpstat udpstat; + + // NEEDED BY: do_icmp... + struct icmpstat icmpstat; + struct icmp_total { + u_long msgs_in; + u_long msgs_out; + } icmp_total = {0, 0}; + + // NEEDED BY: do_ip... + struct ipstat ipstat; + + // NEEDED BY: do_ip6... + struct ip6stat ip6stat; + + // NEEDED BY: do_icmp6... + struct icmp6stat icmp6stat; + struct icmp6_total { + u_long msgs_in; + u_long msgs_out; + } icmp6_total = {0, 0}; + + // NEEDED BY: do_space, do_inodes + struct statfs *mntbuf; + int mntsize; + char mntonname[MNAMELEN + 1]; + + // NEEDED BY: do_uptime + struct timespec boot_time, cur_time; + + // -------------------------------------------------------------------- + + if (last_loadavg_usec <= dt) { + if (likely(do_loadavg)) { + if (unlikely(GETSYSCTL("vm.loadavg", sysload))) { + do_loadavg = 0; + error("DISABLED: system.load"); + } else { + + st = rrdset_find_bytype("system", "load"); + if (unlikely(!st)) { + st = rrdset_create("system", "load", NULL, "load", NULL, "System Load Average", "load", 100, (update_every < MIN_LOADAVG_UPDATE_EVERY) ? MIN_LOADAVG_UPDATE_EVERY : update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "load1", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "load5", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "load15", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "load1", (collected_number) ((double)sysload.ldavg[0] / sysload.fscale * 1000)); + rrddim_set(st, "load5", (collected_number) ((double)sysload.ldavg[1] / sysload.fscale * 1000)); + rrddim_set(st, "load15", (collected_number) ((double)sysload.ldavg[2] / sysload.fscale * 1000)); + rrdset_done(st); + } + } + + last_loadavg_usec = st->update_every * USEC_PER_SEC; + } + else last_loadavg_usec -= dt; + + // -------------------------------------------------------------------- + + if (likely(do_all_processes | do_processes | do_committed)) { + if (unlikely(GETSYSCTL("vm.vmtotal", vmtotal_data))) { + do_all_processes = 0; + error("DISABLED: system.active_processes"); + do_processes = 0; + error("DISABLED: system.processes"); + do_committed = 0; + error("DISABLED: mem.committed"); + } else { + if (likely(do_all_processes)) { + + st = rrdset_find_bytype("system", "active_processes"); + if (unlikely(!st)) { + st = rrdset_create("system", "active_processes", NULL, "processes", NULL, "System Active Processes", "processes", 750, update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "active", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "active", (vmtotal_data.t_rq + vmtotal_data.t_dw + vmtotal_data.t_pw + vmtotal_data.t_sl + vmtotal_data.t_sw)); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_processes)) { + + st = rrdset_find_bytype("system", "processes"); + if (unlikely(!st)) { + st = rrdset_create("system", "processes", NULL, "processes", NULL, "System Processes", "processes", 600, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "running", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "blocked", NULL, -1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "running", vmtotal_data.t_rq); + rrddim_set(st, "blocked", (vmtotal_data.t_dw + vmtotal_data.t_pw)); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_committed)) { + st = rrdset_find("mem.committed"); + if (unlikely(!st)) { + st = rrdset_create("mem", "committed", NULL, "system", NULL, "Committed (Allocated) Memory", "MB", 5000, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "Committed_AS", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "Committed_AS", vmtotal_data.t_rm); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_cpu)) { + if (unlikely(CPUSTATES != 5)) { + error("FREEBSD: There are %d CPU states (5 was expected)", CPUSTATES); + do_cpu = 0; + error("DISABLED: system.cpu"); + } else { + if (unlikely(GETSYSCTL("kern.cp_time", cp_time))) { + do_cpu = 0; + error("DISABLED: system.cpu"); + } else { + + st = rrdset_find_bytype("system", "cpu"); + if (unlikely(!st)) { + st = rrdset_create("system", "cpu", NULL, "cpu", "system.cpu", "Total CPU utilization", "percentage", 100, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "user", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "nice", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "system", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "interrupt", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "idle", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_hide(st, "idle"); + } + else rrdset_next(st); + + rrddim_set(st, "user", cp_time[0]); + rrddim_set(st, "nice", cp_time[1]); + rrddim_set(st, "system", cp_time[2]); + rrddim_set(st, "interrupt", cp_time[3]); + rrddim_set(st, "idle", cp_time[4]); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_cpu_cores)) { + if (unlikely(CPUSTATES != 5)) { + error("FREEBSD: There are %d CPU states (5 was expected)", CPUSTATES); + do_cpu_cores = 0; + error("DISABLED: cpu.cpuXX"); + } else { + if (unlikely(GETSYSCTL("kern.smp.cpus", ncpus))) { + do_cpu_cores = 0; + error("DISABLED: cpu.cpuXX"); + } else { + pcpu_cp_time = reallocz(pcpu_cp_time, sizeof(cp_time) * ncpus); + + for (i = 0; i < ncpus; i++) { + if (unlikely(getsysctl("kern.cp_times", pcpu_cp_time, sizeof(cp_time) * ncpus))) { + do_cpu_cores = 0; + error("DISABLED: cpu.cpuXX"); + break; + } + if (unlikely(ncpus > 9999)) { + error("FREEBSD: There are more than 4 digits in cpu cores number"); + do_cpu_cores = 0; + error("DISABLED: cpu.cpuXX"); + break; + } + snprintfz(cpuid, 8, "cpu%d", i); + + st = rrdset_find_bytype("cpu", cpuid); + if (unlikely(!st)) { + st = rrdset_create("cpu", cpuid, NULL, "utilization", "cpu.cpu", "Core utilization", "percentage", 1000, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "user", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "nice", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "system", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "interrupt", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "idle", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_hide(st, "idle"); + } + else rrdset_next(st); + + rrddim_set(st, "user", pcpu_cp_time[i * 5 + 0]); + rrddim_set(st, "nice", pcpu_cp_time[i * 5 + 1]); + rrddim_set(st, "system", pcpu_cp_time[i * 5 + 2]); + rrddim_set(st, "interrupt", pcpu_cp_time[i * 5 + 3]); + rrddim_set(st, "idle", pcpu_cp_time[i * 5 + 4]); + rrdset_done(st); + } + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_interrupts)) { + if (unlikely(sysctlbyname("hw.intrcnt", NULL, &intrcnt_size, NULL, 0) == -1)) { + error("FREEBSD: sysctl(hw.intrcnt...) failed: %s", strerror(errno)); + do_interrupts = 0; + error("DISABLED: system.intr"); + } else { + nintr = intrcnt_size / sizeof(u_long); + intrcnt = reallocz(intrcnt, nintr * sizeof(u_long)); + if (unlikely(getsysctl("hw.intrcnt", intrcnt, nintr * sizeof(u_long)))){ + do_interrupts = 0; + error("DISABLED: system.intr"); + } else { + for (i = 0; i < nintr; i++) + totalintr += intrcnt[i]; + + st = rrdset_find_bytype("system", "intr"); + if (unlikely(!st)) { + st = rrdset_create("system", "intr", NULL, "interrupts", NULL, "Total Hardware Interrupts", "interrupts/s", 900, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "interrupts", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "interrupts", totalintr); + rrdset_done(st); + + // -------------------------------------------------------------------- + + size = nintr * (MAXCOMLEN +1); + intrnames = reallocz(intrnames, size); + if (unlikely(getsysctl("hw.intrnames", intrnames, size))) { + do_interrupts = 0; + error("DISABLED: system.intr"); + } else { + st = rrdset_find_bytype("system", "interrupts"); + if (unlikely(!st)) + st = rrdset_create("system", "interrupts", NULL, "interrupts", NULL, "System interrupts", "interrupts/s", + 1000, update_every, RRDSET_TYPE_STACKED); + else + rrdset_next(st); + + for (i = 0; i < nintr; i++) { + p = intrnames + i * (MAXCOMLEN + 1); + if (unlikely((intrcnt[i] != 0) && (*(char*)p != 0))) { + rd = rrddim_find(st, p); + if (unlikely(!rd)) + rd = rrddim_add(st, p, NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_set_by_pointer(st, rd, intrcnt[i]); + } + } + rrdset_done(st); + } + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_dev_intr)) { + if (unlikely(GETSYSCTL("vm.stats.sys.v_intr", u_int_data))) { + do_dev_intr = 0; + error("DISABLED: system.dev_intr"); + } else { + + st = rrdset_find_bytype("system", "dev_intr"); + if (unlikely(!st)) { + st = rrdset_create("system", "dev_intr", NULL, "interrupts", NULL, "Device Interrupts", "interrupts/s", 1000, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "interrupts", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "interrupts", u_int_data); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_soft_intr)) { + if (unlikely(GETSYSCTL("vm.stats.sys.v_soft", u_int_data))) { + do_soft_intr = 0; + error("DISABLED: system.dev_intr"); + } else { + + st = rrdset_find_bytype("system", "soft_intr"); + if (unlikely(!st)) { + st = rrdset_create("system", "soft_intr", NULL, "interrupts", NULL, "Software Interrupts", "interrupts/s", 1100, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "interrupts", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "interrupts", u_int_data); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_context)) { + if (unlikely(GETSYSCTL("vm.stats.sys.v_swtch", u_int_data))) { + do_context = 0; + error("DISABLED: system.ctxt"); + } else { + + st = rrdset_find_bytype("system", "ctxt"); + if (unlikely(!st)) { + st = rrdset_create("system", "ctxt", NULL, "processes", NULL, "CPU Context Switches", "context switches/s", 800, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "switches", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "switches", u_int_data); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_forks)) { + if (unlikely(GETSYSCTL("vm.stats.vm.v_forks", u_int_data))) { + do_forks = 0; + error("DISABLED: system.forks"); + } else { + + st = rrdset_find_bytype("system", "forks"); + if (unlikely(!st)) { + st = rrdset_create("system", "forks", NULL, "processes", NULL, "Started Processes", "processes/s", 700, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "started", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "started", u_int_data); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_disk_io)) { + if (unlikely(GETSYSCTL("kern.devstat.numdevs", numdevs))) { + do_disk_io = 0; + error("DISABLED: disk.io"); + } else { + devstat_data = reallocz(devstat_data, sizeof(long) + sizeof(struct devstat) * numdevs); // there is generation number before devstat structures + if (unlikely(getsysctl("kern.devstat.all", devstat_data, sizeof(long) + sizeof(struct devstat) * numdevs))) { + do_disk_io = 0; + error("DISABLED: disk.io"); + } else { + dstat = devstat_data + sizeof(long); // skip generation number + collected_number total_disk_kbytes_read = 0; + collected_number total_disk_kbytes_write = 0; + + for (i = 0; i < numdevs; i++) { + if (((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_DIRECT) || ((dstat[i].device_type & DEVSTAT_TYPE_MASK) == DEVSTAT_TYPE_STORARRAY)) { + sprintf(disk, "%s%d", dstat[i].device_name, dstat[i].unit_number); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype(RRD_TYPE_DISK, disk); + if (unlikely(!st)) { + st = rrdset_create(RRD_TYPE_DISK, disk, NULL, disk, "disk.io", "Disk I/O Bandwidth", "kilobytes/s", 2000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "reads", NULL, 1, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + total_disk_kbytes_read += dstat[i].bytes[DEVSTAT_READ]/KILO_FACTOR; + total_disk_kbytes_write += dstat[i].bytes[DEVSTAT_WRITE]/KILO_FACTOR; + prev_dstat.bytes_read = rrddim_set(st, "reads", dstat[i].bytes[DEVSTAT_READ]); + prev_dstat.bytes_write = rrddim_set(st, "writes", dstat[i].bytes[DEVSTAT_WRITE]); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_ops", disk); + if (unlikely(!st)) { + st = rrdset_create("disk_ops", disk, NULL, disk, "disk.ops", "Disk Completed I/O Operations", "operations/s", 2001, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + prev_dstat.operations_read = rrddim_set(st, "reads", dstat[i].operations[DEVSTAT_READ]); + prev_dstat.operations_write = rrddim_set(st, "writes", dstat[i].operations[DEVSTAT_WRITE]); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_qops", disk); + if (unlikely(!st)) { + st = rrdset_create("disk_qops", disk, NULL, disk, "disk.qops", "Disk Current I/O Operations", "operations", 2002, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "operations", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "operations", dstat[i].start_count - dstat[i].end_count); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_util", disk); + if (unlikely(!st)) { + st = rrdset_create("disk_util", disk, NULL, disk, "disk.util", "Disk Utilization Time", "% of time working", 2004, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "utilization", NULL, 1, 10, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + cur_dstat.busy_time_ms = dstat[i].busy_time.sec * 1000 + dstat[i].busy_time.frac * BINTIME_SCALE; + prev_dstat.busy_time_ms = rrddim_set(st, "utilization", cur_dstat.busy_time_ms); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_iotime", disk); + if (unlikely(!st)) { + st = rrdset_create("disk_iotime", disk, NULL, disk, "disk.iotime", "Disk Total I/O Time", "milliseconds/s", 2022, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + cur_dstat.duration_read_ms = dstat[i].duration[DEVSTAT_READ].sec * 1000 + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + cur_dstat.duration_write_ms = dstat[i].duration[DEVSTAT_WRITE].sec * 1000 + dstat[i].duration[DEVSTAT_READ].frac * BINTIME_SCALE; + prev_dstat.duration_read_ms = rrddim_set(st, "reads", cur_dstat.duration_read_ms); + prev_dstat.duration_write_ms = rrddim_set(st, "writes", cur_dstat.duration_write_ms); + rrdset_done(st); + + // -------------------------------------------------------------------- + // calculate differential charts + // only if this is not the first time we run + + if (likely(dt)) { + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_await", disk); + if (unlikely(!st)) { + st = rrdset_create("disk_await", disk, NULL, disk, "disk.await", "Average Completed I/O Operation Time", "ms per operation", 2005, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "reads", (dstat[i].operations[DEVSTAT_READ] - prev_dstat.operations_read) ? + (cur_dstat.duration_read_ms - prev_dstat.duration_read_ms) / (dstat[i].operations[DEVSTAT_READ] - prev_dstat.operations_read) : 0); + rrddim_set(st, "writes", (dstat[i].operations[DEVSTAT_WRITE] - prev_dstat.operations_write) ? + (cur_dstat.duration_write_ms - prev_dstat.duration_write_ms) / (dstat[i].operations[DEVSTAT_WRITE] - prev_dstat.operations_write) : 0); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_avgsz", disk); + if (unlikely(!st)) { + st = rrdset_create("disk_avgsz", disk, NULL, disk, "disk.avgsz", "Average Completed I/O Operation Bandwidth", "kilobytes per operation", 2006, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "writes", NULL, -1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "reads", (dstat[i].operations[DEVSTAT_READ] - prev_dstat.operations_read) ? + (dstat[i].bytes[DEVSTAT_READ] - prev_dstat.bytes_read) / (dstat[i].operations[DEVSTAT_READ] - prev_dstat.operations_read) : 0); + rrddim_set(st, "writes", (dstat[i].operations[DEVSTAT_WRITE] - prev_dstat.operations_write) ? + (dstat[i].bytes[DEVSTAT_WRITE] - prev_dstat.bytes_write) / (dstat[i].operations[DEVSTAT_WRITE] - prev_dstat.operations_write) : 0); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_svctm", disk); + if (unlikely(!st)) { + st = rrdset_create("disk_svctm", disk, NULL, disk, "disk.svctm", "Average Service Time", "ms per operation", 2007, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "svctm", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "svctm", ((dstat[i].operations[DEVSTAT_READ] - prev_dstat.operations_read) + (dstat[i].operations[DEVSTAT_WRITE] - prev_dstat.operations_write)) ? + (cur_dstat.busy_time_ms - prev_dstat.busy_time_ms) / ((dstat[i].operations[DEVSTAT_READ] - prev_dstat.operations_read) + (dstat[i].operations[DEVSTAT_WRITE] - prev_dstat.operations_write)) : 0); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("system", "io"); + if (unlikely(!st)) { + st = rrdset_create("system", "io", NULL, "disk", NULL, "Disk I/O", "kilobytes/s", 150, update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "in", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "out", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "in", total_disk_kbytes_read); + rrddim_set(st, "out", total_disk_kbytes_write); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + + if (likely(do_swap)) { + mibsize = sizeof mib / sizeof mib[0]; + if (unlikely(sysctlnametomib("vm.swap_info", mib, &mibsize) == -1)) { + error("FREEBSD: sysctl(%s...) failed: %s", "vm.swap_info", strerror(errno)); + do_swap = 0; + error("DISABLED: system.swap"); + } else { + for (i = 0; ; i++) { + mib[mibsize] = i; + size = sizeof(xsw); + if (unlikely(sysctl(mib, mibsize + 1, &xsw, &size, NULL, 0) == -1 )) { + if (unlikely(errno != ENOENT)) { + error("FREEBSD: sysctl(%s...) failed: %s", "vm.swap_info", strerror(errno)); + do_swap = 0; + error("DISABLED: system.swap"); + } else { + if (unlikely(size != sizeof(xsw))) { + error("FREEBSD: sysctl(%s...) expected %lu, got %lu", "vm.swap_info", (unsigned long)sizeof(xsw), (unsigned long)size); + do_swap = 0; + error("DISABLED: system.swap"); + } else break; + } + } + total_xsw.bytes_used += xsw.xsw_used; + total_xsw.bytes_total += xsw.xsw_nblks; + } + + if (likely(do_swap)) { + st = rrdset_find("system.swap"); + if (unlikely(!st)) { + st = rrdset_create("system", "swap", NULL, "swap", NULL, "System Swap", "MB", 201, update_every, RRDSET_TYPE_STACKED); + st->isdetail = 1; + + rrddim_add(st, "free", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "used", total_xsw.bytes_used); + rrddim_set(st, "free", total_xsw.bytes_total - total_xsw.bytes_used); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_ram)) { + if (unlikely(GETSYSCTL("vm.stats.vm.v_active_count", vmmeter_data.v_active_count) || + GETSYSCTL("vm.stats.vm.v_inactive_count", vmmeter_data.v_inactive_count) || + GETSYSCTL("vm.stats.vm.v_wire_count", vmmeter_data.v_wire_count) || + GETSYSCTL("vm.stats.vm.v_cache_count", vmmeter_data.v_cache_count) || + GETSYSCTL("vfs.bufspace", vfs_bufspace_count) || + GETSYSCTL("vm.stats.vm.v_free_count", vmmeter_data.v_free_count))) { + do_ram = 0; + error("DISABLED: system.ram"); + } else { + st = rrdset_find("system.ram"); + if (unlikely(!st)) { + st = rrdset_create("system", "ram", NULL, "ram", NULL, "System RAM", "MB", 200, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "active", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "inactive", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "wired", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "cache", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "buffers", NULL, 1, MEGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "free", NULL, system_pagesize, MEGA_FACTOR, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "active", vmmeter_data.v_active_count); + rrddim_set(st, "inactive", vmmeter_data.v_inactive_count); + rrddim_set(st, "wired", vmmeter_data.v_wire_count); + rrddim_set(st, "cache", vmmeter_data.v_cache_count); + rrddim_set(st, "buffers", vfs_bufspace_count); + rrddim_set(st, "free", vmmeter_data.v_free_count); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_swapio)) { + if (unlikely(GETSYSCTL("vm.stats.vm.v_swappgsin", vmmeter_data.v_swappgsin) || GETSYSCTL("vm.stats.vm.v_swappgsout", vmmeter_data.v_swappgsout))) { + do_swapio = 0; + error("DISABLED: system.swapio"); + } else { + st = rrdset_find("system.swapio"); + if (unlikely(!st)) { + st = rrdset_create("system", "swapio", NULL, "swap", NULL, "Swap I/O", "kilobytes/s", 250, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "in", NULL, system_pagesize, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "out", NULL, -system_pagesize, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "in", vmmeter_data.v_swappgsin); + rrddim_set(st, "out", vmmeter_data.v_swappgsout); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_pgfaults)) { + if (unlikely(GETSYSCTL("vm.stats.vm.v_vm_faults", vmmeter_data.v_vm_faults) || + GETSYSCTL("vm.stats.vm.v_io_faults", vmmeter_data.v_io_faults) || + GETSYSCTL("vm.stats.vm.v_cow_faults", vmmeter_data.v_cow_faults) || + GETSYSCTL("vm.stats.vm.v_cow_optim", vmmeter_data.v_cow_optim) || + GETSYSCTL("vm.stats.vm.v_intrans", vmmeter_data.v_intrans))) { + do_pgfaults = 0; + error("DISABLED: mem.pgfaults"); + } else { + st = rrdset_find("mem.pgfaults"); + if (unlikely(!st)) { + st = rrdset_create("mem", "pgfaults", NULL, "system", NULL, "Memory Page Faults", "page faults/s", 500, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "memory", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "io_requiring", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "cow", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "cow_optimized", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "in_transit", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "memory", vmmeter_data.v_vm_faults); + rrddim_set(st, "io_requiring", vmmeter_data.v_io_faults); + rrddim_set(st, "cow", vmmeter_data.v_cow_faults); + rrddim_set(st, "cow_optimized", vmmeter_data.v_cow_optim); + rrddim_set(st, "in_transit", vmmeter_data.v_intrans); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_ipc_semaphores)) { + if (unlikely(GETSYSCTL("kern.ipc.semmni", ipc_sem.semmni))) { + do_ipc_semaphores = 0; + error("DISABLED: system.ipc_semaphores"); + error("DISABLED: system.ipc_semaphore_arrays"); + } else { + ipc_sem_data = reallocz(ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni); + if (unlikely(getsysctl("kern.ipc.sema", ipc_sem_data, sizeof(struct semid_kernel) * ipc_sem.semmni))) { + do_ipc_semaphores = 0; + error("DISABLED: system.ipc_semaphores"); + error("DISABLED: system.ipc_semaphore_arrays"); + } else { + for (i = 0; i < ipc_sem.semmni; i++) { + if (unlikely(ipc_sem_data[i].u.sem_perm.mode & SEM_ALLOC)) { + ipc_sem.sets += 1; + ipc_sem.semaphores += ipc_sem_data[i].u.sem_nsems; + } + } + + // -------------------------------------------------------------------- + + st = rrdset_find("system.ipc_semaphores"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipc_semaphores", NULL, "ipc semaphores", NULL, "IPC Semaphores", "semaphores", 1000, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "semaphores", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "semaphores", ipc_sem.semaphores); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find("system.ipc_semaphore_arrays"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipc_semaphore_arrays", NULL, "ipc semaphores", NULL, "IPC Semaphore Arrays", "arrays", 1000, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "arrays", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "arrays", ipc_sem.sets); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_ipc_shared_mem)) { + if (unlikely(GETSYSCTL("kern.ipc.shmmni", ipc_shm.shmmni))) { + do_ipc_shared_mem = 0; + error("DISABLED: system.ipc_shared_mem_segs"); + error("DISABLED: system.ipc_shared_mem_size"); + } else { + ipc_shm_data = reallocz(ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni); + if (unlikely(getsysctl("kern.ipc.shmsegs", ipc_shm_data, sizeof(struct shmid_kernel) * ipc_shm.shmmni))) { + do_ipc_shared_mem = 0; + error("DISABLED: system.ipc_shared_mem_segs"); + error("DISABLED: system.ipc_shared_mem_size"); + } else { + for (i = 0; i < ipc_shm.shmmni; i++) { + if (unlikely(ipc_shm_data[i].u.shm_perm.mode & 0x0800)) { + ipc_shm.segs += 1; + ipc_shm.segsize += ipc_shm_data[i].u.shm_segsz; + } + } + + // -------------------------------------------------------------------- + + st = rrdset_find("system.ipc_shared_mem_segs"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipc_shared_mem_segs", NULL, "ipc shared memory", NULL, "IPC Shared Memory Segments", "segments", 1000, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "segments", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "segments", ipc_shm.segs); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find("system.ipc_shared_mem_size"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipc_shared_mem_size", NULL, "ipc shared memory", NULL, "IPC Shared Memory Segments Size", "kilobytes", 1000, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "allocated", NULL, 1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "allocated", ipc_shm.segsize); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_ipc_msg_queues)) { + if (unlikely(GETSYSCTL("kern.ipc.msgmni", ipc_msq.msgmni))) { + do_ipc_msg_queues = 0; + error("DISABLED: system.ipc_msq_queues"); + error("DISABLED: system.ipc_msq_messages"); + error("DISABLED: system.ipc_msq_size"); + } else { + ipc_msq_data = reallocz(ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni); + if (unlikely(getsysctl("kern.ipc.msqids", ipc_msq_data, sizeof(struct msqid_kernel) * ipc_msq.msgmni))) { + do_ipc_msg_queues = 0; + error("DISABLED: system.ipc_msq_queues"); + error("DISABLED: system.ipc_msq_messages"); + error("DISABLED: system.ipc_msq_size"); + } else { + for (i = 0; i < ipc_msq.msgmni; i++) { + if (unlikely(ipc_msq_data[i].u.msg_qbytes != 0)) { + ipc_msq.queues += 1; + ipc_msq.messages += ipc_msq_data[i].u.msg_qnum; + ipc_msq.usedsize += ipc_msq_data[i].u.msg_cbytes; + ipc_msq.allocsize += ipc_msq_data[i].u.msg_qbytes; + } + } + + // -------------------------------------------------------------------- + + st = rrdset_find("system.ipc_msq_queues"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipc_msq_queues", NULL, "ipc message queues", NULL, "Number of IPC Message Queues", "queues", 990, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "queues", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "queues", ipc_msq.queues); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find("system.ipc_msq_messages"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipc_msq_messages", NULL, "ipc message queues", NULL, "Number of Messages in IPC Message Queues", "messages", 1000, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "messages", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "messages", ipc_msq.messages); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find("system.ipc_msq_size"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipc_msq_size", NULL, "ipc message queues", NULL, "Size of IPC Message Queues", "bytes", 1100, rrd_update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "allocated", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "allocated", ipc_msq.allocsize); + rrddim_set(st, "used", ipc_msq.usedsize); + rrdset_done(st); + + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_netisr || do_netisr_per_core)) { + if (unlikely(GETSYSCTL("kern.smp.cpus", ncpus))) { + common_error = 1; + } else if (unlikely(ncpus > 9999)) { + error("FREEBSD: There are more than 4 digits in cpu cores number"); + common_error = 1; + } else if (unlikely(sysctlbyname("net.isr.workstream", NULL, &netisr_workstream_size, NULL, 0) == -1)) { + error("FREEBSD: sysctl(net.isr.workstream...) failed: %s", strerror(errno)); + common_error = 1; + } else if (unlikely(sysctlbyname("net.isr.work", NULL, &netisr_work_size, NULL, 0) == -1)) { + error("FREEBSD: sysctl(net.isr.work...) failed: %s", strerror(errno)); + common_error = 1; + } else { + num_netisr_workstreams = netisr_workstream_size / sizeof(struct sysctl_netisr_workstream); + netisr_workstream = reallocz(netisr_workstream, num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream)); + if (unlikely(getsysctl("net.isr.workstream", netisr_workstream, num_netisr_workstreams * sizeof(struct sysctl_netisr_workstream)))){ + common_error = 1; + } else { + num_netisr_works = netisr_work_size / sizeof(struct sysctl_netisr_work); + netisr_work = reallocz(netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work)); + if (unlikely(getsysctl("net.isr.work", netisr_work, num_netisr_works * sizeof(struct sysctl_netisr_work)))){ + common_error = 1; + } + } + } + if (unlikely(common_error)) { + do_netisr = 0; + error("DISABLED: system.softnet_stat"); + do_netisr_per_core = 0; + error("DISABLED: system.cpuX_softnet_stat"); + common_error = 0; + } else { + netisr_stats = reallocz(netisr_stats, (ncpus + 1) * sizeof(struct netisr_stats)); + bzero(netisr_stats, (ncpus + 1) * sizeof(struct netisr_stats)); + for (i = 0; i < num_netisr_workstreams; i++) { + for (n = 0; n < num_netisr_works; n++) { + if (netisr_workstream[i].snws_wsid == netisr_work[n].snw_wsid) { + netisr_stats[netisr_workstream[i].snws_cpu].dispatched += netisr_work[n].snw_dispatched; + netisr_stats[netisr_workstream[i].snws_cpu].hybrid_dispatched += netisr_work[n].snw_hybrid_dispatched; + netisr_stats[netisr_workstream[i].snws_cpu].qdrops += netisr_work[n].snw_qdrops; + netisr_stats[netisr_workstream[i].snws_cpu].queued += netisr_work[n].snw_queued; + } + } + } + for (i = 0; i < ncpus; i++) { + netisr_stats[ncpus].dispatched += netisr_stats[i].dispatched; + netisr_stats[ncpus].hybrid_dispatched += netisr_stats[i].hybrid_dispatched; + netisr_stats[ncpus].qdrops += netisr_stats[i].qdrops; + netisr_stats[ncpus].queued += netisr_stats[i].queued; + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_netisr)) { + st = rrdset_find_bytype("system", "softnet_stat"); + if (unlikely(!st)) { + st = rrdset_create("system", "softnet_stat", NULL, "softnet_stat", NULL, "System softnet_stat", "events/s", 955, update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "dispatched", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "hybrid_dispatched", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "qdrops", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "queued", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "dispatched", netisr_stats[ncpus].dispatched); + rrddim_set(st, "hybrid_dispatched", netisr_stats[ncpus].hybrid_dispatched); + rrddim_set(st, "qdrops", netisr_stats[ncpus].qdrops); + rrddim_set(st, "queued", netisr_stats[ncpus].queued); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_netisr_per_core)) { + for (i = 0; i < ncpus ;i++) { + snprintfz(netstat_cpuid, 21, "cpu%d_softnet_stat", i); + + st = rrdset_find_bytype("cpu", netstat_cpuid); + if (unlikely(!st)) { + st = rrdset_create("cpu", netstat_cpuid, NULL, "softnet_stat", NULL, "Per CPU netisr statistics", "events/s", 1101 + i, update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "dispatched", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "hybrid_dispatched", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "qdrops", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "queued", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "dispatched", netisr_stats[i].dispatched); + rrddim_set(st, "hybrid_dispatched", netisr_stats[i].hybrid_dispatched); + rrddim_set(st, "qdrops", netisr_stats[i].qdrops); + rrddim_set(st, "queued", netisr_stats[i].queued); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_bandwidth)) { + if (unlikely(getifaddrs(&ifap))) { + error("FREEBSD: getifaddrs()"); + do_bandwidth = 0; + error("DISABLED: system.ipv4"); + } else { + iftot.ift_ibytes = iftot.ift_obytes = 0; + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_INET) + continue; + iftot.ift_ibytes += IFA_DATA(ibytes); + iftot.ift_obytes += IFA_DATA(obytes); + } + + st = rrdset_find("system.ipv4"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipv4", NULL, "network", NULL, "IPv4 Bandwidth", "kilobits/s", 500, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "InOctets", "received", 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutOctets", "sent", -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "InOctets", iftot.ift_ibytes); + rrddim_set(st, "OutOctets", iftot.ift_obytes); + rrdset_done(st); + + // -------------------------------------------------------------------- + + iftot.ift_ibytes = iftot.ift_obytes = 0; + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_INET6) + continue; + iftot.ift_ibytes += IFA_DATA(ibytes); + iftot.ift_obytes += IFA_DATA(obytes); + } + + st = rrdset_find("system.ipv6"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipv6", NULL, "network", NULL, "IPv6 Bandwidth", "kilobits/s", 500, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "sent", iftot.ift_obytes); + rrddim_set(st, "received", iftot.ift_ibytes); + rrdset_done(st); + + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_LINK) + continue; + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net", ifa->ifa_name, NULL, ifa->ifa_name, "net.net", "Bandwidth", "kilobits/s", 7000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", IFA_DATA(ibytes)); + rrddim_set(st, "sent", IFA_DATA(obytes)); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_packets", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_packets", ifa->ifa_name, NULL, ifa->ifa_name, "net.packets", "Packets", "packets/s", 7001, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "multicast_received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "multicast_sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", IFA_DATA(ipackets)); + rrddim_set(st, "sent", IFA_DATA(opackets)); + rrddim_set(st, "multicast_received", IFA_DATA(imcasts)); + rrddim_set(st, "multicast_sent", IFA_DATA(omcasts)); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_errors", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_errors", ifa->ifa_name, NULL, ifa->ifa_name, "net.errors", "Interface Errors", "errors/s", 7002, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "inbound", IFA_DATA(ierrors)); + rrddim_set(st, "outbound", IFA_DATA(oerrors)); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_drops", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_drops", ifa->ifa_name, NULL, ifa->ifa_name, "net.drops", "Interface Drops", "drops/s", 7003, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); +#if __FreeBSD__ >= 11 + rrddim_add(st, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); +#endif + } + else rrdset_next(st); + + rrddim_set(st, "inbound", IFA_DATA(iqdrops)); +#if __FreeBSD__ >= 11 + rrddim_set(st, "outbound", IFA_DATA(oqdrops)); +#endif + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_events", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_events", ifa->ifa_name, NULL, ifa->ifa_name, "net.events", "Network Interface Events", "events/s", 7006, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "frames", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "collisions", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "carrier", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "collisions", IFA_DATA(collisions)); + rrdset_done(st); + } + + freeifaddrs(ifap); + } + } + + // -------------------------------------------------------------------- + + // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html + if (likely(do_tcp_sockets)) { + if (unlikely(GETSYSCTL("net.inet.tcp.states", tcps_states))) { + do_tcp_sockets = 0; + error("DISABLED: ipv4.tcpsock"); + } else { + if (likely(do_tcp_sockets)) { + st = rrdset_find("ipv4.tcpsock"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcpsock", NULL, "tcp", NULL, "IPv4 TCP Connections", + "active connections", 2500, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "CurrEstab", "connections", 1, 1, RRDDIM_ABSOLUTE); + } else + rrdset_next(st); + + rrddim_set(st, "CurrEstab", tcps_states[TCPS_ESTABLISHED]); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html + if (likely(do_tcp_packets || do_tcp_errors || do_tcp_handshake || do_tcpext_connaborts || do_tcpext_ofo || do_tcpext_syscookies || do_ecn)) { + if (unlikely(GETSYSCTL("net.inet.tcp.stats", tcpstat))){ + do_tcp_packets = 0; + error("DISABLED: ipv4.tcppackets"); + do_tcp_errors = 0; + error("DISABLED: ipv4.tcperrors"); + do_tcp_handshake = 0; + error("DISABLED: ipv4.tcphandshake"); + do_tcpext_connaborts = 0; + error("DISABLED: ipv4.tcpconnaborts"); + do_tcpext_ofo = 0; + error("DISABLED: ipv4.tcpofo"); + do_tcpext_syscookies = 0; + error("DISABLED: ipv4.tcpsyncookies"); + do_ecn = 0; + error("DISABLED: ipv4.ecnpkts"); + } else { + if (likely(do_tcp_packets)) { + st = rrdset_find("ipv4.tcppackets"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcppackets", NULL, "tcp", NULL, "IPv4 TCP Packets", + "packets/s", + 2600, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InSegs", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutSegs", "sent", -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InSegs", tcpstat.tcps_rcvtotal); + rrddim_set(st, "OutSegs", tcpstat.tcps_sndtotal); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_tcp_errors)) { + st = rrdset_find("ipv4.tcperrors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcperrors", NULL, "tcp", NULL, "IPv4 TCP Errors", + "packets/s", + 2700, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InErrs", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "RetransSegs", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + +#if __FreeBSD__ >= 11 + rrddim_set(st, "InErrs", tcpstat.tcps_rcvbadoff + tcpstat.tcps_rcvreassfull + tcpstat.tcps_rcvshort); +#else + rrddim_set(st, "InErrs", tcpstat.tcps_rcvbadoff + tcpstat.tcps_rcvshort); +#endif + rrddim_set(st, "InCsumErrors", tcpstat.tcps_rcvbadsum); + rrddim_set(st, "RetransSegs", tcpstat.tcps_sndrexmitpack); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_tcp_handshake)) { + st = rrdset_find("ipv4.tcphandshake"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcphandshake", NULL, "tcp", NULL, + "IPv4 TCP Handshake Issues", + "events/s", 2900, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "EstabResets", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "AttemptFails", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "EstabResets", tcpstat.tcps_drops); + rrddim_set(st, "ActiveOpens", tcpstat.tcps_connattempt); + rrddim_set(st, "PassiveOpens", tcpstat.tcps_accepts); + rrddim_set(st, "AttemptFails", tcpstat.tcps_conndrops); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_tcpext_connaborts == CONFIG_ONDEMAND_YES || (do_tcpext_connaborts == CONFIG_ONDEMAND_ONDEMAND && (tcpstat.tcps_rcvpackafterwin || tcpstat.tcps_rcvafterclose || tcpstat.tcps_rcvmemdrop || tcpstat.tcps_persistdrop || tcpstat.tcps_finwait2_drops))) { + do_tcpext_connaborts = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv4.tcpconnaborts"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcpconnaborts", NULL, "tcp", NULL, "TCP Connection Aborts", "connections/s", 3010, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "TCPAbortOnData", "baddata", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "TCPAbortOnClose", "userclosed", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "TCPAbortOnMemory", "nomemory", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "TCPAbortOnTimeout", "timeout", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "TCPAbortOnLinger", "linger", 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "TCPAbortOnData", tcpstat.tcps_rcvpackafterwin); + rrddim_set(st, "TCPAbortOnClose", tcpstat.tcps_rcvafterclose); + rrddim_set(st, "TCPAbortOnMemory", tcpstat.tcps_rcvmemdrop); + rrddim_set(st, "TCPAbortOnTimeout", tcpstat.tcps_persistdrop); + rrddim_set(st, "TCPAbortOnLinger", tcpstat.tcps_finwait2_drops); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_tcpext_ofo == CONFIG_ONDEMAND_YES || (do_tcpext_ofo == CONFIG_ONDEMAND_ONDEMAND && tcpstat.tcps_rcvoopack)) { + do_tcpext_ofo = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv4.tcpofo"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcpofo", NULL, "tcp", NULL, "TCP Out-Of-Order Queue", "packets/s", 3050, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "TCPOFOQueue", "inqueue", 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "TCPOFOQueue", tcpstat.tcps_rcvoopack); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_tcpext_syscookies == CONFIG_ONDEMAND_YES || (do_tcpext_syscookies == CONFIG_ONDEMAND_ONDEMAND && (tcpstat.tcps_sc_sendcookie || tcpstat.tcps_sc_recvcookie || tcpstat.tcps_sc_zonefail))) { + do_tcpext_syscookies = CONFIG_ONDEMAND_YES; + + st = rrdset_find("ipv4.tcpsyncookies"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcpsyncookies", NULL, "tcp", NULL, "TCP SYN Cookies", "packets/s", 3100, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "SyncookiesRecv", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "SyncookiesSent", "sent", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "SyncookiesFailed", "failed", -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "SyncookiesRecv", tcpstat.tcps_sc_recvcookie); + rrddim_set(st, "SyncookiesSent", tcpstat.tcps_sc_sendcookie); + rrddim_set(st, "SyncookiesFailed", tcpstat.tcps_sc_zonefail); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ecn == CONFIG_ONDEMAND_YES || (do_ecn == CONFIG_ONDEMAND_ONDEMAND && (tcpstat.tcps_ecn_ce || tcpstat.tcps_ecn_ect0 || tcpstat.tcps_ecn_ect1))) { + do_ecn = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv4.ecnpkts"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "ecnpkts", NULL, "ecn", NULL, "IPv4 ECN Statistics", "packets/s", 8700, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InCEPkts", "CEP", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InNoECTPkts", "NoECTP", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InECT0Pkts", "ECTP0", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InECT1Pkts", "ECTP1", 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "InCEPkts", tcpstat.tcps_ecn_ce); + rrddim_set(st, "InNoECTPkts", tcpstat.tcps_ecn_ce - (tcpstat.tcps_ecn_ect0 + tcpstat.tcps_ecn_ect1)); + rrddim_set(st, "InECT0Pkts", tcpstat.tcps_ecn_ect0); + rrddim_set(st, "InECT1Pkts", tcpstat.tcps_ecn_ect1); + rrdset_done(st); + } + + } + } + + // -------------------------------------------------------------------- + + // see http://net-snmp.sourceforge.net/docs/mibs/udp.html + if (likely(do_udp_packets || do_udp_errors)) { + if (unlikely(GETSYSCTL("net.inet.udp.stats", udpstat))) { + do_udp_packets = 0; + error("DISABLED: ipv4.udppackets"); + do_udp_errors = 0; + error("DISABLED: ipv4.udperrors"); + } else { + if (likely(do_udp_packets)) { + st = rrdset_find("ipv4.udppackets"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "udppackets", NULL, "udp", NULL, "IPv4 UDP Packets", + "packets/s", 2601, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InDatagrams", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InDatagrams", udpstat.udps_ipackets); + rrddim_set(st, "OutDatagrams", udpstat.udps_opackets); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_udp_errors)) { + st = rrdset_find("ipv4.udperrors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "udperrors", NULL, "udp", NULL, "IPv4 UDP Errors", "events/s", + 2701, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "NoPorts", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InErrors", udpstat.udps_hdrops + udpstat.udps_badlen); + rrddim_set(st, "NoPorts", udpstat.udps_noport); + rrddim_set(st, "RcvbufErrors", udpstat.udps_fullsock); + rrddim_set(st, "InCsumErrors", udpstat.udps_badsum + udpstat.udps_nosum); + rrddim_set(st, "IgnoredMulti", udpstat.udps_filtermcast); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_icmp_packets || do_icmpmsg)) { + if (unlikely(GETSYSCTL("net.inet.icmp.stats", icmpstat))) { + do_icmp_packets = 0; + error("DISABLED: ipv4.icmp"); + error("DISABLED: ipv4.icmp_errors"); + do_icmpmsg = 0; + error("DISABLED: ipv4.icmpmsg"); + } else { + for (i = 0; i <= ICMP_MAXTYPE; i++) { + icmp_total.msgs_in += icmpstat.icps_inhist[i]; + icmp_total.msgs_out += icmpstat.icps_outhist[i]; + } + icmp_total.msgs_in += icmpstat.icps_badcode + icmpstat.icps_badlen + icmpstat.icps_checksum + icmpstat.icps_tooshort; + + // -------------------------------------------------------------------- + + if (likely(do_icmp_packets)) { + st = rrdset_find("ipv4.icmp"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "icmp", NULL, "icmp", NULL, "IPv4 ICMP Packets", "packets/s", + 2602, + update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InMsgs", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutMsgs", "sent", -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InMsgs", icmp_total.msgs_in); + rrddim_set(st, "OutMsgs", icmp_total.msgs_out); + + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find("ipv4.icmp_errors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "icmp_errors", NULL, "icmp", NULL, "IPv4 ICMP Errors", + "packets/s", + 2603, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutErrors", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InErrors", icmpstat.icps_badcode + icmpstat.icps_badlen + icmpstat.icps_checksum + icmpstat.icps_tooshort); + rrddim_set(st, "OutErrors", icmpstat.icps_error); + rrddim_set(st, "InCsumErrors", icmpstat.icps_checksum); + + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_icmpmsg)) { + st = rrdset_find("ipv4.icmpmsg"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "icmpmsg", NULL, "icmp", NULL, "IPv4 ICMP Messsages", + "packets/s", 2604, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InEchoReps", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchoReps", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InEchos", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchos", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InEchoReps", icmpstat.icps_inhist[ICMP_ECHOREPLY]); + rrddim_set(st, "OutEchoReps", icmpstat.icps_outhist[ICMP_ECHOREPLY]); + rrddim_set(st, "InEchos", icmpstat.icps_inhist[ICMP_ECHO]); + rrddim_set(st, "OutEchos", icmpstat.icps_outhist[ICMP_ECHO]); + + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + // see also http://net-snmp.sourceforge.net/docs/mibs/ip.html + if (likely(do_ip_packets || do_ip_fragsout || do_ip_fragsin || do_ip_errors)) { + if (unlikely(GETSYSCTL("net.inet.ip.stats", ipstat))) { + do_ip_packets = 0; + error("DISABLED: ipv4.packets"); + do_ip_fragsout = 0; + error("DISABLED: ipv4.fragsout"); + do_ip_fragsin = 0; + error("DISABLED: ipv4.fragsin"); + do_ip_errors = 0; + error("DISABLED: ipv4.errors"); + } else { + if (likely(do_ip_packets)) { + st = rrdset_find("ipv4.packets"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "packets", NULL, "packets", NULL, "IPv4 Packets", "packets/s", + 3000, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InReceives", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutRequests", "sent", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InDelivers", "delivered", 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "OutRequests", ipstat.ips_localout); + rrddim_set(st, "InReceives", ipstat.ips_total); + rrddim_set(st, "ForwDatagrams", ipstat.ips_forward); + rrddim_set(st, "InDelivers", ipstat.ips_delivered); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_ip_fragsout)) { + st = rrdset_find("ipv4.fragsout"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "fragsout", NULL, "fragments", NULL, "IPv4 Fragments Sent", + "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "FragOKs", "ok", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "FragFails", "failed", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "FragCreates", "created", 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "FragOKs", ipstat.ips_fragmented); + rrddim_set(st, "FragFails", ipstat.ips_cantfrag); + rrddim_set(st, "FragCreates", ipstat.ips_ofragments); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_ip_fragsin)) { + st = rrdset_find("ipv4.fragsin"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "fragsin", NULL, "fragments", NULL, + "IPv4 Fragments Reassembly", + "packets/s", 3011, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ReasmFails", "failed", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ReasmReqds", "all", 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "ReasmOKs", ipstat.ips_fragments); + rrddim_set(st, "ReasmFails", ipstat.ips_fragdropped); + rrddim_set(st, "ReasmReqds", ipstat.ips_reassembled); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_ip_errors)) { + st = rrdset_find("ipv4.errors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "errors", NULL, "errors", NULL, "IPv4 Errors", "packets/s", + 3002, + update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InDiscards", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDiscards", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InDiscards", ipstat.ips_badsum + ipstat.ips_tooshort + ipstat.ips_toosmall + ipstat.ips_toolong); + rrddim_set(st, "OutDiscards", ipstat.ips_odropped); + rrddim_set(st, "InHdrErrors", ipstat.ips_badhlen + ipstat.ips_badlen + ipstat.ips_badoptions + ipstat.ips_badvers); + rrddim_set(st, "InAddrErrors", ipstat.ips_badaddr); + rrddim_set(st, "InUnknownProtos", ipstat.ips_noproto); + rrddim_set(st, "OutNoRoutes", ipstat.ips_noroute); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_ip6_packets || do_ip6_fragsout || do_ip6_fragsin || do_ip6_errors)) { + if (unlikely(GETSYSCTL("net.inet6.ip6.stats", ip6stat))) { + do_ip6_packets = 0; + error("DISABLED: ipv6.packets"); + do_ip6_fragsout = 0; + error("DISABLED: ipv6.fragsout"); + do_ip6_fragsin = 0; + error("DISABLED: ipv6.fragsin"); + do_ip6_errors = 0; + error("DISABLED: ipv6.errors"); + } else { + if (do_ip6_packets == CONFIG_ONDEMAND_YES || (do_ip6_packets == CONFIG_ONDEMAND_ONDEMAND && + (ip6stat.ip6s_localout || ip6stat.ip6s_total || + ip6stat.ip6s_forward || ip6stat.ip6s_delivered))) { + do_ip6_packets = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.packets"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "packets", NULL, "packets", NULL, "IPv6 Packets", "packets/s", 3000, + update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "forwarded", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "delivers", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "sent", ip6stat.ip6s_localout); + rrddim_set(st, "received", ip6stat.ip6s_total); + rrddim_set(st, "forwarded", ip6stat.ip6s_forward); + rrddim_set(st, "delivers", ip6stat.ip6s_delivered); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ip6_fragsout == CONFIG_ONDEMAND_YES || (do_ip6_fragsout == CONFIG_ONDEMAND_ONDEMAND && + (ip6stat.ip6s_fragmented || ip6stat.ip6s_cantfrag || + ip6stat.ip6s_ofragments))) { + do_ip6_fragsout = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.fragsout"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "fragsout", NULL, "fragments", NULL, "IPv6 Fragments Sent", + "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ok", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "failed", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "all", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "ok", ip6stat.ip6s_fragmented); + rrddim_set(st, "failed", ip6stat.ip6s_cantfrag); + rrddim_set(st, "all", ip6stat.ip6s_ofragments); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ip6_fragsin == CONFIG_ONDEMAND_YES || (do_ip6_fragsin == CONFIG_ONDEMAND_ONDEMAND && + (ip6stat.ip6s_reassembled || ip6stat.ip6s_fragdropped || + ip6stat.ip6s_fragtimeout || ip6stat.ip6s_fragments))) { + do_ip6_fragsin = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.fragsin"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "fragsin", NULL, "fragments", NULL, "IPv6 Fragments Reassembly", + "packets/s", 3011, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ok", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "failed", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "timeout", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "all", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "ok", ip6stat.ip6s_reassembled); + rrddim_set(st, "failed", ip6stat.ip6s_fragdropped); + rrddim_set(st, "timeout", ip6stat.ip6s_fragtimeout); + rrddim_set(st, "all", ip6stat.ip6s_fragments); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ip6_errors == CONFIG_ONDEMAND_YES || (do_ip6_errors == CONFIG_ONDEMAND_ONDEMAND && ( + ip6stat.ip6s_toosmall || + ip6stat.ip6s_odropped || + ip6stat.ip6s_badoptions || + ip6stat.ip6s_badvers || + ip6stat.ip6s_exthdrtoolong || + ip6stat.ip6s_sources_none || + ip6stat.ip6s_tooshort || + ip6stat.ip6s_cantforward || + ip6stat.ip6s_noroute))) { + do_ip6_errors = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.errors"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "errors", NULL, "errors", NULL, "IPv6 Errors", "packets/s", 3002, + update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InDiscards", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDiscards", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InTruncatedPkts", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InNoRoutes", NULL, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InDiscards", ip6stat.ip6s_toosmall); + rrddim_set(st, "OutDiscards", ip6stat.ip6s_odropped); + + rrddim_set(st, "InHdrErrors", + ip6stat.ip6s_badoptions + ip6stat.ip6s_badvers + ip6stat.ip6s_exthdrtoolong); + rrddim_set(st, "InAddrErrors", ip6stat.ip6s_sources_none); + rrddim_set(st, "InTruncatedPkts", ip6stat.ip6s_tooshort); + rrddim_set(st, "InNoRoutes", ip6stat.ip6s_cantforward); + + rrddim_set(st, "OutNoRoutes", ip6stat.ip6s_noroute); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_icmp6 || do_icmp6_redir || do_icmp6_errors || do_icmp6_echos || do_icmp6_router || do_icmp6_neighbor || do_icmp6_types)) { + if (unlikely(GETSYSCTL("net.inet6.icmp6.stats", icmp6stat))) { + do_icmp6 = 0; + error("DISABLED: ipv6.icmp"); + } else { + for (i = 0; i <= ICMP6_MAXTYPE; i++) { + icmp6_total.msgs_in += icmp6stat.icp6s_inhist[i]; + icmp6_total.msgs_out += icmp6stat.icp6s_outhist[i]; + } + icmp6_total.msgs_in += icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen + icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort; + if (do_icmp6 == CONFIG_ONDEMAND_YES || (do_icmp6 == CONFIG_ONDEMAND_ONDEMAND && (icmp6_total.msgs_in || icmp6_total.msgs_out))) { + do_icmp6 = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmp"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmp", NULL, "icmp", NULL, "IPv6 ICMP Messages", + "messages/s", 10000, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "sent", icmp6_total.msgs_in); + rrddim_set(st, "received", icmp6_total.msgs_out); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_redir == CONFIG_ONDEMAND_YES || (do_icmp6_redir == CONFIG_ONDEMAND_ONDEMAND && (icmp6stat.icp6s_inhist[ND_REDIRECT] || icmp6stat.icp6s_outhist[ND_REDIRECT]))) { + do_icmp6_redir = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmpredir"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmpredir", NULL, "icmp", NULL, "IPv6 ICMP Redirects", + "redirects/s", 10050, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "sent", icmp6stat.icp6s_inhist[ND_REDIRECT]); + rrddim_set(st, "received", icmp6stat.icp6s_outhist[ND_REDIRECT]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_errors == CONFIG_ONDEMAND_YES || (do_icmp6_errors == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_badcode || + icmp6stat.icp6s_badlen || + icmp6stat.icp6s_checksum || + icmp6stat.icp6s_tooshort || + icmp6stat.icp6s_error || + icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH] || + icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED] || + icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB] || + icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH] || + icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED] || + icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB]))) { + do_icmp6_errors = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmperrors"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmperrors", NULL, "icmp", NULL, "IPv6 ICMP Errors", "errors/s", 10100, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutErrors", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InDestUnreachs", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InPktTooBigs", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InTimeExcds", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InParmProblems", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDestUnreachs", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutTimeExcds", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutParmProblems", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InErrors", icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen + icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort); + rrddim_set(st, "OutErrors", icmp6stat.icp6s_error); + rrddim_set(st, "InCsumErrors", icmp6stat.icp6s_checksum); + rrddim_set(st, "InDestUnreachs", icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH]); + rrddim_set(st, "InPktTooBigs", icmp6stat.icp6s_badlen); + rrddim_set(st, "InTimeExcds", icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED]); + rrddim_set(st, "InParmProblems", icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB]); + rrddim_set(st, "OutDestUnreachs", icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH]); + rrddim_set(st, "OutTimeExcds", icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED]); + rrddim_set(st, "OutParmProblems", icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_echos == CONFIG_ONDEMAND_YES || (do_icmp6_echos == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST] || + icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST] || + icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY] || + icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY]))) { + do_icmp6_echos = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmpechos"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmpechos", NULL, "icmp", NULL, "IPv6 ICMP Echo", "messages/s", 10200, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InEchos", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchos", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InEchoReplies", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchoReplies", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InEchos", icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST]); + rrddim_set(st, "OutEchos", icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST]); + rrddim_set(st, "InEchoReplies", icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY]); + rrddim_set(st, "OutEchoReplies", icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_router == CONFIG_ONDEMAND_YES || (do_icmp6_router == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT] || + icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT] || + icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT] || + icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT]))) { + do_icmp6_router = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmprouter"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmprouter", NULL, "icmp", NULL, "IPv6 Router Messages", "messages/s", 10400, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InSolicits", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutSolicits", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InSolicits", icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT]); + rrddim_set(st, "OutSolicits", icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT]); + rrddim_set(st, "InAdvertisements", icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT]); + rrddim_set(st, "OutAdvertisements", icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_neighbor == CONFIG_ONDEMAND_YES || (do_icmp6_neighbor == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT] || + icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT] || + icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT] || + icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT]))) { + do_icmp6_neighbor = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmpneighbor"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmpneighbor", NULL, "icmp", NULL, "IPv6 Neighbor Messages", "messages/s", 10500, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InSolicits", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutSolicits", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InSolicits", icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT]); + rrddim_set(st, "OutSolicits", icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT]); + rrddim_set(st, "InAdvertisements", icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT]); + rrddim_set(st, "OutAdvertisements", icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_types == CONFIG_ONDEMAND_YES || (do_icmp6_types == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[1] || + icmp6stat.icp6s_inhist[128] || + icmp6stat.icp6s_inhist[129] || + icmp6stat.icp6s_inhist[136] || + icmp6stat.icp6s_outhist[1] || + icmp6stat.icp6s_outhist[128] || + icmp6stat.icp6s_outhist[129] || + icmp6stat.icp6s_outhist[133] || + icmp6stat.icp6s_outhist[135] || + icmp6stat.icp6s_outhist[136]))) { + do_icmp6_types = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmptypes"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmptypes", NULL, "icmp", NULL, "IPv6 ICMP Types", + "messages/s", 10700, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InType1", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InType128", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InType129", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InType136", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType1", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType128", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType129", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType133", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType135", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType143", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InType1", icmp6stat.icp6s_inhist[1]); + rrddim_set(st, "InType128", icmp6stat.icp6s_inhist[128]); + rrddim_set(st, "InType129", icmp6stat.icp6s_inhist[129]); + rrddim_set(st, "InType136", icmp6stat.icp6s_inhist[136]); + rrddim_set(st, "OutType1", icmp6stat.icp6s_outhist[1]); + rrddim_set(st, "OutType128", icmp6stat.icp6s_outhist[128]); + rrddim_set(st, "OutType129", icmp6stat.icp6s_outhist[129]); + rrddim_set(st, "OutType133", icmp6stat.icp6s_outhist[133]); + rrddim_set(st, "OutType135", icmp6stat.icp6s_outhist[135]); + rrddim_set(st, "OutType143", icmp6stat.icp6s_outhist[143]); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------------- + + if (likely(do_space || do_inodes)) { + // there is no mount info in sysctl MIBs + if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) { + error("FREEBSD: getmntinfo() failed"); + do_space = 0; + error("DISABLED: disk_space.X"); + do_inodes = 0; + error("DISABLED: disk_inodes.X"); + } else { + for (i = 0; i < mntsize; i++) { + if (mntbuf[i].f_flags == MNT_RDONLY || + mntbuf[i].f_blocks == 0 || + // taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes + strcmp(mntbuf[i].f_fstypename, "autofs") == 0 || + strcmp(mntbuf[i].f_fstypename, "procfs") == 0 || + strcmp(mntbuf[i].f_fstypename, "subfs") == 0 || + strcmp(mntbuf[i].f_fstypename, "devfs") == 0 || + strcmp(mntbuf[i].f_fstypename, "none") == 0) + continue; + + // -------------------------------------------------------------------------- + + if (likely(do_space)) { + st = rrdset_find_bytype("disk_space", mntbuf[i].f_mntonname); + if (unlikely(!st)) { + snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); + st = rrdset_create("disk_space", mntbuf[i].f_mntonname, NULL, mntbuf[i].f_mntonname, "disk.space", title, "GB", 2023, + update_every, + RRDSET_TYPE_STACKED); + + rrddim_add(st, "avail", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "reserved_for_root", "reserved for root", mntbuf[i].f_bsize, GIGA_FACTOR, + RRDDIM_ABSOLUTE); + } else + rrdset_next(st); + + rrddim_set(st, "avail", (collected_number) mntbuf[i].f_bavail); + rrddim_set(st, "used", (collected_number) (mntbuf[i].f_blocks - mntbuf[i].f_bfree)); + rrddim_set(st, "reserved_for_root", (collected_number) (mntbuf[i].f_bfree - mntbuf[i].f_bavail)); + rrdset_done(st); + } + + // -------------------------------------------------------------------------- + + if (likely(do_inodes)) { + st = rrdset_find_bytype("disk_inodes", mntbuf[i].f_mntonname); + if (unlikely(!st)) { + snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); + st = rrdset_create("disk_inodes", mntbuf[i].f_mntonname, NULL, mntbuf[i].f_mntonname, "disk.inodes", title, "Inodes", 2024, + update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "avail", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "reserved_for_root", "reserved for root", 1, 1, RRDDIM_ABSOLUTE); + } else + rrdset_next(st); + + rrddim_set(st, "avail", (collected_number) mntbuf[i].f_ffree); + rrddim_set(st, "used", (collected_number) (mntbuf[i].f_files - mntbuf[i].f_ffree)); + rrdset_done(st); + } + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_uptime)) { + if (unlikely(GETSYSCTL("kern.boottime", boot_time))) { + do_uptime = 0; + error("DISABLED: system.uptime"); + } else { + clock_gettime(CLOCK_REALTIME, &cur_time); + st = rrdset_find("system.uptime"); + + if(unlikely(!st)) { + st = rrdset_create("system", "uptime", NULL, "uptime", NULL, "System Uptime", "seconds", 1000, update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "uptime", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "uptime", cur_time.tv_sec - boot_time.tv_sec); + rrdset_done(st); + } + } + + return 0; +} diff --git a/src/global_statistics.c b/src/global_statistics.c index bb2b1f08f..a698615f4 100644 --- a/src/global_statistics.c +++ b/src/global_statistics.c @@ -131,7 +131,7 @@ void global_statistics_charts(void) { if (!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu"); if (!stcpu_thread) { - stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, + stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc", NULL, "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, RRDSET_TYPE_STACKED); diff --git a/src/health.c b/src/health.c index 596b143a4..193312eec 100644..100755 --- a/src/health.c +++ b/src/health.c @@ -6,6 +6,7 @@ struct health_options { const char *health_default_exec; const char *health_default_recipient; const char *log_filename; + size_t log_entries_written; FILE *log_fp; }; @@ -13,6 +14,7 @@ static struct health_options health = { .health_default_exec = PLUGINS_DIR "/alarm-notify.sh", .health_default_recipient = "root", .log_filename = VARLIB_DIR "/health/alarm_log.db", + .log_entries_written = 0, .log_fp = NULL }; @@ -30,11 +32,11 @@ static inline int health_alarm_log_open(void) { if(health.log_fp) { if (setvbuf(health.log_fp, NULL, _IOLBF, 0) != 0) - error("Cannot set line buffering on health log file."); + error("Health: cannot set line buffering on health log file."); return 0; } - error("Cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename); + error("Health: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename); return -1; } @@ -45,51 +47,309 @@ static inline void health_alarm_log_close(void) { } } -static inline void health_log_recreate(void) { - if(health.log_fp != NULL) { +static inline void health_log_rotate(void) { + static size_t rotate_every = 0; + + if(unlikely(rotate_every == 0)) { + rotate_every = (size_t)config_get_number("health", "rotate log every lines", 2000); + if(rotate_every < 100) rotate_every = 100; + } + + if(unlikely(health.log_entries_written > rotate_every)) { health_alarm_log_close(); + char old_filename[FILENAME_MAX + 1]; + snprintfz(old_filename, FILENAME_MAX, "%s.old", health.log_filename); + + if(unlink(old_filename) == -1 && errno != ENOENT) + error("Health: cannot remove old alarms log file '%s'", old_filename); + + if(link(health.log_filename, old_filename) == -1 && errno != ENOENT) + error("Health: cannot move file '%s' to '%s'.", health.log_filename, old_filename); + + if(unlink(health.log_filename) == -1 && errno != ENOENT) + error("Health: cannot remove old alarms log file '%s'", health.log_filename); + // open it with truncate health.log_fp = fopen(health.log_filename, "w"); - if(health.log_fp) fclose(health.log_fp); - else error("Cannot truncate health log '%s'", health.log_filename); + + if(health.log_fp) + fclose(health.log_fp); + else + error("Health: cannot truncate health log '%s'", health.log_filename); health.log_fp = NULL; + health.log_entries_written = 0; health_alarm_log_open(); } } static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) { - (void)host; - (void)ae; - -/* if(likely(health.log_fp)) { - if(unlikely(fprintf(health.log_fp, "A\t%s\t%08x\t%08x\t%08x\t%08x\t%08x\t%08x\t%s\t%s\t%s\t%s\t%s\t%08x\n", - host->hostname, - ae->unique_id, - ae->alarm_id, - ae->alarm_event_id, - (uint32_t)ae->when, - (uint32_t)ae->duration, - (uint32_t)ae->non_clear_duration, - (uint32_t)ae->exec_run_timestamp, - ae->name, - ae->chart, - ae->family, - ae->exec, - ae->recipient - ) < 0)) + health_log_rotate(); + + if(likely(health.log_fp)) { + if(unlikely(fprintf(health.log_fp + , "%c\t%s" + "\t%08x\t%08x\t%08x\t%08x\t%08x" + "\t%08x\t%08x\t%08x" + "\t%08x\t%08x\t%08x" + "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" + "\t%d\t%d\t%d\t%d" + "\t%Lf\t%Lf" + "\n" + , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A' + , host->hostname + + , ae->unique_id + , ae->alarm_id + , ae->alarm_event_id + , ae->updated_by_id + , ae->updates_id + + , (uint32_t)ae->when + , (uint32_t)ae->duration + , (uint32_t)ae->non_clear_duration + , (uint32_t)ae->flags + , (uint32_t)ae->exec_run_timestamp + , (uint32_t)ae->delay_up_to_timestamp + + , (ae->name)?ae->name:"" + , (ae->chart)?ae->chart:"" + , (ae->family)?ae->family:"" + , (ae->exec)?ae->exec:"" + , (ae->recipient)?ae->recipient:"" + , (ae->source)?ae->source:"" + , (ae->units)?ae->units:"" + , (ae->info)?ae->info:"" + + , ae->exec_code + , ae->new_status + , ae->old_status + , ae->delay + + , (long double)ae->new_value + , (long double)ae->old_value + ) < 0)) error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart."); + else { + ae->flags |= HEALTH_ENTRY_FLAG_SAVED; + health.log_entries_written++; + } } -*/ +} + +static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) { + static uint32_t max_unique_id = 0, max_alarm_id = 0; + + errno = 0; + + char *s, *buf = mallocz(65536 + 1); + size_t line = 0, len = 0; + ssize_t loaded = 0, updated = 0, errored = 0, duplicate = 0; + + pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock); + + while((s = fgets_trim_len(buf, 65536, fp, &len))) { + health.log_entries_written++; + line++; + + int max_entries = 30, entries = 0; + char *pointers[max_entries]; + + pointers[entries++] = s++; + while(*s) { + if(unlikely(*s == '\t')) { + *s = '\0'; + pointers[entries++] = ++s; + if(entries >= max_entries) { + error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, filename, max_entries); + break; + } + } + else s++; + } + + if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) { + ALARM_ENTRY *ae = NULL; + + if(entries < 26) { + error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring it.", line, filename, entries); + errored++; + continue; + } + + // check that we have valid ids + uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16); + if(!unique_id) { + error("Health: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", line, filename, unique_id, pointers[2]); + errored++; + continue; + } + + uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16); + if(!alarm_id) { + error("Health: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", line, filename, alarm_id, pointers[3]); + errored++; + continue; + } + + if(unlikely(*pointers[0] == 'A')) { + // make sure it is properly numbered + if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) { + error("Health: line %zu of file '%s' has alarm log entry with %u in wrong order. Ignoring it.", line, filename, unique_id); + errored++; + continue; + } + + ae = callocz(1, sizeof(ALARM_ENTRY)); + } + else if(unlikely(*pointers[0] == 'U')) { + // find the original + for(ae = host->health_log.alarms; ae; ae = ae->next) { + if(unlikely(unique_id == ae->unique_id)) { + if(unlikely(*pointers[0] == 'A')) { + error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u. Using the later." + , line, filename, unique_id); + *pointers[0] = 'U'; + duplicate++; + } + break; + } + else if(unlikely(unique_id > ae->unique_id)) { + // no need to continue + // the linked list is sorted + ae = NULL; + break; + } + } + + // if not found, skip this line + if(!ae) { + // error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, filename, unique_id); + continue; + } + } + + // check for a possible host missmatch + //if(strcmp(pointers[1], host->hostname)) + // error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, filename, pointers[1], host->hostname); + + ae->unique_id = unique_id; + ae->alarm_id = alarm_id; + ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16); + ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16); + ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16); + + ae->when = (uint32_t)strtoul(pointers[7], NULL, 16); + ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16); + ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16); + + ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16); + ae->flags |= HEALTH_ENTRY_FLAG_SAVED; + + ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16); + ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16); + + if(unlikely(ae->name)) freez(ae->name); + ae->name = strdupz(pointers[13]); + ae->hash_name = simple_hash(ae->name); + + if(unlikely(ae->chart)) freez(ae->chart); + ae->chart = strdupz(pointers[14]); + ae->hash_chart = simple_hash(ae->chart); + + if(unlikely(ae->family)) freez(ae->family); + ae->family = strdupz(pointers[15]); + + if(unlikely(ae->exec)) freez(ae->exec); + ae->exec = strdupz(pointers[16]); + if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; } + + if(unlikely(ae->recipient)) freez(ae->recipient); + ae->recipient = strdupz(pointers[17]); + if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; } + + if(unlikely(ae->source)) freez(ae->source); + ae->source = strdupz(pointers[18]); + if(!*ae->source) { freez(ae->source); ae->source = NULL; } + + if(unlikely(ae->units)) freez(ae->units); + ae->units = strdupz(pointers[19]); + if(!*ae->units) { freez(ae->units); ae->units = NULL; } + + if(unlikely(ae->info)) freez(ae->info); + ae->info = strdupz(pointers[20]); + if(!*ae->info) { freez(ae->info); ae->info = NULL; } + + ae->exec_code = str2i(pointers[21]); + ae->new_status = str2i(pointers[22]); + ae->old_status = str2i(pointers[23]); + ae->delay = str2i(pointers[24]); + + ae->new_value = str2l(pointers[25]); + ae->old_value = str2l(pointers[26]); + + // add it to host if not already there + if(unlikely(*pointers[0] == 'A')) { + ae->next = host->health_log.alarms; + host->health_log.alarms = ae; + loaded++; + } + else updated++; + + if(unlikely(ae->unique_id > max_unique_id)) + max_unique_id = ae->unique_id; + + if(unlikely(ae->alarm_id >= max_alarm_id)) + max_alarm_id = ae->alarm_id; + } + else { + error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, filename, pointers[0]); + errored++; + } + } + + pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock); + + freez(buf); + + if(!max_unique_id) max_unique_id = (uint32_t)now_realtime_sec(); + if(!max_alarm_id) max_alarm_id = (uint32_t)now_realtime_sec(); + + host->health_log.next_log_id = max_unique_id + 1; + host->health_log.next_alarm_id = max_alarm_id + 1; + + debug(D_HEALTH, "Health: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", filename, loaded, updated, errored, duplicate); + return loaded; } static inline void health_alarm_log_load(RRDHOST *host) { - (void)host; + health_alarm_log_close(); + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s.old", health.log_filename); + FILE *fp = fopen(filename, "r"); + if(!fp) + error("Health: cannot open health file: %s", filename); + else { + health_alarm_log_read(host, fp, filename); + fclose(fp); + } + + health.log_entries_written = 0; + fp = fopen(health.log_filename, "r"); + if(!fp) + error("Health: cannot open health file: %s", health.log_filename); + else { + health_alarm_log_read(host, fp, health.log_filename); + fclose(fp); + } + + health_alarm_log_open(); } + // ---------------------------------------------------------------------------- // health alarm log management @@ -152,8 +412,8 @@ static inline void health_alarm_log(RRDHOST *host, ALARM_ENTRY *t; for(t = host->health_log.alarms ; t ; t = t->next) { if(t != ae && t->alarm_id == ae->alarm_id) { - if(!(t->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) && !t->updated_by_id) { - t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED; + if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) { + t->flags |= HEALTH_ENTRY_FLAG_UPDATED; t->updated_by_id = ae->unique_id; ae->updates_id = t->unique_id; @@ -163,10 +423,9 @@ static inline void health_alarm_log(RRDHOST *host, health_alarm_log_save(host, t); } - else { - // no need to continue - break; - } + + // no need to continue + break; } } pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock); @@ -226,14 +485,17 @@ static inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) { if(!rv) return; - if(tree) - rrdvar_index_del(tree, rv); + if(tree) { + debug(D_VARIABLES, "Deleting variable '%s'", rv->name); + if(unlikely(!rrdvar_index_del(tree, rv))) + error("Attempted to delete variable '%s' from host '%s', but it is not found.", rv->name, host->hostname); + } freez(rv->name); freez(rv); } -static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, calculated_number *value) { +static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, void *value) { char *variable = strdupz(name); rrdvar_fix_name(variable); uint32_t hash = simple_hash(variable); @@ -258,8 +520,13 @@ static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock * debug(D_VARIABLES, "Variable '%s' created in scope '%s'", variable, scope); } else { + debug(D_VARIABLES, "Variable '%s' is already found in scope '%s'.", variable, scope); + // already exists freez(variable); + + // this is important + // it must return NULL - not the existing variable - or double-free will happen rv = NULL; } @@ -267,10 +534,69 @@ static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock * } // ---------------------------------------------------------------------------- +// CUSTOM VARIABLES + +RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) { + calculated_number *v = callocz(1, sizeof(calculated_number)); + *v = NAN; + RRDVAR *rv = rrdvar_create_and_index("host", &host->variables_root_index, name, RRDVAR_TYPE_CALCULATED_ALLOCATED, v); + if(unlikely(!rv)) { + free(v); + error("Requested variable '%s' already exists - possibly 2 plugins will be updating it at the same time", name); + + char *variable = strdupz(name); + rrdvar_fix_name(variable); + uint32_t hash = simple_hash(variable); + + rv = rrdvar_index_find(&host->variables_root_index, variable, hash); + } + + return rv; +} + +void rrdvar_custom_host_variable_destroy(RRDHOST *host, const char *name) { + char *variable = strdupz(name); + rrdvar_fix_name(variable); + uint32_t hash = simple_hash(variable); + + RRDVAR *rv = rrdvar_index_find(&host->variables_root_index, variable, hash); + freez(variable); + + if(!rv) { + error("Attempted to remove variable '%s' from host '%s', but it does not exist.", name, host->hostname); + return; + } + + if(rv->type != RRDVAR_TYPE_CALCULATED_ALLOCATED) { + error("Attempted to remove variable '%s' from host '%s', but it does not a custom allocated variable.", name, host->hostname); + return; + } + + if(!rrdvar_index_del(&host->variables_root_index, rv)) { + error("Attempted to remove variable '%s' from host '%s', but it cannot be found.", name, host->hostname); + return; + } + + freez(rv->name); + freez(rv->value); + freez(rv); +} + +void rrdvar_custom_host_variable_set(RRDVAR *rv, calculated_number value) { + if(rv->type != RRDVAR_TYPE_CALCULATED_ALLOCATED) + error("requested to set variable '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rv->name, value); + else { + calculated_number *v = rv->value; + *v = value; + } +} + +// ---------------------------------------------------------------------------- // RRDVAR lookup -calculated_number rrdvar2number(RRDVAR *rv) { +static calculated_number rrdvar2number(RRDVAR *rv) { switch(rv->type) { + case RRDVAR_TYPE_CALCULATED_ALLOCATED: case RRDVAR_TYPE_CALCULATED: { calculated_number *n = (calculated_number *)rv->value; return *n; @@ -302,11 +628,6 @@ calculated_number rrdvar2number(RRDVAR *rv) { } } -void dump_variable(void *data) { - RRDVAR *rv = (RRDVAR *)data; - debug(D_HEALTH, "%50s : %20.5Lf", rv->name, rrdvar2number(rv)); -} - int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) { RRDSET *st = rc->rrdset; RRDVAR *rv; @@ -331,162 +652,219 @@ int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, cal return 1; } - debug(D_HEALTH, "Available local chart '%s' variables:", st->id); - avl_traverse_lock(&st->variables_root_index, dump_variable); + return 0; +} - debug(D_HEALTH, "Available family '%s' variables:", st->rrdfamily->family); - avl_traverse_lock(&st->rrdfamily->variables_root_index, dump_variable); +// ---------------------------------------------------------------------------- +// RRDVAR to JSON - debug(D_HEALTH, "Available host '%s' variables:", st->rrdhost->hostname); - avl_traverse_lock(&st->rrdhost->variables_root_index, dump_variable); +struct variable2json_helper { + BUFFER *buf; + size_t counter; +}; + +static int single_variable2json(void *entry, void *data) { + struct variable2json_helper *helper = (struct variable2json_helper *)data; + RRDVAR *rv = (RRDVAR *)entry; + calculated_number value = rrdvar2number(rv); + + if(unlikely(isnan(value) || isinf(value))) + buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": null", helper->counter?",":"", rv->name); + else + buffer_sprintf(helper->buf, "%s\n\t\t\"%s\": %0.5Lf", helper->counter?",":"", rv->name, (long double)value); + + helper->counter++; return 0; } +void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf) { + struct variable2json_helper helper = { + .buf = buf, + .counter = 0 + }; + + buffer_sprintf(buf, "{\n\t\"chart\": \"%s\",\n\t\"chart_name\": \"%s\",\n\t\"chart_context\": \"%s\",\n\t\"chart_variables\": {", st->id, st->name, st->context); + avl_traverse_lock(&st->variables_root_index, single_variable2json, (void *)&helper); + buffer_sprintf(buf, "\n\t},\n\t\"family\": \"%s\",\n\t\"family_variables\": {", st->family); + helper.counter = 0; + avl_traverse_lock(&st->rrdfamily->variables_root_index, single_variable2json, (void *)&helper); + buffer_sprintf(buf, "\n\t},\n\t\"host\": \"%s\",\n\t\"host_variables\": {", st->rrdhost->hostname); + helper.counter = 0; + avl_traverse_lock(&st->rrdhost->variables_root_index, single_variable2json, (void *)&helper); + buffer_strcat(buf, "\n\t}\n}\n"); +} + + // ---------------------------------------------------------------------------- -// RRDSETVAR management +// RRDDIMVAR management +// DIMENSION VARIABLES -RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options) { - debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable); - RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR)); +#define RRDDIMVAR_ID_MAX 1024 - char buffer[RRDVAR_MAX_LENGTH + 1]; - snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, variable); - rs->fullid = strdupz(buffer); +static inline void rrddimvar_free_variables(RRDDIMVAR *rs) { + RRDDIM *rd = rs->rrddim; + RRDSET *st = rd->rrdset; - snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, variable); - rs->fullname = strdupz(buffer); + // CHART VARIABLES FOR THIS DIMENSION - rs->variable = strdupz(variable); + rrdvar_free(st->rrdhost, &st->variables_root_index, rs->var_local_id); + rs->var_local_id = NULL; - rs->type = type; - rs->value = value; - rs->options = options; - rs->rrdset = st; + rrdvar_free(st->rrdhost, &st->variables_root_index, rs->var_local_name); + rs->var_local_name = NULL; - rs->local = rrdvar_create_and_index("local", &st->variables_root_index, rs->variable, rs->type, rs->value); - rs->family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullid, rs->type, rs->value); - rs->host = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullid, rs->type, rs->value); - rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value); - rs->host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value); + // FAMILY VARIABLES FOR THIS DIMENSION - rs->next = st->variables; - st->variables = rs; + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_id); + rs->var_family_id = NULL; - return rs; -} + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_name); + rs->var_family_name = NULL; -void rrdsetvar_rename_all(RRDSET *st) { - debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name); + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_contextid); + rs->var_family_contextid = NULL; - // only these 2 can change name - // rs->family_name - // rs->host_name + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_contextname); + rs->var_family_contextname = NULL; - char buffer[RRDVAR_MAX_LENGTH + 1]; - RRDSETVAR *rs, *next = st->variables; - while((rs = next)) { - next = rs->next; + // HOST VARIABLES FOR THIS DIMENSION - snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable); + rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartidid); + rs->var_host_chartidid = NULL; - if (strcmp(buffer, rs->fullname)) { - // name changed - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name); - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name); + rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartidname); + rs->var_host_chartidname = NULL; - freez(rs->fullname); - rs->fullname = strdupz(st->name); - rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value); - rs->host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value); - } - } + rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartnameid); + rs->var_host_chartnameid = NULL; - rrdsetcalc_link_matching(st); -} + rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_chartnamename); + rs->var_host_chartnamename = NULL; -void rrdsetvar_free(RRDSETVAR *rs) { - RRDSET *st = rs->rrdset; - debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable); + // KEYS - rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local); - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family); - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host); - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name); - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name); + freez(rs->key_id); + rs->key_id = NULL; - if(st->variables == rs) { - st->variables = rs->next; - } - else { - RRDSETVAR *t; - for (t = st->variables; t && t->next != rs; t = t->next); - if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->fullname, st->id); - else t->next = rs->next; - } + freez(rs->key_name); + rs->key_name = NULL; - freez(rs->fullid); - freez(rs->fullname); - freez(rs->variable); - freez(rs); -} + freez(rs->key_fullidid); + rs->key_fullidid = NULL; -// ---------------------------------------------------------------------------- -// RRDDIMVAR management + freez(rs->key_fullidname); + rs->key_fullidname = NULL; -#define RRDDIMVAR_ID_MAX 1024 + freez(rs->key_contextid); + rs->key_contextid = NULL; -RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options) { - RRDSET *st = rd->rrdset; + freez(rs->key_contextname); + rs->key_contextname = NULL; - debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:""); + freez(rs->key_fullnameid); + rs->key_fullnameid = NULL; - if(!prefix) prefix = ""; - if(!suffix) suffix = ""; + freez(rs->key_fullnamename); + rs->key_fullnamename = NULL; +} + +static inline void rrddimvar_create_variables(RRDDIMVAR *rs) { + rrddimvar_free_variables(rs); + + RRDDIM *rd = rs->rrddim; + RRDSET *st = rd->rrdset; char buffer[RRDDIMVAR_ID_MAX + 1]; - RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR)); - rs->prefix = strdupz(prefix); - rs->suffix = strdupz(suffix); + // KEYS snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->id, rs->suffix); - rs->id = strdupz(buffer); + rs->key_id = strdupz(buffer); snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix); - rs->name = strdupz(buffer); + rs->key_name = strdupz(buffer); + + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->key_id); + rs->key_fullidid = strdupz(buffer); + + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->key_name); + rs->key_fullidname = strdupz(buffer); + + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->context, rs->key_id); + rs->key_contextid = strdupz(buffer); + + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->context, rs->key_name); + rs->key_contextname = strdupz(buffer); + + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->key_id); + rs->key_fullnameid = strdupz(buffer); + + snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->key_name); + rs->key_fullnamename = strdupz(buffer); + + // CHART VARIABLES FOR THIS DIMENSION + // ----------------------------------- + // + // dimensions are available as: + // - $id + // - $name + + rs->var_local_id = rrdvar_create_and_index("local", &st->variables_root_index, rs->key_id, rs->type, rs->value); + rs->var_local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->key_name, rs->type, rs->value); + + // FAMILY VARIABLES FOR THIS DIMENSION + // ----------------------------------- + // + // dimensions are available as: + // - $id (only the first, when multiple overlap) + // - $name (only the first, when multiple overlap) + // - $chart-context.id + // - $chart-context.name + + rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_id, rs->type, rs->value); + rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_name, rs->type, rs->value); + rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_contextid, rs->type, rs->value); + rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_contextname, rs->type, rs->value); + + // HOST VARIABLES FOR THIS DIMENSION + // ----------------------------------- + // + // dimensions are available as: + // - $chart-id.id + // - $chart-id.name + // - $chart-name.id + // - $chart-name.name + + rs->var_host_chartidid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullidid, rs->type, rs->value); + rs->var_host_chartidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullidname, rs->type, rs->value); + rs->var_host_chartnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullnameid, rs->type, rs->value); + rs->var_host_chartnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullnamename, rs->type, rs->value); +} - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->id); - rs->fullidid = strdupz(buffer); +RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options) { + RRDSET *st = rd->rrdset; - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->name); - rs->fullidname = strdupz(buffer); + debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:""); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->id); - rs->fullnameid = strdupz(buffer); + if(!prefix) prefix = ""; + if(!suffix) suffix = ""; + + RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR)); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->name); - rs->fullnamename = strdupz(buffer); + rs->prefix = strdupz(prefix); + rs->suffix = strdupz(suffix); rs->type = type; rs->value = value; rs->options = options; rs->rrddim = rd; - rs->local_id = rrdvar_create_and_index("local", &st->variables_root_index, rs->id, rs->type, rs->value); - rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value); - - rs->family_id = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->id, rs->type, rs->value); - rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->name, rs->type, rs->value); - - rs->host_fullidid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidid, rs->type, rs->value); - rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidname, rs->type, rs->value); - rs->host_fullnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnameid, rs->type, rs->value); - rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnamename, rs->type, rs->value); - rs->next = rd->variables; rd->variables = rs; + rrddimvar_create_variables(rs); + return rs; } @@ -497,41 +875,7 @@ void rrddimvar_rename_all(RRDDIM *rd) { RRDDIMVAR *rs, *next = rd->variables; while((rs = next)) { next = rs->next; - - if (strcmp(rd->name, rs->name)) { - char buffer[RRDDIMVAR_ID_MAX + 1]; - // name changed - - // name - rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name); - freez(rs->name); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix); - rs->name = strdupz(buffer); - rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value); - - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname); - freez(rs->fullidname); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->name); - rs->fullidname = strdupz(buffer); - rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, - rs->fullidname, rs->type, rs->value); - - // fullnameid - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid); - freez(rs->fullnameid); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->id); - rs->fullnameid = strdupz(buffer); - rs->host_fullnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, - rs->fullnameid, rs->type, rs->value); - - // fullnamename - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename); - freez(rs->fullnamename); - snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->name); - rs->fullnamename = strdupz(buffer); - rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, - rs->fullnamename, rs->type, rs->value); - } + rrddimvar_create_variables(rs); } } @@ -540,16 +884,7 @@ void rrddimvar_free(RRDDIMVAR *rs) { RRDSET *st = rd->rrdset; debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", st->id, st->name, rd->id, rd->name, rs->prefix, rs->suffix); - rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_id); - rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name); - - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_id); - rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name); - - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidid); - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname); - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid); - rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename); + rrddimvar_free_variables(rs); if(rd->variables == rs) { debug(D_VARIABLES, "RRDDIMSET removing first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name); @@ -559,25 +894,136 @@ void rrddimvar_free(RRDDIMVAR *rs) { debug(D_VARIABLES, "RRDDIMSET removing non-first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name); RRDDIMVAR *t; for (t = rd->variables; t && t->next != rs; t = t->next) ; - if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->name, st->id, rd->id); + if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->key_name, st->id, rd->id); else t->next = rs->next; } freez(rs->prefix); freez(rs->suffix); - freez(rs->id); - freez(rs->name); - freez(rs->fullidid); - freez(rs->fullidname); - freez(rs->fullnameid); - freez(rs->fullnamename); + freez(rs); +} + +// ---------------------------------------------------------------------------- +// RRDSETVAR management +// CHART VARIABLES + +static inline void rrdsetvar_free_variables(RRDSETVAR *rs) { + RRDSET *st = rs->rrdset; + + // CHART + + rrdvar_free(st->rrdhost, &st->variables_root_index, rs->var_local); + rs->var_local = NULL; + + // FAMILY + + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family); + rs->var_family = NULL; + + rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host); + rs->var_host = NULL; + + // HOST + + rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->var_family_name); + rs->var_family_name = NULL; + + rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->var_host_name); + rs->var_host_name = NULL; + + // KEYS + + freez(rs->key_fullid); + rs->key_fullid = NULL; + + freez(rs->key_fullname); + rs->key_fullname = NULL; +} + +static inline void rrdsetvar_create_variables(RRDSETVAR *rs) { + rrdsetvar_free_variables(rs); + + RRDSET *st = rs->rrdset; + + // KEYS + + char buffer[RRDVAR_MAX_LENGTH + 1]; + snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rs->variable); + rs->key_fullid = strdupz(buffer); + + snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable); + rs->key_fullname = strdupz(buffer); + + // CHART + + rs->var_local = rrdvar_create_and_index("local", &st->variables_root_index, rs->variable, rs->type, rs->value); + + // FAMILY + + rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_fullid, rs->type, rs->value); + rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->key_fullname, rs->type, rs->value); + + // HOST + + rs->var_host = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullid, rs->type, rs->value); + rs->var_host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->key_fullname, rs->type, rs->value); + +} + +RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options) { + debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable); + RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR)); + + rs->variable = strdupz(variable); + rs->type = type; + rs->value = value; + rs->options = options; + rs->rrdset = st; + + rs->next = st->variables; + st->variables = rs; + + rrdsetvar_create_variables(rs); + + return rs; +} + +void rrdsetvar_rename_all(RRDSET *st) { + debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name); + + RRDSETVAR *rs, *next = st->variables; + while((rs = next)) { + next = rs->next; + rrdsetvar_create_variables(rs); + } + + rrdsetcalc_link_matching(st); +} + +void rrdsetvar_free(RRDSETVAR *rs) { + RRDSET *st = rs->rrdset; + debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable); + + if(st->variables == rs) { + st->variables = rs->next; + } + else { + RRDSETVAR *t; + for (t = st->variables; t && t->next != rs; t = t->next); + if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->key_fullname, st->id); + else t->next = rs->next; + } + + rrdsetvar_free_variables(rs); + + freez(rs->variable); freez(rs); } // ---------------------------------------------------------------------------- // RRDCALC management -static inline const char *rrdcalc_status2string(int status) { +inline const char *rrdcalc_status2string(int status) { switch(status) { case RRDCALC_STATUS_REMOVED: return "REMOVED"; @@ -609,7 +1055,7 @@ static inline const char *rrdcalc_status2string(int status) { static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, st->rrdhost->hostname); - rc->last_status_change = time(NULL); + rc->last_status_change = now_realtime_sec(); rc->rrdset = st; rc->rrdset_next = st->alarms; @@ -648,7 +1094,7 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) { if(!rc->units) rc->units = strdupz(st->units); { - time_t now = time(NULL); + time_t now = now_realtime_sec(); health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_UNINITIALIZED, rc->source, rc->units, rc->info, 0); } } @@ -686,7 +1132,7 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) { } { - time_t now = time(NULL); + time_t now = now_realtime_sec(); health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info, 0); } @@ -954,7 +1400,8 @@ void rrdcalctemplate_link_matching(RRDSET *st) { RRDCALCTEMPLATE *rt; for(rt = st->rrdhost->templates; rt ; rt = rt->next) { - if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) { + if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context) + && (!rt->family_pattern || simple_pattern_matches(rt->family_pattern, st->family))) { RRDCALC *rc = rrdcalc_create(st->rrdhost, rt, st->id); if(unlikely(!rc)) error("Health tried to create alarm from template '%s', but it failed", rt->name); @@ -990,6 +1437,9 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) { expression_free(rt->warning); expression_free(rt->critical); + freez(rt->family_match); + simple_pattern_free(rt->family_pattern); + freez(rt->name); freez(rt->exec); freez(rt->recipient); @@ -1009,6 +1459,7 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) { #define HEALTH_ALARM_KEY "alarm" #define HEALTH_TEMPLATE_KEY "template" #define HEALTH_ON_KEY "on" +#define HEALTH_FAMILIES_KEY "families" #define HEALTH_LOOKUP_KEY "lookup" #define HEALTH_CALC_KEY "calc" #define HEALTH_EVERY_KEY "every" @@ -1353,6 +1804,16 @@ static inline int health_parse_db_lookup( return 1; } +static inline char *tabs2spaces(char *s) { + char *t = s; + while(*t) { + if(unlikely(*t == '\t')) *t = ' '; + t++; + } + + return s; +} + static inline char *health_source_file(size_t line, const char *path, const char *filename) { char buffer[FILENAME_MAX + 1]; snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename); @@ -1369,13 +1830,14 @@ static inline void strip_quotes(char *s) { int health_readfile(const char *path, const char *filename) { debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename); - static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0, hash_delay = 0; + static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_families = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0, hash_delay = 0; char buffer[HEALTH_CONF_MAX_LINE + 1]; if(unlikely(!hash_alarm)) { hash_alarm = simple_uhash(HEALTH_ALARM_KEY); hash_template = simple_uhash(HEALTH_TEMPLATE_KEY); hash_on = simple_uhash(HEALTH_ON_KEY); + hash_families = simple_uhash(HEALTH_FAMILIES_KEY); hash_calc = simple_uhash(HEALTH_CALC_KEY); hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY); hash_green = simple_uhash(HEALTH_GREEN_KEY); @@ -1405,10 +1867,8 @@ int health_readfile(const char *path, const char *filename) { while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) { int stop_appending = !s; line++; - // info("Line %zu of file '%s/%s': '%s'", line, path, filename, s); s = trim(buffer); if(!s) continue; - // info("Trimmed line %zu of file '%s/%s': '%s'", line, path, filename, s); append = strlen(s); if(!stop_appending && s[append - 1] == '\\') { @@ -1445,7 +1905,6 @@ int health_readfile(const char *path, const char *filename) { continue; } - // info("Health file '%s/%s', key '%s', value '%s'", path, filename, key, value); uint32_t hash = simple_uhash(key); if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) { @@ -1460,7 +1919,7 @@ int health_readfile(const char *path, const char *filename) { rc = callocz(1, sizeof(RRDCALC)); rc->next_event_id = 1; - rc->name = strdupz(value); + rc->name = tabs2spaces(strdupz(value)); rc->hash = simple_hash(rc->name); rc->source = health_source_file(line, path, filename); rc->green = NAN; @@ -1483,7 +1942,7 @@ int health_readfile(const char *path, const char *filename) { rrdcalctemplate_free(&localhost, rt); rt = callocz(1, sizeof(RRDCALCTEMPLATE)); - rt->name = strdupz(value); + rt->name = tabs2spaces(strdupz(value)); rt->hash_name = simple_hash(rt->name); rt->source = health_source_file(line, path, filename); rt->green = NAN; @@ -1497,12 +1956,12 @@ int health_readfile(const char *path, const char *filename) { if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) { if(rc->chart) { if(strcmp(rc->chart, value)) - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, path, filename, rc->name, key, rc->chart, value, value); + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, path, filename, rc->name, key, rc->chart, value, value); freez(rc->chart); } - rc->chart = strdupz(value); + rc->chart = tabs2spaces(strdupz(value)); rc->hash_chart = simple_hash(rc->chart); } else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) { @@ -1512,14 +1971,14 @@ int health_readfile(const char *path, const char *filename) { } else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) { if(!health_parse_duration(value, &rc->update_every)) - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.", + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.", line, path, filename, rc->name, key, value); } else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { char *e; rc->green = strtold(value, &e); if(e && *e) { - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rc->name, key, e); } } @@ -1527,7 +1986,7 @@ int health_readfile(const char *path, const char *filename) { char *e; rc->red = strtold(value, &e); if(e && *e) { - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rc->name, key, e); } } @@ -1561,43 +2020,43 @@ int health_readfile(const char *path, const char *filename) { else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) { if(rc->exec) { if(strcmp(rc->exec, value)) - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rc->name, key, rc->exec, value, value); freez(rc->exec); } - rc->exec = strdupz(value); + rc->exec = tabs2spaces(strdupz(value)); } else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) { if(rc->recipient) { if(strcmp(rc->recipient, value)) - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rc->name, key, rc->recipient, value, value); freez(rc->recipient); } - rc->recipient = strdupz(value); + rc->recipient = tabs2spaces(strdupz(value)); } else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) { if(rc->units) { if(strcmp(rc->units, value)) - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rc->name, key, rc->units, value, value); freez(rc->units); } - rc->units = strdupz(value); + rc->units = tabs2spaces(strdupz(value)); strip_quotes(rc->units); } else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) { if(rc->info) { if(strcmp(rc->info, value)) - info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rc->name, key, rc->info, value, value); freez(rc->info); } - rc->info = strdupz(value); + rc->info = tabs2spaces(strdupz(value)); strip_quotes(rc->info); } else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) { @@ -1612,29 +2071,35 @@ int health_readfile(const char *path, const char *filename) { if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) { if(rt->context) { if(strcmp(rt->context, value)) - info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", - line, path, filename, rt->name, key, rt->context, value, value); + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + line, path, filename, rt->name, key, rt->context, value, value); freez(rt->context); } - rt->context = strdupz(value); + rt->context = tabs2spaces(strdupz(value)); rt->hash_context = simple_hash(rt->context); } + else if(hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) { + freez(rt->family_match); + simple_pattern_free(rt->family_pattern); + + rt->family_match = tabs2spaces(strdupz(value)); + rt->family_pattern = simple_pattern_create(rt->family_match, SIMPLE_PATTERN_EXACT); + } else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) { health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before, - &rt->update_every, - &rt->options, &rt->dimensions); + &rt->update_every, &rt->options, &rt->dimensions); } else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) { if(!health_parse_duration(value, &rt->update_every)) - info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.", + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.", line, path, filename, rt->name, key, value); } else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) { char *e; rt->green = strtold(value, &e); if(e && *e) { - info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rt->name, key, e); } } @@ -1642,7 +2107,7 @@ int health_readfile(const char *path, const char *filename) { char *e; rt->red = strtold(value, &e); if(e && *e) { - info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", + error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.", line, path, filename, rt->name, key, e); } } @@ -1676,43 +2141,43 @@ int health_readfile(const char *path, const char *filename) { else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) { if(rt->exec) { if(strcmp(rt->exec, value)) - info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rt->name, key, rt->exec, value, value); freez(rt->exec); } - rt->exec = strdupz(value); + rt->exec = tabs2spaces(strdupz(value)); } else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) { if(rt->recipient) { if(strcmp(rt->recipient, value)) - info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rt->name, key, rt->recipient, value, value); freez(rt->recipient); } - rt->recipient = strdupz(value); + rt->recipient = tabs2spaces(strdupz(value)); } else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) { if(rt->units) { if(strcmp(rt->units, value)) - info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rt->name, key, rt->units, value, value); freez(rt->units); } - rt->units = strdupz(value); + rt->units = tabs2spaces(strdupz(value)); strip_quotes(rt->units); } else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) { if(rt->info) { if(strcmp(rt->info, value)) - info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", + error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').", line, path, filename, rt->name, key, rt->info, value, value); freez(rt->info); } - rt->info = strdupz(value); + rt->info = tabs2spaces(strdupz(value)); strip_quotes(rt->info); } else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) { @@ -1798,7 +2263,16 @@ void health_init(void) { return; } + char *pathname = config_get("health", "health db directory", VARLIB_DIR "/health"); + if(mkdir(pathname, 0770) == -1 && errno != EEXIST) + fatal("Cannot create directory '%s'.", pathname); + + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/health-log.db", pathname); + health.log_filename = config_get("health", "health db file", filename); + health_alarm_log_load(&localhost); + health_alarm_log_open(); char *path = health_config_dir(); @@ -1865,10 +2339,10 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R ae->name, ae->chart, ae->family, - (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)?"true":"false", - (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)?"true":"false", + (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false", + (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false", (unsigned long)ae->exec_run_timestamp, - (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false", + (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false", ae->exec?ae->exec:health.health_default_exec, ae->recipient?ae->recipient:health.health_default_recipient, ae->exec_code, @@ -2032,7 +2506,7 @@ void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) { host->hostname, (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0, health_enabled?"true":"false", - (unsigned long)time(NULL)); + (unsigned long)now_realtime_sec()); RRDCALC *rc; for(i = 0, rc = host->alarms; rc ; rc = rc->next) { @@ -2085,7 +2559,7 @@ void health_reload(void) { ALARM_ENTRY *t; for(t = localhost.health_log.alarms ; t ; t = t->next) { if(t->new_status != RRDCALC_STATUS_REMOVED) - t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED; + t->flags |= HEALTH_ENTRY_FLAG_UPDATED; } // reset all thresholds to all charts @@ -2115,34 +2589,52 @@ void health_reload(void) { // health main thread and friends static inline int rrdcalc_value2status(calculated_number n) { - if(isnan(n)) return RRDCALC_STATUS_UNDEFINED; + if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED; if(n) return RRDCALC_STATUS_RAISED; return RRDCALC_STATUS_CLEAR; } -static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { - ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED; +#define ALARM_EXEC_COMMAND_LENGTH 8192 - // find the previous notification for the same alarm - ALARM_ENTRY *t; - for(t = ae->next; t ;t = t->next) { - if(t->alarm_id == ae->alarm_id && t->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN) - break; - } +static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { + ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED; - if(t && t->new_status == ae->new_status) { - // don't send the same notification again - info("Health not sending again notification for alarm '%s.%s' status %s", ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); + if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) { + // do not send notifications for internal statuses goto done; } - if((ae->old_status == RRDCALC_STATUS_UNDEFINED && ae->new_status == RRDCALC_STATUS_UNINITIALIZED) - || (ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)) { - info("Health not sending notification for first initialization of alarm '%s.%s' status %s", ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); - goto done; + // find the previous notification for the same alarm + // which we have run the exec script + { + uint32_t id = ae->alarm_id; + ALARM_ENTRY *t; + for(t = ae->next; t ; t = t->next) { + if(t->alarm_id == id && t->flags & HEALTH_ENTRY_FLAG_EXEC_RUN) + break; + } + + if(likely(t)) { + // we have executed this alarm notification in the past + if(t && t->new_status == ae->new_status) { + // don't send the notification for the same status again + debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae->chart, ae->name + , rrdcalc_status2string(ae->new_status)); + goto done; + } + } + else { + // we have not executed this alarm notification in the past + // so, don't send CLEAR notifications + if(unlikely(ae->new_status == RRDCALC_STATUS_CLEAR)) { + debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s" + , ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); + goto done; + } + } } - char buffer[FILENAME_MAX + 1]; + static char command_to_run[ALARM_EXEC_COMMAND_LENGTH + 1]; pid_t command_pid; const char *exec = ae->exec; @@ -2151,7 +2643,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { const char *recipient = ae->recipient; if(!recipient) recipient = health.health_default_recipient; - snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'", + snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'", exec, recipient, host->hostname, @@ -2173,23 +2665,23 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { ae->info?ae->info:"" ); - ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN; - ae->exec_run_timestamp = time(NULL); + ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN; + ae->exec_run_timestamp = now_realtime_sec(); - debug(D_HEALTH, "executing command '%s'", buffer); - FILE *fp = mypopen(buffer, &command_pid); + debug(D_HEALTH, "executing command '%s'", command_to_run); + FILE *fp = mypopen(command_to_run, &command_pid); if(!fp) { - error("HEALTH: Cannot popen(\"%s\", \"r\").", buffer); + error("HEALTH: Cannot popen(\"%s\", \"r\").", command_to_run); goto done; } debug(D_HEALTH, "HEALTH reading from command"); - char *s = fgets(buffer, FILENAME_MAX, fp); + char *s = fgets(command_to_run, FILENAME_MAX, fp); (void)s; ae->exec_code = mypclose(fp, command_pid); debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code); if(ae->exec_code != 0) - ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED; + ae->flags |= HEALTH_ENTRY_FLAG_EXEC_FAILED; done: health_alarm_log_save(host, ae); @@ -2197,7 +2689,7 @@ done: } static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) { - info("Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s", + debug(D_HEALTH, "Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s", ae->chart?ae->chart:"NOCHART", ae->name, ae->new_value, rrdcalc_status2string(ae->old_status), @@ -2210,15 +2702,15 @@ static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) static inline void health_alarm_log_process(RRDHOST *host) { static uint32_t stop_at_id = 0; uint32_t first_waiting = (host->health_log.alarms)?host->health_log.alarms->unique_id:0; - time_t now = time(NULL); + time_t now = now_realtime_sec(); pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock); ALARM_ENTRY *ae; for(ae = host->health_log.alarms; ae && ae->unique_id >= stop_at_id ; ae = ae->next) { if(unlikely( - !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) && - !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) + !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) && + !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED) )) { if(unlikely(ae->unique_id < first_waiting)) @@ -2272,40 +2764,61 @@ static inline void health_alarm_log_process(RRDHOST *host) { } static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) { - if (unlikely(!rc->rrdset)) { + if(unlikely(!rc->rrdset)) { debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name); return 0; } - if (unlikely(!rc->rrdset->last_collected_time.tv_sec)) { - debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not yet collected.", rc->chart?rc->chart:"NOCHART", rc->name); + if(unlikely(rc->next_update > now)) { + if (unlikely(*next_run > rc->next_update)) { + // update the next_run time of the main loop + // to run this alarm precisely the time required + *next_run = rc->next_update; + } + + debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now)); return 0; } - if (unlikely(!rc->update_every)) { + if(unlikely(!rc->update_every)) { debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name); return 0; } - if (unlikely(rc->next_update > now)) { - if (unlikely(*next_run > rc->next_update)) - *next_run = rc->next_update; + if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) { + debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name); + return 0; + } - debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now)); + int update_every = rc->rrdset->update_every; + time_t first = rrdset_first_entry_t(rc->rrdset); + time_t last = rrdset_last_entry_t(rc->rrdset); + + if(unlikely(now + update_every < first /* || now - update_every > last */)) { + debug(D_HEALTH + , "Health not examining alarm '%s.%s' yet (wanted time is out of bounds - we need %lu but got %lu - %lu)." + , rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) now, (unsigned long) first + , (unsigned long) last); return 0; } - // FIXME - // we should check that the DB lookup is possible - // i.e. - // - the duration of the chart includes the required timeframe - // we SHOULD NOT check the dimensions - there might be alarms that refer non-existing dimensions (e.g. cpu steal) + if(RRDCALC_HAS_DB_LOOKUP(rc)) { + time_t needed = now + rc->before + rc->after; + + if(needed + update_every < first || needed - update_every > last) { + debug(D_HEALTH + , "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu)." + , rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) needed, (unsigned long) first + , (unsigned long) last); + return 0; + } + } return 1; } void *health_main(void *ptr) { - (void)ptr; + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; info("HEALTH thread created with task id %d", gettid()); @@ -2326,28 +2839,32 @@ void *health_main(void *ptr) { debug(D_HEALTH, "Health monitoring iteration no %u started", loop); int oldstate, runnable = 0; - time_t now = time(NULL); + time_t now = now_realtime_sec(); time_t next_run = now + min_run_every; RRDCALC *rc; - if (unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)) + if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)) error("Cannot set pthread cancel state to DISABLE."); rrdhost_rdlock(&localhost); // the first loop is to lookup values from the db - for (rc = localhost.alarms; rc; rc = rc->next) { - if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) + for(rc = localhost.alarms; rc; rc = rc->next) { + if(unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) { + if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)) + rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE; continue; + } runnable++; rc->old_value = rc->value; + rc->rrdcalc_flags |= RRDCALC_FLAG_RUNNABLE; // 1. if there is database lookup, do it // 2. if there is calculation expression, run it if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) { - time_t old_db_timestamp = rc->db_before; + /* time_t old_db_timestamp = rc->db_before; */ int value_is_null = 0; int ret = rrd2value(rc->rrdset, wb, &rc->value, @@ -2368,6 +2885,7 @@ void *health_main(void *ptr) { else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR)) rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR; + /* - RRDCALC_FLAG_DB_STALE not currently used if (unlikely(old_db_timestamp == rc->db_before)) { // database is stale @@ -2380,6 +2898,7 @@ void *health_main(void *ptr) { } else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE)) rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE; + */ if (unlikely(value_is_null)) { // collected value is null @@ -2408,23 +2927,24 @@ void *health_main(void *ptr) { rc->value = NAN; - debug(D_HEALTH, "Health alarm '%s.%s': failed to evaluate calculation with error: %s", - rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg)); + debug(D_HEALTH, "Health alarm '%s.%s': expression '%s' failed: %s", + rc->chart?rc->chart:"NOCHART", rc->name, rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)); if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))) { rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR; - error("Health alarm '%s.%s': failed to evaluate calculation with error: %s", - rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg)); + error("Health alarm '%s.%s': expression '%s' failed: %s", + rc->chart?rc->chart:"NOCHART", rc->name, rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)); } } else { if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR)) rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR; - debug(D_HEALTH, "Health alarm '%s.%s': calculation expression gave value " + debug(D_HEALTH, "Health alarm '%s.%s': expression '%s' gave value " CALCULATED_NUMBER_FORMAT ": %s (source: %s)", rc->chart?rc->chart:"NOCHART", rc->name, + rc->calculation->parsed_as, rc->calculation->result, buffer_tostring(rc->calculation->error_msg), rc->source @@ -2436,11 +2956,11 @@ void *health_main(void *ptr) { } rrdhost_unlock(&localhost); - if (unlikely(runnable && !netdata_exit)) { + if(unlikely(runnable && !netdata_exit)) { rrdhost_rdlock(&localhost); - for (rc = localhost.alarms; rc; rc = rc->next) { - if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) + for(rc = localhost.alarms; rc; rc = rc->next) { + if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))) continue; int warning_status = RRDCALC_STATUS_UNDEFINED; @@ -2591,11 +3111,11 @@ void *health_main(void *ptr) { if(unlikely(netdata_exit)) break; - now = time(NULL); + now = now_realtime_sec(); if(now < next_run) { debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now)); - sleep_usec(1000000 * (unsigned long long) (next_run - now)); + sleep_usec(USEC_PER_SEC * (usec_t) (next_run - now)); } else { debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop); @@ -2605,6 +3125,8 @@ void *health_main(void *ptr) { buffer_free(wb); info("HEALTH thread exiting"); + + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } diff --git a/src/health.h b/src/health.h index 9d5834fca..79831d4fc 100644 --- a/src/health.h +++ b/src/health.h @@ -5,11 +5,13 @@ extern int health_enabled; extern int rrdvar_compare(void *a, void *b); -#define RRDVAR_TYPE_CALCULATED 1 -#define RRDVAR_TYPE_TIME_T 2 -#define RRDVAR_TYPE_COLLECTED 3 -#define RRDVAR_TYPE_TOTAL 4 -#define RRDVAR_TYPE_INT 5 +#define RRDVAR_TYPE_CALCULATED 1 +#define RRDVAR_TYPE_TIME_T 2 +#define RRDVAR_TYPE_COLLECTED 3 +#define RRDVAR_TYPE_TOTAL 4 +#define RRDVAR_TYPE_INT 5 +#define RRDVAR_TYPE_CALCULATED_ALLOCATED 6 + // the variables as stored in the variables indexes // there are 3 indexes: @@ -34,8 +36,8 @@ typedef struct rrdvar { // This means, there will be no speed penalty for using // these variables typedef struct rrdsetvar { - char *fullid; // chart type.chart id.variable - char *fullname; // chart type.chart name.variable + char *key_fullid; // chart type.chart id.variable + char *key_fullname; // chart type.chart name.variable char *variable; // variable int type; @@ -43,11 +45,11 @@ typedef struct rrdsetvar { uint32_t options; - RRDVAR *local; - RRDVAR *family; - RRDVAR *host; - RRDVAR *family_name; - RRDVAR *host_name; + RRDVAR *var_local; + RRDVAR *var_family; + RRDVAR *var_host; + RRDVAR *var_family_name; + RRDVAR *var_host_name; struct rrdset *rrdset; @@ -64,28 +66,32 @@ typedef struct rrddimvar { char *prefix; char *suffix; - char *id; // dimension id - char *name; // dimension name - char *fullidid; // chart type.chart id.dimension id - char *fullidname; // chart type.chart id.dimension name - char *fullnameid; // chart type.chart name.dimension id - char *fullnamename; // chart type.chart name.dimension name + char *key_id; // dimension id + char *key_name; // dimension name + char *key_contextid; // context + dimension id + char *key_contextname; // context + dimension name + char *key_fullidid; // chart type.chart id + dimension id + char *key_fullidname; // chart type.chart id + dimension name + char *key_fullnameid; // chart type.chart name + dimension id + char *key_fullnamename; // chart type.chart name + dimension name int type; void *value; uint32_t options; - RRDVAR *local_id; - RRDVAR *local_name; + RRDVAR *var_local_id; + RRDVAR *var_local_name; - RRDVAR *family_id; - RRDVAR *family_name; + RRDVAR *var_family_id; + RRDVAR *var_family_name; + RRDVAR *var_family_contextid; + RRDVAR *var_family_contextname; - RRDVAR *host_fullidid; - RRDVAR *host_fullidname; - RRDVAR *host_fullnameid; - RRDVAR *host_fullnamename; + RRDVAR *var_host_chartidid; + RRDVAR *var_host_chartidname; + RRDVAR *var_host_chartnameid; + RRDVAR *var_host_chartnamename; struct rrddim *rrddim; @@ -115,10 +121,11 @@ typedef struct rrddimvar { #define RRDCALC_FLAG_DB_ERROR 0x00000001 #define RRDCALC_FLAG_DB_NAN 0x00000002 -#define RRDCALC_FLAG_DB_STALE 0x00000004 +/* #define RRDCALC_FLAG_DB_STALE 0x00000004 */ #define RRDCALC_FLAG_CALC_ERROR 0x00000008 #define RRDCALC_FLAG_WARN_ERROR 0x00000010 #define RRDCALC_FLAG_CRIT_ERROR 0x00000020 +#define RRDCALC_FLAG_RUNNABLE 0x00000040 typedef struct rrdcalc { uint32_t id; // the unique id of this alarm @@ -225,6 +232,9 @@ typedef struct rrdcalctemplate { char *context; uint32_t hash_context; + char *family_match; + SIMPLE_PATTERN *family_pattern; + char *source; // the source of this alarm char *units; // the units of the alarm char *info; // a short description of the alarm @@ -264,10 +274,11 @@ typedef struct rrdcalctemplate { #define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after) -#define HEALTH_ENTRY_NOTIFICATIONS_PROCESSED 0x00000001 -#define HEALTH_ENTRY_NOTIFICATIONS_UPDATED 0x00000002 -#define HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN 0x00000004 -#define HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED 0x00000008 +#define HEALTH_ENTRY_FLAG_PROCESSED 0x00000001 +#define HEALTH_ENTRY_FLAG_UPDATED 0x00000002 +#define HEALTH_ENTRY_FLAG_EXEC_RUN 0x00000004 +#define HEALTH_ENTRY_FLAG_EXEC_FAILED 0x00000008 +#define HEALTH_ENTRY_FLAG_SAVED 0x10000000 typedef struct alarm_entry { uint32_t unique_id; @@ -300,7 +311,7 @@ typedef struct alarm_entry { int old_status; int new_status; - uint32_t notifications; + uint32_t flags; int delay; time_t delay_up_to_timestamp; @@ -344,4 +355,12 @@ extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC * extern void health_alarms2json(RRDHOST *host, BUFFER *wb, int all); extern void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after); +void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf); + +extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name); +extern void rrdvar_custom_host_variable_destroy(RRDHOST *host, const char *name); +extern void rrdvar_custom_host_variable_set(RRDVAR *rv, calculated_number value); + +extern const char *rrdcalc_status2string(int status); + #endif //NETDATA_HEALTH_H diff --git a/src/inlined.h b/src/inlined.h new file mode 100644 index 000000000..e776f830e --- /dev/null +++ b/src/inlined.h @@ -0,0 +1,127 @@ +#ifndef NETDATA_INLINED_H +#define NETDATA_INLINED_H + +#include "common.h" + +// for faster execution, allow the compiler to inline +// these functions that are called thousands of times per second + +static inline uint32_t simple_hash(const char *name) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5; + while (*s) { + hval *= 16777619; + hval ^= (uint32_t) *s++; + } + return hval; +} + +static inline uint32_t simple_uhash(const char *name) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5, c; + while ((c = *s++)) { + if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A'; + hval *= 16777619; + hval ^= c; + } + return hval; +} + +static inline int simple_hash_strcmp(const char *name, const char *b, uint32_t *hash) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5; + int ret = 0; + while (*s) { + if(!ret) ret = *s - *b++; + hval *= 16777619; + hval ^= (uint32_t) *s++; + } + *hash = hval; + return ret; +} + +static inline int str2i(const char *s) { + int n = 0; + char c, negative = (*s == '-'); + + for(c = (negative)?*(++s):*s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + + if(unlikely(negative)) + return -n; + + return n; +} + +static inline long str2l(const char *s) { + long n = 0; + char c, negative = (*s == '-'); + + for(c = (negative)?*(++s):*s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + + if(unlikely(negative)) + return -n; + + return n; +} + +static inline unsigned long str2ul(const char *s) { + unsigned long n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + return n; +} + +static inline unsigned long long str2ull(const char *s) { + unsigned long long n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + return n; +} + +#ifdef NETDATA_STRCMP_OVERRIDE +#ifdef strcmp +#undef strcmp +#endif +#define strcmp(a, b) strsame(a, b) +static inline int strsame(const char *a, const char *b) { + if(unlikely(a == b)) return 0; + while(*a && *a == *b) { a++; b++; } + return *a - *b; +} +#endif // NETDATA_STRSAME + +static inline int read_single_number_file(const char *filename, unsigned long long *result) { + char buffer[30 + 1]; + + int fd = open(filename, O_RDONLY, 0666); + if(unlikely(fd == -1)) { + *result = 0; + return 1; + } + + ssize_t r = read(fd, buffer, 30); + if(unlikely(r == -1)) { + *result = 0; + close(fd); + return 2; + } + + close(fd); + buffer[30] = '\0'; + *result = str2ull(buffer); + return 0; +} + +#endif //NETDATA_INLINED_H diff --git a/src/ipc.c b/src/ipc.c new file mode 100644 index 000000000..a5ab342d3 --- /dev/null +++ b/src/ipc.c @@ -0,0 +1,238 @@ +#include "common.h" + +#include <sys/sem.h> +#include <sys/msg.h> +#include <sys/shm.h> + + +#ifndef SEMVMX +#define SEMVMX 32767 /* <= 32767 semaphore maximum value */ +#endif + +/* Some versions of libc only define IPC_INFO when __USE_GNU is defined. */ +#ifndef IPC_INFO +#define IPC_INFO 3 +#endif + +struct ipc_limits { + uint64_t shmmni; /* max number of segments */ + uint64_t shmmax; /* max segment size */ + uint64_t shmall; /* max total shared memory */ + uint64_t shmmin; /* min segment size */ + + int semmni; /* max number of arrays */ + int semmsl; /* max semaphores per array */ + int semmns; /* max semaphores system wide */ + int semopm; /* max ops per semop call */ + unsigned int semvmx; /* semaphore max value (constant) */ + + int msgmni; /* max queues system wide */ + size_t msgmax; /* max size of message */ + int msgmnb; /* default max size of queue */ +}; + +struct ipc_status { + int semusz; /* current number of arrays */ + int semaem; /* current semaphores system wide */ +}; + +/* + * The last arg of semctl is a union semun, but where is it defined? X/OPEN + * tells us to define it ourselves, but until recently Linux include files + * would also define it. + */ +#ifndef HAVE_UNION_SEMUN +/* according to X/OPEN we have to define it ourselves */ +union semun { + int val; + struct semid_ds *buf; + unsigned short int *array; + struct seminfo *__buf; +}; +#endif + +static inline int ipc_sem_get_limits(struct ipc_limits *lim) { + static procfile *ff = NULL; + static int error_shown = 0; + static char filename[FILENAME_MAX + 1] = ""; + + if(unlikely(!filename[0])) + snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/sem", global_host_prefix); + + if(unlikely(!ff)) { + ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) { + if(unlikely(!error_shown)) { + error("IPC: Cannot open file '%s'.", filename); + error_shown = 1; + } + goto ipc; + } + } + + ff = procfile_readall(ff); + if(unlikely(!ff)) { + if(unlikely(!error_shown)) { + error("IPC: Cannot read file '%s'.", filename); + error_shown = 1; + } + goto ipc; + } + + if(procfile_lines(ff) >= 1 && procfile_linewords(ff, 0) >= 4) { + lim->semvmx = SEMVMX; + lim->semmsl = str2i(procfile_lineword(ff, 0, 0)); + lim->semmns = str2i(procfile_lineword(ff, 0, 1)); + lim->semopm = str2i(procfile_lineword(ff, 0, 2)); + lim->semmni = str2i(procfile_lineword(ff, 0, 3)); + return 0; + } + else { + if(unlikely(!error_shown)) { + error("IPC: Invalid content in file '%s'.", filename); + error_shown = 1; + } + goto ipc; + } + +ipc: + // cannot do it from the file + // query IPC + { + struct seminfo seminfo = {.semmni = 0}; + union semun arg = {.array = (ushort *) &seminfo}; + + if(unlikely(semctl(0, 0, IPC_INFO, arg) < 0)) { + error("IPC: Failed to read '%s' and request IPC_INFO with semctl().", filename); + goto error; + } + + lim->semvmx = SEMVMX; + lim->semmni = seminfo.semmni; + lim->semmsl = seminfo.semmsl; + lim->semmns = seminfo.semmns; + lim->semopm = seminfo.semopm; + return 0; + } + +error: + lim->semvmx = 0; + lim->semmni = 0; + lim->semmsl = 0; + lim->semmns = 0; + lim->semopm = 0; + return -1; +} + +/* +printf ("------ Semaphore Limits --------\n"); +printf ("max number of arrays = %d\n", limits.semmni); +printf ("max semaphores per array = %d\n", limits.semmsl); +printf ("max semaphores system wide = %d\n", limits.semmns); +printf ("max ops per semop call = %d\n", limits.semopm); +printf ("semaphore max value = %u\n", limits.semvmx); + +printf ("------ Semaphore Status --------\n"); +printf ("used arrays = %d\n", status.semusz); +printf ("allocated semaphores = %d\n", status.semaem); +*/ + +static inline int ipc_sem_get_status(struct ipc_status *st) { + struct seminfo seminfo; + union semun arg; + + arg.array = (ushort *) (void *) &seminfo; + + if(unlikely(semctl (0, 0, SEM_INFO, arg) < 0)) { + /* kernel not configured for semaphores */ + static int error_shown = 0; + if(unlikely(!error_shown)) { + error("IPC: kernel is not configured for semaphores"); + error_shown = 1; + } + st->semusz = 0; + st->semaem = 0; + return -1; + } + + st->semusz = seminfo.semusz; + st->semaem = seminfo.semaem; + return 0; +} + +int do_ipc(int update_every, usec_t dt) { + (void)dt; + + static int initialized = 0, read_limits_next = 0; + static struct ipc_limits limits; + static struct ipc_status status; + static RRDVAR *arrays_max = NULL, *semaphores_max = NULL; + static RRDSET *semaphores = NULL, *arrays = NULL; + + if(unlikely(!initialized)) { + initialized = 1; + + // make sure it works + if(ipc_sem_get_limits(&limits) == -1) { + error("unable to fetch semaphore limits"); + return 1; + } + + // make sure it works + if(ipc_sem_get_status(&status) == -1) { + error("unable to fetch semaphore statistics"); + return 1; + } + + arrays_max = rrdvar_custom_host_variable_create(&localhost, "ipc.semaphores.arrays.max"); + semaphores_max = rrdvar_custom_host_variable_create(&localhost, "ipc.semaphores.max"); + + if(arrays_max) rrdvar_custom_host_variable_set(arrays_max, limits.semmni); + if(semaphores_max) rrdvar_custom_host_variable_set(semaphores_max, limits.semmns); + + // create the charts + semaphores = rrdset_find("system.ipc_semaphores"); + if(!semaphores) { + semaphores = rrdset_create("system", "ipc_semaphores", NULL, "ipc semaphores", NULL, "IPC Semaphores", "semaphores", 1000, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(semaphores, "semaphores", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + + arrays = rrdset_find("system.ipc_semaphore_arrays"); + if(!arrays) { + arrays = rrdset_create("system", "ipc_semaphore_arrays", NULL, "ipc semaphores", NULL, "IPC Semaphore Arrays", "arrays", 1000, rrd_update_every, RRDSET_TYPE_AREA); + rrddim_add(arrays, "arrays", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + } + + if(unlikely(read_limits_next < 0)) { + if(unlikely(ipc_sem_get_limits(&limits) == -1)) { + error("Unable to fetch semaphore limits."); + } + else { + if(arrays_max) rrdvar_custom_host_variable_set(arrays_max, limits.semmni); + if(semaphores_max) rrdvar_custom_host_variable_set(semaphores_max, limits.semmns); + + arrays->red = limits.semmni; + semaphores->red = limits.semmns; + + read_limits_next = 60 / update_every; + } + } + else + read_limits_next--; + + if(unlikely(ipc_sem_get_status(&status) == -1)) { + error("Unable to get semaphore statistics"); + return 0; + } + + if(semaphores->counter_done) rrdset_next(semaphores); + rrddim_set(semaphores, "semaphores", status.semaem); + rrdset_done(semaphores); + + if(arrays->counter_done) rrdset_next(arrays); + rrddim_set(arrays, "arrays", status.semusz); + rrdset_done(arrays); + + return 0; +} diff --git a/src/ipc.h b/src/ipc.h new file mode 100644 index 000000000..04f9df5cd --- /dev/null +++ b/src/ipc.h @@ -0,0 +1,7 @@ +#ifndef NETDATA_PLUGIN_IPC_H +#define NETDATA_PLUGIN_IPC_H 1 + +extern int do_ipc(int update_every, usec_t dt); + +#endif /* NETDATA_PLUGIN_IPC_H */ + @@ -24,7 +24,7 @@ void syslog_init(void) { } int open_log_file(int fd, FILE **fp, const char *filename, int *enabled_syslog) { - int f, t; + int f; if(!filename || !*filename || !strcmp(filename, "none")) filename = "/dev/null"; @@ -64,7 +64,7 @@ int open_log_file(int fd, FILE **fp, const char *filename, int *enabled_syslog) if(fd != f && fd != -1) { // it automatically closes - t = dup2(f, fd); + int t = dup2(f, fd); if (t == -1) { error("Cannot dup2() new fd %d to old fd %d for '%s'", f, fd, filename); close(f); @@ -125,9 +125,13 @@ int error_log_limit(int reset) { // prevent all logs if the errors per period is 0 if(error_log_errors_per_period == 0) +#ifdef NETDATA_INTERNAL_CHECKS + return 0; +#else return 1; +#endif - time_t now = time(NULL); + time_t now = now_monotonic_sec(); if(!start) start = now; if(reset) { @@ -185,7 +189,11 @@ int error_log_limit(int reset) { prevented++; // prevent logging this error +#ifdef NETDATA_INTERNAL_CHECKS + return 0; +#else return 1; +#endif } return 0; @@ -200,15 +208,15 @@ int error_log_limit(int reset) { void log_date(FILE *out) { - char outstr[24]; + char outstr[26]; time_t t; struct tm *tmp, tmbuf; - t = time(NULL); + t = now_realtime_sec(); tmp = localtime_r(&t, &tmbuf); if (tmp == NULL) return; - if (unlikely(strftime(outstr, sizeof(outstr), "%y-%m-%d %H:%M:%S", tmp) == 0)) return; + if (unlikely(strftime(outstr, sizeof(outstr), "%Y-%m-%d %H:%M:%S", tmp) == 0)) return; fprintf(out, "%s: ", outstr); } @@ -222,7 +230,7 @@ void debug_int( const char *file, const char *function, const unsigned long line log_date(stdout); va_start( args, fmt ); - printf("DEBUG (%04lu@%-10.10s:%-15.15s): %s: ", line, file, function, program_name); + printf("%s: DEBUG (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function); vprintf(fmt, args); va_end( args ); putchar('\n'); @@ -249,8 +257,8 @@ void info_int( const char *file, const char *function, const unsigned long line, log_date(stderr); va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "INFO (%04lu@%-10.10s:%-15.15s): %s: ", line, file, function, program_name); - else fprintf(stderr, "INFO: %s: ", program_name); + if(debug_flags) fprintf(stderr, "%s: INFO: (%04lu@%-10.10s:%-15.15s):", program_name, line, file, function); + else fprintf(stderr, "%s: INFO: ", program_name); vfprintf( stderr, fmt, args ); va_end( args ); @@ -298,8 +306,8 @@ void error_int( const char *prefix, const char *file, const char *function, cons log_date(stderr); va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "%s (%04lu@%-10.10s:%-15.15s): %s: ", prefix, line, file, function, program_name); - else fprintf(stderr, "%s: %s: ", prefix, program_name); + if(debug_flags) fprintf(stderr, "%s: %s: (%04lu@%-10.10s:%-15.15s): ", program_name, prefix, line, file, function); + else fprintf(stderr, "%s: %s: ", program_name, prefix); vfprintf( stderr, fmt, args ); va_end( args ); @@ -325,8 +333,8 @@ void fatal_int( const char *file, const char *function, const unsigned long line log_date(stderr); va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "FATAL (%04lu@%-10.10s:%-15.15s): %s: ", line, file, function, program_name); - else fprintf(stderr, "FATAL: %s: ", program_name); + if(debug_flags) fprintf(stderr, "%s: FATAL: (%04lu@%-10.10s:%-15.15s): ", program_name, line, file, function); + else fprintf(stderr, "%s: FATAL: ", program_name); vfprintf( stderr, fmt, args ); va_end( args ); @@ -25,6 +25,8 @@ #define D_REGISTRY 0x00200000 #define D_VARIABLES 0x00400000 #define D_HEALTH 0x00800000 +#define D_CONNECT_TO 0x01000000 +#define D_SYSTEM 0x80000000 //#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS) //#define DEBUG 0xffffffff @@ -62,10 +64,10 @@ extern void reopen_all_log_files(); #define fatal(args...) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args) extern void log_date(FILE *out); -extern void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) __attribute__ (( format (printf, 4, 5))); -extern void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) __attribute__ (( format (printf, 4, 5))); -extern void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) __attribute__ (( format (printf, 5, 6))); -extern void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) __attribute__ ((noreturn, format (printf, 4, 5))); -extern void log_access( const char *fmt, ... ) __attribute__ (( format (printf, 1, 2))); +extern void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5); +extern void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5); +extern void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(5, 6); +extern void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5); +extern void log_access( const char *fmt, ... ) PRINTFLIKE(1, 2); #endif /* NETDATA_LOG_H */ diff --git a/src/macos_fw.c b/src/macos_fw.c new file mode 100644 index 000000000..a62aa7a7e --- /dev/null +++ b/src/macos_fw.c @@ -0,0 +1,487 @@ +#include "common.h" +#include <CoreFoundation/CoreFoundation.h> +#include <IOKit/IOKitLib.h> +#include <IOKit/storage/IOBlockStorageDriver.h> +#include <IOKit/IOBSD.h> +// NEEDED BY do_space, do_inodes +#include <sys/mount.h> +// NEEDED BY: struct ifaddrs, getifaddrs() +#include <net/if.h> +#include <ifaddrs.h> + +// NEEDED BY: do_bandwidth +#define IFA_DATA(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s) + +#define MAXDRIVENAME 31 + +#define KILO_FACTOR 1024 +#define MEGA_FACTOR 1048576 // 1024 * 1024 +#define GIGA_FACTOR 1073741824 // 1024 * 1024 * 1024 + +int do_macos_iokit(int update_every, usec_t dt) { + (void)dt; + + static int do_io = -1, do_space = -1, do_inodes = -1, do_bandwidth = -1; + + if (unlikely(do_io == -1)) { + do_io = config_get_boolean("plugin:macos:iokit", "disk i/o", 1); + do_space = config_get_boolean("plugin:macos:sysctl", "space usage for all disks", 1); + do_inodes = config_get_boolean("plugin:macos:sysctl", "inodes usage for all disks", 1); + do_bandwidth = config_get_boolean("plugin:macos:sysctl", "bandwidth", 1); + } + + RRDSET *st; + + mach_port_t master_port; + io_registry_entry_t drive, drive_media; + io_iterator_t drive_list; + CFDictionaryRef properties, statistics; + CFStringRef name; + CFNumberRef number; + kern_return_t status; + collected_number total_disk_reads = 0; + collected_number total_disk_writes = 0; + struct diskstat { + char name[MAXDRIVENAME]; + collected_number bytes_read; + collected_number bytes_write; + collected_number reads; + collected_number writes; + collected_number time_read; + collected_number time_write; + collected_number latency_read; + collected_number latency_write; + } diskstat; + struct cur_diskstat { + collected_number duration_read_ns; + collected_number duration_write_ns; + collected_number busy_time_ns; + } cur_diskstat; + struct prev_diskstat { + collected_number bytes_read; + collected_number bytes_write; + collected_number operations_read; + collected_number operations_write; + collected_number duration_read_ns; + collected_number duration_write_ns; + collected_number busy_time_ns; + } prev_diskstat; + + // NEEDED BY: do_space, do_inodes + struct statfs *mntbuf; + int mntsize, i; + char mntonname[MNAMELEN + 1]; + char title[4096 + 1]; + + // NEEDED BY: do_bandwidth + struct ifaddrs *ifa, *ifap; + + /* Get ports and services for drive statistics. */ + if (unlikely(IOMasterPort(bootstrap_port, &master_port))) { + error("MACOS: IOMasterPort() failed"); + do_io = 0; + error("DISABLED: system.io"); + /* Get the list of all drive objects. */ + } else if (unlikely(IOServiceGetMatchingServices(master_port, IOServiceMatching("IOBlockStorageDriver"), &drive_list))) { + error("MACOS: IOServiceGetMatchingServices() failed"); + do_io = 0; + error("DISABLED: system.io"); + } else { + while ((drive = IOIteratorNext(drive_list)) != 0) { + properties = 0; + statistics = 0; + number = 0; + bzero(&diskstat, sizeof(diskstat)); + + /* Get drive media object. */ + status = IORegistryEntryGetChildEntry(drive, kIOServicePlane, &drive_media); + if (unlikely(status != KERN_SUCCESS)) { + IOObjectRelease(drive); + continue; + } + + /* Get drive media properties. */ + if (likely(!IORegistryEntryCreateCFProperties(drive_media, (CFMutableDictionaryRef *)&properties, kCFAllocatorDefault, 0))) { + /* Get disk name. */ + if (likely(name = (CFStringRef)CFDictionaryGetValue(properties, CFSTR(kIOBSDNameKey)))) { + CFStringGetCString(name, diskstat.name, MAXDRIVENAME, kCFStringEncodingUTF8); + } + } + + /* Release. */ + CFRelease(properties); + IOObjectRelease(drive_media); + + /* Obtain the properties for this drive object. */ + if (unlikely(IORegistryEntryCreateCFProperties(drive, (CFMutableDictionaryRef *)&properties, kCFAllocatorDefault, 0))) { + error("MACOS: IORegistryEntryCreateCFProperties() failed"); + do_io = 0; + error("DISABLED: system.io"); + break; + } else if (likely(properties)) { + /* Obtain the statistics from the drive properties. */ + if (likely(statistics = (CFDictionaryRef)CFDictionaryGetValue(properties, CFSTR(kIOBlockStorageDriverStatisticsKey)))) { + + // -------------------------------------------------------------------- + + /* Get bytes read. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsBytesReadKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.bytes_read); + total_disk_reads += diskstat.bytes_read; + } + + /* Get bytes written. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsBytesWrittenKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.bytes_write); + total_disk_writes += diskstat.bytes_write; + } + + st = rrdset_find_bytype("disk", diskstat.name); + if (unlikely(!st)) { + st = rrdset_create("disk", diskstat.name, NULL, diskstat.name, "disk.io", "Disk I/O Bandwidth", "kilobytes/s", 2000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "reads", NULL, 1, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + prev_diskstat.bytes_read = rrddim_set(st, "reads", diskstat.bytes_read); + prev_diskstat.bytes_write = rrddim_set(st, "writes", diskstat.bytes_write); + rrdset_done(st); + + // -------------------------------------------------------------------- + + /* Get number of reads. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsReadsKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.reads); + } + + /* Get number of writes. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsWritesKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes); + } + + st = rrdset_find_bytype("disk_ops", diskstat.name); + if (unlikely(!st)) { + st = rrdset_create("disk_ops", diskstat.name, NULL, diskstat.name, "disk.ops", "Disk Completed I/O Operations", "operations/s", 2001, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + prev_diskstat.operations_read = rrddim_set(st, "reads", diskstat.reads); + prev_diskstat.operations_write = rrddim_set(st, "writes", diskstat.writes); + rrdset_done(st); + + // -------------------------------------------------------------------- + + /* Get reads time. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsTotalReadTimeKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_read); + } + + /* Get writes time. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsTotalWriteTimeKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write); + } + + st = rrdset_find_bytype("disk_util", diskstat.name); + if (unlikely(!st)) { + st = rrdset_create("disk_util", diskstat.name, NULL, diskstat.name, "disk.util", "Disk Utilization Time", "% of time working", 2004, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "utilization", NULL, 1, 10000000, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + cur_diskstat.busy_time_ns = (diskstat.time_read + diskstat.time_write); + prev_diskstat.busy_time_ns = rrddim_set(st, "utilization", cur_diskstat.busy_time_ns); + rrdset_done(st); + + // -------------------------------------------------------------------- + + /* Get reads latency. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsLatentReadTimeKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_read); + } + + /* Get writes latency. */ + if (likely(number = (CFNumberRef)CFDictionaryGetValue(statistics, CFSTR(kIOBlockStorageDriverStatisticsLatentWriteTimeKey)))) { + CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write); + } + + st = rrdset_find_bytype("disk_iotime", diskstat.name); + if (unlikely(!st)) { + st = rrdset_create("disk_iotime", diskstat.name, NULL, diskstat.name, "disk.iotime", "Disk Total I/O Time", "milliseconds/s", 2022, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1000000, RRDDIM_INCREMENTAL); + rrddim_add(st, "writes", NULL, -1, 1000000, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + cur_diskstat.duration_read_ns = diskstat.time_read + diskstat.latency_read; + cur_diskstat.duration_write_ns = diskstat.time_write + diskstat.latency_write; + prev_diskstat.duration_read_ns = rrddim_set(st, "reads", cur_diskstat.duration_read_ns); + prev_diskstat.duration_write_ns = rrddim_set(st, "writes", cur_diskstat.duration_write_ns); + rrdset_done(st); + + // -------------------------------------------------------------------- + // calculate differential charts + // only if this is not the first time we run + + if (likely(dt)) { + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_await", diskstat.name); + if (unlikely(!st)) { + st = rrdset_create("disk_await", diskstat.name, NULL, diskstat.name, "disk.await", "Average Completed I/O Operation Time", "ms per operation", 2005, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1000000, RRDDIM_ABSOLUTE); + rrddim_add(st, "writes", NULL, -1, 1000000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "reads", (diskstat.reads - prev_diskstat.operations_read) ? + (cur_diskstat.duration_read_ns - prev_diskstat.duration_read_ns) / (diskstat.reads - prev_diskstat.operations_read) : 0); + rrddim_set(st, "writes", (diskstat.writes - prev_diskstat.operations_write) ? + (cur_diskstat.duration_write_ns - prev_diskstat.duration_write_ns) / (diskstat.writes - prev_diskstat.operations_write) : 0); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_avgsz", diskstat.name); + if (unlikely(!st)) { + st = rrdset_create("disk_avgsz", diskstat.name, NULL, diskstat.name, "disk.avgsz", "Average Completed I/O Operation Bandwidth", "kilobytes per operation", 2006, update_every, RRDSET_TYPE_AREA); + st->isdetail = 1; + + rrddim_add(st, "reads", NULL, 1, 1024, RRDDIM_ABSOLUTE); + rrddim_add(st, "writes", NULL, -1, 1024, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "reads", (diskstat.reads - prev_diskstat.operations_read) ? + (diskstat.bytes_read - prev_diskstat.bytes_read) / (diskstat.reads - prev_diskstat.operations_read) : 0); + rrddim_set(st, "writes", (diskstat.writes - prev_diskstat.operations_write) ? + (diskstat.bytes_write - prev_diskstat.bytes_write) / (diskstat.writes - prev_diskstat.operations_write) : 0); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("disk_svctm", diskstat.name); + if (unlikely(!st)) { + st = rrdset_create("disk_svctm", diskstat.name, NULL, diskstat.name, "disk.svctm", "Average Service Time", "ms per operation", 2007, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "svctm", NULL, 1, 1000000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "svctm", ((diskstat.reads - prev_diskstat.operations_read) + (diskstat.writes - prev_diskstat.operations_write)) ? + (cur_diskstat.busy_time_ns - prev_diskstat.busy_time_ns) / ((diskstat.reads - prev_diskstat.operations_read) + (diskstat.writes - prev_diskstat.operations_write)) : 0); + rrdset_done(st); + } + } + + /* Release. */ + CFRelease(properties); + } + + /* Release. */ + IOObjectRelease(drive); + } + IOIteratorReset(drive_list); + + /* Release. */ + IOObjectRelease(drive_list); + } + + if (likely(do_io)) { + st = rrdset_find_bytype("system", "io"); + if (unlikely(!st)) { + st = rrdset_create("system", "io", NULL, "disk", NULL, "Disk I/O", "kilobytes/s", 150, update_every, RRDSET_TYPE_AREA); + rrddim_add(st, "in", NULL, 1, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "out", NULL, -1, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "in", total_disk_reads); + rrddim_set(st, "out", total_disk_writes); + rrdset_done(st); + } + + // Can be merged with FreeBSD plugin + // -------------------------------------------------------------------------- + + if (likely(do_space || do_inodes)) { + // there is no mount info in sysctl MIBs + if (unlikely(!(mntsize = getmntinfo(&mntbuf, MNT_NOWAIT)))) { + error("MACOS: getmntinfo() failed"); + do_space = 0; + error("DISABLED: disk_space.X"); + do_inodes = 0; + error("DISABLED: disk_inodes.X"); + } else { + for (i = 0; i < mntsize; i++) { + if (mntbuf[i].f_flags == MNT_RDONLY || + mntbuf[i].f_blocks == 0 || + // taken from gnulib/mountlist.c and shortened to FreeBSD related fstypes + strcmp(mntbuf[i].f_fstypename, "autofs") == 0 || + strcmp(mntbuf[i].f_fstypename, "procfs") == 0 || + strcmp(mntbuf[i].f_fstypename, "subfs") == 0 || + strcmp(mntbuf[i].f_fstypename, "devfs") == 0 || + strcmp(mntbuf[i].f_fstypename, "none") == 0) + continue; + + // -------------------------------------------------------------------------- + + if (likely(do_space)) { + st = rrdset_find_bytype("disk_space", mntbuf[i].f_mntonname); + if (unlikely(!st)) { + snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); + st = rrdset_create("disk_space", mntbuf[i].f_mntonname, NULL, mntbuf[i].f_mntonname, "disk.space", title, "GB", 2023, + update_every, + RRDSET_TYPE_STACKED); + + rrddim_add(st, "avail", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, mntbuf[i].f_bsize, GIGA_FACTOR, RRDDIM_ABSOLUTE); + rrddim_add(st, "reserved_for_root", "reserved for root", mntbuf[i].f_bsize, GIGA_FACTOR, + RRDDIM_ABSOLUTE); + } else + rrdset_next(st); + + rrddim_set(st, "avail", (collected_number) mntbuf[i].f_bavail); + rrddim_set(st, "used", (collected_number) (mntbuf[i].f_blocks - mntbuf[i].f_bfree)); + rrddim_set(st, "reserved_for_root", (collected_number) (mntbuf[i].f_bfree - mntbuf[i].f_bavail)); + rrdset_done(st); + } + + // -------------------------------------------------------------------------- + + if (likely(do_inodes)) { + st = rrdset_find_bytype("disk_inodes", mntbuf[i].f_mntonname); + if (unlikely(!st)) { + snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname); + st = rrdset_create("disk_inodes", mntbuf[i].f_mntonname, NULL, mntbuf[i].f_mntonname, "disk.inodes", title, "Inodes", 2024, + update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "avail", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(st, "reserved_for_root", "reserved for root", 1, 1, RRDDIM_ABSOLUTE); + } else + rrdset_next(st); + + rrddim_set(st, "avail", (collected_number) mntbuf[i].f_ffree); + rrddim_set(st, "used", (collected_number) (mntbuf[i].f_files - mntbuf[i].f_ffree)); + rrdset_done(st); + } + } + } + } + + // Can be merged with FreeBSD plugin + // -------------------------------------------------------------------- + + if (likely(do_bandwidth)) { + if (unlikely(getifaddrs(&ifap))) { + error("MACOS: getifaddrs()"); + do_bandwidth = 0; + error("DISABLED: system.ipv4"); + } else { + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family != AF_LINK) + continue; + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net", ifa->ifa_name, NULL, ifa->ifa_name, "net.net", "Bandwidth", "kilobits/s", 7000, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", IFA_DATA(ibytes)); + rrddim_set(st, "sent", IFA_DATA(obytes)); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_packets", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_packets", ifa->ifa_name, NULL, ifa->ifa_name, "net.packets", "Packets", "packets/s", 7001, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "multicast_received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "multicast_sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "received", IFA_DATA(ipackets)); + rrddim_set(st, "sent", IFA_DATA(opackets)); + rrddim_set(st, "multicast_received", IFA_DATA(imcasts)); + rrddim_set(st, "multicast_sent", IFA_DATA(omcasts)); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_errors", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_errors", ifa->ifa_name, NULL, ifa->ifa_name, "net.errors", "Interface Errors", "errors/s", 7002, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "inbound", IFA_DATA(ierrors)); + rrddim_set(st, "outbound", IFA_DATA(oerrors)); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_drops", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_drops", ifa->ifa_name, NULL, ifa->ifa_name, "net.drops", "Interface Drops", "drops/s", 7003, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "inbound", IFA_DATA(iqdrops)); + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find_bytype("net_events", ifa->ifa_name); + if (unlikely(!st)) { + st = rrdset_create("net_events", ifa->ifa_name, NULL, ifa->ifa_name, "net.events", "Network Interface Events", "events/s", 7006, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "frames", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "collisions", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "carrier", NULL, -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "collisions", IFA_DATA(collisions)); + rrdset_done(st); + } + + freeifaddrs(ifap); + } + } + + + return 0; +} diff --git a/src/macos_mach_smi.c b/src/macos_mach_smi.c new file mode 100644 index 000000000..d86a03220 --- /dev/null +++ b/src/macos_mach_smi.c @@ -0,0 +1,167 @@ +#include "common.h" +#include <mach/mach.h> + +int do_macos_mach_smi(int update_every, usec_t dt) { + (void)dt; + + static int do_cpu = -1, do_ram = - 1, do_swapio = -1, do_pgfaults = -1; + + if (unlikely(do_cpu == -1)) { + do_cpu = config_get_boolean("plugin:macos:mach_smi", "cpu utilization", 1); + do_ram = config_get_boolean("plugin:macos:mach_smi", "system ram", 1); + do_swapio = config_get_boolean("plugin:macos:mach_smi", "swap i/o", 1); + do_pgfaults = config_get_boolean("plugin:macos:mach_smi", "memory page faults", 1); + } + + RRDSET *st; + + kern_return_t kr; + mach_msg_type_number_t count; + host_t host; + vm_size_t system_pagesize; + + + // NEEDED BY: do_cpu + natural_t cp_time[CPU_STATE_MAX]; + + // NEEDED BY: do_ram, do_swapio, do_pgfaults + vm_statistics64_data_t vm_statistics; + + host = mach_host_self(); + kr = host_page_size(host, &system_pagesize); + if (unlikely(kr != KERN_SUCCESS)) + return -1; + + // -------------------------------------------------------------------- + + if (likely(do_cpu)) { + if (unlikely(HOST_CPU_LOAD_INFO_COUNT != 4)) { + error("MACOS: There are %d CPU states (4 was expected)", HOST_CPU_LOAD_INFO_COUNT); + do_cpu = 0; + error("DISABLED: system.cpu"); + } else { + count = HOST_CPU_LOAD_INFO_COUNT; + kr = host_statistics(host, HOST_CPU_LOAD_INFO, (host_info_t)cp_time, &count); + if (unlikely(kr != KERN_SUCCESS)) { + error("MACOS: host_statistics() failed: %s", mach_error_string(kr)); + do_cpu = 0; + error("DISABLED: system.cpu"); + } else { + + st = rrdset_find_bytype("system", "cpu"); + if (unlikely(!st)) { + st = rrdset_create("system", "cpu", NULL, "cpu", "system.cpu", "Total CPU utilization", "percentage", 100, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "user", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "nice", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "system", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_add(st, "idle", NULL, 1, 1, RRDDIM_PCENT_OVER_DIFF_TOTAL); + rrddim_hide(st, "idle"); + } + else rrdset_next(st); + + rrddim_set(st, "user", cp_time[CPU_STATE_USER]); + rrddim_set(st, "nice", cp_time[CPU_STATE_NICE]); + rrddim_set(st, "system", cp_time[CPU_STATE_SYSTEM]); + rrddim_set(st, "idle", cp_time[CPU_STATE_IDLE]); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_ram || do_swapio || do_pgfaults)) { + count = sizeof(vm_statistics64_data_t); + kr = host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&vm_statistics, &count); + if (unlikely(kr != KERN_SUCCESS)) { + error("MACOS: host_statistics64() failed: %s", mach_error_string(kr)); + do_ram = 0; + error("DISABLED: system.ram"); + do_swapio = 0; + error("DISABLED: system.swapio"); + do_pgfaults = 0; + error("DISABLED: mem.pgfaults"); + } else { + if (likely(do_ram)) { + st = rrdset_find("system.ram"); + if (unlikely(!st)) { + st = rrdset_create("system", "ram", NULL, "ram", NULL, "System RAM", "MB", 200, update_every, RRDSET_TYPE_STACKED); + + rrddim_add(st, "active", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "wired", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "throttled", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "compressor", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "inactive", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "purgeable", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "speculative", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "free", NULL, system_pagesize, 1048576, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "active", vm_statistics.active_count); + rrddim_set(st, "wired", vm_statistics.wire_count); + rrddim_set(st, "throttled", vm_statistics.throttled_count); + rrddim_set(st, "compressor", vm_statistics.compressor_page_count); + rrddim_set(st, "inactive", vm_statistics.inactive_count); + rrddim_set(st, "purgeable", vm_statistics.purgeable_count); + rrddim_set(st, "speculative", vm_statistics.speculative_count); + rrddim_set(st, "free", (vm_statistics.free_count - vm_statistics.speculative_count)); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_swapio)) { + st = rrdset_find("system.swapio"); + if (unlikely(!st)) { + st = rrdset_create("system", "swapio", NULL, "swap", NULL, "Swap I/O", "kilobytes/s", 250, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "in", NULL, system_pagesize, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "out", NULL, -system_pagesize, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "in", vm_statistics.swapins); + rrddim_set(st, "out", vm_statistics.swapouts); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_pgfaults)) { + st = rrdset_find("mem.pgfaults"); + if (unlikely(!st)) { + st = rrdset_create("mem", "pgfaults", NULL, "system", NULL, "Memory Page Faults", "page faults/s", 500, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "memory", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "cow", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "pagein", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "pageout", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "compress", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "decompress", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "zero_fill", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "reactivate", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "purge", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "memory", vm_statistics.faults); + rrddim_set(st, "cow", vm_statistics.cow_faults); + rrddim_set(st, "pagein", vm_statistics.pageins); + rrddim_set(st, "pageout", vm_statistics.pageouts); + rrddim_set(st, "compress", vm_statistics.compressions); + rrddim_set(st, "decompress", vm_statistics.decompressions); + rrddim_set(st, "zero_fill", vm_statistics.zero_fill_count); + rrddim_set(st, "reactivate", vm_statistics.reactivations); + rrddim_set(st, "purge", vm_statistics.purges); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + return 0; +} diff --git a/src/macos_sysctl.c b/src/macos_sysctl.c new file mode 100644 index 000000000..955b70757 --- /dev/null +++ b/src/macos_sysctl.c @@ -0,0 +1,1107 @@ +#include "common.h" +#include <sys/sysctl.h> +// NEEDED BY: do_bandwidth +#include <net/route.h> +// NEEDED BY do_tcp... +#include <sys/socketvar.h> +#include <netinet/tcp_var.h> +#include <netinet/tcp_fsm.h> +// NEEDED BY do_udp..., do_ip... +#include <netinet/ip_var.h> +// NEEDED BY do_udp... +#include <netinet/udp.h> +#include <netinet/udp_var.h> +// NEEDED BY do_icmp... +#include <netinet/ip.h> +#include <netinet/ip_icmp.h> +#include <netinet/icmp_var.h> +// NEEDED BY do_icmp6... +#include <netinet/icmp6.h> +// NEEDED BY do_uptime +#include <time.h> + +// MacOS calculates load averages once every 5 seconds +#define MIN_LOADAVG_UPDATE_EVERY 5 + +int do_macos_sysctl(int update_every, usec_t dt) { + (void)dt; + + static int do_loadavg = -1, do_swap = -1, do_bandwidth = -1, + do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1, do_ecn = -1, + do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, + do_udp_packets = -1, do_udp_errors = -1, do_icmp_packets = -1, do_icmpmsg = -1, + do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1, + do_ip6_packets = -1, do_ip6_fragsout = -1, do_ip6_fragsin = -1, do_ip6_errors = -1, + do_icmp6 = -1, do_icmp6_redir = -1, do_icmp6_errors = -1, do_icmp6_echos = -1, + do_icmp6_router = -1, do_icmp6_neighbor = -1, do_icmp6_types = -1, do_uptime = -1; + + + if (unlikely(do_loadavg == -1)) { + do_loadavg = config_get_boolean("plugin:macos:sysctl", "enable load average", 1); + do_swap = config_get_boolean("plugin:macos:sysctl", "system swap", 1); + do_bandwidth = config_get_boolean("plugin:macos:sysctl", "bandwidth", 1); + do_tcp_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 TCP packets", 1); + do_tcp_errors = config_get_boolean("plugin:macos:sysctl", "ipv4 TCP errors", 1); + do_tcp_handshake = config_get_boolean("plugin:macos:sysctl", "ipv4 TCP handshake issues", 1); + do_ecn = config_get_boolean_ondemand("plugin:macos:sysctl", "ECN packets", CONFIG_ONDEMAND_ONDEMAND); + do_tcpext_syscookies = config_get_boolean_ondemand("plugin:macos:sysctl", "TCP SYN cookies", CONFIG_ONDEMAND_ONDEMAND); + do_tcpext_ofo = config_get_boolean_ondemand("plugin:macos:sysctl", "TCP out-of-order queue", CONFIG_ONDEMAND_ONDEMAND); + do_tcpext_connaborts = config_get_boolean_ondemand("plugin:macos:sysctl", "TCP connection aborts", CONFIG_ONDEMAND_ONDEMAND); + do_udp_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 UDP packets", 1); + do_udp_errors = config_get_boolean("plugin:macos:sysctl", "ipv4 UDP errors", 1); + do_icmp_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 ICMP packets", 1); + do_icmpmsg = config_get_boolean("plugin:macos:sysctl", "ipv4 ICMP messages", 1); + do_ip_packets = config_get_boolean("plugin:macos:sysctl", "ipv4 packets", 1); + do_ip_fragsout = config_get_boolean("plugin:macos:sysctl", "ipv4 fragments sent", 1); + do_ip_fragsin = config_get_boolean("plugin:macos:sysctl", "ipv4 fragments assembly", 1); + do_ip_errors = config_get_boolean("plugin:macos:sysctl", "ipv4 errors", 1); + do_ip6_packets = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 packets", CONFIG_ONDEMAND_ONDEMAND); + do_ip6_fragsout = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 fragments sent", CONFIG_ONDEMAND_ONDEMAND); + do_ip6_fragsin = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 fragments assembly", CONFIG_ONDEMAND_ONDEMAND); + do_ip6_errors = config_get_boolean_ondemand("plugin:macos:sysctl", "ipv6 errors", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6 = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_redir = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp redirects", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_errors = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp errors", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_echos = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp echos", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_router = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp router", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_neighbor = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp neighbor", CONFIG_ONDEMAND_ONDEMAND); + do_icmp6_types = config_get_boolean_ondemand("plugin:macos:sysctl", "icmp types", CONFIG_ONDEMAND_ONDEMAND); + do_uptime = config_get_boolean("plugin:macos:sysctl", "system uptime", 1); + } + + RRDSET *st; + + int system_pagesize = getpagesize(); // wouldn't it be better to get value directly from hw.pagesize? + int i, n; + int common_error = 0; + size_t size; + + // NEEDED BY: do_loadavg + static usec_t last_loadavg_usec = 0; + struct loadavg sysload; + + // NEEDED BY: do_swap + struct xsw_usage swap_usage; + + // NEEDED BY: do_bandwidth + int mib[6]; + static char *ifstatdata = NULL; + char *lim, *next; + struct if_msghdr *ifm; + struct iftot { + u_long ift_ibytes; + u_long ift_obytes; + } iftot = {0, 0}; + + // NEEDED BY: do_tcp... + struct tcpstat tcpstat; + uint64_t tcps_states[TCP_NSTATES]; + + // NEEDED BY: do_udp... + struct udpstat udpstat; + + // NEEDED BY: do_icmp... + struct icmpstat icmpstat; + struct icmp_total { + u_long msgs_in; + u_long msgs_out; + } icmp_total = {0, 0}; + + // NEEDED BY: do_ip... + struct ipstat ipstat; + + // NEEDED BY: do_ip6... + /* + * Dirty workaround for /usr/include/netinet6/ip6_var.h absence. + * Struct ip6stat was copied from bsd/netinet6/ip6_var.h from xnu sources. + */ +#define IP6S_SRCRULE_COUNT 16 +#include <netinet6/scope6_var.h> + struct ip6stat { + u_quad_t ip6s_total; /* total packets received */ + u_quad_t ip6s_tooshort; /* packet too short */ + u_quad_t ip6s_toosmall; /* not enough data */ + u_quad_t ip6s_fragments; /* fragments received */ + u_quad_t ip6s_fragdropped; /* frags dropped(dups, out of space) */ + u_quad_t ip6s_fragtimeout; /* fragments timed out */ + u_quad_t ip6s_fragoverflow; /* fragments that exceeded limit */ + u_quad_t ip6s_forward; /* packets forwarded */ + u_quad_t ip6s_cantforward; /* packets rcvd for unreachable dest */ + u_quad_t ip6s_redirectsent; /* packets forwarded on same net */ + u_quad_t ip6s_delivered; /* datagrams delivered to upper level */ + u_quad_t ip6s_localout; /* total ip packets generated here */ + u_quad_t ip6s_odropped; /* lost packets due to nobufs, etc. */ + u_quad_t ip6s_reassembled; /* total packets reassembled ok */ + u_quad_t ip6s_atmfrag_rcvd; /* atomic fragments received */ + u_quad_t ip6s_fragmented; /* datagrams successfully fragmented */ + u_quad_t ip6s_ofragments; /* output fragments created */ + u_quad_t ip6s_cantfrag; /* don't fragment flag was set, etc. */ + u_quad_t ip6s_badoptions; /* error in option processing */ + u_quad_t ip6s_noroute; /* packets discarded due to no route */ + u_quad_t ip6s_badvers; /* ip6 version != 6 */ + u_quad_t ip6s_rawout; /* total raw ip packets generated */ + u_quad_t ip6s_badscope; /* scope error */ + u_quad_t ip6s_notmember; /* don't join this multicast group */ + u_quad_t ip6s_nxthist[256]; /* next header history */ + u_quad_t ip6s_m1; /* one mbuf */ + u_quad_t ip6s_m2m[32]; /* two or more mbuf */ + u_quad_t ip6s_mext1; /* one ext mbuf */ + u_quad_t ip6s_mext2m; /* two or more ext mbuf */ + u_quad_t ip6s_exthdrtoolong; /* ext hdr are not continuous */ + u_quad_t ip6s_nogif; /* no match gif found */ + u_quad_t ip6s_toomanyhdr; /* discarded due to too many headers */ + + /* + * statistics for improvement of the source address selection + * algorithm: + */ + /* number of times that address selection fails */ + u_quad_t ip6s_sources_none; + /* number of times that an address on the outgoing I/F is chosen */ + u_quad_t ip6s_sources_sameif[SCOPE6_ID_MAX]; + /* number of times that an address on a non-outgoing I/F is chosen */ + u_quad_t ip6s_sources_otherif[SCOPE6_ID_MAX]; + /* + * number of times that an address that has the same scope + * from the destination is chosen. + */ + u_quad_t ip6s_sources_samescope[SCOPE6_ID_MAX]; + /* + * number of times that an address that has a different scope + * from the destination is chosen. + */ + u_quad_t ip6s_sources_otherscope[SCOPE6_ID_MAX]; + /* number of times that a deprecated address is chosen */ + u_quad_t ip6s_sources_deprecated[SCOPE6_ID_MAX]; + + u_quad_t ip6s_forward_cachehit; + u_quad_t ip6s_forward_cachemiss; + + /* number of times that each rule of source selection is applied. */ + u_quad_t ip6s_sources_rule[IP6S_SRCRULE_COUNT]; + + /* number of times we ignored address on expensive secondary interfaces */ + u_quad_t ip6s_sources_skip_expensive_secondary_if; + + /* pkt dropped, no mbufs for control data */ + u_quad_t ip6s_pktdropcntrl; + + /* total packets trimmed/adjusted */ + u_quad_t ip6s_adj; + /* hwcksum info discarded during adjustment */ + u_quad_t ip6s_adj_hwcsum_clr; + + /* duplicate address detection collisions */ + u_quad_t ip6s_dad_collide; + + /* DAD NS looped back */ + u_quad_t ip6s_dad_loopcount; + } ip6stat; + + // NEEDED BY: do_icmp6... + struct icmp6stat icmp6stat; + struct icmp6_total { + u_long msgs_in; + u_long msgs_out; + } icmp6_total = {0, 0}; + + // NEEDED BY: do_uptime + struct timespec boot_time, cur_time; + + // -------------------------------------------------------------------- + + if (last_loadavg_usec <= dt) { + if (likely(do_loadavg)) { + if (unlikely(GETSYSCTL("vm.loadavg", sysload))) { + do_loadavg = 0; + error("DISABLED: system.load"); + } else { + + st = rrdset_find_bytype("system", "load"); + if (unlikely(!st)) { + st = rrdset_create("system", "load", NULL, "load", NULL, "System Load Average", "load", 100, (update_every < MIN_LOADAVG_UPDATE_EVERY) ? MIN_LOADAVG_UPDATE_EVERY : update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "load1", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "load5", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(st, "load15", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "load1", (collected_number) ((double)sysload.ldavg[0] / sysload.fscale * 1000)); + rrddim_set(st, "load5", (collected_number) ((double)sysload.ldavg[1] / sysload.fscale * 1000)); + rrddim_set(st, "load15", (collected_number) ((double)sysload.ldavg[2] / sysload.fscale * 1000)); + rrdset_done(st); + } + } + + last_loadavg_usec = st->update_every * USEC_PER_SEC; + } + else last_loadavg_usec -= dt; + + // -------------------------------------------------------------------- + + if (likely(do_swap)) { + if (unlikely(GETSYSCTL("vm.swapusage", swap_usage))) { + do_swap = 0; + error("DISABLED: system.swap"); + } else { + st = rrdset_find("system.swap"); + if (unlikely(!st)) { + st = rrdset_create("system", "swap", NULL, "swap", NULL, "System Swap", "MB", 201, update_every, RRDSET_TYPE_STACKED); + st->isdetail = 1; + + rrddim_add(st, "free", NULL, 1, 1048576, RRDDIM_ABSOLUTE); + rrddim_add(st, "used", NULL, 1, 1048576, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "free", swap_usage.xsu_avail); + rrddim_set(st, "used", swap_usage.xsu_used); + rrdset_done(st); + } + } + + // -------------------------------------------------------------------- + + if (likely(do_bandwidth)) { + mib[0] = CTL_NET; + mib[1] = PF_ROUTE; + mib[2] = 0; + mib[3] = AF_INET; + mib[4] = NET_RT_IFLIST2; + mib[5] = 0; + if (unlikely(sysctl(mib, 6, NULL, &size, NULL, 0))) { + error("MACOS: sysctl(%s...) failed: %s", "net interfaces", strerror(errno)); + do_bandwidth = 0; + error("DISABLED: system.ipv4"); + } else { + ifstatdata = reallocz(ifstatdata, size); + if (unlikely(sysctl(mib, 6, ifstatdata, &size, NULL, 0) < 0)) { + error("MACOS: sysctl(%s...) failed: %s", "net interfaces", strerror(errno)); + do_bandwidth = 0; + error("DISABLED: system.ipv4"); + } else { + lim = ifstatdata + size; + iftot.ift_ibytes = iftot.ift_obytes = 0; + for (next = ifstatdata; next < lim; ) { + ifm = (struct if_msghdr *)next; + next += ifm->ifm_msglen; + + if (ifm->ifm_type == RTM_IFINFO2) { + struct if_msghdr2 *if2m = (struct if_msghdr2 *)ifm; + + iftot.ift_ibytes += if2m->ifm_data.ifi_ibytes; + iftot.ift_obytes += if2m->ifm_data.ifi_obytes; + } + } + st = rrdset_find("system.ipv4"); + if (unlikely(!st)) { + st = rrdset_create("system", "ipv4", NULL, "network", NULL, "IPv4 Bandwidth", "kilobits/s", 500, update_every, RRDSET_TYPE_AREA); + + rrddim_add(st, "InOctets", "received", 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutOctets", "sent", -8, 1024, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "InOctets", iftot.ift_ibytes); + rrddim_set(st, "OutOctets", iftot.ift_obytes); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html + if (likely(do_tcp_packets || do_tcp_errors || do_tcp_handshake || do_tcpext_connaborts || do_tcpext_ofo || do_tcpext_syscookies || do_ecn)) { + if (unlikely(GETSYSCTL("net.inet.tcp.stats", tcpstat))){ + do_tcp_packets = 0; + error("DISABLED: ipv4.tcppackets"); + do_tcp_errors = 0; + error("DISABLED: ipv4.tcperrors"); + do_tcp_handshake = 0; + error("DISABLED: ipv4.tcphandshake"); + do_tcpext_connaborts = 0; + error("DISABLED: ipv4.tcpconnaborts"); + do_tcpext_ofo = 0; + error("DISABLED: ipv4.tcpofo"); + do_tcpext_syscookies = 0; + error("DISABLED: ipv4.tcpsyncookies"); + do_ecn = 0; + error("DISABLED: ipv4.ecnpkts"); + } else { + if (likely(do_tcp_packets)) { + st = rrdset_find("ipv4.tcppackets"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcppackets", NULL, "tcp", NULL, "IPv4 TCP Packets", + "packets/s", + 2600, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InSegs", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutSegs", "sent", -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InSegs", tcpstat.tcps_rcvtotal); + rrddim_set(st, "OutSegs", tcpstat.tcps_sndtotal); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_tcp_errors)) { + st = rrdset_find("ipv4.tcperrors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcperrors", NULL, "tcp", NULL, "IPv4 TCP Errors", + "packets/s", + 2700, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InErrs", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "RetransSegs", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InErrs", tcpstat.tcps_rcvbadoff + tcpstat.tcps_rcvshort); + rrddim_set(st, "InCsumErrors", tcpstat.tcps_rcvbadsum); + rrddim_set(st, "RetransSegs", tcpstat.tcps_sndrexmitpack); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_tcp_handshake)) { + st = rrdset_find("ipv4.tcphandshake"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcphandshake", NULL, "tcp", NULL, + "IPv4 TCP Handshake Issues", + "events/s", 2900, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "EstabResets", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ActiveOpens", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "PassiveOpens", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "AttemptFails", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "EstabResets", tcpstat.tcps_drops); + rrddim_set(st, "ActiveOpens", tcpstat.tcps_connattempt); + rrddim_set(st, "PassiveOpens", tcpstat.tcps_accepts); + rrddim_set(st, "AttemptFails", tcpstat.tcps_conndrops); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_tcpext_connaborts == CONFIG_ONDEMAND_YES || (do_tcpext_connaborts == CONFIG_ONDEMAND_ONDEMAND && (tcpstat.tcps_rcvpackafterwin || tcpstat.tcps_rcvafterclose || tcpstat.tcps_rcvmemdrop || tcpstat.tcps_persistdrop))) { + do_tcpext_connaborts = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv4.tcpconnaborts"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcpconnaborts", NULL, "tcp", NULL, "TCP Connection Aborts", "connections/s", 3010, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "TCPAbortOnData", "baddata", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "TCPAbortOnClose", "userclosed", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "TCPAbortOnMemory", "nomemory", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "TCPAbortOnTimeout", "timeout", 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "TCPAbortOnData", tcpstat.tcps_rcvpackafterwin); + rrddim_set(st, "TCPAbortOnClose", tcpstat.tcps_rcvafterclose); + rrddim_set(st, "TCPAbortOnMemory", tcpstat.tcps_rcvmemdrop); + rrddim_set(st, "TCPAbortOnTimeout", tcpstat.tcps_persistdrop); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_tcpext_ofo == CONFIG_ONDEMAND_YES || (do_tcpext_ofo == CONFIG_ONDEMAND_ONDEMAND && tcpstat.tcps_rcvoopack)) { + do_tcpext_ofo = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv4.tcpofo"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcpofo", NULL, "tcp", NULL, "TCP Out-Of-Order Queue", "packets/s", 3050, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "TCPOFOQueue", "inqueue", 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "TCPOFOQueue", tcpstat.tcps_rcvoopack); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_tcpext_syscookies == CONFIG_ONDEMAND_YES || (do_tcpext_syscookies == CONFIG_ONDEMAND_ONDEMAND && (tcpstat.tcps_sc_sendcookie || tcpstat.tcps_sc_recvcookie || tcpstat.tcps_sc_zonefail))) { + do_tcpext_syscookies = CONFIG_ONDEMAND_YES; + + st = rrdset_find("ipv4.tcpsyncookies"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "tcpsyncookies", NULL, "tcp", NULL, "TCP SYN Cookies", "packets/s", 3100, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "SyncookiesRecv", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "SyncookiesSent", "sent", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "SyncookiesFailed", "failed", -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "SyncookiesRecv", tcpstat.tcps_sc_recvcookie); + rrddim_set(st, "SyncookiesSent", tcpstat.tcps_sc_sendcookie); + rrddim_set(st, "SyncookiesFailed", tcpstat.tcps_sc_zonefail); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ecn == CONFIG_ONDEMAND_YES || (do_ecn == CONFIG_ONDEMAND_ONDEMAND && (tcpstat.tcps_ecn_recv_ce || tcpstat.tcps_ecn_not_supported))) { + do_ecn = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv4.ecnpkts"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "ecnpkts", NULL, "ecn", NULL, "IPv4 ECN Statistics", "packets/s", 8700, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InCEPkts", "CEP", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InNoECTPkts", "NoECTP", -1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st); + + rrddim_set(st, "InCEPkts", tcpstat.tcps_ecn_recv_ce); + rrddim_set(st, "InNoECTPkts", tcpstat.tcps_ecn_not_supported); + rrdset_done(st); + } + + } + } + + // -------------------------------------------------------------------- + + // see http://net-snmp.sourceforge.net/docs/mibs/udp.html + if (likely(do_udp_packets || do_udp_errors)) { + if (unlikely(GETSYSCTL("net.inet.udp.stats", udpstat))) { + do_udp_packets = 0; + error("DISABLED: ipv4.udppackets"); + do_udp_errors = 0; + error("DISABLED: ipv4.udperrors"); + } else { + if (likely(do_udp_packets)) { + st = rrdset_find("ipv4.udppackets"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "udppackets", NULL, "udp", NULL, "IPv4 UDP Packets", + "packets/s", 2601, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InDatagrams", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDatagrams", "sent", -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InDatagrams", udpstat.udps_ipackets); + rrddim_set(st, "OutDatagrams", udpstat.udps_opackets); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_udp_errors)) { + st = rrdset_find("ipv4.udperrors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "udperrors", NULL, "udp", NULL, "IPv4 UDP Errors", "events/s", + 2701, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "RcvbufErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "NoPorts", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "IgnoredMulti", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InErrors", udpstat.udps_hdrops + udpstat.udps_badlen); + rrddim_set(st, "NoPorts", udpstat.udps_noport); + rrddim_set(st, "RcvbufErrors", udpstat.udps_fullsock); + rrddim_set(st, "InCsumErrors", udpstat.udps_badsum + udpstat.udps_nosum); + rrddim_set(st, "IgnoredMulti", udpstat.udps_filtermcast); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_icmp_packets || do_icmpmsg)) { + if (unlikely(GETSYSCTL("net.inet.icmp.stats", icmpstat))) { + do_icmp_packets = 0; + error("DISABLED: ipv4.icmp"); + error("DISABLED: ipv4.icmp_errors"); + do_icmpmsg = 0; + error("DISABLED: ipv4.icmpmsg"); + } else { + for (i = 0; i <= ICMP_MAXTYPE; i++) { + icmp_total.msgs_in += icmpstat.icps_inhist[i]; + icmp_total.msgs_out += icmpstat.icps_outhist[i]; + } + icmp_total.msgs_in += icmpstat.icps_badcode + icmpstat.icps_badlen + icmpstat.icps_checksum + icmpstat.icps_tooshort; + + // -------------------------------------------------------------------- + + if (likely(do_icmp_packets)) { + st = rrdset_find("ipv4.icmp"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "icmp", NULL, "icmp", NULL, "IPv4 ICMP Packets", "packets/s", + 2602, + update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InMsgs", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutMsgs", "sent", -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InMsgs", icmp_total.msgs_in); + rrddim_set(st, "OutMsgs", icmp_total.msgs_out); + + rrdset_done(st); + + // -------------------------------------------------------------------- + + st = rrdset_find("ipv4.icmp_errors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "icmp_errors", NULL, "icmp", NULL, "IPv4 ICMP Errors", + "packets/s", + 2603, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutErrors", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InErrors", icmpstat.icps_badcode + icmpstat.icps_badlen + icmpstat.icps_checksum + icmpstat.icps_tooshort); + rrddim_set(st, "OutErrors", icmpstat.icps_error); + rrddim_set(st, "InCsumErrors", icmpstat.icps_checksum); + + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_icmpmsg)) { + st = rrdset_find("ipv4.icmpmsg"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "icmpmsg", NULL, "icmp", NULL, "IPv4 ICMP Messsages", + "packets/s", 2604, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InEchoReps", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchoReps", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InEchos", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchos", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InEchoReps", icmpstat.icps_inhist[ICMP_ECHOREPLY]); + rrddim_set(st, "OutEchoReps", icmpstat.icps_outhist[ICMP_ECHOREPLY]); + rrddim_set(st, "InEchos", icmpstat.icps_inhist[ICMP_ECHO]); + rrddim_set(st, "OutEchos", icmpstat.icps_outhist[ICMP_ECHO]); + + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + // see also http://net-snmp.sourceforge.net/docs/mibs/ip.html + if (likely(do_ip_packets || do_ip_fragsout || do_ip_fragsin || do_ip_errors)) { + if (unlikely(GETSYSCTL("net.inet.ip.stats", ipstat))) { + do_ip_packets = 0; + error("DISABLED: ipv4.packets"); + do_ip_fragsout = 0; + error("DISABLED: ipv4.fragsout"); + do_ip_fragsin = 0; + error("DISABLED: ipv4.fragsin"); + do_ip_errors = 0; + error("DISABLED: ipv4.errors"); + } else { + if (likely(do_ip_packets)) { + st = rrdset_find("ipv4.packets"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "packets", NULL, "packets", NULL, "IPv4 Packets", "packets/s", + 3000, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InReceives", "received", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutRequests", "sent", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ForwDatagrams", "forwarded", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InDelivers", "delivered", 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "OutRequests", ipstat.ips_localout); + rrddim_set(st, "InReceives", ipstat.ips_total); + rrddim_set(st, "ForwDatagrams", ipstat.ips_forward); + rrddim_set(st, "InDelivers", ipstat.ips_delivered); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_ip_fragsout)) { + st = rrdset_find("ipv4.fragsout"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "fragsout", NULL, "fragments", NULL, "IPv4 Fragments Sent", + "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "FragOKs", "ok", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "FragFails", "failed", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "FragCreates", "created", 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "FragOKs", ipstat.ips_fragmented); + rrddim_set(st, "FragFails", ipstat.ips_cantfrag); + rrddim_set(st, "FragCreates", ipstat.ips_ofragments); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_ip_fragsin)) { + st = rrdset_find("ipv4.fragsin"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "fragsin", NULL, "fragments", NULL, + "IPv4 Fragments Reassembly", + "packets/s", 3011, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ReasmOKs", "ok", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ReasmFails", "failed", -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "ReasmReqds", "all", 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "ReasmOKs", ipstat.ips_fragments); + rrddim_set(st, "ReasmFails", ipstat.ips_fragdropped); + rrddim_set(st, "ReasmReqds", ipstat.ips_reassembled); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (likely(do_ip_errors)) { + st = rrdset_find("ipv4.errors"); + if (unlikely(!st)) { + st = rrdset_create("ipv4", "errors", NULL, "errors", NULL, "IPv4 Errors", "packets/s", + 3002, + update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InDiscards", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDiscards", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InUnknownProtos", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InDiscards", ipstat.ips_badsum + ipstat.ips_tooshort + ipstat.ips_toosmall + ipstat.ips_toolong); + rrddim_set(st, "OutDiscards", ipstat.ips_odropped); + rrddim_set(st, "InHdrErrors", ipstat.ips_badhlen + ipstat.ips_badlen + ipstat.ips_badoptions + ipstat.ips_badvers); + rrddim_set(st, "InAddrErrors", ipstat.ips_badaddr); + rrddim_set(st, "InUnknownProtos", ipstat.ips_noproto); + rrddim_set(st, "OutNoRoutes", ipstat.ips_noroute); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_ip6_packets || do_ip6_fragsout || do_ip6_fragsin || do_ip6_errors)) { + if (unlikely(GETSYSCTL("net.inet6.ip6.stats", ip6stat))) { + do_ip6_packets = 0; + error("DISABLED: ipv6.packets"); + do_ip6_fragsout = 0; + error("DISABLED: ipv6.fragsout"); + do_ip6_fragsin = 0; + error("DISABLED: ipv6.fragsin"); + do_ip6_errors = 0; + error("DISABLED: ipv6.errors"); + } else { + if (do_ip6_packets == CONFIG_ONDEMAND_YES || (do_ip6_packets == CONFIG_ONDEMAND_ONDEMAND && + (ip6stat.ip6s_localout || ip6stat.ip6s_total || + ip6stat.ip6s_forward || ip6stat.ip6s_delivered))) { + do_ip6_packets = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.packets"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "packets", NULL, "packets", NULL, "IPv6 Packets", "packets/s", 3000, + update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "forwarded", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "delivers", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "sent", ip6stat.ip6s_localout); + rrddim_set(st, "received", ip6stat.ip6s_total); + rrddim_set(st, "forwarded", ip6stat.ip6s_forward); + rrddim_set(st, "delivers", ip6stat.ip6s_delivered); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ip6_fragsout == CONFIG_ONDEMAND_YES || (do_ip6_fragsout == CONFIG_ONDEMAND_ONDEMAND && + (ip6stat.ip6s_fragmented || ip6stat.ip6s_cantfrag || + ip6stat.ip6s_ofragments))) { + do_ip6_fragsout = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.fragsout"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "fragsout", NULL, "fragments", NULL, "IPv6 Fragments Sent", + "packets/s", 3010, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ok", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "failed", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "all", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "ok", ip6stat.ip6s_fragmented); + rrddim_set(st, "failed", ip6stat.ip6s_cantfrag); + rrddim_set(st, "all", ip6stat.ip6s_ofragments); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ip6_fragsin == CONFIG_ONDEMAND_YES || (do_ip6_fragsin == CONFIG_ONDEMAND_ONDEMAND && + (ip6stat.ip6s_reassembled || ip6stat.ip6s_fragdropped || + ip6stat.ip6s_fragtimeout || ip6stat.ip6s_fragments))) { + do_ip6_fragsin = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.fragsin"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "fragsin", NULL, "fragments", NULL, "IPv6 Fragments Reassembly", + "packets/s", 3011, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "ok", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "failed", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "timeout", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "all", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "ok", ip6stat.ip6s_reassembled); + rrddim_set(st, "failed", ip6stat.ip6s_fragdropped); + rrddim_set(st, "timeout", ip6stat.ip6s_fragtimeout); + rrddim_set(st, "all", ip6stat.ip6s_fragments); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_ip6_errors == CONFIG_ONDEMAND_YES || (do_ip6_errors == CONFIG_ONDEMAND_ONDEMAND && ( + ip6stat.ip6s_toosmall || + ip6stat.ip6s_odropped || + ip6stat.ip6s_badoptions || + ip6stat.ip6s_badvers || + ip6stat.ip6s_exthdrtoolong || + ip6stat.ip6s_sources_none || + ip6stat.ip6s_tooshort || + ip6stat.ip6s_cantforward || + ip6stat.ip6s_noroute))) { + do_ip6_errors = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.errors"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "errors", NULL, "errors", NULL, "IPv6 Errors", "packets/s", 3002, + update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "InDiscards", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDiscards", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InHdrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InAddrErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InTruncatedPkts", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InNoRoutes", NULL, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "OutNoRoutes", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InDiscards", ip6stat.ip6s_toosmall); + rrddim_set(st, "OutDiscards", ip6stat.ip6s_odropped); + + rrddim_set(st, "InHdrErrors", + ip6stat.ip6s_badoptions + ip6stat.ip6s_badvers + ip6stat.ip6s_exthdrtoolong); + rrddim_set(st, "InAddrErrors", ip6stat.ip6s_sources_none); + rrddim_set(st, "InTruncatedPkts", ip6stat.ip6s_tooshort); + rrddim_set(st, "InNoRoutes", ip6stat.ip6s_cantforward); + + rrddim_set(st, "OutNoRoutes", ip6stat.ip6s_noroute); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_icmp6 || do_icmp6_redir || do_icmp6_errors || do_icmp6_echos || do_icmp6_router || do_icmp6_neighbor || do_icmp6_types)) { + if (unlikely(GETSYSCTL("net.inet6.icmp6.stats", icmp6stat))) { + do_icmp6 = 0; + error("DISABLED: ipv6.icmp"); + } else { + for (i = 0; i <= ICMP6_MAXTYPE; i++) { + icmp6_total.msgs_in += icmp6stat.icp6s_inhist[i]; + icmp6_total.msgs_out += icmp6stat.icp6s_outhist[i]; + } + icmp6_total.msgs_in += icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen + icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort; + if (do_icmp6 == CONFIG_ONDEMAND_YES || (do_icmp6 == CONFIG_ONDEMAND_ONDEMAND && (icmp6_total.msgs_in || icmp6_total.msgs_out))) { + do_icmp6 = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmp"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmp", NULL, "icmp", NULL, "IPv6 ICMP Messages", + "messages/s", 10000, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "sent", icmp6_total.msgs_in); + rrddim_set(st, "received", icmp6_total.msgs_out); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_redir == CONFIG_ONDEMAND_YES || (do_icmp6_redir == CONFIG_ONDEMAND_ONDEMAND && (icmp6stat.icp6s_inhist[ND_REDIRECT] || icmp6stat.icp6s_outhist[ND_REDIRECT]))) { + do_icmp6_redir = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmpredir"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmpredir", NULL, "icmp", NULL, "IPv6 ICMP Redirects", + "redirects/s", 10050, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "sent", icmp6stat.icp6s_inhist[ND_REDIRECT]); + rrddim_set(st, "received", icmp6stat.icp6s_outhist[ND_REDIRECT]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_errors == CONFIG_ONDEMAND_YES || (do_icmp6_errors == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_badcode || + icmp6stat.icp6s_badlen || + icmp6stat.icp6s_checksum || + icmp6stat.icp6s_tooshort || + icmp6stat.icp6s_error || + icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH] || + icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED] || + icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB] || + icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH] || + icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED] || + icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB]))) { + do_icmp6_errors = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmperrors"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmperrors", NULL, "icmp", NULL, "IPv6 ICMP Errors", "errors/s", 10100, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutErrors", NULL, -1, 1, RRDDIM_INCREMENTAL); + + rrddim_add(st, "InCsumErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InDestUnreachs", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InPktTooBigs", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InTimeExcds", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InParmProblems", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutDestUnreachs", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutTimeExcds", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutParmProblems", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InErrors", icmp6stat.icp6s_badcode + icmp6stat.icp6s_badlen + icmp6stat.icp6s_checksum + icmp6stat.icp6s_tooshort); + rrddim_set(st, "OutErrors", icmp6stat.icp6s_error); + rrddim_set(st, "InCsumErrors", icmp6stat.icp6s_checksum); + rrddim_set(st, "InDestUnreachs", icmp6stat.icp6s_inhist[ICMP6_DST_UNREACH]); + rrddim_set(st, "InPktTooBigs", icmp6stat.icp6s_badlen); + rrddim_set(st, "InTimeExcds", icmp6stat.icp6s_inhist[ICMP6_TIME_EXCEEDED]); + rrddim_set(st, "InParmProblems", icmp6stat.icp6s_inhist[ICMP6_PARAM_PROB]); + rrddim_set(st, "OutDestUnreachs", icmp6stat.icp6s_outhist[ICMP6_DST_UNREACH]); + rrddim_set(st, "OutTimeExcds", icmp6stat.icp6s_outhist[ICMP6_TIME_EXCEEDED]); + rrddim_set(st, "OutParmProblems", icmp6stat.icp6s_outhist[ICMP6_PARAM_PROB]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_echos == CONFIG_ONDEMAND_YES || (do_icmp6_echos == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST] || + icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST] || + icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY] || + icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY]))) { + do_icmp6_echos = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmpechos"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmpechos", NULL, "icmp", NULL, "IPv6 ICMP Echo", "messages/s", 10200, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InEchos", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchos", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InEchoReplies", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutEchoReplies", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InEchos", icmp6stat.icp6s_inhist[ICMP6_ECHO_REQUEST]); + rrddim_set(st, "OutEchos", icmp6stat.icp6s_outhist[ICMP6_ECHO_REQUEST]); + rrddim_set(st, "InEchoReplies", icmp6stat.icp6s_inhist[ICMP6_ECHO_REPLY]); + rrddim_set(st, "OutEchoReplies", icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_router == CONFIG_ONDEMAND_YES || (do_icmp6_router == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT] || + icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT] || + icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT] || + icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT]))) { + do_icmp6_router = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmprouter"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmprouter", NULL, "icmp", NULL, "IPv6 Router Messages", "messages/s", 10400, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InSolicits", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutSolicits", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InSolicits", icmp6stat.icp6s_inhist[ND_ROUTER_SOLICIT]); + rrddim_set(st, "OutSolicits", icmp6stat.icp6s_outhist[ND_ROUTER_SOLICIT]); + rrddim_set(st, "InAdvertisements", icmp6stat.icp6s_inhist[ND_ROUTER_ADVERT]); + rrddim_set(st, "OutAdvertisements", icmp6stat.icp6s_outhist[ND_ROUTER_ADVERT]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_neighbor == CONFIG_ONDEMAND_YES || (do_icmp6_neighbor == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT] || + icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT] || + icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT] || + icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT]))) { + do_icmp6_neighbor = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmpneighbor"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmpneighbor", NULL, "icmp", NULL, "IPv6 Neighbor Messages", "messages/s", 10500, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InSolicits", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutSolicits", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InAdvertisements", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutAdvertisements", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InSolicits", icmp6stat.icp6s_inhist[ND_NEIGHBOR_SOLICIT]); + rrddim_set(st, "OutSolicits", icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT]); + rrddim_set(st, "InAdvertisements", icmp6stat.icp6s_inhist[ND_NEIGHBOR_ADVERT]); + rrddim_set(st, "OutAdvertisements", icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT]); + rrdset_done(st); + } + + // -------------------------------------------------------------------- + + if (do_icmp6_types == CONFIG_ONDEMAND_YES || (do_icmp6_types == CONFIG_ONDEMAND_ONDEMAND && ( + icmp6stat.icp6s_inhist[1] || + icmp6stat.icp6s_inhist[128] || + icmp6stat.icp6s_inhist[129] || + icmp6stat.icp6s_inhist[136] || + icmp6stat.icp6s_outhist[1] || + icmp6stat.icp6s_outhist[128] || + icmp6stat.icp6s_outhist[129] || + icmp6stat.icp6s_outhist[133] || + icmp6stat.icp6s_outhist[135] || + icmp6stat.icp6s_outhist[136]))) { + do_icmp6_types = CONFIG_ONDEMAND_YES; + st = rrdset_find("ipv6.icmptypes"); + if (unlikely(!st)) { + st = rrdset_create("ipv6", "icmptypes", NULL, "icmp", NULL, "IPv6 ICMP Types", + "messages/s", 10700, update_every, RRDSET_TYPE_LINE); + + rrddim_add(st, "InType1", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InType128", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InType129", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "InType136", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType1", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType128", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType129", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType133", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType135", NULL, -1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "OutType143", NULL, -1, 1, RRDDIM_INCREMENTAL); + } else + rrdset_next(st); + + rrddim_set(st, "InType1", icmp6stat.icp6s_inhist[1]); + rrddim_set(st, "InType128", icmp6stat.icp6s_inhist[128]); + rrddim_set(st, "InType129", icmp6stat.icp6s_inhist[129]); + rrddim_set(st, "InType136", icmp6stat.icp6s_inhist[136]); + rrddim_set(st, "OutType1", icmp6stat.icp6s_outhist[1]); + rrddim_set(st, "OutType128", icmp6stat.icp6s_outhist[128]); + rrddim_set(st, "OutType129", icmp6stat.icp6s_outhist[129]); + rrddim_set(st, "OutType133", icmp6stat.icp6s_outhist[133]); + rrddim_set(st, "OutType135", icmp6stat.icp6s_outhist[135]); + rrddim_set(st, "OutType143", icmp6stat.icp6s_outhist[143]); + rrdset_done(st); + } + } + } + + // -------------------------------------------------------------------- + + if (likely(do_uptime)) { + if (unlikely(GETSYSCTL("kern.boottime", boot_time))) { + do_uptime = 0; + error("DISABLED: system.uptime"); + } else { + clock_gettime(CLOCK_REALTIME, &cur_time); + st = rrdset_find("system.uptime"); + + if(unlikely(!st)) { + st = rrdset_create("system", "uptime", NULL, "uptime", NULL, "System Uptime", "seconds", 1000, update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "uptime", NULL, 1, 1, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "uptime", cur_time.tv_sec - boot_time.tv_sec); + rrdset_done(st); + } + } + + return 0; +} + +int getsysctl(const char *name, void *ptr, size_t len) +{ + size_t nlen = len; + + if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { + error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} diff --git a/src/main.c b/src/main.c index 3e6aa5046..ca134fcb0 100644 --- a/src/main.c +++ b/src/main.c @@ -7,36 +7,30 @@ void netdata_cleanup_and_exit(int ret) { error_log_limit_unlimited(); - info("Called: netdata_cleanup_and_exit()"); -#ifdef NETDATA_INTERNAL_CHECKS - rrdset_free_all(); -#else + debug(D_EXIT, "Called: netdata_cleanup_and_exit()"); + + // save the database rrdset_save_all(); -#endif - // kill_childs(); + // unlink the pid if(pidfile[0]) { if(unlink(pidfile) != 0) error("Cannot unlink pidfile '%s'.", pidfile); } - info("NetData exiting. Bye bye..."); - exit(ret); -} - -struct netdata_static_thread { - char *name; - - char *config_section; - char *config_name; +#ifdef NETDATA_INTERNAL_CHECKS + // kill all childs + //kill_childs(); - int enabled; + // free database + rrdset_free_all(); +#endif - pthread_t *thread; + info("netdata exiting. Bye bye..."); + exit(ret); +} - void (*init_routine) (void); - void *(*start_routine) (void *); -} static_threads[] = { +struct netdata_static_thread static_threads[] = { #ifdef INTERNAL_PLUGIN_NFACCT // nfacct requires root access // so, we build it as an external plugin with setuid to root @@ -45,9 +39,17 @@ struct netdata_static_thread { {"tc", "plugins", "tc", 1, NULL, NULL, tc_main}, {"idlejitter", "plugins", "idlejitter", 1, NULL, NULL, cpuidlejitter_main}, +#if defined(__FreeBSD__) + {"freebsd", "plugins", "freebsd", 1, NULL, NULL, freebsd_main}, +#elif defined(__APPLE__) + {"macos", "plugins", "macos", 1, NULL, NULL, macos_main}, +#else {"proc", "plugins", "proc", 1, NULL, NULL, proc_main}, + {"diskspace", "plugins", "diskspace", 1, NULL, NULL, proc_diskspace_main}, +#endif /* __FreeBSD__, __APPLE__*/ {"cgroups", "plugins", "cgroups", 1, NULL, NULL, cgroups_main}, {"check", "plugins", "checks", 0, NULL, NULL, checks_main}, + {"backends", NULL, NULL, 1, NULL, NULL, backends_main}, {"health", NULL, NULL, 1, NULL, NULL, health_main}, {"plugins.d", NULL, NULL, 1, NULL, NULL, pluginsd_main}, {"web", NULL, NULL, 1, NULL, NULL, socket_listen_main_multi_threaded}, @@ -149,67 +151,80 @@ int killpid(pid_t pid, int sig) void kill_childs() { + error_log_limit_unlimited(); + siginfo_t info; struct web_client *w; for(w = web_clients; w ; w = w->next) { - debug(D_EXIT, "Stopping web client %s", w->client_ip); + info("Stopping web client %s", w->client_ip); pthread_cancel(w->thread); - pthread_join(w->thread, NULL); + // it is detached + // pthread_join(w->thread, NULL); + + w->obsolete = 1; } int i; for (i = 0; static_threads[i].name != NULL ; i++) { - if(static_threads[i].thread) { - debug(D_EXIT, "Stopping %s thread", static_threads[i].name); + if(static_threads[i].enabled) { + info("Stopping %s thread", static_threads[i].name); pthread_cancel(*static_threads[i].thread); - pthread_join(*static_threads[i].thread, NULL); - static_threads[i].thread = NULL; + // it is detached + // pthread_join(*static_threads[i].thread, NULL); + + static_threads[i].enabled = 0; } } if(tc_child_pid) { - info("Killing tc-qos-helper procees"); + info("Killing tc-qos-helper process %d", tc_child_pid); if(killpid(tc_child_pid, SIGTERM) != -1) waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED); + + tc_child_pid = 0; } - tc_child_pid = 0; struct plugind *cd; for(cd = pluginsd_root ; cd ; cd = cd->next) { - debug(D_EXIT, "Stopping %s plugin thread", cd->id); - pthread_cancel(cd->thread); - pthread_join(cd->thread, NULL); - - if(cd->pid && !cd->obsolete) { - debug(D_EXIT, "killing %s plugin process", cd->id); - if(killpid(cd->pid, SIGTERM) != -1) - waitid(P_PID, (id_t) cd->pid, &info, WEXITED); + if(cd->enabled && !cd->obsolete) { + info("Stopping %s plugin thread", cd->id); + pthread_cancel(cd->thread); + + if(cd->pid) { + info("killing %s plugin child process pid %d", cd->id, cd->pid); + if(killpid(cd->pid, SIGTERM) != -1) + waitid(P_PID, (id_t) cd->pid, &info, WEXITED); + + cd->pid = 0; + } + + cd->obsolete = 1; } } // if, for any reason there is any child exited // catch it here + info("Cleaning up an other children"); waitid(P_PID, 0, &info, WEXITED|WNOHANG); - debug(D_EXIT, "All threads/childs stopped."); + info("All threads/childs stopped."); } struct option_def options[] = { - // opt description arg name default value - {'c', "Load alternate configuration file", "config_file", CONFIG_DIR "/" CONFIG_FILENAME}, - {'D', "Disable fork into background", NULL, NULL}, - {'h', "Display help message", NULL, NULL}, - {'P', "File to save a pid while running", "FILE", NULL}, - {'i', "The IP address to listen to.", "address", "All addresses"}, - {'k', "Check daemon configuration.", NULL, NULL}, - {'p', "Port to listen. Can be from 1 to 65535.", "port_number", "19999"}, - {'s', "Path to access host /proc and /sys when running in a container.", "PATH", NULL}, - {'t', "The frequency in seconds, for data collection. \ -Same as 'update every' config file option.", "seconds", "1"}, - {'u', "System username to run as.", "username", "netdata"}, - {'v', "Version of the program", NULL, NULL}, - {'W', "vendor options.", "stacksize=N|unittest|debug_flags=N", NULL}, + // opt description arg name default value + { 'c', "Configuration file to load.", "filename", CONFIG_DIR "/" CONFIG_FILENAME}, + { 'D', "Do not fork. Run in the foreground.", NULL, "run in the background"}, + { 'h', "Display this help message.", NULL, NULL}, + { 'P', "File to save a pid while running.", "filename", "do not save pid to a file"}, + { 'i', "The IP address to listen to.", "IP", "all IP addresses IPv4 and IPv6"}, + { 'k', "Check health configuration and exit.", NULL, NULL}, + { 'p', "API/Web port to use.", "port", "19999"}, + { 's', "Prefix for /proc and /sys (for containers).", "path", "no prefix"}, + { 't', "The internal clock of netdata.", "seconds", "1"}, + { 'u', "Run as user.", "username", "netdata"}, + { 'v', "Print netdata version and exit.", NULL, NULL}, + { 'W', "See Advanced options below.", "options", NULL}, }; void help(int exitcode) { @@ -231,20 +246,63 @@ void help(int exitcode) { } } - fprintf(stream, "SYNOPSIS: netdata [options]\n"); + if(max_len_arg > 30) max_len_arg = 30; + if(max_len_arg < 20) max_len_arg = 20; + + fprintf(stream, "%s", "\n" + " ^\n" + " |.-. .-. .-. .-. . netdata \n" + " | '-' '-' '-' '-' real-time performance monitoring, done right! \n" + " +----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+--->\n" + "\n" + " Copyright (C) 2016-2017, Costa Tsaousis <costa@tsaousis.gr>\n" + " Released under GNU Public License v3 or later.\n" + " All rights reserved.\n" + "\n" + " Home Page : https://my-netdata.io\n" + " Source Code: https://github.com/firehol/netdata\n" + " Wiki / Docs: https://github.com/firehol/netdata/wiki\n" + " Support : https://github.com/firehol/netdata/issues\n" + " License : https://github.com/firehol/netdata/blob/master/LICENSE.md\n" + "\n" + " Twitter : https://twitter.com/linuxnetdata\n" + " Facebook : https://www.facebook.com/linuxnetdata/\n" + "\n" + " netdata is a https://firehol.org project.\n" + "\n" + "\n" + ); + + fprintf(stream, " SYNOPSIS: netdata [options]\n"); fprintf(stream, "\n"); - fprintf(stream, "Options:\n"); + fprintf(stream, " Options:\n\n"); // Output options description. for( i = 0; i < num_opts; i++ ) { fprintf(stream, " -%c %-*s %s", options[i].val, max_len_arg, options[i].arg_name ? options[i].arg_name : "", options[i].description); if(options[i].default_value) { - fprintf(stream, " Default: %s\n", options[i].default_value); + fprintf(stream, "\n %c %-*s Default: %s\n", ' ', max_len_arg, "", options[i].default_value); } else { fprintf(stream, "\n"); } + fprintf(stream, "\n"); } + fprintf(stream, "\n Advanced options:\n\n" + " -W stacksize=N Set the stacksize (in bytes).\n\n" + " -W debug_flags=N Set runtime tracing to debug.log.\n\n" + " -W unittest Run internal unittests and exit.\n\n" + " -W simple-pattern pattern string\n" + " Check if string matches pattern and exit.\n\n" + ); + + fprintf(stream, "\n Signals netdata handles:\n\n" + " - HUP Close and reopen log files.\n" + " - USR1 Save internal DB to disk.\n" + " - USR2 Reload health configuration.\n" + "\n" + ); + fflush(stream); exit(exitcode); } @@ -329,6 +387,9 @@ int main(int argc, char **argv) string_i++; } } + // terminate optstring + optstring[string_i] ='\0'; + optstring[(num_opts *2)] ='\0'; int opt; while( (opt = getopt(argc, argv, optstring)) != -1 ) { @@ -386,10 +447,54 @@ int main(int argc, char **argv) if(unit_test_storage()) exit(1); fprintf(stderr, "\n\nALL TESTS PASSED\n\n"); exit(0); - } else if(strncmp(optarg, stacksize_string, strlen(stacksize_string)) == 0) { + } + else if(strcmp(optarg, "simple-pattern") == 0) { + if(optind + 2 > argc) { + fprintf(stderr, "%s", "\nUSAGE: -W simple-pattern 'pattern' 'string'\n\n" + " Checks if 'pattern' matches the given 'string'.\n" + " - 'pattern' can be one or more space separated words.\n" + " - each 'word' can contain one or more asterisks.\n" + " - words starting with '!' give negative matches.\n" + " - words are processed left to right\n" + "\n" + "Examples:\n" + "\n" + " > match all veth interfaces, except veth0:\n" + "\n" + " -W simple-pattern '!veth0 veth*' 'veth12'\n" + "\n" + "\n" + " > match all *.ext files directly in /path/:\n" + " (this will not match *.ext files in a subdir of /path/)\n" + "\n" + " -W simple-pattern '!/path/*/*.ext /path/*.ext' '/path/test.ext'\n" + "\n" + ); + exit(1); + } + + const char *heystack = argv[optind]; + const char *needle = argv[optind + 1]; + + SIMPLE_PATTERN *p = simple_pattern_create(heystack + , SIMPLE_PATTERN_EXACT); + int ret = simple_pattern_matches(p, needle); + simple_pattern_free(p); + + if(ret) { + fprintf(stdout, "RESULT: MATCHED - pattern '%s' matches '%s'\n", heystack, needle); + exit(0); + } + else { + fprintf(stdout, "RESULT: NOT MATCHED - pattern '%s' does not match '%s'\n", heystack, needle); + exit(1); + } + } + else if(strncmp(optarg, stacksize_string, strlen(stacksize_string)) == 0) { optarg += strlen(stacksize_string); config_set("global", "pthread stack size", optarg); - } else if(strncmp(optarg, debug_flags_string, strlen(debug_flags_string)) == 0) { + } + else if(strncmp(optarg, debug_flags_string, strlen(debug_flags_string)) == 0) { optarg += strlen(debug_flags_string); config_set("global", "debug flags", optarg); debug_flags = strtoull(optarg, NULL, 0); @@ -460,8 +565,11 @@ int main(int argc, char **argv) if(debug_flags != 0) { struct rlimit rl = { RLIM_INFINITY, RLIM_INFINITY }; if(setrlimit(RLIMIT_CORE, &rl) != 0) - info("Cannot request unlimited core dumps for debugging... Proceeding anyway..."); + error("Cannot request unlimited core dumps for debugging... Proceeding anyway..."); + +#if !(defined(__FreeBSD__) || defined(__APPLE__)) prctl(PR_SET_DUMPABLE, 1, 0, 0, 0); +#endif /* __FreeBSD__ || __APPLE__*/ } // -------------------------------------------------------------------- @@ -520,7 +628,7 @@ int main(int argc, char **argv) rrd_default_history_entries = (int) config_get_number("global", "history", RRD_DEFAULT_HISTORY_ENTRIES); if(rrd_default_history_entries < 5 || rrd_default_history_entries > RRD_HISTORY_ENTRIES_MAX) { - info("Invalid save lines %d given. Defaulting to %d.", rrd_default_history_entries, RRD_DEFAULT_HISTORY_ENTRIES); + error("Invalid history entries %d given. Defaulting to %d.", rrd_default_history_entries, RRD_DEFAULT_HISTORY_ENTRIES); rrd_default_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; } else { @@ -531,7 +639,7 @@ int main(int argc, char **argv) rrd_update_every = (int) config_get_number("global", "update every", UPDATE_EVERY); if(rrd_update_every < 1 || rrd_update_every > 600) { - info("Invalid update timer %d given. Defaulting to %d.", rrd_update_every, UPDATE_EVERY_MAX); + error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", rrd_update_every, UPDATE_EVERY_MAX); rrd_update_every = UPDATE_EVERY; } else debug(D_OPTIONS, "update timer set to %d.", rrd_update_every); @@ -635,17 +743,18 @@ int main(int argc, char **argv) if(debug_flags != 0) { struct rlimit rl = { RLIM_INFINITY, RLIM_INFINITY }; if(setrlimit(RLIMIT_CORE, &rl) != 0) - info("Cannot request unlimited core dumps for debugging... Proceeding anyway..."); + error("Cannot request unlimited core dumps for debugging... Proceeding anyway..."); +#if !(defined(__FreeBSD__) || defined(__APPLE__)) prctl(PR_SET_DUMPABLE, 1, 0, 0, 0); +#endif /* __FreeBSD__ || __APPLE__*/ } #endif /* NETDATA_INTERNAL_CHECKS */ // fork, switch user, create pid file, set process priority if(become_daemon(dont_fork, user) == -1) - fatal("Cannot demonize myself."); - - info("NetData started on pid %d", getpid()); + fatal("Cannot daemonize myself."); + info("netdata started on pid %d.", getpid()); // ------------------------------------------------------------------------ // get default pthread stack size @@ -655,7 +764,7 @@ int main(int argc, char **argv) if(i != 0) fatal("pthread_attr_setstacksize() to %zu bytes, failed with code %d.", wanted_stacksize, i); else - info("Successfully set pthread stacksize to %zu bytes", wanted_stacksize); + debug(D_SYSTEM, "Successfully set pthread stacksize to %zu bytes", wanted_stacksize); } // ------------------------------------------------------------------------ @@ -692,17 +801,19 @@ int main(int argc, char **argv) if(st->enabled) { st->thread = mallocz(sizeof(pthread_t)); - info("Starting thread %s.", st->name); + debug(D_SYSTEM, "Starting thread %s.", st->name); - if(pthread_create(st->thread, &attr, st->start_routine, NULL)) + if(pthread_create(st->thread, &attr, st->start_routine, st)) error("failed to create new thread for %s.", st->name); else if(pthread_detach(*st->thread)) error("Cannot request detach of newly created %s thread.", st->name); } - else info("Not starting thread %s.", st->name); + else debug(D_SYSTEM, "Not starting thread %s.", st->name); } + info("netdata initialization completed. Enjoy real-time performance monitoring!"); + // ------------------------------------------------------------------------ // block signals while initializing threads. sigset_t sigset; @@ -716,7 +827,7 @@ int main(int argc, char **argv) while(1) { pause(); if(netdata_exit) { - info("Exit main loop of netdata."); + debug(D_EXIT, "Exit main loop of netdata."); netdata_cleanup_and_exit(0); exit(0); } diff --git a/src/main.h b/src/main.h index 646827fbd..49afaef12 100644 --- a/src/main.h +++ b/src/main.h @@ -24,8 +24,22 @@ struct option_def { */ extern struct option_def options[]; +struct netdata_static_thread { + char *name; + + char *config_section; + char *config_name; + + volatile int enabled; + + pthread_t *thread; + + void (*init_routine) (void); + void *(*start_routine) (void *); +}; + extern void kill_childs(void); extern int killpid(pid_t pid, int signal); -extern void netdata_cleanup_and_exit(int ret) __attribute__ ((noreturn)); +extern void netdata_cleanup_and_exit(int ret) NORETURN; #endif /* NETDATA_MAIN_H */ diff --git a/src/plugin_checks.c b/src/plugin_checks.c index 007d6565f..fcc542e68 100644 --- a/src/plugin_checks.c +++ b/src/plugin_checks.c @@ -1,8 +1,7 @@ #include "common.h" -void *checks_main(void *ptr) -{ - if(ptr) { ; } +void *checks_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; info("CHECKS thread created with task id %d", gettid()); @@ -12,7 +11,7 @@ void *checks_main(void *ptr) if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("Cannot set pthread cancel state to ENABLE."); - unsigned long long usec = 0, susec = rrd_update_every * 1000000ULL, loop_usec = 0, total_susec = 0; + usec_t usec = 0, susec = rrd_update_every * USEC_PER_SEC, loop_usec = 0, total_susec = 0; struct timeval now, last, loop; RRDSET *check1, *check2, *check3, *apps_cpu = NULL; @@ -30,18 +29,18 @@ void *checks_main(void *ptr) rrddim_add(check3, "netdata", NULL, 1, 1, RRDDIM_ABSOLUTE); rrddim_add(check3, "apps.plugin", NULL, 1, 1, RRDDIM_ABSOLUTE); - gettimeofday(&last, NULL); + now_realtime_timeval(&last); while(1) { usleep(susec); // find the time to sleep in order to wait exactly update_every seconds - gettimeofday(&now, NULL); - loop_usec = usec_dt(&now, &last); + now_realtime_timeval(&now); + loop_usec = dt_usec(&now, &last); usec = loop_usec - susec; debug(D_PROCNETDEV_LOOP, "CHECK: last loop took %llu usec (worked for %llu, sleeped for %llu).", loop_usec, usec, susec); - if(usec < (rrd_update_every * 1000000ULL / 2ULL)) susec = (rrd_update_every * 1000000ULL) - usec; - else susec = rrd_update_every * 1000000ULL / 2ULL; + if(usec < (rrd_update_every * USEC_PER_SEC / 2ULL)) susec = (rrd_update_every * USEC_PER_SEC) - usec; + else susec = rrd_update_every * USEC_PER_SEC / 2ULL; // -------------------------------------------------------------------- // Calculate loop time @@ -71,13 +70,16 @@ void *checks_main(void *ptr) if(!apps_cpu) apps_cpu = rrdset_find("apps.cpu"); if(check3->counter_done) rrdset_next_usec(check3, loop_usec); - gettimeofday(&loop, NULL); - rrddim_set(check3, "caller", (long long) usec_dt(&loop, &check1->last_collected_time)); - rrddim_set(check3, "netdata", (long long) usec_dt(&loop, &check2->last_collected_time)); - if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long) usec_dt(&loop, &apps_cpu->last_collected_time)); + now_realtime_timeval(&loop); + rrddim_set(check3, "caller", (long long) dt_usec(&loop, &check1->last_collected_time)); + rrddim_set(check3, "netdata", (long long) dt_usec(&loop, &check2->last_collected_time)); + if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long) dt_usec(&loop, &apps_cpu->last_collected_time)); rrdset_done(check3); } + info("CHECKS thread exiting"); + + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } diff --git a/src/plugin_freebsd.c b/src/plugin_freebsd.c new file mode 100644 index 000000000..bdc3599ea --- /dev/null +++ b/src/plugin_freebsd.c @@ -0,0 +1,79 @@ +#include "common.h" + +void *freebsd_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + + info("FREEBSD Plugin thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + // disable (by default) various interface that are not needed + /* + config_get_boolean("plugin:proc:/proc/net/dev:lo", "enabled", 0); + config_get_boolean("plugin:proc:/proc/net/dev:fireqos_monitor", "enabled", 0); + */ + + // when ZERO, attempt to do it + int vdo_cpu_netdata = !config_get_boolean("plugin:freebsd", "netdata server resources", 1); + int vdo_freebsd_sysctl = !config_get_boolean("plugin:freebsd", "sysctl", 1); + + // keep track of the time each module was called + unsigned long long sutime_freebsd_sysctl = 0ULL; + + usec_t step = rrd_update_every * USEC_PER_SEC; + for(;;) { + usec_t now = now_realtime_usec(); + usec_t next = now - (now % step) + step; + + while(now < next) { + sleep_usec(next - now); + now = now_realtime_usec(); + } + + if(unlikely(netdata_exit)) break; + + // BEGIN -- the job to be done + + if(!vdo_freebsd_sysctl) { + debug(D_PROCNETDEV_LOOP, "FREEBSD: calling do_freebsd_sysctl()."); + now = now_realtime_usec(); + vdo_freebsd_sysctl = do_freebsd_sysctl(rrd_update_every, (sutime_freebsd_sysctl > 0)?now - sutime_freebsd_sysctl:0ULL); + sutime_freebsd_sysctl = now; + } + if(unlikely(netdata_exit)) break; + + // END -- the job is done + + // -------------------------------------------------------------------- + + if(!vdo_cpu_netdata) { + global_statistics_charts(); + registry_statistics(); + } + } + + info("FREEBSD thread exiting"); + + static_thread->enabled = 0; + pthread_exit(NULL); + return NULL; +} + +int getsysctl(const char *name, void *ptr, size_t len) +{ + size_t nlen = len; + + if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { + error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} diff --git a/src/plugin_freebsd.h b/src/plugin_freebsd.h new file mode 100644 index 000000000..e4767a091 --- /dev/null +++ b/src/plugin_freebsd.h @@ -0,0 +1,14 @@ +#ifndef NETDATA_PLUGIN_FREEBSD_H +#define NETDATA_PLUGIN_FREEBSD_H 1 + +#include <sys/sysctl.h> + +#define GETSYSCTL(name, var) getsysctl(name, &(var), sizeof(var)) + +void *freebsd_main(void *ptr); + +int getsysctl(const char *name, void *ptr, size_t len); + +extern int do_freebsd_sysctl(int update_every, usec_t dt); + +#endif /* NETDATA_PLUGIN_FREEBSD_H */ diff --git a/src/plugin_idlejitter.c b/src/plugin_idlejitter.c index 30c6d8708..7d4a4c189 100644 --- a/src/plugin_idlejitter.c +++ b/src/plugin_idlejitter.c @@ -2,11 +2,10 @@ #define CPU_IDLEJITTER_SLEEP_TIME_MS 20 -void *cpuidlejitter_main(void *ptr) -{ - if(ptr) { ; } +void *cpuidlejitter_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; - info("CPU Idle Jitter thread created with task id %d", gettid()); + info("IDLEJITTER thread created with task id %d", gettid()); if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) error("Cannot set pthread cancel type to DEFERRED."); @@ -29,16 +28,16 @@ void *cpuidlejitter_main(void *ptr) struct timeval before, after; unsigned long long counter; for(counter = 0; 1 ;counter++) { - unsigned long long usec = 0, susec = 0; + usec_t usec = 0, susec = 0; - while(susec < (rrd_update_every * 1000000ULL)) { + while(susec < (rrd_update_every * USEC_PER_SEC)) { - gettimeofday(&before, NULL); - usleep(sleep_ms * 1000); - gettimeofday(&after, NULL); + now_realtime_timeval(&before); + sleep_usec(sleep_ms * 1000); + now_realtime_timeval(&after); // calculate the time it took for a full loop - usec = usec_dt(&after, &before); + usec = dt_usec(&after, &before); susec += usec; } usec -= (sleep_ms * 1000); @@ -48,6 +47,9 @@ void *cpuidlejitter_main(void *ptr) rrdset_done(st); } + info("IDLEJITTER thread exiting"); + + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } diff --git a/src/plugin_macos.c b/src/plugin_macos.c new file mode 100644 index 000000000..3955c1414 --- /dev/null +++ b/src/plugin_macos.c @@ -0,0 +1,84 @@ +#include "common.h" + +void *macos_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + + info("MACOS Plugin thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + // disable (by default) various interface that are not needed + /* + config_get_boolean("plugin:proc:/proc/net/dev:lo", "enabled", 0); + config_get_boolean("plugin:proc:/proc/net/dev:fireqos_monitor", "enabled", 0); + */ + + // when ZERO, attempt to do it + int vdo_cpu_netdata = !config_get_boolean("plugin:macos", "netdata server resources", 1); + int vdo_macos_sysctl = !config_get_boolean("plugin:macos", "sysctl", 1); + int vdo_macos_mach_smi = !config_get_boolean("plugin:macos", "mach system management interface", 1); + int vdo_macos_iokit = !config_get_boolean("plugin:macos", "iokit", 1); + + // keep track of the time each module was called + unsigned long long sutime_macos_sysctl = 0ULL; + unsigned long long sutime_macos_mach_smi = 0ULL; + unsigned long long sutime_macos_iokit = 0ULL; + + usec_t step = rrd_update_every * USEC_PER_SEC; + for(;;) { + usec_t now = now_realtime_usec(); + usec_t next = now - (now % step) + step; + + while(now < next) { + sleep_usec(next - now); + now = now_realtime_usec(); + } + + if(unlikely(netdata_exit)) break; + + // BEGIN -- the job to be done + + if(!vdo_macos_sysctl) { + debug(D_PROCNETDEV_LOOP, "MACOS: calling do_macos_sysctl()."); + now = now_realtime_usec(); + vdo_macos_sysctl = do_macos_sysctl(rrd_update_every, (sutime_macos_sysctl > 0)?now - sutime_macos_sysctl:0ULL); + sutime_macos_sysctl = now; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_macos_mach_smi) { + debug(D_PROCNETDEV_LOOP, "MACOS: calling do_macos_mach_smi()."); + now = now_realtime_usec(); + vdo_macos_mach_smi = do_macos_mach_smi(rrd_update_every, (sutime_macos_mach_smi > 0)?now - sutime_macos_mach_smi:0ULL); + sutime_macos_mach_smi = now; + } + if(unlikely(netdata_exit)) break; + + if(!vdo_macos_iokit) { + debug(D_PROCNETDEV_LOOP, "MACOS: calling do_macos_iokit()."); + now = now_realtime_usec(); + vdo_macos_iokit = do_macos_iokit(rrd_update_every, (sutime_macos_iokit > 0)?now - sutime_macos_iokit:0ULL); + sutime_macos_iokit = now; + } + if(unlikely(netdata_exit)) break; + + // END -- the job is done + + // -------------------------------------------------------------------- + + if(!vdo_cpu_netdata) { + global_statistics_charts(); + registry_statistics(); + } + } + + info("MACOS thread exiting"); + + static_thread->enabled = 0; + pthread_exit(NULL); + return NULL; +} diff --git a/src/plugin_macos.h b/src/plugin_macos.h new file mode 100644 index 000000000..a21e5601d --- /dev/null +++ b/src/plugin_macos.h @@ -0,0 +1,14 @@ +#ifndef NETDATA_PLUGIN_MACOS_H +#define NETDATA_PLUGIN_MACOS_H 1 + +void *macos_main(void *ptr); + +#define GETSYSCTL(name, var) getsysctl(name, &(var), sizeof(var)) + +extern int getsysctl(const char *name, void *ptr, size_t len); + +extern int do_macos_sysctl(int update_every, usec_t dt); +extern int do_macos_mach_smi(int update_every, usec_t dt); +extern int do_macos_iokit(int update_every, usec_t dt); + +#endif /* NETDATA_PLUGIN_MACOS_H */ diff --git a/src/plugin_nfacct.c b/src/plugin_nfacct.c index 7843161d3..7aae33c0c 100644 --- a/src/plugin_nfacct.c +++ b/src/plugin_nfacct.c @@ -55,7 +55,7 @@ static int nfacct_callback(const struct nlmsghdr *nlh, void *data) { } void *nfacct_main(void *ptr) { - if(ptr) { ; } + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; info("NFACCT thread created with task id %d", gettid()); @@ -70,30 +70,27 @@ void *nfacct_main(void *ptr) { struct nlmsghdr *nlh = NULL; unsigned int seq = 0, portid = 0; - seq = time(NULL) - 1; + seq = now_realtime_sec() - 1; nl = mnl_socket_open(NETLINK_NETFILTER); if(!nl) { error("nfacct.plugin: mnl_socket_open() failed"); - pthread_exit(NULL); - return NULL; + goto cleanup; } if(mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { - mnl_socket_close(nl); error("nfacct.plugin: mnl_socket_bind() failed"); - pthread_exit(NULL); - return NULL; + goto cleanup; } portid = mnl_socket_get_portid(nl); // ------------------------------------------------------------------------ struct timeval last, now; - unsigned long long usec = 0, susec = 0; + usec_t usec = 0, susec = 0; RRDSET *st = NULL; - gettimeofday(&last, NULL); + now_realtime_timeval(&last); // ------------------------------------------------------------------------ @@ -104,16 +101,13 @@ void *nfacct_main(void *ptr) { nlh = nfacct_nlmsg_build_hdr(buf, NFNL_MSG_ACCT_GET, NLM_F_DUMP, seq); if(!nlh) { - mnl_socket_close(nl); error("nfacct.plugin: nfacct_nlmsg_build_hdr() failed"); - pthread_exit(NULL); - return NULL; + goto cleanup; } if(mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { error("nfacct.plugin: mnl_socket_send"); - pthread_exit(NULL); - return NULL; + goto cleanup; } if(nfacct_list) nfacct_list->len = 0; @@ -125,14 +119,13 @@ void *nfacct_main(void *ptr) { if (ret == -1) { error("nfacct.plugin: error communicating with kernel."); - pthread_exit(NULL); - return NULL; + goto cleanup; } // -------------------------------------------------------------------- - gettimeofday(&now, NULL); - usec = usec_dt(&now, &last) - susec; + now_realtime_timeval(&now); + usec = dt_usec(&now, &last) - susec; debug(D_NFACCT_LOOP, "nfacct.plugin: last loop took %llu usec (worked for %llu, sleeped for %llu).", usec + susec, usec, susec); if(usec < (rrd_update_every * 1000000ULL / 2ULL)) susec = (rrd_update_every * 1000000ULL) - usec; @@ -191,7 +184,12 @@ void *nfacct_main(void *ptr) { memmove(&last, &now, sizeof(struct timeval)); } - mnl_socket_close(nl); +cleanup: + info("NFACCT thread exiting"); + + if(nl) mnl_socket_close(nl); + + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } diff --git a/src/plugin_proc.c b/src/plugin_proc.c index a50a22514..9b66b7c28 100644 --- a/src/plugin_proc.c +++ b/src/plugin_proc.c @@ -1,241 +1,146 @@ #include "common.h" -void *proc_main(void *ptr) -{ - (void)ptr; +static struct proc_module { + const char *name; + const char *dim; - info("PROC Plugin thread created with task id %d", gettid()); + int enabled; - if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) - error("Cannot set pthread cancel type to DEFERRED."); + int (*func)(int update_every, usec_t dt); + usec_t last_run_usec; + usec_t duration; - if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) - error("Cannot set pthread cancel state to ENABLE."); + RRDDIM *rd; - // disable (by default) various interface that are not needed - config_get_boolean("plugin:proc:/proc/net/dev:lo", "enabled", 0); - config_get_boolean("plugin:proc:/proc/net/dev:fireqos_monitor", "enabled", 0); - - // when ZERO, attempt to do it - int vdo_proc_net_dev = !config_get_boolean("plugin:proc", "/proc/net/dev", 1); - int vdo_proc_diskstats = !config_get_boolean("plugin:proc", "/proc/diskstats", 1); - int vdo_proc_net_snmp = !config_get_boolean("plugin:proc", "/proc/net/snmp", 1); - int vdo_proc_net_snmp6 = !config_get_boolean("plugin:proc", "/proc/net/snmp6", 1); - int vdo_proc_net_netstat = !config_get_boolean("plugin:proc", "/proc/net/netstat", 1); - int vdo_proc_net_stat_conntrack = !config_get_boolean("plugin:proc", "/proc/net/stat/conntrack", 1); - int vdo_proc_net_ip_vs_stats = !config_get_boolean("plugin:proc", "/proc/net/ip_vs/stats", 1); - int vdo_proc_net_stat_synproxy = !config_get_boolean("plugin:proc", "/proc/net/stat/synproxy", 1); - int vdo_proc_stat = !config_get_boolean("plugin:proc", "/proc/stat", 1); - int vdo_proc_meminfo = !config_get_boolean("plugin:proc", "/proc/meminfo", 1); - int vdo_proc_vmstat = !config_get_boolean("plugin:proc", "/proc/vmstat", 1); - int vdo_proc_net_rpc_nfs = !config_get_boolean("plugin:proc", "/proc/net/rpc/nfs", 1); - int vdo_proc_net_rpc_nfsd = !config_get_boolean("plugin:proc", "/proc/net/rpc/nfsd", 1); - int vdo_proc_sys_kernel_random_entropy_avail = !config_get_boolean("plugin:proc", "/proc/sys/kernel/random/entropy_avail", 1); - int vdo_proc_interrupts = !config_get_boolean("plugin:proc", "/proc/interrupts", 1); - int vdo_proc_softirqs = !config_get_boolean("plugin:proc", "/proc/softirqs", 1); - int vdo_proc_net_softnet_stat = !config_get_boolean("plugin:proc", "/proc/net/softnet_stat", 1); - int vdo_proc_loadavg = !config_get_boolean("plugin:proc", "/proc/loadavg", 1); - int vdo_sys_kernel_mm_ksm = !config_get_boolean("plugin:proc", "/sys/kernel/mm/ksm", 1); - int vdo_cpu_netdata = !config_get_boolean("plugin:proc", "netdata server resources", 1); - - // keep track of the time each module was called - unsigned long long sutime_proc_net_dev = 0ULL; - unsigned long long sutime_proc_diskstats = 0ULL; - unsigned long long sutime_proc_net_snmp = 0ULL; - unsigned long long sutime_proc_net_snmp6 = 0ULL; - unsigned long long sutime_proc_net_netstat = 0ULL; - unsigned long long sutime_proc_net_stat_conntrack = 0ULL; - unsigned long long sutime_proc_net_ip_vs_stats = 0ULL; - unsigned long long sutime_proc_net_stat_synproxy = 0ULL; - unsigned long long sutime_proc_stat = 0ULL; - unsigned long long sutime_proc_meminfo = 0ULL; - unsigned long long sutime_proc_vmstat = 0ULL; - unsigned long long sutime_proc_net_rpc_nfs = 0ULL; - unsigned long long sutime_proc_net_rpc_nfsd = 0ULL; - unsigned long long sutime_proc_sys_kernel_random_entropy_avail = 0ULL; - unsigned long long sutime_proc_interrupts = 0ULL; - unsigned long long sutime_proc_softirqs = 0ULL; - unsigned long long sutime_proc_net_softnet_stat = 0ULL; - unsigned long long sutime_proc_loadavg = 0ULL; - unsigned long long sutime_sys_kernel_mm_ksm = 0ULL; - - // the next time we will run - aligned properly - unsigned long long sunext = (time(NULL) - (time(NULL) % rrd_update_every) + rrd_update_every) * 1000000ULL; - unsigned long long sunow; +} proc_modules[] = { - for(;;) { - if(unlikely(netdata_exit)) break; + // system metrics + { .name = "/proc/stat", .dim = "stat", .func = do_proc_stat }, + { .name = "/proc/uptime", .dim = "uptime", .func = do_proc_uptime }, + { .name = "/proc/loadavg", .dim = "loadavg", .func = do_proc_loadavg }, + { .name = "/proc/sys/kernel/random/entropy_avail", .dim = "entropy", .func = do_proc_sys_kernel_random_entropy_avail }, - // delay until it is our time to run - while((sunow = time_usec()) < sunext) - sleep_usec(sunext - sunow); + // CPU metrics + { .name = "/proc/interrupts", .dim = "interrupts", .func = do_proc_interrupts }, + { .name = "/proc/softirqs", .dim = "softirqs", .func = do_proc_softirqs }, - // find the next time we need to run - while(time_usec() > sunext) - sunext += rrd_update_every * 1000000ULL; + // memory metrics + { .name = "/proc/vmstat", .dim = "vmstat", .func = do_proc_vmstat }, + { .name = "/proc/meminfo", .dim = "meminfo", .func = do_proc_meminfo }, + { .name = "/sys/kernel/mm/ksm", .dim = "ksm", .func = do_sys_kernel_mm_ksm }, + { .name = "/sys/devices/system/edac/mc", .dim = "ecc", .func = do_proc_sys_devices_system_edac_mc }, + { .name = "/sys/devices/system/node", .dim = "numa", .func = do_proc_sys_devices_system_node }, - if(unlikely(netdata_exit)) break; + // network metrics + { .name = "/proc/net/dev", .dim = "netdev", .func = do_proc_net_dev }, + { .name = "/proc/net/netstat", .dim = "netstat", .func = do_proc_net_netstat }, + { .name = "/proc/net/snmp", .dim = "snmp", .func = do_proc_net_snmp }, + { .name = "/proc/net/snmp6", .dim = "snmp6", .func = do_proc_net_snmp6 }, + { .name = "/proc/net/softnet_stat", .dim = "softnet", .func = do_proc_net_softnet_stat }, + { .name = "/proc/net/ip_vs/stats", .dim = "ipvs", .func = do_proc_net_ip_vs_stats }, - // BEGIN -- the job to be done + // firewall metrics + { .name = "/proc/net/stat/conntrack", .dim = "conntrack", .func = do_proc_net_stat_conntrack }, + { .name = "/proc/net/stat/synproxy", .dim = "synproxy", .func = do_proc_net_stat_synproxy }, - if(!vdo_sys_kernel_mm_ksm) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_kernel_mm_ksm()."); + // disk metrics + { .name = "/proc/diskstats", .dim = "diskstats", .func = do_proc_diskstats }, - sunow = time_usec(); - vdo_sys_kernel_mm_ksm = do_sys_kernel_mm_ksm(rrd_update_every, (sutime_sys_kernel_mm_ksm > 0)?sunow - sutime_sys_kernel_mm_ksm:0ULL); - sutime_sys_kernel_mm_ksm = sunow; - } - if(unlikely(netdata_exit)) break; + // NFS metrics + { .name = "/proc/net/rpc/nfsd", .dim = "nfsd", .func = do_proc_net_rpc_nfsd }, + { .name = "/proc/net/rpc/nfs", .dim = "nfs", .func = do_proc_net_rpc_nfs }, - if(!vdo_proc_loadavg) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_loadavg()."); - sunow = time_usec(); - vdo_proc_loadavg = do_proc_loadavg(rrd_update_every, (sutime_proc_loadavg > 0)?sunow - sutime_proc_loadavg:0ULL); - sutime_proc_loadavg = sunow; - } - if(unlikely(netdata_exit)) break; + // IPC metrics + { .name = "ipc", .dim = "ipc", .func = do_ipc }, - if(!vdo_proc_interrupts) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_interrupts()."); - sunow = time_usec(); - vdo_proc_interrupts = do_proc_interrupts(rrd_update_every, (sutime_proc_interrupts > 0)?sunow - sutime_proc_interrupts:0ULL); - sutime_proc_interrupts = sunow; - } - if(unlikely(netdata_exit)) break; + // the terminator of this array + { .name = NULL, .dim = NULL, .func = NULL } +}; - if(!vdo_proc_softirqs) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_softirqs()."); - sunow = time_usec(); - vdo_proc_softirqs = do_proc_softirqs(rrd_update_every, (sutime_proc_softirqs > 0)?sunow - sutime_proc_softirqs:0ULL); - sutime_proc_softirqs = sunow; - } - if(unlikely(netdata_exit)) break; +void *proc_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; - if(!vdo_proc_net_softnet_stat) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_softnet_stat()."); - sunow = time_usec(); - vdo_proc_net_softnet_stat = do_proc_net_softnet_stat(rrd_update_every, (sutime_proc_net_softnet_stat > 0)?sunow - sutime_proc_net_softnet_stat:0ULL); - sutime_proc_net_softnet_stat = sunow; - } - if(unlikely(netdata_exit)) break; + info("PROC Plugin thread created with task id %d", gettid()); - if(!vdo_proc_sys_kernel_random_entropy_avail) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_sys_kernel_random_entropy_avail()."); - sunow = time_usec(); - vdo_proc_sys_kernel_random_entropy_avail = do_proc_sys_kernel_random_entropy_avail(rrd_update_every, (sutime_proc_sys_kernel_random_entropy_avail > 0)?sunow - sutime_proc_sys_kernel_random_entropy_avail:0ULL); - sutime_proc_sys_kernel_random_entropy_avail = sunow; - } - if(unlikely(netdata_exit)) break; + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); - if(!vdo_proc_net_dev) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_dev()."); - sunow = time_usec(); - vdo_proc_net_dev = do_proc_net_dev(rrd_update_every, (sutime_proc_net_dev > 0)?sunow - sutime_proc_net_dev:0ULL); - sutime_proc_net_dev = sunow; - } - if(unlikely(netdata_exit)) break; + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); - if(!vdo_proc_diskstats) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_diskstats()."); - sunow = time_usec(); - vdo_proc_diskstats = do_proc_diskstats(rrd_update_every, (sutime_proc_diskstats > 0)?sunow - sutime_proc_diskstats:0ULL); - sutime_proc_diskstats = sunow; - } - if(unlikely(netdata_exit)) break; + int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1); - if(!vdo_proc_net_snmp) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp()."); - sunow = time_usec(); - vdo_proc_net_snmp = do_proc_net_snmp(rrd_update_every, (sutime_proc_net_snmp > 0)?sunow - sutime_proc_net_snmp:0ULL); - sutime_proc_net_snmp = sunow; - } - if(unlikely(netdata_exit)) break; + // check the enabled status for each module + int i; + for(i = 0 ; proc_modules[i].name ;i++) { + struct proc_module *pm = &proc_modules[i]; - if(!vdo_proc_net_snmp6) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp6()."); - sunow = time_usec(); - vdo_proc_net_snmp6 = do_proc_net_snmp6(rrd_update_every, (sutime_proc_net_snmp6 > 0)?sunow - sutime_proc_net_snmp6:0ULL); - sutime_proc_net_snmp6 = sunow; - } - if(unlikely(netdata_exit)) break; + pm->enabled = config_get_boolean("plugin:proc", pm->name, 1); + pm->last_run_usec = 0ULL; + pm->duration = 0ULL; + pm->rd = NULL; + } - if(!vdo_proc_net_netstat) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_netstat()."); - sunow = time_usec(); - vdo_proc_net_netstat = do_proc_net_netstat(rrd_update_every, (sutime_proc_net_netstat > 0)?sunow - sutime_proc_net_netstat:0ULL); - sutime_proc_net_netstat = sunow; - } - if(unlikely(netdata_exit)) break; + usec_t step = rrd_update_every * USEC_PER_SEC; + for(;;) { + usec_t now = now_monotonic_usec(); + usec_t next = now - (now % step) + step; - if(!vdo_proc_net_stat_conntrack) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_stat_conntrack()."); - sunow = time_usec(); - vdo_proc_net_stat_conntrack = do_proc_net_stat_conntrack(rrd_update_every, (sutime_proc_net_stat_conntrack > 0)?sunow - sutime_proc_net_stat_conntrack:0ULL); - sutime_proc_net_stat_conntrack = sunow; + while(now < next) { + sleep_usec(next - now); + now = now_monotonic_usec(); } - if(unlikely(netdata_exit)) break; - if(!vdo_proc_net_ip_vs_stats) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_ip_vs_stats()."); - sunow = time_usec(); - vdo_proc_net_ip_vs_stats = do_proc_net_ip_vs_stats(rrd_update_every, (sutime_proc_net_ip_vs_stats > 0)?sunow - sutime_proc_net_ip_vs_stats:0ULL); - sutime_proc_net_ip_vs_stats = sunow; - } if(unlikely(netdata_exit)) break; - if(!vdo_proc_net_stat_synproxy) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_stat_synproxy()."); - sunow = time_usec(); - vdo_proc_net_stat_synproxy = do_proc_net_stat_synproxy(rrd_update_every, (sutime_proc_net_stat_synproxy > 0)?sunow - sutime_proc_net_stat_synproxy:0ULL); - sutime_proc_net_stat_synproxy = sunow; - } - if(unlikely(netdata_exit)) break; + // BEGIN -- the job to be done - if(!vdo_proc_stat) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_stat()."); - sunow = time_usec(); - vdo_proc_stat = do_proc_stat(rrd_update_every, (sutime_proc_stat > 0)?sunow - sutime_proc_stat:0ULL); - sutime_proc_stat = sunow; - } - if(unlikely(netdata_exit)) break; + for(i = 0 ; proc_modules[i].name ;i++) { + struct proc_module *pm = &proc_modules[i]; + if(unlikely(!pm->enabled)) continue; - if(!vdo_proc_meminfo) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_meminfo()."); - sunow = time_usec(); - vdo_proc_meminfo = do_proc_meminfo(rrd_update_every, (sutime_proc_meminfo > 0)?sunow - sutime_proc_meminfo:0ULL); - sutime_proc_meminfo = sunow; - } - if(unlikely(netdata_exit)) break; + debug(D_PROCNETDEV_LOOP, "PROC calling %s.", pm->name); - if(!vdo_proc_vmstat) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_vmstat()."); - sunow = time_usec(); - vdo_proc_vmstat = do_proc_vmstat(rrd_update_every, (sutime_proc_vmstat > 0)?sunow - sutime_proc_vmstat:0ULL); - sutime_proc_vmstat = sunow; - } - if(unlikely(netdata_exit)) break; + pm->enabled = !pm->func(rrd_update_every, (pm->last_run_usec > 0)?now - pm->last_run_usec:0ULL); + pm->last_run_usec = now; - if(!vdo_proc_net_rpc_nfsd) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_rpc_nfsd()."); - sunow = time_usec(); - vdo_proc_net_rpc_nfsd = do_proc_net_rpc_nfsd(rrd_update_every, (sutime_proc_net_rpc_nfsd > 0)?sunow - sutime_proc_net_rpc_nfsd:0ULL); - sutime_proc_net_rpc_nfsd = sunow; - } - if(unlikely(netdata_exit)) break; + now = now_monotonic_usec(); + pm->duration = now - pm->last_run_usec; - if(!vdo_proc_net_rpc_nfs) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_rpc_nfs()."); - sunow = time_usec(); - vdo_proc_net_rpc_nfs = do_proc_net_rpc_nfs(rrd_update_every, (sutime_proc_net_rpc_nfs > 0)?sunow - sutime_proc_net_rpc_nfs:0ULL); - sutime_proc_net_rpc_nfs = sunow; + if(unlikely(netdata_exit)) break; } - if(unlikely(netdata_exit)) break; // END -- the job is done // -------------------------------------------------------------------- - if(!vdo_cpu_netdata) { + if(vdo_cpu_netdata) { + static RRDSET *st = NULL; + + if(unlikely(!st)) { + st = rrdset_find_bytype("netdata", "plugin_proc_modules"); + + if(!st) { + st = rrdset_create("netdata", "plugin_proc_modules", NULL, "proc", NULL, "NetData Proc Plugin Modules Durations", "milliseconds/run", 132001, rrd_update_every, RRDSET_TYPE_STACKED); + + for(i = 0 ; proc_modules[i].name ;i++) { + struct proc_module *pm = &proc_modules[i]; + if(unlikely(!pm->enabled)) continue; + + pm->rd = rrddim_add(st, pm->dim, NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + } + } + else rrdset_next(st); + + for(i = 0 ; proc_modules[i].name ;i++) { + struct proc_module *pm = &proc_modules[i]; + if(unlikely(!pm->enabled)) continue; + + rrddim_set_by_pointer(st, pm->rd, pm->duration); + } + rrdset_done(st); + global_statistics_charts(); registry_statistics(); } @@ -243,6 +148,41 @@ void *proc_main(void *ptr) info("PROC thread exiting"); + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } + +int get_numa_node_count(void) +{ + static int numa_node_count = -1; + + if (numa_node_count != -1) + return numa_node_count; + + numa_node_count = 0; + + char name[FILENAME_MAX + 1]; + snprintfz(name, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/devices/system/node"); + char *dirname = config_get("plugin:proc:/sys/devices/system/node", "directory to monitor", name); + + DIR *dir = opendir(dirname); + if(dir) { + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type != DT_DIR) + continue; + + if(strncmp(de->d_name, "node", 4) != 0) + continue; + + if(!isdigit(de->d_name[4])) + continue; + + numa_node_count++; + } + closedir(dir); + } + + return numa_node_count; +} diff --git a/src/plugin_proc.h b/src/plugin_proc.h index f72a9970f..5dee7853c 100644 --- a/src/plugin_proc.h +++ b/src/plugin_proc.h @@ -3,24 +3,29 @@ void *proc_main(void *ptr); -extern int do_proc_net_dev(int update_every, unsigned long long dt); -extern int do_proc_diskstats(int update_every, unsigned long long dt); -extern int do_proc_net_snmp(int update_every, unsigned long long dt); -extern int do_proc_net_snmp6(int update_every, unsigned long long dt); -extern int do_proc_net_netstat(int update_every, unsigned long long dt); -extern int do_proc_net_stat_conntrack(int update_every, unsigned long long dt); -extern int do_proc_net_ip_vs_stats(int update_every, unsigned long long dt); -extern int do_proc_stat(int update_every, unsigned long long dt); -extern int do_proc_meminfo(int update_every, unsigned long long dt); -extern int do_proc_vmstat(int update_every, unsigned long long dt); -extern int do_proc_net_rpc_nfs(int update_every, unsigned long long dt); -extern int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt); -extern int do_proc_sys_kernel_random_entropy_avail(int update_every, unsigned long long dt); -extern int do_proc_interrupts(int update_every, unsigned long long dt); -extern int do_proc_softirqs(int update_every, unsigned long long dt); -extern int do_sys_kernel_mm_ksm(int update_every, unsigned long long dt); -extern int do_proc_loadavg(int update_every, unsigned long long dt); -extern int do_proc_net_stat_synproxy(int update_every, unsigned long long dt); -extern int do_proc_net_softnet_stat(int update_every, unsigned long long dt); +extern int do_proc_net_dev(int update_every, usec_t dt); +extern int do_proc_diskstats(int update_every, usec_t dt); +extern int do_proc_net_snmp(int update_every, usec_t dt); +extern int do_proc_net_snmp6(int update_every, usec_t dt); +extern int do_proc_net_netstat(int update_every, usec_t dt); +extern int do_proc_net_stat_conntrack(int update_every, usec_t dt); +extern int do_proc_net_ip_vs_stats(int update_every, usec_t dt); +extern int do_proc_stat(int update_every, usec_t dt); +extern int do_proc_meminfo(int update_every, usec_t dt); +extern int do_proc_vmstat(int update_every, usec_t dt); +extern int do_proc_net_rpc_nfs(int update_every, usec_t dt); +extern int do_proc_net_rpc_nfsd(int update_every, usec_t dt); +extern int do_proc_sys_kernel_random_entropy_avail(int update_every, usec_t dt); +extern int do_proc_interrupts(int update_every, usec_t dt); +extern int do_proc_softirqs(int update_every, usec_t dt); +extern int do_sys_kernel_mm_ksm(int update_every, usec_t dt); +extern int do_proc_loadavg(int update_every, usec_t dt); +extern int do_proc_net_stat_synproxy(int update_every, usec_t dt); +extern int do_proc_net_softnet_stat(int update_every, usec_t dt); +extern int do_proc_uptime(int update_every, usec_t dt); +extern int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt); +extern int do_proc_sys_devices_system_node(int update_every, usec_t dt); + +extern int get_numa_node_count(void); #endif /* NETDATA_PLUGIN_PROC_H */ diff --git a/src/plugin_proc_diskspace.c b/src/plugin_proc_diskspace.c new file mode 100644 index 000000000..43e6dd7c5 --- /dev/null +++ b/src/plugin_proc_diskspace.c @@ -0,0 +1,324 @@ +#include "common.h" + +#define DELAULT_EXLUDED_PATHS "/proc/* /sys/* /var/run/user/* /run/user/*" + +static struct mountinfo *disk_mountinfo_root = NULL; +static int check_for_new_mountpoints_every = 15; + +static inline void mountinfo_reload(int force) { + static time_t last_loaded = 0; + time_t now = now_realtime_sec(); + + if(force || now - last_loaded >= check_for_new_mountpoints_every) { + // mountinfo_free() can be called with NULL disk_mountinfo_root + mountinfo_free(disk_mountinfo_root); + + // re-read mountinfo in case something changed + disk_mountinfo_root = mountinfo_read(1); + + last_loaded = now; + } +} + +// Data to be stored in DICTIONARY mount_points used by do_disk_space_stats(). +// This DICTIONARY is used to lookup the settings of the mount point on each iteration. +struct mount_point_metadata { + int do_space; + int do_inodes; + + size_t collected; // the number of times this has been collected + + RRDSET *st_space; + RRDDIM *rd_space_used; + RRDDIM *rd_space_avail; + RRDDIM *rd_space_reserved; + + RRDSET *st_inodes; + RRDDIM *rd_inodes_used; + RRDDIM *rd_inodes_avail; + RRDDIM *rd_inodes_reserved; +}; + +static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) { + const char *family = mi->mount_point; + const char *disk = mi->persistent_id; + + static DICTIONARY *mount_points = NULL; + static SIMPLE_PATTERN *excluded_mountpoints = NULL; + int do_space, do_inodes; + + if(unlikely(!mount_points)) { + const char *s; + SIMPLE_PREFIX_MODE mode = SIMPLE_PATTERN_EXACT; + + if(config_exists("plugin:proc:/proc/diskstats", "exclude space metrics on paths") && !config_exists("plugin:proc:diskspace", "exclude space metrics on paths")) { + // the config exists in the old section + s = config_get("plugin:proc:/proc/diskstats", "exclude space metrics on paths", DELAULT_EXLUDED_PATHS); + mode = SIMPLE_PATTERN_PREFIX; + } + else + s = config_get("plugin:proc:diskspace", "exclude space metrics on paths", DELAULT_EXLUDED_PATHS); + + mount_points = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + excluded_mountpoints = simple_pattern_create(s, mode); + } + + struct mount_point_metadata *m = dictionary_get(mount_points, mi->mount_point); + if(unlikely(!m)) { + char var_name[4096 + 1]; + snprintfz(var_name, 4096, "plugin:proc:diskspace:%s", mi->mount_point); + + int def_space = config_get_boolean_ondemand("plugin:proc:diskspace", "space usage for all disks", CONFIG_ONDEMAND_ONDEMAND); + int def_inodes = config_get_boolean_ondemand("plugin:proc:diskspace", "inodes usage for all disks", CONFIG_ONDEMAND_ONDEMAND); + + if(unlikely(simple_pattern_matches(excluded_mountpoints, mi->mount_point))) { + def_space = CONFIG_ONDEMAND_NO; + def_inodes = CONFIG_ONDEMAND_NO; + } + + do_space = config_get_boolean_ondemand(var_name, "space usage", def_space); + do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", def_inodes); + + struct mount_point_metadata mp = { + .do_space = do_space, + .do_inodes = do_inodes, + + .collected = 0, + + .st_space = NULL, + .rd_space_avail = NULL, + .rd_space_used = NULL, + .rd_space_reserved = NULL, + + .st_inodes = NULL, + .rd_inodes_avail = NULL, + .rd_inodes_used = NULL, + .rd_inodes_reserved = NULL + }; + + m = dictionary_set(mount_points, mi->mount_point, &mp, sizeof(struct mount_point_metadata)); + } + else { + do_space = m->do_space; + do_inodes = m->do_inodes; + } + + if(unlikely(do_space == CONFIG_ONDEMAND_NO && do_inodes == CONFIG_ONDEMAND_NO)) + return; + + if(unlikely(mi->flags & MOUNTINFO_READONLY && !m->collected)) + return; + + struct statvfs buff_statvfs; + if (statvfs(mi->mount_point, &buff_statvfs) < 0) { + error("Failed statvfs() for '%s' (disk '%s')", mi->mount_point, disk); + return; + } + + // logic found at get_fs_usage() in coreutils + unsigned long bsize = (buff_statvfs.f_frsize) ? buff_statvfs.f_frsize : buff_statvfs.f_bsize; + + fsblkcnt_t bavail = buff_statvfs.f_bavail; + fsblkcnt_t btotal = buff_statvfs.f_blocks; + fsblkcnt_t bavail_root = buff_statvfs.f_bfree; + fsblkcnt_t breserved_root = bavail_root - bavail; + fsblkcnt_t bused; + if(likely(btotal >= bavail_root)) + bused = btotal - bavail_root; + else + bused = bavail_root - btotal; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(btotal != bavail + breserved_root + bused)) + error("Disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused); +#endif + + // -------------------------------------------------------------------------- + + fsfilcnt_t favail = buff_statvfs.f_favail; + fsfilcnt_t ftotal = buff_statvfs.f_files; + fsfilcnt_t favail_root = buff_statvfs.f_ffree; + fsfilcnt_t freserved_root = favail_root - favail; + fsfilcnt_t fused = ftotal - favail_root; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(btotal != bavail + breserved_root + bused)) + error("Disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused); +#endif + + // -------------------------------------------------------------------------- + + int rendered = 0; + + if(do_space == CONFIG_ONDEMAND_YES || (do_space == CONFIG_ONDEMAND_ONDEMAND && (bavail || breserved_root || bused))) { + if(unlikely(!m->st_space)) { + m->do_space = CONFIG_ONDEMAND_YES; + m->st_space = rrdset_find_bytype("disk_space", disk); + if(unlikely(!m->st_space)) { + char title[4096 + 1]; + snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source); + m->st_space = rrdset_create("disk_space", disk, NULL, family, "disk.space", title, "GB", 2023, update_every, RRDSET_TYPE_STACKED); + } + + m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL, bsize, 1024 * 1024 * 1024, RRDDIM_ABSOLUTE); + m->rd_space_used = rrddim_add(m->st_space, "used", NULL, bsize, 1024 * 1024 * 1024, RRDDIM_ABSOLUTE); + m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root", bsize, 1024 * 1024 * 1024, RRDDIM_ABSOLUTE); + } + else + rrdset_next(m->st_space); + + rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number)bavail); + rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number)bused); + rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number)breserved_root); + rrdset_done(m->st_space); + + rendered++; + } + + // -------------------------------------------------------------------------- + + if(do_inodes == CONFIG_ONDEMAND_YES || (do_inodes == CONFIG_ONDEMAND_ONDEMAND && (favail || freserved_root || fused))) { + if(unlikely(!m->st_inodes)) { + m->do_inodes = CONFIG_ONDEMAND_YES; + m->st_inodes = rrdset_find_bytype("disk_inodes", disk); + if(unlikely(!m->st_inodes)) { + char title[4096 + 1]; + snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source); + m->st_inodes = rrdset_create("disk_inodes", disk, NULL, family, "disk.inodes", title, "Inodes", 2024, update_every, RRDSET_TYPE_STACKED); + } + + m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRDDIM_ABSOLUTE); + m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRDDIM_ABSOLUTE); + m->rd_inodes_reserved = rrddim_add(m->st_inodes, "reserved_for_root", "reserved for root", 1, 1, RRDDIM_ABSOLUTE); + } + else + rrdset_next(m->st_inodes); + + rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number)favail); + rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number)fused); + rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_reserved, (collected_number)freserved_root); + rrdset_done(m->st_inodes); + + rendered++; + } + + // -------------------------------------------------------------------------- + + if(likely(rendered)) + m->collected++; +} + +void *proc_diskspace_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + + info("DISKSPACE thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("Cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("Cannot set pthread cancel state to ENABLE."); + + int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1); + + int update_every = (int)config_get_number("plugin:proc:diskspace", "update every", rrd_update_every); + if(update_every < rrd_update_every) + update_every = rrd_update_every; + + check_for_new_mountpoints_every = (int)config_get_number("plugin:proc:diskspace", "check for new mount points every", check_for_new_mountpoints_every); + if(check_for_new_mountpoints_every < update_every) + check_for_new_mountpoints_every = update_every; + + struct rusage thread; + + usec_t last = 0, dt = 0; + usec_t step = update_every * USEC_PER_SEC; + for(;;) { + usec_t now = now_monotonic_usec(); + usec_t next = now - (now % step) + step; + + dt = (last)?now - last:0; + + while(now < next) { + sleep_usec(next - now); + now = now_monotonic_usec(); + } + + last = now; + + if(unlikely(netdata_exit)) break; + + + // -------------------------------------------------------------------------- + // this is smart enough not to reload it every time + + mountinfo_reload(0); + + + // -------------------------------------------------------------------------- + // disk space metrics + + struct mountinfo *mi; + for(mi = disk_mountinfo_root; mi; mi = mi->next) { + + if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND | MOUNTINFO_IS_SAME_DEV | MOUNTINFO_NO_STAT | MOUNTINFO_NO_SIZE))) + continue; + + do_disk_space_stats(mi, update_every); + if(unlikely(netdata_exit)) break; + } + + if(unlikely(netdata_exit)) break; + + if(vdo_cpu_netdata) { + static RRDSET *stcpu_thread = NULL, *st_duration = NULL; + static RRDDIM *rd_user = NULL, *rd_system = NULL, *rd_duration = NULL; + + // ---------------------------------------------------------------- + + getrusage(RUSAGE_THREAD, &thread); + + if(!stcpu_thread) { + stcpu_thread = rrdset_find("netdata.plugin_diskspace"); + if(!stcpu_thread) stcpu_thread = rrdset_create("netdata", "plugin_diskspace", NULL, "diskspace", NULL + , "NetData Disk Space Plugin CPU usage", "milliseconds/s", 132020 + , update_every, RRDSET_TYPE_STACKED); + + rd_user = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); + } + else + rrdset_next(stcpu_thread); + + rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set_by_pointer(stcpu_thread, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrdset_done(stcpu_thread); + + // ---------------------------------------------------------------- + + if(!st_duration) { + st_duration = rrdset_find("netdata.plugin_diskspace_dt"); + if(!st_duration) st_duration = rrdset_create("netdata", "plugin_diskspace_dt", NULL, "diskspace", NULL + , "NetData Disk Space Plugin Duration", "milliseconds/run", 132021 + , update_every, RRDSET_TYPE_AREA); + + rd_duration = rrddim_add(st_duration, "duration", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + else + rrdset_next(st_duration); + + rrddim_set_by_pointer(st_duration, rd_duration, dt); + rrdset_done(st_duration); + + // ---------------------------------------------------------------- + + if(unlikely(netdata_exit)) break; + } + } + + info("DISKSPACE thread exiting"); + + static_thread->enabled = 0; + pthread_exit(NULL); + return NULL; +} diff --git a/src/plugin_proc_diskspace.h b/src/plugin_proc_diskspace.h new file mode 100644 index 000000000..dcec28f75 --- /dev/null +++ b/src/plugin_proc_diskspace.h @@ -0,0 +1,6 @@ +#ifndef NETDATA_PLUGIN_PROC_DISKSPACE_H +#define NETDATA_PLUGIN_PROC_DISKSPACE_H + +extern void *proc_diskspace_main(void *ptr); + +#endif //NETDATA_PLUGIN_PROC_DISKSPACE_H diff --git a/src/plugin_tc.c b/src/plugin_tc.c index 399fcd6db..0fa595320 100644 --- a/src/plugin_tc.c +++ b/src/plugin_tc.c @@ -44,7 +44,7 @@ struct tc_class { char name_updated; char updated; // updated bytes - int seen; // seen in the tc list (even without bytes) + int unupdated; // the number of times, this has been found un-updated struct tc_class *next; struct tc_class *prev; @@ -100,8 +100,8 @@ avl_tree tc_device_root_index = { tc_device_compare }; -#define tc_device_index_add(st) avl_insert(&tc_device_root_index, (avl *)(st)) -#define tc_device_index_del(st) avl_remove(&tc_device_root_index, (avl *)(st)) +#define tc_device_index_add(st) (struct tc_device *)avl_insert(&tc_device_root_index, (avl *)(st)) +#define tc_device_index_del(st) (struct tc_device *)avl_remove(&tc_device_root_index, (avl *)(st)) static inline struct tc_device *tc_device_index_find(const char *id, uint32_t hash) { struct tc_device tmp; @@ -121,8 +121,8 @@ static int tc_class_compare(void* a, void* b) { else return strcmp(((struct tc_class *)a)->id, ((struct tc_class *)b)->id); } -#define tc_class_index_add(st, rd) avl_insert(&((st)->classes_index), (avl *)(rd)) -#define tc_class_index_del(st, rd) avl_remove(&((st)->classes_index), (avl *)(rd)) +#define tc_class_index_add(st, rd) (struct tc_class *)avl_insert(&((st)->classes_index), (avl *)(rd)) +#define tc_class_index_del(st, rd) (struct tc_class *)avl_remove(&((st)->classes_index), (avl *)(rd)) static inline struct tc_class *tc_class_index_find(struct tc_device *st, const char *id, uint32_t hash) { struct tc_class tmp; @@ -144,9 +144,10 @@ static inline void tc_class_free(struct tc_device *n, struct tc_class *c) { if(c->next) c->next->prev = c->prev; if(c->prev) c->prev->next = c->next; - debug(D_TC_LOOP, "Removing from device '%s' class '%s', parentid '%s', leafid '%s', seen=%d", n->id, c->id, c->parentid?c->parentid:"", c->leafid?c->leafid:"", c->seen); + debug(D_TC_LOOP, "Removing from device '%s' class '%s', parentid '%s', leafid '%s', unused=%d", n->id, c->id, c->parentid?c->parentid:"", c->leafid?c->leafid:"", c->unupdated); - tc_class_index_del(n, c); + if(unlikely(tc_class_index_del(n, c) != c)) + error("plugin_tc: INTERNAL ERROR: attempt remove class '%s' from device '%s': removed a different calls", c->id, n->id); freez(c->id); freez(c->name); @@ -159,7 +160,7 @@ static inline void tc_device_classes_cleanup(struct tc_device *d) { static int cleanup_every = 999; if(unlikely(cleanup_every > 0)) { - cleanup_every = (int) config_get_number("plugin:tc", "cleanup unused classes every", 60); + cleanup_every = (int) config_get_number("plugin:tc", "cleanup unused classes every", 120); if(cleanup_every < 0) cleanup_every = -cleanup_every; } @@ -168,7 +169,7 @@ static inline void tc_device_classes_cleanup(struct tc_device *d) { struct tc_class *c = d->classes; while(c) { - if(unlikely(cleanup_every > 0 && c->seen >= cleanup_every)) { + if(unlikely(cleanup_every && c->unupdated >= cleanup_every)) { struct tc_class *nc = c->next; tc_class_free(d, c); c = nc; @@ -203,6 +204,11 @@ static inline void tc_device_commit(struct tc_device *d) { for(c = d->classes ; c ; c = c->next) { c->isleaf = 1; c->hasparent = 0; + + if(unlikely(!c->updated)) + c->unupdated++; + else + c->unupdated = 0; } // mark the classes as leafs and parents @@ -256,22 +262,22 @@ static inline void tc_device_commit(struct tc_device *d) { if(unlikely(d->enabled == (char)-1)) { char var_name[CONFIG_MAX_NAME + 1]; snprintfz(var_name, CONFIG_MAX_NAME, "qos for %s", d->id); - d->enabled = config_get_boolean_ondemand("plugin:tc", var_name, enable_new_interfaces); + d->enabled = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_new_interfaces); snprintfz(var_name, CONFIG_MAX_NAME, "traffic chart for %s", d->id); - d->enabled_bytes = config_get_boolean_ondemand("plugin:tc", var_name, enable_bytes); + d->enabled_bytes = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_bytes); snprintfz(var_name, CONFIG_MAX_NAME, "packets chart for %s", d->id); - d->enabled_packets = config_get_boolean_ondemand("plugin:tc", var_name, enable_packets); + d->enabled_packets = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_packets); snprintfz(var_name, CONFIG_MAX_NAME, "dropped packets chart for %s", d->id); - d->enabled_dropped = config_get_boolean_ondemand("plugin:tc", var_name, enable_dropped); + d->enabled_dropped = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_dropped); snprintfz(var_name, CONFIG_MAX_NAME, "tokens chart for %s", d->id); - d->enabled_tokens = config_get_boolean_ondemand("plugin:tc", var_name, enable_tokens); + d->enabled_tokens = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_tokens); snprintfz(var_name, CONFIG_MAX_NAME, "ctokens chart for %s", d->id); - d->enabled_ctokens = config_get_boolean_ondemand("plugin:tc", var_name, enable_ctokens); + d->enabled_ctokens = (char)config_get_boolean_ondemand("plugin:tc", var_name, enable_ctokens); } debug(D_TC_LOOP, "TC: evaluating TC device '%s'. enabled = %d/%d (bytes: %d/%d, packets: %d/%d, dropped: %d/%d, tokens: %d/%d, ctokens: %d/%d), classes = %d (bytes = %llu, packets = %llu, dropped = %llu, tokens = %llu, ctokens = %llu).", @@ -306,7 +312,7 @@ static inline void tc_device_commit(struct tc_device *d) { } else { debug(D_TC_LOOP, "TC: Updating chart for device '%s'", d->name?d->name:d->id); - rrdset_next_plugins(d->st_bytes); + rrdset_next(d->st_bytes); if(unlikely(d->name_updated && d->name && strcmp(d->id, d->name) != 0)) { rrdset_set_name(d->st_bytes, d->name); @@ -321,8 +327,6 @@ static inline void tc_device_commit(struct tc_device *d) { if(unlikely(!c->updated)) continue; if(c->isleaf && c->hasparent) { - c->seen++; - if(unlikely(!c->rd_bytes)) { c->rd_bytes = rrddim_find(d->st_bytes, c->id); if(unlikely(!c->rd_bytes)) { @@ -367,7 +371,7 @@ static inline void tc_device_commit(struct tc_device *d) { } else { debug(D_TC_LOOP, "TC: Updating _packets chart for device '%s'", d->name?d->name:d->id); - rrdset_next_plugins(d->st_packets); + rrdset_next(d->st_packets); // FIXME // update the family @@ -421,7 +425,7 @@ static inline void tc_device_commit(struct tc_device *d) { } else { debug(D_TC_LOOP, "TC: Updating _dropped chart for device '%s'", d->name?d->name:d->id); - rrdset_next_plugins(d->st_dropped); + rrdset_next(d->st_dropped); // FIXME // update the family @@ -475,7 +479,7 @@ static inline void tc_device_commit(struct tc_device *d) { } else { debug(D_TC_LOOP, "TC: Updating _tokens chart for device '%s'", d->name?d->name:d->id); - rrdset_next_plugins(d->st_tokens); + rrdset_next(d->st_tokens); // FIXME // update the family @@ -529,7 +533,7 @@ static inline void tc_device_commit(struct tc_device *d) { } else { debug(D_TC_LOOP, "TC: Updating _ctokens chart for device '%s'", d->name?d->name:d->id); - rrdset_next_plugins(d->st_ctokens); + rrdset_next(d->st_ctokens); // FIXME // update the family @@ -619,7 +623,8 @@ static inline struct tc_device *tc_device_create(char *id) d->enabled = (char)-1; avl_init(&d->classes_index, tc_class_compare); - tc_device_index_add(d); + if(unlikely(tc_device_index_add(d) != d)) + error("plugin_tc: INTERNAL ERROR: removing device '%s' removed a different device.", d->id); if(!tc_device_root) { tc_device_root = d; @@ -660,11 +665,9 @@ static inline struct tc_class *tc_class_add(struct tc_device *n, char *id, char c->leaf_hash = simple_hash(c->leafid); } - tc_class_index_add(n, c); + if(unlikely(tc_class_index_add(n, c) != c)) + error("plugin_tc: INTERNAL ERROR: attempt index class '%s' on device '%s': already exists", c->id, n->id); } - - c->seen = 1; - return(c); } @@ -677,7 +680,8 @@ static inline void tc_device_free(struct tc_device *n) else tc_device_root = n->prev; } - tc_device_index_del(n); + if(unlikely(tc_device_index_del(n) != n)) + error("plugin_tc: INTERNAL ERROR: removing device '%s' removed a different device.", n->id); while(n->classes) tc_class_free(n, n->classes); @@ -743,9 +747,9 @@ static inline void tc_split_words(char *str, char **words, int max_words) { while(i < max_words) words[i++] = NULL; } -pid_t tc_child_pid = 0; +volatile pid_t tc_child_pid = 0; void *tc_main(void *ptr) { - (void)ptr; + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; info("TC thread created with task id %d", gettid()); @@ -789,11 +793,10 @@ void *tc_main(void *ptr) { snprintfz(buffer, TC_LINE_MAX, "exec %s %d", tc_script, rrd_update_every); debug(D_TC_LOOP, "executing '%s'", buffer); - fp = mypopen(buffer, &tc_child_pid); + fp = mypopen(buffer, (pid_t *)&tc_child_pid); if(unlikely(!fp)) { error("TC: Cannot popen(\"%s\", \"r\").", buffer); - pthread_exit(NULL); - return NULL; + goto cleanup; } while(fgets(buffer, TC_LINE_MAX, fp) != NULL) { @@ -884,7 +887,7 @@ void *tc_main(void *ptr) { else if(unlikely(device && class && first_hash == SENT_HASH && strcmp(words[0], "Sent") == 0)) { // debug(D_TC_LOOP, "SENT line '%s'", words[1]); if(likely(words[1] && *words[1])) { - class->bytes = strtoull(words[1], NULL, 10); + class->bytes = str2ull(words[1]); class->updated = 1; } else { @@ -892,35 +895,35 @@ void *tc_main(void *ptr) { } if(likely(words[3] && *words[3])) - class->packets = strtoull(words[3], NULL, 10); + class->packets = str2ull(words[3]); if(likely(words[6] && *words[6])) - class->dropped = strtoull(words[6], NULL, 10); + class->dropped = str2ull(words[6]); if(likely(words[8] && *words[8])) - class->overlimits = strtoull(words[8], NULL, 10); + class->overlimits = str2ull(words[8]); if(likely(words[10] && *words[10])) - class->requeues = strtoull(words[8], NULL, 10); + class->requeues = str2ull(words[8]); } else if(unlikely(device && class && class->updated && first_hash == LENDED_HASH && strcmp(words[0], "lended:") == 0)) { // debug(D_TC_LOOP, "LENDED line '%s'", words[1]); if(likely(words[1] && *words[1])) - class->lended = strtoull(words[1], NULL, 10); + class->lended = str2ull(words[1]); if(likely(words[3] && *words[3])) - class->borrowed = strtoull(words[3], NULL, 10); + class->borrowed = str2ull(words[3]); if(likely(words[5] && *words[5])) - class->giants = strtoull(words[5], NULL, 10); + class->giants = str2ull(words[5]); } else if(unlikely(device && class && class->updated && first_hash == TOKENS_HASH && strcmp(words[0], "tokens:") == 0)) { // debug(D_TC_LOOP, "TOKENS line '%s'", words[1]); if(likely(words[1] && *words[1])) - class->tokens = strtoull(words[1], NULL, 10); + class->tokens = str2ull(words[1]); if(likely(words[3] && *words[3])) - class->ctokens = strtoull(words[3], NULL, 10); + class->ctokens = str2ull(words[3]); } else if(unlikely(device && first_hash == SETDEVICENAME_HASH && strcmp(words[0], "SETDEVICENAME") == 0)) { // debug(D_TC_LOOP, "SETDEVICENAME line '%s'", words[1]); @@ -983,7 +986,7 @@ void *tc_main(void *ptr) { } // fgets() failed or loop broke - int code = mypclose(fp, tc_child_pid); + int code = mypclose(fp, (pid_t)tc_child_pid); tc_child_pid = 0; if(unlikely(device)) { @@ -994,8 +997,7 @@ void *tc_main(void *ptr) { if(unlikely(netdata_exit)) { tc_device_free_all(); - pthread_exit(NULL); - return NULL; + goto cleanup; } if(code == 1 || code == 127) { @@ -1004,13 +1006,16 @@ void *tc_main(void *ptr) { error("TC: tc-qos-helper.sh exited with code %d. Disabling it.", code); tc_device_free_all(); - pthread_exit(NULL); - return NULL; + goto cleanup; } sleep((unsigned int) rrd_update_every); } +cleanup: + info("TC thread exiting"); + + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } diff --git a/src/plugin_tc.h b/src/plugin_tc.h index c3abbddd0..9a0a19cce 100644 --- a/src/plugin_tc.h +++ b/src/plugin_tc.h @@ -1,7 +1,7 @@ #ifndef NETDATA_PLUGIN_TC_H #define NETDATA_PLUGIN_TC_H 1 -extern pid_t tc_child_pid; +extern volatile pid_t tc_child_pid; extern void *tc_main(void *ptr); #endif /* NETDATA_PLUGIN_TC_H */ diff --git a/src/plugins_d.c b/src/plugins_d.c index 0030e2216..4b83b5281 100644 --- a/src/plugins_d.c +++ b/src/plugins_d.c @@ -87,15 +87,16 @@ static int pluginsd_split_words(char *str, char **words, int max_words) { void *pluginsd_worker_thread(void *arg) { struct plugind *cd = (struct plugind *)arg; + cd->obsolete = 0; + char line[PLUGINSD_LINE_MAX + 1]; #ifdef DETACH_PLUGINS_FROM_NETDATA - unsigned long long usec = 0, susec = 0; + usec_t usec = 0, susec = 0; struct timeval last = {0, 0} , now = {0, 0}; #endif char *words[MAX_WORDS] = { NULL }; - uint32_t SET_HASH = simple_hash("SET"); uint32_t BEGIN_HASH = simple_hash("BEGIN"); uint32_t END_HASH = simple_hash("END"); uint32_t FLUSH_HASH = simple_hash("FLUSH"); @@ -121,7 +122,6 @@ void *pluginsd_worker_thread(void *arg) info("PLUGINSD: '%s' running on pid %d", cd->fullfilename, cd->pid); RRDSET *st = NULL; - char *s; uint32_t hash; while(likely(fgets(line, PLUGINSD_LINE_MAX, fp) != NULL)) { @@ -132,7 +132,7 @@ void *pluginsd_worker_thread(void *arg) // debug(D_PLUGINSD, "PLUGINSD: %s: %s", cd->filename, line); int w = pluginsd_split_words(line, words, MAX_WORDS); - s = words[0]; + char *s = words[0]; if(unlikely(!s || !*s || !w)) { // debug(D_PLUGINSD, "PLUGINSD: empty line"); continue; @@ -140,9 +140,7 @@ void *pluginsd_worker_thread(void *arg) // debug(D_PLUGINSD, "PLUGINSD: words 0='%s' 1='%s' 2='%s' 3='%s' 4='%s' 5='%s' 6='%s' 7='%s' 8='%s' 9='%s'", words[0], words[1], words[2], words[3], words[4], words[5], words[6], words[7], words[8], words[9]); - hash = simple_hash(s); - - if(likely(hash == SET_HASH && !strcmp(s, "SET"))) { + if(likely(!simple_hash_strcmp(s, "SET", &hash))) { char *dimension = words[1]; char *value = words[2]; @@ -186,10 +184,10 @@ void *pluginsd_worker_thread(void *arg) } if(likely(st->counter_done)) { - unsigned long long microseconds = 0; - if(microseconds_txt && *microseconds_txt) microseconds = strtoull(microseconds_txt, NULL, 10); + usec_t microseconds = 0; + if(microseconds_txt && *microseconds_txt) microseconds = str2ull(microseconds_txt); if(microseconds) rrdset_next_usec(st, microseconds); - else rrdset_next_plugins(st); + else rrdset_next(st); } } else if(likely(hash == END_HASH && !strcmp(s, "END"))) { @@ -241,10 +239,10 @@ void *pluginsd_worker_thread(void *arg) } int priority = 1000; - if(likely(priority_s)) priority = atoi(priority_s); + if(likely(priority_s)) priority = str2i(priority_s); int update_every = cd->update_every; - if(likely(update_every_s)) update_every = atoi(update_every_s); + if(likely(update_every_s)) update_every = str2i(update_every_s); if(unlikely(!update_every)) update_every = cd->update_every; int chart_type = RRDSET_TYPE_LINE; @@ -326,7 +324,7 @@ void *pluginsd_worker_thread(void *arg) else if(unlikely(st->debug)) debug(D_PLUGINSD, "PLUGINSD: dimension %s/%s already exists. Not adding it again.", st->id, id); } else if(unlikely(hash == DISABLE_HASH && !strcmp(s, "DISABLE"))) { - error("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename); + info("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename); cd->enabled = 0; killpid(cd->pid, SIGTERM); break; @@ -342,17 +340,17 @@ void *pluginsd_worker_thread(void *arg) else if(likely(hash == STOPPING_WAKE_ME_UP_PLEASE_HASH && !strcmp(s, "STOPPING_WAKE_ME_UP_PLEASE"))) { error("PLUGINSD: '%s' (pid %d) called STOPPING_WAKE_ME_UP_PLEASE.", cd->fullfilename, cd->pid); - gettimeofday(&now, NULL); + now_realtime_timeval(&now); if(unlikely(!usec && !susec)) { // our first run - susec = cd->rrd_update_every * 1000000ULL; + susec = cd->rrd_update_every * USEC_PER_SEC; } else { // second+ run - usec = usec_dt(&now, &last) - susec; + usec = dt_usec(&now, &last) - susec; error("PLUGINSD: %s last loop took %llu usec (worked for %llu, sleeped for %llu).\n", cd->fullfilename, usec + susec, usec, susec); - if(unlikely(usec < (rrd_update_every * 1000000ULL / 2ULL))) susec = (rrd_update_every * 1000000ULL) - usec; - else susec = rrd_update_every * 1000000ULL / 2ULL; + if(unlikely(usec < (rrd_update_every * USEC_PER_SEC / 2ULL))) susec = (rrd_update_every * USEC_PER_SEC) - usec; + else susec = rrd_update_every * USEC_PER_SEC / 2ULL; } error("PLUGINSD: %s sleeping for %llu. Will kill with SIGCONT pid %d to wake it up.\n", cd->fullfilename, susec, cd->pid); @@ -394,7 +392,7 @@ void *pluginsd_worker_thread(void *arg) // we have collected something if(likely(cd->serial_failures <= 10)) { - error("PLUGINSD: '%s' exited with error code %d, but has given useful output in the past (%zu times). Waiting a bit before starting it again.", cd->fullfilename, code, cd->successful_collections); + error("PLUGINSD: '%s' exited with error code %d, but has given useful output in the past (%zu times). %s", cd->fullfilename, code, cd->successful_collections, cd->enabled?"Waiting a bit before starting it again.":"Will not start it again - it is disabled."); sleep((unsigned int) (cd->update_every * 10)); } else { @@ -410,7 +408,7 @@ void *pluginsd_worker_thread(void *arg) // we have collected nothing so far if(likely(cd->serial_failures <= 10)) { - error("PLUGINSD: '%s' (pid %d) does not generate useful output but it reports success (exits with 0). Waiting a bit before starting it again.", cd->fullfilename, cd->pid); + error("PLUGINSD: '%s' (pid %d) does not generate useful output but it reports success (exits with 0). %s.", cd->fullfilename, cd->pid, cd->enabled?"Waiting a bit before starting it again.":"Will not start it again - it is disabled."); sleep((unsigned int) (cd->update_every * 10)); } else { @@ -434,7 +432,7 @@ void *pluginsd_worker_thread(void *arg) } void *pluginsd_main(void *ptr) { - (void)ptr; + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; info("PLUGINS.D thread created with task id %d", gettid()); @@ -462,8 +460,7 @@ void *pluginsd_main(void *ptr) { dir = opendir(dir_name); if(unlikely(!dir)) { error("Cannot open directory '%s'.", dir_name); - pthread_exit(NULL); - return NULL; + goto cleanup; } while(likely((file = readdir(dir)))) { @@ -490,9 +487,9 @@ void *pluginsd_main(void *ptr) { } // check if it runs already - for(cd = pluginsd_root ; likely(cd) ; cd = cd->next) { + for(cd = pluginsd_root ; cd ; cd = cd->next) if(unlikely(strcmp(cd->filename, file->d_name) == 0)) break; - } + if(likely(cd && !cd->obsolete)) { debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is already running", cd->filename); continue; @@ -510,7 +507,7 @@ void *pluginsd_main(void *ptr) { cd->enabled = enabled; cd->update_every = (int) config_get_number(cd->id, "update every", rrd_update_every); - cd->started_t = time(NULL); + cd->started_t = now_realtime_sec(); char *def = ""; snprintfz(cd->cmd, PLUGINSD_CMD_MAX, "exec %s %d %s", cd->fullfilename, cd->update_every, config_get(cd->id, "command options", def)); @@ -518,26 +515,29 @@ void *pluginsd_main(void *ptr) { // link it if(likely(pluginsd_root)) cd->next = pluginsd_root; pluginsd_root = cd; - } - cd->obsolete = 0; - if(unlikely(!cd->enabled)) continue; - - // spawn a new thread for it - if(unlikely(pthread_create(&cd->thread, NULL, pluginsd_worker_thread, cd) != 0)) { - error("PLUGINSD: failed to create new thread for plugin '%s'.", cd->filename); + // it is not currently running cd->obsolete = 1; + + if(cd->enabled) { + // spawn a new thread for it + if(unlikely(pthread_create(&cd->thread, NULL, pluginsd_worker_thread, cd) != 0)) + error("PLUGINSD: failed to create new thread for plugin '%s'.", cd->filename); + + else if(unlikely(pthread_detach(cd->thread) != 0)) + error("PLUGINSD: Cannot request detach of newly created thread for plugin '%s'.", cd->filename); + } } - else if(unlikely(pthread_detach(cd->thread) != 0)) - error("PLUGINSD: Cannot request detach of newly created thread for plugin '%s'.", cd->filename); } closedir(dir); sleep((unsigned int) scan_frequency); } +cleanup: info("PLUGINS.D thread exiting"); + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } diff --git a/src/plugins_d.h b/src/plugins_d.h index 6f1fbd6e1..3c74355a3 100644 --- a/src/plugins_d.h +++ b/src/plugins_d.h @@ -23,8 +23,8 @@ struct plugind { // without collecting values int update_every; // the plugin default data collection frequency - int obsolete; // do not touch this structure after setting this to 1 - int enabled; // if this is enabled or not + volatile int obsolete; // do not touch this structure after setting this to 1 + volatile int enabled; // if this is enabled or not time_t started_t; diff --git a/src/popen.c b/src/popen.c index ad8d7596f..8448b7311 100644 --- a/src/popen.c +++ b/src/popen.c @@ -138,7 +138,7 @@ FILE *mypopen(const char *command, pid_t *pidptr) error("pre-execution of command '%s' on pid %d: failed to set default signal handler for SIGUSR2.", command, getpid()); } - info("executing command: '%s' on pid %d.", command, getpid()); + debug(D_CHILDS, "executing command: '%s' on pid %d.", command, getpid()); execl("/bin/sh", "sh", "-c", command, NULL); exit(1); } diff --git a/src/proc_diskstats.c b/src/proc_diskstats.c index 459d5a133..9ccac6dc7 100644 --- a/src/proc_diskstats.c +++ b/src/proc_diskstats.c @@ -12,6 +12,7 @@ static struct disk { unsigned long minor; int sector_size; int type; + char *mount_point; // disk options caching @@ -23,18 +24,16 @@ static struct disk { int do_qops; int do_util; int do_backlog; - int do_space; - int do_inodes; struct disk *next; } *disk_root = NULL; -static struct mountinfo *disk_mountinfo_root = NULL; - static struct disk *get_disk(unsigned long major, unsigned long minor, char *disk) { static char path_to_get_hw_sector_size[FILENAME_MAX + 1] = ""; static char path_to_get_hw_sector_size_partitions[FILENAME_MAX + 1] = ""; static char path_find_block_device[FILENAME_MAX + 1] = ""; + static struct mountinfo *disk_mountinfo_root = NULL; + struct disk *d; // search for it in our RAM list. @@ -43,11 +42,7 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis // should not be that many, it should be acceptable for(d = disk_root; d ; d = d->next) if(unlikely(d->major == major && d->minor == minor)) - break; - - // if we found it, return it - if(likely(d)) - return d; + return d; // not found // create a new disk structure @@ -62,7 +57,7 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis d->next = NULL; // append it to the list - if(!disk_root) + if(unlikely(!disk_root)) disk_root = d; else { struct disk *last; @@ -84,18 +79,19 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis // find if it is a partition // by checking if /sys/dev/block/MAJOR:MINOR/partition is readable. snprintfz(buffer, FILENAME_MAX, path_find_block_device, major, minor, "partition"); - if(access(buffer, R_OK) == 0) { + if(likely(access(buffer, R_OK) == 0)) { d->type = DISK_TYPE_PARTITION; - } else { + } + else { // find if it is a container // by checking if /sys/dev/block/MAJOR:MINOR/slaves has entries snprintfz(buffer, FILENAME_MAX, path_find_block_device, major, minor, "slaves/"); - DIR *dirp = opendir(buffer); - if (dirp != NULL) { + DIR *dirp = opendir(buffer); + if(likely(dirp != NULL)) { struct dirent *dp; while( (dp = readdir(dirp)) ) { // . and .. are also files in empty folders. - if(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) { + if(unlikely(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0)) { continue; } @@ -104,7 +100,7 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis // Stop the loop after we found one file. break; } - if(closedir(dirp) == -1) + if(unlikely(closedir(dirp) == -1)) error("Unable to close dir %s", buffer); } } @@ -115,30 +111,25 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis // mountinfo_find() can be called with NULL disk_mountinfo_root struct mountinfo *mi = mountinfo_find(disk_mountinfo_root, d->major, d->minor); if(unlikely(!mi)) { - // mountinfo_free() can be called with NULL disk_mountinfo_root + // mountinfo_free can be called with NULL mountinfo_free(disk_mountinfo_root); - - // re-read mountinfo in case something changed - disk_mountinfo_root = mountinfo_read(); - - // search again for this disk + disk_mountinfo_root = mountinfo_read(0); mi = mountinfo_find(disk_mountinfo_root, d->major, d->minor); } - if(mi) + if(unlikely(mi)) d->mount_point = strdupz(mi->mount_point); - // no need to check for NULL else d->mount_point = NULL; // ------------------------------------------------------------------------ // find the disk sector size - if(!path_to_get_hw_sector_size[0]) { + if(unlikely(!path_to_get_hw_sector_size[0])) { snprintfz(buffer, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/block/%s/queue/hw_sector_size"); snprintfz(path_to_get_hw_sector_size, FILENAME_MAX, "%s", config_get("plugin:proc:/proc/diskstats", "path to get h/w sector size", buffer)); } - if(!path_to_get_hw_sector_size_partitions[0]) { + if(unlikely(!path_to_get_hw_sector_size_partitions[0])) { snprintfz(buffer, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/dev/block/%lu:%lu/subsystem/%s/../queue/hw_sector_size"); snprintfz(path_to_get_hw_sector_size_partitions, FILENAME_MAX, "%s", config_get("plugin:proc:/proc/diskstats", "path to get h/w sector size for partitions", buffer)); } @@ -149,21 +140,21 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis // replace all / with ! for(t = tf; *t ;t++) - if(*t == '/') *t = '!'; + if(unlikely(*t == '/')) *t = '!'; - if(d->type == DISK_TYPE_PARTITION) + if(likely(d->type == DISK_TYPE_PARTITION)) snprintfz(buffer, FILENAME_MAX, path_to_get_hw_sector_size_partitions, d->major, d->minor, tf); else snprintfz(buffer, FILENAME_MAX, path_to_get_hw_sector_size, tf); FILE *fpss = fopen(buffer, "r"); - if(fpss) { + if(likely(fpss)) { char buffer2[1024 + 1]; char *tmp = fgets(buffer2, 1024, fpss); - if(tmp) { - d->sector_size = atoi(tmp); - if(d->sector_size <= 0) { + if(likely(tmp)) { + d->sector_size = str2i(tmp); + if(unlikely(d->sector_size <= 0)) { error("Invalid sector size %d for device %s in %s. Assuming 512.", d->sector_size, d->disk, buffer); d->sector_size = 512; } @@ -178,26 +169,41 @@ static struct disk *get_disk(unsigned long major, unsigned long minor, char *dis return d; } -static inline int select_positive_option(int option1, int option2) { - if(option1 == CONFIG_ONDEMAND_YES || option2 == CONFIG_ONDEMAND_YES) - return CONFIG_ONDEMAND_YES; - else if(option1 == CONFIG_ONDEMAND_ONDEMAND || option2 == CONFIG_ONDEMAND_ONDEMAND) - return CONFIG_ONDEMAND_ONDEMAND; +static inline int is_major_enabled(int major) { + static char *major_configs = NULL; + static size_t major_size = 0; + + if(major < 0) return 1; - return CONFIG_ONDEMAND_NO; + size_t wanted_size = (size_t)major + 1; + + if(major_size < wanted_size) { + major_configs = reallocz(major_configs, wanted_size); + + size_t i; + for(i = major_size; i < wanted_size ; i++) + major_configs[i] = -1; + + major_size = wanted_size; + } + + if(major_configs[major] == -1) { + char buffer[CONFIG_MAX_NAME + 1]; + snprintfz(buffer, CONFIG_MAX_NAME, "performance metrics for disks with major %d", major); + major_configs[major] = (char)config_get_boolean("plugin:proc:/proc/diskstats", buffer, 1); + } + + return major_configs[major]; } -int do_proc_diskstats(int update_every, unsigned long long dt) { +int do_proc_diskstats(int update_every, usec_t dt) { + (void)dt; + static procfile *ff = NULL; - static struct statvfs buff_statvfs; - static struct stat buff_stat; static int global_enable_new_disks_detected_at_runtime = CONFIG_ONDEMAND_YES, global_enable_performance_for_physical_disks = CONFIG_ONDEMAND_ONDEMAND, - global_enable_performance_for_virtual_disks = CONFIG_ONDEMAND_NO, + global_enable_performance_for_virtual_disks = CONFIG_ONDEMAND_ONDEMAND, global_enable_performance_for_partitions = CONFIG_ONDEMAND_NO, - global_enable_performance_for_mountpoints = CONFIG_ONDEMAND_NO, - global_enable_performance_for_virtual_mountpoints = CONFIG_ONDEMAND_ONDEMAND, - global_enable_space_for_mountpoints = CONFIG_ONDEMAND_ONDEMAND, global_do_io = CONFIG_ONDEMAND_ONDEMAND, global_do_ops = CONFIG_ONDEMAND_ONDEMAND, global_do_mops = CONFIG_ONDEMAND_ONDEMAND, @@ -205,8 +211,6 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { global_do_qops = CONFIG_ONDEMAND_ONDEMAND, global_do_util = CONFIG_ONDEMAND_ONDEMAND, global_do_backlog = CONFIG_ONDEMAND_ONDEMAND, - global_do_space = CONFIG_ONDEMAND_ONDEMAND, - global_do_inodes = CONFIG_ONDEMAND_ONDEMAND, globals_initialized = 0; if(unlikely(!globals_initialized)) { @@ -215,9 +219,6 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { global_enable_performance_for_physical_disks = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "performance metrics for physical disks", global_enable_performance_for_physical_disks); global_enable_performance_for_virtual_disks = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "performance metrics for virtual disks", global_enable_performance_for_virtual_disks); global_enable_performance_for_partitions = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "performance metrics for partitions", global_enable_performance_for_partitions); - global_enable_performance_for_mountpoints = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "performance metrics for mounted filesystems", global_enable_performance_for_mountpoints); - global_enable_performance_for_virtual_mountpoints = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "performance metrics for mounted virtual disks", global_enable_performance_for_virtual_mountpoints); - global_enable_space_for_mountpoints = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "space metrics for mounted filesystems", global_enable_space_for_mountpoints); global_do_io = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "bandwidth for all disks", global_do_io); global_do_ops = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "operations for all disks", global_do_ops); @@ -226,79 +227,77 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { global_do_qops = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "queued operations for all disks", global_do_qops); global_do_util = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "utilization percentage for all disks", global_do_util); global_do_backlog = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "backlog for all disks", global_do_backlog); - global_do_space = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "space usage for all disks", global_do_space); - global_do_inodes = config_get_boolean_ondemand("plugin:proc:/proc/diskstats", "inodes usage for all disks", global_do_inodes); globals_initialized = 1; } - if(!ff) { + // -------------------------------------------------------------------------- + + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/diskstats"); ff = procfile_open(config_get("plugin:proc:/proc/diskstats", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); } - if(!ff) return 1; + if(unlikely(!ff)) return 0; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + size_t lines = procfile_lines(ff), l; for(l = 0; l < lines ;l++) { // -------------------------------------------------------------------------- // Read parameters char *disk; - unsigned long long major = 0, minor = 0, - reads = 0, mreads = 0, readsectors = 0, readms = 0, + unsigned long major = 0, minor = 0; + + collected_number reads = 0, mreads = 0, readsectors = 0, readms = 0, writes = 0, mwrites = 0, writesectors = 0, writems = 0, - queued_ios = 0, busy_ms = 0, backlog_ms = 0, - space_avail = 0, space_avail_root = 0, space_used = 0, - inodes_avail = 0, inodes_avail_root = 0, inodes_used = 0; + queued_ios = 0, busy_ms = 0, backlog_ms = 0; - unsigned long long last_reads = 0, last_readsectors = 0, last_readms = 0, + collected_number last_reads = 0, last_readsectors = 0, last_readms = 0, last_writes = 0, last_writesectors = 0, last_writems = 0, last_busy_ms = 0; - words = procfile_linewords(ff, l); - if(words < 14) continue; + size_t words = procfile_linewords(ff, l); + if(unlikely(words < 14)) continue; - major = strtoull(procfile_lineword(ff, l, 0), NULL, 10); - minor = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + major = str2ul(procfile_lineword(ff, l, 0)); + minor = str2ul(procfile_lineword(ff, l, 1)); disk = procfile_lineword(ff, l, 2); // # of reads completed # of writes completed // This is the total number of reads or writes completed successfully. - reads = strtoull(procfile_lineword(ff, l, 3), NULL, 10); // rd_ios - writes = strtoull(procfile_lineword(ff, l, 7), NULL, 10); // wr_ios + reads = str2ull(procfile_lineword(ff, l, 3)); // rd_ios + writes = str2ull(procfile_lineword(ff, l, 7)); // wr_ios // # of reads merged # of writes merged // Reads and writes which are adjacent to each other may be merged for // efficiency. Thus two 4K reads may become one 8K read before it is // ultimately handed to the disk, and so it will be counted (and queued) - mreads = strtoull(procfile_lineword(ff, l, 4), NULL, 10); // rd_merges_or_rd_sec - mwrites = strtoull(procfile_lineword(ff, l, 8), NULL, 10); // wr_merges + mreads = str2ull(procfile_lineword(ff, l, 4)); // rd_merges_or_rd_sec + mwrites = str2ull(procfile_lineword(ff, l, 8)); // wr_merges // # of sectors read # of sectors written // This is the total number of sectors read or written successfully. - readsectors = strtoull(procfile_lineword(ff, l, 5), NULL, 10); // rd_sec_or_wr_ios - writesectors = strtoull(procfile_lineword(ff, l, 9), NULL, 10); // wr_sec + readsectors = str2ull(procfile_lineword(ff, l, 5)); // rd_sec_or_wr_ios + writesectors = str2ull(procfile_lineword(ff, l, 9)); // wr_sec // # of milliseconds spent reading # of milliseconds spent writing // This is the total number of milliseconds spent by all reads or writes (as // measured from __make_request() to end_that_request_last()). - readms = strtoull(procfile_lineword(ff, l, 6), NULL, 10); // rd_ticks_or_wr_sec - writems = strtoull(procfile_lineword(ff, l, 10), NULL, 10); // wr_ticks + readms = str2ull(procfile_lineword(ff, l, 6)); // rd_ticks_or_wr_sec + writems = str2ull(procfile_lineword(ff, l, 10)); // wr_ticks // # of I/Os currently in progress // The only field that should go to zero. Incremented as requests are // given to appropriate struct request_queue and decremented as they finish. - queued_ios = strtoull(procfile_lineword(ff, l, 11), NULL, 10); // ios_pgr + queued_ios = str2ull(procfile_lineword(ff, l, 11)); // ios_pgr // # of milliseconds spent doing I/Os // This field increases so long as field queued_ios is nonzero. - busy_ms = strtoull(procfile_lineword(ff, l, 12), NULL, 10); // tot_ticks + busy_ms = str2ull(procfile_lineword(ff, l, 12)); // tot_ticks // weighted # of milliseconds spent doing I/Os // This field is incremented at each I/O start, I/O completion, I/O @@ -306,7 +305,7 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { // (field queued_ios) times the number of milliseconds spent doing I/O since the // last update of this field. This can provide an easy measure of both // I/O completion time and the backlog that may be accumulating. - backlog_ms = strtoull(procfile_lineword(ff, l, 13), NULL, 10); // rq_ticks + backlog_ms = str2ull(procfile_lineword(ff, l, 13)); // rq_ticks // -------------------------------------------------------------------------- @@ -336,7 +335,7 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { snprintfz(var_name, 4096, "plugin:proc:/proc/diskstats:%s", disk); int def_enable = config_get_boolean_ondemand(var_name, "enable", global_enable_new_disks_detected_at_runtime); - if(def_enable == CONFIG_ONDEMAND_NO) { + if(unlikely(def_enable == CONFIG_ONDEMAND_NO)) { // the user does not want any metrics for this disk d->do_io = CONFIG_ONDEMAND_NO; d->do_ops = CONFIG_ONDEMAND_NO; @@ -345,15 +344,12 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_qops = CONFIG_ONDEMAND_NO; d->do_util = CONFIG_ONDEMAND_NO; d->do_backlog = CONFIG_ONDEMAND_NO; - d->do_space = CONFIG_ONDEMAND_NO; - d->do_inodes = CONFIG_ONDEMAND_NO; } else { // this disk is enabled // check its direct settings int def_performance = CONFIG_ONDEMAND_ONDEMAND; - int def_space = (d->mount_point)?CONFIG_ONDEMAND_ONDEMAND:CONFIG_ONDEMAND_NO; // since this is 'on demand' we can figure the performance settings // based on the type of disk @@ -369,15 +365,12 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { case DISK_TYPE_CONTAINER: def_performance = global_enable_performance_for_virtual_disks; - - if(d->mount_point) - def_performance = select_positive_option(def_performance, global_enable_performance_for_virtual_mountpoints); - break; } - if(d->mount_point) - def_performance = select_positive_option(def_performance, global_enable_performance_for_mountpoints); + // check if we have to disable performance for this disk + if(def_performance) + def_performance = is_major_enabled((int)major); // ------------------------------------------------------------ // now we have def_performance and def_space @@ -396,7 +389,7 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { ddo_backlog = CONFIG_ONDEMAND_NO; // we enable individual performance charts only when def_performance is not disabled - if(def_performance != CONFIG_ONDEMAND_NO) { + if(unlikely(def_performance != CONFIG_ONDEMAND_NO)) { ddo_io = global_do_io, ddo_ops = global_do_ops, ddo_mops = global_do_mops, @@ -413,23 +406,6 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_qops = config_get_boolean_ondemand(var_name, "queued operations", ddo_qops); d->do_util = config_get_boolean_ondemand(var_name, "utilization percentage", ddo_util); d->do_backlog = config_get_boolean_ondemand(var_name, "backlog", ddo_backlog); - - // def_space - if(d->mount_point) { - // check the user configuration (this will also show our 'on demand' decision) - def_space = config_get_boolean_ondemand(var_name, "enable space metrics", def_space); - - int ddo_space = def_space, - ddo_inodes = def_space; - - d->do_space = config_get_boolean_ondemand(var_name, "space usage", ddo_space); - d->do_inodes = config_get_boolean_ondemand(var_name, "inodes usage", ddo_inodes); - } - else { - // don't show settings for this disk - d->do_space = CONFIG_ONDEMAND_NO; - d->do_inodes = CONFIG_ONDEMAND_NO; - } } d->configured = 1; @@ -444,13 +420,13 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_io = CONFIG_ONDEMAND_YES; st = rrdset_find_bytype(RRD_TYPE_DISK, disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_DISK, disk, NULL, family, "disk.io", "Disk I/O Bandwidth", "kilobytes/s", 2000, update_every, RRDSET_TYPE_AREA); rrddim_add(st, "reads", NULL, d->sector_size, 1024, RRDDIM_INCREMENTAL); rrddim_add(st, "writes", NULL, d->sector_size * -1, 1024, RRDDIM_INCREMENTAL); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); last_readsectors = rrddim_set(st, "reads", readsectors); last_writesectors = rrddim_set(st, "writes", writesectors); @@ -463,14 +439,14 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_ops = CONFIG_ONDEMAND_YES; st = rrdset_find_bytype("disk_ops", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_ops", disk, NULL, family, "disk.ops", "Disk Completed I/O Operations", "operations/s", 2001, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); last_reads = rrddim_set(st, "reads", reads); last_writes = rrddim_set(st, "writes", writes); @@ -483,13 +459,13 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_qops = CONFIG_ONDEMAND_YES; st = rrdset_find_bytype("disk_qops", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_qops", disk, NULL, family, "disk.qops", "Disk Current I/O Operations", "operations", 2002, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; rrddim_add(st, "operations", NULL, 1, 1, RRDDIM_ABSOLUTE); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); rrddim_set(st, "operations", queued_ios); rrdset_done(st); @@ -501,13 +477,13 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_backlog = CONFIG_ONDEMAND_YES; st = rrdset_find_bytype("disk_backlog", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_backlog", disk, NULL, family, "disk.backlog", "Disk Backlog", "backlog (ms)", 2003, update_every, RRDSET_TYPE_AREA); st->isdetail = 1; rrddim_add(st, "backlog", NULL, 1, 10, RRDDIM_INCREMENTAL); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); rrddim_set(st, "backlog", backlog_ms); rrdset_done(st); @@ -519,13 +495,13 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_util = CONFIG_ONDEMAND_YES; st = rrdset_find_bytype("disk_util", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_util", disk, NULL, family, "disk.util", "Disk Utilization Time", "% of time working", 2004, update_every, RRDSET_TYPE_AREA); st->isdetail = 1; rrddim_add(st, "utilization", NULL, 1, 10, RRDDIM_INCREMENTAL); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); last_busy_ms = rrddim_set(st, "utilization", busy_ms); rrdset_done(st); @@ -537,14 +513,14 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_mops = CONFIG_ONDEMAND_YES; st = rrdset_find_bytype("disk_mops", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_mops", disk, NULL, family, "disk.mops", "Disk Merged Operations", "merged operations/s", 2021, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); rrddim_set(st, "reads", mreads); rrddim_set(st, "writes", mwrites); @@ -557,14 +533,14 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { d->do_iotime = CONFIG_ONDEMAND_YES; st = rrdset_find_bytype("disk_iotime", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_iotime", disk, NULL, family, "disk.iotime", "Disk Total I/O Time", "milliseconds/s", 2022, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_INCREMENTAL); rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); last_readms = rrddim_set(st, "reads", readms); last_writems = rrddim_set(st, "writes", writems); @@ -575,18 +551,18 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { // calculate differential charts // only if this is not the first time we run - if(dt) { + if(likely(dt)) { if( (d->do_iotime == CONFIG_ONDEMAND_YES || (d->do_iotime == CONFIG_ONDEMAND_ONDEMAND && (readms || writems))) && (d->do_ops == CONFIG_ONDEMAND_YES || (d->do_ops == CONFIG_ONDEMAND_ONDEMAND && (reads || writes)))) { st = rrdset_find_bytype("disk_await", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_await", disk, NULL, family, "disk.await", "Average Completed I/O Operation Time", "ms per operation", 2005, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; rrddim_add(st, "reads", NULL, 1, 1, RRDDIM_ABSOLUTE); rrddim_add(st, "writes", NULL, -1, 1, RRDDIM_ABSOLUTE); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); rrddim_set(st, "reads", (reads - last_reads) ? (readms - last_readms) / (reads - last_reads) : 0); rrddim_set(st, "writes", (writes - last_writes) ? (writems - last_writems) / (writes - last_writes) : 0); @@ -596,14 +572,14 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { if( (d->do_io == CONFIG_ONDEMAND_YES || (d->do_io == CONFIG_ONDEMAND_ONDEMAND && (readsectors || writesectors))) && (d->do_ops == CONFIG_ONDEMAND_YES || (d->do_ops == CONFIG_ONDEMAND_ONDEMAND && (reads || writes)))) { st = rrdset_find_bytype("disk_avgsz", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_avgsz", disk, NULL, family, "disk.avgsz", "Average Completed I/O Operation Bandwidth", "kilobytes per operation", 2006, update_every, RRDSET_TYPE_AREA); st->isdetail = 1; rrddim_add(st, "reads", NULL, d->sector_size, 1024, RRDDIM_ABSOLUTE); rrddim_add(st, "writes", NULL, d->sector_size * -1, 1024, RRDDIM_ABSOLUTE); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); rrddim_set(st, "reads", (reads - last_reads) ? (readsectors - last_readsectors) / (reads - last_reads) : 0); rrddim_set(st, "writes", (writes - last_writes) ? (writesectors - last_writesectors) / (writes - last_writes) : 0); @@ -613,87 +589,18 @@ int do_proc_diskstats(int update_every, unsigned long long dt) { if( (d->do_util == CONFIG_ONDEMAND_YES || (d->do_util == CONFIG_ONDEMAND_ONDEMAND && busy_ms)) && (d->do_ops == CONFIG_ONDEMAND_YES || (d->do_ops == CONFIG_ONDEMAND_ONDEMAND && (reads || writes)))) { st = rrdset_find_bytype("disk_svctm", disk); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("disk_svctm", disk, NULL, family, "disk.svctm", "Average Service Time", "ms per operation", 2007, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; rrddim_add(st, "svctm", NULL, 1, 1, RRDDIM_ABSOLUTE); } - else rrdset_next_usec(st, dt); + else rrdset_next(st); rrddim_set(st, "svctm", ((reads - last_reads) + (writes - last_writes)) ? (busy_ms - last_busy_ms) / ((reads - last_reads) + (writes - last_writes)) : 0); rrdset_done(st); } } - - // -------------------------------------------------------------------------- - // space metrics - - if(d->mount_point && (d->do_space || d->do_inodes) ) { - // collect space metrics using statvfs - - if (statvfs(d->mount_point, &buff_statvfs) < 0) - error("Failed statvfs() for '%s' (disk '%s')", d->mount_point, d->disk); - else { - space_avail = buff_statvfs.f_bavail * buff_statvfs.f_bsize; - space_avail_root = (buff_statvfs.f_bfree - buff_statvfs.f_bavail) * buff_statvfs.f_bsize; - space_used = (buff_statvfs.f_blocks - buff_statvfs.f_bfree) * buff_statvfs.f_bsize; - - inodes_avail = buff_statvfs.f_favail; - inodes_avail_root = buff_statvfs.f_ffree - buff_statvfs.f_favail; - inodes_used = buff_statvfs.f_files - buff_statvfs.f_ffree; - - // verify we collected the metrics for the right disk. - // if not the mountpoint has changed. - - if(stat(d->mount_point, &buff_stat) == -1) - error("Failed to stat() for '%s' (disk '%s')", d->mount_point, d->disk); - else { - if(major(buff_stat.st_dev) == major && minor(buff_stat.st_dev) == minor) { - - // -------------------------------------------------------------------------- - - if(d->do_space == CONFIG_ONDEMAND_YES || (d->do_space == CONFIG_ONDEMAND_ONDEMAND && (space_avail || space_avail_root || space_used))) { - st = rrdset_find_bytype("disk_space", disk); - if(!st) { - st = rrdset_create("disk_space", disk, NULL, family, "disk.space", "Disk Space Usage", "GB", 2023, update_every, RRDSET_TYPE_STACKED); - st->isdetail = 1; - - rrddim_add(st, "avail", NULL, 1, 1024*1024*1024, RRDDIM_ABSOLUTE); - rrddim_add(st, "used" , NULL, 1, 1024*1024*1024, RRDDIM_ABSOLUTE); - rrddim_add(st, "reserved_for_root", "reserved for root", 1, 1024*1024*1024, RRDDIM_ABSOLUTE); - } - else rrdset_next_usec(st, dt); - - rrddim_set(st, "avail", space_avail); - rrddim_set(st, "used", space_used); - rrddim_set(st, "reserved_for_root", space_avail_root); - rrdset_done(st); - } - - // -------------------------------------------------------------------------- - - if(d->do_inodes == CONFIG_ONDEMAND_YES || (d->do_inodes == CONFIG_ONDEMAND_ONDEMAND && (inodes_avail || inodes_avail_root || inodes_used))) { - st = rrdset_find_bytype("disk_inodes", disk); - if(!st) { - st = rrdset_create("disk_inodes", disk, NULL, family, "disk.inodes", "Disk Inodes Usage", "Inodes", 2024, update_every, RRDSET_TYPE_STACKED); - st->isdetail = 1; - - rrddim_add(st, "avail", NULL, 1, 1, RRDDIM_ABSOLUTE); - rrddim_add(st, "used" , NULL, 1, 1, RRDDIM_ABSOLUTE); - rrddim_add(st, "reserved_for_root", "reserved for root", 1, 1, RRDDIM_ABSOLUTE); - } - else rrdset_next_usec(st, dt); - - rrddim_set(st, "avail", inodes_avail); - rrddim_set(st, "used", inodes_used); - rrddim_set(st, "reserved_for_root", inodes_avail_root); - rrdset_done(st); - } - } - } - } - } } return 0; diff --git a/src/proc_interrupts.c b/src/proc_interrupts.c index f277a5a90..f663c0fdd 100644 --- a/src/proc_interrupts.c +++ b/src/proc_interrupts.c @@ -2,70 +2,92 @@ #define MAX_INTERRUPT_NAME 50 +struct cpu_interrupt { + unsigned long long value; + RRDDIM *rd; +}; + struct interrupt { int used; char *id; char name[MAX_INTERRUPT_NAME + 1]; + RRDDIM *rd; unsigned long long total; - unsigned long long value[]; + struct cpu_interrupt cpu[]; }; // since each interrupt is variable in size // we use this to calculate its record size -#define recordsize(cpus) (sizeof(struct interrupt) + (cpus * sizeof(unsigned long long))) +#define recordsize(cpus) (sizeof(struct interrupt) + (cpus * sizeof(struct cpu_interrupt))) // given a base, get a pointer to each record #define irrindex(base, line, cpus) ((struct interrupt *)&((char *)(base))[line * recordsize(cpus)]) -static inline struct interrupt *get_interrupts_array(int lines, int cpus) { +static inline struct interrupt *get_interrupts_array(size_t lines, int cpus) { static struct interrupt *irrs = NULL; - static int allocated = 0; + static size_t allocated = 0; + + if(unlikely(lines != allocated)) { + size_t l; + int c; - if(lines > allocated) { irrs = (struct interrupt *)reallocz(irrs, lines * recordsize(cpus)); + + // reset all interrupt RRDDIM pointers as any line could have shifted + for(l = 0; l < lines ;l++) { + struct interrupt *irr = irrindex(irrs, l, cpus); + irr->rd = NULL; + irr->name[0] = '\0'; + for(c = 0; c < cpus ;c++) + irr->cpu[c].rd = NULL; + } + allocated = lines; } return irrs; } -int do_proc_interrupts(int update_every, unsigned long long dt) { +int do_proc_interrupts(int update_every, usec_t dt) { + (void)dt; static procfile *ff = NULL; static int cpus = -1, do_per_core = -1; struct interrupt *irrs = NULL; - if(dt) {}; - - if(do_per_core == -1) do_per_core = config_get_boolean("plugin:proc:/proc/interrupts", "interrupts per core", 1); + if(unlikely(do_per_core == -1)) + do_per_core = config_get_boolean("plugin:proc:/proc/interrupts", "interrupts per core", 1); - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/interrupts"); ff = procfile_open(config_get("plugin:proc:/proc/interrupts", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); } - if(!ff) return 1; + if(unlikely(!ff)) + return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time - uint32_t lines = procfile_lines(ff), l; - uint32_t words = procfile_linewords(ff, 0), w; + size_t lines = procfile_lines(ff), l; + size_t words = procfile_linewords(ff, 0); - if(!lines) { + if(unlikely(!lines)) { error("Cannot read /proc/interrupts, zero lines reported."); return 1; } // find how many CPUs are there - if(cpus == -1) { + if(unlikely(cpus == -1)) { + uint32_t w; cpus = 0; for(w = 0; w < words ; w++) { - if(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0) + if(likely(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0)) cpus++; } } - if(!cpus) { + if(unlikely(!cpus)) { error("PLUGIN: PROC_INTERRUPTS: Cannot find the number of CPUs in /proc/interrupts"); return 1; } @@ -81,31 +103,36 @@ int do_proc_interrupts(int update_every, unsigned long long dt) { irr->total = 0; words = procfile_linewords(ff, l); - if(!words) continue; + if(unlikely(!words)) continue; irr->id = procfile_lineword(ff, l, 0); - if(!irr->id || !irr->id[0]) continue; + if(unlikely(!irr->id || !irr->id[0])) continue; - int idlen = strlen(irr->id); - if(irr->id[idlen - 1] == ':') + size_t idlen = strlen(irr->id); + if(unlikely(idlen && irr->id[idlen - 1] == ':')) irr->id[idlen - 1] = '\0'; int c; for(c = 0; c < cpus ;c++) { - if((c + 1) < (int)words) - irr->value[c] = strtoull(procfile_lineword(ff, l, (uint32_t)(c + 1)), NULL, 10); + if(likely((c + 1) < (int)words)) + irr->cpu[c].value = str2ull(procfile_lineword(ff, l, (uint32_t)(c + 1))); else - irr->value[c] = 0; + irr->cpu[c].value = 0; - irr->total += irr->value[c]; + irr->total += irr->cpu[c].value; } - if(isdigit(irr->id[0]) && (uint32_t)(cpus + 2) < words) { + if(unlikely(isdigit(irr->id[0]) && (uint32_t)(cpus + 2) < words)) { strncpyz(irr->name, procfile_lineword(ff, l, words - 1), MAX_INTERRUPT_NAME); - int nlen = strlen(irr->name); - if(nlen < (MAX_INTERRUPT_NAME-1)) { + size_t nlen = strlen(irr->name); + idlen = strlen(irr->id); + if(likely(nlen + 1 + idlen <= MAX_INTERRUPT_NAME)) { irr->name[nlen] = '_'; - strncpyz(&irr->name[nlen + 1], irr->id, MAX_INTERRUPT_NAME - nlen); + strncpyz(&irr->name[nlen + 1], irr->id, MAX_INTERRUPT_NAME - nlen - 1); + } + else { + irr->name[MAX_INTERRUPT_NAME - idlen - 1] = '_'; + strncpyz(&irr->name[MAX_INTERRUPT_NAME - idlen], irr->id, idlen); } } else { @@ -120,49 +147,59 @@ int do_proc_interrupts(int update_every, unsigned long long dt) { // -------------------------------------------------------------------- st = rrdset_find_bytype("system", "interrupts"); - if(!st) { - st = rrdset_create("system", "interrupts", NULL, "interrupts", NULL, "System interrupts", "interrupts/s", 1000, update_every, RRDSET_TYPE_STACKED); - - for(l = 0; l < lines ;l++) { - struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); - } - } + if(unlikely(!st)) st = rrdset_create("system", "interrupts", NULL, "interrupts", NULL, "System interrupts", "interrupts/s", 1000, update_every, RRDSET_TYPE_STACKED); else rrdset_next(st); for(l = 0; l < lines ;l++) { struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_set(st, irr->id, irr->total); + if(unlikely(!irr->used)) continue; + // some interrupt may have changed without changing the total number of lines + // if the same number of interrupts have been added and removed between two + // calls of this function. + if(unlikely(!irr->rd || strncmp(irr->rd->name, irr->name, MAX_INTERRUPT_NAME) != 0)) { + irr->rd = rrddim_find(st, irr->id); + if(unlikely(!irr->rd)) + irr->rd = rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); + else + rrddim_set_name(st, irr->rd, irr->name); + + // also reset per cpu RRDDIMs to avoid repeating strncmp() in the per core loop + if(likely(do_per_core)) { + int c; + for (c = 0; c < cpus ;c++) + irr->cpu[c].rd = NULL; + } + } + rrddim_set_by_pointer(st, irr->rd, irr->total); } rrdset_done(st); - if(do_per_core) { + if(likely(do_per_core)) { int c; - for(c = 0; c < cpus ; c++) { + for(c = 0; c < cpus ;c++) { char id[50+1]; snprintfz(id, 50, "cpu%d_interrupts", c); st = rrdset_find_bytype("cpu", id); - if(!st) { + if(unlikely(!st)) { char title[100+1]; snprintfz(title, 100, "CPU%d Interrupts", c); st = rrdset_create("cpu", id, NULL, "interrupts", "cpu.interrupts", title, "interrupts/s", 1100 + c, update_every, RRDSET_TYPE_STACKED); - - for(l = 0; l < lines ;l++) { - struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); - } } else rrdset_next(st); for(l = 0; l < lines ;l++) { struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_set(st, irr->id, irr->value[c]); + if(unlikely(!irr->used)) continue; + if(unlikely(!irr->cpu[c].rd)) { + irr->cpu[c].rd = rrddim_find(st, irr->id); + if(unlikely(!irr->cpu[c].rd)) + irr->cpu[c].rd = rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); + else + rrddim_set_name(st, irr->cpu[c].rd, irr->name); + } + rrddim_set_by_pointer(st, irr->cpu[c].rd, irr->cpu[c].value); } rrdset_done(st); } diff --git a/src/proc_loadavg.c b/src/proc_loadavg.c index 44ea70191..4326ffb7d 100644 --- a/src/proc_loadavg.c +++ b/src/proc_loadavg.c @@ -3,30 +3,35 @@ // linux calculates this once every 5 seconds #define MIN_LOADAVG_UPDATE_EVERY 5 -int do_proc_loadavg(int update_every, unsigned long long dt) { +int do_proc_loadavg(int update_every, usec_t dt) { static procfile *ff = NULL; static int do_loadavg = -1, do_all_processes = -1; + static usec_t last_loadavg_usec = 0; + static RRDSET *load_chart = NULL, *processes_chart = NULL; - if(dt) {}; - - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/loadavg"); + ff = procfile_open(config_get("plugin:proc:/proc/loadavg", "filename to monitor", filename), " \t,:|/", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) + return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time - if(do_loadavg == -1) do_loadavg = config_get_boolean("plugin:proc:/proc/loadavg", "enable load average", 1); - if(do_all_processes == -1) do_all_processes = config_get_boolean("plugin:proc:/proc/loadavg", "enable total processes", 1); + if(unlikely(do_loadavg == -1)) { + do_loadavg = config_get_boolean("plugin:proc:/proc/loadavg", "enable load average", 1); + do_all_processes = config_get_boolean("plugin:proc:/proc/loadavg", "enable total processes", 1); + } - if(procfile_lines(ff) < 1) { + if(unlikely(procfile_lines(ff) < 1)) { error("/proc/loadavg has no lines."); return 1; } - if(procfile_linewords(ff, 0) < 6) { + if(unlikely(procfile_linewords(ff, 0) < 6)) { error("/proc/loadavg has less than 6 words in it."); return 1; } @@ -35,45 +40,51 @@ int do_proc_loadavg(int update_every, unsigned long long dt) { double load5 = strtod(procfile_lineword(ff, 0, 1), NULL); double load15 = strtod(procfile_lineword(ff, 0, 2), NULL); - //unsigned long long running_processes = strtoull(procfile_lineword(ff, 0, 3), NULL, 10); - unsigned long long active_processes = strtoull(procfile_lineword(ff, 0, 4), NULL, 10); - //unsigned long long next_pid = strtoull(procfile_lineword(ff, 0, 5), NULL, 10); + //unsigned long long running_processes = str2ull(procfile_lineword(ff, 0, 3)); + unsigned long long active_processes = str2ull(procfile_lineword(ff, 0, 4)); + //unsigned long long next_pid = str2ull(procfile_lineword(ff, 0, 5)); - RRDSET *st; - // -------------------------------------------------------------------- - if(do_loadavg) { - st = rrdset_find_byname("system.load"); - if(!st) { - st = rrdset_create("system", "load", NULL, "load", NULL, "System Load Average", "load", 100, (update_every < MIN_LOADAVG_UPDATE_EVERY)?MIN_LOADAVG_UPDATE_EVERY:update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "load1", NULL, 1, 1000, RRDDIM_ABSOLUTE); - rrddim_add(st, "load5", NULL, 1, 1000, RRDDIM_ABSOLUTE); - rrddim_add(st, "load15", NULL, 1, 1000, RRDDIM_ABSOLUTE); + if(last_loadavg_usec <= dt) { + if(likely(do_loadavg)) { + if(unlikely(!load_chart)) { + load_chart = rrdset_find_byname("system.load"); + if(unlikely(!load_chart)) { + load_chart = rrdset_create("system", "load", NULL, "load", NULL, "System Load Average", "load", 100, (update_every < MIN_LOADAVG_UPDATE_EVERY) ? MIN_LOADAVG_UPDATE_EVERY : update_every, RRDSET_TYPE_LINE); + rrddim_add(load_chart, "load1", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(load_chart, "load5", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(load_chart, "load15", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + } + else + rrdset_next(load_chart); + + rrddim_set(load_chart, "load1", (collected_number) (load1 * 1000)); + rrddim_set(load_chart, "load5", (collected_number) (load5 * 1000)); + rrddim_set(load_chart, "load15", (collected_number) (load15 * 1000)); + rrdset_done(load_chart); } - else rrdset_next(st); - rrddim_set(st, "load1", load1 * 1000); - rrddim_set(st, "load5", load5 * 1000); - rrddim_set(st, "load15", load15 * 1000); - rrdset_done(st); + last_loadavg_usec = load_chart->update_every * USEC_PER_SEC; } + else last_loadavg_usec -= dt; // -------------------------------------------------------------------- - if(do_all_processes) { - st = rrdset_find_byname("system.active_processes"); - if(!st) { - st = rrdset_create("system", "active_processes", NULL, "processes", NULL, "System Active Processes", "processes", 750, update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "active", NULL, 1, 1, RRDDIM_ABSOLUTE); + if(likely(do_all_processes)) { + if(unlikely(!processes_chart)) { + processes_chart = rrdset_find_byname("system.active_processes"); + if(unlikely(!processes_chart)) { + processes_chart = rrdset_create("system", "active_processes", NULL, "processes", NULL, "System Active Processes", "processes", 750, update_every, RRDSET_TYPE_LINE); + rrddim_add(processes_chart, "active", NULL, 1, 1, RRDDIM_ABSOLUTE); + } } - else rrdset_next(st); + else rrdset_next(processes_chart); - rrddim_set(st, "active", active_processes); - rrdset_done(st); + rrddim_set(processes_chart, "active", active_processes); + rrdset_done(processes_chart); } return 0; diff --git a/src/proc_meminfo.c b/src/proc_meminfo.c index 999c9538d..19ba8da3c 100644 --- a/src/proc_meminfo.c +++ b/src/proc_meminfo.c @@ -1,96 +1,135 @@ #include "common.h" -#define MAX_PROC_MEMINFO_LINE 4096 -#define MAX_PROC_MEMINFO_NAME 1024 +int do_proc_meminfo(int update_every, usec_t dt) { + (void)dt; -int do_proc_meminfo(int update_every, unsigned long long dt) { static procfile *ff = NULL; - static int do_ram = -1, do_swap = -1, do_hwcorrupt = -1, do_committed = -1, do_writeback = -1, do_kernel = -1, do_slab = -1; - if(do_ram == -1) do_ram = config_get_boolean("plugin:proc:/proc/meminfo", "system ram", 1); - if(do_swap == -1) do_swap = config_get_boolean_ondemand("plugin:proc:/proc/meminfo", "system swap", CONFIG_ONDEMAND_ONDEMAND); - if(do_hwcorrupt == -1) do_hwcorrupt = config_get_boolean_ondemand("plugin:proc:/proc/meminfo", "hardware corrupted ECC", CONFIG_ONDEMAND_ONDEMAND); - if(do_committed == -1) do_committed = config_get_boolean("plugin:proc:/proc/meminfo", "committed memory", 1); - if(do_writeback == -1) do_writeback = config_get_boolean("plugin:proc:/proc/meminfo", "writeback memory", 1); - if(do_kernel == -1) do_kernel = config_get_boolean("plugin:proc:/proc/meminfo", "kernel memory", 1); - if(do_slab == -1) do_slab = config_get_boolean("plugin:proc:/proc/meminfo", "slab memory", 1); - - (void)dt; + static ARL_BASE *arl_base = NULL; + static ARL_ENTRY *arl_hwcorrupted = NULL; + + static unsigned long long + MemTotal = 0, + MemFree = 0, + Buffers = 0, + Cached = 0, + //SwapCached = 0, + //Active = 0, + //Inactive = 0, + //ActiveAnon = 0, + //InactiveAnon = 0, + //ActiveFile = 0, + //InactiveFile = 0, + //Unevictable = 0, + //Mlocked = 0, + SwapTotal = 0, + SwapFree = 0, + Dirty = 0, + Writeback = 0, + //AnonPages = 0, + //Mapped = 0, + //Shmem = 0, + Slab = 0, + SReclaimable = 0, + SUnreclaim = 0, + KernelStack = 0, + PageTables = 0, + NFS_Unstable = 0, + Bounce = 0, + WritebackTmp = 0, + //CommitLimit = 0, + Committed_AS = 0, + //VmallocTotal = 0, + VmallocUsed = 0, + //VmallocChunk = 0, + //AnonHugePages = 0, + //HugePages_Total = 0, + //HugePages_Free = 0, + //HugePages_Rsvd = 0, + //HugePages_Surp = 0, + //Hugepagesize = 0, + //DirectMap4k = 0, + //DirectMap2M = 0, + HardwareCorrupted = 0; + + if(unlikely(!arl_base)) { + do_ram = config_get_boolean("plugin:proc:/proc/meminfo", "system ram", 1); + do_swap = config_get_boolean_ondemand("plugin:proc:/proc/meminfo", "system swap", CONFIG_ONDEMAND_ONDEMAND); + do_hwcorrupt = config_get_boolean_ondemand("plugin:proc:/proc/meminfo", "hardware corrupted ECC", CONFIG_ONDEMAND_ONDEMAND); + do_committed = config_get_boolean("plugin:proc:/proc/meminfo", "committed memory", 1); + do_writeback = config_get_boolean("plugin:proc:/proc/meminfo", "writeback memory", 1); + do_kernel = config_get_boolean("plugin:proc:/proc/meminfo", "kernel memory", 1); + do_slab = config_get_boolean("plugin:proc:/proc/meminfo", "slab memory", 1); + + arl_base = arl_create("meminfo", NULL, 60); + arl_expect(arl_base, "MemTotal", &MemTotal); + arl_expect(arl_base, "MemFree", &MemFree); + arl_expect(arl_base, "Buffers", &Buffers); + arl_expect(arl_base, "Cached", &Cached); + //arl_expect(arl_base, "SwapCached", &SwapCached); + //arl_expect(arl_base, "Active", &Active); + //arl_expect(arl_base, "Inactive", &Inactive); + //arl_expect(arl_base, "ActiveAnon", &ActiveAnon); + //arl_expect(arl_base, "InactiveAnon", &InactiveAnon); + //arl_expect(arl_base, "ActiveFile", &ActiveFile); + //arl_expect(arl_base, "InactiveFile", &InactiveFile); + //arl_expect(arl_base, "Unevictable", &Unevictable); + //arl_expect(arl_base, "Mlocked", &Mlocked); + arl_expect(arl_base, "SwapTotal", &SwapTotal); + arl_expect(arl_base, "SwapFree", &SwapFree); + arl_expect(arl_base, "Dirty", &Dirty); + arl_expect(arl_base, "Writeback", &Writeback); + //arl_expect(arl_base, "AnonPages", &AnonPages); + //arl_expect(arl_base, "Mapped", &Mapped); + //arl_expect(arl_base, "Shmem", &Shmem); + arl_expect(arl_base, "Slab", &Slab); + arl_expect(arl_base, "SReclaimable", &SReclaimable); + arl_expect(arl_base, "SUnreclaim", &SUnreclaim); + arl_expect(arl_base, "KernelStack", &KernelStack); + arl_expect(arl_base, "PageTables", &PageTables); + arl_expect(arl_base, "NFS_Unstable", &NFS_Unstable); + arl_expect(arl_base, "Bounce", &Bounce); + arl_expect(arl_base, "WritebackTmp", &WritebackTmp); + //arl_expect(arl_base, "CommitLimit", &CommitLimit); + arl_expect(arl_base, "Committed_AS", &Committed_AS); + //arl_expect(arl_base, "VmallocTotal", &VmallocTotal); + arl_expect(arl_base, "VmallocUsed", &VmallocUsed); + //arl_expect(arl_base, "VmallocChunk", &VmallocChunk); + arl_hwcorrupted = arl_expect(arl_base, "HardwareCorrupted", &HardwareCorrupted); + //arl_expect(arl_base, "AnonHugePages", &AnonHugePages); + //arl_expect(arl_base, "HugePages_Total", &HugePages_Total); + //arl_expect(arl_base, "HugePages_Free", &HugePages_Free); + //arl_expect(arl_base, "HugePages_Rsvd", &HugePages_Rsvd); + //arl_expect(arl_base, "HugePages_Surp", &HugePages_Surp); + //arl_expect(arl_base, "Hugepagesize", &Hugepagesize); + //arl_expect(arl_base, "DirectMap4k", &DirectMap4k); + //arl_expect(arl_base, "DirectMap2M", &DirectMap2M); + } - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/meminfo"); ff = procfile_open(config_get("plugin:proc:/proc/meminfo", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) + return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time - - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time - int hwcorrupted = 0; + size_t lines = procfile_lines(ff), l; - unsigned long long MemTotal = 0, MemFree = 0, Buffers = 0, Cached = 0, SwapCached = 0, - Active = 0, Inactive = 0, ActiveAnon = 0, InactiveAnon = 0, ActiveFile = 0, InactiveFile = 0, - Unevictable = 0, Mlocked = 0, SwapTotal = 0, SwapFree = 0, Dirty = 0, Writeback = 0, AnonPages = 0, - Mapped = 0, Shmem = 0, Slab = 0, SReclaimable = 0, SUnreclaim = 0, KernelStack = 0, PageTables = 0, - NFS_Unstable = 0, Bounce = 0, WritebackTmp = 0, CommitLimit = 0, Committed_AS = 0, - VmallocTotal = 0, VmallocUsed = 0, VmallocChunk = 0, - AnonHugePages = 0, HugePages_Total = 0, HugePages_Free = 0, HugePages_Rsvd = 0, HugePages_Surp = 0, Hugepagesize = 0, - DirectMap4k = 0, DirectMap2M = 0, HardwareCorrupted = 0; + arl_begin(arl_base); for(l = 0; l < lines ;l++) { - words = procfile_linewords(ff, l); - if(words < 2) continue; - - char *name = procfile_lineword(ff, l, 0); - unsigned long long value = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - - if(!MemTotal && strcmp(name, "MemTotal") == 0) MemTotal = value; - else if(!MemFree && strcmp(name, "MemFree") == 0) MemFree = value; - else if(!Buffers && strcmp(name, "Buffers") == 0) Buffers = value; - else if(!Cached && strcmp(name, "Cached") == 0) Cached = value; - else if(!SwapCached && strcmp(name, "SwapCached") == 0) SwapCached = value; - else if(!Active && strcmp(name, "Active") == 0) Active = value; - else if(!Inactive && strcmp(name, "Inactive") == 0) Inactive = value; - else if(!ActiveAnon && strcmp(name, "ActiveAnon") == 0) ActiveAnon = value; - else if(!InactiveAnon && strcmp(name, "InactiveAnon") == 0) InactiveAnon = value; - else if(!ActiveFile && strcmp(name, "ActiveFile") == 0) ActiveFile = value; - else if(!InactiveFile && strcmp(name, "InactiveFile") == 0) InactiveFile = value; - else if(!Unevictable && strcmp(name, "Unevictable") == 0) Unevictable = value; - else if(!Mlocked && strcmp(name, "Mlocked") == 0) Mlocked = value; - else if(!SwapTotal && strcmp(name, "SwapTotal") == 0) SwapTotal = value; - else if(!SwapFree && strcmp(name, "SwapFree") == 0) SwapFree = value; - else if(!Dirty && strcmp(name, "Dirty") == 0) Dirty = value; - else if(!Writeback && strcmp(name, "Writeback") == 0) Writeback = value; - else if(!AnonPages && strcmp(name, "AnonPages") == 0) AnonPages = value; - else if(!Mapped && strcmp(name, "Mapped") == 0) Mapped = value; - else if(!Shmem && strcmp(name, "Shmem") == 0) Shmem = value; - else if(!Slab && strcmp(name, "Slab") == 0) Slab = value; - else if(!SReclaimable && strcmp(name, "SReclaimable") == 0) SReclaimable = value; - else if(!SUnreclaim && strcmp(name, "SUnreclaim") == 0) SUnreclaim = value; - else if(!KernelStack && strcmp(name, "KernelStack") == 0) KernelStack = value; - else if(!PageTables && strcmp(name, "PageTables") == 0) PageTables = value; - else if(!NFS_Unstable && strcmp(name, "NFS_Unstable") == 0) NFS_Unstable = value; - else if(!Bounce && strcmp(name, "Bounce") == 0) Bounce = value; - else if(!WritebackTmp && strcmp(name, "WritebackTmp") == 0) WritebackTmp = value; - else if(!CommitLimit && strcmp(name, "CommitLimit") == 0) CommitLimit = value; - else if(!Committed_AS && strcmp(name, "Committed_AS") == 0) Committed_AS = value; - else if(!VmallocTotal && strcmp(name, "VmallocTotal") == 0) VmallocTotal = value; - else if(!VmallocUsed && strcmp(name, "VmallocUsed") == 0) VmallocUsed = value; - else if(!VmallocChunk && strcmp(name, "VmallocChunk") == 0) VmallocChunk = value; - else if(!HardwareCorrupted && strcmp(name, "HardwareCorrupted") == 0) { HardwareCorrupted = value; hwcorrupted = 1; } - else if(!AnonHugePages && strcmp(name, "AnonHugePages") == 0) AnonHugePages = value; - else if(!HugePages_Total && strcmp(name, "HugePages_Total") == 0) HugePages_Total = value; - else if(!HugePages_Free && strcmp(name, "HugePages_Free") == 0) HugePages_Free = value; - else if(!HugePages_Rsvd && strcmp(name, "HugePages_Rsvd") == 0) HugePages_Rsvd = value; - else if(!HugePages_Surp && strcmp(name, "HugePages_Surp") == 0) HugePages_Surp = value; - else if(!Hugepagesize && strcmp(name, "Hugepagesize") == 0) Hugepagesize = value; - else if(!DirectMap4k && strcmp(name, "DirectMap4k") == 0) DirectMap4k = value; - else if(!DirectMap2M && strcmp(name, "DirectMap2M") == 0) DirectMap2M = value; + size_t words = procfile_linewords(ff, l); + if(unlikely(words < 2)) continue; + + if(unlikely(arl_check(arl_base, + procfile_lineword(ff, l, 0), + procfile_lineword(ff, l, 1)))) break; } RRDSET *st; @@ -143,12 +182,12 @@ int do_proc_meminfo(int update_every, unsigned long long dt) { // -------------------------------------------------------------------- - if(hwcorrupted && do_hwcorrupt && HardwareCorrupted > 0) { + if(arl_hwcorrupted->flags & ARL_ENTRY_FLAG_FOUND && (do_hwcorrupt == CONFIG_ONDEMAND_YES || (do_hwcorrupt == CONFIG_ONDEMAND_ONDEMAND && HardwareCorrupted > 0))) { do_hwcorrupt = CONFIG_ONDEMAND_YES; st = rrdset_find("mem.hwcorrupt"); if(!st) { - st = rrdset_create("mem", "hwcorrupt", NULL, "errors", NULL, "Hardware Corrupted ECC", "MB", 9000, update_every, RRDSET_TYPE_LINE); + st = rrdset_create("mem", "hwcorrupt", NULL, "ecc", NULL, "Hardware Corrupted ECC", "MB", 9000, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; rrddim_add(st, "HardwareCorrupted", NULL, 1, 1024, RRDDIM_ABSOLUTE); diff --git a/src/proc_net_dev.c b/src/proc_net_dev.c index 53981182a..82661abd4 100644 --- a/src/proc_net_dev.c +++ b/src/proc_net_dev.c @@ -1,234 +1,369 @@ #include "common.h" -int do_proc_net_dev(int update_every, unsigned long long dt) { +struct netdev { + char *name; + uint32_t hash; + size_t len; + + // flags + int configured; + int enabled; + + int do_bandwidth; + int do_packets; + int do_errors; + int do_drops; + int do_fifo; + int do_compressed; + int do_events; + + // data collected + unsigned long long rbytes; + unsigned long long rpackets; + unsigned long long rerrors; + unsigned long long rdrops; + unsigned long long rfifo; + unsigned long long rframe; + unsigned long long rcompressed; + unsigned long long rmulticast; + + unsigned long long tbytes; + unsigned long long tpackets; + unsigned long long terrors; + unsigned long long tdrops; + unsigned long long tfifo; + unsigned long long tcollisions; + unsigned long long tcarrier; + unsigned long long tcompressed; + + // charts + RRDSET *st_bandwidth; + RRDSET *st_packets; + RRDSET *st_errors; + RRDSET *st_drops; + RRDSET *st_fifo; + RRDSET *st_compressed; + RRDSET *st_events; + + // dimensions + RRDDIM *rd_rbytes; + RRDDIM *rd_rpackets; + RRDDIM *rd_rerrors; + RRDDIM *rd_rdrops; + RRDDIM *rd_rfifo; + RRDDIM *rd_rframe; + RRDDIM *rd_rcompressed; + RRDDIM *rd_rmulticast; + + RRDDIM *rd_tbytes; + RRDDIM *rd_tpackets; + RRDDIM *rd_terrors; + RRDDIM *rd_tdrops; + RRDDIM *rd_tfifo; + RRDDIM *rd_tcollisions; + RRDDIM *rd_tcarrier; + RRDDIM *rd_tcompressed; + + struct netdev *next; +}; + +static struct netdev *netdev_root = NULL; + +static struct netdev *get_netdev(const char *name) { + static struct netdev *last = NULL; + struct netdev *d; + + uint32_t hash = simple_hash(name); + + // search it, from the last position to the end + for(d = last ; d ; d = d->next) { + if(unlikely(hash == d->hash && !strcmp(name, d->name))) { + last = d->next; + return d; + } + } + + // search it from the beginning to the last position we used + for(d = netdev_root ; d != last ; d = d->next) { + if(unlikely(hash == d->hash && !strcmp(name, d->name))) { + last = d->next; + return d; + } + } + + // create a new one + d = callocz(1, sizeof(struct netdev)); + d->name = strdupz(name); + d->hash = simple_hash(d->name); + d->len = strlen(d->name); + + // link it to the end + if(netdev_root) { + struct netdev *e; + for(e = netdev_root; e->next ; e = e->next) ; + e->next = d; + } + else + netdev_root = d; + + return d; +} + +int do_proc_net_dev(int update_every, usec_t dt) { + (void)dt; + + static SIMPLE_PATTERN *disabled_list = NULL; static procfile *ff = NULL; - static int enable_new_interfaces = -1, enable_ifb_interfaces = -1; + static int enable_new_interfaces = -1; static int do_bandwidth = -1, do_packets = -1, do_errors = -1, do_drops = -1, do_fifo = -1, do_compressed = -1, do_events = -1; - if(dt) {}; + if(unlikely(enable_new_interfaces == -1)) { + enable_new_interfaces = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "enable new interfaces detected at runtime", CONFIG_ONDEMAND_ONDEMAND); + + do_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "bandwidth for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + do_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "packets for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + do_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "errors for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + do_drops = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "drops for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + do_fifo = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "fifo for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + do_compressed = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "compressed packets for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + do_events = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "frames, collisions, carrier counters for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + + disabled_list = simple_pattern_create( + config_get("plugin:proc:/proc/net/dev", "disable by default interfaces matching", "lo fireqos* *-ifb") + , SIMPLE_PATTERN_EXACT); + } - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/dev"); ff = procfile_open(config_get("plugin:proc:/proc/net/dev", "filename to monitor", filename), " \t,:|", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - if(enable_new_interfaces == -1) enable_new_interfaces = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "enable new interfaces detected at runtime", CONFIG_ONDEMAND_ONDEMAND); - if(enable_ifb_interfaces == -1) enable_ifb_interfaces = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "enable ifb interfaces", CONFIG_ONDEMAND_NO); + size_t lines = procfile_lines(ff), l; + for(l = 2; l < lines ;l++) { + // require 17 words on each line + if(unlikely(procfile_linewords(ff, l) < 17)) continue; - if(do_bandwidth == -1) do_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "bandwidth for all interfaces", CONFIG_ONDEMAND_ONDEMAND); - if(do_packets == -1) do_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "packets for all interfaces", CONFIG_ONDEMAND_ONDEMAND); - if(do_errors == -1) do_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "errors for all interfaces", CONFIG_ONDEMAND_ONDEMAND); - if(do_drops == -1) do_drops = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "drops for all interfaces", CONFIG_ONDEMAND_ONDEMAND); - if(do_fifo == -1) do_fifo = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "fifo for all interfaces", CONFIG_ONDEMAND_ONDEMAND); - if(do_compressed == -1) do_compressed = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "compressed packets for all interfaces", CONFIG_ONDEMAND_ONDEMAND); - if(do_events == -1) do_events = config_get_boolean_ondemand("plugin:proc:/proc/net/dev", "frames, collisions, carrier counters for all interfaces", CONFIG_ONDEMAND_ONDEMAND); + struct netdev *d = get_netdev(procfile_lineword(ff, l, 0)); - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + if(unlikely(!d->configured)) { + // this is the first time we see this interface - char *iface; - unsigned long long rbytes, rpackets, rerrors, rdrops, rfifo, rframe, rcompressed, rmulticast; - unsigned long long tbytes, tpackets, terrors, tdrops, tfifo, tcollisions, tcarrier, tcompressed; + // remember we configured it + d->configured = 1; - for(l = 2; l < lines ;l++) { - words = procfile_linewords(ff, l); - if(words < 17) continue; - - iface = procfile_lineword(ff, l, 0); - - rbytes = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - rpackets = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - rerrors = strtoull(procfile_lineword(ff, l, 3), NULL, 10); - rdrops = strtoull(procfile_lineword(ff, l, 4), NULL, 10); - rfifo = strtoull(procfile_lineword(ff, l, 5), NULL, 10); - rframe = strtoull(procfile_lineword(ff, l, 6), NULL, 10); - rcompressed = strtoull(procfile_lineword(ff, l, 7), NULL, 10); - rmulticast = strtoull(procfile_lineword(ff, l, 8), NULL, 10); - - tbytes = strtoull(procfile_lineword(ff, l, 9), NULL, 10); - tpackets = strtoull(procfile_lineword(ff, l, 10), NULL, 10); - terrors = strtoull(procfile_lineword(ff, l, 11), NULL, 10); - tdrops = strtoull(procfile_lineword(ff, l, 12), NULL, 10); - tfifo = strtoull(procfile_lineword(ff, l, 13), NULL, 10); - tcollisions = strtoull(procfile_lineword(ff, l, 14), NULL, 10); - tcarrier = strtoull(procfile_lineword(ff, l, 15), NULL, 10); - tcompressed = strtoull(procfile_lineword(ff, l, 16), NULL, 10); - - int ddo_bandwidth = do_bandwidth, ddo_packets = do_packets, ddo_errors = do_errors, ddo_drops = do_drops, ddo_fifo = do_fifo, ddo_compressed = do_compressed, ddo_events = do_events; - - int default_enable = enable_new_interfaces; - - // prevent unused interfaces from creating charts - if(strcmp(iface, "lo") == 0) - default_enable = 0; - else { - int len = strlen(iface); - if(len >= 4 && strcmp(&iface[len-4], "-ifb") == 0) - default_enable = enable_ifb_interfaces; - } + d->enabled = enable_new_interfaces; + + if(d->enabled) + d->enabled = !simple_pattern_matches(disabled_list, d->name); - // check if the user wants it - { char var_name[512 + 1]; - snprintfz(var_name, 512, "plugin:proc:/proc/net/dev:%s", iface); - default_enable = config_get_boolean_ondemand(var_name, "enabled", default_enable); - if(default_enable == CONFIG_ONDEMAND_NO) continue; - if(default_enable == CONFIG_ONDEMAND_ONDEMAND && !rbytes && !tbytes) continue; - - ddo_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", ddo_bandwidth); - ddo_packets = config_get_boolean_ondemand(var_name, "packets", ddo_packets); - ddo_errors = config_get_boolean_ondemand(var_name, "errors", ddo_errors); - ddo_drops = config_get_boolean_ondemand(var_name, "drops", ddo_drops); - ddo_fifo = config_get_boolean_ondemand(var_name, "fifo", ddo_fifo); - ddo_compressed = config_get_boolean_ondemand(var_name, "compressed", ddo_compressed); - ddo_events = config_get_boolean_ondemand(var_name, "events", ddo_events); - - if(ddo_bandwidth == CONFIG_ONDEMAND_ONDEMAND && rbytes == 0 && tbytes == 0) ddo_bandwidth = 0; - if(ddo_errors == CONFIG_ONDEMAND_ONDEMAND && rerrors == 0 && terrors == 0) ddo_errors = 0; - if(ddo_drops == CONFIG_ONDEMAND_ONDEMAND && rdrops == 0 && tdrops == 0) ddo_drops = 0; - if(ddo_fifo == CONFIG_ONDEMAND_ONDEMAND && rfifo == 0 && tfifo == 0) ddo_fifo = 0; - if(ddo_compressed == CONFIG_ONDEMAND_ONDEMAND && rcompressed == 0 && tcompressed == 0) ddo_compressed = 0; - if(ddo_events == CONFIG_ONDEMAND_ONDEMAND && rframe == 0 && tcollisions == 0 && tcarrier == 0) ddo_events = 0; - - // for absolute values, we need to switch the setting to 'yes' - // to allow it refresh from now on - // if(ddo_fifo == CONFIG_ONDEMAND_ONDEMAND) config_set(var_name, "fifo", "yes"); + snprintfz(var_name, 512, "plugin:proc:/proc/net/dev:%s", d->name); + d->enabled = config_get_boolean_ondemand(var_name, "enabled", d->enabled); + + if(d->enabled == CONFIG_ONDEMAND_NO) + continue; + + d->do_bandwidth = config_get_boolean_ondemand(var_name, "bandwidth", do_bandwidth); + d->do_packets = config_get_boolean_ondemand(var_name, "packets", do_packets); + d->do_errors = config_get_boolean_ondemand(var_name, "errors", do_errors); + d->do_drops = config_get_boolean_ondemand(var_name, "drops", do_drops); + d->do_fifo = config_get_boolean_ondemand(var_name, "fifo", do_fifo); + d->do_compressed = config_get_boolean_ondemand(var_name, "compressed", do_compressed); + d->do_events = config_get_boolean_ondemand(var_name, "events", do_events); } - RRDSET *st; + if(unlikely(!d->enabled)) + continue; + + d->rbytes = str2ull(procfile_lineword(ff, l, 1)); + d->rpackets = str2ull(procfile_lineword(ff, l, 2)); + d->rerrors = str2ull(procfile_lineword(ff, l, 3)); + d->rdrops = str2ull(procfile_lineword(ff, l, 4)); + d->rfifo = str2ull(procfile_lineword(ff, l, 5)); + d->rframe = str2ull(procfile_lineword(ff, l, 6)); + d->rcompressed = str2ull(procfile_lineword(ff, l, 7)); + d->rmulticast = str2ull(procfile_lineword(ff, l, 8)); + + d->tbytes = str2ull(procfile_lineword(ff, l, 9)); + d->tpackets = str2ull(procfile_lineword(ff, l, 10)); + d->terrors = str2ull(procfile_lineword(ff, l, 11)); + d->tdrops = str2ull(procfile_lineword(ff, l, 12)); + d->tfifo = str2ull(procfile_lineword(ff, l, 13)); + d->tcollisions = str2ull(procfile_lineword(ff, l, 14)); + d->tcarrier = str2ull(procfile_lineword(ff, l, 15)); + d->tcompressed = str2ull(procfile_lineword(ff, l, 16)); // -------------------------------------------------------------------- - if(ddo_bandwidth) { - st = rrdset_find_bytype("net", iface); - if(!st) { - st = rrdset_create("net", iface, NULL, iface, "net.net", "Bandwidth", "kilobits/s", 7000, update_every, RRDSET_TYPE_AREA); + if(unlikely((d->do_bandwidth == CONFIG_ONDEMAND_ONDEMAND && (d->rbytes || d->tbytes)))) + d->do_bandwidth = CONFIG_ONDEMAND_YES; - rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); + if(d->do_bandwidth == CONFIG_ONDEMAND_YES) { + if(unlikely(!d->st_bandwidth)) { + d->st_bandwidth = rrdset_find_bytype("net", d->name); + + if(!d->st_bandwidth) + d->st_bandwidth = rrdset_create("net", d->name, NULL, d->name, "net.net", "Bandwidth", "kilobits/s", 7000, update_every, RRDSET_TYPE_AREA); + + d->rd_rbytes = rrddim_add(d->st_bandwidth, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); + d->rd_tbytes = rrddim_add(d->st_bandwidth, "sent", NULL, -8, 1024, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else rrdset_next(d->st_bandwidth); - rrddim_set(st, "received", rbytes); - rrddim_set(st, "sent", tbytes); - rrdset_done(st); + rrddim_set_by_pointer(d->st_bandwidth, d->rd_rbytes, d->rbytes); + rrddim_set_by_pointer(d->st_bandwidth, d->rd_tbytes, d->tbytes); + rrdset_done(d->st_bandwidth); } // -------------------------------------------------------------------- - if(ddo_packets) { - st = rrdset_find_bytype("net_packets", iface); - if(!st) { - st = rrdset_create("net_packets", iface, NULL, iface, "net.packets", "Packets", "packets/s", 7001, update_every, RRDSET_TYPE_LINE); - st->isdetail = 1; + if(unlikely((d->do_packets == CONFIG_ONDEMAND_ONDEMAND && (d->rpackets || d->tpackets || d->rmulticast)))) + d->do_packets = CONFIG_ONDEMAND_YES; + + if(d->do_packets == CONFIG_ONDEMAND_YES) { + if(unlikely(!d->st_packets)) { + d->st_packets = rrdset_find_bytype("net_packets", d->name); + + if(!d->st_packets) + d->st_packets = rrdset_create("net_packets", d->name, NULL, d->name, "net.packets", "Packets", "packets/s", 7001, update_every, RRDSET_TYPE_LINE); + + d->st_packets->isdetail = 1; - rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "multicast", NULL, 1, 1, RRDDIM_INCREMENTAL); + d->rd_rpackets = rrddim_add(d->st_packets, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + d->rd_tpackets = rrddim_add(d->st_packets, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + d->rd_rmulticast = rrddim_add(d->st_packets, "multicast", NULL, 1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else rrdset_next(d->st_packets); - rrddim_set(st, "received", rpackets); - rrddim_set(st, "sent", tpackets); - rrddim_set(st, "multicast", rmulticast); - rrdset_done(st); + rrddim_set_by_pointer(d->st_packets, d->rd_rpackets, d->rpackets); + rrddim_set_by_pointer(d->st_packets, d->rd_tpackets, d->tpackets); + rrddim_set_by_pointer(d->st_packets, d->rd_rmulticast, d->rmulticast); + rrdset_done(d->st_packets); } // -------------------------------------------------------------------- - if(ddo_errors) { - st = rrdset_find_bytype("net_errors", iface); - if(!st) { - st = rrdset_create("net_errors", iface, NULL, iface, "net.errors", "Interface Errors", "errors/s", 7002, update_every, RRDSET_TYPE_LINE); - st->isdetail = 1; + if(unlikely((d->do_errors == CONFIG_ONDEMAND_ONDEMAND && (d->rerrors || d->terrors)))) + d->do_errors = CONFIG_ONDEMAND_YES; - rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); + if(d->do_errors == CONFIG_ONDEMAND_YES) { + if(unlikely(!d->st_errors)) { + d->st_errors = rrdset_find_bytype("net_errors", d->name); + + if(!d->st_errors) + d->st_errors = rrdset_create("net_errors", d->name, NULL, d->name, "net.errors", "Interface Errors", "errors/s", 7002, update_every, RRDSET_TYPE_LINE); + + d->st_errors->isdetail = 1; + + d->rd_rerrors = rrddim_add(d->st_errors, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); + d->rd_terrors = rrddim_add(d->st_errors, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else rrdset_next(d->st_errors); - rrddim_set(st, "inbound", rerrors); - rrddim_set(st, "outbound", terrors); - rrdset_done(st); + rrddim_set_by_pointer(d->st_errors, d->rd_rerrors, d->rerrors); + rrddim_set_by_pointer(d->st_errors, d->rd_terrors, d->terrors); + rrdset_done(d->st_errors); } // -------------------------------------------------------------------- - if(ddo_drops) { - st = rrdset_find_bytype("net_drops", iface); - if(!st) { - st = rrdset_create("net_drops", iface, NULL, iface, "net.drops", "Interface Drops", "drops/s", 7003, update_every, RRDSET_TYPE_LINE); - st->isdetail = 1; + if(unlikely((d->do_drops == CONFIG_ONDEMAND_ONDEMAND && (d->rdrops || d->tdrops)))) + d->do_drops = CONFIG_ONDEMAND_YES; + + if(d->do_drops == CONFIG_ONDEMAND_YES) { + if(unlikely(!d->st_drops)) { + d->st_drops = rrdset_find_bytype("net_drops", d->name); - rrddim_add(st, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); + if(!d->st_drops) + d->st_drops = rrdset_create("net_drops", d->name, NULL, d->name, "net.drops", "Interface Drops", "drops/s", 7003, update_every, RRDSET_TYPE_LINE); + + d->st_drops->isdetail = 1; + + d->rd_rdrops = rrddim_add(d->st_drops, "inbound", NULL, 1, 1, RRDDIM_INCREMENTAL); + d->rd_tdrops = rrddim_add(d->st_drops, "outbound", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else rrdset_next(d->st_drops); - rrddim_set(st, "inbound", rdrops); - rrddim_set(st, "outbound", tdrops); - rrdset_done(st); + rrddim_set_by_pointer(d->st_drops, d->rd_rdrops, d->rdrops); + rrddim_set_by_pointer(d->st_drops, d->rd_tdrops, d->tdrops); + rrdset_done(d->st_drops); } // -------------------------------------------------------------------- - if(ddo_fifo) { - st = rrdset_find_bytype("net_fifo", iface); - if(!st) { - st = rrdset_create("net_fifo", iface, NULL, iface, "net.fifo", "Interface FIFO Buffer Errors", "errors", 7004, update_every, RRDSET_TYPE_LINE); - st->isdetail = 1; + if(unlikely((d->do_fifo == CONFIG_ONDEMAND_ONDEMAND && (d->rfifo || d->tfifo)))) + d->do_fifo = CONFIG_ONDEMAND_YES; + + if(d->do_fifo == CONFIG_ONDEMAND_YES) { + if(unlikely(!d->st_fifo)) { + d->st_fifo = rrdset_find_bytype("net_fifo", d->name); + + if(!d->st_fifo) + d->st_fifo = rrdset_create("net_fifo", d->name, NULL, d->name, "net.fifo", "Interface FIFO Buffer Errors", "errors", 7004, update_every, RRDSET_TYPE_LINE); + + d->st_fifo->isdetail = 1; - rrddim_add(st, "receive", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "transmit", NULL, -1, 1, RRDDIM_INCREMENTAL); + d->rd_rfifo = rrddim_add(d->st_fifo, "receive", NULL, 1, 1, RRDDIM_INCREMENTAL); + d->rd_tfifo = rrddim_add(d->st_fifo, "transmit", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else rrdset_next(d->st_fifo); - rrddim_set(st, "receive", rfifo); - rrddim_set(st, "transmit", tfifo); - rrdset_done(st); + rrddim_set_by_pointer(d->st_fifo, d->rd_rfifo, d->rfifo); + rrddim_set_by_pointer(d->st_fifo, d->rd_tfifo, d->tfifo); + rrdset_done(d->st_fifo); } // -------------------------------------------------------------------- - if(ddo_compressed) { - st = rrdset_find_bytype("net_compressed", iface); - if(!st) { - st = rrdset_create("net_compressed", iface, NULL, iface, "net.compressed", "Compressed Packets", "packets/s", 7005, update_every, RRDSET_TYPE_LINE); - st->isdetail = 1; + if(unlikely((d->do_compressed == CONFIG_ONDEMAND_ONDEMAND && (d->rcompressed || d->tcompressed)))) + d->do_compressed = CONFIG_ONDEMAND_YES; - rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); + if(d->do_compressed == CONFIG_ONDEMAND_YES) { + if(unlikely(!d->st_compressed)) { + d->st_compressed = rrdset_find_bytype("net_compressed", d->name); + if(!d->st_compressed) + d->st_compressed = rrdset_create("net_compressed", d->name, NULL, d->name, "net.compressed", "Compressed Packets", "packets/s", 7005, update_every, RRDSET_TYPE_LINE); + + d->st_compressed->isdetail = 1; + + d->rd_rcompressed = rrddim_add(d->st_compressed, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); + d->rd_tcompressed = rrddim_add(d->st_compressed, "sent", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else rrdset_next(d->st_compressed); - rrddim_set(st, "received", rcompressed); - rrddim_set(st, "sent", tcompressed); - rrdset_done(st); + rrddim_set_by_pointer(d->st_compressed, d->rd_rcompressed, d->rcompressed); + rrddim_set_by_pointer(d->st_compressed, d->rd_tcompressed, d->tcompressed); + rrdset_done(d->st_compressed); } // -------------------------------------------------------------------- - if(ddo_events) { - st = rrdset_find_bytype("net_events", iface); - if(!st) { - st = rrdset_create("net_events", iface, NULL, iface, "net.events", "Network Interface Events", "events/s", 7006, update_every, RRDSET_TYPE_LINE); - st->isdetail = 1; + if(unlikely((d->do_events == CONFIG_ONDEMAND_ONDEMAND && (d->rframe || d->tcollisions || d->tcarrier)))) + d->do_events = CONFIG_ONDEMAND_YES; + + if(d->do_events == CONFIG_ONDEMAND_YES) { + if(unlikely(!d->st_events)) { + d->st_events = rrdset_find_bytype("net_events", d->name); + if(!d->st_events) + d->st_events = rrdset_create("net_events", d->name, NULL, d->name, "net.events", "Network Interface Events", "events/s", 7006, update_every, RRDSET_TYPE_LINE); + + d->st_events->isdetail = 1; - rrddim_add(st, "frames", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "collisions", NULL, -1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "carrier", NULL, -1, 1, RRDDIM_INCREMENTAL); + d->rd_rframe = rrddim_add(d->st_events, "frames", NULL, 1, 1, RRDDIM_INCREMENTAL); + d->rd_tcollisions = rrddim_add(d->st_events, "collisions", NULL, -1, 1, RRDDIM_INCREMENTAL); + d->rd_tcarrier = rrddim_add(d->st_events, "carrier", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else rrdset_next(d->st_events); - rrddim_set(st, "frames", rframe); - rrddim_set(st, "collisions", tcollisions); - rrddim_set(st, "carrier", tcarrier); - rrdset_done(st); + rrddim_set_by_pointer(d->st_events, d->rd_rframe, d->rframe); + rrddim_set_by_pointer(d->st_events, d->rd_tcollisions, d->tcollisions); + rrddim_set_by_pointer(d->st_events, d->rd_tcarrier, d->tcarrier); + rrdset_done(d->st_events); } } diff --git a/src/proc_net_ip_vs_stats.c b/src/proc_net_ip_vs_stats.c index de3e0e460..34cadaea7 100644 --- a/src/proc_net_ip_vs_stats.c +++ b/src/proc_net_ip_vs_stats.c @@ -1,9 +1,8 @@ #include "common.h" #define RRD_TYPE_NET_IPVS "ipvs" -#define RRD_TYPE_NET_IPVS_LEN strlen(RRD_TYPE_NET_IPVS) -int do_proc_net_ip_vs_stats(int update_every, unsigned long long dt) { +int do_proc_net_ip_vs_stats(int update_every, usec_t dt) { static int do_bandwidth = -1, do_sockets = -1, do_packets = -1; static procfile *ff = NULL; diff --git a/src/proc_net_netstat.c b/src/proc_net_netstat.c index ea38acf29..8741a71c9 100644 --- a/src/proc_net_netstat.c +++ b/src/proc_net_netstat.c @@ -1,197 +1,22 @@ #include "common.h" - -struct netstat_columns { - char *name; - uint32_t hash; - unsigned long long value; - int multiplier; // not needed everywhere - char *label; // not needed everywhere -}; - -static struct netstat_columns tcpext_data[] = { - { "SyncookiesSent", 0, 0, 1, NULL }, - { "SyncookiesRecv", 0, 0, 1, NULL }, - { "SyncookiesFailed", 0, 0, 1, NULL }, - { "EmbryonicRsts", 0, 0, 1, NULL }, - { "PruneCalled", 0, 0, 1, NULL }, - { "RcvPruned", 0, 0, 1, NULL }, - { "OfoPruned", 0, 0, 1, NULL }, - { "OutOfWindowIcmps", 0, 0, 1, NULL }, - { "LockDroppedIcmps", 0, 0, 1, NULL }, - { "ArpFilter", 0, 0, 1, NULL }, - { "TW", 0, 0, 1, NULL }, - { "TWRecycled", 0, 0, 1, NULL }, - { "TWKilled", 0, 0, 1, NULL }, - { "PAWSPassive", 0, 0, 1, NULL }, - { "PAWSActive", 0, 0, 1, NULL }, - { "PAWSEstab", 0, 0, 1, NULL }, - { "DelayedACKs", 0, 0, 1, NULL }, - { "DelayedACKLocked", 0, 0, 1, NULL }, - { "DelayedACKLost", 0, 0, 1, NULL }, - { "ListenOverflows", 0, 0, 1, NULL }, - { "ListenDrops", 0, 0, 1, NULL }, - { "TCPPrequeued", 0, 0, 1, NULL }, - { "TCPDirectCopyFromBacklog", 0, 0, 1, NULL }, - { "TCPDirectCopyFromPrequeue", 0, 0, 1, NULL }, - { "TCPPrequeueDropped", 0, 0, 1, NULL }, - { "TCPHPHits", 0, 0, 1, NULL }, - { "TCPHPHitsToUser", 0, 0, 1, NULL }, - { "TCPPureAcks", 0, 0, 1, NULL }, - { "TCPHPAcks", 0, 0, 1, NULL }, - { "TCPRenoRecovery", 0, 0, 1, NULL }, - { "TCPSackRecovery", 0, 0, 1, NULL }, - { "TCPSACKReneging", 0, 0, 1, NULL }, - { "TCPFACKReorder", 0, 0, 1, NULL }, - { "TCPSACKReorder", 0, 0, 1, NULL }, - { "TCPRenoReorder", 0, 0, 1, NULL }, - { "TCPTSReorder", 0, 0, 1, NULL }, - { "TCPFullUndo", 0, 0, 1, NULL }, - { "TCPPartialUndo", 0, 0, 1, NULL }, - { "TCPDSACKUndo", 0, 0, 1, NULL }, - { "TCPLossUndo", 0, 0, 1, NULL }, - { "TCPLostRetransmit", 0, 0, 1, NULL }, - { "TCPRenoFailures", 0, 0, 1, NULL }, - { "TCPSackFailures", 0, 0, 1, NULL }, - { "TCPLossFailures", 0, 0, 1, NULL }, - { "TCPFastRetrans", 0, 0, 1, NULL }, - { "TCPForwardRetrans", 0, 0, 1, NULL }, - { "TCPSlowStartRetrans", 0, 0, 1, NULL }, - { "TCPTimeouts", 0, 0, 1, NULL }, - { "TCPLossProbes", 0, 0, 1, NULL }, - { "TCPLossProbeRecovery", 0, 0, 1, NULL }, - { "TCPRenoRecoveryFail", 0, 0, 1, NULL }, - { "TCPSackRecoveryFail", 0, 0, 1, NULL }, - { "TCPSchedulerFailed", 0, 0, 1, NULL }, - { "TCPRcvCollapsed", 0, 0, 1, NULL }, - { "TCPDSACKOldSent", 0, 0, 1, NULL }, - { "TCPDSACKOfoSent", 0, 0, 1, NULL }, - { "TCPDSACKRecv", 0, 0, 1, NULL }, - { "TCPDSACKOfoRecv", 0, 0, 1, NULL }, - { "TCPAbortOnData", 0, 0, 1, NULL }, - { "TCPAbortOnClose", 0, 0, 1, NULL }, - { "TCPAbortOnMemory", 0, 0, 1, NULL }, - { "TCPAbortOnTimeout", 0, 0, 1, NULL }, - { "TCPAbortOnLinger", 0, 0, 1, NULL }, - { "TCPAbortFailed", 0, 0, 1, NULL }, - { "TCPMemoryPressures", 0, 0, 1, NULL }, - { "TCPSACKDiscard", 0, 0, 1, NULL }, - { "TCPDSACKIgnoredOld", 0, 0, 1, NULL }, - { "TCPDSACKIgnoredNoUndo", 0, 0, 1, NULL }, - { "TCPSpuriousRTOs", 0, 0, 1, NULL }, - { "TCPMD5NotFound", 0, 0, 1, NULL }, - { "TCPMD5Unexpected", 0, 0, 1, NULL }, - { "TCPSackShifted", 0, 0, 1, NULL }, - { "TCPSackMerged", 0, 0, 1, NULL }, - { "TCPSackShiftFallback", 0, 0, 1, NULL }, - { "TCPBacklogDrop", 0, 0, 1, NULL }, - { "TCPMinTTLDrop", 0, 0, 1, NULL }, - { "TCPDeferAcceptDrop", 0, 0, 1, NULL }, - { "IPReversePathFilter", 0, 0, 1, NULL }, - { "TCPTimeWaitOverflow", 0, 0, 1, NULL }, - { "TCPReqQFullDoCookies", 0, 0, 1, NULL }, - { "TCPReqQFullDrop", 0, 0, 1, NULL }, - { "TCPRetransFail", 0, 0, 1, NULL }, - { "TCPRcvCoalesce", 0, 0, 1, NULL }, - { "TCPOFOQueue", 0, 0, 1, NULL }, - { "TCPOFODrop", 0, 0, 1, NULL }, - { "TCPOFOMerge", 0, 0, 1, NULL }, - { "TCPChallengeACK", 0, 0, 1, NULL }, - { "TCPSYNChallenge", 0, 0, 1, NULL }, - { "TCPFastOpenActive", 0, 0, 1, NULL }, - { "TCPFastOpenActiveFail", 0, 0, 1, NULL }, - { "TCPFastOpenPassive", 0, 0, 1, NULL }, - { "TCPFastOpenPassiveFail", 0, 0, 1, NULL }, - { "TCPFastOpenListenOverflow", 0, 0, 1, NULL }, - { "TCPFastOpenCookieReqd", 0, 0, 1, NULL }, - { "TCPSpuriousRtxHostQueues", 0, 0, 1, NULL }, - { "BusyPollRxPackets", 0, 0, 1, NULL }, - { "TCPAutoCorking", 0, 0, 1, NULL }, - { "TCPFromZeroWindowAdv", 0, 0, 1, NULL }, - { "TCPToZeroWindowAdv", 0, 0, 1, NULL }, - { "TCPWantZeroWindowAdv", 0, 0, 1, NULL }, - { "TCPSynRetrans", 0, 0, 1, NULL }, - { "TCPOrigDataSent", 0, 0, 1, NULL }, - { "TCPHystartTrainDetect", 0, 0, 1, NULL }, - { "TCPHystartTrainCwnd", 0, 0, 1, NULL }, - { "TCPHystartDelayDetect", 0, 0, 1, NULL }, - { "TCPHystartDelayCwnd", 0, 0, 1, NULL }, - { "TCPACKSkippedSynRecv", 0, 0, 1, NULL }, - { "TCPACKSkippedPAWS", 0, 0, 1, NULL }, - { "TCPACKSkippedSeq", 0, 0, 1, NULL }, - { "TCPACKSkippedFinWait2", 0, 0, 1, NULL }, - { "TCPACKSkippedTimeWait", 0, 0, 1, NULL }, - { "TCPACKSkippedChallenge", 0, 0, 1, NULL }, - { "TCPWinProbe", 0, 0, 1, NULL }, - { "TCPKeepAlive", 0, 0, 1, NULL }, - { "TCPMTUPFail", 0, 0, 1, NULL }, - { "TCPMTUPSuccess", 0, 0, 1, NULL }, - { NULL, 0, 0, 0, NULL } -}; - -static struct netstat_columns ipext_data[] = { - { "InNoRoutes", 0, 0, 1, NULL }, - { "InTruncatedPkts", 0, 0, 1, NULL }, - { "InMcastPkts", 0, 0, 1, NULL }, - { "OutMcastPkts", 0, 0, 1, NULL }, - { "InBcastPkts", 0, 0, 1, NULL }, - { "OutBcastPkts", 0, 0, 1, NULL }, - { "InOctets", 0, 0, 1, NULL }, - { "OutOctets", 0, 0, 1, NULL }, - { "InMcastOctets", 0, 0, 1, NULL }, - { "OutMcastOctets", 0, 0, 1, NULL }, - { "InBcastOctets", 0, 0, 1, NULL }, - { "OutBcastOctets", 0, 0, 1, NULL }, - { "InCsumErrors", 0, 0, 1, NULL }, - { "InNoECTPkts", 0, 0, 1, NULL }, - { "InECT1Pkts", 0, 0, 1, NULL }, - { "InECT0Pkts", 0, 0, 1, NULL }, - { "InCEPkts", 0, 0, 1, NULL }, - { NULL, 0, 0, 0, NULL } -}; - -static void hash_array(struct netstat_columns *nc) { - int i; - - for(i = 0; nc[i].name ;i++) - nc[i].hash = simple_hash(nc[i].name); -} - -static unsigned long long *netstat_columns_find(struct netstat_columns *nc, const char *name) { - uint32_t i, hash = simple_hash(name); - - for(i = 0; nc[i].name ;i++) - if(unlikely(nc[i].hash == hash && !strcmp(nc[i].name, name))) - return &nc[i].value; - - fatal("Cannot find key '%s' in /proc/net/netstat internal array.", name); -} - -static void parse_line_pair(procfile *ff, struct netstat_columns *nc, uint32_t header_line, uint32_t values_line) { - uint32_t hwords = procfile_linewords(ff, header_line); - uint32_t vwords = procfile_linewords(ff, values_line); - uint32_t w, i; +static void parse_line_pair(procfile *ff, ARL_BASE *base, size_t header_line, size_t values_line) { + size_t hwords = procfile_linewords(ff, header_line); + size_t vwords = procfile_linewords(ff, values_line); + size_t w; if(unlikely(vwords > hwords)) { - error("File /proc/net/netstat on header line %u has %u words, but on value line %u has %u words.", header_line, hwords, values_line, vwords); + error("File /proc/net/netstat on header line %zu has %zu words, but on value line %zu has %zu words.", header_line, hwords, values_line, vwords); vwords = hwords; } for(w = 1; w < vwords ;w++) { - char *key = procfile_lineword(ff, header_line, w); - uint32_t hash = simple_hash(key); - - for(i = 0 ; nc[i].name ;i++) { - if(unlikely(hash == nc[i].hash && !strcmp(key, nc[i].name))) { - nc[i].value = strtoull(procfile_lineword(ff, values_line, w), NULL, 10); - break; - } - } + if(unlikely(arl_check(base, procfile_lineword(ff, header_line, w), procfile_lineword(ff, values_line, w)))) + break; } } - -int do_proc_net_netstat(int update_every, unsigned long long dt) { +int do_proc_net_netstat(int update_every, usec_t dt) { (void)dt; static int do_bandwidth = -1, do_inerrors = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, do_bcast_p = -1, do_ecn = -1, \ @@ -199,178 +24,80 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { static uint32_t hash_ipext = 0, hash_tcpext = 0; static procfile *ff = NULL; - static unsigned long long *tcpext_TCPRenoReorder = NULL; - static unsigned long long *tcpext_TCPFACKReorder = NULL; - static unsigned long long *tcpext_TCPSACKReorder = NULL; - static unsigned long long *tcpext_TCPTSReorder = NULL; - - static unsigned long long *tcpext_SyncookiesSent = NULL; - static unsigned long long *tcpext_SyncookiesRecv = NULL; - static unsigned long long *tcpext_SyncookiesFailed = NULL; + static ARL_BASE *arl_tcpext = NULL; + static ARL_BASE *arl_ipext = NULL; + + // -------------------------------------------------------------------- + // IPv4 + + // IPv4 bandwidth + static unsigned long long ipext_InOctets = 0; + static unsigned long long ipext_OutOctets = 0; + + // IPv4 input errors + static unsigned long long ipext_InNoRoutes = 0; + static unsigned long long ipext_InTruncatedPkts = 0; + static unsigned long long ipext_InCsumErrors = 0; + + // IPv4 multicast bandwidth + static unsigned long long ipext_InMcastOctets = 0; + static unsigned long long ipext_OutMcastOctets = 0; + + // IPv4 multicast packets + static unsigned long long ipext_InMcastPkts = 0; + static unsigned long long ipext_OutMcastPkts = 0; + + // IPv4 broadcast bandwidth + static unsigned long long ipext_InBcastOctets = 0; + static unsigned long long ipext_OutBcastOctets = 0; + + // IPv4 broadcast packets + static unsigned long long ipext_InBcastPkts = 0; + static unsigned long long ipext_OutBcastPkts = 0; + + // IPv4 ECN + static unsigned long long ipext_InNoECTPkts = 0; + static unsigned long long ipext_InECT1Pkts = 0; + static unsigned long long ipext_InECT0Pkts = 0; + static unsigned long long ipext_InCEPkts = 0; + + // -------------------------------------------------------------------- + // IPv4 TCP + + // IPv4 TCP Reordering + static unsigned long long tcpext_TCPRenoReorder = 0; + static unsigned long long tcpext_TCPFACKReorder = 0; + static unsigned long long tcpext_TCPSACKReorder = 0; + static unsigned long long tcpext_TCPTSReorder = 0; + + // IPv4 TCP SYN Cookies + static unsigned long long tcpext_SyncookiesSent = 0; + static unsigned long long tcpext_SyncookiesRecv = 0; + static unsigned long long tcpext_SyncookiesFailed = 0; + + // IPv4 TCP Out Of Order Queue + // http://www.spinics.net/lists/netdev/msg204696.html + static unsigned long long tcpext_TCPOFOQueue = 0; // Number of packets queued in OFO queue + static unsigned long long tcpext_TCPOFODrop = 0; // Number of packets meant to be queued in OFO but dropped because socket rcvbuf limit hit. + static unsigned long long tcpext_TCPOFOMerge = 0; // Number of packets in OFO that were merged with other packets. + static unsigned long long tcpext_OfoPruned = 0; // packets dropped from out-of-order queue because of socket buffer overrun + + // IPv4 TCP connection resets + // https://github.com/ecki/net-tools/blob/bd8bceaed2311651710331a7f8990c3e31be9840/statistics.c + static unsigned long long tcpext_TCPAbortOnData = 0; // connections reset due to unexpected data + static unsigned long long tcpext_TCPAbortOnClose = 0; // connections reset due to early user close + static unsigned long long tcpext_TCPAbortOnMemory = 0; // connections aborted due to memory pressure + static unsigned long long tcpext_TCPAbortOnTimeout = 0; // connections aborted due to timeout + static unsigned long long tcpext_TCPAbortOnLinger = 0; // connections aborted after user close in linger timeout + static unsigned long long tcpext_TCPAbortFailed = 0; // times unable to send RST due to no memory + + // IPv4 TCP memory pressures + static unsigned long long tcpext_TCPMemoryPressures = 0; + + if(unlikely(!arl_ipext)) { + hash_ipext = simple_hash("IpExt"); + hash_tcpext = simple_hash("TcpExt"); - static unsigned long long *tcpext_TCPOFOQueue = NULL; // Number of packets queued in OFO queue - static unsigned long long *tcpext_TCPOFODrop = NULL; // Number of packets meant to be queued in OFO but dropped because socket rcvbuf limit hit. - static unsigned long long *tcpext_TCPOFOMerge = NULL; // Number of packets in OFO that were merged with other packets. - static unsigned long long *tcpext_OfoPruned = NULL; // packets dropped from out-of-order queue because of socket buffer overrun - - static unsigned long long *tcpext_TCPAbortOnData = NULL; // connections reset due to unexpected data - static unsigned long long *tcpext_TCPAbortOnClose = NULL; // connections reset due to early user close - static unsigned long long *tcpext_TCPAbortOnMemory = NULL; // connections aborted due to memory pressure - static unsigned long long *tcpext_TCPAbortOnTimeout = NULL; // connections aborted due to timeout - static unsigned long long *tcpext_TCPAbortOnLinger = NULL; // connections aborted after user close in linger timeout - static unsigned long long *tcpext_TCPAbortFailed = NULL; // times unable to send RST due to no memory - - static unsigned long long *tcpext_TCPMemoryPressures = NULL; - -/* - // connection rejects - static unsigned long long *tcpext_PAWSActive = NULL; // active connections rejected because of time stamp - static unsigned long long *tcpext_PAWSPassive = NULL; // passive connections rejected because of time stamp - - static unsigned long long *tcpext_TCPTimeouts = NULL; - - static unsigned long long *tcpext_TCPDSACKUndo = NULL; - static unsigned long long *tcpext_TCPDSACKOldSent = NULL; - static unsigned long long *tcpext_TCPDSACKOfoSent = NULL; - static unsigned long long *tcpext_TCPDSACKRecv = NULL; - static unsigned long long *tcpext_TCPDSACKOfoRecv = NULL; - static unsigned long long *tcpext_TCPDSACKIgnoredOld = NULL; - static unsigned long long *tcpext_TCPDSACKIgnoredNoUndo = NULL; - - - static unsigned long long *tcpext_EmbryonicRsts = NULL; - - static unsigned long long *tcpext_PruneCalled = NULL; - static unsigned long long *tcpext_RcvPruned = NULL; - static unsigned long long *tcpext_OutOfWindowIcmps = NULL; - static unsigned long long *tcpext_LockDroppedIcmps = NULL; - static unsigned long long *tcpext_ArpFilter = NULL; - - static unsigned long long *tcpext_TW = NULL; - static unsigned long long *tcpext_TWRecycled = NULL; - static unsigned long long *tcpext_TWKilled = NULL; - - static unsigned long long *tcpext_PAWSEstab = NULL; - - static unsigned long long *tcpext_DelayedACKs = NULL; - static unsigned long long *tcpext_DelayedACKLocked = NULL; - static unsigned long long *tcpext_DelayedACKLost = NULL; - - static unsigned long long *tcpext_ListenOverflows = NULL; - static unsigned long long *tcpext_ListenDrops = NULL; - - static unsigned long long *tcpext_TCPPrequeued = NULL; - - static unsigned long long *tcpext_TCPDirectCopyFromBacklog = NULL; - static unsigned long long *tcpext_TCPDirectCopyFromPrequeue = NULL; - static unsigned long long *tcpext_TCPPrequeueDropped = NULL; - - static unsigned long long *tcpext_TCPHPHits = NULL; - static unsigned long long *tcpext_TCPHPHitsToUser = NULL; - static unsigned long long *tcpext_TCPHPAcks = NULL; - - static unsigned long long *tcpext_TCPPureAcks = NULL; - static unsigned long long *tcpext_TCPRenoRecovery = NULL; - - static unsigned long long *tcpext_TCPSackRecovery = NULL; - static unsigned long long *tcpext_TCPSackFailures = NULL; - static unsigned long long *tcpext_TCPSACKReneging = NULL; - static unsigned long long *tcpext_TCPSackRecoveryFail = NULL; - static unsigned long long *tcpext_TCPSACKDiscard = NULL; - static unsigned long long *tcpext_TCPSackShifted = NULL; - static unsigned long long *tcpext_TCPSackMerged = NULL; - static unsigned long long *tcpext_TCPSackShiftFallback = NULL; - - - static unsigned long long *tcpext_TCPFullUndo = NULL; - static unsigned long long *tcpext_TCPPartialUndo = NULL; - - static unsigned long long *tcpext_TCPLossUndo = NULL; - static unsigned long long *tcpext_TCPLostRetransmit = NULL; - - static unsigned long long *tcpext_TCPRenoFailures = NULL; - - static unsigned long long *tcpext_TCPLossFailures = NULL; - static unsigned long long *tcpext_TCPFastRetrans = NULL; - static unsigned long long *tcpext_TCPForwardRetrans = NULL; - static unsigned long long *tcpext_TCPSlowStartRetrans = NULL; - static unsigned long long *tcpext_TCPLossProbes = NULL; - static unsigned long long *tcpext_TCPLossProbeRecovery = NULL; - static unsigned long long *tcpext_TCPRenoRecoveryFail = NULL; - static unsigned long long *tcpext_TCPSchedulerFailed = NULL; - static unsigned long long *tcpext_TCPRcvCollapsed = NULL; - - static unsigned long long *tcpext_TCPSpuriousRTOs = NULL; - static unsigned long long *tcpext_TCPMD5NotFound = NULL; - static unsigned long long *tcpext_TCPMD5Unexpected = NULL; - - static unsigned long long *tcpext_TCPBacklogDrop = NULL; - static unsigned long long *tcpext_TCPMinTTLDrop = NULL; - static unsigned long long *tcpext_TCPDeferAcceptDrop = NULL; - static unsigned long long *tcpext_IPReversePathFilter = NULL; - static unsigned long long *tcpext_TCPTimeWaitOverflow = NULL; - static unsigned long long *tcpext_TCPReqQFullDoCookies = NULL; - static unsigned long long *tcpext_TCPReqQFullDrop = NULL; - static unsigned long long *tcpext_TCPRetransFail = NULL; - static unsigned long long *tcpext_TCPRcvCoalesce = NULL; - - static unsigned long long *tcpext_TCPChallengeACK = NULL; - static unsigned long long *tcpext_TCPSYNChallenge = NULL; - - static unsigned long long *tcpext_TCPFastOpenActive = NULL; - static unsigned long long *tcpext_TCPFastOpenActiveFail = NULL; - static unsigned long long *tcpext_TCPFastOpenPassive = NULL; - static unsigned long long *tcpext_TCPFastOpenPassiveFail = NULL; - static unsigned long long *tcpext_TCPFastOpenListenOverflow = NULL; - static unsigned long long *tcpext_TCPFastOpenCookieReqd = NULL; - - static unsigned long long *tcpext_TCPSpuriousRtxHostQueues = NULL; - static unsigned long long *tcpext_BusyPollRxPackets = NULL; - static unsigned long long *tcpext_TCPAutoCorking = NULL; - static unsigned long long *tcpext_TCPFromZeroWindowAdv = NULL; - static unsigned long long *tcpext_TCPToZeroWindowAdv = NULL; - static unsigned long long *tcpext_TCPWantZeroWindowAdv = NULL; - static unsigned long long *tcpext_TCPSynRetrans = NULL; - static unsigned long long *tcpext_TCPOrigDataSent = NULL; - - static unsigned long long *tcpext_TCPHystartTrainDetect = NULL; - static unsigned long long *tcpext_TCPHystartTrainCwnd = NULL; - static unsigned long long *tcpext_TCPHystartDelayDetect = NULL; - static unsigned long long *tcpext_TCPHystartDelayCwnd = NULL; - - static unsigned long long *tcpext_TCPACKSkippedSynRecv = NULL; - static unsigned long long *tcpext_TCPACKSkippedPAWS = NULL; - static unsigned long long *tcpext_TCPACKSkippedSeq = NULL; - static unsigned long long *tcpext_TCPACKSkippedFinWait2 = NULL; - static unsigned long long *tcpext_TCPACKSkippedTimeWait = NULL; - static unsigned long long *tcpext_TCPACKSkippedChallenge = NULL; - - static unsigned long long *tcpext_TCPWinProbe = NULL; - static unsigned long long *tcpext_TCPKeepAlive = NULL; - - static unsigned long long *tcpext_TCPMTUPFail = NULL; - static unsigned long long *tcpext_TCPMTUPSuccess = NULL; -*/ - - static unsigned long long *ipext_InNoRoutes = NULL; - static unsigned long long *ipext_InTruncatedPkts = NULL; - static unsigned long long *ipext_InMcastPkts = NULL; - static unsigned long long *ipext_OutMcastPkts = NULL; - static unsigned long long *ipext_InBcastPkts = NULL; - static unsigned long long *ipext_OutBcastPkts = NULL; - static unsigned long long *ipext_InOctets = NULL; - static unsigned long long *ipext_OutOctets = NULL; - static unsigned long long *ipext_InMcastOctets = NULL; - static unsigned long long *ipext_OutMcastOctets = NULL; - static unsigned long long *ipext_InBcastOctets = NULL; - static unsigned long long *ipext_OutBcastOctets = NULL; - static unsigned long long *ipext_InCsumErrors = NULL; - static unsigned long long *ipext_InNoECTPkts = NULL; - static unsigned long long *ipext_InECT1Pkts = NULL; - static unsigned long long *ipext_InECT0Pkts = NULL; - static unsigned long long *ipext_InCEPkts = NULL; - - if(unlikely(do_bandwidth == -1)) { do_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "bandwidth", CONFIG_ONDEMAND_ONDEMAND); do_inerrors = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "input errors", CONFIG_ONDEMAND_ONDEMAND); do_mcast = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "multicast bandwidth", CONFIG_ONDEMAND_ONDEMAND); @@ -385,200 +112,124 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { do_tcpext_connaborts = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP connection aborts", CONFIG_ONDEMAND_ONDEMAND); do_tcpext_memory = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP memory pressures", CONFIG_ONDEMAND_ONDEMAND); - hash_ipext = simple_hash("IpExt"); - hash_tcpext = simple_hash("TcpExt"); + arl_ipext = arl_create("netstat/ipext", NULL, 60); + arl_tcpext = arl_create("netstat/tcpext", NULL, 60); + + // -------------------------------------------------------------------- + // IPv4 + + if(do_bandwidth != CONFIG_ONDEMAND_NO) { + arl_expect(arl_ipext, "InOctets", &ipext_InOctets); + arl_expect(arl_ipext, "OutOctets", &ipext_OutOctets); + } + + if(do_inerrors != CONFIG_ONDEMAND_NO) { + arl_expect(arl_ipext, "InNoRoutes", &ipext_InNoRoutes); + arl_expect(arl_ipext, "InTruncatedPkts", &ipext_InTruncatedPkts); + arl_expect(arl_ipext, "InCsumErrors", &ipext_InCsumErrors); + } - hash_array(ipext_data); - hash_array(tcpext_data); - - // Reordering - tcpext_TCPFACKReorder = netstat_columns_find(tcpext_data, "TCPFACKReorder"); - tcpext_TCPSACKReorder = netstat_columns_find(tcpext_data, "TCPSACKReorder"); - tcpext_TCPRenoReorder = netstat_columns_find(tcpext_data, "TCPRenoReorder"); - tcpext_TCPTSReorder = netstat_columns_find(tcpext_data, "TCPTSReorder"); - - // SYN Cookies - tcpext_SyncookiesSent = netstat_columns_find(tcpext_data, "SyncookiesSent"); - tcpext_SyncookiesRecv = netstat_columns_find(tcpext_data, "SyncookiesRecv"); - tcpext_SyncookiesFailed = netstat_columns_find(tcpext_data, "SyncookiesFailed"); - - // Out Of Order Queue - // http://www.spinics.net/lists/netdev/msg204696.html - tcpext_TCPOFOQueue = netstat_columns_find(tcpext_data, "TCPOFOQueue"); // Number of packets queued in OFO queue - tcpext_TCPOFODrop = netstat_columns_find(tcpext_data, "TCPOFODrop"); // Number of packets meant to be queued in OFO but dropped because socket rcvbuf limit hit. - tcpext_TCPOFOMerge = netstat_columns_find(tcpext_data, "TCPOFOMerge"); // Number of packets in OFO that were merged with other packets. - tcpext_OfoPruned = netstat_columns_find(tcpext_data, "OfoPruned"); // packets dropped from out-of-order queue because of socket buffer overrun - - // connection resets - // https://github.com/ecki/net-tools/blob/bd8bceaed2311651710331a7f8990c3e31be9840/statistics.c - tcpext_TCPAbortOnData = netstat_columns_find(tcpext_data, "TCPAbortOnData"); // connections reset due to unexpected data - tcpext_TCPAbortOnClose = netstat_columns_find(tcpext_data, "TCPAbortOnClose"); // connections reset due to early user close - tcpext_TCPAbortOnMemory = netstat_columns_find(tcpext_data, "TCPAbortOnMemory"); // connections aborted due to memory pressure - tcpext_TCPAbortOnTimeout = netstat_columns_find(tcpext_data, "TCPAbortOnTimeout"); // connections aborted due to timeout - tcpext_TCPAbortOnLinger = netstat_columns_find(tcpext_data, "TCPAbortOnLinger"); // connections aborted after user close in linger timeout - tcpext_TCPAbortFailed = netstat_columns_find(tcpext_data, "TCPAbortFailed"); // times unable to send RST due to no memory - - tcpext_TCPMemoryPressures = netstat_columns_find(tcpext_data, "TCPMemoryPressures"); - - /* - tcpext_EmbryonicRsts = netstat_columns_find(tcpext_data, "EmbryonicRsts"); - tcpext_PruneCalled = netstat_columns_find(tcpext_data, "PruneCalled"); - tcpext_RcvPruned = netstat_columns_find(tcpext_data, "RcvPruned"); - tcpext_OutOfWindowIcmps = netstat_columns_find(tcpext_data, "OutOfWindowIcmps"); - tcpext_LockDroppedIcmps = netstat_columns_find(tcpext_data, "LockDroppedIcmps"); - tcpext_ArpFilter = netstat_columns_find(tcpext_data, "ArpFilter"); - tcpext_TW = netstat_columns_find(tcpext_data, "TW"); - tcpext_TWRecycled = netstat_columns_find(tcpext_data, "TWRecycled"); - tcpext_TWKilled = netstat_columns_find(tcpext_data, "TWKilled"); - tcpext_PAWSPassive = netstat_columns_find(tcpext_data, "PAWSPassive"); - tcpext_PAWSActive = netstat_columns_find(tcpext_data, "PAWSActive"); - tcpext_PAWSEstab = netstat_columns_find(tcpext_data, "PAWSEstab"); - tcpext_DelayedACKs = netstat_columns_find(tcpext_data, "DelayedACKs"); - tcpext_DelayedACKLocked = netstat_columns_find(tcpext_data, "DelayedACKLocked"); - tcpext_DelayedACKLost = netstat_columns_find(tcpext_data, "DelayedACKLost"); - tcpext_ListenOverflows = netstat_columns_find(tcpext_data, "ListenOverflows"); - tcpext_ListenDrops = netstat_columns_find(tcpext_data, "ListenDrops"); - tcpext_TCPPrequeued = netstat_columns_find(tcpext_data, "TCPPrequeued"); - tcpext_TCPDirectCopyFromBacklog = netstat_columns_find(tcpext_data, "TCPDirectCopyFromBacklog"); - tcpext_TCPDirectCopyFromPrequeue = netstat_columns_find(tcpext_data, "TCPDirectCopyFromPrequeue"); - tcpext_TCPPrequeueDropped = netstat_columns_find(tcpext_data, "TCPPrequeueDropped"); - tcpext_TCPHPHits = netstat_columns_find(tcpext_data, "TCPHPHits"); - tcpext_TCPHPHitsToUser = netstat_columns_find(tcpext_data, "TCPHPHitsToUser"); - tcpext_TCPPureAcks = netstat_columns_find(tcpext_data, "TCPPureAcks"); - tcpext_TCPHPAcks = netstat_columns_find(tcpext_data, "TCPHPAcks"); - tcpext_TCPRenoRecovery = netstat_columns_find(tcpext_data, "TCPRenoRecovery"); - tcpext_TCPSackRecovery = netstat_columns_find(tcpext_data, "TCPSackRecovery"); - tcpext_TCPSACKReneging = netstat_columns_find(tcpext_data, "TCPSACKReneging"); - tcpext_TCPFullUndo = netstat_columns_find(tcpext_data, "TCPFullUndo"); - tcpext_TCPPartialUndo = netstat_columns_find(tcpext_data, "TCPPartialUndo"); - tcpext_TCPDSACKUndo = netstat_columns_find(tcpext_data, "TCPDSACKUndo"); - tcpext_TCPLossUndo = netstat_columns_find(tcpext_data, "TCPLossUndo"); - tcpext_TCPLostRetransmit = netstat_columns_find(tcpext_data, "TCPLostRetransmit"); - tcpext_TCPRenoFailures = netstat_columns_find(tcpext_data, "TCPRenoFailures"); - tcpext_TCPSackFailures = netstat_columns_find(tcpext_data, "TCPSackFailures"); - tcpext_TCPLossFailures = netstat_columns_find(tcpext_data, "TCPLossFailures"); - tcpext_TCPFastRetrans = netstat_columns_find(tcpext_data, "TCPFastRetrans"); - tcpext_TCPForwardRetrans = netstat_columns_find(tcpext_data, "TCPForwardRetrans"); - tcpext_TCPSlowStartRetrans = netstat_columns_find(tcpext_data, "TCPSlowStartRetrans"); - tcpext_TCPTimeouts = netstat_columns_find(tcpext_data, "TCPTimeouts"); - tcpext_TCPLossProbes = netstat_columns_find(tcpext_data, "TCPLossProbes"); - tcpext_TCPLossProbeRecovery = netstat_columns_find(tcpext_data, "TCPLossProbeRecovery"); - tcpext_TCPRenoRecoveryFail = netstat_columns_find(tcpext_data, "TCPRenoRecoveryFail"); - tcpext_TCPSackRecoveryFail = netstat_columns_find(tcpext_data, "TCPSackRecoveryFail"); - tcpext_TCPSchedulerFailed = netstat_columns_find(tcpext_data, "TCPSchedulerFailed"); - tcpext_TCPRcvCollapsed = netstat_columns_find(tcpext_data, "TCPRcvCollapsed"); - tcpext_TCPDSACKOldSent = netstat_columns_find(tcpext_data, "TCPDSACKOldSent"); - tcpext_TCPDSACKOfoSent = netstat_columns_find(tcpext_data, "TCPDSACKOfoSent"); - tcpext_TCPDSACKRecv = netstat_columns_find(tcpext_data, "TCPDSACKRecv"); - tcpext_TCPDSACKOfoRecv = netstat_columns_find(tcpext_data, "TCPDSACKOfoRecv"); - tcpext_TCPSACKDiscard = netstat_columns_find(tcpext_data, "TCPSACKDiscard"); - tcpext_TCPDSACKIgnoredOld = netstat_columns_find(tcpext_data, "TCPDSACKIgnoredOld"); - tcpext_TCPDSACKIgnoredNoUndo = netstat_columns_find(tcpext_data, "TCPDSACKIgnoredNoUndo"); - tcpext_TCPSpuriousRTOs = netstat_columns_find(tcpext_data, "TCPSpuriousRTOs"); - tcpext_TCPMD5NotFound = netstat_columns_find(tcpext_data, "TCPMD5NotFound"); - tcpext_TCPMD5Unexpected = netstat_columns_find(tcpext_data, "TCPMD5Unexpected"); - tcpext_TCPSackShifted = netstat_columns_find(tcpext_data, "TCPSackShifted"); - tcpext_TCPSackMerged = netstat_columns_find(tcpext_data, "TCPSackMerged"); - tcpext_TCPSackShiftFallback = netstat_columns_find(tcpext_data, "TCPSackShiftFallback"); - tcpext_TCPBacklogDrop = netstat_columns_find(tcpext_data, "TCPBacklogDrop"); - tcpext_TCPMinTTLDrop = netstat_columns_find(tcpext_data, "TCPMinTTLDrop"); - tcpext_TCPDeferAcceptDrop = netstat_columns_find(tcpext_data, "TCPDeferAcceptDrop"); - tcpext_IPReversePathFilter = netstat_columns_find(tcpext_data, "IPReversePathFilter"); - tcpext_TCPTimeWaitOverflow = netstat_columns_find(tcpext_data, "TCPTimeWaitOverflow"); - tcpext_TCPReqQFullDoCookies = netstat_columns_find(tcpext_data, "TCPReqQFullDoCookies"); - tcpext_TCPReqQFullDrop = netstat_columns_find(tcpext_data, "TCPReqQFullDrop"); - tcpext_TCPRetransFail = netstat_columns_find(tcpext_data, "TCPRetransFail"); - tcpext_TCPRcvCoalesce = netstat_columns_find(tcpext_data, "TCPRcvCoalesce"); - tcpext_TCPChallengeACK = netstat_columns_find(tcpext_data, "TCPChallengeACK"); - tcpext_TCPSYNChallenge = netstat_columns_find(tcpext_data, "TCPSYNChallenge"); - tcpext_TCPFastOpenActive = netstat_columns_find(tcpext_data, "TCPFastOpenActive"); - tcpext_TCPFastOpenActiveFail = netstat_columns_find(tcpext_data, "TCPFastOpenActiveFail"); - tcpext_TCPFastOpenPassive = netstat_columns_find(tcpext_data, "TCPFastOpenPassive"); - tcpext_TCPFastOpenPassiveFail = netstat_columns_find(tcpext_data, "TCPFastOpenPassiveFail"); - tcpext_TCPFastOpenListenOverflow = netstat_columns_find(tcpext_data, "TCPFastOpenListenOverflow"); - tcpext_TCPFastOpenCookieReqd = netstat_columns_find(tcpext_data, "TCPFastOpenCookieReqd"); - tcpext_TCPSpuriousRtxHostQueues = netstat_columns_find(tcpext_data, "TCPSpuriousRtxHostQueues"); - tcpext_BusyPollRxPackets = netstat_columns_find(tcpext_data, "BusyPollRxPackets"); - tcpext_TCPAutoCorking = netstat_columns_find(tcpext_data, "TCPAutoCorking"); - tcpext_TCPFromZeroWindowAdv = netstat_columns_find(tcpext_data, "TCPFromZeroWindowAdv"); - tcpext_TCPToZeroWindowAdv = netstat_columns_find(tcpext_data, "TCPToZeroWindowAdv"); - tcpext_TCPWantZeroWindowAdv = netstat_columns_find(tcpext_data, "TCPWantZeroWindowAdv"); - tcpext_TCPSynRetrans = netstat_columns_find(tcpext_data, "TCPSynRetrans"); - tcpext_TCPOrigDataSent = netstat_columns_find(tcpext_data, "TCPOrigDataSent"); - tcpext_TCPHystartTrainDetect = netstat_columns_find(tcpext_data, "TCPHystartTrainDetect"); - tcpext_TCPHystartTrainCwnd = netstat_columns_find(tcpext_data, "TCPHystartTrainCwnd"); - tcpext_TCPHystartDelayDetect = netstat_columns_find(tcpext_data, "TCPHystartDelayDetect"); - tcpext_TCPHystartDelayCwnd = netstat_columns_find(tcpext_data, "TCPHystartDelayCwnd"); - tcpext_TCPACKSkippedSynRecv = netstat_columns_find(tcpext_data, "TCPACKSkippedSynRecv"); - tcpext_TCPACKSkippedPAWS = netstat_columns_find(tcpext_data, "TCPACKSkippedPAWS"); - tcpext_TCPACKSkippedSeq = netstat_columns_find(tcpext_data, "TCPACKSkippedSeq"); - tcpext_TCPACKSkippedFinWait2 = netstat_columns_find(tcpext_data, "TCPACKSkippedFinWait2"); - tcpext_TCPACKSkippedTimeWait = netstat_columns_find(tcpext_data, "TCPACKSkippedTimeWait"); - tcpext_TCPACKSkippedChallenge = netstat_columns_find(tcpext_data, "TCPACKSkippedChallenge"); - tcpext_TCPWinProbe = netstat_columns_find(tcpext_data, "TCPWinProbe"); - tcpext_TCPKeepAlive = netstat_columns_find(tcpext_data, "TCPKeepAlive"); - tcpext_TCPMTUPFail = netstat_columns_find(tcpext_data, "TCPMTUPFail"); - tcpext_TCPMTUPSuccess = netstat_columns_find(tcpext_data, "TCPMTUPSuccess"); -*/ - ipext_InNoRoutes = netstat_columns_find(ipext_data, "InNoRoutes"); - ipext_InTruncatedPkts = netstat_columns_find(ipext_data, "InTruncatedPkts"); - ipext_InMcastPkts = netstat_columns_find(ipext_data, "InMcastPkts"); - ipext_OutMcastPkts = netstat_columns_find(ipext_data, "OutMcastPkts"); - ipext_InBcastPkts = netstat_columns_find(ipext_data, "InBcastPkts"); - ipext_OutBcastPkts = netstat_columns_find(ipext_data, "OutBcastPkts"); - ipext_InOctets = netstat_columns_find(ipext_data, "InOctets"); - ipext_OutOctets = netstat_columns_find(ipext_data, "OutOctets"); - ipext_InMcastOctets = netstat_columns_find(ipext_data, "InMcastOctets"); - ipext_OutMcastOctets = netstat_columns_find(ipext_data, "OutMcastOctets"); - ipext_InBcastOctets = netstat_columns_find(ipext_data, "InBcastOctets"); - ipext_OutBcastOctets = netstat_columns_find(ipext_data, "OutBcastOctets"); - ipext_InCsumErrors = netstat_columns_find(ipext_data, "InCsumErrors"); - ipext_InNoECTPkts = netstat_columns_find(ipext_data, "InNoECTPkts"); - ipext_InECT1Pkts = netstat_columns_find(ipext_data, "InECT1Pkts"); - ipext_InECT0Pkts = netstat_columns_find(ipext_data, "InECT0Pkts"); - ipext_InCEPkts = netstat_columns_find(ipext_data, "InCEPkts"); + if(do_mcast != CONFIG_ONDEMAND_NO) { + arl_expect(arl_ipext, "InMcastOctets", &ipext_InMcastOctets); + arl_expect(arl_ipext, "OutMcastOctets", &ipext_OutMcastOctets); + } + + if(do_mcast_p != CONFIG_ONDEMAND_NO) { + arl_expect(arl_ipext, "InMcastPkts", &ipext_InMcastPkts); + arl_expect(arl_ipext, "OutMcastPkts", &ipext_OutMcastPkts); + } + + if(do_bcast != CONFIG_ONDEMAND_NO) { + arl_expect(arl_ipext, "InBcastPkts", &ipext_InBcastPkts); + arl_expect(arl_ipext, "OutBcastPkts", &ipext_OutBcastPkts); + } + + if(do_bcast_p != CONFIG_ONDEMAND_NO) { + arl_expect(arl_ipext, "InBcastOctets", &ipext_InBcastOctets); + arl_expect(arl_ipext, "OutBcastOctets", &ipext_OutBcastOctets); + } + + if(do_ecn != CONFIG_ONDEMAND_NO) { + arl_expect(arl_ipext, "InNoECTPkts", &ipext_InNoECTPkts); + arl_expect(arl_ipext, "InECT1Pkts", &ipext_InECT1Pkts); + arl_expect(arl_ipext, "InECT0Pkts", &ipext_InECT0Pkts); + arl_expect(arl_ipext, "InCEPkts", &ipext_InCEPkts); + } + + // -------------------------------------------------------------------- + // IPv4 TCP + + if(do_tcpext_reorder != CONFIG_ONDEMAND_NO) { + arl_expect(arl_tcpext, "TCPFACKReorder", &tcpext_TCPFACKReorder); + arl_expect(arl_tcpext, "TCPSACKReorder", &tcpext_TCPSACKReorder); + arl_expect(arl_tcpext, "TCPRenoReorder", &tcpext_TCPRenoReorder); + arl_expect(arl_tcpext, "TCPTSReorder", &tcpext_TCPTSReorder); + } + + if(do_tcpext_syscookies != CONFIG_ONDEMAND_NO) { + arl_expect(arl_tcpext, "SyncookiesSent", &tcpext_SyncookiesSent); + arl_expect(arl_tcpext, "SyncookiesRecv", &tcpext_SyncookiesRecv); + arl_expect(arl_tcpext, "SyncookiesFailed", &tcpext_SyncookiesFailed); + } + + if(do_tcpext_ofo != CONFIG_ONDEMAND_NO) { + arl_expect(arl_tcpext, "TCPOFOQueue", &tcpext_TCPOFOQueue); + arl_expect(arl_tcpext, "TCPOFODrop", &tcpext_TCPOFODrop); + arl_expect(arl_tcpext, "TCPOFOMerge", &tcpext_TCPOFOMerge); + arl_expect(arl_tcpext, "OfoPruned", &tcpext_OfoPruned); + } + + if(do_tcpext_connaborts != CONFIG_ONDEMAND_NO) { + arl_expect(arl_tcpext, "TCPAbortOnData", &tcpext_TCPAbortOnData); + arl_expect(arl_tcpext, "TCPAbortOnClose", &tcpext_TCPAbortOnClose); + arl_expect(arl_tcpext, "TCPAbortOnMemory", &tcpext_TCPAbortOnMemory); + arl_expect(arl_tcpext, "TCPAbortOnTimeout", &tcpext_TCPAbortOnTimeout); + arl_expect(arl_tcpext, "TCPAbortOnLinger", &tcpext_TCPAbortOnLinger); + arl_expect(arl_tcpext, "TCPAbortFailed", &tcpext_TCPAbortFailed); + } + + if(do_tcpext_memory != CONFIG_ONDEMAND_NO) { + arl_expect(arl_tcpext, "TCPMemoryPressures", &tcpext_TCPMemoryPressures); + } } if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/netstat"); ff = procfile_open(config_get("plugin:proc:/proc/net/netstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; } - if(unlikely(!ff)) return 1; ff = procfile_readall(ff); if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + size_t lines = procfile_lines(ff), l; + size_t words; for(l = 0; l < lines ;l++) { char *key = procfile_lineword(ff, l, 0); uint32_t hash = simple_hash(key); if(unlikely(hash == hash_ipext && strcmp(key, "IpExt") == 0)) { - uint32_t h = l++; + size_t h = l++; - if(strcmp(procfile_lineword(ff, l, 0), "IpExt") != 0) { - error("Cannot read IpExt line from /proc/net/netstat."); - break; - } words = procfile_linewords(ff, l); - if(words < 2) { - error("Cannot read /proc/net/netstat IpExt line. Expected 2+ params, read %u.", words); + if(unlikely(words < 2)) { + error("Cannot read /proc/net/netstat IpExt line. Expected 2+ params, read %zu.", words); continue; } - parse_line_pair(ff, ipext_data, h, l); + arl_begin(arl_ipext); + parse_line_pair(ff, arl_ipext, h, l); RRDSET *st; // -------------------------------------------------------------------- - if(do_bandwidth == CONFIG_ONDEMAND_YES || (do_bandwidth == CONFIG_ONDEMAND_ONDEMAND && (*ipext_InOctets || *ipext_OutOctets))) { + if(do_bandwidth == CONFIG_ONDEMAND_YES || (do_bandwidth == CONFIG_ONDEMAND_ONDEMAND && (ipext_InOctets || ipext_OutOctets))) { do_bandwidth = CONFIG_ONDEMAND_YES; st = rrdset_find("system.ipv4"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("system", "ipv4", NULL, "network", NULL, "IPv4 Bandwidth", "kilobits/s", 500, update_every, RRDSET_TYPE_AREA); rrddim_add(st, "InOctets", "received", 8, 1024, RRDDIM_INCREMENTAL); @@ -586,17 +237,17 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "InOctets", *ipext_InOctets); - rrddim_set(st, "OutOctets", *ipext_OutOctets); + rrddim_set(st, "InOctets", ipext_InOctets); + rrddim_set(st, "OutOctets", ipext_OutOctets); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_inerrors == CONFIG_ONDEMAND_YES || (do_inerrors == CONFIG_ONDEMAND_ONDEMAND && (*ipext_InNoRoutes || *ipext_InTruncatedPkts))) { + if(do_inerrors == CONFIG_ONDEMAND_YES || (do_inerrors == CONFIG_ONDEMAND_ONDEMAND && (ipext_InNoRoutes || ipext_InTruncatedPkts))) { do_inerrors = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.inerrors"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "inerrors", NULL, "errors", NULL, "IPv4 Input Errors", "packets/s", 4000, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -606,18 +257,18 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "InNoRoutes", *ipext_InNoRoutes); - rrddim_set(st, "InTruncatedPkts", *ipext_InTruncatedPkts); - rrddim_set(st, "InCsumErrors", *ipext_InCsumErrors); + rrddim_set(st, "InNoRoutes", ipext_InNoRoutes); + rrddim_set(st, "InTruncatedPkts", ipext_InTruncatedPkts); + rrddim_set(st, "InCsumErrors", ipext_InCsumErrors); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_mcast == CONFIG_ONDEMAND_YES || (do_mcast == CONFIG_ONDEMAND_ONDEMAND && (*ipext_InMcastOctets || *ipext_OutMcastOctets))) { + if(do_mcast == CONFIG_ONDEMAND_YES || (do_mcast == CONFIG_ONDEMAND_ONDEMAND && (ipext_InMcastOctets || ipext_OutMcastOctets))) { do_mcast = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.mcast"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "mcast", NULL, "multicast", NULL, "IPv4 Multicast Bandwidth", "kilobits/s", 9000, update_every, RRDSET_TYPE_AREA); st->isdetail = 1; @@ -626,17 +277,17 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "InMcastOctets", *ipext_InMcastOctets); - rrddim_set(st, "OutMcastOctets", *ipext_OutMcastOctets); + rrddim_set(st, "InMcastOctets", ipext_InMcastOctets); + rrddim_set(st, "OutMcastOctets", ipext_OutMcastOctets); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_bcast == CONFIG_ONDEMAND_YES || (do_bcast == CONFIG_ONDEMAND_ONDEMAND && (*ipext_InBcastOctets || *ipext_OutBcastOctets))) { + if(do_bcast == CONFIG_ONDEMAND_YES || (do_bcast == CONFIG_ONDEMAND_ONDEMAND && (ipext_InBcastOctets || ipext_OutBcastOctets))) { do_bcast = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.bcast"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "bcast", NULL, "broadcast", NULL, "IPv4 Broadcast Bandwidth", "kilobits/s", 8000, update_every, RRDSET_TYPE_AREA); st->isdetail = 1; @@ -645,17 +296,17 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "InBcastOctets", *ipext_InBcastOctets); - rrddim_set(st, "OutBcastOctets", *ipext_OutBcastOctets); + rrddim_set(st, "InBcastOctets", ipext_InBcastOctets); + rrddim_set(st, "OutBcastOctets", ipext_OutBcastOctets); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_mcast_p == CONFIG_ONDEMAND_YES || (do_mcast_p == CONFIG_ONDEMAND_ONDEMAND && (*ipext_InMcastPkts || *ipext_OutMcastPkts))) { + if(do_mcast_p == CONFIG_ONDEMAND_YES || (do_mcast_p == CONFIG_ONDEMAND_ONDEMAND && (ipext_InMcastPkts || ipext_OutMcastPkts))) { do_mcast_p = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.mcastpkts"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "mcastpkts", NULL, "multicast", NULL, "IPv4 Multicast Packets", "packets/s", 8600, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -664,17 +315,17 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "InMcastPkts", *ipext_InMcastPkts); - rrddim_set(st, "OutMcastPkts", *ipext_OutMcastPkts); + rrddim_set(st, "InMcastPkts", ipext_InMcastPkts); + rrddim_set(st, "OutMcastPkts", ipext_OutMcastPkts); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_bcast_p == CONFIG_ONDEMAND_YES || (do_bcast_p == CONFIG_ONDEMAND_ONDEMAND && (*ipext_InBcastPkts || *ipext_OutBcastPkts))) { + if(do_bcast_p == CONFIG_ONDEMAND_YES || (do_bcast_p == CONFIG_ONDEMAND_ONDEMAND && (ipext_InBcastPkts || ipext_OutBcastPkts))) { do_bcast_p = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.bcastpkts"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "bcastpkts", NULL, "broadcast", NULL, "IPv4 Broadcast Packets", "packets/s", 8500, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -683,17 +334,17 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "InBcastPkts", *ipext_InBcastPkts); - rrddim_set(st, "OutBcastPkts", *ipext_OutBcastPkts); + rrddim_set(st, "InBcastPkts", ipext_InBcastPkts); + rrddim_set(st, "OutBcastPkts", ipext_OutBcastPkts); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_ecn == CONFIG_ONDEMAND_YES || (do_ecn == CONFIG_ONDEMAND_ONDEMAND && (*ipext_InCEPkts || *ipext_InECT0Pkts || *ipext_InECT1Pkts || *ipext_InNoECTPkts))) { + if(do_ecn == CONFIG_ONDEMAND_YES || (do_ecn == CONFIG_ONDEMAND_ONDEMAND && (ipext_InCEPkts || ipext_InECT0Pkts || ipext_InECT1Pkts || ipext_InNoECTPkts))) { do_ecn = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.ecnpkts"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "ecnpkts", NULL, "ecn", NULL, "IPv4 ECN Statistics", "packets/s", 8700, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -704,52 +355,49 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "InCEPkts", *ipext_InCEPkts); - rrddim_set(st, "InNoECTPkts", *ipext_InNoECTPkts); - rrddim_set(st, "InECT0Pkts", *ipext_InECT0Pkts); - rrddim_set(st, "InECT1Pkts", *ipext_InECT1Pkts); + rrddim_set(st, "InCEPkts", ipext_InCEPkts); + rrddim_set(st, "InNoECTPkts", ipext_InNoECTPkts); + rrddim_set(st, "InECT0Pkts", ipext_InECT0Pkts); + rrddim_set(st, "InECT1Pkts", ipext_InECT1Pkts); rrdset_done(st); } } else if(unlikely(hash == hash_tcpext && strcmp(key, "TcpExt") == 0)) { - uint32_t h = l++; + size_t h = l++; - if(strcmp(procfile_lineword(ff, l, 0), "TcpExt") != 0) { - error("Cannot read TcpExt line from /proc/net/netstat."); - break; - } words = procfile_linewords(ff, l); - if(words < 2) { - error("Cannot read /proc/net/netstat TcpExt line. Expected 2+ params, read %u.", words); + if(unlikely(words < 2)) { + error("Cannot read /proc/net/netstat TcpExt line. Expected 2+ params, read %zu.", words); continue; } - parse_line_pair(ff, tcpext_data, h, l); + arl_begin(arl_tcpext); + parse_line_pair(ff, arl_tcpext, h, l); RRDSET *st; // -------------------------------------------------------------------- - if(do_tcpext_memory == CONFIG_ONDEMAND_YES || (do_tcpext_memory == CONFIG_ONDEMAND_ONDEMAND && (*tcpext_TCPMemoryPressures))) { + if(do_tcpext_memory == CONFIG_ONDEMAND_YES || (do_tcpext_memory == CONFIG_ONDEMAND_ONDEMAND && (tcpext_TCPMemoryPressures))) { do_tcpext_memory = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.tcpmemorypressures"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "tcpmemorypressures", NULL, "tcp", NULL, "TCP Memory Pressures", "events/s", 3000, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "TCPMemoryPressures", "pressures", 1, 1, RRDDIM_INCREMENTAL); } else rrdset_next(st); - rrddim_set(st, "TCPMemoryPressures", *tcpext_TCPMemoryPressures); + rrddim_set(st, "TCPMemoryPressures", tcpext_TCPMemoryPressures); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_tcpext_connaborts == CONFIG_ONDEMAND_YES || (do_tcpext_connaborts == CONFIG_ONDEMAND_ONDEMAND && (*tcpext_TCPAbortOnData || *tcpext_TCPAbortOnClose || *tcpext_TCPAbortOnMemory || *tcpext_TCPAbortOnTimeout || *tcpext_TCPAbortOnLinger || *tcpext_TCPAbortFailed))) { + if(do_tcpext_connaborts == CONFIG_ONDEMAND_YES || (do_tcpext_connaborts == CONFIG_ONDEMAND_ONDEMAND && (tcpext_TCPAbortOnData || tcpext_TCPAbortOnClose || tcpext_TCPAbortOnMemory || tcpext_TCPAbortOnTimeout || tcpext_TCPAbortOnLinger || tcpext_TCPAbortFailed))) { do_tcpext_connaborts = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.tcpconnaborts"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "tcpconnaborts", NULL, "tcp", NULL, "TCP Connection Aborts", "connections/s", 3010, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "TCPAbortOnData", "baddata", 1, 1, RRDDIM_INCREMENTAL); @@ -761,20 +409,20 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "TCPAbortOnData", *tcpext_TCPAbortOnData); - rrddim_set(st, "TCPAbortOnClose", *tcpext_TCPAbortOnClose); - rrddim_set(st, "TCPAbortOnMemory", *tcpext_TCPAbortOnMemory); - rrddim_set(st, "TCPAbortOnTimeout", *tcpext_TCPAbortOnTimeout); - rrddim_set(st, "TCPAbortOnLinger", *tcpext_TCPAbortOnLinger); - rrddim_set(st, "TCPAbortFailed", *tcpext_TCPAbortFailed); + rrddim_set(st, "TCPAbortOnData", tcpext_TCPAbortOnData); + rrddim_set(st, "TCPAbortOnClose", tcpext_TCPAbortOnClose); + rrddim_set(st, "TCPAbortOnMemory", tcpext_TCPAbortOnMemory); + rrddim_set(st, "TCPAbortOnTimeout", tcpext_TCPAbortOnTimeout); + rrddim_set(st, "TCPAbortOnLinger", tcpext_TCPAbortOnLinger); + rrddim_set(st, "TCPAbortFailed", tcpext_TCPAbortFailed); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_tcpext_reorder == CONFIG_ONDEMAND_YES || (do_tcpext_reorder == CONFIG_ONDEMAND_ONDEMAND && (*tcpext_TCPRenoReorder || *tcpext_TCPFACKReorder || *tcpext_TCPSACKReorder || *tcpext_TCPTSReorder))) { + if(do_tcpext_reorder == CONFIG_ONDEMAND_YES || (do_tcpext_reorder == CONFIG_ONDEMAND_ONDEMAND && (tcpext_TCPRenoReorder || tcpext_TCPFACKReorder || tcpext_TCPSACKReorder || tcpext_TCPTSReorder))) { do_tcpext_reorder = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.tcpreorders"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "tcpreorders", NULL, "tcp", NULL, "TCP Reordered Packets by Detection Method", "packets/s", 3020, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "TCPTSReorder", "timestamp", 1, 1, RRDDIM_INCREMENTAL); @@ -784,19 +432,19 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "TCPTSReorder", *tcpext_TCPTSReorder); - rrddim_set(st, "TCPSACKReorder", *tcpext_TCPSACKReorder); - rrddim_set(st, "TCPFACKReorder", *tcpext_TCPFACKReorder); - rrddim_set(st, "TCPRenoReorder", *tcpext_TCPRenoReorder); + rrddim_set(st, "TCPTSReorder", tcpext_TCPTSReorder); + rrddim_set(st, "TCPSACKReorder", tcpext_TCPSACKReorder); + rrddim_set(st, "TCPFACKReorder", tcpext_TCPFACKReorder); + rrddim_set(st, "TCPRenoReorder", tcpext_TCPRenoReorder); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_tcpext_ofo == CONFIG_ONDEMAND_YES || (do_tcpext_ofo == CONFIG_ONDEMAND_ONDEMAND && (*tcpext_TCPOFOQueue || *tcpext_TCPOFODrop || *tcpext_TCPOFOMerge))) { + if(do_tcpext_ofo == CONFIG_ONDEMAND_YES || (do_tcpext_ofo == CONFIG_ONDEMAND_ONDEMAND && (tcpext_TCPOFOQueue || tcpext_TCPOFODrop || tcpext_TCPOFOMerge))) { do_tcpext_ofo = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.tcpofo"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "tcpofo", NULL, "tcp", NULL, "TCP Out-Of-Order Queue", "packets/s", 3050, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "TCPOFOQueue", "inqueue", 1, 1, RRDDIM_INCREMENTAL); @@ -806,19 +454,19 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "TCPOFOQueue", *tcpext_TCPOFOQueue); - rrddim_set(st, "TCPOFODrop", *tcpext_TCPOFODrop); - rrddim_set(st, "TCPOFOMerge", *tcpext_TCPOFOMerge); - rrddim_set(st, "OfoPruned", *tcpext_OfoPruned); + rrddim_set(st, "TCPOFOQueue", tcpext_TCPOFOQueue); + rrddim_set(st, "TCPOFODrop", tcpext_TCPOFODrop); + rrddim_set(st, "TCPOFOMerge", tcpext_TCPOFOMerge); + rrddim_set(st, "OfoPruned", tcpext_OfoPruned); rrdset_done(st); } // -------------------------------------------------------------------- - if(do_tcpext_syscookies == CONFIG_ONDEMAND_YES || (do_tcpext_syscookies == CONFIG_ONDEMAND_ONDEMAND && (*tcpext_SyncookiesSent || *tcpext_SyncookiesRecv || *tcpext_SyncookiesFailed))) { + if(do_tcpext_syscookies == CONFIG_ONDEMAND_YES || (do_tcpext_syscookies == CONFIG_ONDEMAND_ONDEMAND && (tcpext_SyncookiesSent || tcpext_SyncookiesRecv || tcpext_SyncookiesFailed))) { do_tcpext_syscookies = CONFIG_ONDEMAND_YES; st = rrdset_find("ipv4.tcpsyncookies"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("ipv4", "tcpsyncookies", NULL, "tcp", NULL, "TCP SYN Cookies", "packets/s", 3100, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "SyncookiesRecv", "received", 1, 1, RRDDIM_INCREMENTAL); @@ -827,9 +475,9 @@ int do_proc_net_netstat(int update_every, unsigned long long dt) { } else rrdset_next(st); - rrddim_set(st, "SyncookiesRecv", *tcpext_SyncookiesRecv); - rrddim_set(st, "SyncookiesSent", *tcpext_SyncookiesSent); - rrddim_set(st, "SyncookiesFailed", *tcpext_SyncookiesFailed); + rrddim_set(st, "SyncookiesRecv", tcpext_SyncookiesRecv); + rrddim_set(st, "SyncookiesSent", tcpext_SyncookiesSent); + rrddim_set(st, "SyncookiesFailed", tcpext_SyncookiesFailed); rrdset_done(st); } diff --git a/src/proc_net_rpc_nfs.c b/src/proc_net_rpc_nfs.c index 98acdd814..9dba08d56 100644 --- a/src/proc_net_rpc_nfs.c +++ b/src/proc_net_rpc_nfs.c @@ -127,7 +127,7 @@ struct nfs_procs nfs_proc4_values[] = { { "", 0ULL, 0 } }; -int do_proc_net_rpc_nfs(int update_every, unsigned long long dt) { +int do_proc_net_rpc_nfs(int update_every, usec_t dt) { (void)dt; static procfile *ff = NULL; @@ -151,36 +151,35 @@ int do_proc_net_rpc_nfs(int update_every, unsigned long long dt) { if(do_proc4 == -1) do_proc4 = config_get_boolean("plugin:proc:/proc/net/rpc/nfs", "NFS v4 procedures", 1); // if they are enabled, reset them to 1 - // later we do them =2 to avoid doing strcmp for all lines + // later we do them =2 to avoid doing strcmp() for all lines if(do_net) do_net = 1; if(do_rpc) do_rpc = 1; if(do_proc2) do_proc2 = 1; if(do_proc3) do_proc3 = 1; if(do_proc4) do_proc4 = 1; - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + size_t lines = procfile_lines(ff), l; char *type; unsigned long long net_count = 0, net_udp_count = 0, net_tcp_count = 0, net_tcp_connections = 0; unsigned long long rpc_calls = 0, rpc_retransmits = 0, rpc_auth_refresh = 0; for(l = 0; l < lines ;l++) { - words = procfile_linewords(ff, l); + size_t words = procfile_linewords(ff, l); if(!words) continue; type = procfile_lineword(ff, l, 0); if(do_net == 1 && strcmp(type, "net") == 0) { if(words < 5) { - error("%s line of /proc/net/rpc/nfs has %u words, expected %d", type, words, 5); + error("%s line of /proc/net/rpc/nfs has %zu words, expected %d", type, words, 5); continue; } - net_count = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - net_udp_count = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - net_tcp_count = strtoull(procfile_lineword(ff, l, 3), NULL, 10); - net_tcp_connections = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + net_count = str2ull(procfile_lineword(ff, l, 1)); + net_udp_count = str2ull(procfile_lineword(ff, l, 2)); + net_tcp_count = str2ull(procfile_lineword(ff, l, 3)); + net_tcp_connections = str2ull(procfile_lineword(ff, l, 4)); unsigned long long sum = net_count + net_udp_count + net_tcp_count + net_tcp_connections; if(sum == 0ULL) do_net = -1; @@ -188,13 +187,13 @@ int do_proc_net_rpc_nfs(int update_every, unsigned long long dt) { } else if(do_rpc == 1 && strcmp(type, "rpc") == 0) { if(words < 4) { - error("%s line of /proc/net/rpc/nfs has %u words, expected %d", type, words, 6); + error("%s line of /proc/net/rpc/nfs has %zu words, expected %d", type, words, 6); continue; } - rpc_calls = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - rpc_retransmits = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - rpc_auth_refresh = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + rpc_calls = str2ull(procfile_lineword(ff, l, 1)); + rpc_retransmits = str2ull(procfile_lineword(ff, l, 2)); + rpc_auth_refresh = str2ull(procfile_lineword(ff, l, 3)); unsigned long long sum = rpc_calls + rpc_retransmits + rpc_auth_refresh; if(sum == 0ULL) do_rpc = -1; @@ -207,7 +206,7 @@ int do_proc_net_rpc_nfs(int update_every, unsigned long long dt) { unsigned long long sum = 0; unsigned int i, j; for(i = 0, j = 2; j < words && nfs_proc2_values[i].name[0] ; i++, j++) { - nfs_proc2_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfs_proc2_values[i].value = str2ull(procfile_lineword(ff, l, j)); nfs_proc2_values[i].present = 1; sum += nfs_proc2_values[i].value; } @@ -228,7 +227,7 @@ int do_proc_net_rpc_nfs(int update_every, unsigned long long dt) { unsigned long long sum = 0; unsigned int i, j; for(i = 0, j = 2; j < words && nfs_proc3_values[i].name[0] ; i++, j++) { - nfs_proc3_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfs_proc3_values[i].value = str2ull(procfile_lineword(ff, l, j)); nfs_proc3_values[i].present = 1; sum += nfs_proc3_values[i].value; } @@ -249,7 +248,7 @@ int do_proc_net_rpc_nfs(int update_every, unsigned long long dt) { unsigned long long sum = 0; unsigned int i, j; for(i = 0, j = 2; j < words && nfs_proc4_values[i].name[0] ; i++, j++) { - nfs_proc4_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfs_proc4_values[i].value = str2ull(procfile_lineword(ff, l, j)); nfs_proc4_values[i].present = 1; sum += nfs_proc4_values[i].value; } diff --git a/src/proc_net_rpc_nfsd.c b/src/proc_net_rpc_nfsd.c index 817e6c86a..02a8c8f90 100644 --- a/src/proc_net_rpc_nfsd.c +++ b/src/proc_net_rpc_nfsd.c @@ -209,7 +209,7 @@ struct nfsd_procs nfsd4_ops_values[] = { }; -int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { +int do_proc_net_rpc_nfsd(int update_every, usec_t dt) { static procfile *ff = NULL; static int do_rc = -1, do_fh = -1, do_io = -1, do_th = -1, do_ra = -1, do_net = -1, do_rpc = -1, do_proc2 = -1, do_proc3 = -1, do_proc4 = -1, do_proc4ops = -1; static int ra_warning = 0, th_warning = 0, proc2_warning = 0, proc3_warning = 0, proc4_warning = 0, proc4ops_warning = 0; @@ -239,7 +239,7 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { if(do_proc4ops == -1) do_proc4ops = config_get_boolean("plugin:proc:/proc/net/rpc/nfsd", "NFS v4 operations", 1); // if they are enabled, reset them to 1 - // later we do them =2 to avoid doing strcmp for all lines + // later we do them =2 to avoid doing strcmp() for all lines if(do_rc) do_rc = 1; if(do_fh) do_fh = 1; if(do_io) do_io = 1; @@ -252,8 +252,7 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { if(do_proc4) do_proc4 = 1; if(do_proc4ops) do_proc4ops = 1; - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + size_t lines = procfile_lines(ff), l; char *type; unsigned long long rc_hits = 0, rc_misses = 0, rc_nocache = 0; @@ -265,20 +264,20 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { unsigned long long rpc_calls = 0, rpc_bad_format = 0, rpc_bad_auth = 0, rpc_bad_client = 0; for(l = 0; l < lines ;l++) { - words = procfile_linewords(ff, l); + size_t words = procfile_linewords(ff, l); if(!words) continue; type = procfile_lineword(ff, l, 0); if(do_rc == 1 && strcmp(type, "rc") == 0) { if(words < 4) { - error("%s line of /proc/net/rpc/nfsd has %u words, expected %d", type, words, 4); + error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 4); continue; } - rc_hits = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - rc_misses = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - rc_nocache = strtoull(procfile_lineword(ff, l, 3), NULL, 10); + rc_hits = str2ull(procfile_lineword(ff, l, 1)); + rc_misses = str2ull(procfile_lineword(ff, l, 2)); + rc_nocache = str2ull(procfile_lineword(ff, l, 3)); unsigned long long sum = rc_hits + rc_misses + rc_nocache; if(sum == 0ULL) do_rc = -1; @@ -286,15 +285,15 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { } else if(do_fh == 1 && strcmp(type, "fh") == 0) { if(words < 6) { - error("%s line of /proc/net/rpc/nfsd has %u words, expected %d", type, words, 6); + error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 6); continue; } - fh_stale = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - fh_total_lookups = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - fh_anonymous_lookups = strtoull(procfile_lineword(ff, l, 3), NULL, 10); - fh_dir_not_in_dcache = strtoull(procfile_lineword(ff, l, 4), NULL, 10); - fh_non_dir_not_in_dcache = strtoull(procfile_lineword(ff, l, 5), NULL, 10); + fh_stale = str2ull(procfile_lineword(ff, l, 1)); + fh_total_lookups = str2ull(procfile_lineword(ff, l, 2)); + fh_anonymous_lookups = str2ull(procfile_lineword(ff, l, 3)); + fh_dir_not_in_dcache = str2ull(procfile_lineword(ff, l, 4)); + fh_non_dir_not_in_dcache = str2ull(procfile_lineword(ff, l, 5)); unsigned long long sum = fh_stale + fh_total_lookups + fh_anonymous_lookups + fh_dir_not_in_dcache + fh_non_dir_not_in_dcache; if(sum == 0ULL) do_fh = -1; @@ -302,12 +301,12 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { } else if(do_io == 1 && strcmp(type, "io") == 0) { if(words < 3) { - error("%s line of /proc/net/rpc/nfsd has %u words, expected %d", type, words, 3); + error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 3); continue; } - io_read = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - io_write = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + io_read = str2ull(procfile_lineword(ff, l, 1)); + io_write = str2ull(procfile_lineword(ff, l, 2)); unsigned long long sum = io_read + io_write; if(sum == 0ULL) do_io = -1; @@ -315,12 +314,12 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { } else if(do_th == 1 && strcmp(type, "th") == 0) { if(words < 13) { - error("%s line of /proc/net/rpc/nfsd has %u words, expected %d", type, words, 13); + error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 13); continue; } - th_threads = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - th_fullcnt = strtoull(procfile_lineword(ff, l, 2), NULL, 10); + th_threads = str2ull(procfile_lineword(ff, l, 1)); + th_fullcnt = str2ull(procfile_lineword(ff, l, 2)); th_hist10 = (unsigned long long)(atof(procfile_lineword(ff, l, 3)) * 1000.0); th_hist20 = (unsigned long long)(atof(procfile_lineword(ff, l, 4)) * 1000.0); th_hist30 = (unsigned long long)(atof(procfile_lineword(ff, l, 5)) * 1000.0); @@ -346,22 +345,22 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { } else if(do_ra == 1 && strcmp(type, "ra") == 0) { if(words < 13) { - error("%s line of /proc/net/rpc/nfsd has %u words, expected %d", type, words, 13); + error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 13); continue; } - ra_size = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - ra_hist10 = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - ra_hist20 = strtoull(procfile_lineword(ff, l, 3), NULL, 10); - ra_hist30 = strtoull(procfile_lineword(ff, l, 4), NULL, 10); - ra_hist40 = strtoull(procfile_lineword(ff, l, 5), NULL, 10); - ra_hist50 = strtoull(procfile_lineword(ff, l, 6), NULL, 10); - ra_hist60 = strtoull(procfile_lineword(ff, l, 7), NULL, 10); - ra_hist70 = strtoull(procfile_lineword(ff, l, 8), NULL, 10); - ra_hist80 = strtoull(procfile_lineword(ff, l, 9), NULL, 10); - ra_hist90 = strtoull(procfile_lineword(ff, l, 10), NULL, 10); - ra_hist100 = strtoull(procfile_lineword(ff, l, 11), NULL, 10); - ra_none = strtoull(procfile_lineword(ff, l, 12), NULL, 10); + ra_size = str2ull(procfile_lineword(ff, l, 1)); + ra_hist10 = str2ull(procfile_lineword(ff, l, 2)); + ra_hist20 = str2ull(procfile_lineword(ff, l, 3)); + ra_hist30 = str2ull(procfile_lineword(ff, l, 4)); + ra_hist40 = str2ull(procfile_lineword(ff, l, 5)); + ra_hist50 = str2ull(procfile_lineword(ff, l, 6)); + ra_hist60 = str2ull(procfile_lineword(ff, l, 7)); + ra_hist70 = str2ull(procfile_lineword(ff, l, 8)); + ra_hist80 = str2ull(procfile_lineword(ff, l, 9)); + ra_hist90 = str2ull(procfile_lineword(ff, l, 10)); + ra_hist100 = str2ull(procfile_lineword(ff, l, 11)); + ra_none = str2ull(procfile_lineword(ff, l, 12)); unsigned long long sum = ra_hist10 + ra_hist20 + ra_hist30 + ra_hist40 + ra_hist50 + ra_hist60 + ra_hist70 + ra_hist80 + ra_hist90 + ra_hist100 + ra_none; if(sum == 0ULL) { @@ -375,14 +374,14 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { } else if(do_net == 1 && strcmp(type, "net") == 0) { if(words < 5) { - error("%s line of /proc/net/rpc/nfsd has %u words, expected %d", type, words, 5); + error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 5); continue; } - net_count = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - net_udp_count = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - net_tcp_count = strtoull(procfile_lineword(ff, l, 3), NULL, 10); - net_tcp_connections = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + net_count = str2ull(procfile_lineword(ff, l, 1)); + net_udp_count = str2ull(procfile_lineword(ff, l, 2)); + net_tcp_count = str2ull(procfile_lineword(ff, l, 3)); + net_tcp_connections = str2ull(procfile_lineword(ff, l, 4)); unsigned long long sum = net_count + net_udp_count + net_tcp_count + net_tcp_connections; if(sum == 0ULL) do_net = -1; @@ -390,14 +389,14 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { } else if(do_rpc == 1 && strcmp(type, "rpc") == 0) { if(words < 6) { - error("%s line of /proc/net/rpc/nfsd has %u words, expected %d", type, words, 6); + error("%s line of /proc/net/rpc/nfsd has %zu words, expected %d", type, words, 6); continue; } - rpc_calls = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - rpc_bad_format = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - rpc_bad_auth = strtoull(procfile_lineword(ff, l, 3), NULL, 10); - rpc_bad_client = strtoull(procfile_lineword(ff, l, 4), NULL, 10); + rpc_calls = str2ull(procfile_lineword(ff, l, 1)); + rpc_bad_format = str2ull(procfile_lineword(ff, l, 2)); + rpc_bad_auth = str2ull(procfile_lineword(ff, l, 3)); + rpc_bad_client = str2ull(procfile_lineword(ff, l, 4)); unsigned long long sum = rpc_calls + rpc_bad_format + rpc_bad_auth + rpc_bad_client; if(sum == 0ULL) do_rpc = -1; @@ -410,7 +409,7 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { unsigned long long sum = 0; unsigned int i, j; for(i = 0, j = 2; j < words && nfsd_proc2_values[i].name[0] ; i++, j++) { - nfsd_proc2_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd_proc2_values[i].value = str2ull(procfile_lineword(ff, l, j)); nfsd_proc2_values[i].present = 1; sum += nfsd_proc2_values[i].value; } @@ -431,7 +430,7 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { unsigned long long sum = 0; unsigned int i, j; for(i = 0, j = 2; j < words && nfsd_proc3_values[i].name[0] ; i++, j++) { - nfsd_proc3_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd_proc3_values[i].value = str2ull(procfile_lineword(ff, l, j)); nfsd_proc3_values[i].present = 1; sum += nfsd_proc3_values[i].value; } @@ -452,7 +451,7 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { unsigned long long sum = 0; unsigned int i, j; for(i = 0, j = 2; j < words && nfsd_proc4_values[i].name[0] ; i++, j++) { - nfsd_proc4_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd_proc4_values[i].value = str2ull(procfile_lineword(ff, l, j)); nfsd_proc4_values[i].present = 1; sum += nfsd_proc4_values[i].value; } @@ -473,7 +472,7 @@ int do_proc_net_rpc_nfsd(int update_every, unsigned long long dt) { unsigned long long sum = 0; unsigned int i, j; for(i = 0, j = 2; j < words && nfsd4_ops_values[i].name[0] ; i++, j++) { - nfsd4_ops_values[i].value = strtoull(procfile_lineword(ff, l, j), NULL, 10); + nfsd4_ops_values[i].value = str2ull(procfile_lineword(ff, l, j)); nfsd4_ops_values[i].present = 1; sum += nfsd4_ops_values[i].value; } diff --git a/src/proc_net_snmp.c b/src/proc_net_snmp.c index a75c0a96a..cd5c250ae 100644 --- a/src/proc_net_snmp.c +++ b/src/proc_net_snmp.c @@ -1,9 +1,6 @@ #include "common.h" #define RRD_TYPE_NET_SNMP "ipv4" -#define RRD_TYPE_NET_SNMP_LEN strlen(RRD_TYPE_NET_SNMP) - -#define NETSTAT_PRESENT 0x00000001 struct netstat_columns { char *name; @@ -174,13 +171,13 @@ static unsigned long long *netstat_columns_find(struct netstat_columns *nc, cons fatal("Cannot find key '%s' in /proc/net/snmp internal array.", name); } -static void parse_line_pair(procfile *ff, struct netstat_columns *nc, uint32_t header_line, uint32_t values_line) { - uint32_t hwords = procfile_linewords(ff, header_line); - uint32_t vwords = procfile_linewords(ff, values_line); - uint32_t w, i; +static void parse_line_pair(procfile *ff, struct netstat_columns *nc, size_t header_line, size_t values_line) { + size_t hwords = procfile_linewords(ff, header_line); + size_t vwords = procfile_linewords(ff, values_line); + size_t w, i; if(unlikely(vwords > hwords)) { - error("File /proc/net/snmp on header line %u has %u words, but on value line %u has %u words.", header_line, hwords, values_line, vwords); + error("File /proc/net/snmp on header line %zu has %zu words, but on value line %zu has %zu words.", header_line, hwords, values_line, vwords); vwords = hwords; } @@ -190,14 +187,14 @@ static void parse_line_pair(procfile *ff, struct netstat_columns *nc, uint32_t h for(i = 0 ; nc[i].name ;i++) { if(unlikely(hash == nc[i].hash && !strcmp(key, nc[i].name))) { - nc[i].value = strtoull(procfile_lineword(ff, values_line, w), NULL, 10); + nc[i].value = str2ull(procfile_lineword(ff, values_line, w)); break; } } } } -int do_proc_net_snmp(int update_every, unsigned long long dt) { +int do_proc_net_snmp(int update_every, usec_t dt) { (void)dt; static procfile *ff = NULL; @@ -360,14 +357,14 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/snmp"); ff = procfile_open(config_get("plugin:proc:/proc/net/snmp", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; } - if(unlikely(!ff)) return 1; ff = procfile_readall(ff); if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + size_t lines = procfile_lines(ff), l; + size_t words; RRDSET *st; @@ -376,7 +373,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { uint32_t hash = simple_hash(key); if(unlikely(hash == hash_ip && strcmp(key, "Ip") == 0)) { - uint32_t h = l++; + size_t h = l++; if(strcmp(procfile_lineword(ff, l, 0), "Ip") != 0) { error("Cannot read Ip line from /proc/net/snmp."); @@ -385,7 +382,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { words = procfile_linewords(ff, l); if(words < 3) { - error("Cannot read /proc/net/snmp Ip line. Expected 3+ params, read %u.", words); + error("Cannot read /proc/net/snmp Ip line. Expected 3+ params, read %zu.", words); continue; } @@ -482,7 +479,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { } } else if(unlikely(hash == hash_icmp && strcmp(key, "Icmp") == 0)) { - uint32_t h = l++; + size_t h = l++; if(strcmp(procfile_lineword(ff, l, 0), "Icmp") != 0) { error("Cannot read Icmp line from /proc/net/snmp."); @@ -491,7 +488,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { words = procfile_linewords(ff, l); if(words < 3) { - error("Cannot read /proc/net/snmp Icmp line. Expected 3+ params, read %u.", words); + error("Cannot read /proc/net/snmp Icmp line. Expected 3+ params, read %zu.", words); continue; } @@ -532,7 +529,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { } } else if(unlikely(hash == hash_icmpmsg && strcmp(key, "IcmpMsg") == 0)) { - uint32_t h = l++; + size_t h = l++; if(strcmp(procfile_lineword(ff, l, 0), "IcmpMsg") != 0) { error("Cannot read IcmpMsg line from /proc/net/snmp."); @@ -562,7 +559,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { } } else if(unlikely(hash == hash_tcp && strcmp(key, "Tcp") == 0)) { - uint32_t h = l++; + size_t h = l++; if(strcmp(procfile_lineword(ff, l, 0), "Tcp") != 0) { error("Cannot read Tcp line from /proc/net/snmp."); @@ -571,7 +568,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { words = procfile_linewords(ff, l); if(words < 3) { - error("Cannot read /proc/net/snmp Tcp line. Expected 3+ params, read %u.", words); + error("Cannot read /proc/net/snmp Tcp line. Expected 3+ params, read %zu.", words); continue; } @@ -655,7 +652,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { } } else if(unlikely(hash == hash_udp && strcmp(key, "Udp") == 0)) { - uint32_t h = l++; + size_t h = l++; if(strcmp(procfile_lineword(ff, l, 0), "Udp") != 0) { error("Cannot read Udp line from /proc/net/snmp."); @@ -664,7 +661,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { words = procfile_linewords(ff, l); if(words < 3) { - error("Cannot read /proc/net/snmp Udp line. Expected 3+ params, read %u.", words); + error("Cannot read /proc/net/snmp Udp line. Expected 3+ params, read %zu.", words); continue; } @@ -715,7 +712,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { } } else if(unlikely(hash == hash_udplite && strcmp(key, "UdpLite") == 0)) { - uint32_t h = l++; + size_t h = l++; if(strcmp(procfile_lineword(ff, l, 0), "UdpLite") != 0) { error("Cannot read UdpLite line from /proc/net/snmp."); @@ -724,7 +721,7 @@ int do_proc_net_snmp(int update_every, unsigned long long dt) { words = procfile_linewords(ff, l); if(words < 3) { - error("Cannot read /proc/net/snmp UdpLite line. Expected 3+ params, read %u.", words); + error("Cannot read /proc/net/snmp UdpLite line. Expected 3+ params, read %zu.", words); continue; } diff --git a/src/proc_net_snmp6.c b/src/proc_net_snmp6.c index 97dc20edd..51d7121a1 100644 --- a/src/proc_net_snmp6.c +++ b/src/proc_net_snmp6.c @@ -1,456 +1,275 @@ #include "common.h" #define RRD_TYPE_NET_SNMP6 "ipv6" -#define RRD_TYPE_NET_SNMP6_LEN strlen(RRD_TYPE_NET_SNMP6) -int do_proc_net_snmp6(int update_every, unsigned long long dt) { +int do_proc_net_snmp6(int update_every, usec_t dt) { + (void)dt; + static procfile *ff = NULL; - static int gen_hashes = -1; - - static int do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1, - do_udplite_packets = -1, do_udplite_errors = -1, - do_udp_packets = -1, do_udp_errors = -1, - do_bandwidth = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, - do_icmp = -1, do_icmp_redir = -1, do_icmp_errors = -1, do_icmp_echos = -1, do_icmp_groupmemb = -1, - do_icmp_router = -1, do_icmp_neighbor = -1, do_icmp_mldv2 = -1, do_icmp_types = -1, do_ect = -1; - - static uint32_t hash_Ip6InReceives = -1; - - static uint32_t hash_Ip6InHdrErrors = -1; - static uint32_t hash_Ip6InTooBigErrors = -1; - static uint32_t hash_Ip6InNoRoutes = -1; - static uint32_t hash_Ip6InAddrErrors = -1; - static uint32_t hash_Ip6InUnknownProtos = -1; - static uint32_t hash_Ip6InTruncatedPkts = -1; - static uint32_t hash_Ip6InDiscards = -1; - static uint32_t hash_Ip6InDelivers = -1; - - static uint32_t hash_Ip6OutForwDatagrams = -1; - static uint32_t hash_Ip6OutRequests = -1; - static uint32_t hash_Ip6OutDiscards = -1; - static uint32_t hash_Ip6OutNoRoutes = -1; - - static uint32_t hash_Ip6ReasmTimeout = -1; - static uint32_t hash_Ip6ReasmReqds = -1; - static uint32_t hash_Ip6ReasmOKs = -1; - static uint32_t hash_Ip6ReasmFails = -1; - - static uint32_t hash_Ip6FragOKs = -1; - static uint32_t hash_Ip6FragFails = -1; - static uint32_t hash_Ip6FragCreates = -1; - - static uint32_t hash_Ip6InMcastPkts = -1; - static uint32_t hash_Ip6OutMcastPkts = -1; - - static uint32_t hash_Ip6InOctets = -1; - static uint32_t hash_Ip6OutOctets = -1; - - static uint32_t hash_Ip6InMcastOctets = -1; - static uint32_t hash_Ip6OutMcastOctets = -1; - static uint32_t hash_Ip6InBcastOctets = -1; - static uint32_t hash_Ip6OutBcastOctets = -1; - - static uint32_t hash_Ip6InNoECTPkts = -1; - static uint32_t hash_Ip6InECT1Pkts = -1; - static uint32_t hash_Ip6InECT0Pkts = -1; - static uint32_t hash_Ip6InCEPkts = -1; - - static uint32_t hash_Icmp6InMsgs = -1; - static uint32_t hash_Icmp6InErrors = -1; - static uint32_t hash_Icmp6OutMsgs = -1; - static uint32_t hash_Icmp6OutErrors = -1; - static uint32_t hash_Icmp6InCsumErrors = -1; - static uint32_t hash_Icmp6InDestUnreachs = -1; - static uint32_t hash_Icmp6InPktTooBigs = -1; - static uint32_t hash_Icmp6InTimeExcds = -1; - static uint32_t hash_Icmp6InParmProblems = -1; - static uint32_t hash_Icmp6InEchos = -1; - static uint32_t hash_Icmp6InEchoReplies = -1; - static uint32_t hash_Icmp6InGroupMembQueries = -1; - static uint32_t hash_Icmp6InGroupMembResponses = -1; - static uint32_t hash_Icmp6InGroupMembReductions = -1; - static uint32_t hash_Icmp6InRouterSolicits = -1; - static uint32_t hash_Icmp6InRouterAdvertisements = -1; - static uint32_t hash_Icmp6InNeighborSolicits = -1; - static uint32_t hash_Icmp6InNeighborAdvertisements = -1; - static uint32_t hash_Icmp6InRedirects = -1; - static uint32_t hash_Icmp6InMLDv2Reports = -1; - static uint32_t hash_Icmp6OutDestUnreachs = -1; - static uint32_t hash_Icmp6OutPktTooBigs = -1; - static uint32_t hash_Icmp6OutTimeExcds = -1; - static uint32_t hash_Icmp6OutParmProblems = -1; - static uint32_t hash_Icmp6OutEchos = -1; - static uint32_t hash_Icmp6OutEchoReplies = -1; - static uint32_t hash_Icmp6OutGroupMembQueries = -1; - static uint32_t hash_Icmp6OutGroupMembResponses = -1; - static uint32_t hash_Icmp6OutGroupMembReductions = -1; - static uint32_t hash_Icmp6OutRouterSolicits = -1; - static uint32_t hash_Icmp6OutRouterAdvertisements = -1; - static uint32_t hash_Icmp6OutNeighborSolicits = -1; - static uint32_t hash_Icmp6OutNeighborAdvertisements = -1; - static uint32_t hash_Icmp6OutRedirects = -1; - static uint32_t hash_Icmp6OutMLDv2Reports = -1; - static uint32_t hash_Icmp6InType1 = -1; - static uint32_t hash_Icmp6InType128 = -1; - static uint32_t hash_Icmp6InType129 = -1; - static uint32_t hash_Icmp6InType136 = -1; - static uint32_t hash_Icmp6OutType1 = -1; - static uint32_t hash_Icmp6OutType128 = -1; - static uint32_t hash_Icmp6OutType129 = -1; - static uint32_t hash_Icmp6OutType133 = -1; - static uint32_t hash_Icmp6OutType135 = -1; - static uint32_t hash_Icmp6OutType143 = -1; - - static uint32_t hash_Udp6InDatagrams = -1; - static uint32_t hash_Udp6NoPorts = -1; - static uint32_t hash_Udp6InErrors = -1; - static uint32_t hash_Udp6OutDatagrams = -1; - static uint32_t hash_Udp6RcvbufErrors = -1; - static uint32_t hash_Udp6SndbufErrors = -1; - static uint32_t hash_Udp6InCsumErrors = -1; - static uint32_t hash_Udp6IgnoredMulti = -1; - - static uint32_t hash_UdpLite6InDatagrams = -1; - static uint32_t hash_UdpLite6NoPorts = -1; - static uint32_t hash_UdpLite6InErrors = -1; - static uint32_t hash_UdpLite6OutDatagrams = -1; - static uint32_t hash_UdpLite6RcvbufErrors = -1; - static uint32_t hash_UdpLite6SndbufErrors = -1; - static uint32_t hash_UdpLite6InCsumErrors = -1; - - if(gen_hashes != 1) { - gen_hashes = 1; - hash_Ip6InReceives = simple_hash("Ip6InReceives"); - hash_Ip6InHdrErrors = simple_hash("Ip6InHdrErrors"); - hash_Ip6InTooBigErrors = simple_hash("Ip6InTooBigErrors"); - hash_Ip6InNoRoutes = simple_hash("Ip6InNoRoutes"); - hash_Ip6InAddrErrors = simple_hash("Ip6InAddrErrors"); - hash_Ip6InUnknownProtos = simple_hash("Ip6InUnknownProtos"); - hash_Ip6InTruncatedPkts = simple_hash("Ip6InTruncatedPkts"); - hash_Ip6InDiscards = simple_hash("Ip6InDiscards"); - hash_Ip6InDelivers = simple_hash("Ip6InDelivers"); - hash_Ip6OutForwDatagrams = simple_hash("Ip6OutForwDatagrams"); - hash_Ip6OutRequests = simple_hash("Ip6OutRequests"); - hash_Ip6OutDiscards = simple_hash("Ip6OutDiscards"); - hash_Ip6OutNoRoutes = simple_hash("Ip6OutNoRoutes"); - hash_Ip6ReasmTimeout = simple_hash("Ip6ReasmTimeout"); - hash_Ip6ReasmReqds = simple_hash("Ip6ReasmReqds"); - hash_Ip6ReasmOKs = simple_hash("Ip6ReasmOKs"); - hash_Ip6ReasmFails = simple_hash("Ip6ReasmFails"); - hash_Ip6FragOKs = simple_hash("Ip6FragOKs"); - hash_Ip6FragFails = simple_hash("Ip6FragFails"); - hash_Ip6FragCreates = simple_hash("Ip6FragCreates"); - hash_Ip6InMcastPkts = simple_hash("Ip6InMcastPkts"); - hash_Ip6OutMcastPkts = simple_hash("Ip6OutMcastPkts"); - hash_Ip6InOctets = simple_hash("Ip6InOctets"); - hash_Ip6OutOctets = simple_hash("Ip6OutOctets"); - hash_Ip6InMcastOctets = simple_hash("Ip6InMcastOctets"); - hash_Ip6OutMcastOctets = simple_hash("Ip6OutMcastOctets"); - hash_Ip6InBcastOctets = simple_hash("Ip6InBcastOctets"); - hash_Ip6OutBcastOctets = simple_hash("Ip6OutBcastOctets"); - hash_Ip6InNoECTPkts = simple_hash("Ip6InNoECTPkts"); - hash_Ip6InECT1Pkts = simple_hash("Ip6InECT1Pkts"); - hash_Ip6InECT0Pkts = simple_hash("Ip6InECT0Pkts"); - hash_Ip6InCEPkts = simple_hash("Ip6InCEPkts"); - hash_Icmp6InMsgs = simple_hash("Icmp6InMsgs"); - hash_Icmp6InErrors = simple_hash("Icmp6InErrors"); - hash_Icmp6OutMsgs = simple_hash("Icmp6OutMsgs"); - hash_Icmp6OutErrors = simple_hash("Icmp6OutErrors"); - hash_Icmp6InCsumErrors = simple_hash("Icmp6InCsumErrors"); - hash_Icmp6InDestUnreachs = simple_hash("Icmp6InDestUnreachs"); - hash_Icmp6InPktTooBigs = simple_hash("Icmp6InPktTooBigs"); - hash_Icmp6InTimeExcds = simple_hash("Icmp6InTimeExcds"); - hash_Icmp6InParmProblems = simple_hash("Icmp6InParmProblems"); - hash_Icmp6InEchos = simple_hash("Icmp6InEchos"); - hash_Icmp6InEchoReplies = simple_hash("Icmp6InEchoReplies"); - hash_Icmp6InGroupMembQueries = simple_hash("Icmp6InGroupMembQueries"); - hash_Icmp6InGroupMembResponses = simple_hash("Icmp6InGroupMembResponses"); - hash_Icmp6InGroupMembReductions = simple_hash("Icmp6InGroupMembReductions"); - hash_Icmp6InRouterSolicits = simple_hash("Icmp6InRouterSolicits"); - hash_Icmp6InRouterAdvertisements = simple_hash("Icmp6InRouterAdvertisements"); - hash_Icmp6InNeighborSolicits = simple_hash("Icmp6InNeighborSolicits"); - hash_Icmp6InNeighborAdvertisements = simple_hash("Icmp6InNeighborAdvertisements"); - hash_Icmp6InRedirects = simple_hash("Icmp6InRedirects"); - hash_Icmp6InMLDv2Reports = simple_hash("Icmp6InMLDv2Reports"); - hash_Icmp6OutDestUnreachs = simple_hash("Icmp6OutDestUnreachs"); - hash_Icmp6OutPktTooBigs = simple_hash("Icmp6OutPktTooBigs"); - hash_Icmp6OutTimeExcds = simple_hash("Icmp6OutTimeExcds"); - hash_Icmp6OutParmProblems = simple_hash("Icmp6OutParmProblems"); - hash_Icmp6OutEchos = simple_hash("Icmp6OutEchos"); - hash_Icmp6OutEchoReplies = simple_hash("Icmp6OutEchoReplies"); - hash_Icmp6OutGroupMembQueries = simple_hash("Icmp6OutGroupMembQueries"); - hash_Icmp6OutGroupMembResponses = simple_hash("Icmp6OutGroupMembResponses"); - hash_Icmp6OutGroupMembReductions = simple_hash("Icmp6OutGroupMembReductions"); - hash_Icmp6OutRouterSolicits = simple_hash("Icmp6OutRouterSolicits"); - hash_Icmp6OutRouterAdvertisements = simple_hash("Icmp6OutRouterAdvertisements"); - hash_Icmp6OutNeighborSolicits = simple_hash("Icmp6OutNeighborSolicits"); - hash_Icmp6OutNeighborAdvertisements = simple_hash("Icmp6OutNeighborAdvertisements"); - hash_Icmp6OutRedirects = simple_hash("Icmp6OutRedirects"); - hash_Icmp6OutMLDv2Reports = simple_hash("Icmp6OutMLDv2Reports"); - hash_Icmp6InType1 = simple_hash("Icmp6InType1"); - hash_Icmp6InType128 = simple_hash("Icmp6InType128"); - hash_Icmp6InType129 = simple_hash("Icmp6InType129"); - hash_Icmp6InType136 = simple_hash("Icmp6InType136"); - hash_Icmp6OutType1 = simple_hash("Icmp6OutType1"); - hash_Icmp6OutType128 = simple_hash("Icmp6OutType128"); - hash_Icmp6OutType129 = simple_hash("Icmp6OutType129"); - hash_Icmp6OutType133 = simple_hash("Icmp6OutType133"); - hash_Icmp6OutType135 = simple_hash("Icmp6OutType135"); - hash_Icmp6OutType143 = simple_hash("Icmp6OutType143"); - hash_Udp6InDatagrams = simple_hash("Udp6InDatagrams"); - hash_Udp6NoPorts = simple_hash("Udp6NoPorts"); - hash_Udp6InErrors = simple_hash("Udp6InErrors"); - hash_Udp6OutDatagrams = simple_hash("Udp6OutDatagrams"); - hash_Udp6RcvbufErrors = simple_hash("Udp6RcvbufErrors"); - hash_Udp6SndbufErrors = simple_hash("Udp6SndbufErrors"); - hash_Udp6InCsumErrors = simple_hash("Udp6InCsumErrors"); - hash_Udp6IgnoredMulti = simple_hash("Udp6IgnoredMulti"); - hash_UdpLite6InDatagrams = simple_hash("UdpLite6InDatagrams"); - hash_UdpLite6NoPorts = simple_hash("UdpLite6NoPorts"); - hash_UdpLite6InErrors = simple_hash("UdpLite6InErrors"); - hash_UdpLite6OutDatagrams = simple_hash("UdpLite6OutDatagrams"); - hash_UdpLite6RcvbufErrors = simple_hash("UdpLite6RcvbufErrors"); - hash_UdpLite6SndbufErrors = simple_hash("UdpLite6SndbufErrors"); - hash_UdpLite6InCsumErrors = simple_hash("UdpLite6InCsumErrors"); + + static int do_ip_packets = -1, + do_ip_fragsout = -1, + do_ip_fragsin = -1, + do_ip_errors = -1, + do_udplite_packets = -1, + do_udplite_errors = -1, + do_udp_packets = -1, + do_udp_errors = -1, + do_bandwidth = -1, + do_mcast = -1, + do_bcast = -1, + do_mcast_p = -1, + do_icmp = -1, + do_icmp_redir = -1, + do_icmp_errors = -1, + do_icmp_echos = -1, + do_icmp_groupmemb = -1, + do_icmp_router = -1, + do_icmp_neighbor = -1, + do_icmp_mldv2 = -1, + do_icmp_types = -1, + do_ect = -1; + + static ARL_BASE *arl_base = NULL; + + static unsigned long long Ip6InReceives = 0ULL; + static unsigned long long Ip6InHdrErrors = 0ULL; + static unsigned long long Ip6InTooBigErrors = 0ULL; + static unsigned long long Ip6InNoRoutes = 0ULL; + static unsigned long long Ip6InAddrErrors = 0ULL; + static unsigned long long Ip6InUnknownProtos = 0ULL; + static unsigned long long Ip6InTruncatedPkts = 0ULL; + static unsigned long long Ip6InDiscards = 0ULL; + static unsigned long long Ip6InDelivers = 0ULL; + static unsigned long long Ip6OutForwDatagrams = 0ULL; + static unsigned long long Ip6OutRequests = 0ULL; + static unsigned long long Ip6OutDiscards = 0ULL; + static unsigned long long Ip6OutNoRoutes = 0ULL; + static unsigned long long Ip6ReasmTimeout = 0ULL; + static unsigned long long Ip6ReasmReqds = 0ULL; + static unsigned long long Ip6ReasmOKs = 0ULL; + static unsigned long long Ip6ReasmFails = 0ULL; + static unsigned long long Ip6FragOKs = 0ULL; + static unsigned long long Ip6FragFails = 0ULL; + static unsigned long long Ip6FragCreates = 0ULL; + static unsigned long long Ip6InMcastPkts = 0ULL; + static unsigned long long Ip6OutMcastPkts = 0ULL; + static unsigned long long Ip6InOctets = 0ULL; + static unsigned long long Ip6OutOctets = 0ULL; + static unsigned long long Ip6InMcastOctets = 0ULL; + static unsigned long long Ip6OutMcastOctets = 0ULL; + static unsigned long long Ip6InBcastOctets = 0ULL; + static unsigned long long Ip6OutBcastOctets = 0ULL; + static unsigned long long Ip6InNoECTPkts = 0ULL; + static unsigned long long Ip6InECT1Pkts = 0ULL; + static unsigned long long Ip6InECT0Pkts = 0ULL; + static unsigned long long Ip6InCEPkts = 0ULL; + static unsigned long long Icmp6InMsgs = 0ULL; + static unsigned long long Icmp6InErrors = 0ULL; + static unsigned long long Icmp6OutMsgs = 0ULL; + static unsigned long long Icmp6OutErrors = 0ULL; + static unsigned long long Icmp6InCsumErrors = 0ULL; + static unsigned long long Icmp6InDestUnreachs = 0ULL; + static unsigned long long Icmp6InPktTooBigs = 0ULL; + static unsigned long long Icmp6InTimeExcds = 0ULL; + static unsigned long long Icmp6InParmProblems = 0ULL; + static unsigned long long Icmp6InEchos = 0ULL; + static unsigned long long Icmp6InEchoReplies = 0ULL; + static unsigned long long Icmp6InGroupMembQueries = 0ULL; + static unsigned long long Icmp6InGroupMembResponses = 0ULL; + static unsigned long long Icmp6InGroupMembReductions = 0ULL; + static unsigned long long Icmp6InRouterSolicits = 0ULL; + static unsigned long long Icmp6InRouterAdvertisements = 0ULL; + static unsigned long long Icmp6InNeighborSolicits = 0ULL; + static unsigned long long Icmp6InNeighborAdvertisements = 0ULL; + static unsigned long long Icmp6InRedirects = 0ULL; + static unsigned long long Icmp6InMLDv2Reports = 0ULL; + static unsigned long long Icmp6OutDestUnreachs = 0ULL; + static unsigned long long Icmp6OutPktTooBigs = 0ULL; + static unsigned long long Icmp6OutTimeExcds = 0ULL; + static unsigned long long Icmp6OutParmProblems = 0ULL; + static unsigned long long Icmp6OutEchos = 0ULL; + static unsigned long long Icmp6OutEchoReplies = 0ULL; + static unsigned long long Icmp6OutGroupMembQueries = 0ULL; + static unsigned long long Icmp6OutGroupMembResponses = 0ULL; + static unsigned long long Icmp6OutGroupMembReductions = 0ULL; + static unsigned long long Icmp6OutRouterSolicits = 0ULL; + static unsigned long long Icmp6OutRouterAdvertisements = 0ULL; + static unsigned long long Icmp6OutNeighborSolicits = 0ULL; + static unsigned long long Icmp6OutNeighborAdvertisements = 0ULL; + static unsigned long long Icmp6OutRedirects = 0ULL; + static unsigned long long Icmp6OutMLDv2Reports = 0ULL; + static unsigned long long Icmp6InType1 = 0ULL; + static unsigned long long Icmp6InType128 = 0ULL; + static unsigned long long Icmp6InType129 = 0ULL; + static unsigned long long Icmp6InType136 = 0ULL; + static unsigned long long Icmp6OutType1 = 0ULL; + static unsigned long long Icmp6OutType128 = 0ULL; + static unsigned long long Icmp6OutType129 = 0ULL; + static unsigned long long Icmp6OutType133 = 0ULL; + static unsigned long long Icmp6OutType135 = 0ULL; + static unsigned long long Icmp6OutType143 = 0ULL; + static unsigned long long Udp6InDatagrams = 0ULL; + static unsigned long long Udp6NoPorts = 0ULL; + static unsigned long long Udp6InErrors = 0ULL; + static unsigned long long Udp6OutDatagrams = 0ULL; + static unsigned long long Udp6RcvbufErrors = 0ULL; + static unsigned long long Udp6SndbufErrors = 0ULL; + static unsigned long long Udp6InCsumErrors = 0ULL; + static unsigned long long Udp6IgnoredMulti = 0ULL; + static unsigned long long UdpLite6InDatagrams = 0ULL; + static unsigned long long UdpLite6NoPorts = 0ULL; + static unsigned long long UdpLite6InErrors = 0ULL; + static unsigned long long UdpLite6OutDatagrams = 0ULL; + static unsigned long long UdpLite6RcvbufErrors = 0ULL; + static unsigned long long UdpLite6SndbufErrors = 0ULL; + static unsigned long long UdpLite6InCsumErrors = 0ULL; + + if(unlikely(!arl_base)) { + do_ip_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 packets", CONFIG_ONDEMAND_ONDEMAND); + do_ip_fragsout = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 fragments sent", CONFIG_ONDEMAND_ONDEMAND); + do_ip_fragsin = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 fragments assembly", CONFIG_ONDEMAND_ONDEMAND); + do_ip_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 errors", CONFIG_ONDEMAND_ONDEMAND); + do_udp_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDP packets", CONFIG_ONDEMAND_ONDEMAND); + do_udp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDP errors", CONFIG_ONDEMAND_ONDEMAND); + do_udplite_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDPlite packets", CONFIG_ONDEMAND_ONDEMAND); + do_udplite_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDPlite errors", CONFIG_ONDEMAND_ONDEMAND); + do_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "bandwidth", CONFIG_ONDEMAND_ONDEMAND); + do_mcast = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "multicast bandwidth", CONFIG_ONDEMAND_ONDEMAND); + do_bcast = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "broadcast bandwidth", CONFIG_ONDEMAND_ONDEMAND); + do_mcast_p = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "multicast packets", CONFIG_ONDEMAND_ONDEMAND); + do_icmp = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_redir = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp redirects", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp errors", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_echos = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp echos", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_groupmemb = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp group membership", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_router = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp router", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_neighbor = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp neighbor", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_mldv2 = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp mldv2", CONFIG_ONDEMAND_ONDEMAND); + do_icmp_types = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp types", CONFIG_ONDEMAND_ONDEMAND); + do_ect = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ect", CONFIG_ONDEMAND_ONDEMAND); + + arl_base = arl_create("snmp6", NULL, 60); + arl_expect(arl_base, "Ip6InReceives", &Ip6InReceives); + arl_expect(arl_base, "Ip6InHdrErrors", &Ip6InHdrErrors); + arl_expect(arl_base, "Ip6InTooBigErrors", &Ip6InTooBigErrors); + arl_expect(arl_base, "Ip6InNoRoutes", &Ip6InNoRoutes); + arl_expect(arl_base, "Ip6InAddrErrors", &Ip6InAddrErrors); + arl_expect(arl_base, "Ip6InUnknownProtos", &Ip6InUnknownProtos); + arl_expect(arl_base, "Ip6InTruncatedPkts", &Ip6InTruncatedPkts); + arl_expect(arl_base, "Ip6InDiscards", &Ip6InDiscards); + arl_expect(arl_base, "Ip6InDelivers", &Ip6InDelivers); + arl_expect(arl_base, "Ip6OutForwDatagrams", &Ip6OutForwDatagrams); + arl_expect(arl_base, "Ip6OutRequests", &Ip6OutRequests); + arl_expect(arl_base, "Ip6OutDiscards", &Ip6OutDiscards); + arl_expect(arl_base, "Ip6OutNoRoutes", &Ip6OutNoRoutes); + arl_expect(arl_base, "Ip6ReasmTimeout", &Ip6ReasmTimeout); + arl_expect(arl_base, "Ip6ReasmReqds", &Ip6ReasmReqds); + arl_expect(arl_base, "Ip6ReasmOKs", &Ip6ReasmOKs); + arl_expect(arl_base, "Ip6ReasmFails", &Ip6ReasmFails); + arl_expect(arl_base, "Ip6FragOKs", &Ip6FragOKs); + arl_expect(arl_base, "Ip6FragFails", &Ip6FragFails); + arl_expect(arl_base, "Ip6FragCreates", &Ip6FragCreates); + arl_expect(arl_base, "Ip6InMcastPkts", &Ip6InMcastPkts); + arl_expect(arl_base, "Ip6OutMcastPkts", &Ip6OutMcastPkts); + arl_expect(arl_base, "Ip6InOctets", &Ip6InOctets); + arl_expect(arl_base, "Ip6OutOctets", &Ip6OutOctets); + arl_expect(arl_base, "Ip6InMcastOctets", &Ip6InMcastOctets); + arl_expect(arl_base, "Ip6OutMcastOctets", &Ip6OutMcastOctets); + arl_expect(arl_base, "Ip6InBcastOctets", &Ip6InBcastOctets); + arl_expect(arl_base, "Ip6OutBcastOctets", &Ip6OutBcastOctets); + arl_expect(arl_base, "Ip6InNoECTPkts", &Ip6InNoECTPkts); + arl_expect(arl_base, "Ip6InECT1Pkts", &Ip6InECT1Pkts); + arl_expect(arl_base, "Ip6InECT0Pkts", &Ip6InECT0Pkts); + arl_expect(arl_base, "Ip6InCEPkts", &Ip6InCEPkts); + arl_expect(arl_base, "Icmp6InMsgs", &Icmp6InMsgs); + arl_expect(arl_base, "Icmp6InErrors", &Icmp6InErrors); + arl_expect(arl_base, "Icmp6OutMsgs", &Icmp6OutMsgs); + arl_expect(arl_base, "Icmp6OutErrors", &Icmp6OutErrors); + arl_expect(arl_base, "Icmp6InCsumErrors", &Icmp6InCsumErrors); + arl_expect(arl_base, "Icmp6InDestUnreachs", &Icmp6InDestUnreachs); + arl_expect(arl_base, "Icmp6InPktTooBigs", &Icmp6InPktTooBigs); + arl_expect(arl_base, "Icmp6InTimeExcds", &Icmp6InTimeExcds); + arl_expect(arl_base, "Icmp6InParmProblems", &Icmp6InParmProblems); + arl_expect(arl_base, "Icmp6InEchos", &Icmp6InEchos); + arl_expect(arl_base, "Icmp6InEchoReplies", &Icmp6InEchoReplies); + arl_expect(arl_base, "Icmp6InGroupMembQueries", &Icmp6InGroupMembQueries); + arl_expect(arl_base, "Icmp6InGroupMembResponses", &Icmp6InGroupMembResponses); + arl_expect(arl_base, "Icmp6InGroupMembReductions", &Icmp6InGroupMembReductions); + arl_expect(arl_base, "Icmp6InRouterSolicits", &Icmp6InRouterSolicits); + arl_expect(arl_base, "Icmp6InRouterAdvertisements", &Icmp6InRouterAdvertisements); + arl_expect(arl_base, "Icmp6InNeighborSolicits", &Icmp6InNeighborSolicits); + arl_expect(arl_base, "Icmp6InNeighborAdvertisements", &Icmp6InNeighborAdvertisements); + arl_expect(arl_base, "Icmp6InRedirects", &Icmp6InRedirects); + arl_expect(arl_base, "Icmp6InMLDv2Reports", &Icmp6InMLDv2Reports); + arl_expect(arl_base, "Icmp6OutDestUnreachs", &Icmp6OutDestUnreachs); + arl_expect(arl_base, "Icmp6OutPktTooBigs", &Icmp6OutPktTooBigs); + arl_expect(arl_base, "Icmp6OutTimeExcds", &Icmp6OutTimeExcds); + arl_expect(arl_base, "Icmp6OutParmProblems", &Icmp6OutParmProblems); + arl_expect(arl_base, "Icmp6OutEchos", &Icmp6OutEchos); + arl_expect(arl_base, "Icmp6OutEchoReplies", &Icmp6OutEchoReplies); + arl_expect(arl_base, "Icmp6OutGroupMembQueries", &Icmp6OutGroupMembQueries); + arl_expect(arl_base, "Icmp6OutGroupMembResponses", &Icmp6OutGroupMembResponses); + arl_expect(arl_base, "Icmp6OutGroupMembReductions", &Icmp6OutGroupMembReductions); + arl_expect(arl_base, "Icmp6OutRouterSolicits", &Icmp6OutRouterSolicits); + arl_expect(arl_base, "Icmp6OutRouterAdvertisements", &Icmp6OutRouterAdvertisements); + arl_expect(arl_base, "Icmp6OutNeighborSolicits", &Icmp6OutNeighborSolicits); + arl_expect(arl_base, "Icmp6OutNeighborAdvertisements", &Icmp6OutNeighborAdvertisements); + arl_expect(arl_base, "Icmp6OutRedirects", &Icmp6OutRedirects); + arl_expect(arl_base, "Icmp6OutMLDv2Reports", &Icmp6OutMLDv2Reports); + arl_expect(arl_base, "Icmp6InType1", &Icmp6InType1); + arl_expect(arl_base, "Icmp6InType128", &Icmp6InType128); + arl_expect(arl_base, "Icmp6InType129", &Icmp6InType129); + arl_expect(arl_base, "Icmp6InType136", &Icmp6InType136); + arl_expect(arl_base, "Icmp6OutType1", &Icmp6OutType1); + arl_expect(arl_base, "Icmp6OutType128", &Icmp6OutType128); + arl_expect(arl_base, "Icmp6OutType129", &Icmp6OutType129); + arl_expect(arl_base, "Icmp6OutType133", &Icmp6OutType133); + arl_expect(arl_base, "Icmp6OutType135", &Icmp6OutType135); + arl_expect(arl_base, "Icmp6OutType143", &Icmp6OutType143); + arl_expect(arl_base, "Udp6InDatagrams", &Udp6InDatagrams); + arl_expect(arl_base, "Udp6NoPorts", &Udp6NoPorts); + arl_expect(arl_base, "Udp6InErrors", &Udp6InErrors); + arl_expect(arl_base, "Udp6OutDatagrams", &Udp6OutDatagrams); + arl_expect(arl_base, "Udp6RcvbufErrors", &Udp6RcvbufErrors); + arl_expect(arl_base, "Udp6SndbufErrors", &Udp6SndbufErrors); + arl_expect(arl_base, "Udp6InCsumErrors", &Udp6InCsumErrors); + arl_expect(arl_base, "Udp6IgnoredMulti", &Udp6IgnoredMulti); + arl_expect(arl_base, "UdpLite6InDatagrams", &UdpLite6InDatagrams); + arl_expect(arl_base, "UdpLite6NoPorts", &UdpLite6NoPorts); + arl_expect(arl_base, "UdpLite6InErrors", &UdpLite6InErrors); + arl_expect(arl_base, "UdpLite6OutDatagrams", &UdpLite6OutDatagrams); + arl_expect(arl_base, "UdpLite6RcvbufErrors", &UdpLite6RcvbufErrors); + arl_expect(arl_base, "UdpLite6SndbufErrors", &UdpLite6SndbufErrors); + arl_expect(arl_base, "UdpLite6InCsumErrors", &UdpLite6InCsumErrors); } - if(do_ip_packets == -1) do_ip_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 packets", CONFIG_ONDEMAND_ONDEMAND); - if(do_ip_fragsout == -1) do_ip_fragsout = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 fragments sent", CONFIG_ONDEMAND_ONDEMAND); - if(do_ip_fragsin == -1) do_ip_fragsin = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 fragments assembly", CONFIG_ONDEMAND_ONDEMAND); - if(do_ip_errors == -1) do_ip_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 errors", CONFIG_ONDEMAND_ONDEMAND); - if(do_udp_packets == -1) do_udp_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDP packets", CONFIG_ONDEMAND_ONDEMAND); - if(do_udp_errors == -1) do_udp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDP errors", CONFIG_ONDEMAND_ONDEMAND); - if(do_udplite_packets == -1) do_udplite_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDPlite packets", CONFIG_ONDEMAND_ONDEMAND); - if(do_udplite_errors == -1) do_udplite_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ipv6 UDPlite errors", CONFIG_ONDEMAND_ONDEMAND); - if(do_bandwidth == -1) do_bandwidth = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "bandwidth", CONFIG_ONDEMAND_ONDEMAND); - if(do_mcast == -1) do_mcast = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "multicast bandwidth", CONFIG_ONDEMAND_ONDEMAND); - if(do_bcast == -1) do_bcast = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "broadcast bandwidth", CONFIG_ONDEMAND_ONDEMAND); - if(do_mcast_p == -1) do_mcast_p = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "multicast packets", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp == -1) do_icmp = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_redir == -1) do_icmp_redir = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp redirects", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_errors == -1) do_icmp_errors = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp errors", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_echos == -1) do_icmp_echos = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp echos", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_groupmemb == -1) do_icmp_groupmemb = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp group membership", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_router == -1) do_icmp_router = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp router", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_neighbor == -1) do_icmp_neighbor = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp neighbor", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_mldv2 == -1) do_icmp_mldv2 = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp mldv2", CONFIG_ONDEMAND_ONDEMAND); - if(do_icmp_types == -1) do_icmp_types = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "icmp types", CONFIG_ONDEMAND_ONDEMAND); - if(do_ect == -1) do_ect = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp6", "ect", CONFIG_ONDEMAND_ONDEMAND); - - if(dt) {}; - - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/snmp6"); ff = procfile_open(config_get("plugin:proc:/proc/net/snmp6", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) + return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time - - uint32_t lines = procfile_lines(ff), l; - uint32_t words; - - unsigned long long Ip6InReceives = 0ULL; - unsigned long long Ip6InHdrErrors = 0ULL; - unsigned long long Ip6InTooBigErrors = 0ULL; - unsigned long long Ip6InNoRoutes = 0ULL; - unsigned long long Ip6InAddrErrors = 0ULL; - unsigned long long Ip6InUnknownProtos = 0ULL; - unsigned long long Ip6InTruncatedPkts = 0ULL; - unsigned long long Ip6InDiscards = 0ULL; - unsigned long long Ip6InDelivers = 0ULL; - unsigned long long Ip6OutForwDatagrams = 0ULL; - unsigned long long Ip6OutRequests = 0ULL; - unsigned long long Ip6OutDiscards = 0ULL; - unsigned long long Ip6OutNoRoutes = 0ULL; - unsigned long long Ip6ReasmTimeout = 0ULL; - unsigned long long Ip6ReasmReqds = 0ULL; - unsigned long long Ip6ReasmOKs = 0ULL; - unsigned long long Ip6ReasmFails = 0ULL; - unsigned long long Ip6FragOKs = 0ULL; - unsigned long long Ip6FragFails = 0ULL; - unsigned long long Ip6FragCreates = 0ULL; - unsigned long long Ip6InMcastPkts = 0ULL; - unsigned long long Ip6OutMcastPkts = 0ULL; - unsigned long long Ip6InOctets = 0ULL; - unsigned long long Ip6OutOctets = 0ULL; - unsigned long long Ip6InMcastOctets = 0ULL; - unsigned long long Ip6OutMcastOctets = 0ULL; - unsigned long long Ip6InBcastOctets = 0ULL; - unsigned long long Ip6OutBcastOctets = 0ULL; - unsigned long long Ip6InNoECTPkts = 0ULL; - unsigned long long Ip6InECT1Pkts = 0ULL; - unsigned long long Ip6InECT0Pkts = 0ULL; - unsigned long long Ip6InCEPkts = 0ULL; - unsigned long long Icmp6InMsgs = 0ULL; - unsigned long long Icmp6InErrors = 0ULL; - unsigned long long Icmp6OutMsgs = 0ULL; - unsigned long long Icmp6OutErrors = 0ULL; - unsigned long long Icmp6InCsumErrors = 0ULL; - unsigned long long Icmp6InDestUnreachs = 0ULL; - unsigned long long Icmp6InPktTooBigs = 0ULL; - unsigned long long Icmp6InTimeExcds = 0ULL; - unsigned long long Icmp6InParmProblems = 0ULL; - unsigned long long Icmp6InEchos = 0ULL; - unsigned long long Icmp6InEchoReplies = 0ULL; - unsigned long long Icmp6InGroupMembQueries = 0ULL; - unsigned long long Icmp6InGroupMembResponses = 0ULL; - unsigned long long Icmp6InGroupMembReductions = 0ULL; - unsigned long long Icmp6InRouterSolicits = 0ULL; - unsigned long long Icmp6InRouterAdvertisements = 0ULL; - unsigned long long Icmp6InNeighborSolicits = 0ULL; - unsigned long long Icmp6InNeighborAdvertisements = 0ULL; - unsigned long long Icmp6InRedirects = 0ULL; - unsigned long long Icmp6InMLDv2Reports = 0ULL; - unsigned long long Icmp6OutDestUnreachs = 0ULL; - unsigned long long Icmp6OutPktTooBigs = 0ULL; - unsigned long long Icmp6OutTimeExcds = 0ULL; - unsigned long long Icmp6OutParmProblems = 0ULL; - unsigned long long Icmp6OutEchos = 0ULL; - unsigned long long Icmp6OutEchoReplies = 0ULL; - unsigned long long Icmp6OutGroupMembQueries = 0ULL; - unsigned long long Icmp6OutGroupMembResponses = 0ULL; - unsigned long long Icmp6OutGroupMembReductions = 0ULL; - unsigned long long Icmp6OutRouterSolicits = 0ULL; - unsigned long long Icmp6OutRouterAdvertisements = 0ULL; - unsigned long long Icmp6OutNeighborSolicits = 0ULL; - unsigned long long Icmp6OutNeighborAdvertisements = 0ULL; - unsigned long long Icmp6OutRedirects = 0ULL; - unsigned long long Icmp6OutMLDv2Reports = 0ULL; - unsigned long long Icmp6InType1 = 0ULL; - unsigned long long Icmp6InType128 = 0ULL; - unsigned long long Icmp6InType129 = 0ULL; - unsigned long long Icmp6InType136 = 0ULL; - unsigned long long Icmp6OutType1 = 0ULL; - unsigned long long Icmp6OutType128 = 0ULL; - unsigned long long Icmp6OutType129 = 0ULL; - unsigned long long Icmp6OutType133 = 0ULL; - unsigned long long Icmp6OutType135 = 0ULL; - unsigned long long Icmp6OutType143 = 0ULL; - unsigned long long Udp6InDatagrams = 0ULL; - unsigned long long Udp6NoPorts = 0ULL; - unsigned long long Udp6InErrors = 0ULL; - unsigned long long Udp6OutDatagrams = 0ULL; - unsigned long long Udp6RcvbufErrors = 0ULL; - unsigned long long Udp6SndbufErrors = 0ULL; - unsigned long long Udp6InCsumErrors = 0ULL; - unsigned long long Udp6IgnoredMulti = 0ULL; - unsigned long long UdpLite6InDatagrams = 0ULL; - unsigned long long UdpLite6NoPorts = 0ULL; - unsigned long long UdpLite6InErrors = 0ULL; - unsigned long long UdpLite6OutDatagrams = 0ULL; - unsigned long long UdpLite6RcvbufErrors = 0ULL; - unsigned long long UdpLite6SndbufErrors = 0ULL; - unsigned long long UdpLite6InCsumErrors = 0ULL; + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time + + size_t lines = procfile_lines(ff), l; + + arl_begin(arl_base); for(l = 0; l < lines ;l++) { - words = procfile_linewords(ff, l); - if(words < 2) { - if(words) error("Cannot read /proc/net/snmp6 line %u. Expected 2 params, read %u.", l, words); + size_t words = procfile_linewords(ff, l); + if(unlikely(words < 2)) { + if(unlikely(words)) error("Cannot read /proc/net/snmp6 line %zu. Expected 2 params, read %zu.", l, words); continue; } - char *name = procfile_lineword(ff, l, 0); - char * value = procfile_lineword(ff, l, 1); - if(!name || !*name || !value || !*value) continue; - - uint32_t hash = simple_hash(name); - - if(0) ; - else if(hash == hash_Ip6InReceives && strcmp(name, "Ip6InReceives") == 0) Ip6InReceives = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InHdrErrors && strcmp(name, "Ip6InHdrErrors") == 0) Ip6InHdrErrors = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InTooBigErrors && strcmp(name, "Ip6InTooBigErrors") == 0) Ip6InTooBigErrors = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InNoRoutes && strcmp(name, "Ip6InNoRoutes") == 0) Ip6InNoRoutes = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InAddrErrors && strcmp(name, "Ip6InAddrErrors") == 0) Ip6InAddrErrors = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InUnknownProtos && strcmp(name, "Ip6InUnknownProtos") == 0) Ip6InUnknownProtos = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InTruncatedPkts && strcmp(name, "Ip6InTruncatedPkts") == 0) Ip6InTruncatedPkts = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InDiscards && strcmp(name, "Ip6InDiscards") == 0) Ip6InDiscards = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InDelivers && strcmp(name, "Ip6InDelivers") == 0) Ip6InDelivers = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutForwDatagrams && strcmp(name, "Ip6OutForwDatagrams") == 0) Ip6OutForwDatagrams = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutRequests && strcmp(name, "Ip6OutRequests") == 0) Ip6OutRequests = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutDiscards && strcmp(name, "Ip6OutDiscards") == 0) Ip6OutDiscards = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutNoRoutes && strcmp(name, "Ip6OutNoRoutes") == 0) Ip6OutNoRoutes = strtoull(value, NULL, 10); - else if(hash == hash_Ip6ReasmTimeout && strcmp(name, "Ip6ReasmTimeout") == 0) Ip6ReasmTimeout = strtoull(value, NULL, 10); - else if(hash == hash_Ip6ReasmReqds && strcmp(name, "Ip6ReasmReqds") == 0) Ip6ReasmReqds = strtoull(value, NULL, 10); - else if(hash == hash_Ip6ReasmOKs && strcmp(name, "Ip6ReasmOKs") == 0) Ip6ReasmOKs = strtoull(value, NULL, 10); - else if(hash == hash_Ip6ReasmFails && strcmp(name, "Ip6ReasmFails") == 0) Ip6ReasmFails = strtoull(value, NULL, 10); - else if(hash == hash_Ip6FragOKs && strcmp(name, "Ip6FragOKs") == 0) Ip6FragOKs = strtoull(value, NULL, 10); - else if(hash == hash_Ip6FragFails && strcmp(name, "Ip6FragFails") == 0) Ip6FragFails = strtoull(value, NULL, 10); - else if(hash == hash_Ip6FragCreates && strcmp(name, "Ip6FragCreates") == 0) Ip6FragCreates = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InMcastPkts && strcmp(name, "Ip6InMcastPkts") == 0) Ip6InMcastPkts = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutMcastPkts && strcmp(name, "Ip6OutMcastPkts") == 0) Ip6OutMcastPkts = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InOctets && strcmp(name, "Ip6InOctets") == 0) Ip6InOctets = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutOctets && strcmp(name, "Ip6OutOctets") == 0) Ip6OutOctets = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InMcastOctets && strcmp(name, "Ip6InMcastOctets") == 0) Ip6InMcastOctets = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutMcastOctets && strcmp(name, "Ip6OutMcastOctets") == 0) Ip6OutMcastOctets = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InBcastOctets && strcmp(name, "Ip6InBcastOctets") == 0) Ip6InBcastOctets = strtoull(value, NULL, 10); - else if(hash == hash_Ip6OutBcastOctets && strcmp(name, "Ip6OutBcastOctets") == 0) Ip6OutBcastOctets = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InNoECTPkts && strcmp(name, "Ip6InNoECTPkts") == 0) Ip6InNoECTPkts = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InECT1Pkts && strcmp(name, "Ip6InECT1Pkts") == 0) Ip6InECT1Pkts = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InECT0Pkts && strcmp(name, "Ip6InECT0Pkts") == 0) Ip6InECT0Pkts = strtoull(value, NULL, 10); - else if(hash == hash_Ip6InCEPkts && strcmp(name, "Ip6InCEPkts") == 0) Ip6InCEPkts = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InMsgs && strcmp(name, "Icmp6InMsgs") == 0) Icmp6InMsgs = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InErrors && strcmp(name, "Icmp6InErrors") == 0) Icmp6InErrors = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutMsgs && strcmp(name, "Icmp6OutMsgs") == 0) Icmp6OutMsgs = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutErrors && strcmp(name, "Icmp6OutErrors") == 0) Icmp6OutErrors = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InCsumErrors && strcmp(name, "Icmp6InCsumErrors") == 0) Icmp6InCsumErrors = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InDestUnreachs && strcmp(name, "Icmp6InDestUnreachs") == 0) Icmp6InDestUnreachs = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InPktTooBigs && strcmp(name, "Icmp6InPktTooBigs") == 0) Icmp6InPktTooBigs = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InTimeExcds && strcmp(name, "Icmp6InTimeExcds") == 0) Icmp6InTimeExcds = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InParmProblems && strcmp(name, "Icmp6InParmProblems") == 0) Icmp6InParmProblems = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InEchos && strcmp(name, "Icmp6InEchos") == 0) Icmp6InEchos = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InEchoReplies && strcmp(name, "Icmp6InEchoReplies") == 0) Icmp6InEchoReplies = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InGroupMembQueries && strcmp(name, "Icmp6InGroupMembQueries") == 0) Icmp6InGroupMembQueries = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InGroupMembResponses && strcmp(name, "Icmp6InGroupMembResponses") == 0) Icmp6InGroupMembResponses = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InGroupMembReductions && strcmp(name, "Icmp6InGroupMembReductions") == 0) Icmp6InGroupMembReductions = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InRouterSolicits && strcmp(name, "Icmp6InRouterSolicits") == 0) Icmp6InRouterSolicits = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InRouterAdvertisements && strcmp(name, "Icmp6InRouterAdvertisements") == 0) Icmp6InRouterAdvertisements = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InNeighborSolicits && strcmp(name, "Icmp6InNeighborSolicits") == 0) Icmp6InNeighborSolicits = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InNeighborAdvertisements && strcmp(name, "Icmp6InNeighborAdvertisements") == 0) Icmp6InNeighborAdvertisements = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InRedirects && strcmp(name, "Icmp6InRedirects") == 0) Icmp6InRedirects = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InMLDv2Reports && strcmp(name, "Icmp6InMLDv2Reports") == 0) Icmp6InMLDv2Reports = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutDestUnreachs && strcmp(name, "Icmp6OutDestUnreachs") == 0) Icmp6OutDestUnreachs = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutPktTooBigs && strcmp(name, "Icmp6OutPktTooBigs") == 0) Icmp6OutPktTooBigs = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutTimeExcds && strcmp(name, "Icmp6OutTimeExcds") == 0) Icmp6OutTimeExcds = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutParmProblems && strcmp(name, "Icmp6OutParmProblems") == 0) Icmp6OutParmProblems = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutEchos && strcmp(name, "Icmp6OutEchos") == 0) Icmp6OutEchos = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutEchoReplies && strcmp(name, "Icmp6OutEchoReplies") == 0) Icmp6OutEchoReplies = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutGroupMembQueries && strcmp(name, "Icmp6OutGroupMembQueries") == 0) Icmp6OutGroupMembQueries = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutGroupMembResponses && strcmp(name, "Icmp6OutGroupMembResponses") == 0) Icmp6OutGroupMembResponses = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutGroupMembReductions && strcmp(name, "Icmp6OutGroupMembReductions") == 0) Icmp6OutGroupMembReductions = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutRouterSolicits && strcmp(name, "Icmp6OutRouterSolicits") == 0) Icmp6OutRouterSolicits = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutRouterAdvertisements && strcmp(name, "Icmp6OutRouterAdvertisements") == 0) Icmp6OutRouterAdvertisements = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutNeighborSolicits && strcmp(name, "Icmp6OutNeighborSolicits") == 0) Icmp6OutNeighborSolicits = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutNeighborAdvertisements && strcmp(name, "Icmp6OutNeighborAdvertisements") == 0) Icmp6OutNeighborAdvertisements = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutRedirects && strcmp(name, "Icmp6OutRedirects") == 0) Icmp6OutRedirects = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutMLDv2Reports && strcmp(name, "Icmp6OutMLDv2Reports") == 0) Icmp6OutMLDv2Reports = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InType1 && strcmp(name, "Icmp6InType1") == 0) Icmp6InType1 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InType128 && strcmp(name, "Icmp6InType128") == 0) Icmp6InType128 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InType129 && strcmp(name, "Icmp6InType129") == 0) Icmp6InType129 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6InType136 && strcmp(name, "Icmp6InType136") == 0) Icmp6InType136 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutType1 && strcmp(name, "Icmp6OutType1") == 0) Icmp6OutType1 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutType128 && strcmp(name, "Icmp6OutType128") == 0) Icmp6OutType128 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutType129 && strcmp(name, "Icmp6OutType129") == 0) Icmp6OutType129 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutType133 && strcmp(name, "Icmp6OutType133") == 0) Icmp6OutType133 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutType135 && strcmp(name, "Icmp6OutType135") == 0) Icmp6OutType135 = strtoull(value, NULL, 10); - else if(hash == hash_Icmp6OutType143 && strcmp(name, "Icmp6OutType143") == 0) Icmp6OutType143 = strtoull(value, NULL, 10); - else if(hash == hash_Udp6InDatagrams && strcmp(name, "Udp6InDatagrams") == 0) Udp6InDatagrams = strtoull(value, NULL, 10); - else if(hash == hash_Udp6NoPorts && strcmp(name, "Udp6NoPorts") == 0) Udp6NoPorts = strtoull(value, NULL, 10); - else if(hash == hash_Udp6InErrors && strcmp(name, "Udp6InErrors") == 0) Udp6InErrors = strtoull(value, NULL, 10); - else if(hash == hash_Udp6OutDatagrams && strcmp(name, "Udp6OutDatagrams") == 0) Udp6OutDatagrams = strtoull(value, NULL, 10); - else if(hash == hash_Udp6RcvbufErrors && strcmp(name, "Udp6RcvbufErrors") == 0) Udp6RcvbufErrors = strtoull(value, NULL, 10); - else if(hash == hash_Udp6SndbufErrors && strcmp(name, "Udp6SndbufErrors") == 0) Udp6SndbufErrors = strtoull(value, NULL, 10); - else if(hash == hash_Udp6InCsumErrors && strcmp(name, "Udp6InCsumErrors") == 0) Udp6InCsumErrors = strtoull(value, NULL, 10); - else if(hash == hash_Udp6IgnoredMulti && strcmp(name, "Udp6IgnoredMulti") == 0) Udp6IgnoredMulti = strtoull(value, NULL, 10); - else if(hash == hash_UdpLite6InDatagrams && strcmp(name, "UdpLite6InDatagrams") == 0) UdpLite6InDatagrams = strtoull(value, NULL, 10); - else if(hash == hash_UdpLite6NoPorts && strcmp(name, "UdpLite6NoPorts") == 0) UdpLite6NoPorts = strtoull(value, NULL, 10); - else if(hash == hash_UdpLite6InErrors && strcmp(name, "UdpLite6InErrors") == 0) UdpLite6InErrors = strtoull(value, NULL, 10); - else if(hash == hash_UdpLite6OutDatagrams && strcmp(name, "UdpLite6OutDatagrams") == 0) UdpLite6OutDatagrams = strtoull(value, NULL, 10); - else if(hash == hash_UdpLite6RcvbufErrors && strcmp(name, "UdpLite6RcvbufErrors") == 0) UdpLite6RcvbufErrors = strtoull(value, NULL, 10); - else if(hash == hash_UdpLite6SndbufErrors && strcmp(name, "UdpLite6SndbufErrors") == 0) UdpLite6SndbufErrors = strtoull(value, NULL, 10); - else if(hash == hash_UdpLite6InCsumErrors && strcmp(name, "UdpLite6InCsumErrors") == 0) UdpLite6InCsumErrors = strtoull(value, NULL, 10); + if(unlikely(arl_check(arl_base, + procfile_lineword(ff, l, 0), + procfile_lineword(ff, l, 1)))) break; } RRDSET *st; @@ -460,7 +279,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_bandwidth == CONFIG_ONDEMAND_YES || (do_bandwidth == CONFIG_ONDEMAND_ONDEMAND && (Ip6InOctets || Ip6OutOctets))) { do_bandwidth = CONFIG_ONDEMAND_YES; st = rrdset_find("system.ipv6"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("system", "ipv6", NULL, "network", NULL, "IPv6 Bandwidth", "kilobits/s", 500, update_every, RRDSET_TYPE_AREA); rrddim_add(st, "received", NULL, 8, 1024, RRDDIM_INCREMENTAL); @@ -478,7 +297,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_ip_packets == CONFIG_ONDEMAND_YES || (do_ip_packets == CONFIG_ONDEMAND_ONDEMAND && (Ip6InReceives || Ip6OutRequests || Ip6InDelivers || Ip6OutForwDatagrams))) { do_ip_packets = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".packets"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "packets", NULL, "packets", NULL, "IPv6 Packets", "packets/s", 3000, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -500,7 +319,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_ip_fragsout == CONFIG_ONDEMAND_YES || (do_ip_fragsout == CONFIG_ONDEMAND_ONDEMAND && (Ip6FragOKs || Ip6FragFails || Ip6FragCreates))) { do_ip_fragsout = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".fragsout"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "fragsout", NULL, "fragments", NULL, "IPv6 Fragments Sent", "packets/s", 3010, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -527,7 +346,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_ip_fragsin = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".fragsin"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "fragsin", NULL, "fragments", NULL, "IPv6 Fragments Reassembly", "packets/s", 3011, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -560,7 +379,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_ip_errors = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".errors"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "errors", NULL, "errors", NULL, "IPv6 Errors", "packets/s", 3002, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -597,7 +416,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_udp_packets == CONFIG_ONDEMAND_YES || (do_udp_packets == CONFIG_ONDEMAND_ONDEMAND && (Udp6InDatagrams || Udp6OutDatagrams))) { do_udp_packets = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".udppackets"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "udppackets", NULL, "udp", NULL, "IPv6 UDP Packets", "packets/s", 3601, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -623,7 +442,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_udp_errors = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".udperrors"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "udperrors", NULL, "udp", NULL, "IPv6 UDP Errors", "events/s", 3701, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -650,7 +469,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_udplite_packets == CONFIG_ONDEMAND_YES || (do_udplite_packets == CONFIG_ONDEMAND_ONDEMAND && (UdpLite6InDatagrams || UdpLite6OutDatagrams))) { do_udplite_packets = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".udplitepackets"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "udplitepackets", NULL, "udplite", NULL, "IPv6 UDPlite Packets", "packets/s", 3601, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -676,7 +495,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_udplite_errors = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".udpliteerrors"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "udpliteerrors", NULL, "udplite", NULL, "IPv6 UDP Lite Errors", "events/s", 3701, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -701,7 +520,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_mcast == CONFIG_ONDEMAND_YES || (do_mcast == CONFIG_ONDEMAND_ONDEMAND && (Ip6OutMcastOctets || Ip6InMcastOctets))) { do_mcast = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".mcast"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "mcast", NULL, "multicast", NULL, "IPv6 Multicast Bandwidth", "kilobits/s", 9000, update_every, RRDSET_TYPE_AREA); st->isdetail = 1; @@ -720,7 +539,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_bcast == CONFIG_ONDEMAND_YES || (do_bcast == CONFIG_ONDEMAND_ONDEMAND && (Ip6OutBcastOctets || Ip6InBcastOctets))) { do_bcast = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".bcast"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "bcast", NULL, "broadcast", NULL, "IPv6 Broadcast Bandwidth", "kilobits/s", 8000, update_every, RRDSET_TYPE_AREA); st->isdetail = 1; @@ -739,7 +558,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_mcast_p == CONFIG_ONDEMAND_YES || (do_mcast_p == CONFIG_ONDEMAND_ONDEMAND && (Ip6OutMcastPkts || Ip6InMcastPkts))) { do_mcast_p = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".mcastpkts"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "mcastpkts", NULL, "multicast", NULL, "IPv6 Multicast Packets", "packets/s", 9500, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -758,7 +577,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_icmp == CONFIG_ONDEMAND_YES || (do_icmp == CONFIG_ONDEMAND_ONDEMAND && (Icmp6InMsgs || Icmp6OutMsgs))) { do_icmp = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmp"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmp", NULL, "icmp", NULL, "IPv6 ICMP Messages", "messages/s", 10000, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -776,7 +595,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_icmp_redir == CONFIG_ONDEMAND_YES || (do_icmp_redir == CONFIG_ONDEMAND_ONDEMAND && (Icmp6InRedirects || Icmp6OutRedirects))) { do_icmp_redir = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmpredir"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmpredir", NULL, "icmp", NULL, "IPv6 ICMP Redirects", "redirects/s", 10050, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -807,7 +626,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_icmp_errors = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmperrors"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmperrors", NULL, "icmp", NULL, "IPv6 ICMP Errors", "errors/s", 10100, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "InErrors", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -850,7 +669,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_icmp_echos = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmpechos"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmpechos", NULL, "icmp", NULL, "IPv6 ICMP Echo", "messages/s", 10200, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "InEchos", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -880,7 +699,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_icmp_groupmemb = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".groupmemb"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "groupmemb", NULL, "icmp", NULL, "IPv6 ICMP Group Membership", "messages/s", 10300, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "InQueries", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -912,7 +731,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_icmp_router = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmprouter"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmprouter", NULL, "icmp", NULL, "IPv6 Router Messages", "messages/s", 10400, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "InSolicits", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -940,7 +759,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_icmp_neighbor = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmpneighbor"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmpneighbor", NULL, "icmp", NULL, "IPv6 Neighbor Messages", "messages/s", 10500, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "InSolicits", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -962,7 +781,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { if(do_icmp_mldv2 == CONFIG_ONDEMAND_YES || (do_icmp_mldv2 == CONFIG_ONDEMAND_ONDEMAND && (Icmp6InMLDv2Reports || Icmp6OutMLDv2Reports))) { do_icmp_mldv2 = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmpmldv2"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmpmldv2", NULL, "icmp", NULL, "IPv6 ICMP MLDv2 Reports", "reports/s", 10600, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -992,7 +811,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_icmp_types = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".icmptypes"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "icmptypes", NULL, "icmp", NULL, "IPv6 ICMP Types", "messages/s", 10700, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "InType1", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -1032,7 +851,7 @@ int do_proc_net_snmp6(int update_every, unsigned long long dt) { ))) { do_ect = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_SNMP6 ".ect"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_SNMP6, "ect", NULL, "packets", NULL, "IPv6 ECT Packets", "packets/s", 10800, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "InNoECTPkts", NULL, 1, 1, RRDDIM_INCREMENTAL); diff --git a/src/proc_net_softnet_stat.c b/src/proc_net_softnet_stat.c index b01315863..2f4eb3e66 100644 --- a/src/proc_net_softnet_stat.c +++ b/src/proc_net_softnet_stat.c @@ -1,6 +1,6 @@ #include "common.h" -static inline char *softnet_column_name(uint32_t column) { +static inline char *softnet_column_name(size_t column) { switch(column) { // https://github.com/torvalds/linux/blob/a7fd20d1c476af4563e66865213474a2f9f473a4/net/core/net-procfs.c#L161-L166 case 0: return "processed"; @@ -12,35 +12,36 @@ static inline char *softnet_column_name(uint32_t column) { } } -int do_proc_net_softnet_stat(int update_every, unsigned long long dt) { +int do_proc_net_softnet_stat(int update_every, usec_t dt) { (void)dt; static procfile *ff = NULL; static int do_per_core = -1; - static uint32_t allocated_lines = 0, allocated_columns = 0, *data = NULL; + static size_t allocated_lines = 0, allocated_columns = 0; + static uint32_t *data = NULL; - if(do_per_core == -1) do_per_core = config_get_boolean("plugin:proc:/proc/net/softnet_stat", "softnet_stat per core", 1); + if(unlikely(do_per_core == -1)) do_per_core = config_get_boolean("plugin:proc:/proc/net/softnet_stat", "softnet_stat per core", 1); - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/softnet_stat"); ff = procfile_open(config_get("plugin:proc:/proc/net/softnet_stat", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - uint32_t lines = procfile_lines(ff), l; - uint32_t words = procfile_linewords(ff, 0), w; + size_t lines = procfile_lines(ff), l; + size_t words = procfile_linewords(ff, 0), w; - if(!lines || !words) { - error("Cannot read /proc/net/softnet_stat, %u lines and %u columns reported.", lines, words); + if(unlikely(!lines || !words)) { + error("Cannot read /proc/net/softnet_stat, %zu lines and %zu columns reported.", lines, words); return 1; } - if(lines > 200) lines = 200; - if(words > 50) words = 50; + if(unlikely(lines > 200)) lines = 200; + if(unlikely(words > 50)) words = 50; if(unlikely(!data || lines > allocated_lines || words > allocated_columns)) { freez(data); @@ -55,20 +56,21 @@ int do_proc_net_softnet_stat(int update_every, unsigned long long dt) { // parse the values for(l = 0; l < lines ;l++) { words = procfile_linewords(ff, l); - if(!words) continue; + if(unlikely(!words)) continue; - if(words > allocated_columns) words = allocated_columns; + if(unlikely(words > allocated_columns)) + words = allocated_columns; for(w = 0; w < words ; w++) { if(unlikely(softnet_column_name(w))) { - uint32_t t = strtoul(procfile_lineword(ff, l, w), NULL, 16); + uint32_t t = (uint32_t)strtoul(procfile_lineword(ff, l, w), NULL, 16); data[w] += t; data[((l + 1) * allocated_columns) + w] = t; } } } - if(data[(lines * allocated_columns)] == 0) + if(unlikely(data[(lines * allocated_columns)] == 0)) lines--; RRDSET *st; @@ -76,7 +78,7 @@ int do_proc_net_softnet_stat(int update_every, unsigned long long dt) { // -------------------------------------------------------------------- st = rrdset_find_bytype("system", "softnet_stat"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("system", "softnet_stat", NULL, "softnet_stat", NULL, "System softnet_stat", "events/s", 955, update_every, RRDSET_TYPE_LINE); for(w = 0; w < allocated_columns ;w++) if(unlikely(softnet_column_name(w))) @@ -93,12 +95,12 @@ int do_proc_net_softnet_stat(int update_every, unsigned long long dt) { if(do_per_core) { for(l = 0; l < lines ;l++) { char id[50+1]; - snprintfz(id, 50, "cpu%u_softnet_stat", l); + snprintfz(id, 50, "cpu%zu_softnet_stat", l); st = rrdset_find_bytype("cpu", id); - if(!st) { + if(unlikely(!st)) { char title[100+1]; - snprintfz(title, 100, "CPU%u softnet_stat", l); + snprintfz(title, 100, "CPU%zu softnet_stat", l); st = rrdset_create("cpu", id, NULL, "softnet_stat", NULL, title, "events/s", 4101 + l, update_every, RRDSET_TYPE_LINE); for(w = 0; w < allocated_columns ;w++) diff --git a/src/proc_net_stat_conntrack.c b/src/proc_net_stat_conntrack.c index 54e250bf2..b9c724983 100644 --- a/src/proc_net_stat_conntrack.c +++ b/src/proc_net_stat_conntrack.c @@ -2,83 +2,127 @@ #define RRD_TYPE_NET_STAT_NETFILTER "netfilter" #define RRD_TYPE_NET_STAT_CONNTRACK "conntrack" -#define RRD_TYPE_NET_STAT_CONNTRACK_LEN strlen(RRD_TYPE_NET_STAT_CONNTRACK) -int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { +int do_proc_net_stat_conntrack(int update_every, usec_t dt) { static procfile *ff = NULL; static int do_sockets = -1, do_new = -1, do_changes = -1, do_expect = -1, do_search = -1, do_errors = -1; + static usec_t get_max_every = 10 * USEC_PER_SEC, usec_since_last_max = 0; + static int read_full = 1; + static char *nf_conntrack_filename, *nf_conntrack_count_filename, *nf_conntrack_max_filename; + static RRDVAR *rrdvar_max = NULL; - if(do_sockets == -1) do_sockets = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connections", 1); - if(do_new == -1) do_new = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter new connections", 1); - if(do_changes == -1) do_changes = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection changes", 1); - if(do_expect == -1) do_expect = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection expectations", 1); - if(do_search == -1) do_search = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection searches", 1); - if(do_errors == -1) do_errors = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter errors", 1); - - if(dt) {}; + unsigned long long aentries = 0, asearched = 0, afound = 0, anew = 0, ainvalid = 0, aignore = 0, adelete = 0, adelete_list = 0, + ainsert = 0, ainsert_failed = 0, adrop = 0, aearly_drop = 0, aicmp_error = 0, aexpect_new = 0, aexpect_create = 0, aexpect_delete = 0, asearch_restart = 0; - if(!ff) { + if(unlikely(do_sockets == -1)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/stat/nf_conntrack"); - ff = procfile_open(config_get("plugin:proc:/proc/net/stat/nf_conntrack", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); - } - if(!ff) return 1; + nf_conntrack_filename = config_get("plugin:proc:/proc/net/stat/nf_conntrack", "filename to monitor", filename); - ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/sys/net/netfilter/nf_conntrack_max"); + nf_conntrack_max_filename = config_get("plugin:proc:/proc/sys/net/netfilter/nf_conntrack_max", "filename to monitor", filename); + usec_since_last_max = get_max_every = config_get_number("plugin:proc:/proc/sys/net/netfilter/nf_conntrack_max", "read every seconds", 10) * USEC_PER_SEC; - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + read_full = 1; + ff = procfile_open(nf_conntrack_filename, " \t:", PROCFILE_FLAG_DEFAULT); + if(!ff) read_full = 0; - unsigned long long aentries = 0, asearched = 0, afound = 0, anew = 0, ainvalid = 0, aignore = 0, adelete = 0, adelete_list = 0, - ainsert = 0, ainsert_failed = 0, adrop = 0, aearly_drop = 0, aicmp_error = 0, aexpect_new = 0, aexpect_create = 0, aexpect_delete = 0, asearch_restart = 0; + do_new = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter new connections", read_full); + do_changes = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection changes", read_full); + do_expect = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection expectations", read_full); + do_search = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection searches", read_full); + do_errors = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter errors", read_full); + + do_sockets = 1; + if(!read_full) { + snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/sys/net/netfilter/nf_conntrack_count"); + nf_conntrack_count_filename = config_get("plugin:proc:/proc/sys/net/netfilter/nf_conntrack_count", "filename to monitor", filename); + + if(read_single_number_file(nf_conntrack_count_filename, &aentries)) + do_sockets = 0; + } + + do_sockets = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connections", do_sockets); - for(l = 1; l < lines ;l++) { - words = procfile_linewords(ff, l); - if(words < 17) { - if(words) error("Cannot read /proc/net/stat/nf_conntrack line. Expected 17 params, read %u.", words); - continue; + if(!do_sockets && !read_full) + return 1; + + rrdvar_max = rrdvar_custom_host_variable_create(&localhost, "netfilter.conntrack.max"); + } + + if(likely(read_full)) { + if(unlikely(!ff)) { + ff = procfile_open(nf_conntrack_filename, " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time + } + + ff = procfile_readall(ff); + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time + + size_t lines = procfile_lines(ff), l; + + for(l = 1; l < lines ;l++) { + size_t words = procfile_linewords(ff, l); + if(unlikely(words < 17)) { + if(unlikely(words)) error("Cannot read /proc/net/stat/nf_conntrack line. Expected 17 params, read %zu.", words); + continue; + } + + unsigned long long tentries = 0, tsearched = 0, tfound = 0, tnew = 0, tinvalid = 0, tignore = 0, tdelete = 0, tdelete_list = 0, tinsert = 0, tinsert_failed = 0, tdrop = 0, tearly_drop = 0, ticmp_error = 0, texpect_new = 0, texpect_create = 0, texpect_delete = 0, tsearch_restart = 0; + + tentries = strtoull(procfile_lineword(ff, l, 0), NULL, 16); + tsearched = strtoull(procfile_lineword(ff, l, 1), NULL, 16); + tfound = strtoull(procfile_lineword(ff, l, 2), NULL, 16); + tnew = strtoull(procfile_lineword(ff, l, 3), NULL, 16); + tinvalid = strtoull(procfile_lineword(ff, l, 4), NULL, 16); + tignore = strtoull(procfile_lineword(ff, l, 5), NULL, 16); + tdelete = strtoull(procfile_lineword(ff, l, 6), NULL, 16); + tdelete_list = strtoull(procfile_lineword(ff, l, 7), NULL, 16); + tinsert = strtoull(procfile_lineword(ff, l, 8), NULL, 16); + tinsert_failed = strtoull(procfile_lineword(ff, l, 9), NULL, 16); + tdrop = strtoull(procfile_lineword(ff, l, 10), NULL, 16); + tearly_drop = strtoull(procfile_lineword(ff, l, 11), NULL, 16); + ticmp_error = strtoull(procfile_lineword(ff, l, 12), NULL, 16); + texpect_new = strtoull(procfile_lineword(ff, l, 13), NULL, 16); + texpect_create = strtoull(procfile_lineword(ff, l, 14), NULL, 16); + texpect_delete = strtoull(procfile_lineword(ff, l, 15), NULL, 16); + tsearch_restart = strtoull(procfile_lineword(ff, l, 16), NULL, 16); + + if(unlikely(!aentries)) aentries = tentries; + + // sum all the cpus together + asearched += tsearched; // conntrack.search + afound += tfound; // conntrack.search + anew += tnew; // conntrack.new + ainvalid += tinvalid; // conntrack.new + aignore += tignore; // conntrack.new + adelete += tdelete; // conntrack.changes + adelete_list += tdelete_list; // conntrack.changes + ainsert += tinsert; // conntrack.changes + ainsert_failed += tinsert_failed; // conntrack.errors + adrop += tdrop; // conntrack.errors + aearly_drop += tearly_drop; // conntrack.errors + aicmp_error += ticmp_error; // conntrack.errors + aexpect_new += texpect_new; // conntrack.expect + aexpect_create += texpect_create; // conntrack.expect + aexpect_delete += texpect_delete; // conntrack.expect + asearch_restart += tsearch_restart; // conntrack.search } + } + else { + if(unlikely(read_single_number_file(nf_conntrack_count_filename, &aentries))) + return 0; // we return 0, so that we will retry to open it next time + } + + usec_since_last_max += dt; + if(unlikely(rrdvar_max && usec_since_last_max >= get_max_every)) { + usec_since_last_max = 0; - unsigned long long tentries = 0, tsearched = 0, tfound = 0, tnew = 0, tinvalid = 0, tignore = 0, tdelete = 0, tdelete_list = 0, tinsert = 0, tinsert_failed = 0, tdrop = 0, tearly_drop = 0, ticmp_error = 0, texpect_new = 0, texpect_create = 0, texpect_delete = 0, tsearch_restart = 0; - - tentries = strtoull(procfile_lineword(ff, l, 0), NULL, 16); - tsearched = strtoull(procfile_lineword(ff, l, 1), NULL, 16); - tfound = strtoull(procfile_lineword(ff, l, 2), NULL, 16); - tnew = strtoull(procfile_lineword(ff, l, 3), NULL, 16); - tinvalid = strtoull(procfile_lineword(ff, l, 4), NULL, 16); - tignore = strtoull(procfile_lineword(ff, l, 5), NULL, 16); - tdelete = strtoull(procfile_lineword(ff, l, 6), NULL, 16); - tdelete_list = strtoull(procfile_lineword(ff, l, 7), NULL, 16); - tinsert = strtoull(procfile_lineword(ff, l, 8), NULL, 16); - tinsert_failed = strtoull(procfile_lineword(ff, l, 9), NULL, 16); - tdrop = strtoull(procfile_lineword(ff, l, 10), NULL, 16); - tearly_drop = strtoull(procfile_lineword(ff, l, 11), NULL, 16); - ticmp_error = strtoull(procfile_lineword(ff, l, 12), NULL, 16); - texpect_new = strtoull(procfile_lineword(ff, l, 13), NULL, 16); - texpect_create = strtoull(procfile_lineword(ff, l, 14), NULL, 16); - texpect_delete = strtoull(procfile_lineword(ff, l, 15), NULL, 16); - tsearch_restart = strtoull(procfile_lineword(ff, l, 16), NULL, 16); - - if(!aentries) aentries = tentries; - - // sum all the cpus together - asearched += tsearched; // conntrack.search - afound += tfound; // conntrack.search - anew += tnew; // conntrack.new - ainvalid += tinvalid; // conntrack.new - aignore += tignore; // conntrack.new - adelete += tdelete; // conntrack.changes - adelete_list += tdelete_list; // conntrack.changes - ainsert += tinsert; // conntrack.changes - ainsert_failed += tinsert_failed; // conntrack.errors - adrop += tdrop; // conntrack.errors - aearly_drop += tearly_drop; // conntrack.errors - aicmp_error += ticmp_error; // conntrack.errors - aexpect_new += texpect_new; // conntrack.expect - aexpect_create += texpect_create; // conntrack.expect - aexpect_delete += texpect_delete; // conntrack.expect - asearch_restart += tsearch_restart; // conntrack.search + unsigned long long max; + if(likely(!read_single_number_file(nf_conntrack_max_filename, &max))) + rrdvar_custom_host_variable_set(rrdvar_max, max); } RRDSET *st; @@ -87,7 +131,7 @@ int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { if(do_sockets) { st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_sockets"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_sockets", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Connections", "active connections", 3000, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "connections", NULL, 1, 1, RRDDIM_ABSOLUTE); @@ -102,7 +146,7 @@ int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { if(do_new) { st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_new"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_new", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker New Connections", "connections/s", 3001, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "new", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -121,7 +165,7 @@ int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { if(do_changes) { st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_changes"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_changes", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Changes", "changes/s", 3002, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -141,7 +185,7 @@ int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { if(do_expect) { st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_expect"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_expect", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Expectations", "expectations/s", 3003, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -161,7 +205,7 @@ int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { if(do_search) { st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_search"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_search", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Searches", "searches/s", 3010, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; @@ -181,7 +225,7 @@ int do_proc_net_stat_conntrack(int update_every, unsigned long long dt) { if(do_errors) { st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_CONNTRACK "_errors"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_CONNTRACK "_errors", NULL, RRD_TYPE_NET_STAT_CONNTRACK, NULL, "Connection Tracker Errors", "events/s", 3005, update_every, RRDSET_TYPE_LINE); st->isdetail = 1; diff --git a/src/proc_net_stat_synproxy.c b/src/proc_net_stat_synproxy.c index 102805f70..6bb0a3c69 100644 --- a/src/proc_net_stat_synproxy.c +++ b/src/proc_net_stat_synproxy.c @@ -2,32 +2,35 @@ #define RRD_TYPE_NET_STAT_NETFILTER "netfilter" #define RRD_TYPE_NET_STAT_SYNPROXY "synproxy" -#define RRD_TYPE_NET_STAT_SYNPROXY_LEN strlen(RRD_TYPE_NET_STAT_SYNPROXY) -int do_proc_net_stat_synproxy(int update_every, unsigned long long dt) { +int do_proc_net_stat_synproxy(int update_every, usec_t dt) { + (void)dt; + static int do_entries = -1, do_cookies = -1, do_syns = -1, do_reopened = -1; static procfile *ff = NULL; - if(do_entries == -1) do_entries = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY entries", CONFIG_ONDEMAND_ONDEMAND); - if(do_cookies == -1) do_cookies = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY cookies", CONFIG_ONDEMAND_ONDEMAND); - if(do_syns == -1) do_syns = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY SYN received", CONFIG_ONDEMAND_ONDEMAND); - if(do_reopened == -1) do_reopened = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY connections reopened", CONFIG_ONDEMAND_ONDEMAND); - - if(dt) {}; + if(unlikely(do_entries == -1)) { + do_entries = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY entries", CONFIG_ONDEMAND_ONDEMAND); + do_cookies = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY cookies", CONFIG_ONDEMAND_ONDEMAND); + do_syns = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY SYN received", CONFIG_ONDEMAND_ONDEMAND); + do_reopened = config_get_boolean_ondemand("plugin:proc:/proc/net/stat/synproxy", "SYNPROXY connections reopened", CONFIG_ONDEMAND_ONDEMAND); + } - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/net/stat/synproxy"); ff = procfile_open(config_get("plugin:proc:/proc/net/stat/synproxy", "filename to monitor", filename), " \t,:|", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) + return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time // make sure we have 3 lines size_t lines = procfile_lines(ff), l; - if(lines < 2) { + if(unlikely(lines < 2)) { error("/proc/net/stat/synproxy has %zu lines, expected no less than 2. Disabling it.", lines); return 1; } @@ -36,8 +39,9 @@ int do_proc_net_stat_synproxy(int update_every, unsigned long long dt) { // synproxy gives its values per CPU for(l = 1; l < lines ;l++) { - int words = procfile_linewords(ff, l); - if(words < 6) continue; + size_t words = procfile_linewords(ff, l); + if(unlikely(words < 6)) + continue; entries += strtoull(procfile_lineword(ff, l, 0), NULL, 16); syn_received += strtoull(procfile_lineword(ff, l, 1), NULL, 16); @@ -57,7 +61,7 @@ int do_proc_net_stat_synproxy(int update_every, unsigned long long dt) { do_entries = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_entries"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_entries", NULL, RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY Entries Used", "entries", 3304, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "entries", NULL, 1, 1, RRDDIM_ABSOLUTE); @@ -74,7 +78,7 @@ int do_proc_net_stat_synproxy(int update_every, unsigned long long dt) { do_syns = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_syn_received"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_syn_received", NULL, RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY SYN Packets received", "SYN/s", 3301, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "received", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -91,7 +95,7 @@ int do_proc_net_stat_synproxy(int update_every, unsigned long long dt) { do_reopened = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_conn_reopened"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_conn_reopened", NULL, RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY Connections Reopened", "connections/s", 3303, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "reopened", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -108,7 +112,7 @@ int do_proc_net_stat_synproxy(int update_every, unsigned long long dt) { do_cookies = CONFIG_ONDEMAND_YES; st = rrdset_find(RRD_TYPE_NET_STAT_NETFILTER "." RRD_TYPE_NET_STAT_SYNPROXY "_cookies"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create(RRD_TYPE_NET_STAT_NETFILTER, RRD_TYPE_NET_STAT_SYNPROXY "_cookies", NULL, RRD_TYPE_NET_STAT_SYNPROXY, NULL, "SYNPROXY TCP Cookies", "cookies/s", 3302, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "valid", NULL, 1, 1, RRDDIM_INCREMENTAL); diff --git a/src/proc_self_mountinfo.c b/src/proc_self_mountinfo.c index 51aea7aee..d07f22510 100644 --- a/src/proc_self_mountinfo.c +++ b/src/proc_self_mountinfo.c @@ -1,12 +1,55 @@ #include "common.h" +// ---------------------------------------------------------------------------- +// taken from gnulib/mountlist.c + +#ifndef ME_REMOTE +/* A file system is "remote" if its Fs_name contains a ':' + or if (it is of type (smbfs or cifs) and its Fs_name starts with '//') + or Fs_name is equal to "-hosts" (used by autofs to mount remote fs). */ +# define ME_REMOTE(Fs_name, Fs_type) \ + (strchr (Fs_name, ':') != NULL \ + || ((Fs_name)[0] == '/' \ + && (Fs_name)[1] == '/' \ + && (strcmp (Fs_type, "smbfs") == 0 \ + || strcmp (Fs_type, "cifs") == 0)) \ + || (strcmp("-hosts", Fs_name) == 0)) +#endif + +#define ME_DUMMY_0(Fs_name, Fs_type) \ + (strcmp (Fs_type, "autofs") == 0 \ + || strcmp (Fs_type, "proc") == 0 \ + || strcmp (Fs_type, "subfs") == 0 \ + /* for Linux 2.6/3.x */ \ + || strcmp (Fs_type, "debugfs") == 0 \ + || strcmp (Fs_type, "devpts") == 0 \ + || strcmp (Fs_type, "fusectl") == 0 \ + || strcmp (Fs_type, "mqueue") == 0 \ + || strcmp (Fs_type, "rpc_pipefs") == 0 \ + || strcmp (Fs_type, "sysfs") == 0 \ + /* FreeBSD, Linux 2.4 */ \ + || strcmp (Fs_type, "devfs") == 0 \ + /* for NetBSD 3.0 */ \ + || strcmp (Fs_type, "kernfs") == 0 \ + /* for Irix 6.5 */ \ + || strcmp (Fs_type, "ignore") == 0) + +/* Historically, we have marked as "dummy" any file system of type "none", + but now that programs like du need to know about bind-mounted directories, + we grant an exception to any with "bind" in its list of mount options. + I.e., those are *not* dummy entries. */ +# define ME_DUMMY(Fs_name, Fs_type) \ + (ME_DUMMY_0 (Fs_name, Fs_type) || strcmp (Fs_type, "none") == 0) + +// ---------------------------------------------------------------------------- + // find the mount info with the given major:minor // in the supplied linked list of mountinfo structures struct mountinfo *mountinfo_find(struct mountinfo *root, unsigned long major, unsigned long minor) { struct mountinfo *mi; for(mi = root; mi ; mi = mi->next) - if(mi->major == major && mi->minor == minor) + if(unlikely(mi->major == major && mi->minor == minor)) return mi; return NULL; @@ -19,12 +62,12 @@ struct mountinfo *mountinfo_find_by_filesystem_mount_source(struct mountinfo *ro uint32_t filesystem_hash = simple_hash(filesystem), mount_source_hash = simple_hash(mount_source); for(mi = root; mi ; mi = mi->next) - if(mi->filesystem + if(unlikely(mi->filesystem && mi->mount_source && mi->filesystem_hash == filesystem_hash && mi->mount_source_hash == mount_source_hash && !strcmp(mi->filesystem, filesystem) - && !strcmp(mi->mount_source, mount_source)) + && !strcmp(mi->mount_source, mount_source))) return mi; return NULL; @@ -37,10 +80,10 @@ struct mountinfo *mountinfo_find_by_filesystem_super_option(struct mountinfo *ro size_t solen = strlen(super_options); for(mi = root; mi ; mi = mi->next) - if(mi->filesystem + if(unlikely(mi->filesystem && mi->super_options && mi->filesystem_hash == filesystem_hash - && !strcmp(mi->filesystem, filesystem)) { + && !strcmp(mi->filesystem, filesystem))) { // super_options is a comma separated list char *s = mi->super_options, *e; @@ -49,7 +92,7 @@ struct mountinfo *mountinfo_find_by_filesystem_super_option(struct mountinfo *ro while(*e && *e != ',') e++; size_t len = e - s; - if(len == solen && !strncmp(s, super_options, len)) + if(unlikely(len == solen && !strncmp(s, super_options, len))) return mi; if(*e == ',') s = ++e; @@ -72,6 +115,7 @@ void mountinfo_free(struct mountinfo *mi) { freez(mi->root); freez(mi->mount_point); freez(mi->mount_options); + freez(mi->persistent_id); /* if(mi->optional_fields_count) { @@ -113,56 +157,65 @@ static char *strdupz_decoding_octal(const char *string) { return buffer; } -// read the whole mountinfo into a linked list -struct mountinfo *mountinfo_read() { - procfile *ff = NULL; +static inline int is_read_only(const char *s) { + if(!s) return 0; + + size_t len = strlen(s); + if(len < 2) return 0; + if(len == 2) { + if(!strcmp(s, "ro")) return 1; + return 0; + } + if(!strncmp(s, "ro,", 3)) return 1; + if(!strncmp(&s[len - 3], ",ro", 3)) return 1; + if(strstr(s, ",ro,")) return 1; + return 0; +} +// read the whole mountinfo into a linked list +struct mountinfo *mountinfo_read(int do_statvfs) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s/proc/self/mountinfo", global_host_prefix); - ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT); - if(!ff) { + procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) { snprintfz(filename, FILENAME_MAX, "%s/proc/1/mountinfo", global_host_prefix); ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT); - if(!ff) return NULL; + if(unlikely(!ff)) return NULL; } ff = procfile_readall(ff); - if(!ff) return NULL; + if(unlikely(!ff)) + return NULL; struct mountinfo *root = NULL, *last = NULL, *mi = NULL; unsigned long l, lines = procfile_lines(ff); for(l = 0; l < lines ;l++) { - if(procfile_linewords(ff, l) < 5) + if(unlikely(procfile_linewords(ff, l) < 5)) continue; mi = mallocz(sizeof(struct mountinfo)); - if(unlikely(!root)) - root = last = mi; - else - last->next = mi; - - last = mi; - mi->next = NULL; - unsigned long w = 0; - mi->id = strtoul(procfile_lineword(ff, l, w), NULL, 10); w++; - mi->parentid = strtoul(procfile_lineword(ff, l, w), NULL, 10); w++; + mi->id = str2ul(procfile_lineword(ff, l, w)); w++; + mi->parentid = str2ul(procfile_lineword(ff, l, w)); w++; char *major = procfile_lineword(ff, l, w), *minor; w++; for(minor = major; *minor && *minor != ':' ;minor++) ; - if(!*minor) { + if(unlikely(!*minor)) { error("Cannot parse major:minor on '%s' at line %lu of '%s'", major, l + 1, filename); + freez(mi); continue; } *minor = '\0'; minor++; - mi->major = strtoul(major, NULL, 10); - mi->minor = strtoul(minor, NULL, 10); + mi->flags = 0; + + mi->major = str2ul(major); + mi->minor = str2ul(minor); mi->root = strdupz(procfile_lineword(ff, l, w)); w++; mi->root_hash = simple_hash(mi->root); @@ -170,8 +223,15 @@ struct mountinfo *mountinfo_read() { mi->mount_point = strdupz_decoding_octal(procfile_lineword(ff, l, w)); w++; mi->mount_point_hash = simple_hash(mi->mount_point); + mi->persistent_id = strdupz(mi->mount_point); + netdata_fix_chart_id(mi->persistent_id); + mi->persistent_id_hash = simple_hash(mi->persistent_id); + mi->mount_options = strdupz(procfile_lineword(ff, l, w)); w++; + if(unlikely(is_read_only(mi->mount_options))) + mi->flags |= MOUNTINFO_READONLY; + // count the optional fields /* unsigned long wo = w; @@ -189,14 +249,11 @@ struct mountinfo *mountinfo_read() { // we have some optional fields // read them into a new array of pointers; - mi->optional_fields = malloc(mi->optional_fields_count * sizeof(char *)); - if(unlikely(!mi->optional_fields)) - fatal("Cannot allocate memory for %d mountinfo optional fields", mi->optional_fields_count); + mi->optional_fields = mallocz(mi->optional_fields_count * sizeof(char *)); int i; for(i = 0; i < mi->optional_fields_count ; i++) { - *mi->optional_fields[wo] = strdup(procfile_lineword(ff, l, w)); - if(!mi->optional_fields[wo]) fatal("Cannot allocate memory"); + *mi->optional_fields[wo] = strdupz(procfile_lineword(ff, l, w)); wo++; } } @@ -210,33 +267,134 @@ struct mountinfo *mountinfo_read() { mi->filesystem = strdupz(procfile_lineword(ff, l, w)); w++; mi->filesystem_hash = simple_hash(mi->filesystem); - mi->mount_source = strdupz(procfile_lineword(ff, l, w)); w++; + mi->mount_source = strdupz_decoding_octal(procfile_lineword(ff, l, w)); w++; mi->mount_source_hash = simple_hash(mi->mount_source); mi->super_options = strdupz(procfile_lineword(ff, l, w)); w++; + + if(unlikely(is_read_only(mi->super_options))) + mi->flags |= MOUNTINFO_READONLY; + + if(unlikely(ME_DUMMY(mi->mount_source, mi->filesystem))) + mi->flags |= MOUNTINFO_IS_DUMMY; + + if(unlikely(ME_REMOTE(mi->mount_source, mi->filesystem))) + mi->flags |= MOUNTINFO_IS_REMOTE; + + // mark as BIND the duplicates (i.e. same filesystem + same source) + if(do_statvfs) { + struct stat buf; + if(unlikely(stat(mi->mount_point, &buf) == -1)) { + mi->st_dev = 0; + mi->flags |= MOUNTINFO_NO_STAT; + } + else { + mi->st_dev = buf.st_dev; + + struct mountinfo *mt; + for(mt = root; mt; mt = mt->next) { + if(unlikely(mt->st_dev == mi->st_dev && !(mi->flags & MOUNTINFO_NO_STAT))) { + if(strlen(mi->mount_point) < strlen(mt->mount_point)) + mt->flags |= MOUNTINFO_IS_SAME_DEV; + else + mi->flags |= MOUNTINFO_IS_SAME_DEV; + } + } + } + } + else { + mi->st_dev = 0; + } } else { mi->filesystem = NULL; + mi->filesystem_hash = 0; + mi->mount_source = NULL; + mi->mount_source_hash = 0; + mi->super_options = NULL; + + mi->st_dev = 0; + } + + // check if it has size + if(do_statvfs) { + struct statvfs buff_statvfs; + if(unlikely(statvfs(mi->mount_point, &buff_statvfs) < 0)) { + mi->flags |= MOUNTINFO_NO_STAT; + } + else if(unlikely(!buff_statvfs.f_blocks /* || !buff_statvfs.f_files */)) { + mi->flags |= MOUNTINFO_NO_SIZE; + } } + // link it + if(unlikely(!root)) + root = mi; + else + last->next = mi; + + last = mi; + mi->next = NULL; + /* - info("MOUNTINFO: %u %u %u:%u root '%s', mount point '%s', mount options '%s', filesystem '%s', mount source '%s', super options '%s'", +#ifdef NETDATA_INTERNAL_CHECKS + fprintf(stderr, "MOUNTINFO: %ld %ld %lu:%lu root '%s', persistent id '%s', mount point '%s', mount options '%s', filesystem '%s', mount source '%s', super options '%s'%s%s%s%s%s%s\n", mi->id, mi->parentid, mi->major, mi->minor, mi->root, - mi->mount_point, - mi->mount_options, - mi->filesystem, - mi->mount_source, - mi->super_options + mi->persistent_id, + (mi->mount_point)?mi->mount_point:"", + (mi->mount_options)?mi->mount_options:"", + (mi->filesystem)?mi->filesystem:"", + (mi->mount_source)?mi->mount_source:"", + (mi->super_options)?mi->super_options:"", + (mi->flags & MOUNTINFO_IS_DUMMY)?" DUMMY":"", + (mi->flags & MOUNTINFO_IS_BIND)?" BIND":"", + (mi->flags & MOUNTINFO_IS_REMOTE)?" REMOTE":"", + (mi->flags & MOUNTINFO_NO_STAT)?" NOSTAT":"", + (mi->flags & MOUNTINFO_NO_SIZE)?" NOSIZE":"", + (mi->flags & MOUNTINFO_IS_SAME_DEV)?" SAMEDEV":"" ); +#endif */ } +/* find if the mount options have "bind" in them + { + FILE *fp = setmntent(MOUNTED, "r"); + if (fp != NULL) { + struct mntent mntbuf; + struct mntent *mnt; + char buf[4096 + 1]; + + while ((mnt = getmntent_r(fp, &mntbuf, buf, 4096))) { + char *bind = hasmntopt(mnt, "bind"); + if(unlikely(bind)) { + struct mountinfo *mi; + for(mi = root; mi ; mi = mi->next) { + if(unlikely(strcmp(mnt->mnt_dir, mi->mount_point) == 0)) { + fprintf(stderr, "Mount point '%s' is BIND\n", mi->mount_point); + mi->flags |= MOUNTINFO_IS_BIND; + break; + } + } + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(!mi)) { + error("Mount point '%s' not found in /proc/self/mountinfo", mnt->mnt_dir); + } +#endif + } + } + endmntent(fp); + } + } +*/ + procfile_close(ff); return root; } diff --git a/src/proc_self_mountinfo.h b/src/proc_self_mountinfo.h index c2d9688c1..00cf699ab 100644 --- a/src/proc_self_mountinfo.h +++ b/src/proc_self_mountinfo.h @@ -1,12 +1,23 @@ #ifndef NETDATA_PROC_SELF_MOUNTINFO_H #define NETDATA_PROC_SELF_MOUNTINFO_H 1 +#define MOUNTINFO_IS_DUMMY 0x00000001 +#define MOUNTINFO_IS_REMOTE 0x00000002 +#define MOUNTINFO_IS_BIND 0x00000004 +#define MOUNTINFO_IS_SAME_DEV 0x00000008 +#define MOUNTINFO_NO_STAT 0x00000010 +#define MOUNTINFO_NO_SIZE 0x00000020 +#define MOUNTINFO_READONLY 0x00000040 + struct mountinfo { long id; // mount ID: unique identifier of the mount (may be reused after umount(2)). long parentid; // parent ID: ID of parent mount (or of self for the top of the mount tree). unsigned long major; // major:minor: value of st_dev for files on filesystem (see stat(2)). unsigned long minor; + char *persistent_id; // a calculated persistent id for the mount point + uint32_t persistent_id_hash; + char *root; // root: root of the mount within the filesystem. uint32_t root_hash; @@ -27,6 +38,10 @@ struct mountinfo { char *super_options; // super options: per-superblock options. + uint32_t flags; + + dev_t st_dev; // id of device as given by stat() + struct mountinfo *next; }; @@ -35,6 +50,6 @@ extern struct mountinfo *mountinfo_find_by_filesystem_mount_source(struct mounti extern struct mountinfo *mountinfo_find_by_filesystem_super_option(struct mountinfo *root, const char *filesystem, const char *super_options); extern void mountinfo_free(struct mountinfo *mi); -extern struct mountinfo *mountinfo_read(); +extern struct mountinfo *mountinfo_read(int do_statvfs); #endif /* NETDATA_PROC_SELF_MOUNTINFO_H */
\ No newline at end of file diff --git a/src/proc_softirqs.c b/src/proc_softirqs.c index ebbbf2aeb..c7b10d70d 100644 --- a/src/proc_softirqs.c +++ b/src/proc_softirqs.c @@ -2,71 +2,89 @@ #define MAX_INTERRUPT_NAME 50 +struct cpu_interrupt { + unsigned long long value; + RRDDIM *rd; +}; + struct interrupt { int used; char *id; char name[MAX_INTERRUPT_NAME + 1]; + RRDDIM *rd; unsigned long long total; - unsigned long long value[]; + struct cpu_interrupt cpu[]; }; // since each interrupt is variable in size // we use this to calculate its record size -#define recordsize(cpus) (sizeof(struct interrupt) + (cpus * sizeof(unsigned long long))) +#define recordsize(cpus) (sizeof(struct interrupt) + (cpus * sizeof(struct cpu_interrupt))) // given a base, get a pointer to each record #define irrindex(base, line, cpus) ((struct interrupt *)&((char *)(base))[line * recordsize(cpus)]) -static inline struct interrupt *get_interrupts_array(int lines, int cpus) { +static inline struct interrupt *get_interrupts_array(size_t lines, int cpus) { static struct interrupt *irrs = NULL; - static int allocated = 0; + static size_t allocated = 0; + + if(unlikely(lines != allocated)) { + uint32_t l; + int c; - if(lines > allocated) { irrs = (struct interrupt *)reallocz(irrs, lines * recordsize(cpus)); + + // reset all interrupt RRDDIM pointers as any line could have shifted + for(l = 0; l < lines ;l++) { + struct interrupt *irr = irrindex(irrs, l, cpus); + irr->rd = NULL; + irr->name[0] = '\0'; + for(c = 0; c < cpus ;c++) + irr->cpu[c].rd = NULL; + } + allocated = lines; } return irrs; } -int do_proc_softirqs(int update_every, unsigned long long dt) { +int do_proc_softirqs(int update_every, usec_t dt) { + (void)dt; static procfile *ff = NULL; static int cpus = -1, do_per_core = -1; - struct interrupt *irrs = NULL; - if(dt) {}; + if(unlikely(do_per_core == -1)) do_per_core = config_get_boolean("plugin:proc:/proc/softirqs", "interrupts per core", 1); - if(do_per_core == -1) do_per_core = config_get_boolean("plugin:proc:/proc/softirqs", "interrupts per core", 1); - - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/softirqs"); ff = procfile_open(config_get("plugin:proc:/proc/softirqs", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - uint32_t lines = procfile_lines(ff), l; - uint32_t words = procfile_linewords(ff, 0), w; + size_t lines = procfile_lines(ff), l; + size_t words = procfile_linewords(ff, 0); - if(!lines) { + if(unlikely(!lines)) { error("Cannot read /proc/softirqs, zero lines reported."); return 1; } // find how many CPUs are there - if(cpus == -1) { + if(unlikely(cpus == -1)) { + uint32_t w; cpus = 0; for(w = 0; w < words ; w++) { - if(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0) + if(likely(strncmp(procfile_lineword(ff, 0, w), "CPU", 3) == 0)) cpus++; } } - if(!cpus) { + if(unlikely(!cpus)) { error("PLUGIN: PROC_SOFTIRQS: Cannot find the number of CPUs in /proc/softirqs"); return 1; } @@ -82,23 +100,23 @@ int do_proc_softirqs(int update_every, unsigned long long dt) { irr->total = 0; words = procfile_linewords(ff, l); - if(!words) continue; + if(unlikely(!words)) continue; irr->id = procfile_lineword(ff, l, 0); - if(!irr->id || !irr->id[0]) continue; + if(unlikely(!irr->id || !irr->id[0])) continue; - int idlen = strlen(irr->id); - if(irr->id[idlen - 1] == ':') + size_t idlen = strlen(irr->id); + if(unlikely(idlen && irr->id[idlen - 1] == ':')) irr->id[idlen - 1] = '\0'; int c; for(c = 0; c < cpus ;c++) { - if((c + 1) < (int)words) - irr->value[c] = strtoull(procfile_lineword(ff, l, (uint32_t)(c + 1)), NULL, 10); + if(likely((c + 1) < (int)words)) + irr->cpu[c].value = str2ull(procfile_lineword(ff, l, (uint32_t)(c + 1))); else - irr->value[c] = 0; + irr->cpu[c].value = 0; - irr->total += irr->value[c]; + irr->total += irr->cpu[c].value; } strncpyz(irr->name, irr->id, MAX_INTERRUPT_NAME); @@ -111,21 +129,30 @@ int do_proc_softirqs(int update_every, unsigned long long dt) { // -------------------------------------------------------------------- st = rrdset_find_bytype("system", "softirqs"); - if(!st) { - st = rrdset_create("system", "softirqs", NULL, "softirqs", NULL, "System softirqs", "softirqs/s", 950, update_every, RRDSET_TYPE_STACKED); - - for(l = 0; l < lines ;l++) { - struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); - } - } + if(unlikely(!st)) st = rrdset_create("system", "softirqs", NULL, "softirqs", NULL, "System softirqs", "softirqs/s", 950, update_every, RRDSET_TYPE_STACKED); else rrdset_next(st); for(l = 0; l < lines ;l++) { struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_set(st, irr->id, irr->total); + if(unlikely(!irr->used)) continue; + // some interrupt may have changed without changing the total number of lines + // if the same number of interrupts have been added and removed between two + // calls of this function. + if(unlikely(!irr->rd || strncmp(irr->name, irr->rd->name, MAX_INTERRUPT_NAME) != 0)) { + irr->rd = rrddim_find(st, irr->id); + if(unlikely(!irr->rd)) + irr->rd = rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); + else + rrddim_set_name(st, irr->rd, irr->name); + + // also reset per cpu RRDDIMs to avoid repeating strncmp() in the per core loop + if(likely(do_per_core)) { + int c; + for (c = 0; c < cpus ;c++) + irr->cpu[c].rd = NULL; + } + } + rrddim_set_by_pointer(st, irr->rd, irr->total); } rrdset_done(st); @@ -137,32 +164,33 @@ int do_proc_softirqs(int update_every, unsigned long long dt) { snprintfz(id, 50, "cpu%d_softirqs", c); st = rrdset_find_bytype("cpu", id); - if(!st) { + if(unlikely(!st)) { // find if everything is zero unsigned long long core_sum = 0 ; for(l = 0; l < lines ;l++) { struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - core_sum += irr->value[c]; + if(unlikely(!irr->used)) continue; + core_sum += irr->cpu[c].value; } - if(core_sum == 0) continue; // try next core + if(unlikely(core_sum == 0)) continue; // try next core char title[100+1]; snprintfz(title, 100, "CPU%d softirqs", c); st = rrdset_create("cpu", id, NULL, "softirqs", "cpu.softirqs", title, "softirqs/s", 3000 + c, update_every, RRDSET_TYPE_STACKED); - - for(l = 0; l < lines ;l++) { - struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); - } } else rrdset_next(st); for(l = 0; l < lines ;l++) { struct interrupt *irr = irrindex(irrs, l, cpus); - if(!irr->used) continue; - rrddim_set(st, irr->id, irr->value[c]); + if(unlikely(!irr->used)) continue; + if(unlikely(!irr->cpu[c].rd)) { + irr->cpu[c].rd = rrddim_find(st, irr->id); + if(unlikely(!irr->cpu[c].rd)) + irr->cpu[c].rd = rrddim_add(st, irr->id, irr->name, 1, 1, RRDDIM_INCREMENTAL); + else + rrddim_set_name(st, irr->cpu[c].rd, irr->name); + } + rrddim_set_by_pointer(st, irr->cpu[c].rd, irr->cpu[c].value); } rrdset_done(st); } diff --git a/src/proc_stat.c b/src/proc_stat.c index 88cb820b3..f7e6d5bc0 100644 --- a/src/proc_stat.c +++ b/src/proc_stat.c @@ -1,6 +1,6 @@ #include "common.h" -int do_proc_stat(int update_every, unsigned long long dt) { +int do_proc_stat(int update_every, usec_t dt) { (void)dt; static procfile *ff = NULL; @@ -32,8 +32,8 @@ int do_proc_stat(int update_every, unsigned long long dt) { ff = procfile_readall(ff); if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - uint32_t lines = procfile_lines(ff), l; - uint32_t words; + size_t lines = procfile_lines(ff), l; + size_t words; unsigned long long processes = 0, running = 0 , blocked = 0; RRDSET *st; @@ -46,7 +46,7 @@ int do_proc_stat(int update_every, unsigned long long dt) { if(likely(row_key[0] == 'c' && row_key[1] == 'p' && row_key[2] == 'u')) { words = procfile_linewords(ff, l); if(unlikely(words < 9)) { - error("Cannot read /proc/stat cpu line. Expected 9 params, read %u.", words); + error("Cannot read /proc/stat cpu line. Expected 9 params, read %zu.", words); continue; } @@ -54,19 +54,19 @@ int do_proc_stat(int update_every, unsigned long long dt) { unsigned long long user = 0, nice = 0, system = 0, idle = 0, iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guest_nice = 0; id = row_key; - user = strtoull(procfile_lineword(ff, l, 1), NULL, 10); - nice = strtoull(procfile_lineword(ff, l, 2), NULL, 10); - system = strtoull(procfile_lineword(ff, l, 3), NULL, 10); - idle = strtoull(procfile_lineword(ff, l, 4), NULL, 10); - iowait = strtoull(procfile_lineword(ff, l, 5), NULL, 10); - irq = strtoull(procfile_lineword(ff, l, 6), NULL, 10); - softirq = strtoull(procfile_lineword(ff, l, 7), NULL, 10); - steal = strtoull(procfile_lineword(ff, l, 8), NULL, 10); - - guest = strtoull(procfile_lineword(ff, l, 9), NULL, 10); + user = str2ull(procfile_lineword(ff, l, 1)); + nice = str2ull(procfile_lineword(ff, l, 2)); + system = str2ull(procfile_lineword(ff, l, 3)); + idle = str2ull(procfile_lineword(ff, l, 4)); + iowait = str2ull(procfile_lineword(ff, l, 5)); + irq = str2ull(procfile_lineword(ff, l, 6)); + softirq = str2ull(procfile_lineword(ff, l, 7)); + steal = str2ull(procfile_lineword(ff, l, 8)); + + guest = str2ull(procfile_lineword(ff, l, 9)); user -= guest; - guest_nice = strtoull(procfile_lineword(ff, l, 10), NULL, 10); + guest_nice = str2ull(procfile_lineword(ff, l, 10)); nice -= guest_nice; char *title, *type, *context, *family; @@ -126,8 +126,8 @@ int do_proc_stat(int update_every, unsigned long long dt) { rrdset_done(st); } } - else if(hash == hash_intr && strcmp(row_key, "intr") == 0) { - unsigned long long value = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + else if(unlikely(hash == hash_intr && strcmp(row_key, "intr") == 0)) { + unsigned long long value = str2ull(procfile_lineword(ff, l, 1)); // -------------------------------------------------------------------- @@ -145,8 +145,8 @@ int do_proc_stat(int update_every, unsigned long long dt) { rrdset_done(st); } } - else if(hash == hash_ctxt && strcmp(row_key, "ctxt") == 0) { - unsigned long long value = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + else if(unlikely(hash == hash_ctxt && strcmp(row_key, "ctxt") == 0)) { + unsigned long long value = str2ull(procfile_lineword(ff, l, 1)); // -------------------------------------------------------------------- @@ -163,14 +163,14 @@ int do_proc_stat(int update_every, unsigned long long dt) { rrdset_done(st); } } - else if(hash == hash_processes && !processes && strcmp(row_key, "processes") == 0) { - processes = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + else if(unlikely(hash == hash_processes && !processes && strcmp(row_key, "processes") == 0)) { + processes = str2ull(procfile_lineword(ff, l, 1)); } - else if(hash == hash_procs_running && !running && strcmp(row_key, "procs_running") == 0) { - running = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + else if(unlikely(hash == hash_procs_running && !running && strcmp(row_key, "procs_running") == 0)) { + running = str2ull(procfile_lineword(ff, l, 1)); } - else if(hash == hash_procs_blocked && !blocked && strcmp(row_key, "procs_blocked") == 0) { - blocked = strtoull(procfile_lineword(ff, l, 1), NULL, 10); + else if(unlikely(hash == hash_procs_blocked && !blocked && strcmp(row_key, "procs_blocked") == 0)) { + blocked = str2ull(procfile_lineword(ff, l, 1)); } } diff --git a/src/proc_sys_kernel_random_entropy_avail.c b/src/proc_sys_kernel_random_entropy_avail.c index 9515dad61..388406e0b 100644 --- a/src/proc_sys_kernel_random_entropy_avail.c +++ b/src/proc_sys_kernel_random_entropy_avail.c @@ -1,24 +1,24 @@ #include "common.h" -int do_proc_sys_kernel_random_entropy_avail(int update_every, unsigned long long dt) { - static procfile *ff = NULL; +int do_proc_sys_kernel_random_entropy_avail(int update_every, usec_t dt) { + (void)dt; - if(dt) {} ; + static procfile *ff = NULL; - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/sys/kernel/random/entropy_avail"); ff = procfile_open(config_get("plugin:proc:/proc/sys/kernel/random/entropy_avail", "filename to monitor", filename), "", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time - unsigned long long entropy = strtoull(procfile_lineword(ff, 0, 0), NULL, 10); + unsigned long long entropy = str2ull(procfile_lineword(ff, 0, 0)); RRDSET *st = rrdset_find_bytype("system", "entropy"); - if(!st) { + if(unlikely(!st)) { st = rrdset_create("system", "entropy", NULL, "entropy", NULL, "Available Entropy", "entropy", 1000, update_every, RRDSET_TYPE_LINE); rrddim_add(st, "entropy", NULL, 1, 1, RRDDIM_ABSOLUTE); } diff --git a/src/proc_uptime.c b/src/proc_uptime.c new file mode 100644 index 000000000..9f341a33f --- /dev/null +++ b/src/proc_uptime.c @@ -0,0 +1,54 @@ +#include "common.h" + +int do_proc_uptime(int update_every, usec_t dt) { + (void)dt; + + static RRDSET *st = NULL; + collected_number uptime = 0; + +#ifdef CLOCK_BOOTTIME_IS_AVAILABLE + uptime = now_boottime_usec() / 1000; +#else + static procfile *ff = NULL; + + if(unlikely(!ff)) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/uptime"); + + ff = procfile_open(config_get("plugin:proc:/proc/uptime", "filename to monitor", filename), " \t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) + return 1; + } + + ff = procfile_readall(ff); + if(unlikely(!ff)) + return 0; // we return 0, so that we will retry to open it next time + + if(unlikely(procfile_lines(ff) < 1)) { + error("/proc/uptime has no lines."); + return 1; + } + if(unlikely(procfile_linewords(ff, 0) < 1)) { + error("/proc/uptime has less than 1 word in it."); + return 1; + } + + uptime = (collected_number)(strtold(procfile_lineword(ff, 0, 0), NULL) * 1000.0); +#endif + + // -------------------------------------------------------------------- + + if(unlikely(!st)) + st = rrdset_find("system.uptime"); + + if(unlikely(!st)) { + st = rrdset_create("system", "uptime", NULL, "uptime", NULL, "System Uptime", "seconds", 1000, update_every, RRDSET_TYPE_LINE); + rrddim_add(st, "uptime", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + else rrdset_next(st); + + rrddim_set(st, "uptime", uptime); + rrdset_done(st); + + return 0; +} diff --git a/src/proc_vmstat.c b/src/proc_vmstat.c index 5f4e5aadd..ea917b989 100644 --- a/src/proc_vmstat.c +++ b/src/proc_vmstat.c @@ -1,416 +1,88 @@ #include "common.h" -int do_proc_vmstat(int update_every, unsigned long long dt) { +int do_proc_vmstat(int update_every, usec_t dt) { + (void)dt; + static procfile *ff = NULL; - static int do_swapio = -1, do_io = -1, do_pgfaults = -1, gen_hashes = -1; - - // static uint32_t hash_allocstall = -1; - // static uint32_t hash_compact_blocks_moved = -1; - // static uint32_t hash_compact_fail = -1; - // static uint32_t hash_compact_pagemigrate_failed = -1; - // static uint32_t hash_compact_pages_moved = -1; - // static uint32_t hash_compact_stall = -1; - // static uint32_t hash_compact_success = -1; - // static uint32_t hash_htlb_buddy_alloc_fail = -1; - // static uint32_t hash_htlb_buddy_alloc_success = -1; - // static uint32_t hash_kswapd_high_wmark_hit_quickly = -1; - // static uint32_t hash_kswapd_inodesteal = -1; - // static uint32_t hash_kswapd_low_wmark_hit_quickly = -1; - // static uint32_t hash_kswapd_skip_congestion_wait = -1; - // static uint32_t hash_nr_active_anon = -1; - // static uint32_t hash_nr_active_file = -1; - // static uint32_t hash_nr_anon_pages = -1; - // static uint32_t hash_nr_anon_transparent_hugepages = -1; - // static uint32_t hash_nr_bounce = -1; - // static uint32_t hash_nr_dirtied = -1; - // static uint32_t hash_nr_dirty = -1; - // static uint32_t hash_nr_dirty_background_threshold = -1; - // static uint32_t hash_nr_dirty_threshold = -1; - // static uint32_t hash_nr_file_pages = -1; - // static uint32_t hash_nr_free_pages = -1; - // static uint32_t hash_nr_inactive_anon = -1; - // static uint32_t hash_nr_inactive_file = -1; - // static uint32_t hash_nr_isolated_anon = -1; - // static uint32_t hash_nr_isolated_file = -1; - // static uint32_t hash_nr_kernel_stack = -1; - // static uint32_t hash_nr_mapped = -1; - // static uint32_t hash_nr_mlock = -1; - // static uint32_t hash_nr_page_table_pages = -1; - // static uint32_t hash_nr_shmem = -1; - // static uint32_t hash_nr_slab_reclaimable = -1; - // static uint32_t hash_nr_slab_unreclaimable = -1; - // static uint32_t hash_nr_unevictable = -1; - // static uint32_t hash_nr_unstable = -1; - // static uint32_t hash_nr_vmscan_immediate_reclaim = -1; - // static uint32_t hash_nr_vmscan_write = -1; - // static uint32_t hash_nr_writeback = -1; - // static uint32_t hash_nr_writeback_temp = -1; - // static uint32_t hash_nr_written = -1; - // static uint32_t hash_pageoutrun = -1; - // static uint32_t hash_pgactivate = -1; - // static uint32_t hash_pgalloc_dma = -1; - // static uint32_t hash_pgalloc_dma32 = -1; - // static uint32_t hash_pgalloc_movable = -1; - // static uint32_t hash_pgalloc_normal = -1; - // static uint32_t hash_pgdeactivate = -1; - static uint32_t hash_pgfault = -1; - // static uint32_t hash_pgfree = -1; - // static uint32_t hash_pginodesteal = -1; - static uint32_t hash_pgmajfault = -1; - static uint32_t hash_pgpgin = -1; - static uint32_t hash_pgpgout = -1; - // static uint32_t hash_pgrefill_dma = -1; - // static uint32_t hash_pgrefill_dma32 = -1; - // static uint32_t hash_pgrefill_movable = -1; - // static uint32_t hash_pgrefill_normal = -1; - // static uint32_t hash_pgrotated = -1; - // static uint32_t hash_pgscan_direct_dma = -1; - // static uint32_t hash_pgscan_direct_dma32 = -1; - // static uint32_t hash_pgscan_direct_movable = -1; - // static uint32_t hash_pgscan_direct_normal = -1; - // static uint32_t hash_pgscan_kswapd_dma = -1; - // static uint32_t hash_pgscan_kswapd_dma32 = -1; - // static uint32_t hash_pgscan_kswapd_movable = -1; - // static uint32_t hash_pgscan_kswapd_normal = -1; - // static uint32_t hash_pgsteal_direct_dma = -1; - // static uint32_t hash_pgsteal_direct_dma32 = -1; - // static uint32_t hash_pgsteal_direct_movable = -1; - // static uint32_t hash_pgsteal_direct_normal = -1; - // static uint32_t hash_pgsteal_kswapd_dma = -1; - // static uint32_t hash_pgsteal_kswapd_dma32 = -1; - // static uint32_t hash_pgsteal_kswapd_movable = -1; - // static uint32_t hash_pgsteal_kswapd_normal = -1; - static uint32_t hash_pswpin = -1; - static uint32_t hash_pswpout = -1; - // static uint32_t hash_slabs_scanned = -1; - // static uint32_t hash_thp_collapse_alloc = -1; - // static uint32_t hash_thp_collapse_alloc_failed = -1; - // static uint32_t hash_thp_fault_alloc = -1; - // static uint32_t hash_thp_fault_fallback = -1; - // static uint32_t hash_thp_split = -1; - // static uint32_t hash_unevictable_pgs_cleared = -1; - // static uint32_t hash_unevictable_pgs_culled = -1; - // static uint32_t hash_unevictable_pgs_mlocked = -1; - // static uint32_t hash_unevictable_pgs_mlockfreed = -1; - // static uint32_t hash_unevictable_pgs_munlocked = -1; - // static uint32_t hash_unevictable_pgs_rescued = -1; - // static uint32_t hash_unevictable_pgs_scanned = -1; - // static uint32_t hash_unevictable_pgs_stranded = -1; - - if(gen_hashes != 1) { - gen_hashes = 1; - // hash_allocstall = simple_hash("allocstall"); - // hash_compact_blocks_moved = simple_hash("compact_blocks_moved"); - // hash_compact_fail = simple_hash("compact_fail"); - // hash_compact_pagemigrate_failed = simple_hash("compact_pagemigrate_failed"); - // hash_compact_pages_moved = simple_hash("compact_pages_moved"); - // hash_compact_stall = simple_hash("compact_stall"); - // hash_compact_success = simple_hash("compact_success"); - // hash_htlb_buddy_alloc_fail = simple_hash("htlb_buddy_alloc_fail"); - // hash_htlb_buddy_alloc_success = simple_hash("htlb_buddy_alloc_success"); - // hash_kswapd_high_wmark_hit_quickly = simple_hash("kswapd_high_wmark_hit_quickly"); - // hash_kswapd_inodesteal = simple_hash("kswapd_inodesteal"); - // hash_kswapd_low_wmark_hit_quickly = simple_hash("kswapd_low_wmark_hit_quickly"); - // hash_kswapd_skip_congestion_wait = simple_hash("kswapd_skip_congestion_wait"); - // hash_nr_active_anon = simple_hash("nr_active_anon"); - // hash_nr_active_file = simple_hash("nr_active_file"); - // hash_nr_anon_pages = simple_hash("nr_anon_pages"); - // hash_nr_anon_transparent_hugepages = simple_hash("nr_anon_transparent_hugepages"); - // hash_nr_bounce = simple_hash("nr_bounce"); - // hash_nr_dirtied = simple_hash("nr_dirtied"); - // hash_nr_dirty = simple_hash("nr_dirty"); - // hash_nr_dirty_background_threshold = simple_hash("nr_dirty_background_threshold"); - // hash_nr_dirty_threshold = simple_hash("nr_dirty_threshold"); - // hash_nr_file_pages = simple_hash("nr_file_pages"); - // hash_nr_free_pages = simple_hash("nr_free_pages"); - // hash_nr_inactive_anon = simple_hash("nr_inactive_anon"); - // hash_nr_inactive_file = simple_hash("nr_inactive_file"); - // hash_nr_isolated_anon = simple_hash("nr_isolated_anon"); - // hash_nr_isolated_file = simple_hash("nr_isolated_file"); - // hash_nr_kernel_stack = simple_hash("nr_kernel_stack"); - // hash_nr_mapped = simple_hash("nr_mapped"); - // hash_nr_mlock = simple_hash("nr_mlock"); - // hash_nr_page_table_pages = simple_hash("nr_page_table_pages"); - // hash_nr_shmem = simple_hash("nr_shmem"); - // hash_nr_slab_reclaimable = simple_hash("nr_slab_reclaimable"); - // hash_nr_slab_unreclaimable = simple_hash("nr_slab_unreclaimable"); - // hash_nr_unevictable = simple_hash("nr_unevictable"); - // hash_nr_unstable = simple_hash("nr_unstable"); - // hash_nr_vmscan_immediate_reclaim = simple_hash("nr_vmscan_immediate_reclaim"); - // hash_nr_vmscan_write = simple_hash("nr_vmscan_write"); - // hash_nr_writeback = simple_hash("nr_writeback"); - // hash_nr_writeback_temp = simple_hash("nr_writeback_temp"); - // hash_nr_written = simple_hash("nr_written"); - // hash_pageoutrun = simple_hash("pageoutrun"); - // hash_pgactivate = simple_hash("pgactivate"); - // hash_pgalloc_dma = simple_hash("pgalloc_dma"); - // hash_pgalloc_dma32 = simple_hash("pgalloc_dma32"); - // hash_pgalloc_movable = simple_hash("pgalloc_movable"); - // hash_pgalloc_normal = simple_hash("pgalloc_normal"); - // hash_pgdeactivate = simple_hash("pgdeactivate"); - hash_pgfault = simple_hash("pgfault"); - // hash_pgfree = simple_hash("pgfree"); - // hash_pginodesteal = simple_hash("pginodesteal"); - hash_pgmajfault = simple_hash("pgmajfault"); - hash_pgpgin = simple_hash("pgpgin"); - hash_pgpgout = simple_hash("pgpgout"); - // hash_pgrefill_dma = simple_hash("pgrefill_dma"); - // hash_pgrefill_dma32 = simple_hash("pgrefill_dma32"); - // hash_pgrefill_movable = simple_hash("pgrefill_movable"); - // hash_pgrefill_normal = simple_hash("pgrefill_normal"); - // hash_pgrotated = simple_hash("pgrotated"); - // hash_pgscan_direct_dma = simple_hash("pgscan_direct_dma"); - // hash_pgscan_direct_dma32 = simple_hash("pgscan_direct_dma32"); - // hash_pgscan_direct_movable = simple_hash("pgscan_direct_movable"); - // hash_pgscan_direct_normal = simple_hash("pgscan_direct_normal"); - // hash_pgscan_kswapd_dma = simple_hash("pgscan_kswapd_dma"); - // hash_pgscan_kswapd_dma32 = simple_hash("pgscan_kswapd_dma32"); - // hash_pgscan_kswapd_movable = simple_hash("pgscan_kswapd_movable"); - // hash_pgscan_kswapd_normal = simple_hash("pgscan_kswapd_normal"); - // hash_pgsteal_direct_dma = simple_hash("pgsteal_direct_dma"); - // hash_pgsteal_direct_dma32 = simple_hash("pgsteal_direct_dma32"); - // hash_pgsteal_direct_movable = simple_hash("pgsteal_direct_movable"); - // hash_pgsteal_direct_normal = simple_hash("pgsteal_direct_normal"); - // hash_pgsteal_kswapd_dma = simple_hash("pgsteal_kswapd_dma"); - // hash_pgsteal_kswapd_dma32 = simple_hash("pgsteal_kswapd_dma32"); - // hash_pgsteal_kswapd_movable = simple_hash("pgsteal_kswapd_movable"); - // hash_pgsteal_kswapd_normal = simple_hash("pgsteal_kswapd_normal"); - hash_pswpin = simple_hash("pswpin"); - hash_pswpout = simple_hash("pswpout"); - // hash_slabs_scanned = simple_hash("slabs_scanned"); - // hash_thp_collapse_alloc = simple_hash("thp_collapse_alloc"); - // hash_thp_collapse_alloc_failed = simple_hash("thp_collapse_alloc_failed"); - // hash_thp_fault_alloc = simple_hash("thp_fault_alloc"); - // hash_thp_fault_fallback = simple_hash("thp_fault_fallback"); - // hash_thp_split = simple_hash("thp_split"); - // hash_unevictable_pgs_cleared = simple_hash("unevictable_pgs_cleared"); - // hash_unevictable_pgs_culled = simple_hash("unevictable_pgs_culled"); - // hash_unevictable_pgs_mlocked = simple_hash("unevictable_pgs_mlocked"); - // hash_unevictable_pgs_mlockfreed = simple_hash("unevictable_pgs_mlockfreed"); - // hash_unevictable_pgs_munlocked = simple_hash("unevictable_pgs_munlocked"); - // hash_unevictable_pgs_rescued = simple_hash("unevictable_pgs_rescued"); - // hash_unevictable_pgs_scanned = simple_hash("unevictable_pgs_scanned"); - // hash_unevictable_pgs_stranded = simple_hash("unevictable_pgs_stranded"); + static int do_swapio = -1, do_io = -1, do_pgfaults = -1, do_numa = -1; + static int has_numa = -1; + + static ARL_BASE *arl_base = NULL; + static unsigned long long numa_foreign = 0ULL; + static unsigned long long numa_hint_faults = 0ULL; + static unsigned long long numa_hint_faults_local = 0ULL; + static unsigned long long numa_huge_pte_updates = 0ULL; + static unsigned long long numa_interleave = 0ULL; + static unsigned long long numa_local = 0ULL; + static unsigned long long numa_other = 0ULL; + static unsigned long long numa_pages_migrated = 0ULL; + static unsigned long long numa_pte_updates = 0ULL; + static unsigned long long pgfault = 0ULL; + static unsigned long long pgmajfault = 0ULL; + static unsigned long long pgpgin = 0ULL; + static unsigned long long pgpgout = 0ULL; + static unsigned long long pswpin = 0ULL; + static unsigned long long pswpout = 0ULL; + + if(unlikely(!arl_base)) { + do_swapio = config_get_boolean_ondemand("plugin:proc:/proc/vmstat", "swap i/o", CONFIG_ONDEMAND_ONDEMAND); + do_io = config_get_boolean("plugin:proc:/proc/vmstat", "disk i/o", 1); + do_pgfaults = config_get_boolean("plugin:proc:/proc/vmstat", "memory page faults", 1); + do_numa = config_get_boolean_ondemand("plugin:proc:/proc/vmstat", "system-wide numa metric summary", CONFIG_ONDEMAND_ONDEMAND); + + + arl_base = arl_create("vmstat", NULL, 60); + arl_expect(arl_base, "pgfault", &pgfault); + arl_expect(arl_base, "pgmajfault", &pgmajfault); + arl_expect(arl_base, "pgpgin", &pgpgin); + arl_expect(arl_base, "pgpgout", &pgpgout); + arl_expect(arl_base, "pswpin", &pswpin); + arl_expect(arl_base, "pswpout", &pswpout); + + if(do_numa == CONFIG_ONDEMAND_YES || (do_numa == CONFIG_ONDEMAND_ONDEMAND && get_numa_node_count() >= 2)) { + arl_expect(arl_base, "numa_foreign", &numa_foreign); + arl_expect(arl_base, "numa_hint_faults_local", &numa_hint_faults_local); + arl_expect(arl_base, "numa_hint_faults", &numa_hint_faults); + arl_expect(arl_base, "numa_huge_pte_updates", &numa_huge_pte_updates); + arl_expect(arl_base, "numa_interleave", &numa_interleave); + arl_expect(arl_base, "numa_local", &numa_local); + arl_expect(arl_base, "numa_other", &numa_other); + arl_expect(arl_base, "numa_pages_migrated", &numa_pages_migrated); + arl_expect(arl_base, "numa_pte_updates", &numa_pte_updates); + } + else { + // Do not expect numa metrics when they are not needed. + // By not adding them, the ARL will stop processing the file + // when all the expected metrics are collected. + // Also ARL will not parse their values. + has_numa = 0; + do_numa = CONFIG_ONDEMAND_NO; + } } - if(do_swapio == -1) do_swapio = config_get_boolean_ondemand("plugin:proc:/proc/vmstat", "swap i/o", CONFIG_ONDEMAND_ONDEMAND); - if(do_io == -1) do_io = config_get_boolean("plugin:proc:/proc/vmstat", "disk i/o", 1); - if(do_pgfaults == -1) do_pgfaults = config_get_boolean("plugin:proc:/proc/vmstat", "memory page faults", 1); - - (void)dt; - - if(!ff) { + if(unlikely(!ff)) { char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, "/proc/vmstat"); ff = procfile_open(config_get("plugin:proc:/proc/vmstat", "filename to monitor", filename), " \t:", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 1; } - if(!ff) return 1; ff = procfile_readall(ff); - if(!ff) return 0; // we return 0, so that we will retry to open it next time - - uint32_t lines = procfile_lines(ff), l; - uint32_t words; - - // unsigned long long allocstall = 0ULL; - // unsigned long long compact_blocks_moved = 0ULL; - // unsigned long long compact_fail = 0ULL; - // unsigned long long compact_pagemigrate_failed = 0ULL; - // unsigned long long compact_pages_moved = 0ULL; - // unsigned long long compact_stall = 0ULL; - // unsigned long long compact_success = 0ULL; - // unsigned long long htlb_buddy_alloc_fail = 0ULL; - // unsigned long long htlb_buddy_alloc_success = 0ULL; - // unsigned long long kswapd_high_wmark_hit_quickly = 0ULL; - // unsigned long long kswapd_inodesteal = 0ULL; - // unsigned long long kswapd_low_wmark_hit_quickly = 0ULL; - // unsigned long long kswapd_skip_congestion_wait = 0ULL; - // unsigned long long nr_active_anon = 0ULL; - // unsigned long long nr_active_file = 0ULL; - // unsigned long long nr_anon_pages = 0ULL; - // unsigned long long nr_anon_transparent_hugepages = 0ULL; - // unsigned long long nr_bounce = 0ULL; - // unsigned long long nr_dirtied = 0ULL; - // unsigned long long nr_dirty = 0ULL; - // unsigned long long nr_dirty_background_threshold = 0ULL; - // unsigned long long nr_dirty_threshold = 0ULL; - // unsigned long long nr_file_pages = 0ULL; - // unsigned long long nr_free_pages = 0ULL; - // unsigned long long nr_inactive_anon = 0ULL; - // unsigned long long nr_inactive_file = 0ULL; - // unsigned long long nr_isolated_anon = 0ULL; - // unsigned long long nr_isolated_file = 0ULL; - // unsigned long long nr_kernel_stack = 0ULL; - // unsigned long long nr_mapped = 0ULL; - // unsigned long long nr_mlock = 0ULL; - // unsigned long long nr_page_table_pages = 0ULL; - // unsigned long long nr_shmem = 0ULL; - // unsigned long long nr_slab_reclaimable = 0ULL; - // unsigned long long nr_slab_unreclaimable = 0ULL; - // unsigned long long nr_unevictable = 0ULL; - // unsigned long long nr_unstable = 0ULL; - // unsigned long long nr_vmscan_immediate_reclaim = 0ULL; - // unsigned long long nr_vmscan_write = 0ULL; - // unsigned long long nr_writeback = 0ULL; - // unsigned long long nr_writeback_temp = 0ULL; - // unsigned long long nr_written = 0ULL; - // unsigned long long pageoutrun = 0ULL; - // unsigned long long pgactivate = 0ULL; - // unsigned long long pgalloc_dma = 0ULL; - // unsigned long long pgalloc_dma32 = 0ULL; - // unsigned long long pgalloc_movable = 0ULL; - // unsigned long long pgalloc_normal = 0ULL; - // unsigned long long pgdeactivate = 0ULL; - unsigned long long pgfault = 0ULL; - // unsigned long long pgfree = 0ULL; - // unsigned long long pginodesteal = 0ULL; - unsigned long long pgmajfault = 0ULL; - unsigned long long pgpgin = 0ULL; - unsigned long long pgpgout = 0ULL; - // unsigned long long pgrefill_dma = 0ULL; - // unsigned long long pgrefill_dma32 = 0ULL; - // unsigned long long pgrefill_movable = 0ULL; - // unsigned long long pgrefill_normal = 0ULL; - // unsigned long long pgrotated = 0ULL; - // unsigned long long pgscan_direct_dma = 0ULL; - // unsigned long long pgscan_direct_dma32 = 0ULL; - // unsigned long long pgscan_direct_movable = 0ULL; - // unsigned long long pgscan_direct_normal = 0ULL; - // unsigned long long pgscan_kswapd_dma = 0ULL; - // unsigned long long pgscan_kswapd_dma32 = 0ULL; - // unsigned long long pgscan_kswapd_movable = 0ULL; - // unsigned long long pgscan_kswapd_normal = 0ULL; - // unsigned long long pgsteal_direct_dma = 0ULL; - // unsigned long long pgsteal_direct_dma32 = 0ULL; - // unsigned long long pgsteal_direct_movable = 0ULL; - // unsigned long long pgsteal_direct_normal = 0ULL; - // unsigned long long pgsteal_kswapd_dma = 0ULL; - // unsigned long long pgsteal_kswapd_dma32 = 0ULL; - // unsigned long long pgsteal_kswapd_movable = 0ULL; - // unsigned long long pgsteal_kswapd_normal = 0ULL; - unsigned long long pswpin = 0ULL; - unsigned long long pswpout = 0ULL; - // unsigned long long slabs_scanned = 0ULL; - // unsigned long long thp_collapse_alloc = 0ULL; - // unsigned long long thp_collapse_alloc_failed = 0ULL; - // unsigned long long thp_fault_alloc = 0ULL; - // unsigned long long thp_fault_fallback = 0ULL; - // unsigned long long thp_split = 0ULL; - // unsigned long long unevictable_pgs_cleared = 0ULL; - // unsigned long long unevictable_pgs_culled = 0ULL; - // unsigned long long unevictable_pgs_mlocked = 0ULL; - // unsigned long long unevictable_pgs_mlockfreed = 0ULL; - // unsigned long long unevictable_pgs_munlocked = 0ULL; - // unsigned long long unevictable_pgs_rescued = 0ULL; - // unsigned long long unevictable_pgs_scanned = 0ULL; - // unsigned long long unevictable_pgs_stranded = 0ULL; + if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time + + size_t lines = procfile_lines(ff), l; + arl_begin(arl_base); for(l = 0; l < lines ;l++) { - words = procfile_linewords(ff, l); - if(words < 2) { - if(words) error("Cannot read /proc/vmstat line %u. Expected 2 params, read %u.", l, words); + size_t words = procfile_linewords(ff, l); + if(unlikely(words < 2)) { + if(unlikely(words)) error("Cannot read /proc/vmstat line %zu. Expected 2 params, read %zu.", l, words); continue; } - char *name = procfile_lineword(ff, l, 0); - char * value = procfile_lineword(ff, l, 1); - if(!name || !*name || !value || !*value) continue; - - uint32_t hash = simple_hash(name); - - if(0) ; - // else if(hash == hash_allocstall && strcmp(name, "allocstall") == 0) allocstall = strtoull(value, NULL, 10); - // else if(hash == hash_compact_blocks_moved && strcmp(name, "compact_blocks_moved") == 0) compact_blocks_moved = strtoull(value, NULL, 10); - // else if(hash == hash_compact_fail && strcmp(name, "compact_fail") == 0) compact_fail = strtoull(value, NULL, 10); - // else if(hash == hash_compact_pagemigrate_failed && strcmp(name, "compact_pagemigrate_failed") == 0) compact_pagemigrate_failed = strtoull(value, NULL, 10); - // else if(hash == hash_compact_pages_moved && strcmp(name, "compact_pages_moved") == 0) compact_pages_moved = strtoull(value, NULL, 10); - // else if(hash == hash_compact_stall && strcmp(name, "compact_stall") == 0) compact_stall = strtoull(value, NULL, 10); - // else if(hash == hash_compact_success && strcmp(name, "compact_success") == 0) compact_success = strtoull(value, NULL, 10); - // else if(hash == hash_htlb_buddy_alloc_fail && strcmp(name, "htlb_buddy_alloc_fail") == 0) htlb_buddy_alloc_fail = strtoull(value, NULL, 10); - // else if(hash == hash_htlb_buddy_alloc_success && strcmp(name, "htlb_buddy_alloc_success") == 0) htlb_buddy_alloc_success = strtoull(value, NULL, 10); - // else if(hash == hash_kswapd_high_wmark_hit_quickly && strcmp(name, "kswapd_high_wmark_hit_quickly") == 0) kswapd_high_wmark_hit_quickly = strtoull(value, NULL, 10); - // else if(hash == hash_kswapd_inodesteal && strcmp(name, "kswapd_inodesteal") == 0) kswapd_inodesteal = strtoull(value, NULL, 10); - // else if(hash == hash_kswapd_low_wmark_hit_quickly && strcmp(name, "kswapd_low_wmark_hit_quickly") == 0) kswapd_low_wmark_hit_quickly = strtoull(value, NULL, 10); - // else if(hash == hash_kswapd_skip_congestion_wait && strcmp(name, "kswapd_skip_congestion_wait") == 0) kswapd_skip_congestion_wait = strtoull(value, NULL, 10); - // else if(hash == hash_nr_active_anon && strcmp(name, "nr_active_anon") == 0) nr_active_anon = strtoull(value, NULL, 10); - // else if(hash == hash_nr_active_file && strcmp(name, "nr_active_file") == 0) nr_active_file = strtoull(value, NULL, 10); - // else if(hash == hash_nr_anon_pages && strcmp(name, "nr_anon_pages") == 0) nr_anon_pages = strtoull(value, NULL, 10); - // else if(hash == hash_nr_anon_transparent_hugepages && strcmp(name, "nr_anon_transparent_hugepages") == 0) nr_anon_transparent_hugepages = strtoull(value, NULL, 10); - // else if(hash == hash_nr_bounce && strcmp(name, "nr_bounce") == 0) nr_bounce = strtoull(value, NULL, 10); - // else if(hash == hash_nr_dirtied && strcmp(name, "nr_dirtied") == 0) nr_dirtied = strtoull(value, NULL, 10); - // else if(hash == hash_nr_dirty && strcmp(name, "nr_dirty") == 0) nr_dirty = strtoull(value, NULL, 10); - // else if(hash == hash_nr_dirty_background_threshold && strcmp(name, "nr_dirty_background_threshold") == 0) nr_dirty_background_threshold = strtoull(value, NULL, 10); - // else if(hash == hash_nr_dirty_threshold && strcmp(name, "nr_dirty_threshold") == 0) nr_dirty_threshold = strtoull(value, NULL, 10); - // else if(hash == hash_nr_file_pages && strcmp(name, "nr_file_pages") == 0) nr_file_pages = strtoull(value, NULL, 10); - // else if(hash == hash_nr_free_pages && strcmp(name, "nr_free_pages") == 0) nr_free_pages = strtoull(value, NULL, 10); - // else if(hash == hash_nr_inactive_anon && strcmp(name, "nr_inactive_anon") == 0) nr_inactive_anon = strtoull(value, NULL, 10); - // else if(hash == hash_nr_inactive_file && strcmp(name, "nr_inactive_file") == 0) nr_inactive_file = strtoull(value, NULL, 10); - // else if(hash == hash_nr_isolated_anon && strcmp(name, "nr_isolated_anon") == 0) nr_isolated_anon = strtoull(value, NULL, 10); - // else if(hash == hash_nr_isolated_file && strcmp(name, "nr_isolated_file") == 0) nr_isolated_file = strtoull(value, NULL, 10); - // else if(hash == hash_nr_kernel_stack && strcmp(name, "nr_kernel_stack") == 0) nr_kernel_stack = strtoull(value, NULL, 10); - // else if(hash == hash_nr_mapped && strcmp(name, "nr_mapped") == 0) nr_mapped = strtoull(value, NULL, 10); - // else if(hash == hash_nr_mlock && strcmp(name, "nr_mlock") == 0) nr_mlock = strtoull(value, NULL, 10); - // else if(hash == hash_nr_page_table_pages && strcmp(name, "nr_page_table_pages") == 0) nr_page_table_pages = strtoull(value, NULL, 10); - // else if(hash == hash_nr_shmem && strcmp(name, "nr_shmem") == 0) nr_shmem = strtoull(value, NULL, 10); - // else if(hash == hash_nr_slab_reclaimable && strcmp(name, "nr_slab_reclaimable") == 0) nr_slab_reclaimable = strtoull(value, NULL, 10); - // else if(hash == hash_nr_slab_unreclaimable && strcmp(name, "nr_slab_unreclaimable") == 0) nr_slab_unreclaimable = strtoull(value, NULL, 10); - // else if(hash == hash_nr_unevictable && strcmp(name, "nr_unevictable") == 0) nr_unevictable = strtoull(value, NULL, 10); - // else if(hash == hash_nr_unstable && strcmp(name, "nr_unstable") == 0) nr_unstable = strtoull(value, NULL, 10); - // else if(hash == hash_nr_vmscan_immediate_reclaim && strcmp(name, "nr_vmscan_immediate_reclaim") == 0) nr_vmscan_immediate_reclaim = strtoull(value, NULL, 10); - // else if(hash == hash_nr_vmscan_write && strcmp(name, "nr_vmscan_write") == 0) nr_vmscan_write = strtoull(value, NULL, 10); - // else if(hash == hash_nr_writeback && strcmp(name, "nr_writeback") == 0) nr_writeback = strtoull(value, NULL, 10); - // else if(hash == hash_nr_writeback_temp && strcmp(name, "nr_writeback_temp") == 0) nr_writeback_temp = strtoull(value, NULL, 10); - // else if(hash == hash_nr_written && strcmp(name, "nr_written") == 0) nr_written = strtoull(value, NULL, 10); - // else if(hash == hash_pageoutrun && strcmp(name, "pageoutrun") == 0) pageoutrun = strtoull(value, NULL, 10); - // else if(hash == hash_pgactivate && strcmp(name, "pgactivate") == 0) pgactivate = strtoull(value, NULL, 10); - // else if(hash == hash_pgalloc_dma && strcmp(name, "pgalloc_dma") == 0) pgalloc_dma = strtoull(value, NULL, 10); - // else if(hash == hash_pgalloc_dma32 && strcmp(name, "pgalloc_dma32") == 0) pgalloc_dma32 = strtoull(value, NULL, 10); - // else if(hash == hash_pgalloc_movable && strcmp(name, "pgalloc_movable") == 0) pgalloc_movable = strtoull(value, NULL, 10); - // else if(hash == hash_pgalloc_normal && strcmp(name, "pgalloc_normal") == 0) pgalloc_normal = strtoull(value, NULL, 10); - // else if(hash == hash_pgdeactivate && strcmp(name, "pgdeactivate") == 0) pgdeactivate = strtoull(value, NULL, 10); - else if(hash == hash_pgfault && strcmp(name, "pgfault") == 0) pgfault = strtoull(value, NULL, 10); - // else if(hash == hash_pgfree && strcmp(name, "pgfree") == 0) pgfree = strtoull(value, NULL, 10); - // else if(hash == hash_pginodesteal && strcmp(name, "pginodesteal") == 0) pginodesteal = strtoull(value, NULL, 10); - else if(hash == hash_pgmajfault && strcmp(name, "pgmajfault") == 0) pgmajfault = strtoull(value, NULL, 10); - else if(hash == hash_pgpgin && strcmp(name, "pgpgin") == 0) pgpgin = strtoull(value, NULL, 10); - else if(hash == hash_pgpgout && strcmp(name, "pgpgout") == 0) pgpgout = strtoull(value, NULL, 10); - // else if(hash == hash_pgrefill_dma && strcmp(name, "pgrefill_dma") == 0) pgrefill_dma = strtoull(value, NULL, 10); - // else if(hash == hash_pgrefill_dma32 && strcmp(name, "pgrefill_dma32") == 0) pgrefill_dma32 = strtoull(value, NULL, 10); - // else if(hash == hash_pgrefill_movable && strcmp(name, "pgrefill_movable") == 0) pgrefill_movable = strtoull(value, NULL, 10); - // else if(hash == hash_pgrefill_normal && strcmp(name, "pgrefill_normal") == 0) pgrefill_normal = strtoull(value, NULL, 10); - // else if(hash == hash_pgrotated && strcmp(name, "pgrotated") == 0) pgrotated = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_direct_dma && strcmp(name, "pgscan_direct_dma") == 0) pgscan_direct_dma = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_direct_dma32 && strcmp(name, "pgscan_direct_dma32") == 0) pgscan_direct_dma32 = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_direct_movable && strcmp(name, "pgscan_direct_movable") == 0) pgscan_direct_movable = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_direct_normal && strcmp(name, "pgscan_direct_normal") == 0) pgscan_direct_normal = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_kswapd_dma && strcmp(name, "pgscan_kswapd_dma") == 0) pgscan_kswapd_dma = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_kswapd_dma32 && strcmp(name, "pgscan_kswapd_dma32") == 0) pgscan_kswapd_dma32 = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_kswapd_movable && strcmp(name, "pgscan_kswapd_movable") == 0) pgscan_kswapd_movable = strtoull(value, NULL, 10); - // else if(hash == hash_pgscan_kswapd_normal && strcmp(name, "pgscan_kswapd_normal") == 0) pgscan_kswapd_normal = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_direct_dma && strcmp(name, "pgsteal_direct_dma") == 0) pgsteal_direct_dma = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_direct_dma32 && strcmp(name, "pgsteal_direct_dma32") == 0) pgsteal_direct_dma32 = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_direct_movable && strcmp(name, "pgsteal_direct_movable") == 0) pgsteal_direct_movable = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_direct_normal && strcmp(name, "pgsteal_direct_normal") == 0) pgsteal_direct_normal = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_kswapd_dma && strcmp(name, "pgsteal_kswapd_dma") == 0) pgsteal_kswapd_dma = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_kswapd_dma32 && strcmp(name, "pgsteal_kswapd_dma32") == 0) pgsteal_kswapd_dma32 = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_kswapd_movable && strcmp(name, "pgsteal_kswapd_movable") == 0) pgsteal_kswapd_movable = strtoull(value, NULL, 10); - // else if(hash == hash_pgsteal_kswapd_normal && strcmp(name, "pgsteal_kswapd_normal") == 0) pgsteal_kswapd_normal = strtoull(value, NULL, 10); - else if(hash == hash_pswpin && strcmp(name, "pswpin") == 0) pswpin = strtoull(value, NULL, 10); - else if(hash == hash_pswpout && strcmp(name, "pswpout") == 0) pswpout = strtoull(value, NULL, 10); - // else if(hash == hash_slabs_scanned && strcmp(name, "slabs_scanned") == 0) slabs_scanned = strtoull(value, NULL, 10); - // else if(hash == hash_thp_collapse_alloc && strcmp(name, "thp_collapse_alloc") == 0) thp_collapse_alloc = strtoull(value, NULL, 10); - // else if(hash == hash_thp_collapse_alloc_failed && strcmp(name, "thp_collapse_alloc_failed") == 0) thp_collapse_alloc_failed = strtoull(value, NULL, 10); - // else if(hash == hash_thp_fault_alloc && strcmp(name, "thp_fault_alloc") == 0) thp_fault_alloc = strtoull(value, NULL, 10); - // else if(hash == hash_thp_fault_fallback && strcmp(name, "thp_fault_fallback") == 0) thp_fault_fallback = strtoull(value, NULL, 10); - // else if(hash == hash_thp_split && strcmp(name, "thp_split") == 0) thp_split = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_cleared && strcmp(name, "unevictable_pgs_cleared") == 0) unevictable_pgs_cleared = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_culled && strcmp(name, "unevictable_pgs_culled") == 0) unevictable_pgs_culled = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_mlocked && strcmp(name, "unevictable_pgs_mlocked") == 0) unevictable_pgs_mlocked = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_mlockfreed && strcmp(name, "unevictable_pgs_mlockfreed") == 0) unevictable_pgs_mlockfreed = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_munlocked && strcmp(name, "unevictable_pgs_munlocked") == 0) unevictable_pgs_munlocked = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_rescued && strcmp(name, "unevictable_pgs_rescued") == 0) unevictable_pgs_rescued = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_scanned && strcmp(name, "unevictable_pgs_scanned") == 0) unevictable_pgs_scanned = strtoull(value, NULL, 10); - // else if(hash == hash_unevictable_pgs_stranded && strcmp(name, "unevictable_pgs_stranded") == 0) unevictable_pgs_stranded = strtoull(value, NULL, 10); + if(unlikely(arl_check(arl_base, + procfile_lineword(ff, l, 0), + procfile_lineword(ff, l, 1)))) break; } // -------------------------------------------------------------------- @@ -419,7 +91,7 @@ int do_proc_vmstat(int update_every, unsigned long long dt) { do_swapio = CONFIG_ONDEMAND_YES; static RRDSET *st_swapio = NULL; - if(!st_swapio) { + if(unlikely(!st_swapio)) { st_swapio = rrdset_create("system", "swapio", NULL, "swap", NULL, "Swap I/O", "kilobytes/s", 250, update_every, RRDSET_TYPE_AREA); rrddim_add(st_swapio, "in", NULL, sysconf(_SC_PAGESIZE), 1024, RRDDIM_INCREMENTAL); @@ -436,7 +108,7 @@ int do_proc_vmstat(int update_every, unsigned long long dt) { if(do_io) { static RRDSET *st_io = NULL; - if(!st_io) { + if(unlikely(!st_io)) { st_io = rrdset_create("system", "io", NULL, "disk", NULL, "Disk I/O", "kilobytes/s", 150, update_every, RRDSET_TYPE_AREA); rrddim_add(st_io, "in", NULL, 1, 1, RRDDIM_INCREMENTAL); @@ -453,7 +125,7 @@ int do_proc_vmstat(int update_every, unsigned long long dt) { if(do_pgfaults) { static RRDSET *st_pgfaults = NULL; - if(!st_pgfaults) { + if(unlikely(!st_pgfaults)) { st_pgfaults = rrdset_create("mem", "pgfaults", NULL, "system", NULL, "Memory Page Faults", "page faults/s", 500, update_every, RRDSET_TYPE_LINE); st_pgfaults->isdetail = 1; @@ -467,6 +139,54 @@ int do_proc_vmstat(int update_every, unsigned long long dt) { rrdset_done(st_pgfaults); } + // -------------------------------------------------------------------- + + // Ondemand criteria for NUMA. Since this won't change at run time, we + // check it only once. We check whether the node count is >= 2 because + // single-node systems have uninteresting statistics (since all accesses + // are local). + if(unlikely(has_numa == -1)) + has_numa = (numa_local || numa_foreign || numa_interleave || numa_other || numa_pte_updates || + numa_huge_pte_updates || numa_hint_faults || numa_hint_faults_local || numa_pages_migrated) ? 1 : 0; + + if(do_numa == CONFIG_ONDEMAND_YES || (do_numa == CONFIG_ONDEMAND_ONDEMAND && has_numa)) { + do_numa = CONFIG_ONDEMAND_YES; + + static RRDSET *st_numa = NULL; + if(unlikely(!st_numa)) { + st_numa = rrdset_create("mem", "numa", NULL, "numa", NULL, "NUMA events", "events/s", 800, update_every, RRDSET_TYPE_LINE); + st_numa->isdetail = 1; + + // These depend on CONFIG_NUMA in the kernel. + rrddim_add(st_numa, "local", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_numa, "foreign", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_numa, "interleave", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_numa, "other", NULL, 1, 1, RRDDIM_INCREMENTAL); + + // The following stats depend on CONFIG_NUMA_BALANCING in the + // kernel. + rrddim_add(st_numa, "pte updates", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_numa, "huge pte updates", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_numa, "hint faults", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_numa, "hint faults local", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st_numa, "pages migrated", NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else rrdset_next(st_numa); + + rrddim_set(st_numa, "local", numa_local); + rrddim_set(st_numa, "foreign", numa_foreign); + rrddim_set(st_numa, "interleave", numa_interleave); + rrddim_set(st_numa, "other", numa_other); + + rrddim_set(st_numa, "pte updates", numa_pte_updates); + rrddim_set(st_numa, "huge pte updates", numa_huge_pte_updates); + rrddim_set(st_numa, "hint faults", numa_hint_faults); + rrddim_set(st_numa, "hint faults local", numa_hint_faults_local); + rrddim_set(st_numa, "pages migrated", numa_pages_migrated); + + rrdset_done(st_numa); + } + return 0; } diff --git a/src/procfile.c b/src/procfile.c index e2aa60582..6f52bf465 100644 --- a/src/procfile.c +++ b/src/procfile.c @@ -1,4 +1,5 @@ #include "common.h" +#include "procfile.h" #define PF_PREFIX "PROCFILE" @@ -10,15 +11,15 @@ int procfile_adaptive_initial_allocation = 0; // if adaptive allocation is set, these store the // max values we have seen so far -uint32_t procfile_max_lines = PFLINES_INCREASE_STEP; -uint32_t procfile_max_words = PFWORDS_INCREASE_STEP; +size_t procfile_max_lines = PFLINES_INCREASE_STEP; +size_t procfile_max_words = PFWORDS_INCREASE_STEP; size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER; // ---------------------------------------------------------------------------- // An array of words - -pfwords *pfwords_add(pfwords *fw, char *str) { +static inline pfwords *pfwords_add(pfwords *fw, char *str) NEVERNULL; +static inline pfwords *pfwords_add(pfwords *fw, char *str) { // debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str); if(unlikely(fw->len == fw->size)) { @@ -33,10 +34,11 @@ pfwords *pfwords_add(pfwords *fw, char *str) { return fw; } -pfwords *pfwords_new(void) { +static inline pfwords *pfwords_new(void) NEVERNULL; +static inline pfwords *pfwords_new(void) { // debug(D_PROCFILE, PF_PREFIX ": initializing words"); - uint32_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP; + size_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP; pfwords *new = mallocz(sizeof(pfwords) + size * sizeof(char *)); new->len = 0; @@ -44,12 +46,12 @@ pfwords *pfwords_new(void) { return new; } -void pfwords_reset(pfwords *fw) { +static inline void pfwords_reset(pfwords *fw) { // debug(D_PROCFILE, PF_PREFIX ": reseting words"); fw->len = 0; } -void pfwords_free(pfwords *fw) { +static inline void pfwords_free(pfwords *fw) { // debug(D_PROCFILE, PF_PREFIX ": freeing words"); freez(fw); @@ -59,7 +61,8 @@ void pfwords_free(pfwords *fw) { // ---------------------------------------------------------------------------- // An array of lines -pflines *pflines_add(pflines *fl, uint32_t first_word) { +static inline pflines *pflines_add(pflines *fl, size_t first_word) NEVERNULL; +static inline pflines *pflines_add(pflines *fl, size_t first_word) { // debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word); if(unlikely(fl->len == fl->size)) { @@ -75,10 +78,11 @@ pflines *pflines_add(pflines *fl, uint32_t first_word) { return fl; } -pflines *pflines_new(void) { +static inline pflines *pflines_new(void) NEVERNULL; +static inline pflines *pflines_new(void) { // debug(D_PROCFILE, PF_PREFIX ": initializing lines"); - uint32_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_words : PFLINES_INCREASE_STEP; + size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_words : PFLINES_INCREASE_STEP; pflines *new = mallocz(sizeof(pflines) + size * sizeof(ffline)); new->len = 0; @@ -86,13 +90,13 @@ pflines *pflines_new(void) { return new; } -void pflines_reset(pflines *fl) { +static inline void pflines_reset(pflines *fl) { // debug(D_PROCFILE, PF_PREFIX ": reseting lines"); fl->len = 0; } -void pflines_free(pflines *fl) { +static inline void pflines_free(pflines *fl) { // debug(D_PROCFILE, PF_PREFIX ": freeing lines"); freez(fl); @@ -119,20 +123,20 @@ void procfile_close(procfile *ff) { freez(ff); } -procfile *procfile_parser(procfile *ff) { - debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename); +static inline void procfile_parser(procfile *ff) { + // debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename); - char *s = ff->data, *e = &ff->data[ff->len], *t = ff->data, quote = 0; - uint32_t l = 0, w = 0; - int opened = 0; + register char *s = ff->data, *e = &ff->data[ff->len], *t = ff->data; + register char *separators = ff->separators; + char quote = 0; + size_t l = 0, w = 0, opened = 0; ff->lines = pflines_add(ff->lines, w); - if(unlikely(!ff->lines)) goto cleanup; while(likely(s < e)) { // we are not at the end - switch(ff->separators[(uint8_t)(*s)]) { + switch(separators[(unsigned char)(*s)]) { case PF_CHAR_IS_OPEN: if(s == t) { opened++; @@ -144,7 +148,7 @@ procfile *procfile_parser(procfile *ff) { } else s++; - continue; + break; case PF_CHAR_IS_CLOSE: if(opened) { @@ -153,8 +157,6 @@ procfile *procfile_parser(procfile *ff) { if(!opened) { *s = '\0'; ff->words = pfwords_add(ff->words, t); - if(unlikely(!ff->words)) goto cleanup; - ff->lines->lines[l].words++; w++; @@ -165,7 +167,7 @@ procfile *procfile_parser(procfile *ff) { } else s++; - continue; + break; case PF_CHAR_IS_QUOTE: if(unlikely(!quote && s == t)) { @@ -179,8 +181,6 @@ procfile *procfile_parser(procfile *ff) { *s = '\0'; ff->words = pfwords_add(ff->words, t); - if(unlikely(!ff->words)) goto cleanup; - ff->lines->lines[l].words++; w++; @@ -188,55 +188,50 @@ procfile *procfile_parser(procfile *ff) { } else s++; - continue; + break; case PF_CHAR_IS_SEPARATOR: if(unlikely(quote || opened)) { // we are inside a quote s++; - continue; + break; } if(unlikely(s == t)) { // skip all leading white spaces t = ++s; - continue; + break; } // end of word *s = '\0'; ff->words = pfwords_add(ff->words, t); - if(unlikely(!ff->words)) goto cleanup; - ff->lines->lines[l].words++; w++; t = ++s; - continue; + break; case PF_CHAR_IS_NEWLINE: // end of line *s = '\0'; ff->words = pfwords_add(ff->words, t); - if(unlikely(!ff->words)) goto cleanup; - ff->lines->lines[l].words++; w++; // debug(D_PROCFILE, PF_PREFIX ": ended line %d with %d words", l, ff->lines->lines[l].words); ff->lines = pflines_add(ff->lines, w); - if(unlikely(!ff->lines)) goto cleanup; l++; t = ++s; - continue; + break; default: s++; - continue; + break; } } @@ -250,31 +245,21 @@ procfile *procfile_parser(procfile *ff) { } ff->words = pfwords_add(ff->words, t); - if(unlikely(!ff->words)) goto cleanup; - ff->lines->lines[l].words++; - w++; } - - return ff; - -cleanup: - error(PF_PREFIX ": Failed to parse file '%s'", ff->filename); - procfile_close(ff); - return NULL; } procfile *procfile_readall(procfile *ff) { debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename); - ssize_t s, r = 1, x; + ssize_t r = 1; ff->len = 0; while(likely(r > 0)) { - s = ff->len; - x = ff->size - s; + ssize_t s = ff->len; + ssize_t x = ff->size - s; - if(!x) { + if(unlikely(!x)) { debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", ff->filename); ff = reallocz(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER); @@ -301,8 +286,7 @@ procfile *procfile_readall(procfile *ff) { pflines_reset(ff->lines); pfwords_reset(ff->words); - - ff = procfile_parser(ff); + procfile_parser(ff); if(unlikely(procfile_adaptive_initial_allocation)) { if(unlikely(ff->len > procfile_max_allocation)) procfile_max_allocation = ff->len; @@ -316,13 +300,13 @@ procfile *procfile_readall(procfile *ff) { static void procfile_set_separators(procfile *ff, const char *separators) { static char def[256] = { [0 ... 255] = 0 }; - int i; if(unlikely(!def[255])) { // this is thread safe // we check that the last byte is non-zero // if it is zero, multiple threads may be executing this at the same time // setting in def[] the exact same values + int i; for(i = 0; likely(i < 256) ;i++) { if(unlikely(i == '\n' || i == '\r')) def[i] = PF_CHAR_IS_NEWLINE; else if(unlikely(isspace(i) || !isprint(i))) def[i] = PF_CHAR_IS_SEPARATOR; @@ -348,7 +332,7 @@ void procfile_set_quotes(procfile *ff, const char *quotes) { // remove all quotes int i; for(i = 0; i < 256 ; i++) - if(ff->separators[i] == PF_CHAR_IS_QUOTE) + if(unlikely(ff->separators[i] == PF_CHAR_IS_QUOTE)) ff->separators[i] = PF_CHAR_IS_WORD; // if nothing given, return @@ -366,7 +350,7 @@ void procfile_set_open_close(procfile *ff, const char *open, const char *close) // remove all open/close int i; for(i = 0; i < 256 ; i++) - if(ff->separators[i] == PF_CHAR_IS_OPEN || ff->separators[i] == PF_CHAR_IS_CLOSE) + if(unlikely(ff->separators[i] == PF_CHAR_IS_OPEN || ff->separators[i] == PF_CHAR_IS_CLOSE)) ff->separators[i] = PF_CHAR_IS_WORD; // if nothing given, return @@ -405,12 +389,6 @@ procfile *procfile_open(const char *filename, const char *separators, uint32_t f ff->lines = pflines_new(); ff->words = pfwords_new(); - if(unlikely(!ff->lines || !ff->words)) { - error(PF_PREFIX ": Cannot initialize parser for file '%s'", filename); - procfile_close(ff); - return NULL; - } - procfile_set_separators(ff, separators); debug(D_PROCFILE, "File '%s' opened.", filename); @@ -442,20 +420,20 @@ procfile *procfile_reopen(procfile *ff, const char *filename, const char *separa // example parsing of procfile data void procfile_print(procfile *ff) { - uint32_t lines = procfile_lines(ff), l; - uint32_t words, w; + size_t lines = procfile_lines(ff), l; char *s; - debug(D_PROCFILE, "File '%s' with %u lines and %u words", ff->filename, ff->lines->len, ff->words->len); + debug(D_PROCFILE, "File '%s' with %zu lines and %zu words", ff->filename, ff->lines->len, ff->words->len); for(l = 0; likely(l < lines) ;l++) { - words = procfile_linewords(ff, l); + size_t words = procfile_linewords(ff, l); - debug(D_PROCFILE, " line %u starts at word %u and has %u words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); + debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); + size_t w; for(w = 0; likely(w < words) ;w++) { s = procfile_lineword(ff, l, w); - debug(D_PROCFILE, " [%u.%u] '%s'", l, w, s); + debug(D_PROCFILE, " [%zu.%zu] '%s'", l, w, s); } } } diff --git a/src/procfile.h b/src/procfile.h index 5e00b2584..a586ba48d 100644 --- a/src/procfile.h +++ b/src/procfile.h @@ -30,8 +30,8 @@ // An array of words typedef struct { - uint32_t len; // used entries - uint32_t size; // capacity + size_t len; // used entries + size_t size; // capacity char *words[]; // array of pointers } pfwords; @@ -40,15 +40,15 @@ typedef struct { // An array of lines typedef struct { - uint32_t words; // how many words this line has - uint32_t first; // the id of the first word of this line - // in the words array + size_t words; // how many words this line has + size_t first; // the id of the first word of this line + // in the words array } ffline; typedef struct { - uint32_t len; // used entries - uint32_t size; // capacity - ffline lines[]; // array of lines + size_t len; // used entries + size_t size; // capacity + ffline lines[]; // array of lines } pflines; @@ -61,13 +61,13 @@ typedef struct { typedef struct { char filename[FILENAME_MAX + 1]; uint32_t flags; - int fd; // the file desriptor - size_t len; // the bytes we have placed into data - size_t size; // the bytes we have allocated for data + int fd; // the file desriptor + size_t len; // the bytes we have placed into data + size_t size; // the bytes we have allocated for data pflines *lines; pfwords *words; char separators[256]; - char data[]; // allocated buffer to keep file contents + char data[]; // allocated buffer to keep file contents } procfile; // close the proc file and free all related memory diff --git a/src/registry.c b/src/registry.c index a0fb629ad..d223cd6f1 100644 --- a/src/registry.c +++ b/src/registry.c @@ -1,1015 +1,29 @@ #include "common.h" -// ---------------------------------------------------------------------------- -// TODO -// -// 1. the default tracking cookie expires in 1 year, but the persons are not -// removed from the db - this means the database only grows - ideally the -// database should be cleaned in registry_save() for both on-disk and -// on-memory entries. -// -// Cleanup: -// i. Find all the PERSONs that have expired cookie -// ii. For each of their PERSON_URLs: -// - decrement the linked MACHINE links -// - if the linked MACHINE has no other links, remove the linked MACHINE too -// - remove the PERSON_URL -// -// 2. add protection to prevent abusing the registry by flooding it with -// requests to fill the memory and crash it. -// -// Possible protections: -// - limit the number of URLs per person -// - limit the number of URLs per machine -// - limit the number of persons -// - limit the number of machines -// - [DONE] limit the size of URLs -// - [DONE] limit the size of PERSON_URL names -// - limit the number of requests that add data to the registry, -// per client IP per hour -// -// 3. lower memory requirements -// -// - embed avl structures directly into registry objects, instead of DICTIONARY -// - store GUIDs in memory as UUID instead of char * -// (this will also remove the index hash, since UUIDs can be compared directly) -// - do not track persons using the demo machines only -// (i.e. start tracking them only when they access a non-demo machine) -// - [DONE] do not track custom dashboards by default - -#define REGISTRY_URL_FLAGS_DEFAULT 0x00 -#define REGISTRY_URL_FLAGS_EXPIRED 0x01 - -#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - -// ---------------------------------------------------------------------------- -// COMMON structures - -struct registry { - int enabled; - - char machine_guid[36 + 1]; - - // entries counters / statistics - unsigned long long persons_count; - unsigned long long machines_count; - unsigned long long usages_count; - unsigned long long urls_count; - unsigned long long persons_urls_count; - unsigned long long machines_urls_count; - unsigned long long log_count; - - // memory counters / statistics - unsigned long long persons_memory; - unsigned long long machines_memory; - unsigned long long urls_memory; - unsigned long long persons_urls_memory; - unsigned long long machines_urls_memory; - - // configuration - unsigned long long save_registry_every_entries; - char *registry_domain; - char *hostname; - char *registry_to_announce; - time_t persons_expiration; // seconds to expire idle persons - int verify_cookies_redirects; - - size_t max_url_length; - size_t max_name_length; - - // file/path names - char *pathname; - char *db_filename; - char *log_filename; - char *machine_guid_filename; - - // open files - FILE *log_fp; - - // the database - DICTIONARY *persons; // dictionary of PERSON *, with key the PERSON.guid - DICTIONARY *machines; // dictionary of MACHINE *, with key the MACHINE.guid - DICTIONARY *urls; // dictionary of URL *, with key the URL.url - - // concurrency locking - // we keep different locks for different things - // so that many tasks can be completed in parallel - pthread_mutex_t persons_lock; - pthread_mutex_t machines_lock; - pthread_mutex_t urls_lock; - pthread_mutex_t person_urls_lock; - pthread_mutex_t machine_urls_lock; - pthread_mutex_t log_lock; -} registry; - - -// ---------------------------------------------------------------------------- -// URL structures -// Save memory by de-duplicating URLs -// so instead of storing URLs all over the place -// we store them here and we keep pointers elsewhere - -struct url { - uint32_t links; // the number of links to this URL - when none is left, we free it - uint16_t len; // the length of the URL in bytes - char url[1]; // the URL - dynamically allocated to more size -}; -typedef struct url URL; - - -// ---------------------------------------------------------------------------- -// MACHINE structures - -// For each MACHINE-URL pair we keep this -struct machine_url { - URL *url; // de-duplicated URL -// DICTIONARY *persons; // dictionary of PERSON * - - uint8_t flags; - uint32_t first_t; // the first time we saw this - uint32_t last_t; // the last time we saw this - uint32_t usages; // how many times this has been accessed -}; -typedef struct machine_url MACHINE_URL; - -// A machine -struct machine { - char guid[36 + 1]; // the GUID - - uint32_t links; // the number of PERSON_URLs linked to this machine - - DICTIONARY *urls; // MACHINE_URL * - - uint32_t first_t; // the first time we saw this - uint32_t last_t; // the last time we saw this - uint32_t usages; // how many times this has been accessed -}; -typedef struct machine MACHINE; - - -// ---------------------------------------------------------------------------- -// PERSON structures - -// for each PERSON-URL pair we keep this -struct person_url { - URL *url; // de-duplicated URL - MACHINE *machine; // link the MACHINE of this URL - - uint8_t flags; - uint32_t first_t; // the first time we saw this - uint32_t last_t; // the last time we saw this - uint32_t usages; // how many times this has been accessed - - char name[1]; // the name of the URL, as known by the user - // dynamically allocated to fit properly -}; -typedef struct person_url PERSON_URL; - -// A person -struct person { - char guid[36 + 1]; // the person GUID - - DICTIONARY *urls; // dictionary of PERSON_URL * - - uint32_t first_t; // the first time we saw this - uint32_t last_t; // the last time we saw this - uint32_t usages; // how many times this has been accessed -}; -typedef struct person PERSON; +#include "registry_internals.h" +#define REGISTRY_STATUS_OK "ok" +#define REGISTRY_STATUS_FAILED "failed" +#define REGISTRY_STATUS_DISABLED "disabled" // ---------------------------------------------------------------------------- // REGISTRY concurrency locking -static inline void registry_persons_lock(void) { - pthread_mutex_lock(®istry.persons_lock); -} - -static inline void registry_persons_unlock(void) { - pthread_mutex_unlock(®istry.persons_lock); -} - -static inline void registry_machines_lock(void) { - pthread_mutex_lock(®istry.machines_lock); -} - -static inline void registry_machines_unlock(void) { - pthread_mutex_unlock(®istry.machines_lock); -} - -static inline void registry_urls_lock(void) { - pthread_mutex_lock(®istry.urls_lock); -} - -static inline void registry_urls_unlock(void) { - pthread_mutex_unlock(®istry.urls_lock); -} - -// ideally, we should not lock the whole registry for -// updating a person's urls. -// however, to save the memory required for keeping a -// mutex (40 bytes) per person, we do... -static inline void registry_person_urls_lock(PERSON *p) { - (void)p; - pthread_mutex_lock(®istry.person_urls_lock); -} - -static inline void registry_person_urls_unlock(PERSON *p) { - (void)p; - pthread_mutex_unlock(®istry.person_urls_lock); -} - -// ideally, we should not lock the whole registry for -// updating a machine's urls. -// however, to save the memory required for keeping a -// mutex (40 bytes) per machine, we do... -static inline void registry_machine_urls_lock(MACHINE *m) { - (void)m; - pthread_mutex_lock(®istry.machine_urls_lock); -} - -static inline void registry_machine_urls_unlock(MACHINE *m) { - (void)m; - pthread_mutex_unlock(®istry.machine_urls_lock); -} - -static inline void registry_log_lock(void) { - pthread_mutex_lock(®istry.log_lock); -} - -static inline void registry_log_unlock(void) { - pthread_mutex_unlock(®istry.log_lock); -} - - -// ---------------------------------------------------------------------------- -// common functions - -// parse a GUID and re-generated to be always lower case -// this is used as a protection against the variations of GUIDs -static inline int registry_regenerate_guid(const char *guid, char *result) { - uuid_t uuid; - if(unlikely(uuid_parse(guid, uuid) == -1)) { - info("Registry: GUID '%s' is not a valid GUID.", guid); - return -1; - } - else { - uuid_unparse_lower(uuid, result); - -#ifdef NETDATA_INTERNAL_CHECKS - if(strcmp(guid, result)) - info("Registry: source GUID '%s' and re-generated GUID '%s' differ!", guid, result); -#endif /* NETDATA_INTERNAL_CHECKS */ - } - - return 0; -} - -// make sure the names of the machines / URLs do not contain any tabs -// (which are used as our separator in the database files) -// and are properly trimmed (before and after) -static inline char *registry_fix_machine_name(char *name, size_t *len) { - char *s = name?name:""; - - // skip leading spaces - while(*s && isspace(*s)) s++; - - // make sure all spaces are a SPACE - char *t = s; - while(*t) { - if(unlikely(isspace(*t))) - *t = ' '; - - t++; - } - - // remove trailing spaces - while(--t >= s) { - if(*t == ' ') - *t = '\0'; - else - break; - } - t++; - - if(likely(len)) - *len = (t - s); - - return s; -} - -static inline char *registry_fix_url(char *url, size_t *len) { - return registry_fix_machine_name(url, len); -} - - -// ---------------------------------------------------------------------------- -// forward definition of functions - -extern PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when); -extern PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when); - - -// ---------------------------------------------------------------------------- -// URL - -static inline URL *registry_url_allocate_nolock(const char *url, size_t urllen) { - // protection from too big URLs - if(urllen > registry.max_url_length) - urllen = registry.max_url_length; - - debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): allocating %zu bytes", url, sizeof(URL) + urllen); - URL *u = mallocz(sizeof(URL) + urllen); - - // a simple strcpy() should do the job - // but I prefer to be safe, since the caller specified urllen - u->len = (uint16_t)urllen; - strncpyz(u->url, url, u->len); - u->links = 0; - - registry.urls_memory += sizeof(URL) + urllen; - - debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): indexing it", url); - dictionary_set(registry.urls, u->url, u, sizeof(URL)); - - return u; -} - -static inline URL *registry_url_get_nolock(const char *url, size_t urllen) { - debug(D_REGISTRY, "Registry: registry_url_get_nolock('%s')", url); - - URL *u = dictionary_get(registry.urls, url); - if(!u) { - u = registry_url_allocate_nolock(url, urllen); - registry.urls_count++; - } - - return u; -} - -static inline URL *registry_url_get(const char *url, size_t urllen) { - debug(D_REGISTRY, "Registry: registry_url_get('%s')", url); - - registry_urls_lock(); - - URL *u = registry_url_get_nolock(url, urllen); - - registry_urls_unlock(); - - return u; -} - -static inline void registry_url_link_nolock(URL *u) { - u->links++; - debug(D_REGISTRY, "Registry: registry_url_link_nolock('%s'): URL has now %u links", u->url, u->links); -} - -static inline void registry_url_unlink_nolock(URL *u) { - u->links--; - if(!u->links) { - debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): No more links for this URL", u->url); - dictionary_del(registry.urls, u->url); - freez(u); - } - else - debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): URL has %u links left", u->url, u->links); -} - - -// ---------------------------------------------------------------------------- -// MACHINE - -static inline MACHINE *registry_machine_find(const char *machine_guid) { - debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid); - return dictionary_get(registry.machines, machine_guid); -} - -static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, time_t when) { - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(MACHINE_URL)); - - MACHINE_URL *mu = mallocz(sizeof(MACHINE_URL)); - - // mu->persons = dictionary_create(DICTIONARY_FLAGS); - // dictionary_set(mu->persons, p->guid, p, sizeof(PERSON)); - - mu->first_t = mu->last_t = (uint32_t)when; - mu->usages = 1; - mu->url = u; - mu->flags = REGISTRY_URL_FLAGS_DEFAULT; - - registry.machines_urls_memory += sizeof(MACHINE_URL); - - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url); - dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL)); - registry_url_link_nolock(u); - - return mu; -} - -static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) { - debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(MACHINE)); - - MACHINE *m = mallocz(sizeof(MACHINE)); - - strncpyz(m->guid, machine_guid, 36); - - debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid); - m->urls = dictionary_create(DICTIONARY_FLAGS); - - m->first_t = m->last_t = (uint32_t)when; - m->usages = 0; - - registry.machines_memory += sizeof(MACHINE); - - registry.machines_count++; - dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE)); - - return m; -} - -// 1. validate machine GUID -// 2. if it is valid, find it or create it and return it -// 3. if it is not valid, return NULL -static inline MACHINE *registry_machine_get(const char *machine_guid, time_t when) { - MACHINE *m = NULL; - - registry_machines_lock(); - - if(likely(machine_guid && *machine_guid)) { - // validate it is a GUID - char buf[36 + 1]; - if(unlikely(registry_regenerate_guid(machine_guid, buf) == -1)) - info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid); - else { - machine_guid = buf; - m = registry_machine_find(machine_guid); - if(!m) m = registry_machine_allocate(machine_guid, when); - } - } - - registry_machines_unlock(); - - return m; -} - - -// ---------------------------------------------------------------------------- -// PERSON - -static inline PERSON *registry_person_find(const char *person_guid) { - debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid); - return dictionary_get(registry.persons, person_guid); -} - -static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) { - // protection from too big names - if(namelen > registry.max_name_length) - namelen = registry.max_name_length; - - debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, - sizeof(PERSON_URL) + namelen); - - PERSON_URL *pu = mallocz(sizeof(PERSON_URL) + namelen); - - // a simple strcpy() should do the job - // but I prefer to be safe, since the caller specified urllen - strncpyz(pu->name, name, namelen); - - pu->machine = m; - pu->first_t = pu->last_t = when; - pu->usages = 1; - pu->url = u; - pu->flags = REGISTRY_URL_FLAGS_DEFAULT; - m->links++; - - registry.persons_urls_memory += sizeof(PERSON_URL) + namelen; - - debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url); - dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL)); - registry_url_link_nolock(u); - - return pu; -} - -static inline PERSON_URL *registry_person_url_reallocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when, PERSON_URL *pu) { - // this function is needed to change the name of a PERSON_URL - - debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, - sizeof(PERSON_URL) + namelen); - - PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when); - tpu->first_t = pu->first_t; - tpu->last_t = pu->last_t; - tpu->usages = pu->usages; - - // ok, these are a hack - since the registry_person_url_allocate() is - // adding these, we have to subtract them - tpu->machine->links--; - registry.persons_urls_memory -= sizeof(PERSON_URL) + strlen(pu->name); - registry_url_unlink_nolock(u); - - freez(pu); - - return tpu; -} - -static inline PERSON *registry_person_allocate(const char *person_guid, time_t when) { - PERSON *p = NULL; - - debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(PERSON)); - - p = mallocz(sizeof(PERSON)); - - if(!person_guid) { - for (; ;) { - uuid_t uuid; - uuid_generate(uuid); - uuid_unparse_lower(uuid, p->guid); - - debug(D_REGISTRY, "Registry: Checking if the generated person guid '%s' is unique", p->guid); - if (!dictionary_get(registry.persons, p->guid)) { - debug(D_REGISTRY, "Registry: generated person guid '%s' is unique", p->guid); - break; - } - else - info("Registry: generated person guid '%s' found in the registry. Retrying...", p->guid); - } - } - else - strncpyz(p->guid, person_guid, 36); - - debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid); - p->urls = dictionary_create(DICTIONARY_FLAGS); - - p->first_t = p->last_t = when; - p->usages = 0; - - registry.persons_memory += sizeof(PERSON); - - registry.persons_count++; - dictionary_set(registry.persons, p->guid, p, sizeof(PERSON)); - - return p; -} - - -// 1. validate person GUID -// 2. if it is valid, find it -// 3. if it is not valid, create a new one -// 4. return it -static inline PERSON *registry_person_get(const char *person_guid, time_t when) { - PERSON *p = NULL; - - registry_persons_lock(); - - if(person_guid && *person_guid) { - char buf[36 + 1]; - // validate it is a GUID - if(unlikely(registry_regenerate_guid(person_guid, buf) == -1)) - info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid); - else { - person_guid = buf; - p = registry_person_find(person_guid); - } - } - - if(!p) p = registry_person_allocate(NULL, when); - - registry_persons_unlock(); - - return p; -} - -// ---------------------------------------------------------------------------- -// LINKING OF OBJECTS - -static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) { - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url); - - registry_person_urls_lock(p); - - PERSON_URL *pu = dictionary_get(p->urls, u->url); - if(!pu) { - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url); - pu = registry_person_url_allocate(p, m, u, name, namelen, when); - registry.persons_urls_count++; - } - else { - debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url); - pu->usages++; - if(likely(pu->last_t < (uint32_t)when)) pu->last_t = when; - - if(pu->machine != m) { - MACHINE_URL *mu = dictionary_get(pu->machine->urls, u->url); - if(mu) { - info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.", - p->guid, m->guid, u->url, pu->machine->guid); - mu->flags |= REGISTRY_URL_FLAGS_EXPIRED; - } - else { - info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - but the URL is not linked to the old machine.", - p->guid, m->guid, u->url, pu->machine->guid); - } - - pu->machine->links--; - pu->machine = m; - } - - if(strcmp(pu->name, name)) { - // the name of the PERSON_URL has changed ! - pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu); - } - } - - p->usages++; - if(likely(p->last_t < (uint32_t)when)) p->last_t = when; - - if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) { - info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url); - pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED; - } - - registry_person_urls_unlock(p); - - return pu; -} - -static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) { - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url); - - registry_machine_urls_lock(m); - - MACHINE_URL *mu = dictionary_get(m->urls, u->url); - if(!mu) { - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url); - mu = registry_machine_url_allocate(m, u, when); - registry.machines_urls_count++; - } - else { - debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url); - mu->usages++; - if(likely(mu->last_t < (uint32_t)when)) mu->last_t = when; - } - - //debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing person in machine", p->guid, m->guid, u->url); - //dictionary_set(mu->persons, p->guid, p, sizeof(PERSON)); - - m->usages++; - if(likely(m->last_t < (uint32_t)when)) m->last_t = when; - - if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) { - info("registry_machine_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url); - mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED; - } - - registry_machine_urls_unlock(m); - - return mu; -} - -// ---------------------------------------------------------------------------- -// REGISTRY LOG LOAD/SAVE - -static inline int registry_should_save_db(void) { - debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries); - return registry.log_count > registry.save_registry_every_entries; -} - -static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u, char *name) { - if(likely(registry.log_fp)) { - // we lock only if the file is open - // to allow replaying the log at registry_log_load() - registry_log_lock(); - - if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n", - action, - p->last_t, - p->guid, - m->guid, - name, - u->url) < 0)) - error("Registry: failed to save log. Registry data may be lost in case of abnormal restart."); - - // we increase the counter even on failures - // so that the registry will be saved periodically - registry.log_count++; - - registry_log_unlock(); - - // this must be outside the log_lock(), or a deadlock will happen. - // registry_save() checks the same inside the log_lock, so only - // one thread will save the db - if(unlikely(registry_should_save_db())) - registry_save(); - } -} - -static inline int registry_log_open_nolock(void) { - if(registry.log_fp) - fclose(registry.log_fp); - - registry.log_fp = fopen(registry.log_filename, "a"); - - if(registry.log_fp) { - if (setvbuf(registry.log_fp, NULL, _IOLBF, 0) != 0) - error("Cannot set line buffering on registry log file."); - return 0; - } - - error("Cannot open registry log file '%s'. Registry data will be lost in case of netdata or server crash.", registry.log_filename); - return -1; -} - -static inline void registry_log_close_nolock(void) { - if(registry.log_fp) { - fclose(registry.log_fp); - registry.log_fp = NULL; - } -} - -static inline void registry_log_recreate_nolock(void) { - if(registry.log_fp != NULL) { - registry_log_close_nolock(); - - // open it with truncate - registry.log_fp = fopen(registry.log_filename, "w"); - if(registry.log_fp) fclose(registry.log_fp); - else error("Cannot truncate registry log '%s'", registry.log_filename); - - registry.log_fp = NULL; - - registry_log_open_nolock(); - } -} - -int registry_log_load(void) { - char *s, buf[4096 + 1]; - size_t line = -1; - - // closing the log is required here - // otherwise we will append to it the values we read - registry_log_close_nolock(); - - debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename); - FILE *fp = fopen(registry.log_filename, "r"); - if(!fp) - error("Registry: cannot open registry file: %s", registry.log_filename); - else { - line = 0; - size_t len = 0; - while ((s = fgets_trim_len(buf, 4096, fp, &len))) { - line++; - - switch (s[0]) { - case 'A': // accesses - case 'D': // deletes - - // verify it is valid - if (unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) { - error("Registry: log line %zu is wrong (len = %zu).", line, len); - continue; - } - s[1] = s[10] = s[47] = s[84] = '\0'; - - // get the variables - time_t when = strtoul(&s[2], NULL, 16); - char *person_guid = &s[11]; - char *machine_guid = &s[48]; - char *name = &s[85]; - - // skip the name to find the url - char *url = name; - while(*url && *url != '\t') url++; - if(!*url) { - error("Registry: log line %zu does not have a url.", line); - continue; - } - *url++ = '\0'; - - // make sure the person exists - // without this, a new person guid will be created - PERSON *p = registry_person_find(person_guid); - if(!p) p = registry_person_allocate(person_guid, when); - - if(s[0] == 'A') - registry_request_access(p->guid, machine_guid, url, name, when); - else - registry_request_delete(p->guid, machine_guid, url, name, when); - - registry.log_count++; - break; - - default: - error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.log_filename, s); - break; - } - } - - fclose(fp); - } - - // open the log again - registry_log_open_nolock(); - - return line; -} - - -// ---------------------------------------------------------------------------- -// REGISTRY REQUESTS - -PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) { - debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url); - - MACHINE *m = registry_machine_get(machine_guid, when); - if(!m) return NULL; - - // make sure the name is valid - size_t namelen; - name = registry_fix_machine_name(name, &namelen); - - size_t urllen; - url = registry_fix_url(url, &urllen); - - URL *u = registry_url_get(url, urllen); - PERSON *p = registry_person_get(person_guid, when); - - registry_person_link_to_url(p, m, u, name, namelen, when); - registry_machine_link_to_url(p, m, u, when); - - registry_log('A', p, m, u, name); - - registry.usages_count++; - return p; -} - -// verify the person, the machine and the URL exist in our DB -PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, PERSON **pp, MACHINE **mm) { - char pbuf[36 + 1], mbuf[36 + 1]; - - if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) { - info("Registry Request Verification: invalid request! person: '%s', machine '%s', url '%s'", person_guid?person_guid:"UNSET", machine_guid?machine_guid:"UNSET", url?url:"UNSET"); - return NULL; - } - - // normalize the url - url = registry_fix_url(url, NULL); - - // make sure the person GUID is valid - if(registry_regenerate_guid(person_guid, pbuf) == -1) { - info("Registry Request Verification: invalid person GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); - return NULL; - } - person_guid = pbuf; - - // make sure the machine GUID is valid - if(registry_regenerate_guid(machine_guid, mbuf) == -1) { - info("Registry Request Verification: invalid machine GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); - return NULL; - } - machine_guid = mbuf; - - // make sure the machine exists - MACHINE *m = registry_machine_find(machine_guid); - if(!m) { - info("Registry Request Verification: machine not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); - return NULL; - } - if(mm) *mm = m; - - // make sure the person exist - PERSON *p = registry_person_find(person_guid); - if(!p) { - info("Registry Request Verification: person not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); - return NULL; - } - if(pp) *pp = p; - - PERSON_URL *pu = dictionary_get(p->urls, url); - if(!pu) { - info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); - return NULL; - } - return pu; +static inline void registry_lock(void) { + pthread_mutex_lock(®istry.lock); } -PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) { - (void)when; - - PERSON *p = NULL; - MACHINE *m = NULL; - PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m); - if(!pu || !p || !m) return NULL; - - // normalize the url - delete_url = registry_fix_url(delete_url, NULL); - - // make sure the user is not deleting the url it uses - if(!strcmp(delete_url, pu->url->url)) { - info("Registry Delete Request: delete URL is the one currently accessed, person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url); - return NULL; - } - - registry_person_urls_lock(p); - - PERSON_URL *dpu = dictionary_get(p->urls, delete_url); - if(!dpu) { - info("Registry Delete Request: URL not found for person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url); - registry_person_urls_unlock(p); - return NULL; - } - - registry_log('D', p, m, pu->url, dpu->url->url); - - dictionary_del(p->urls, dpu->url->url); - registry_url_unlink_nolock(dpu->url); - freez(dpu); - - registry_person_urls_unlock(p); - return p; -} - - -// a structure to pass to the dictionary_get_all() callback handler -struct machine_request_callback_data { - MACHINE *find_this_machine; - PERSON_URL *result; -}; - -// the callback function -// this will be run for every PERSON_URL of this PERSON -int machine_request_callback(void *entry, void *data) { - PERSON_URL *mypu = (PERSON_URL *)entry; - struct machine_request_callback_data *myrdata = (struct machine_request_callback_data *)data; - - if(mypu->machine == myrdata->find_this_machine) { - myrdata->result = mypu; - return -1; // this will also stop the walk through - } - - return 0; // continue -} - -MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) { - (void)when; - - char mbuf[36 + 1]; - - PERSON *p = NULL; - MACHINE *m = NULL; - PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m); - if(!pu || !p || !m) return NULL; - - // make sure the machine GUID is valid - if(registry_regenerate_guid(request_machine, mbuf) == -1) { - info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine); - return NULL; - } - request_machine = mbuf; - - // make sure the machine exists - m = registry_machine_find(request_machine); - if(!m) { - info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, machine_guid, pu->url->url, request_machine); - return NULL; - } - - // Verify the user has in the past accessed this machine - // We will walk through the PERSON_URLs to find the machine - // linking to our machine - - // a structure to pass to the dictionary_get_all() callback handler - struct machine_request_callback_data rdata = { m, NULL }; - - // request a walk through on the dictionary - // no need for locking here, the underlying dictionary has its own - dictionary_get_all(p->urls, machine_request_callback, &rdata); - - if(rdata.result) - return m; - - return NULL; +static inline void registry_unlock(void) { + pthread_mutex_unlock(®istry.lock); } // ---------------------------------------------------------------------------- -// REGISTRY JSON generation - -#define REGISTRY_STATUS_OK "ok" -#define REGISTRY_STATUS_FAILED "failed" -#define REGISTRY_STATUS_DISABLED "disabled" - -int registry_verify_cookies_redirects(void) { - return registry.verify_cookies_redirects; -} - -const char *registry_to_announce(void) { - return registry.registry_to_announce; -} +// COOKIES -void registry_set_cookie(struct web_client *w, const char *guid) { +static void registry_set_cookie(struct web_client *w, const char *guid) { char edate[100]; - time_t et = time(NULL) + registry.persons_expiration; + time_t et = now_realtime_sec() + registry.persons_expiration; struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf); strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm); @@ -1019,52 +33,50 @@ void registry_set_cookie(struct web_client *w, const char *guid) { snprintfz(w->cookie2, COOKIE_MAX, NETDATA_REGISTRY_COOKIE_NAME "=%s; Domain=%s; Expires=%s", guid, registry.registry_domain, edate); } -static inline void registry_set_person_cookie(struct web_client *w, PERSON *p) { +static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PERSON *p) { registry_set_cookie(w, p->guid); } + +// ---------------------------------------------------------------------------- +// JSON GENERATION + static inline void registry_json_header(struct web_client *w, const char *action, const char *status) { buffer_flush(w->response.data); w->response.data->contenttype = CT_APPLICATION_JSON; buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"", - action, status, registry.hostname, registry.machine_guid); + action, status, registry.hostname, registry.machine_guid); } static inline void registry_json_footer(struct web_client *w) { buffer_strcat(w->response.data, "\n}\n"); } -int registry_request_hello_json(struct web_client *w) { - registry_json_header(w, "hello", REGISTRY_STATUS_OK); - - buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"", - registry.registry_to_announce); - - registry_json_footer(w); - return 200; -} - static inline int registry_json_disabled(struct web_client *w, const char *action) { registry_json_header(w, action, REGISTRY_STATUS_DISABLED); buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"", - registry.registry_to_announce); + registry.registry_to_announce); registry_json_footer(w); return 200; } + +// ---------------------------------------------------------------------------- +// CALLBACKS FOR WALKING THROUGH REGISTRY OBJECTS + // structure used be the callbacks below struct registry_json_walk_person_urls_callback { - PERSON *p; - MACHINE *m; + REGISTRY_PERSON *p; + REGISTRY_MACHINE *m; struct web_client *w; int count; }; // callback for rendering PERSON_URLs -static inline int registry_json_person_url_callback(void *entry, void *data) { - PERSON_URL *pu = (PERSON_URL *)entry; +static int registry_json_person_url_callback(void *entry, void *data) { + REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry; struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data; struct web_client *w = c->w; @@ -1072,37 +84,92 @@ static inline int registry_json_person_url_callback(void *entry, void *data) { buffer_strcat(w->response.data, ","); buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u, \"%s\" ]", - pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->name); + pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->machine_name); - return 1; + return 0; } // callback for rendering MACHINE_URLs -static inline int registry_json_machine_url_callback(void *entry, void *data) { - MACHINE_URL *mu = (MACHINE_URL *)entry; +static int registry_json_machine_url_callback(void *entry, void *data) { + REGISTRY_MACHINE_URL *mu = (REGISTRY_MACHINE_URL *)entry; struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data; struct web_client *w = c->w; - MACHINE *m = c->m; + REGISTRY_MACHINE *m = c->m; if(unlikely(c->count++)) buffer_strcat(w->response.data, ","); buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u ]", - m->guid, mu->url->url, mu->last_t, mu->usages); + m->guid, mu->url->url, mu->last_t, mu->usages); return 1; } +// ---------------------------------------------------------------------------- + +// structure used be the callbacks below +struct registry_person_url_callback_verify_machine_exists_data { + REGISTRY_MACHINE *m; + int count; +}; + +static inline int registry_person_url_callback_verify_machine_exists(void *entry, void *data) { + struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data; + REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry; + REGISTRY_MACHINE *m = d->m; + + if(pu->machine == m) + d->count++; + + return 0; +} + +// ---------------------------------------------------------------------------- +// public HELLO request + +int registry_request_hello_json(struct web_client *w) { + registry_json_header(w, "hello", REGISTRY_STATUS_OK); + + buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"", + registry.registry_to_announce); + + registry_json_footer(w); + return 200; +} + +// ---------------------------------------------------------------------------- +//public ACCESS request + +#define REGISTRY_VERIFY_COOKIES_GUID "give-me-back-this-cookie-now--please" // the main method for registering an access int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) { - if(!registry.enabled) + if(unlikely(!registry.enabled)) return registry_json_disabled(w, "access"); - PERSON *p = registry_request_access(person_guid, machine_guid, url, name, when); + // ------------------------------------------------------------------------ + // verify the browser supports cookies + + if(registry.verify_cookies_redirects > 0 && !person_guid[0]) { + buffer_flush(w->response.data); + registry_set_cookie(w, REGISTRY_VERIFY_COOKIES_GUID); + w->response.data->contenttype = CT_APPLICATION_JSON; + buffer_sprintf(w->response.data, "{ \"status\": \"redirect\", \"registry\": \"%s\" }", registry.registry_to_announce); + return 200; + } + + if(unlikely(person_guid[0] && !strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID))) + person_guid[0] = '\0'; + + // ------------------------------------------------------------------------ + + registry_lock(); + + REGISTRY_PERSON *p = registry_request_access(person_guid, machine_guid, url, name, when); if(!p) { registry_json_header(w, "access", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 412; } @@ -1114,40 +181,54 @@ int registry_request_access_json(struct web_client *w, char *person_guid, char * buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\",\n\t\"urls\": [", p->guid); struct registry_json_walk_person_urls_callback c = { p, NULL, w, 0 }; - dictionary_get_all(p->urls, registry_json_person_url_callback, &c); + avl_traverse(&p->person_urls, registry_json_person_url_callback, &c); buffer_strcat(w->response.data, "\n\t]\n"); registry_json_footer(w); + registry_unlock(); return 200; } +// ---------------------------------------------------------------------------- +// public DELETE request + // the main method for deleting a URL from a person int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) { if(!registry.enabled) return registry_json_disabled(w, "delete"); - PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when); + registry_lock(); + + REGISTRY_PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when); if(!p) { registry_json_header(w, "delete", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 412; } // generate the response registry_json_header(w, "delete", REGISTRY_STATUS_OK); registry_json_footer(w); + registry_unlock(); return 200; } +// ---------------------------------------------------------------------------- +// public SEARCH request + // the main method for searching the URLs of a netdata int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) { if(!registry.enabled) return registry_json_disabled(w, "search"); - MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when); + registry_lock(); + + REGISTRY_MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when); if(!m) { registry_json_header(w, "search", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 404; } @@ -1155,75 +236,69 @@ int registry_request_search_json(struct web_client *w, char *person_guid, char * buffer_strcat(w->response.data, ",\n\t\"urls\": ["); struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 }; - dictionary_get_all(m->urls, registry_json_machine_url_callback, &c); + dictionary_get_all(m->machine_urls, registry_json_machine_url_callback, &c); buffer_strcat(w->response.data, "\n\t]\n"); registry_json_footer(w); + registry_unlock(); return 200; } -// structure used be the callbacks below -struct registry_person_url_callback_verify_machine_exists_data { - MACHINE *m; - int count; -}; - -int registry_person_url_callback_verify_machine_exists(void *entry, void *data) { - struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data; - PERSON_URL *pu = (PERSON_URL *)entry; - MACHINE *m = d->m; - - if(pu->machine == m) - d->count++; - - return 0; -} +// ---------------------------------------------------------------------------- +// SWITCH REQUEST // the main method for switching user identity int registry_request_switch_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when) { + if(!registry.enabled) + return registry_json_disabled(w, "switch"); + (void)url; (void)when; - if(!registry.enabled) - return registry_json_disabled(w, "switch"); + registry_lock(); - PERSON *op = registry_person_find(person_guid); + REGISTRY_PERSON *op = registry_person_find(person_guid); if(!op) { registry_json_header(w, "switch", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 430; } - PERSON *np = registry_person_find(new_person_guid); + REGISTRY_PERSON *np = registry_person_find(new_person_guid); if(!np) { registry_json_header(w, "switch", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 431; } - MACHINE *m = registry_machine_find(machine_guid); + REGISTRY_MACHINE *m = registry_machine_find(machine_guid); if(!m) { registry_json_header(w, "switch", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 432; } struct registry_person_url_callback_verify_machine_exists_data data = { m, 0 }; // verify the old person has access to this machine - dictionary_get_all(op->urls, registry_person_url_callback_verify_machine_exists, &data); + avl_traverse(&op->person_urls, registry_person_url_callback_verify_machine_exists, &data); if(!data.count) { registry_json_header(w, "switch", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 433; } // verify the new person has access to this machine data.count = 0; - dictionary_get_all(np->urls, registry_person_url_callback_verify_machine_exists, &data); + avl_traverse(&np->person_urls, registry_person_url_callback_verify_machine_exists, &data); if(!data.count) { registry_json_header(w, "switch", REGISTRY_STATUS_FAILED); registry_json_footer(w); + registry_unlock(); return 434; } @@ -1235,582 +310,9 @@ int registry_request_switch_json(struct web_client *w, char *person_guid, char * registry_json_header(w, "switch", REGISTRY_STATUS_OK); buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\"", np->guid); registry_json_footer(w); - return 200; -} - - -// ---------------------------------------------------------------------------- -// REGISTRY THIS MACHINE UNIQUE ID - -static inline int is_machine_guid_blacklisted(const char *guid) { - // these are machine GUIDs that have been included in distribution packages. - // we blacklist them here, so that the next version of netdata will generate - // new ones. - - if(!strcmp(guid, "8a795b0c-2311-11e6-8563-000c295076a6") - || !strcmp(guid, "4aed1458-1c3e-11e6-a53f-000c290fc8f5") - ) { - error("Blacklisted machine GUID '%s' found.", guid); - return 1; - } - - return 0; -} - -char *registry_get_this_machine_guid(void) { - if(likely(registry.machine_guid[0])) - return registry.machine_guid; - - // read it from disk - int fd = open(registry.machine_guid_filename, O_RDONLY); - if(fd != -1) { - char buf[36 + 1]; - if(read(fd, buf, 36) != 36) - error("Failed to read machine GUID from '%s'", registry.machine_guid_filename); - else { - buf[36] = '\0'; - if(registry_regenerate_guid(buf, registry.machine_guid) == -1) { - error("Failed to validate machine GUID '%s' from '%s'. Ignoring it - this might mean this netdata will appear as duplicate in the registry.", - buf, registry.machine_guid_filename); - - registry.machine_guid[0] = '\0'; - } - else if(is_machine_guid_blacklisted(registry.machine_guid)) - registry.machine_guid[0] = '\0'; - } - close(fd); - } - - // generate a new one? - if(!registry.machine_guid[0]) { - uuid_t uuid; - - uuid_generate_time(uuid); - uuid_unparse_lower(uuid, registry.machine_guid); - registry.machine_guid[36] = '\0'; - - // save it - fd = open(registry.machine_guid_filename, O_WRONLY|O_CREAT|O_TRUNC, 444); - if(fd == -1) - fatal("Cannot create unique machine id file '%s'. Please fix this.", registry.machine_guid_filename); - - if(write(fd, registry.machine_guid, 36) != 36) - fatal("Cannot write the unique machine id file '%s'. Please fix this.", registry.machine_guid_filename); - - close(fd); - } - - setenv("NETDATA_REGISTRY_UNIQUE_ID", registry.machine_guid, 1); - - return registry.machine_guid; -} - - -// ---------------------------------------------------------------------------- -// REGISTRY LOAD/SAVE - -int registry_machine_save_url(void *entry, void *file) { - MACHINE_URL *mu = entry; - FILE *fp = file; - - debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url); - - int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n", - mu->first_t, - mu->last_t, - mu->usages, - mu->flags, - mu->url->url - ); - - // error handling is done at registry_save() - - return ret; -} - -int registry_machine_save(void *entry, void *file) { - MACHINE *m = entry; - FILE *fp = file; - - debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid); - - int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n", - m->first_t, - m->last_t, - m->usages, - m->guid - ); - - if(ret >= 0) { - int ret2 = dictionary_get_all(m->urls, registry_machine_save_url, fp); - if(ret2 < 0) return ret2; - ret += ret2; - } - - // error handling is done at registry_save() - - return ret; -} - -static inline int registry_person_save_url(void *entry, void *file) { - PERSON_URL *pu = entry; - FILE *fp = file; - - debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url); - - int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n", - pu->first_t, - pu->last_t, - pu->usages, - pu->flags, - pu->machine->guid, - pu->name, - pu->url->url - ); - - // error handling is done at registry_save() - - return ret; -} - -static inline int registry_person_save(void *entry, void *file) { - PERSON *p = entry; - FILE *fp = file; - - debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid); - - int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n", - p->first_t, - p->last_t, - p->usages, - p->guid - ); - - if(ret >= 0) { - int ret2 = dictionary_get_all(p->urls, registry_person_save_url, fp); - if (ret2 < 0) return ret2; - ret += ret2; - } - - // error handling is done at registry_save() - - return ret; -} - -int registry_save(void) { - if(!registry.enabled) return -1; - - // make sure the log is not updated - registry_log_lock(); - - if(unlikely(!registry_should_save_db())) { - registry_log_unlock(); - return -2; - } - - error_log_limit_unlimited(); - - char tmp_filename[FILENAME_MAX + 1]; - char old_filename[FILENAME_MAX + 1]; - - snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename); - snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename); - - debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename); - FILE *fp = fopen(tmp_filename, "w"); - if(!fp) { - error("Registry: Cannot create file: %s", tmp_filename); - registry_log_unlock(); - error_log_limit_reset(); - return -1; - } - - // dictionary_get_all() has its own locking, so this is safe to do - debug(D_REGISTRY, "Saving all machines"); - int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp); - if(bytes1 < 0) { - error("Registry: Cannot save registry machines - return value %d", bytes1); - fclose(fp); - registry_log_unlock(); - error_log_limit_reset(); - return bytes1; - } - debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1); - - debug(D_REGISTRY, "Saving all persons"); - int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp); - if(bytes2 < 0) { - error("Registry: Cannot save registry persons - return value %d", bytes2); - fclose(fp); - registry_log_unlock(); - error_log_limit_reset(); - return bytes2; - } - debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2); - - // save the totals - fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n", - registry.persons_count, - registry.machines_count, - registry.usages_count + 1, // this is required - it is lost on db rotation - registry.urls_count, - registry.persons_urls_count, - registry.machines_urls_count - ); - - fclose(fp); - - errno = 0; - - // remove the .old db - debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename); - if(unlink(old_filename) == -1 && errno != ENOENT) - error("Registry: cannot remove old registry file '%s'", old_filename); - - // rename the db to .old - debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename); - if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT) - error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, registry.db_filename); - - else { - // remove the database (it is saved in .old) - debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename); - if (unlink(registry.db_filename) == -1 && errno != ENOENT) - error("Registry: cannot remove old registry file '%s'", registry.db_filename); - - // move the .tmp to make it active - debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename); - if (link(tmp_filename, registry.db_filename) == -1) { - error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, - registry.db_filename); - - // move the .old back - debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename); - if(link(old_filename, registry.db_filename) == -1) - error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename); - } - else { - debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename); - if(unlink(tmp_filename) == -1) - error("Registry: cannot remove tmp registry file '%s'", tmp_filename); - - // it has been moved successfully - // discard the current registry log - registry_log_recreate_nolock(); - registry.log_count = 0; - } - } - - // continue operations - registry_log_unlock(); - error_log_limit_reset(); - - return -1; -} - -static inline size_t registry_load(void) { - char *s, buf[4096 + 1]; - PERSON *p = NULL; - MACHINE *m = NULL; - URL *u = NULL; - size_t line = 0; - - debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename); - FILE *fp = fopen(registry.db_filename, "r"); - if(!fp) { - error("Registry: cannot open registry file: '%s'", registry.db_filename); - return 0; - } - - size_t len = 0; - buf[4096] = '\0'; - while((s = fgets_trim_len(buf, 4096, fp, &len))) { - line++; - - debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s); - switch(*s) { - case 'T': // totals - if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) { - error("Registry totals line %zu is wrong (len = %zu).", line, len); - continue; - } - registry.persons_count = strtoull(&s[2], NULL, 16); - registry.machines_count = strtoull(&s[19], NULL, 16); - registry.usages_count = strtoull(&s[36], NULL, 16); - registry.urls_count = strtoull(&s[53], NULL, 16); - registry.persons_urls_count = strtoull(&s[70], NULL, 16); - registry.machines_urls_count = strtoull(&s[87], NULL, 16); - break; - - case 'P': // person - m = NULL; - // verify it is valid - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) { - error("Registry person line %zu is wrong (len = %zu).", line, len); - continue; - } - - s[1] = s[10] = s[19] = s[28] = '\0'; - p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16)); - p->last_t = strtoul(&s[11], NULL, 16); - p->usages = strtoul(&s[20], NULL, 16); - debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages); - break; - - case 'M': // machine - p = NULL; - // verify it is valid - if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) { - error("Registry person line %zu is wrong (len = %zu).", line, len); - continue; - } - - s[1] = s[10] = s[19] = s[28] = '\0'; - m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16)); - m->last_t = strtoul(&s[11], NULL, 16); - m->usages = strtoul(&s[20], NULL, 16); - debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages); - break; - - case 'U': // person URL - if(unlikely(!p)) { - error("Registry: ignoring line %zu, no person loaded: %s", line, s); - continue; - } - - // verify it is valid - if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') { - error("Registry person URL line %zu is wrong (len = %zu).", line, len); - continue; - } - - s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0'; - - // skip the name to find the url - char *url = &s[69]; - while(*url && *url != '\t') url++; - if(!*url) { - error("Registry person URL line %zu does not have a url.", line); - continue; - } - *url++ = '\0'; - - // u = registry_url_allocate_nolock(url, strlen(url)); - u = registry_url_get_nolock(url, strlen(url)); - - time_t first_t = strtoul(&s[2], NULL, 16); - - m = registry_machine_find(&s[32]); - if(!m) m = registry_machine_allocate(&s[32], first_t); - - PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t); - pu->last_t = strtoul(&s[11], NULL, 16); - pu->usages = strtoul(&s[20], NULL, 16); - pu->flags = strtoul(&s[29], NULL, 16); - debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags); - break; - - case 'V': // machine URL - if(unlikely(!m)) { - error("Registry: ignoring line %zu, no machine loaded: %s", line, s); - continue; - } - - // verify it is valid - if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') { - error("Registry person URL line %zu is wrong (len = %zu).", line, len); - continue; - } - - s[1] = s[10] = s[19] = s[28] = s[31] = '\0'; - // u = registry_url_allocate_nolock(&s[32], strlen(&s[32])); - u = registry_url_get_nolock(&s[32], strlen(&s[32])); - - MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16)); - mu->last_t = strtoul(&s[11], NULL, 16); - mu->usages = strtoul(&s[20], NULL, 16); - mu->flags = strtoul(&s[29], NULL, 16); - debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags); - break; - - default: - error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s); - break; - } - } - fclose(fp); - - return line; -} - -// ---------------------------------------------------------------------------- -// REGISTRY - -int registry_init(void) { - char filename[FILENAME_MAX + 1]; - - // registry enabled? - registry.enabled = config_get_boolean("registry", "enabled", 0); - - // pathnames - registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR "/registry"); - if(mkdir(registry.pathname, 0770) == -1 && errno != EEXIST) - fatal("Cannot create directory '%s'.", registry.pathname); - - // filenames - snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname); - registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename); - registry_get_this_machine_guid(); - - snprintfz(filename, FILENAME_MAX, "%s/registry.db", registry.pathname); - registry.db_filename = config_get("registry", "registry db file", filename); - - snprintfz(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname); - registry.log_filename = config_get("registry", "registry log file", filename); - - // configuration options - registry.save_registry_every_entries = config_get_number("registry", "registry save db every new entries", 1000000); - registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400; - registry.registry_domain = config_get("registry", "registry domain", ""); - registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io"); - registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", localhost.hostname)); - registry.verify_cookies_redirects = config_get_boolean("registry", "verify browser cookies support", 1); - - setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1); - setenv("NETDATA_REGISTRY_URL", registry.registry_to_announce, 1); - - registry.max_url_length = config_get_number("registry", "max URL length", 1024); - if(registry.max_url_length < 10) { - registry.max_url_length = 10; - config_set_number("registry", "max URL length", registry.max_url_length); - } - - registry.max_name_length = config_get_number("registry", "max URL name length", 50); - if(registry.max_name_length < 10) { - registry.max_name_length = 10; - config_set_number("registry", "max URL name length", registry.max_name_length); - } - - // initialize entries counters - registry.persons_count = 0; - registry.machines_count = 0; - registry.usages_count = 0; - registry.urls_count = 0; - registry.persons_urls_count = 0; - registry.machines_urls_count = 0; - - // initialize memory counters - registry.persons_memory = 0; - registry.machines_memory = 0; - registry.urls_memory = 0; - registry.persons_urls_memory = 0; - registry.machines_urls_memory = 0; - - // initialize locks - pthread_mutex_init(®istry.persons_lock, NULL); - pthread_mutex_init(®istry.machines_lock, NULL); - pthread_mutex_init(®istry.urls_lock, NULL); - pthread_mutex_init(®istry.person_urls_lock, NULL); - pthread_mutex_init(®istry.machine_urls_lock, NULL); - - // create dictionaries - registry.persons = dictionary_create(DICTIONARY_FLAGS); - registry.machines = dictionary_create(DICTIONARY_FLAGS); - registry.urls = dictionary_create(DICTIONARY_FLAGS); - - // load the registry database - if(registry.enabled) { - registry_log_open_nolock(); - registry_load(); - registry_log_load(); - - if(unlikely(registry_should_save_db())) - registry_save(); - } - - return 0; -} - -void registry_free(void) { - if(!registry.enabled) return; - - // we need to destroy the dictionaries ourselves - // since the dictionaries use memory we allocated - - while(registry.persons->values_index.root) { - PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value; - - // fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages); - - while(p->urls->values_index.root) { - PERSON_URL *pu = ((NAME_VALUE *)p->urls->values_index.root)->value; - - // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags); - - debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid); - dictionary_del(p->urls, pu->url->url); - - debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url); - registry_url_unlink_nolock(pu->url); - - debug(D_REGISTRY, "Registry: freeing person url"); - freez(pu); - } - - debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid); - dictionary_del(registry.persons, p->guid); - - debug(D_REGISTRY, "Registry: destroying URL dictionary of person '%s'", p->guid); - dictionary_destroy(p->urls); - - debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid); - freez(p); - } - - while(registry.machines->values_index.root) { - MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value; - - // fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages); - - while(m->urls->values_index.root) { - MACHINE_URL *mu = ((NAME_VALUE *)m->urls->values_index.root)->value; - - // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags); - - //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url); - //dictionary_destroy(mu->persons); - - debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid); - dictionary_del(m->urls, mu->url->url); - - debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url); - registry_url_unlink_nolock(mu->url); - - debug(D_REGISTRY, "Registry: freeing machine url"); - freez(mu); - } - - debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid); - dictionary_del(registry.machines, m->guid); - - debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid); - dictionary_destroy(m->urls); - - debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid); - freez(m); - } - - // and free the memory of remaining dictionary structures - - debug(D_REGISTRY, "Registry: destroying persons dictionary"); - dictionary_destroy(registry.persons); - - debug(D_REGISTRY, "Registry: destroying machines dictionary"); - dictionary_destroy(registry.machines); - - debug(D_REGISTRY, "Registry: destroying urls dictionary"); - dictionary_destroy(registry.urls); + registry_unlock(); + return 200; } // ---------------------------------------------------------------------------- @@ -1869,8 +371,8 @@ void registry_statistics(void) { rrddim_set(stm, "persons", registry.persons_memory + registry.persons_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY)); rrddim_set(stm, "machines", registry.machines_memory + registry.machines_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY)); - rrddim_set(stm, "urls", registry.urls_memory + registry.urls_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY)); - rrddim_set(stm, "persons_urls", registry.persons_urls_memory + registry.persons_count * sizeof(DICTIONARY) + registry.persons_urls_count * sizeof(NAME_VALUE)); + rrddim_set(stm, "urls", registry.urls_memory); + rrddim_set(stm, "persons_urls", registry.persons_urls_memory); rrddim_set(stm, "machines_urls", registry.machines_urls_memory + registry.machines_count * sizeof(DICTIONARY) + registry.machines_urls_count * sizeof(NAME_VALUE)); rrdset_done(stm); } diff --git a/src/registry.h b/src/registry.h index c2b57a23d..4947486c4 100644 --- a/src/registry.h +++ b/src/registry.h @@ -1,25 +1,72 @@ +/* + * netdata registry + * + * this header file describes the public interface + * to the netdata registry + * + * only these high level functions are exposed + * + */ + +// ---------------------------------------------------------------------------- +// TODO +// +// 1. the default tracking cookie expires in 1 year, but the persons are not +// removed from the db - this means the database only grows - ideally the +// database should be cleaned in registry_db_save() for both on-disk and +// on-memory entries. +// +// Cleanup: +// i. Find all the PERSONs that have expired cookie +// ii. For each of their PERSON_URLs: +// - decrement the linked MACHINE links +// - if the linked MACHINE has no other links, remove the linked MACHINE too +// - remove the PERSON_URL +// +// 2. add protection to prevent abusing the registry by flooding it with +// requests to fill the memory and crash it. +// +// Possible protections: +// - limit the number of URLs per person +// - limit the number of URLs per machine +// - limit the number of persons +// - limit the number of machines +// - [DONE] limit the size of URLs +// - [DONE] limit the size of PERSON_URL names +// - limit the number of requests that add data to the registry, +// per client IP per hour +// +// 3. lower memory requirements +// +// - embed avl structures directly into registry objects, instead of DICTIONARY +// [DONE for PERSON_URLs, PENDING for MACHINE_URLs] +// - store GUIDs in memory as UUID instead of char * +// - do not track persons using the demo machines only +// (i.e. start tracking them only when they access a non-demo machine) +// - [DONE] do not track custom dashboards by default + + #ifndef NETDATA_REGISTRY_H #define NETDATA_REGISTRY_H 1 #define NETDATA_REGISTRY_COOKIE_NAME "netdata_registry_id" -extern void registry_set_cookie(struct web_client *w, const char *guid); -extern const char *registry_to_announce(void); -extern int registry_verify_cookies_redirects(void); +// initialize the registry +// should only happen when netdata starts +extern int registry_init(void); + +// free all data held by the registry +// should only happen when netdata exits +extern void registry_free(void); +// HTTP requests handled by the registry extern int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when); extern int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when); extern int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when); extern int registry_request_switch_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when); extern int registry_request_hello_json(struct web_client *w); -extern int registry_init(void); -extern void registry_free(void); -extern int registry_save(void); - -extern char *registry_get_this_machine_guid(void); - +// update the registry monitoring charts extern void registry_statistics(void); - #endif /* NETDATA_REGISTRY_H */ diff --git a/src/registry_db.c b/src/registry_db.c new file mode 100644 index 000000000..de6c634c3 --- /dev/null +++ b/src/registry_db.c @@ -0,0 +1,343 @@ +#include "registry_internals.h" + +int registry_db_should_be_saved(void) { + debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries); + return registry.log_count > registry.save_registry_every_entries; +} + +// ---------------------------------------------------------------------------- +// INTERNAL FUNCTIONS FOR SAVING REGISTRY OBJECTS + +static int registry_machine_save_url(void *entry, void *file) { + REGISTRY_MACHINE_URL *mu = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url); + + int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n", + mu->first_t, + mu->last_t, + mu->usages, + mu->flags, + mu->url->url + ); + + // error handling is done at registry_db_save() + + return ret; +} + +static int registry_machine_save(void *entry, void *file) { + REGISTRY_MACHINE *m = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid); + + int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n", + m->first_t, + m->last_t, + m->usages, + m->guid + ); + + if(ret >= 0) { + int ret2 = dictionary_get_all(m->machine_urls, registry_machine_save_url, fp); + if(ret2 < 0) return ret2; + ret += ret2; + } + + // error handling is done at registry_db_save() + + return ret; +} + +static inline int registry_person_save_url(void *entry, void *file) { + REGISTRY_PERSON_URL *pu = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url); + + int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n", + pu->first_t, + pu->last_t, + pu->usages, + pu->flags, + pu->machine->guid, + pu->machine_name, + pu->url->url + ); + + // error handling is done at registry_db_save() + + return ret; +} + +static inline int registry_person_save(void *entry, void *file) { + REGISTRY_PERSON *p = entry; + FILE *fp = file; + + debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid); + + int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n", + p->first_t, + p->last_t, + p->usages, + p->guid + ); + + if(ret >= 0) { + //int ret2 = dictionary_get_all(p->person_urls, registry_person_save_url, fp); + int ret2 = avl_traverse(&p->person_urls, registry_person_save_url, fp); + if (ret2 < 0) return ret2; + ret += ret2; + } + + // error handling is done at registry_db_save() + + return ret; +} + +// ---------------------------------------------------------------------------- +// SAVE THE REGISTRY DATABASE + +int registry_db_save(void) { + if(unlikely(!registry.enabled)) + return -1; + + if(unlikely(!registry_db_should_be_saved())) + return -2; + + error_log_limit_unlimited(); + + char tmp_filename[FILENAME_MAX + 1]; + char old_filename[FILENAME_MAX + 1]; + + snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename); + snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename); + + debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename); + FILE *fp = fopen(tmp_filename, "w"); + if(!fp) { + error("Registry: Cannot create file: %s", tmp_filename); + error_log_limit_reset(); + return -1; + } + + // dictionary_get_all() has its own locking, so this is safe to do + + debug(D_REGISTRY, "Saving all machines"); + int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp); + if(bytes1 < 0) { + error("Registry: Cannot save registry machines - return value %d", bytes1); + fclose(fp); + error_log_limit_reset(); + return bytes1; + } + debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1); + + debug(D_REGISTRY, "Saving all persons"); + int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp); + if(bytes2 < 0) { + error("Registry: Cannot save registry persons - return value %d", bytes2); + fclose(fp); + error_log_limit_reset(); + return bytes2; + } + debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2); + + // save the totals + fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n", + registry.persons_count, + registry.machines_count, + registry.usages_count + 1, // this is required - it is lost on db rotation + registry.urls_count, + registry.persons_urls_count, + registry.machines_urls_count + ); + + fclose(fp); + + errno = 0; + + // remove the .old db + debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename); + if(unlink(old_filename) == -1 && errno != ENOENT) + error("Registry: cannot remove old registry file '%s'", old_filename); + + // rename the db to .old + debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename); + if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT) + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", registry.db_filename, old_filename); + + else { + // remove the database (it is saved in .old) + debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename); + if (unlink(registry.db_filename) == -1 && errno != ENOENT) + error("Registry: cannot remove old registry file '%s'", registry.db_filename); + + // move the .tmp to make it active + debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename); + if (link(tmp_filename, registry.db_filename) == -1) { + error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, + registry.db_filename); + + // move the .old back + debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename); + if(link(old_filename, registry.db_filename) == -1) + error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename); + } + else { + debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename); + if(unlink(tmp_filename) == -1) + error("Registry: cannot remove tmp registry file '%s'", tmp_filename); + + // it has been moved successfully + // discard the current registry log + registry_log_recreate(); + registry.log_count = 0; + } + } + + // continue operations + error_log_limit_reset(); + + return -1; +} + +// ---------------------------------------------------------------------------- +// LOAD THE REGISTRY DATABASE + +size_t registry_db_load(void) { + char *s, buf[4096 + 1]; + REGISTRY_PERSON *p = NULL; + REGISTRY_MACHINE *m = NULL; + REGISTRY_URL *u = NULL; + size_t line = 0; + + debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename); + FILE *fp = fopen(registry.db_filename, "r"); + if(!fp) { + error("Registry: cannot open registry file: '%s'", registry.db_filename); + return 0; + } + + size_t len = 0; + buf[4096] = '\0'; + while((s = fgets_trim_len(buf, 4096, fp, &len))) { + line++; + + debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s); + switch(*s) { + case 'T': // totals + if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) { + error("Registry totals line %zu is wrong (len = %zu).", line, len); + continue; + } + registry.persons_count = strtoull(&s[2], NULL, 16); + registry.machines_count = strtoull(&s[19], NULL, 16); + registry.usages_count = strtoull(&s[36], NULL, 16); + registry.urls_count = strtoull(&s[53], NULL, 16); + registry.persons_urls_count = strtoull(&s[70], NULL, 16); + registry.machines_urls_count = strtoull(&s[87], NULL, 16); + break; + + case 'P': // person + m = NULL; + // verify it is valid + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) { + error("Registry person line %zu is wrong (len = %zu).", line, len); + continue; + } + + s[1] = s[10] = s[19] = s[28] = '\0'; + p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16)); + p->last_t = (uint32_t)strtoul(&s[11], NULL, 16); + p->usages = (uint32_t)strtoul(&s[20], NULL, 16); + debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages); + break; + + case 'M': // machine + p = NULL; + // verify it is valid + if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) { + error("Registry person line %zu is wrong (len = %zu).", line, len); + continue; + } + + s[1] = s[10] = s[19] = s[28] = '\0'; + m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16)); + m->last_t = (uint32_t)strtoul(&s[11], NULL, 16); + m->usages = (uint32_t)strtoul(&s[20], NULL, 16); + debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages); + break; + + case 'U': // person URL + if(unlikely(!p)) { + error("Registry: ignoring line %zu, no person loaded: %s", line, s); + continue; + } + + // verify it is valid + if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') { + error("Registry person URL line %zu is wrong (len = %zu).", line, len); + continue; + } + + s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0'; + + // skip the name to find the url + char *url = &s[69]; + while(*url && *url != '\t') url++; + if(!*url) { + error("Registry person URL line %zu does not have a url.", line); + continue; + } + *url++ = '\0'; + + // u = registry_url_allocate_nolock(url, strlen(url)); + u = registry_url_get(url, strlen(url)); + + time_t first_t = strtoul(&s[2], NULL, 16); + + m = registry_machine_find(&s[32]); + if(!m) m = registry_machine_allocate(&s[32], first_t); + + REGISTRY_PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t); + pu->last_t = (uint32_t)strtoul(&s[11], NULL, 16); + pu->usages = (uint32_t)strtoul(&s[20], NULL, 16); + pu->flags = (uint8_t)strtoul(&s[29], NULL, 16); + debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->machine_name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags); + break; + + case 'V': // machine URL + if(unlikely(!m)) { + error("Registry: ignoring line %zu, no machine loaded: %s", line, s); + continue; + } + + // verify it is valid + if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') { + error("Registry person URL line %zu is wrong (len = %zu).", line, len); + continue; + } + + s[1] = s[10] = s[19] = s[28] = s[31] = '\0'; + // u = registry_url_allocate_nolock(&s[32], strlen(&s[32])); + u = registry_url_get(&s[32], strlen(&s[32])); + + REGISTRY_MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16)); + mu->last_t = (uint32_t)strtoul(&s[11], NULL, 16); + mu->usages = (uint32_t)strtoul(&s[20], NULL, 16); + mu->flags = (uint8_t)strtoul(&s[29], NULL, 16); + debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags); + break; + + default: + error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s); + break; + } + } + fclose(fp); + + return line; +} diff --git a/src/registry_init.c b/src/registry_init.c new file mode 100644 index 000000000..fb61acd08 --- /dev/null +++ b/src/registry_init.c @@ -0,0 +1,136 @@ +#include "registry_internals.h" + +int registry_init(void) { + char filename[FILENAME_MAX + 1]; + + // registry enabled? + registry.enabled = config_get_boolean("registry", "enabled", 0); + + // pathnames + registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR "/registry"); + if(mkdir(registry.pathname, 0770) == -1 && errno != EEXIST) + fatal("Cannot create directory '%s'.", registry.pathname); + + // filenames + snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname); + registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename); + registry_get_this_machine_guid(); + + snprintfz(filename, FILENAME_MAX, "%s/registry.db", registry.pathname); + registry.db_filename = config_get("registry", "registry db file", filename); + + snprintfz(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname); + registry.log_filename = config_get("registry", "registry log file", filename); + + // configuration options + registry.save_registry_every_entries = (unsigned long long)config_get_number("registry", "registry save db every new entries", 1000000); + registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400; + registry.registry_domain = config_get("registry", "registry domain", ""); + registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io"); + registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", localhost.hostname)); + registry.verify_cookies_redirects = config_get_boolean("registry", "verify browser cookies support", 1); + + setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1); + setenv("NETDATA_REGISTRY_URL", registry.registry_to_announce, 1); + + registry.max_url_length = (size_t)config_get_number("registry", "max URL length", 1024); + if(registry.max_url_length < 10) { + registry.max_url_length = 10; + config_set_number("registry", "max URL length", (long long)registry.max_url_length); + } + + registry.max_name_length = (size_t)config_get_number("registry", "max URL name length", 50); + if(registry.max_name_length < 10) { + registry.max_name_length = 10; + config_set_number("registry", "max URL name length", (long long)registry.max_name_length); + } + + // initialize entries counters + registry.persons_count = 0; + registry.machines_count = 0; + registry.usages_count = 0; + registry.urls_count = 0; + registry.persons_urls_count = 0; + registry.machines_urls_count = 0; + + // initialize memory counters + registry.persons_memory = 0; + registry.machines_memory = 0; + registry.urls_memory = 0; + registry.persons_urls_memory = 0; + registry.machines_urls_memory = 0; + + // initialize locks + pthread_mutex_init(®istry.lock, NULL); + + // create dictionaries + registry.persons = dictionary_create(DICTIONARY_FLAGS); + registry.machines = dictionary_create(DICTIONARY_FLAGS); + avl_init(®istry.registry_urls_root_index, registry_url_compare); + + // load the registry database + if(registry.enabled) { + registry_log_open(); + registry_db_load(); + registry_log_load(); + + if(unlikely(registry_db_should_be_saved())) + registry_db_save(); + } + + return 0; +} + +void registry_free(void) { + if(!registry.enabled) return; + + // we need to destroy the dictionaries ourselves + // since the dictionaries use memory we allocated + + while(registry.persons->values_index.root) { + REGISTRY_PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value; + registry_person_del(p); + } + + while(registry.machines->values_index.root) { + REGISTRY_MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value; + + // fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages); + + while(m->machine_urls->values_index.root) { + REGISTRY_MACHINE_URL *mu = ((NAME_VALUE *)m->machine_urls->values_index.root)->value; + + // fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags); + + //debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url); + //dictionary_destroy(mu->persons); + + debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid); + dictionary_del(m->machine_urls, mu->url->url); + + debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url); + registry_url_unlink(mu->url); + + debug(D_REGISTRY, "Registry: freeing machine url"); + freez(mu); + } + + debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid); + dictionary_del(registry.machines, m->guid); + + debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid); + dictionary_destroy(m->machine_urls); + + debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid); + freez(m); + } + + // and free the memory of remaining dictionary structures + + debug(D_REGISTRY, "Registry: destroying persons dictionary"); + dictionary_destroy(registry.persons); + + debug(D_REGISTRY, "Registry: destroying machines dictionary"); + dictionary_destroy(registry.machines); +} + diff --git a/src/registry_internals.c b/src/registry_internals.c new file mode 100644 index 000000000..d32d549e2 --- /dev/null +++ b/src/registry_internals.c @@ -0,0 +1,323 @@ +#include "registry_internals.h" + +struct registry registry; + +// ---------------------------------------------------------------------------- +// common functions + +// parse a GUID and re-generated to be always lower case +// this is used as a protection against the variations of GUIDs +int registry_regenerate_guid(const char *guid, char *result) { + uuid_t uuid; + if(unlikely(uuid_parse(guid, uuid) == -1)) { + info("Registry: GUID '%s' is not a valid GUID.", guid); + return -1; + } + else { + uuid_unparse_lower(uuid, result); + +#ifdef NETDATA_INTERNAL_CHECKS + if(strcmp(guid, result)) + info("Registry: source GUID '%s' and re-generated GUID '%s' differ!", guid, result); +#endif /* NETDATA_INTERNAL_CHECKS */ + } + + return 0; +} + +// make sure the names of the machines / URLs do not contain any tabs +// (which are used as our separator in the database files) +// and are properly trimmed (before and after) +static inline char *registry_fix_machine_name(char *name, size_t *len) { + char *s = name?name:""; + + // skip leading spaces + while(*s && isspace(*s)) s++; + + // make sure all spaces are a SPACE + char *t = s; + while(*t) { + if(unlikely(isspace(*t))) + *t = ' '; + + t++; + } + + // remove trailing spaces + while(--t >= s) { + if(*t == ' ') + *t = '\0'; + else + break; + } + t++; + + if(likely(len)) + *len = (t - s); + + return s; +} + +static inline char *registry_fix_url(char *url, size_t *len) { + size_t l = 0; + char *s = registry_fix_machine_name(url, &l); + + // protection from too big URLs + if(l > registry.max_url_length) { + l = registry.max_url_length; + s[l] = '\0'; + } + + if(len) *len = l; + return s; +} + + +// ---------------------------------------------------------------------------- +// forward definition of functions + +extern REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when); +extern REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when); + + +// ---------------------------------------------------------------------------- +// HELPERS + +// verify the person, the machine and the URL exist in our DB +REGISTRY_PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, REGISTRY_PERSON **pp, REGISTRY_MACHINE **mm) { + char pbuf[GUID_LEN + 1], mbuf[GUID_LEN + 1]; + + if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) { + info("Registry Request Verification: invalid request! person: '%s', machine '%s', url '%s'", person_guid?person_guid:"UNSET", machine_guid?machine_guid:"UNSET", url?url:"UNSET"); + return NULL; + } + + // normalize the url + url = registry_fix_url(url, NULL); + + // make sure the person GUID is valid + if(registry_regenerate_guid(person_guid, pbuf) == -1) { + info("Registry Request Verification: invalid person GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); + return NULL; + } + person_guid = pbuf; + + // make sure the machine GUID is valid + if(registry_regenerate_guid(machine_guid, mbuf) == -1) { + info("Registry Request Verification: invalid machine GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); + return NULL; + } + machine_guid = mbuf; + + // make sure the machine exists + REGISTRY_MACHINE *m = registry_machine_find(machine_guid); + if(!m) { + info("Registry Request Verification: machine not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); + return NULL; + } + if(mm) *mm = m; + + // make sure the person exist + REGISTRY_PERSON *p = registry_person_find(person_guid); + if(!p) { + info("Registry Request Verification: person not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); + return NULL; + } + if(pp) *pp = p; + + REGISTRY_PERSON_URL *pu = registry_person_url_index_find(p, url); + if(!pu) { + info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url); + return NULL; + } + return pu; +} + + +// ---------------------------------------------------------------------------- +// REGISTRY REQUESTS + +REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) { + debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url); + + REGISTRY_MACHINE *m = registry_machine_get(machine_guid, when); + if(!m) return NULL; + + // make sure the name is valid + size_t namelen; + name = registry_fix_machine_name(name, &namelen); + + size_t urllen; + url = registry_fix_url(url, &urllen); + + REGISTRY_PERSON *p = registry_person_get(person_guid, when); + + REGISTRY_URL *u = registry_url_get(url, urllen); + registry_person_link_to_url(p, m, u, name, namelen, when); + registry_machine_link_to_url(m, u, when); + + registry_log('A', p, m, u, name); + + registry.usages_count++; + + return p; +} + +REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) { + (void) when; + + REGISTRY_PERSON *p = NULL; + REGISTRY_MACHINE *m = NULL; + REGISTRY_PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m); + if(!pu || !p || !m) return NULL; + + // normalize the url + delete_url = registry_fix_url(delete_url, NULL); + + // make sure the user is not deleting the url it uses + if(!strcmp(delete_url, pu->url->url)) { + info("Registry Delete Request: delete URL is the one currently accessed, person: '%s', machine '%s', url '%s', delete url '%s'" + , p->guid, m->guid, pu->url->url, delete_url); + return NULL; + } + + REGISTRY_PERSON_URL *dpu = registry_person_url_index_find(p, delete_url); + if(!dpu) { + info("Registry Delete Request: URL not found for person: '%s', machine '%s', url '%s', delete url '%s'", p->guid + , m->guid, pu->url->url, delete_url); + return NULL; + } + + registry_log('D', p, m, pu->url, dpu->url->url); + registry_person_unlink_from_url(p, dpu); + + return p; +} + + +// a structure to pass to the dictionary_get_all() callback handler +struct machine_request_callback_data { + REGISTRY_MACHINE *find_this_machine; + REGISTRY_PERSON_URL *result; +}; + +// the callback function +// this will be run for every PERSON_URL of this PERSON +static int machine_request_callback(void *entry, void *data) { + REGISTRY_PERSON_URL *mypu = (REGISTRY_PERSON_URL *)entry; + struct machine_request_callback_data *myrdata = (struct machine_request_callback_data *)data; + + if(mypu->machine == myrdata->find_this_machine) { + myrdata->result = mypu; + return -1; // this will also stop the walk through + } + + return 0; // continue +} + +REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) { + (void)when; + + char mbuf[GUID_LEN + 1]; + + REGISTRY_PERSON *p = NULL; + REGISTRY_MACHINE *m = NULL; + REGISTRY_PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m); + if(!pu || !p || !m) return NULL; + + // make sure the machine GUID is valid + if(registry_regenerate_guid(request_machine, mbuf) == -1) { + info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine); + return NULL; + } + request_machine = mbuf; + + // make sure the machine exists + m = registry_machine_find(request_machine); + if(!m) { + info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, machine_guid, pu->url->url, request_machine); + return NULL; + } + + // Verify the user has in the past accessed this machine + // We will walk through the PERSON_URLs to find the machine + // linking to our machine + + // a structure to pass to the dictionary_get_all() callback handler + struct machine_request_callback_data rdata = { m, NULL }; + + // request a walk through on the dictionary + avl_traverse(&p->person_urls, machine_request_callback, &rdata); + + if(rdata.result) + return m; + + return NULL; +} + + +// ---------------------------------------------------------------------------- +// REGISTRY THIS MACHINE UNIQUE ID + +static inline int is_machine_guid_blacklisted(const char *guid) { + // these are machine GUIDs that have been included in distribution packages. + // we blacklist them here, so that the next version of netdata will generate + // new ones. + + if(!strcmp(guid, "8a795b0c-2311-11e6-8563-000c295076a6") + || !strcmp(guid, "4aed1458-1c3e-11e6-a53f-000c290fc8f5") + ) { + error("Blacklisted machine GUID '%s' found.", guid); + return 1; + } + + return 0; +} + +char *registry_get_this_machine_guid(void) { + if(likely(registry.machine_guid[0])) + return registry.machine_guid; + + // read it from disk + int fd = open(registry.machine_guid_filename, O_RDONLY); + if(fd != -1) { + char buf[GUID_LEN + 1]; + if(read(fd, buf, GUID_LEN) != GUID_LEN) + error("Failed to read machine GUID from '%s'", registry.machine_guid_filename); + else { + buf[GUID_LEN] = '\0'; + if(registry_regenerate_guid(buf, registry.machine_guid) == -1) { + error("Failed to validate machine GUID '%s' from '%s'. Ignoring it - this might mean this netdata will appear as duplicate in the registry.", + buf, registry.machine_guid_filename); + + registry.machine_guid[0] = '\0'; + } + else if(is_machine_guid_blacklisted(registry.machine_guid)) + registry.machine_guid[0] = '\0'; + } + close(fd); + } + + // generate a new one? + if(!registry.machine_guid[0]) { + uuid_t uuid; + + uuid_generate_time(uuid); + uuid_unparse_lower(uuid, registry.machine_guid); + registry.machine_guid[GUID_LEN] = '\0'; + + // save it + fd = open(registry.machine_guid_filename, O_WRONLY|O_CREAT|O_TRUNC, 444); + if(fd == -1) + fatal("Cannot create unique machine id file '%s'. Please fix this.", registry.machine_guid_filename); + + if(write(fd, registry.machine_guid, GUID_LEN) != GUID_LEN) + fatal("Cannot write the unique machine id file '%s'. Please fix this.", registry.machine_guid_filename); + + close(fd); + } + + setenv("NETDATA_REGISTRY_UNIQUE_ID", registry.machine_guid, 1); + + return registry.machine_guid; +} diff --git a/src/registry_internals.h b/src/registry_internals.h new file mode 100644 index 000000000..9c0b74452 --- /dev/null +++ b/src/registry_internals.h @@ -0,0 +1,92 @@ +#include "common.h" + +#ifndef NETDATA_REGISTRY_INTERNALS_H_H +#define NETDATA_REGISTRY_INTERNALS_H_H + +#define REGISTRY_URL_FLAGS_DEFAULT 0x00 +#define REGISTRY_URL_FLAGS_EXPIRED 0x01 + +#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED + +// ---------------------------------------------------------------------------- +// COMMON structures + +struct registry { + int enabled; + + char machine_guid[GUID_LEN + 1]; + + // entries counters / statistics + unsigned long long persons_count; + unsigned long long machines_count; + unsigned long long usages_count; + unsigned long long urls_count; + unsigned long long persons_urls_count; + unsigned long long machines_urls_count; + unsigned long long log_count; + + // memory counters / statistics + unsigned long long persons_memory; + unsigned long long machines_memory; + unsigned long long urls_memory; + unsigned long long persons_urls_memory; + unsigned long long machines_urls_memory; + + // configuration + unsigned long long save_registry_every_entries; + char *registry_domain; + char *hostname; + char *registry_to_announce; + time_t persons_expiration; // seconds to expire idle persons + int verify_cookies_redirects; + + size_t max_url_length; + size_t max_name_length; + + // file/path names + char *pathname; + char *db_filename; + char *log_filename; + char *machine_guid_filename; + + // open files + FILE *log_fp; + + // the database + DICTIONARY *persons; // dictionary of REGISTRY_PERSON *, with key the REGISTRY_PERSON.guid + DICTIONARY *machines; // dictionary of REGISTRY_MACHINE *, with key the REGISTRY_MACHINE.guid + + avl_tree registry_urls_root_index; + + pthread_mutex_t lock; +}; + +extern int registry_regenerate_guid(const char *guid, char *result); + +#include "registry_url.h" +#include "registry_machine.h" +#include "registry_person.h" +#include "registry.h" + +extern struct registry registry; + +extern char *registry_get_this_machine_guid(void); + +// REGISTRY LOW-LEVEL REQUESTS (in registry-internals.c) +extern REGISTRY_PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when); +extern REGISTRY_PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when); +extern REGISTRY_MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when); + +// REGISTRY LOG (in registry_log.c) +extern void registry_log(const char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name); +extern int registry_log_open(void); +extern void registry_log_close(void); +extern void registry_log_recreate(void); +extern ssize_t registry_log_load(void); + +// REGISTRY DB (in registry_db.c) +extern int registry_db_save(void); +extern size_t registry_db_load(void); +extern int registry_db_should_be_saved(void); + +#endif //NETDATA_REGISTRY_INTERNALS_H_H diff --git a/src/registry_log.c b/src/registry_log.c new file mode 100644 index 000000000..3229a34b4 --- /dev/null +++ b/src/registry_log.c @@ -0,0 +1,133 @@ +#include "registry_internals.h" + +void registry_log(const char action, REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name) { + if(likely(registry.log_fp)) { + if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n", + action, + p->last_t, + p->guid, + m->guid, + name, + u->url) < 0)) + error("Registry: failed to save log. Registry data may be lost in case of abnormal restart."); + + // we increase the counter even on failures + // so that the registry will be saved periodically + registry.log_count++; + + // this must be outside the log_lock(), or a deadlock will happen. + // registry_db_save() checks the same inside the log_lock, so only + // one thread will save the db + if(unlikely(registry_db_should_be_saved())) + registry_db_save(); + } +} + +int registry_log_open(void) { + if(registry.log_fp) + fclose(registry.log_fp); + + registry.log_fp = fopen(registry.log_filename, "a"); + if(registry.log_fp) { + if (setvbuf(registry.log_fp, NULL, _IOLBF, 0) != 0) + error("Cannot set line buffering on registry log file."); + return 0; + } + + error("Cannot open registry log file '%s'. Registry data will be lost in case of netdata or server crash.", registry.log_filename); + return -1; +} + +void registry_log_close(void) { + if(registry.log_fp) { + fclose(registry.log_fp); + registry.log_fp = NULL; + } +} + +void registry_log_recreate(void) { + if(registry.log_fp != NULL) { + registry_log_close(); + + // open it with truncate + registry.log_fp = fopen(registry.log_filename, "w"); + if(registry.log_fp) fclose(registry.log_fp); + else error("Cannot truncate registry log '%s'", registry.log_filename); + + registry.log_fp = NULL; + registry_log_open(); + } +} + +ssize_t registry_log_load(void) { + ssize_t line = -1; + + // closing the log is required here + // otherwise we will append to it the values we read + registry_log_close(); + + debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename); + FILE *fp = fopen(registry.log_filename, "r"); + if(!fp) + error("Registry: cannot open registry file: %s", registry.log_filename); + else { + char *s, buf[4096 + 1]; + line = 0; + size_t len = 0; + + while ((s = fgets_trim_len(buf, 4096, fp, &len))) { + line++; + + switch (s[0]) { + case 'A': // accesses + case 'D': // deletes + + // verify it is valid + if (unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) { + error("Registry: log line %zd is wrong (len = %zu).", line, len); + continue; + } + s[1] = s[10] = s[47] = s[84] = '\0'; + + // get the variables + time_t when = strtoul(&s[2], NULL, 16); + char *person_guid = &s[11]; + char *machine_guid = &s[48]; + char *name = &s[85]; + + // skip the name to find the url + char *url = name; + while(*url && *url != '\t') url++; + if(!*url) { + error("Registry: log line %zd does not have a url.", line); + continue; + } + *url++ = '\0'; + + // make sure the person exists + // without this, a new person guid will be created + REGISTRY_PERSON *p = registry_person_find(person_guid); + if(!p) p = registry_person_allocate(person_guid, when); + + if(s[0] == 'A') + registry_request_access(p->guid, machine_guid, url, name, when); + else + registry_request_delete(p->guid, machine_guid, url, name, when); + + registry.log_count++; + break; + + default: + error("Registry: ignoring line %zd of filename '%s': %s.", line, registry.log_filename, s); + break; + } + } + + fclose(fp); + } + + // open the log again + registry_log_open(); + + return line; +} diff --git a/src/registry_machine.c b/src/registry_machine.c new file mode 100644 index 000000000..3510736df --- /dev/null +++ b/src/registry_machine.c @@ -0,0 +1,101 @@ +#include "registry_internals.h" + +// ---------------------------------------------------------------------------- +// MACHINE + +REGISTRY_MACHINE *registry_machine_find(const char *machine_guid) { + debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid); + return dictionary_get(registry.machines, machine_guid); +} + +REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) { + debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(REGISTRY_MACHINE_URL)); + + REGISTRY_MACHINE_URL *mu = mallocz(sizeof(REGISTRY_MACHINE_URL)); + + mu->first_t = mu->last_t = (uint32_t)when; + mu->usages = 1; + mu->url = u; + mu->flags = REGISTRY_URL_FLAGS_DEFAULT; + + registry.machines_urls_memory += sizeof(REGISTRY_MACHINE_URL); + + debug(D_REGISTRY, "registry_machine_url_allocate('%s', '%s'): indexing URL in machine", m->guid, u->url); + dictionary_set(m->machine_urls, u->url, mu, sizeof(REGISTRY_MACHINE_URL)); + + registry_url_link(u); + + return mu; +} + +REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) { + debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(REGISTRY_MACHINE)); + + REGISTRY_MACHINE *m = mallocz(sizeof(REGISTRY_MACHINE)); + + strncpyz(m->guid, machine_guid, GUID_LEN); + + debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid); + m->machine_urls = dictionary_create(DICTIONARY_FLAGS); + + m->first_t = m->last_t = (uint32_t)when; + m->usages = 0; + + registry.machines_memory += sizeof(REGISTRY_MACHINE); + + registry.machines_count++; + dictionary_set(registry.machines, m->guid, m, sizeof(REGISTRY_MACHINE)); + + return m; +} + +// 1. validate machine GUID +// 2. if it is valid, find it or create it and return it +// 3. if it is not valid, return NULL +REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when) { + REGISTRY_MACHINE *m = NULL; + + if(likely(machine_guid && *machine_guid)) { + // validate it is a GUID + char buf[GUID_LEN + 1]; + if(unlikely(registry_regenerate_guid(machine_guid, buf) == -1)) + info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid); + else { + machine_guid = buf; + m = registry_machine_find(machine_guid); + if(!m) m = registry_machine_allocate(machine_guid, when); + } + } + + return m; +} + + +// ---------------------------------------------------------------------------- +// LINKING OF OBJECTS + +REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when) { + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): searching for URL in machine", m->guid, u->url); + + REGISTRY_MACHINE_URL *mu = dictionary_get(m->machine_urls, u->url); + if(!mu) { + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): not found", m->guid, u->url); + mu = registry_machine_url_allocate(m, u, when); + registry.machines_urls_count++; + } + else { + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): found", m->guid, u->url); + mu->usages++; + if(likely(mu->last_t < (uint32_t)when)) mu->last_t = (uint32_t)when; + } + + m->usages++; + if(likely(m->last_t < (uint32_t)when)) m->last_t = (uint32_t)when; + + if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) { + debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): accessing an expired URL.", m->guid, u->url); + mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED; + } + + return mu; +} diff --git a/src/registry_machine.h b/src/registry_machine.h new file mode 100644 index 000000000..be824d168 --- /dev/null +++ b/src/registry_machine.h @@ -0,0 +1,41 @@ +#ifndef NETDATA_REGISTRY_MACHINE_H +#define NETDATA_REGISTRY_MACHINE_H + +#include "registry_internals.h" + +// ---------------------------------------------------------------------------- +// MACHINE structures + +// For each MACHINE-URL pair we keep this +struct registry_machine_url { + REGISTRY_URL *url; // de-duplicated URL + + uint8_t flags; + + uint32_t first_t; // the first time we saw this + uint32_t last_t; // the last time we saw this + uint32_t usages; // how many times this has been accessed +}; +typedef struct registry_machine_url REGISTRY_MACHINE_URL; + +// A machine +struct registry_machine { + char guid[GUID_LEN + 1]; // the GUID + + uint32_t links; // the number of REGISTRY_PERSON_URL linked to this machine + + DICTIONARY *machine_urls; // MACHINE_URL * + + uint32_t first_t; // the first time we saw this + uint32_t last_t; // the last time we saw this + uint32_t usages; // how many times this has been accessed +}; +typedef struct registry_machine REGISTRY_MACHINE; + +extern REGISTRY_MACHINE *registry_machine_find(const char *machine_guid); +extern REGISTRY_MACHINE_URL *registry_machine_url_allocate(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when); +extern REGISTRY_MACHINE *registry_machine_allocate(const char *machine_guid, time_t when); +extern REGISTRY_MACHINE *registry_machine_get(const char *machine_guid, time_t when); +extern REGISTRY_MACHINE_URL *registry_machine_link_to_url(REGISTRY_MACHINE *m, REGISTRY_URL *u, time_t when); + +#endif //NETDATA_REGISTRY_MACHINE_H diff --git a/src/registry_person.c b/src/registry_person.c new file mode 100644 index 000000000..5f9099c9a --- /dev/null +++ b/src/registry_person.c @@ -0,0 +1,264 @@ +#include "registry_internals.h" + +// ---------------------------------------------------------------------------- +// PERSON_URL INDEX + +int person_url_compare(void *a, void *b) { + register uint32_t hash1 = ((REGISTRY_PERSON_URL *)a)->url->hash; + register uint32_t hash2 = ((REGISTRY_PERSON_URL *)b)->url->hash; + + if(hash1 < hash2) return -1; + else if(hash1 > hash2) return 1; + else return strcmp(((REGISTRY_PERSON_URL *)a)->url->url, ((REGISTRY_PERSON_URL *)b)->url->url); +} + +inline REGISTRY_PERSON_URL *registry_person_url_index_find(REGISTRY_PERSON *p, const char *url) { + debug(D_REGISTRY, "Registry: registry_person_url_index_find('%s', '%s')", p->guid, url); + + char buf[sizeof(REGISTRY_URL) + strlen(url)]; + + REGISTRY_URL *u = (REGISTRY_URL *)&buf; + strcpy(u->url, url); + u->hash = simple_hash(u->url); + + REGISTRY_PERSON_URL tpu = { .url = u }; + + REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)avl_search(&p->person_urls, (void *)&tpu); + return pu; +} + +inline REGISTRY_PERSON_URL *registry_person_url_index_add(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) { + debug(D_REGISTRY, "Registry: registry_person_url_index_add('%s', '%s')", p->guid, pu->url->url); + REGISTRY_PERSON_URL *tpu = (REGISTRY_PERSON_URL *)avl_insert(&(p->person_urls), (avl *)(pu)); + if(tpu != pu) + error("Registry: registry_person_url_index_add('%s', '%s') already exists as '%s'", p->guid, pu->url->url, tpu->url->url); + + return tpu; +} + +inline REGISTRY_PERSON_URL *registry_person_url_index_del(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) { + debug(D_REGISTRY, "Registry: registry_person_url_index_del('%s', '%s')", p->guid, pu->url->url); + REGISTRY_PERSON_URL *tpu = (REGISTRY_PERSON_URL *)avl_remove(&(p->person_urls), (avl *)(pu)); + if(!tpu) + error("Registry: registry_person_url_index_del('%s', '%s') deleted nothing", p->guid, pu->url->url); + else if(tpu != pu) + error("Registry: registry_person_url_index_del('%s', '%s') deleted wrong URL '%s'", p->guid, pu->url->url, tpu->url->url); + + return tpu; +} + +// ---------------------------------------------------------------------------- +// PERSON_URL + +REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) { + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(REGISTRY_PERSON_URL) + namelen); + + // protection from too big names + if(namelen > registry.max_name_length) + namelen = registry.max_name_length; + + REGISTRY_PERSON_URL *pu = mallocz(sizeof(REGISTRY_PERSON_URL) + namelen); + + // a simple strcpy() should do the job + // but I prefer to be safe, since the caller specified urllen + strncpyz(pu->machine_name, name, namelen); + + pu->machine = m; + pu->first_t = pu->last_t = (uint32_t)when; + pu->usages = 1; + pu->url = u; + pu->flags = REGISTRY_URL_FLAGS_DEFAULT; + m->links++; + + registry.persons_urls_memory += sizeof(REGISTRY_PERSON_URL) + namelen; + + debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url); + REGISTRY_PERSON_URL *tpu = registry_person_url_index_add(p, pu); + if(tpu != pu) { + error("Registry: Attempted to add duplicate person url '%s' with name '%s' to person '%s'", u->url, name, p->guid); + free(pu); + pu = tpu; + } + else + registry_url_link(u); + + return pu; +} + +void registry_person_url_free(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) { + debug(D_REGISTRY, "registry_person_url_free('%s', '%s')", p->guid, pu->url->url); + + REGISTRY_PERSON_URL *tpu = registry_person_url_index_del(p, pu); + if(tpu) { + registry_url_unlink(tpu->url); + tpu->machine->links--; + registry.persons_urls_memory -= sizeof(REGISTRY_PERSON_URL) + strlen(tpu->machine_name); + freez(tpu); + } +} + +// this function is needed to change the name of a PERSON_URL +REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when, REGISTRY_PERSON_URL *pu) { + debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url, sizeof(REGISTRY_PERSON_URL) + namelen); + + // keep a backup + REGISTRY_PERSON_URL pu2 = { + .first_t = pu->first_t, + .last_t = pu->last_t, + .usages = pu->usages, + .flags = pu->flags, + .machine = pu->machine, + .machine_name = "" + }; + + // remove the existing one from the index + registry_person_url_free(p, pu); + pu = &pu2; + + // allocate a new one + REGISTRY_PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when); + tpu->first_t = pu->first_t; + tpu->last_t = pu->last_t; + tpu->usages = pu->usages; + tpu->flags = pu->flags; + + return tpu; +} + + +// ---------------------------------------------------------------------------- +// PERSON + +REGISTRY_PERSON *registry_person_find(const char *person_guid) { + debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid); + return dictionary_get(registry.persons, person_guid); +} + +REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when) { + debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(REGISTRY_PERSON)); + + REGISTRY_PERSON *p = mallocz(sizeof(REGISTRY_PERSON)); + if(!person_guid) { + for(;;) { + uuid_t uuid; + uuid_generate(uuid); + uuid_unparse_lower(uuid, p->guid); + + debug(D_REGISTRY, "Registry: Checking if the generated person guid '%s' is unique", p->guid); + if (!dictionary_get(registry.persons, p->guid)) { + debug(D_REGISTRY, "Registry: generated person guid '%s' is unique", p->guid); + break; + } + else + info("Registry: generated person guid '%s' found in the registry. Retrying...", p->guid); + } + } + else + strncpyz(p->guid, person_guid, GUID_LEN); + + debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid); + avl_init(&p->person_urls, person_url_compare); + + p->first_t = p->last_t = (uint32_t)when; + p->usages = 0; + + registry.persons_memory += sizeof(REGISTRY_PERSON); + + registry.persons_count++; + dictionary_set(registry.persons, p->guid, p, sizeof(REGISTRY_PERSON)); + + return p; +} + + +// 1. validate person GUID +// 2. if it is valid, find it +// 3. if it is not valid, create a new one +// 4. return it +REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when) { + debug(D_REGISTRY, "Registry: registry_person_get('%s'): creating dictionary of urls", person_guid); + + REGISTRY_PERSON *p = NULL; + + if(person_guid && *person_guid) { + char buf[GUID_LEN + 1]; + // validate it is a GUID + if(unlikely(registry_regenerate_guid(person_guid, buf) == -1)) + info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid); + else { + person_guid = buf; + p = registry_person_find(person_guid); + } + } + + if(!p) p = registry_person_allocate(NULL, when); + + return p; +} + +void registry_person_del(REGISTRY_PERSON *p) { + debug(D_REGISTRY, "Registry: registry_person_del('%s'): creating dictionary of urls", p->guid); + + while(p->person_urls.root) + registry_person_unlink_from_url(p, (REGISTRY_PERSON_URL *)p->person_urls.root); + + debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid); + dictionary_del(registry.persons, p->guid); + + debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid); + freez(p); +} + +// ---------------------------------------------------------------------------- +// LINKING OF OBJECTS + +REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when) { + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url); + + REGISTRY_PERSON_URL *pu = registry_person_url_index_find(p, u->url); + if(!pu) { + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url); + pu = registry_person_url_allocate(p, m, u, name, namelen, when); + registry.persons_urls_count++; + } + else { + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url); + pu->usages++; + if(likely(pu->last_t < (uint32_t)when)) pu->last_t = (uint32_t)when; + + if(pu->machine != m) { + REGISTRY_MACHINE_URL *mu = dictionary_get(pu->machine->machine_urls, u->url); + if(mu) { + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.", + p->guid, m->guid, u->url, pu->machine->guid); + mu->flags |= REGISTRY_URL_FLAGS_EXPIRED; + } + else { + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - but the URL is not linked to the old machine.", + p->guid, m->guid, u->url, pu->machine->guid); + } + + pu->machine->links--; + pu->machine = m; + } + + if(strcmp(pu->machine_name, name)) { + // the name of the PERSON_URL has changed ! + pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu); + } + } + + p->usages++; + if(likely(p->last_t < (uint32_t)when)) p->last_t = (uint32_t)when; + + if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) { + debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url); + pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED; + } + + return pu; +} + +void registry_person_unlink_from_url(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) { + registry_person_url_free(p, pu); +} diff --git a/src/registry_person.h b/src/registry_person.h new file mode 100644 index 000000000..5f6cc2443 --- /dev/null +++ b/src/registry_person.h @@ -0,0 +1,60 @@ +#ifndef NETDATA_REGISTRY_PERSON_H +#define NETDATA_REGISTRY_PERSON_H + +#include "registry_internals.h" + +// ---------------------------------------------------------------------------- +// PERSON structures + +// for each PERSON-URL pair we keep this +struct registry_person_url { + avl avl; // binary tree node + + REGISTRY_URL *url; // de-duplicated URL + REGISTRY_MACHINE *machine; // link the MACHINE of this URL + + uint8_t flags; + + uint32_t first_t; // the first time we saw this + uint32_t last_t; // the last time we saw this + uint32_t usages; // how many times this has been accessed + + char machine_name[1]; // the name of the machine, as known by the user + // dynamically allocated to fit properly +}; +typedef struct registry_person_url REGISTRY_PERSON_URL; + +// A person +struct registry_person { + char guid[GUID_LEN + 1]; // the person GUID + + avl_tree person_urls; // dictionary of PERSON_URLs + + uint32_t first_t; // the first time we saw this + uint32_t last_t; // the last time we saw this + uint32_t usages; // how many times this has been accessed + + //uint32_t flags; + //char *email; +}; +typedef struct registry_person REGISTRY_PERSON; + +// PERSON_URL +extern REGISTRY_PERSON_URL *registry_person_url_index_find(REGISTRY_PERSON *p, const char *url); +extern REGISTRY_PERSON_URL *registry_person_url_index_add(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) NEVERNULL WARNUNUSED; +extern REGISTRY_PERSON_URL *registry_person_url_index_del(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu) WARNUNUSED; + +extern REGISTRY_PERSON_URL *registry_person_url_allocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when); +extern REGISTRY_PERSON_URL *registry_person_url_reallocate(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when, REGISTRY_PERSON_URL *pu); + +// PERSON +extern REGISTRY_PERSON *registry_person_find(const char *person_guid); +extern REGISTRY_PERSON *registry_person_allocate(const char *person_guid, time_t when); +extern REGISTRY_PERSON *registry_person_get(const char *person_guid, time_t when); +extern void registry_person_del(REGISTRY_PERSON *p); + +// LINKING PERSON -> PERSON_URL +extern REGISTRY_PERSON_URL *registry_person_link_to_url(REGISTRY_PERSON *p, REGISTRY_MACHINE *m, REGISTRY_URL *u, char *name, size_t namelen, time_t when); +extern void registry_person_unlink_from_url(REGISTRY_PERSON *p, REGISTRY_PERSON_URL *pu); + +#endif //NETDATA_REGISTRY_PERSON_H diff --git a/src/registry_url.c b/src/registry_url.c new file mode 100644 index 000000000..52d36a898 --- /dev/null +++ b/src/registry_url.c @@ -0,0 +1,85 @@ +#include "registry_internals.h" + +// ---------------------------------------------------------------------------- +// REGISTRY_URL + +int registry_url_compare(void *a, void *b) { + if(((REGISTRY_URL *)a)->hash < ((REGISTRY_URL *)b)->hash) return -1; + else if(((REGISTRY_URL *)a)->hash > ((REGISTRY_URL *)b)->hash) return 1; + else return strcmp(((REGISTRY_URL *)a)->url, ((REGISTRY_URL *)b)->url); +} + +inline REGISTRY_URL *registry_url_index_add(REGISTRY_URL *u) { + return (REGISTRY_URL *)avl_insert(&(registry.registry_urls_root_index), (avl *)(u)); +} + +inline REGISTRY_URL *registry_url_index_del(REGISTRY_URL *u) { + return (REGISTRY_URL *)avl_remove(&(registry.registry_urls_root_index), (avl *)(u)); +} + +REGISTRY_URL *registry_url_get(const char *url, size_t urllen) { + // protection from too big URLs + if(urllen > registry.max_url_length) + urllen = registry.max_url_length; + + debug(D_REGISTRY, "Registry: registry_url_get('%s', %zu)", url, urllen); + + char buf[sizeof(REGISTRY_URL) + urllen]; // no need for +1, 1 is already in REGISTRY_URL + REGISTRY_URL *n = (REGISTRY_URL *)&buf[0]; + n->len = (uint16_t)urllen; + strncpyz(n->url, url, n->len); + n->hash = simple_hash(n->url); + + REGISTRY_URL *u = (REGISTRY_URL *)avl_search(&(registry.registry_urls_root_index), (avl *)n); + if(!u) { + debug(D_REGISTRY, "Registry: registry_url_get('%s', %zu): allocating %zu bytes", url, urllen, sizeof(REGISTRY_URL) + urllen); + u = callocz(1, sizeof(REGISTRY_URL) + urllen); // no need for +1, 1 is already in REGISTRY_URL + + // a simple strcpy() should do the job + // but I prefer to be safe, since the caller specified urllen + u->len = (uint16_t)urllen; + strncpyz(u->url, url, u->len); + u->links = 0; + u->hash = simple_hash(u->url); + + registry.urls_memory += sizeof(REGISTRY_URL) + urllen; // no need for +1, 1 is already in REGISTRY_URL + + debug(D_REGISTRY, "Registry: registry_url_get('%s'): indexing it", url); + n = registry_url_index_add(u); + if(n != u) { + error("INTERNAL ERROR: registry_url_get(): url '%s' already exists in the registry as '%s'", u->url, n->url); + free(u); + u = n; + } + else + registry.urls_count++; + } + + return u; +} + +void registry_url_link(REGISTRY_URL *u) { + u->links++; + debug(D_REGISTRY, "Registry: registry_url_link('%s'): URL has now %u links", u->url, u->links); +} + +void registry_url_unlink(REGISTRY_URL *u) { + u->links--; + if(!u->links) { + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): No more links for this URL", u->url); + REGISTRY_URL *n = registry_url_index_del(u); + if(!n) { + error("INTERNAL ERROR: registry_url_unlink('%s'): cannot find url in index", u->url); + } + else { + if(n != u) { + error("INTERNAL ERROR: registry_url_unlink('%s'): deleted different url '%s'", u->url, n->url); + } + + registry.urls_memory -= sizeof(REGISTRY_URL) + n->len; // no need for +1, 1 is already in REGISTRY_URL + freez(n); + } + } + else + debug(D_REGISTRY, "Registry: registry_url_unlink('%s'): URL has %u links left", u->url, u->links); +} diff --git a/src/registry_url.h b/src/registry_url.h new file mode 100644 index 000000000..5ac52f5a5 --- /dev/null +++ b/src/registry_url.h @@ -0,0 +1,33 @@ +#ifndef NETDATA_REGISTRY_URL_H +#define NETDATA_REGISTRY_URL_H + +#include "registry_internals.h" + +// ---------------------------------------------------------------------------- +// URL structures +// Save memory by de-duplicating URLs +// so instead of storing URLs all over the place +// we store them here and we keep pointers elsewhere + +struct registry_url { + avl avl; + uint32_t hash; // the index hash + + uint32_t links; // the number of links to this URL - when none is left, we free it + + uint16_t len; // the length of the URL in bytes + char url[1]; // the URL - dynamically allocated to more size +}; +typedef struct registry_url REGISTRY_URL; + +// REGISTRY_URL INDEX +extern int registry_url_compare(void *a, void *b); +extern REGISTRY_URL *registry_url_index_del(REGISTRY_URL *u) WARNUNUSED; +extern REGISTRY_URL *registry_url_index_add(REGISTRY_URL *u) NEVERNULL WARNUNUSED; + +// REGISTRY_URL MANAGEMENT +extern REGISTRY_URL *registry_url_get(const char *url, size_t urllen) NEVERNULL; +extern void registry_url_link(REGISTRY_URL *u); +extern void registry_url_unlink(REGISTRY_URL *u); + +#endif //NETDATA_REGISTRY_URL_H @@ -56,7 +56,7 @@ RRDHOST localhost = { void rrdhost_init(char *hostname) { localhost.hostname = hostname; localhost.health_log.next_log_id = - localhost.health_log.next_alarm_id = time(NULL); + localhost.health_log.next_alarm_id = now_realtime_sec(); } void rrdhost_rwlock(RRDHOST *host) { @@ -118,7 +118,7 @@ RRDFAMILY *rrdfamily_create(const char *id) { RRDFAMILY *ret = rrdfamily_index_add(&localhost, rc); if(ret != rc) - fatal("INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", rc->family, (ret)?ret->family:"NONE"); + fatal("RRDFAMILY: INTERNAL ERROR: Expected to INSERT RRDFAMILY '%s' into index, but inserted '%s'.", rc->family, (ret)?ret->family:"NONE"); } rc->use_count++; @@ -130,10 +130,10 @@ void rrdfamily_free(RRDFAMILY *rc) { if(!rc->use_count) { RRDFAMILY *ret = rrdfamily_index_del(&localhost, rc); if(ret != rc) - fatal("INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", rc->family, (ret)?ret->family:"NONE"); + fatal("RRDFAMILY: INTERNAL ERROR: Expected to DELETE RRDFAMILY '%s' from index, but deleted '%s'.", rc->family, (ret)?ret->family:"NONE"); if(rc->variables_root_index.avl_tree.root != NULL) - fatal("INTERNAL ERROR: Variables index of RRDFAMILY '%s' that is freed, is not empty.", rc->family); + fatal("RRDFAMILY: INTERNAL ERROR: Variables index of RRDFAMILY '%s' that is freed, is not empty.", rc->family); freez((void *)rc->family); freez(rc); @@ -222,8 +222,8 @@ static int rrddim_compare(void* a, void* b) { else return strcmp(((RRDDIM *)a)->id, ((RRDDIM *)b)->id); } -#define rrddim_index_add(st, rd) avl_insert_lock(&((st)->dimensions_index), (avl *)(rd)) -#define rrddim_index_del(st,rd ) avl_remove_lock(&((st)->dimensions_index), (avl *)(rd)) +#define rrddim_index_add(st, rd) (RRDDIM *)avl_insert_lock(&((st)->dimensions_index), (avl *)(rd)) +#define rrddim_index_del(st,rd ) (RRDDIM *)avl_remove_lock(&((st)->dimensions_index), (avl *)(rd)) static RRDDIM *rrddim_index_find(RRDSET *st, const char *id, uint32_t hash) { RRDDIM tmp; @@ -353,22 +353,36 @@ char *rrdset_strncpyz_name(char *to, const char *from, size_t length) void rrdset_set_name(RRDSET *st, const char *name) { - debug(D_RRD_CALLS, "rrdset_set_name() old: %s, new: %s", st->name, name); + if(unlikely(st->name && !strcmp(st->name, name))) + return; - if(st->name) { - rrdset_index_del_name(&localhost, st); - rrdsetvar_rename_all(st); - } + debug(D_RRD_CALLS, "rrdset_set_name() old: %s, new: %s", st->name, name); char b[CONFIG_MAX_VALUE + 1]; char n[RRD_ID_LENGTH_MAX + 1]; snprintfz(n, RRD_ID_LENGTH_MAX, "%s.%s", st->type, name); rrdset_strncpyz_name(b, n, CONFIG_MAX_VALUE); - st->name = config_get(st->id, "name", b); - st->hash_name = simple_hash(st->name); - rrdset_index_add_name(&localhost, st); + if(st->name) { + rrdset_index_del_name(&localhost, st); + st->name = config_set_default(st->id, "name", b); + st->hash_name = simple_hash(st->name); + rrdsetvar_rename_all(st); + } + else { + st->name = config_get(st->id, "name", b); + st->hash_name = simple_hash(st->name); + } + + pthread_rwlock_wrlock(&st->rwlock); + RRDDIM *rd; + for(rd = st->dimensions; rd ;rd = rd->next) + rrddimvar_rename_all(rd); + pthread_rwlock_unlock(&st->rwlock); + + if(unlikely(rrdset_index_add_name(&localhost, st) != st)) + error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name); } // ---------------------------------------------------------------------------- @@ -425,7 +439,7 @@ void rrdset_reset(RRDSET *st) memset(rd->values, 0, rd->entries * sizeof(storage_number)); } } -static long align_entries_to_pagesize(long entries) { +static inline long align_entries_to_pagesize(long entries) { if(entries < 5) entries = 5; if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX; @@ -447,6 +461,11 @@ static long align_entries_to_pagesize(long entries) { #endif } +static inline void timeval_align(struct timeval *tv, int update_every) { + tv->tv_sec -= tv->tv_sec % update_every; + tv->tv_usec = 500000; +} + RRDSET *rrdset_create(const char *type, const char *id, const char *name, const char *family, const char *context, const char *title, const char *units, long priority, int update_every, int chart_type) { if(!type || !type[0]) { @@ -461,11 +480,10 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const char fullid[RRD_ID_LENGTH_MAX + 1]; char fullfilename[FILENAME_MAX + 1]; - RRDSET *st = NULL; snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id); - st = rrdset_find(fullid); + RRDSET *st = rrdset_find(fullid); if(st) { error("Cannot create rrd stats for '%s', it already exists.", fullid); return st; @@ -508,11 +526,15 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const error("File %s does not have the desired update frequency. Clearing it.", fullfilename); memset(st, 0, size); } - else if((time(NULL) - st->last_updated.tv_sec) > update_every * entries) { + else if((now_realtime_sec() - st->last_updated.tv_sec) > update_every * entries) { errno = 0; error("File %s is too old. Clearing it.", fullfilename); memset(st, 0, size); } + + // make sure the database is aligned + if(st->last_updated.tv_sec) + timeval_align(&st->last_updated, update_every); } if(st) { @@ -527,6 +549,11 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const st->mapped = rrd_memory_mode; st->variables = NULL; st->alarms = NULL; + memset(&st->rwlock, 0, sizeof(pthread_rwlock_t)); + memset(&st->avl, 0, sizeof(avl)); + memset(&st->avlname, 0, sizeof(avl)); + memset(&st->variables_root_index, 0, sizeof(avl_tree_lock)); + memset(&st->dimensions_index, 0, sizeof(avl_tree_lock)); } else { st = callocz(1, size); @@ -588,8 +615,10 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const { char varvalue[CONFIG_MAX_VALUE + 1]; + char varvalue2[CONFIG_MAX_VALUE + 1]; snprintfz(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name); - st->title = config_get(st->id, "title", varvalue); + json_escape_string(varvalue2, varvalue, sizeof(varvalue2)); + st->title = config_get(st->id, "title", varvalue2); } st->rrdfamily = rrdfamily_create(st->family); @@ -606,7 +635,8 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const rrdsetvar_create(st, "update_every", RRDVAR_TYPE_INT, &st->update_every, 0); } - rrdset_index_add(&localhost, st); + if(unlikely(rrdset_index_add(&localhost, st) != st)) + error("RRDSET: INTERNAL ERROR: attempt to index duplicate chart '%s'", st->id); rrdsetcalc_link_matching(st); rrdcalctemplate_link_matching(st); @@ -618,21 +648,29 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier, long divisor, int algorithm) { + RRDDIM *rd = rrddim_find(st, id); + if(rd) { + debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>"); + return rd; + } + char filename[FILENAME_MAX + 1]; char fullfilename[FILENAME_MAX + 1]; char varname[CONFIG_MAX_NAME + 1]; - RRDDIM *rd = NULL; unsigned long size = sizeof(RRDDIM) + (st->entries * sizeof(storage_number)); debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id); rrdset_strncpyz_name(filename, id, FILENAME_MAX); snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename); - if(rrd_memory_mode != RRD_MEMORY_MODE_RAM) rd = (RRDDIM *)mymmap(fullfilename, size, ((rrd_memory_mode == RRD_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE), 1); + + if(rrd_memory_mode != RRD_MEMORY_MODE_RAM) + rd = (RRDDIM *)mymmap(fullfilename, size, ((rrd_memory_mode == RRD_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE), 1); + if(rd) { struct timeval now; - gettimeofday(&now, NULL); + now_realtime_timeval(&now); if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) { errno = 0; @@ -664,7 +702,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier error("File %s does not have the same refresh frequency. Clearing it.", fullfilename); memset(rd, 0, size); } - else if(usec_dt(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) { + else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) { errno = 0; error("File %s is too old. Clearing it.", fullfilename); memset(rd, 0, size); @@ -685,6 +723,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier rd->variables = NULL; rd->next = NULL; rd->name = NULL; + memset(&rd->avl, 0, sizeof(avl)); } else { // if we didn't manage to get a mmap'd dimension, just create one @@ -748,18 +787,22 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier pthread_rwlock_unlock(&st->rwlock); - rrddim_index_add(st, rd); + if(unlikely(rrddim_index_add(st, rd) != rd)) + error("RRDDIM: INTERNAL ERROR: attempt to index duplicate dimension '%s' on chart '%s'", rd->id, st->id); return(rd); } void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) { - debug(D_RRD_CALLS, "rrddim_set_name() %s.%s", st->name, rd->name); + if(unlikely(rd->name && !strcmp(rd->name, name))) + return; + + debug(D_RRD_CALLS, "rrddim_set_name() from %s.%s to %s.%s", st->name, rd->name, st->name, name); char varname[CONFIG_MAX_NAME + 1]; snprintfz(varname, CONFIG_MAX_NAME, "dim %s name", rd->id); - config_set_default(st->id, varname, name); + rd->name = config_set_default(st->id, varname, name); rrddimvar_rename_all(rd); } @@ -784,7 +827,8 @@ void rrddim_free(RRDSET *st, RRDDIM *rd) while(rd->variables) rrddimvar_free(rd->variables); - rrddim_index_del(st, rd); + if(unlikely(rrddim_index_del(st, rd) != rd)) + error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id); // free(rd->annotations); if(rd->mapped == RRD_MEMORY_MODE_SAVE) { @@ -825,7 +869,10 @@ void rrdset_free_all(void) while(st->dimensions) rrddim_free(st, st->dimensions); - rrdset_index_del(&localhost, st); + if(unlikely(rrdset_index_del(&localhost, st) != st)) + error("RRDSET: INTERNAL ERROR: attempt to remove from index chart '%s', removed a different chart.", st->id); + + rrdset_index_del_name(&localhost, st); st->rrdfamily->use_count--; if(!st->rrdfamily->use_count) @@ -833,14 +880,7 @@ void rrdset_free_all(void) pthread_rwlock_unlock(&st->rwlock); - if(st->mapped == RRD_MEMORY_MODE_SAVE) { - debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename); - savememory(st->cache_filename, st, st->memsize); - - debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name); - munmap(st, st->memsize); - } - else if(st->mapped == RRD_MEMORY_MODE_MAP) { + if(st->mapped == RRD_MEMORY_MODE_SAVE || st->mapped == RRD_MEMORY_MODE_MAP) { debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name); munmap(st, st->memsize); } @@ -862,9 +902,12 @@ void rrdset_save_all(void) { RRDSET *st; RRDDIM *rd; + // we get an write lock + // to ensure only one thread is saving the database rrdhost_rwlock(&localhost); + for(st = localhost.rrdset_root; st ; st = st->next) { - pthread_rwlock_wrlock(&st->rwlock); + pthread_rwlock_rdlock(&st->rwlock); if(st->mapped == RRD_MEMORY_MODE_SAVE) { debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename); @@ -880,6 +923,7 @@ void rrdset_save_all(void) { pthread_rwlock_unlock(&st->rwlock); } + rrdhost_unlock(&localhost); } @@ -953,7 +997,7 @@ collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number { debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value); - gettimeofday(&rd->last_collected_time, NULL); + now_realtime_timeval(&rd->last_collected_time); rd->collected_value = value; rd->updated = 1; rd->counter++; @@ -974,38 +1018,61 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value) return rrddim_set_by_pointer(st, rd, value); } -void rrdset_next_usec(RRDSET *st, unsigned long long microseconds) +void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds) { - if(!microseconds) rrdset_next(st); - else { - debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds); - - if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds); - st->usec_since_last_update = microseconds; + if(unlikely(!st->last_collected_time.tv_sec || !microseconds)) { + // the first entry + microseconds = st->update_every * USEC_PER_SEC; } + st->usec_since_last_update = microseconds; } -void rrdset_next(RRDSET *st) +void rrdset_next_usec(RRDSET *st, usec_t microseconds) { - unsigned long long microseconds = 0; + struct timeval now; + now_realtime_timeval(&now); - if(likely(st->last_collected_time.tv_sec)) { - struct timeval now; - gettimeofday(&now, NULL); - microseconds = usec_dt(&now, &st->last_collected_time); + if(unlikely(!st->last_collected_time.tv_sec)) { + // the first entry + microseconds = st->update_every * USEC_PER_SEC; + } + else if(unlikely(!microseconds)) { + // no dt given by the plugin + microseconds = dt_usec(&now, &st->last_collected_time); } - // prevent infinite loop - else microseconds = st->update_every * 1000000ULL; + else { + // microseconds has the time since the last collection + usec_t now_usec = timeval_usec(&now); + usec_t last_usec = timeval_usec(&st->last_collected_time); + usec_t since_last_usec = dt_usec(&now, &st->last_collected_time); - rrdset_next_usec(st, microseconds); -} + // verify the microseconds given is good + if(unlikely(microseconds > since_last_usec)) { + debug(D_RRD_CALLS, "dt %llu usec given is too big - it leads %llu usec to the future, for chart '%s' (%s).", microseconds, microseconds - since_last_usec, st->name, st->id); -void rrdset_next_plugins(RRDSET *st) -{ - rrdset_next(st); +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(last_usec + microseconds > now_usec + 1000)) + error("dt %llu usec given is too big - it leads %llu usec to the future, for chart '%s' (%s).", microseconds, microseconds - since_last_usec, st->name, st->id); +#endif + + microseconds = since_last_usec; + } + else if(unlikely(microseconds < since_last_usec * 0.8)) { + debug(D_RRD_CALLS, "dt %llu usec given is too small - expected %llu usec up to -20%%, for chart '%s' (%s).", microseconds, since_last_usec, st->name, st->id); + +#ifdef NETDATA_INTERNAL_CHECKS + error("dt %llu usec given is too small - expected %llu usec up to -20%%, for chart '%s' (%s).", microseconds, since_last_usec, st->name, st->id); +#endif + microseconds = since_last_usec; + } + } + debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds); + + if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds); + st->usec_since_last_update = microseconds; } -unsigned long long rrdset_done(RRDSET *st) +usec_t rrdset_done(RRDSET *st) { if(unlikely(netdata_exit)) return 0; @@ -1023,12 +1090,12 @@ unsigned long long rrdset_done(RRDSET *st) unsigned int stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done() - unsigned long long + usec_t last_collect_ut, // the timestamp in microseconds, of the last collected value now_collect_ut, // the timestamp in microseconds, of this collected value (this is NOW) last_stored_ut, // the timestamp in microseconds, of the last stored entry in the db next_store_ut, // the timestamp in microseconds, of the next entry to store in the db - update_every_ut = st->update_every * 1000000ULL; // st->update_every in microseconds + update_every_ut = st->update_every * USEC_PER_SEC; // st->update_every in microseconds if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &pthreadoldcancelstate) != 0)) error("Cannot set pthread cancel state to DISABLE."); @@ -1044,7 +1111,7 @@ unsigned long long rrdset_done(RRDSET *st) // check if the chart has a long time to be updated if(unlikely(st->usec_since_last_update > st->entries * update_every_ut)) { - info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); + info("%s: took too long to be updated (%0.3Lf secs). Resetting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); rrdset_reset(st); st->usec_since_last_update = update_every_ut; first_entry = 1; @@ -1055,8 +1122,10 @@ unsigned long long rrdset_done(RRDSET *st) if(unlikely(!st->last_collected_time.tv_sec)) { // it is the first entry // set the last_collected_time to now - gettimeofday(&st->last_collected_time, NULL); - last_collect_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - update_every_ut; + now_realtime_timeval(&st->last_collected_time); + timeval_align(&st->last_collected_time, st->update_every); + + last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec - update_every_ut; // the first entry should not be stored store_this_entry = 0; @@ -1067,10 +1136,10 @@ unsigned long long rrdset_done(RRDSET *st) else { // it is not the first entry // calculate the proper last_collected_time, using usec_since_last_update - last_collect_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; - unsigned long long ut = last_collect_ut + st->usec_since_last_update; - st->last_collected_time.tv_sec = (time_t) (ut / 1000000ULL); - st->last_collected_time.tv_usec = (suseconds_t) (ut % 1000000ULL); + last_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec; + usec_t ut = last_collect_ut + st->usec_since_last_update; + st->last_collected_time.tv_sec = (time_t) (ut / USEC_PER_SEC); + st->last_collected_time.tv_usec = (suseconds_t) (ut % USEC_PER_SEC); } // if this set has not been updated in the past @@ -1078,9 +1147,9 @@ unsigned long long rrdset_done(RRDSET *st) if(unlikely(!st->last_updated.tv_sec)) { // it has never been updated before // set a fake last_updated, in the past using usec_since_last_update - unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; - st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); - st->last_updated.tv_usec = (suseconds_t) (ut % 1000000ULL); + usec_t ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec - st->usec_since_last_update; + st->last_updated.tv_sec = (time_t) (ut / USEC_PER_SEC); + st->last_updated.tv_usec = (suseconds_t) (ut % USEC_PER_SEC); // the first entry should not be stored store_this_entry = 0; @@ -1090,17 +1159,18 @@ unsigned long long rrdset_done(RRDSET *st) } // check if we will re-write the entire data set - if(unlikely(usec_dt(&st->last_collected_time, &st->last_updated) > st->entries * update_every_ut)) { - info("%s: too old data (last updated at %ld.%ld, last collected at %ld.%ld). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec); + if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->entries * update_every_ut)) { + info("%s: too old data (last updated at %ld.%ld, last collected at %ld.%ld). Resetting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec); rrdset_reset(st); st->usec_since_last_update = update_every_ut; - gettimeofday(&st->last_collected_time, NULL); + now_realtime_timeval(&st->last_collected_time); + timeval_align(&st->last_collected_time, st->update_every); - unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; - st->last_updated.tv_sec = (time_t) (ut / 1000000ULL); - st->last_updated.tv_usec = (suseconds_t) (ut % 1000000ULL); + usec_t ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec - st->usec_since_last_update; + st->last_updated.tv_sec = (time_t) (ut / USEC_PER_SEC); + st->last_updated.tv_usec = (suseconds_t) (ut % USEC_PER_SEC); // the first entry should not be stored store_this_entry = 0; @@ -1111,9 +1181,9 @@ unsigned long long rrdset_done(RRDSET *st) // last_stored_ut = the last time we added a value to the storage // now_collect_ut = the time the current value has been collected // next_store_ut = the time of the next interpolation point - last_stored_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec; - now_collect_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; - next_store_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL; + last_stored_ut = st->last_updated.tv_sec * USEC_PER_SEC + st->last_updated.tv_usec; + now_collect_ut = st->last_collected_time.tv_sec * USEC_PER_SEC + st->last_collected_time.tv_usec; + next_store_ut = (st->last_updated.tv_sec + st->update_every) * USEC_PER_SEC; if(unlikely(st->debug)) { debug(D_RRD_STATS, "%s: last_collect_ut = %0.3Lf (last collection time)", st->name, (long double)last_collect_ut/1000000.0); @@ -1311,9 +1381,12 @@ unsigned long long rrdset_done(RRDSET *st) if(unlikely(now_collect_ut < next_store_ut)) { // this is collected in the same interpolation point if(unlikely(st->debug)) debug(D_RRD_STATS, "%s: THIS IS IN THE SAME INTERPOLATION POINT", st->name); +#ifdef NETDATA_INTERNAL_CHECKS + info("%s is collected in the same interpolation point: short by %llu microseconds", st->name, next_store_ut - now_collect_ut); +#endif } - unsigned long long first_ut = last_stored_ut; + usec_t first_ut = last_stored_ut; long long iterations = (now_collect_ut - last_stored_ut) / (update_every_ut); if((now_collect_ut % (update_every_ut)) == 0) iterations++; @@ -1327,7 +1400,7 @@ unsigned long long rrdset_done(RRDSET *st) debug(D_RRD_STATS, "%s: next_store_ut = %0.3Lf (next interpolation point)", st->name, (long double)next_store_ut/1000000.0); } - st->last_updated.tv_sec = (time_t) (next_store_ut / 1000000ULL); + st->last_updated.tv_sec = (time_t) (next_store_ut / USEC_PER_SEC); st->last_updated.tv_usec = 0; for( rd = st->dimensions ; likely(rd) ; rd = rd->next ) { @@ -244,7 +244,7 @@ struct rrdset { uint32_t hash_name; // a simple hash on the name - unsigned long long usec_since_last_update; // the time in microseconds since the last collection of data + usec_t usec_since_last_update; // the time in microseconds since the last collection of data struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) struct timeval last_collected_time; // when did this data set last collected values @@ -355,11 +355,11 @@ extern RRDSET *rrdset_find(const char *id); extern RRDSET *rrdset_find_bytype(const char *type, const char *id); extern RRDSET *rrdset_find_byname(const char *name); -extern void rrdset_next_usec(RRDSET *st, unsigned long long microseconds); -extern void rrdset_next(RRDSET *st); -extern void rrdset_next_plugins(RRDSET *st); +extern void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds); +extern void rrdset_next_usec(RRDSET *st, usec_t microseconds); +#define rrdset_next(st) rrdset_next_usec(st, 0ULL) -extern unsigned long long rrdset_done(RRDSET *st); +extern usec_t rrdset_done(RRDSET *st); // get the total duration in seconds of the round robin database #define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every )) diff --git a/src/rrd2json.c b/src/rrd2json.c index 474b5915d..067475006 100644 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -1,6 +1,6 @@ #include "common.h" -void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) +void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used) { pthread_rwlock_rdlock(&st->rwlock); @@ -41,7 +41,7 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) unsigned long memory = st->memsize; - int c = 0; + size_t dimensions = 0; RRDDIM *rd; for(rd = st->dimensions; rd ; rd = rd->next) { if(rd->flags & RRDDIM_FLAG_HIDDEN) continue; @@ -51,14 +51,17 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) buffer_sprintf(wb, "%s" "\t\t\t\t\"%s\": { \"name\": \"%s\" }" - , c?",\n":"" + , dimensions?",\n":"" , rd->id , rd->name ); - c++; + dimensions++; } + if(dimensions_count) *dimensions_count += dimensions; + if(memory_used) *memory_used += memory; + buffer_strcat(wb, "\n\t\t\t},\n\t\t\t\"green\": "); buffer_rrd_value(wb, st->green); buffer_strcat(wb, ",\n\t\t\t\"red\": "); @@ -71,9 +74,13 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) pthread_rwlock_unlock(&st->rwlock); } +void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) { + rrd_stats_api_v1_chart_with_data(st, wb, NULL, NULL); +} + void rrd_stats_api_v1_charts(BUFFER *wb) { - long c; + size_t c, dimensions = 0, memory = 0, alarms = 0; RRDSET *st; buffer_sprintf(wb, "{\n" @@ -93,19 +100,202 @@ void rrd_stats_api_v1_charts(BUFFER *wb) buffer_strcat(wb, "\n\t\t\""); buffer_strcat(wb, st->id); buffer_strcat(wb, "\": "); - rrd_stats_api_v1_chart(st, wb); + rrd_stats_api_v1_chart_with_data(st, wb, &dimensions, &memory); c++; } } + + RRDCALC *rc; + for(rc = localhost.alarms; rc ; rc = rc->next) { + if(rc->rrdset) + alarms++; + } pthread_rwlock_unlock(&localhost.rrdset_root_rwlock); - buffer_strcat(wb, "\n\t}\n}\n"); + buffer_sprintf(wb, "\n\t}" + ",\n\t\"charts_count\": %zu" + ",\n\t\"dimensions_count\": %zu" + ",\n\t\"alarms_count\": %zu" + ",\n\t\"rrd_memory_bytes\": %zu" + "\n}\n" + , c + , dimensions + , alarms + , memory + ); } +// ---------------------------------------------------------------------------- +// PROMETHEUS +// /api/v1/allmetrics?format=prometheus + +static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) { + size_t n; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(unlikely(!isalnum(c))) *d = '_'; + else *d = c; + } + *d = '\0'; + + return n; +} + +#define PROMETHEUS_ELEMENT_MAX 256 + +void rrd_stats_api_v1_charts_allmetrics_prometheus(BUFFER *wb) +{ + pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock); + + char host[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_name_copy(host, config_get("global", "hostname", "localhost"), PROMETHEUS_ELEMENT_MAX); + + // for each chart + RRDSET *st; + for(st = localhost.rrdset_root; st ; st = st->next) { + char chart[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_name_copy(chart, st->id, PROMETHEUS_ELEMENT_MAX); + + buffer_strcat(wb, "\n"); + if(st->enabled && st->dimensions) { + pthread_rwlock_rdlock(&st->rwlock); + + // for each dimension + RRDDIM *rd; + for(rd = st->dimensions; rd ; rd = rd->next) { + if(rd->counter) { + char dimension[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_name_copy(dimension, rd->id, PROMETHEUS_ELEMENT_MAX); + + // buffer_sprintf(wb, "# HELP %s.%s %s\n", st->id, rd->id, st->units); + + switch(rd->algorithm) { + case RRDDIM_INCREMENTAL: + case RRDDIM_PCENT_OVER_DIFF_TOTAL: + buffer_sprintf(wb, "# TYPE %s_%s counter\n", chart, dimension); + break; + + default: + buffer_sprintf(wb, "# TYPE %s_%s gauge\n", chart, dimension); + break; + } + + // calculated_number n = (calculated_number)rd->last_collected_value * (calculated_number)(abs(rd->multiplier)) / (calculated_number)(abs(rd->divisor)); + // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n, + // (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000))); + + buffer_sprintf(wb, "%s_%s{instance=\"%s\"} " COLLECTED_NUMBER_FORMAT " %llu\n", + chart, dimension, host, rd->last_collected_value, + (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000))); + + } + } + + pthread_rwlock_unlock(&st->rwlock); + } + } + + pthread_rwlock_unlock(&localhost.rrdset_root_rwlock); +} + +// ---------------------------------------------------------------------------- +// BASH +// /api/v1/allmetrics?format=bash + +static inline size_t shell_name_copy(char *d, const char *s, size_t usable) { + size_t n; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(unlikely(!isalnum(c))) *d = '_'; + else *d = (char)toupper(c); + } + *d = '\0'; + + return n; +} + +#define SHELL_ELEMENT_MAX 100 + +void rrd_stats_api_v1_charts_allmetrics_shell(BUFFER *wb) +{ + pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock); + + char host[SHELL_ELEMENT_MAX + 1]; + shell_name_copy(host, config_get("global", "hostname", "localhost"), SHELL_ELEMENT_MAX); + + // for each chart + RRDSET *st; + for(st = localhost.rrdset_root; st ; st = st->next) { + calculated_number total = 0.0; + char chart[SHELL_ELEMENT_MAX + 1]; + shell_name_copy(chart, st->id, SHELL_ELEMENT_MAX); + + buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name); + if(st->enabled && st->dimensions) { + pthread_rwlock_rdlock(&st->rwlock); + + // for each dimension + RRDDIM *rd; + for(rd = st->dimensions; rd ; rd = rd->next) { + if(rd->counter) { + char dimension[SHELL_ELEMENT_MAX + 1]; + shell_name_copy(dimension, rd->id, SHELL_ELEMENT_MAX); + + calculated_number n = rd->last_stored_value; + + if(isnan(n) || isinf(n)) + buffer_sprintf(wb, "NETDATA_%s_%s=\"\" # %s\n", chart, dimension, st->units); + else { + if(rd->multiplier < 0 || rd->divisor < 0) n = -n; + n = roundl(n); + if(!(rd->flags & RRDDIM_FLAG_HIDDEN)) total += n; + buffer_sprintf(wb, "NETDATA_%s_%s=\"%0.0Lf\" # %s\n", chart, dimension, n, st->units); + } + } + } + + total = roundl(total); + buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"%0.0Lf\" # %s\n", chart, total, st->units); + pthread_rwlock_unlock(&st->rwlock); + } + } + + buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n"); + + RRDCALC *rc; + for(rc = localhost.alarms; rc ;rc = rc->next) { + if(!rc->rrdset) continue; + + char chart[SHELL_ELEMENT_MAX + 1]; + shell_name_copy(chart, rc->rrdset->id, SHELL_ELEMENT_MAX); + + char alarm[SHELL_ELEMENT_MAX + 1]; + shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX); + + calculated_number n = rc->value; + + if(isnan(n) || isinf(n)) + buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\" # %s\n", chart, alarm, rc->units); + else { + n = roundl(n); + buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"%0.0Lf\" # %s\n", chart, alarm, n, rc->units); + } + + buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status)); + } + + pthread_rwlock_unlock(&localhost.rrdset_root_rwlock); +} + +// ---------------------------------------------------------------------------- unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb) { - time_t now = time(NULL); + time_t now = now_realtime_sec(); pthread_rwlock_rdlock(&st->rwlock); @@ -434,17 +624,21 @@ void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims) { + (void)dims; + if(options & RRDR_OPTION_NONZERO) { long i; - if(dims && *dims) { + // commented due to #1514 + + //if(dims && *dims) { // the caller wants specific dimensions // disable NONZERO option // to make sure we don't accidentally prevent // the specific dimensions from being returned - i = 0; - } - else { + // i = 0; + //} + //else { // find how many dimensions are not zero long c; RRDDIM *rd; @@ -453,7 +647,7 @@ uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims) if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue; i++; } - } + //} // if with nonzero we get i = 0 (no dimensions will be returned) // disable nonzero to show all dimensions @@ -1158,7 +1352,7 @@ inline static void rrdr_free(RRDR *r) freez(r); } -inline void rrdr_done(RRDR *r) +static inline void rrdr_done(RRDR *r) { r->rows = r->c + 1; r->c = 0; @@ -1209,19 +1403,32 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g time_t last_entry_t = rrdset_last_entry_t(st); if(before == 0 && after == 0) { + // dump the all the data before = last_entry_t; after = first_entry_t; absolute_period_requested = 0; } - // allow relative for before and after (smaller than 3 years) - if(((before < 0)?-before:before) <= (3 * 365 * 86400)) { - before = last_entry_t + before; + // allow relative for before (smaller than API_RELATIVE_TIME_MAX) + if(((before < 0)?-before:before) <= API_RELATIVE_TIME_MAX) { + if(abs(before) % st->update_every) { + // make sure it is multiple of st->update_every + if(before < 0) before = before - st->update_every - before % st->update_every; + else before = before + st->update_every - before % st->update_every; + } + if(before > 0) before = first_entry_t + before; + else before = last_entry_t + before; absolute_period_requested = 0; } - if(((after < 0)?-after:after) <= (3 * 365 * 86400)) { + // allow relative for after (smaller than API_RELATIVE_TIME_MAX) + if(((after < 0)?-after:after) <= API_RELATIVE_TIME_MAX) { if(after == 0) after = -st->update_every; + if(abs(after) % st->update_every) { + // make sure it is multiple of st->update_every + if(after < 0) after = after - st->update_every - after % st->update_every; + else after = after + st->update_every - after % st->update_every; + } after = before + after; absolute_period_requested = 0; } diff --git a/src/rrd2json.h b/src/rrd2json.h index e3e5cb690..7b1401970 100644 --- a/src/rrd2json.h +++ b/src/rrd2json.h @@ -2,7 +2,8 @@ #define NETDATA_RRD2JSON_H 1 #define HOSTNAME_MAX 1024 -extern char *hostname; + +#define API_RELATIVE_TIME_MAX (3 * 365 * 86400) // type of JSON generations #define DATASOURCE_INVALID -1 @@ -30,6 +31,12 @@ extern char *hostname; #define DATASOURCE_FORMAT_SSV_COMMA "ssvcomma" #define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray" +#define ALLMETRICS_FORMAT_SHELL "shell" +#define ALLMETRICS_FORMAT_PROMETHEUS "prometheus" + +#define ALLMETRICS_SHELL 1 +#define ALLMETRICS_PROMETHEUS 2 + #define GROUP_UNDEFINED 0 #define GROUP_AVERAGE 1 #define GROUP_MIN 2 @@ -54,6 +61,9 @@ extern char *hostname; extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); extern void rrd_stats_api_v1_charts(BUFFER *wb); +extern void rrd_stats_api_v1_charts_allmetrics_shell(BUFFER *wb); +extern void rrd_stats_api_v1_charts_allmetrics_prometheus(BUFFER *wb); + extern unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb); extern void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb); diff --git a/src/simple_pattern.c b/src/simple_pattern.c new file mode 100644 index 000000000..7e4424297 --- /dev/null +++ b/src/simple_pattern.c @@ -0,0 +1,197 @@ +#include "common.h" + +struct simple_pattern { + const char *match; + size_t len; + + SIMPLE_PREFIX_MODE mode; + char negative; + + struct simple_pattern *child; + + struct simple_pattern *next; +}; + +static inline struct simple_pattern *parse_pattern(const char *str, SIMPLE_PREFIX_MODE default_mode) { + SIMPLE_PREFIX_MODE mode; + struct simple_pattern *child = NULL; + + char *buf = strdupz(str); + char *s = buf, *c = buf; + + // skip asterisks in front + while(*c == '*') c++; + + // find the next asterisk + while(*c && *c != '*') c++; + + // do we have an asterisk in the middle? + if(*c == '*' && c[1] != '\0') { + // yes, we have + child = parse_pattern(c, default_mode); + c[1] = '\0'; + } + + // check what this one matches + + size_t len = strlen(s); + if(len >= 2 && *s == '*' && s[len - 1] == '*') { + s[len - 1] = '\0'; + s++; + mode = SIMPLE_PATTERN_SUBSTRING; + } + else if(len >= 1 && *s == '*') { + s++; + mode = SIMPLE_PATTERN_SUFFIX; + } + else if(len >= 1 && s[len - 1] == '*') { + s[len - 1] = '\0'; + mode = SIMPLE_PATTERN_PREFIX; + } + else + mode = default_mode; + + // allocate the structure + struct simple_pattern *m = callocz(1, sizeof(struct simple_pattern)); + if(*s) { + m->match = strdupz(s); + m->len = strlen(m->match); + m->mode = mode; + } + else { + m->mode = SIMPLE_PATTERN_SUBSTRING; + } + + m->child = child; + + freez(buf); + + return m; +} + +SIMPLE_PATTERN *simple_pattern_create(const char *list, SIMPLE_PREFIX_MODE default_mode) { + struct simple_pattern *root = NULL, *last = NULL; + + if(unlikely(!list || !*list)) return root; + + char *buf = strdupz(list); + if(buf && *buf) { + char *s = buf; + + while(s && *s) { + char negative = 0; + + // skip all spaces + while(isspace(*s)) s++; + + if(*s == '!') { + negative = 1; + s++; + } + + // empty string + if(unlikely(!*s)) break; + + // find the next space + char *c = s; + while(*c && !isspace(*c)) c++; + + // find the next word + char *n; + if(likely(*c)) n = c + 1; + else n = NULL; + + // terminate our string + *c = '\0'; + + struct simple_pattern *m = parse_pattern(s, default_mode); + m->negative = negative; + + if(likely(n)) *c = ' '; + + // link it at the end + if(unlikely(!root)) + root = last = m; + else { + last->next = m; + last = m; + } + + // prepare for next loop + s = n; + } + } + + freez(buf); + return (SIMPLE_PATTERN *)root; +} + +static inline int match_pattern(struct simple_pattern *m, const char *str, size_t len) { + char *s; + + if(m->len <= len) { + switch(m->mode) { + case SIMPLE_PATTERN_SUBSTRING: + if(!m->len) return 1; + if((s = strstr(str, m->match))) { + if(!m->child) return 1; + return match_pattern(m->child, &s[m->len], len - (s - str) - m->len); + } + break; + + case SIMPLE_PATTERN_PREFIX: + if(unlikely(strncmp(str, m->match, m->len) == 0)) { + if(!m->child) return 1; + return match_pattern(m->child, &str[m->len], len - m->len); + } + break; + + case SIMPLE_PATTERN_SUFFIX: + if(unlikely(strcmp(&str[len - m->len], m->match) == 0)) { + if(!m->child) return 1; + return 0; + } + break; + + case SIMPLE_PATTERN_EXACT: + default: + if(unlikely(strcmp(str, m->match) == 0)) { + if(!m->child) return 1; + return 0; + } + break; + } + } + + return 0; +} + +int simple_pattern_matches(SIMPLE_PATTERN *list, const char *str) { + struct simple_pattern *m, *root = (struct simple_pattern *)list; + + if(unlikely(!root)) return 0; + + size_t len = strlen(str); + for(m = root; m ; m = m->next) + if(match_pattern(m, str, len)) { + if(m->negative) return 0; + return 1; + } + + return 0; +} + +static inline void free_pattern(struct simple_pattern *m) { + if(!m) return; + + if(m->next) free_pattern(m->next); + if(m->child) free_pattern(m->child); + freez((void *)m->match); + freez(m); +} + +void simple_pattern_free(SIMPLE_PATTERN *list) { + if(!list) return; + + free_pattern(((struct simple_pattern *)list)->next); +} diff --git a/src/simple_pattern.h b/src/simple_pattern.h new file mode 100644 index 000000000..3768c5089 --- /dev/null +++ b/src/simple_pattern.h @@ -0,0 +1,25 @@ +#ifndef NETDATA_SIMPLE_PATTERN_H +#define NETDATA_SIMPLE_PATTERN_H + +typedef enum { + SIMPLE_PATTERN_EXACT, + SIMPLE_PATTERN_PREFIX, + SIMPLE_PATTERN_SUFFIX, + SIMPLE_PATTERN_SUBSTRING +} SIMPLE_PREFIX_MODE; + +typedef void SIMPLE_PATTERN; + +// create a simple_pattern from the string given +// default_mode is used in cases where EXACT matches, without an asterisk, +// should be considered PREFIX matches. +extern SIMPLE_PATTERN *simple_pattern_create(const char *list, SIMPLE_PREFIX_MODE default_mode); + +// test if string str is matched from the pattern +extern int simple_pattern_matches(SIMPLE_PATTERN *list, const char *str); + +// free a simple_pattern that was created with simple_pattern_create() +// list can be NULL, in which case, this does nothing. +extern void simple_pattern_free(SIMPLE_PATTERN *list); + +#endif //NETDATA_SIMPLE_PATTERN_H diff --git a/src/socket.c b/src/socket.c new file mode 100644 index 000000000..643811e44 --- /dev/null +++ b/src/socket.c @@ -0,0 +1,179 @@ +#include "common.h" + +// connect_to() +// +// definition format: +// +// [PROTOCOL:]IP[%INTERFACE][:PORT] +// +// PROTOCOL = tcp or udp +// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6) +// INTERFACE = for IPv6 only, the network interface to use +// PORT = port number or service name + +int connect_to(const char *definition, int default_port, struct timeval *timeout) { + struct addrinfo hints; + struct addrinfo *ai_head = NULL, *ai = NULL; + + char buffer[strlen(definition) + 1]; + strcpy(buffer, definition); + + char default_service[10 + 1]; + snprintfz(default_service, 10, "%d", default_port); + + char *host = buffer, *service = default_service, *interface = ""; + int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; + uint32_t scope_id = 0; + + if(strncmp(host, "tcp:", 4) == 0) { + host += 4; + protocol = IPPROTO_TCP; + socktype = SOCK_STREAM; + } + else if(strncmp(host, "udp:", 4) == 0) { + host += 4; + protocol = IPPROTO_UDP; + socktype = SOCK_DGRAM; + } + + char *e = host; + if(*e == '[') { + e = ++host; + while(*e && *e != ']') e++; + if(*e == ']') { + *e = '\0'; + e++; + } + } + else { + while(*e && *e != ':' && *e != '%') e++; + } + + if(*e == '%') { + *e = '\0'; + e++; + interface = e; + while(*e && *e != ':') e++; + } + + if(*e == ':') { + *e = '\0'; + e++; + service = e; + } + + debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP); + + if(!*host) { + error("Definition '%s' does not specify a host.", definition); + return -1; + } + + if(*interface) { + scope_id = if_nametoindex(interface); + if(!scope_id) + error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + } + + if(!*service) + service = default_service; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = socktype; + hints.ai_protocol = protocol; + + int ai_err = getaddrinfo(host, service, &hints, &ai_head); + if (ai_err != 0) { + error("Cannot resolve host '%s', port '%s': %s", host, service, gai_strerror(ai_err)); + return -1; + } + + int fd = -1; + for (ai = ai_head; ai != NULL && fd == -1; ai = ai->ai_next) { + + if (ai->ai_family == PF_INET6) { + struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr; + if(pSadrIn6->sin6_scope_id == 0) { + pSadrIn6->sin6_scope_id = scope_id; + } + } + + char hostBfr[NI_MAXHOST + 1]; + char servBfr[NI_MAXSERV + 1]; + + getnameinfo(ai->ai_addr, + ai->ai_addrlen, + hostBfr, + sizeof(hostBfr), + servBfr, + sizeof(servBfr), + NI_NUMERICHOST | NI_NUMERICSERV); + + debug(D_CONNECT_TO, "Address info: host = '%s', service = '%s', ai_flags = 0x%02X, ai_family = %d (PF_INET = %d, PF_INET6 = %d), ai_socktype = %d (SOCK_STREAM = %d, SOCK_DGRAM = %d), ai_protocol = %d (IPPROTO_TCP = %d, IPPROTO_UDP = %d), ai_addrlen = %lu (sockaddr_in = %lu, sockaddr_in6 = %lu)", + hostBfr, + servBfr, + (unsigned int)ai->ai_flags, + ai->ai_family, + PF_INET, + PF_INET6, + ai->ai_socktype, + SOCK_STREAM, + SOCK_DGRAM, + ai->ai_protocol, + IPPROTO_TCP, + IPPROTO_UDP, + (unsigned long)ai->ai_addrlen, + (unsigned long)sizeof(struct sockaddr_in), + (unsigned long)sizeof(struct sockaddr_in6)); + + switch (ai->ai_addr->sa_family) { + case PF_INET: { + struct sockaddr_in *pSadrIn = (struct sockaddr_in *)ai->ai_addr; + debug(D_CONNECT_TO, "ai_addr = sin_family: %d (AF_INET = %d, AF_INET6 = %d), sin_addr: '%s', sin_port: '%s'", + pSadrIn->sin_family, + AF_INET, + AF_INET6, + hostBfr, + servBfr); + break; + } + + case PF_INET6: { + struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr; + debug(D_CONNECT_TO,"ai_addr = sin6_family: %d (AF_INET = %d, AF_INET6 = %d), sin6_addr: '%s', sin6_port: '%s', sin6_flowinfo: %u, sin6_scope_id: %u", + pSadrIn6->sin6_family, + AF_INET, + AF_INET6, + hostBfr, + servBfr, + pSadrIn6->sin6_flowinfo, + pSadrIn6->sin6_scope_id); + break; + } + + default: { + debug(D_CONNECT_TO, "Unknown protocol family %d.", ai->ai_family); + continue; + } + } + + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if(fd != -1) { + if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *)timeout, sizeof(struct timeval)) < 0) + error("Failed to set timeout on the socket to ip '%s' port '%s'", hostBfr, servBfr); + + if(connect(fd, ai->ai_addr, ai->ai_addrlen) < 0) { + error("Failed to connect to '%s', port '%s'", hostBfr, servBfr); + close(fd); + fd = -1; + } + + debug(D_CONNECT_TO, "Connected to '%s' on port '%s'.", hostBfr, servBfr); + } + } + + freeaddrinfo(ai_head); + + return fd; +} diff --git a/src/socket.h b/src/socket.h new file mode 100644 index 000000000..791c0ce54 --- /dev/null +++ b/src/socket.h @@ -0,0 +1,10 @@ +// +// Created by costa on 24/12/2016. +// + +#ifndef NETDATA_SOCKET_H +#define NETDATA_SOCKET_H + +extern int connect_to(const char *definition, int default_port, struct timeval *timeout); + +#endif //NETDATA_SOCKET_H diff --git a/src/sys_devices_system_edac_mc.c b/src/sys_devices_system_edac_mc.c new file mode 100644 index 000000000..c764615f1 --- /dev/null +++ b/src/sys_devices_system_edac_mc.c @@ -0,0 +1,183 @@ +#include "common.h" + +struct mc { + char *name; + char ce_updated; + char ue_updated; + + char *ce_count_filename; + char *ue_count_filename; + + procfile *ce_ff; + procfile *ue_ff; + + collected_number ce_count; + collected_number ue_count; + + RRDDIM *ce_rd; + RRDDIM *ue_rd; + + struct mc *next; +}; +static struct mc *mc_root = NULL; + +static void find_all_mc() { + char name[FILENAME_MAX + 1]; + snprintfz(name, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/devices/system/edac/mc"); + char *dirname = config_get("plugin:proc:/sys/devices/system/edac/mc", "directory to monitor", name); + + DIR *dir = opendir(dirname); + if(!dir) { + error("Cannot read ECC memory errors directory '%s'", dirname); + return; + } + + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR && de->d_name[0] == 'm' && de->d_name[1] == 'c' && isdigit(de->d_name[2])) { + struct mc *m = callocz(1, sizeof(struct mc)); + m->name = strdupz(de->d_name); + + struct stat st; + + snprintfz(name, FILENAME_MAX, "%s/%s/ce_count", dirname, de->d_name); + if(stat(name, &st) != -1) + m->ce_count_filename = strdupz(name); + + snprintfz(name, FILENAME_MAX, "%s/%s/ue_count", dirname, de->d_name); + if(stat(name, &st) != -1) + m->ue_count_filename = strdupz(name); + + if(!m->ce_count_filename && !m->ue_count_filename) { + freez(m->name); + freez(m); + } + else { + m->next = mc_root; + mc_root = m; + } + } + } + + closedir(dir); +} + +int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt) { + (void)dt; + + if(unlikely(mc_root == NULL)) { + find_all_mc(); + if(unlikely(mc_root == NULL)) + return 1; + } + + static int do_ce = -1, do_ue = -1; + calculated_number ce_sum = 0, ue_sum = 0; + struct mc *m; + + if(unlikely(do_ce == -1)) { + do_ce = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory correctable errors", CONFIG_ONDEMAND_ONDEMAND); + do_ue = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/edac/mc", "enable ECC memory uncorrectable errors", CONFIG_ONDEMAND_ONDEMAND); + } + + if(do_ce != CONFIG_ONDEMAND_NO) { + for(m = mc_root; m; m = m->next) { + if(m->ce_count_filename) { + m->ce_updated = 0; + + if(unlikely(!m->ce_ff)) { + m->ce_ff = procfile_open(m->ce_count_filename, " \t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!m->ce_ff)) + continue; + } + + m->ce_ff = procfile_readall(m->ce_ff); + if(unlikely(!m->ce_ff || procfile_lines(m->ce_ff) < 1 || procfile_linewords(m->ce_ff, 0) < 1)) + continue; + + m->ce_count = str2ull(procfile_lineword(m->ce_ff, 0, 0)); + ce_sum += m->ce_count; + m->ce_updated = 1; + } + } + } + + if(do_ue != CONFIG_ONDEMAND_NO) { + for(m = mc_root; m; m = m->next) { + if(m->ue_count_filename) { + m->ue_updated = 0; + + if(unlikely(!m->ue_ff)) { + m->ue_ff = procfile_open(m->ue_count_filename, " \t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!m->ue_ff)) + continue; + } + + m->ue_ff = procfile_readall(m->ue_ff); + if(unlikely(!m->ue_ff || procfile_lines(m->ue_ff) < 1 || procfile_linewords(m->ue_ff, 0) < 1)) + continue; + + m->ue_count = str2ull(procfile_lineword(m->ue_ff, 0, 0)); + ue_sum += m->ue_count; + m->ue_updated = 1; + } + } + } + + // -------------------------------------------------------------------- + + if(do_ce == CONFIG_ONDEMAND_YES || (do_ce == CONFIG_ONDEMAND_ONDEMAND && ce_sum > 0)) { + do_ce = CONFIG_ONDEMAND_YES; + + static RRDSET *ce_st = NULL; + + if(unlikely(!ce_st)) { + ce_st = rrdset_find("mem.ecc_ce"); + if(unlikely(!ce_st)) + ce_st = rrdset_create("mem", "ecc_ce", NULL, "ecc", NULL, "ECC Memory Correctable Errors", "errors", + 6600, update_every, RRDSET_TYPE_LINE); + + for(m = mc_root; m; m = m->next) + if(m->ce_count_filename) + m->ce_rd = rrddim_add(ce_st, m->name, NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else + rrdset_next(ce_st); + + for(m = mc_root; m; m = m->next) + if(m->ce_count_filename && m->ce_updated) + rrddim_set_by_pointer(ce_st, m->ce_rd, m->ce_count); + + rrdset_done(ce_st); + } + + // -------------------------------------------------------------------- + + if(do_ue == CONFIG_ONDEMAND_YES || (do_ue == CONFIG_ONDEMAND_ONDEMAND && ue_sum > 0)) { + do_ue = CONFIG_ONDEMAND_YES; + + static RRDSET *ue_st = NULL; + + if(unlikely(!ue_st)) { + ue_st = rrdset_find("mem.ecc_ue"); + + if(unlikely(!ue_st)) + ue_st = rrdset_create("mem", "ecc_ue", NULL, "ecc", NULL, "ECC Memory Uncorrectable Errors", "errors", + 6610, update_every, RRDSET_TYPE_LINE); + + for(m = mc_root; m; m = m->next) + if(m->ue_count_filename) + m->ue_rd = rrddim_add(ue_st, m->name, NULL, 1, 1, RRDDIM_INCREMENTAL); + } + else + rrdset_next(ue_st); + + for(m = mc_root; m; m = m->next) + if(m->ue_count_filename && m->ue_updated) + rrddim_set_by_pointer(ue_st, m->ue_rd, m->ue_count); + + rrdset_done(ue_st); + } + + return 0; +} diff --git a/src/sys_devices_system_node.c b/src/sys_devices_system_node.c new file mode 100644 index 000000000..18c3fcd3a --- /dev/null +++ b/src/sys_devices_system_node.c @@ -0,0 +1,127 @@ +#include "common.h" + +struct node { + char *name; + char *numastat_filename; + procfile *numastat_ff; + RRDSET *numastat_st; + struct node *next; +}; +static struct node *numa_root = NULL; + +static int find_all_nodes() { + int numa_node_count = 0; + char name[FILENAME_MAX + 1]; + snprintfz(name, FILENAME_MAX, "%s%s", global_host_prefix, "/sys/devices/system/node"); + char *dirname = config_get("plugin:proc:/sys/devices/system/node", "directory to monitor", name); + + DIR *dir = opendir(dirname); + if(!dir) { + error("Cannot read NUMA node directory '%s'", dirname); + return 0; + } + + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type != DT_DIR) + continue; + + if(strncmp(de->d_name, "node", 4) != 0) + continue; + + if(!isdigit(de->d_name[4])) + continue; + + numa_node_count++; + + struct node *m = callocz(1, sizeof(struct node)); + m->name = strdupz(de->d_name); + + struct stat st; + + snprintfz(name, FILENAME_MAX, "%s/%s/numastat", dirname, de->d_name); + if(stat(name, &st) == -1) { + freez(m->name); + freez(m); + continue; + } + + m->numastat_filename = strdupz(name); + + m->next = numa_root; + numa_root = m; + } + + closedir(dir); + + return numa_node_count; +} + +int do_proc_sys_devices_system_node(int update_every, usec_t dt) { + (void)dt; + + static int numa_node_count = 0; + + if(unlikely(numa_root == NULL)) { + numa_node_count = find_all_nodes(update_every); + if(unlikely(numa_root == NULL)) + return 1; + } + + static int do_numastat = -1; + struct node *m; + + if(unlikely(do_numastat == -1)) { + do_numastat = config_get_boolean_ondemand("plugin:proc:/sys/devices/system/node", "enable per-node numa metrics", CONFIG_ONDEMAND_ONDEMAND); + } + + if(do_numastat == CONFIG_ONDEMAND_YES || (do_numastat == CONFIG_ONDEMAND_ONDEMAND && numa_node_count >= 2)) { + for(m = numa_root; m; m = m->next) { + if(m->numastat_filename) { + if(unlikely(!m->numastat_ff)) { + m->numastat_ff = procfile_open(m->numastat_filename, " ", PROCFILE_FLAG_DEFAULT); + if(unlikely(!m->numastat_ff)) + continue; + } + + m->numastat_ff = procfile_readall(m->numastat_ff); + if(unlikely(!m->numastat_ff || procfile_lines(m->numastat_ff) < 1 || procfile_linewords(m->numastat_ff, 0) < 1)) + continue; + + procfile *ff = m->numastat_ff; + + RRDSET *st = m->numastat_st; + if(unlikely(!st)) { + st = rrdset_create("mem", m->name, NULL, "numa", NULL, "NUMA events", "events/s", 1000, update_every, RRDSET_TYPE_LINE); + st->isdetail = 1; + + rrddim_add(st, "local_node", "local", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "numa_foreign", "foreign", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "interleave_hit", "interleave", 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(st, "other_node", "other", 1, 1, RRDDIM_INCREMENTAL); + + m->numastat_st = st; + } + else rrdset_next(st); + + uint32_t lines = procfile_lines(ff), l; + for(l = 0; l < lines; l++) { + uint32_t words = procfile_linewords(ff, l); + if(unlikely(words < 2)) { + if(unlikely(words)) error("Cannot read %s numastat line %u. Expected 2 params, read %u.", m->name, l, words); + continue; + } + + char *name = procfile_lineword(ff, l, 0); + char *value = procfile_lineword(ff, l, 1); + if (unlikely(!name || !*name || !value || !*value)) continue; + + rrddim_set(st, name, strtoull(value, NULL, 10)); + } + rrdset_done(st); + } + } + } + + return 0; +} diff --git a/src/sys_fs_cgroup.c b/src/sys_fs_cgroup.c index 298f38a38..2b7254f47 100644 --- a/src/sys_fs_cgroup.c +++ b/src/sys_fs_cgroup.c @@ -3,13 +3,38 @@ // ---------------------------------------------------------------------------- // cgroup globals +#define CHART_PRIORITY_SYSTEMD_SERVICES 19000 +#define CHART_PRIORITY_CONTAINERS 40000 + +static long system_page_size = 4096; // system will be queried via sysconf() in configuration() + static int cgroup_enable_cpuacct_stat = CONFIG_ONDEMAND_ONDEMAND; static int cgroup_enable_cpuacct_usage = CONFIG_ONDEMAND_ONDEMAND; static int cgroup_enable_memory = CONFIG_ONDEMAND_ONDEMAND; -static int cgroup_enable_devices = CONFIG_ONDEMAND_ONDEMAND; -static int cgroup_enable_blkio = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_detailed_memory = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_memory_failcnt = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_swap = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_blkio_io = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_blkio_ops = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_blkio_throttle_io = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_blkio_throttle_ops = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_blkio_merged_ops = CONFIG_ONDEMAND_ONDEMAND; +static int cgroup_enable_blkio_queued_ops = CONFIG_ONDEMAND_ONDEMAND; + +static int cgroup_enable_systemd_services = CONFIG_ONDEMAND_YES; +static int cgroup_enable_systemd_services_detailed_memory = CONFIG_ONDEMAND_NO; +static int cgroup_used_memory_without_cache = CONFIG_ONDEMAND_YES; + +static int cgroup_search_in_devices = 1; + static int cgroup_enable_new_cgroups_detected_at_runtime = 1; static int cgroup_check_for_new_every = 10; +static int cgroup_update_every = 1; + +static int cgroup_recheck_zero_blkio_every_iterations = 10; +static int cgroup_recheck_zero_mem_failcnt_every_iterations = 10; +static int cgroup_recheck_zero_mem_detailed_every_iterations = 10; + static char *cgroup_cpuacct_base = NULL; static char *cgroup_blkio_base = NULL; static char *cgroup_memory_base = NULL; @@ -19,16 +44,59 @@ static int cgroup_root_count = 0; static int cgroup_root_max = 500; static int cgroup_max_depth = 0; +static SIMPLE_PATTERN *enabled_cgroup_patterns = NULL; +static SIMPLE_PATTERN *enabled_cgroup_paths = NULL; +static SIMPLE_PATTERN *enabled_cgroup_renames = NULL; +static SIMPLE_PATTERN *systemd_services_cgroups = NULL; + +static char *cgroups_rename_script = PLUGINS_DIR "/cgroup-name.sh"; + +static uint32_t Read_hash = 0; +static uint32_t Write_hash = 0; +static uint32_t user_hash = 0; +static uint32_t system_hash = 0; + void read_cgroup_plugin_configuration() { - cgroup_check_for_new_every = config_get_number("plugin:cgroups", "check for new cgroups every", cgroup_check_for_new_every); + system_page_size = sysconf(_SC_PAGESIZE); + + Read_hash = simple_hash("Read"); + Write_hash = simple_hash("Write"); + user_hash = simple_hash("user"); + system_hash = simple_hash("system"); + + cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", rrd_update_every); + if(cgroup_update_every < rrd_update_every) + cgroup_update_every = rrd_update_every; + + cgroup_check_for_new_every = (int)config_get_number("plugin:cgroups", "check for new cgroups every", cgroup_check_for_new_every * cgroup_update_every); + if(cgroup_check_for_new_every < cgroup_update_every) + cgroup_check_for_new_every = cgroup_update_every; + + cgroup_enable_cpuacct_stat = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct stat (total CPU)", cgroup_enable_cpuacct_stat); + cgroup_enable_cpuacct_usage = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct usage (per core CPU)", cgroup_enable_cpuacct_usage); + + cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory (used mem including cache)", cgroup_enable_memory); + cgroup_enable_detailed_memory = config_get_boolean_ondemand("plugin:cgroups", "enable detailed memory", cgroup_enable_detailed_memory); + cgroup_enable_memory_failcnt = config_get_boolean_ondemand("plugin:cgroups", "enable memory limits fail count", cgroup_enable_memory_failcnt); + cgroup_enable_swap = config_get_boolean_ondemand("plugin:cgroups", "enable swap memory", cgroup_enable_swap); + + cgroup_enable_blkio_io = config_get_boolean_ondemand("plugin:cgroups", "enable blkio bandwidth", cgroup_enable_blkio_io); + cgroup_enable_blkio_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio operations", cgroup_enable_blkio_ops); + cgroup_enable_blkio_throttle_io = config_get_boolean_ondemand("plugin:cgroups", "enable blkio throttle bandwidth", cgroup_enable_blkio_throttle_io); + cgroup_enable_blkio_throttle_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio throttle operations", cgroup_enable_blkio_throttle_ops); + cgroup_enable_blkio_queued_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio queued operations", cgroup_enable_blkio_queued_ops); + cgroup_enable_blkio_merged_ops = config_get_boolean_ondemand("plugin:cgroups", "enable blkio merged operations", cgroup_enable_blkio_merged_ops); - cgroup_enable_cpuacct_stat = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct stat", cgroup_enable_cpuacct_stat); - cgroup_enable_cpuacct_usage = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct usage", cgroup_enable_cpuacct_usage); - cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory", cgroup_enable_memory); - cgroup_enable_blkio = config_get_boolean_ondemand("plugin:cgroups", "enable blkio", cgroup_enable_blkio); + cgroup_recheck_zero_blkio_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero blkio every iterations", cgroup_recheck_zero_blkio_every_iterations); + cgroup_recheck_zero_mem_failcnt_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero memory failcnt every iterations", cgroup_recheck_zero_mem_failcnt_every_iterations); + cgroup_recheck_zero_mem_detailed_every_iterations = (int)config_get_number("plugin:cgroups", "recheck zero detailed memory every iterations", cgroup_recheck_zero_mem_detailed_every_iterations); + + cgroup_enable_systemd_services = config_get_boolean("plugin:cgroups", "enable systemd services", cgroup_enable_systemd_services); + cgroup_enable_systemd_services_detailed_memory = config_get_boolean("plugin:cgroups", "enable systemd services detailed memory", cgroup_enable_systemd_services_detailed_memory); + cgroup_used_memory_without_cache = config_get_boolean("plugin:cgroups", "report used memory without cache", cgroup_used_memory_without_cache); char filename[FILENAME_MAX + 1], *s; - struct mountinfo *mi, *root = mountinfo_read(); + struct mountinfo *mi, *root = mountinfo_read(0); mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "cpuacct"); if(!mi) mi = mountinfo_find_by_filesystem_mount_source(root, "cgroup", "cpuacct"); @@ -70,11 +138,67 @@ void read_cgroup_plugin_configuration() { snprintfz(filename, FILENAME_MAX, "%s%s", global_host_prefix, s); cgroup_devices_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/devices", filename); - cgroup_root_max = config_get_number("plugin:cgroups", "max cgroups to allow", cgroup_root_max); - cgroup_max_depth = config_get_number("plugin:cgroups", "max cgroups depth to monitor", cgroup_max_depth); + cgroup_root_max = (int)config_get_number("plugin:cgroups", "max cgroups to allow", cgroup_root_max); + cgroup_max_depth = (int)config_get_number("plugin:cgroups", "max cgroups depth to monitor", cgroup_max_depth); cgroup_enable_new_cgroups_detected_at_runtime = config_get_boolean("plugin:cgroups", "enable new cgroups detected at run time", cgroup_enable_new_cgroups_detected_at_runtime); + enabled_cgroup_patterns = simple_pattern_create( + config_get("plugin:cgroups", "enable by default cgroups matching", + " !*.mount " + " !*.partition " + " !*.scope " + " !*.service " + " !*.slice " + " !*.swap " + " !*.user " + " !/ " + " !/docker " + " !/libvirt " + " !/lxc " + " !/lxc/*/ns " // #1397 + " !/machine " + " !/qemu " + " !/system " + " !/systemd " + " !/user " + " * " // enable anything else + ), SIMPLE_PATTERN_EXACT); + + enabled_cgroup_paths = simple_pattern_create( + config_get("plugin:cgroups", "search for cgroups in subpaths matching", + " !*-qemu " // #345 + " !/init.scope " + " !/system " + " !/systemd " + " !/user " + " !/user.slice " + " * " + ), SIMPLE_PATTERN_EXACT); + + cgroups_rename_script = config_get("plugin:cgroups", "script to get cgroup names", cgroups_rename_script); + + enabled_cgroup_renames = simple_pattern_create( + config_get("plugin:cgroups", "run script to rename cgroups matching", + " !/ " + " !*.mount " + " !*.partition " + " !*.scope " + " !*.service " + " !*.slice " + " !*.swap " + " !*.user " + " * " + ), SIMPLE_PATTERN_EXACT); + + if(cgroup_enable_systemd_services) { + systemd_services_cgroups = simple_pattern_create( + config_get("plugin:cgroups", "cgroups to match as systemd services", + " !/system.slice/*/*.service " + " /system.slice/*.service " + ), SIMPLE_PATTERN_EXACT); + } + mountinfo_free(root); } @@ -83,6 +207,8 @@ void read_cgroup_plugin_configuration() { struct blkio { int updated; + int enabled; // CONFIG_ONDEMAND_YES or CONFIG_ONDEMAND_ONDEMAND + int delay_counter; char *filename; @@ -97,12 +223,32 @@ struct blkio { // https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt struct memory { - int updated; + ARL_BASE *arl_base; + ARL_ENTRY *arl_dirty; + ARL_ENTRY *arl_swap; - char *filename; + int updated_detailed; + int updated_usage_in_bytes; + int updated_msw_usage_in_bytes; + int updated_failcnt; + + int enabled_detailed; // CONFIG_ONDEMAND_YES or CONFIG_ONDEMAND_ONDEMAND + int enabled_usage_in_bytes; // CONFIG_ONDEMAND_YES or CONFIG_ONDEMAND_ONDEMAND + int enabled_msw_usage_in_bytes; // CONFIG_ONDEMAND_YES or CONFIG_ONDEMAND_ONDEMAND + int enabled_failcnt; // CONFIG_ONDEMAND_YES or CONFIG_ONDEMAND_ONDEMAND - int has_dirty_swap; + int delay_counter_detailed; + int delay_counter_failcnt; + char *filename_detailed; + char *filename_usage_in_bytes; + char *filename_msw_usage_in_bytes; + char *filename_failcnt; + + int detailed_has_dirty; + int detailed_has_swap; + + // detailed metrics unsigned long long cache; unsigned long long rss; unsigned long long rss_huge; @@ -139,22 +285,16 @@ struct memory { unsigned long long total_unevictable; */ - int usage_in_bytes_updated; - char *filename_usage_in_bytes; + // single file metrics unsigned long long usage_in_bytes; - - int msw_usage_in_bytes_updated; - char *filename_msw_usage_in_bytes; unsigned long long msw_usage_in_bytes; - - int failcnt_updated; - char *filename_failcnt; unsigned long long failcnt; }; // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt struct cpuacct_stat { int updated; + int enabled; // CONFIG_ONDEMAND_YES or CONFIG_ONDEMAND_ONDEMAND char *filename; @@ -165,6 +305,7 @@ struct cpuacct_stat { // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt struct cpuacct_usage { int updated; + int enabled; // CONFIG_ONDEMAND_YES or CONFIG_ONDEMAND_ONDEMAND char *filename; @@ -172,7 +313,8 @@ struct cpuacct_usage { unsigned long long *cpu_percpu; }; -#define CGROUP_OPTIONS_DISABLED_DUPLICATE 0x00000001 +#define CGROUP_OPTIONS_DISABLED_DUPLICATE 0x00000001 +#define CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE 0x00000002 struct cgroup { uint32_t options; @@ -202,6 +344,51 @@ struct cgroup { struct blkio io_merged; // operations struct blkio io_queued; // operations + // per cgroup charts + RRDSET *st_cpu; + RRDSET *st_cpu_per_core; + RRDSET *st_mem; + RRDSET *st_writeback; + RRDSET *st_mem_activity; + RRDSET *st_pgfaults; + RRDSET *st_mem_usage; + RRDSET *st_mem_failcnt; + RRDSET *st_io; + RRDSET *st_serviced_ops; + RRDSET *st_throttle_io; + RRDSET *st_throttle_serviced_ops; + RRDSET *st_queued_ops; + RRDSET *st_merged_ops; + + // services + RRDDIM *rd_cpu; + RRDDIM *rd_mem_usage; + RRDDIM *rd_mem_failcnt; + RRDDIM *rd_swap_usage; + + RRDDIM *rd_mem_detailed_cache; + RRDDIM *rd_mem_detailed_rss; + RRDDIM *rd_mem_detailed_mapped; + RRDDIM *rd_mem_detailed_writeback; + RRDDIM *rd_mem_detailed_pgpgin; + RRDDIM *rd_mem_detailed_pgpgout; + RRDDIM *rd_mem_detailed_pgfault; + RRDDIM *rd_mem_detailed_pgmajfault; + + RRDDIM *rd_io_service_bytes_read; + RRDDIM *rd_io_serviced_read; + RRDDIM *rd_throttle_io_read; + RRDDIM *rd_throttle_io_serviced_read; + RRDDIM *rd_io_queued_read; + RRDDIM *rd_io_merged_read; + + RRDDIM *rd_io_service_bytes_write; + RRDDIM *rd_io_serviced_write; + RRDDIM *rd_throttle_io_write; + RRDDIM *rd_throttle_io_serviced_write; + RRDDIM *rd_io_queued_write; + RRDDIM *rd_io_merged_write; + struct cgroup *next; } *cgroup_root = NULL; @@ -209,29 +396,27 @@ struct cgroup { // ---------------------------------------------------------------------------- // read values from /sys -void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) { +static inline void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) { static procfile *ff = NULL; - static uint32_t user_hash = 0; - static uint32_t system_hash = 0; - - if(unlikely(user_hash == 0)) { - user_hash = simple_hash("user"); - system_hash = simple_hash("system"); - } - - cp->updated = 0; - if(cp->filename) { + if(likely(cp->filename)) { ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT); - if(!ff) return; + if(unlikely(!ff)) { + cp->updated = 0; + return; + } ff = procfile_readall(ff); - if(!ff) return; + if(unlikely(!ff)) { + cp->updated = 0; + return; + } unsigned long i, lines = procfile_lines(ff); - if(lines < 1) { + if(unlikely(lines < 1)) { error("File '%s' should have 1+ lines.", cp->filename); + cp->updated = 0; return; } @@ -239,37 +424,47 @@ void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) { char *s = procfile_lineword(ff, i, 0); uint32_t hash = simple_hash(s); - if(hash == user_hash && !strcmp(s, "user")) - cp->user = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + if(unlikely(hash == user_hash && !strcmp(s, "user"))) + cp->user = str2ull(procfile_lineword(ff, i, 1)); - else if(hash == system_hash && !strcmp(s, "system")) - cp->system = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + else if(unlikely(hash == system_hash && !strcmp(s, "system"))) + cp->system = str2ull(procfile_lineword(ff, i, 1)); } cp->updated = 1; - // fprintf(stderr, "READ '%s': user: %llu, system: %llu\n", cp->filename, cp->user, cp->system); + if(unlikely(cp->enabled == CONFIG_ONDEMAND_ONDEMAND && (cp->user || cp->system))) + cp->enabled = CONFIG_ONDEMAND_YES; } } -void cgroup_read_cpuacct_usage(struct cpuacct_usage *ca) { +static inline void cgroup_read_cpuacct_usage(struct cpuacct_usage *ca) { static procfile *ff = NULL; - ca->updated = 0; - if(ca->filename) { + if(likely(ca->filename)) { ff = procfile_reopen(ff, ca->filename, NULL, PROCFILE_FLAG_DEFAULT); - if(!ff) return; + if(unlikely(!ff)) { + ca->updated = 0; + return; + } ff = procfile_readall(ff); - if(!ff) return; + if(unlikely(!ff)) { + ca->updated = 0; + return; + } - if(procfile_lines(ff) < 1) { - error("File '%s' should have 1+ lines but has %u.", ca->filename, procfile_lines(ff)); + if(unlikely(procfile_lines(ff) < 1)) { + error("File '%s' should have 1+ lines but has %zu.", ca->filename, procfile_lines(ff)); + ca->updated = 0; return; } unsigned long i = procfile_linewords(ff, 0); - if(i <= 0) return; + if(unlikely(i == 0)) { + return; + ca->updated = 0; + } // we may have 1 more CPU reported while(i > 0) { @@ -278,54 +473,52 @@ void cgroup_read_cpuacct_usage(struct cpuacct_usage *ca) { else break; } - if(i != ca->cpus) { + if(unlikely(i != ca->cpus)) { freez(ca->cpu_percpu); ca->cpu_percpu = mallocz(sizeof(unsigned long long) * i); ca->cpus = (unsigned int)i; } + unsigned long long total = 0; for(i = 0; i < ca->cpus ;i++) { - ca->cpu_percpu[i] = strtoull(procfile_lineword(ff, 0, i), NULL, 10); - // fprintf(stderr, "READ '%s': cpu%d/%d: %llu ('%s')\n", ca->filename, i, ca->cpus, ca->cpu_percpu[i], procfile_lineword(ff, 0, i)); + unsigned long long n = str2ull(procfile_lineword(ff, 0, i)); + ca->cpu_percpu[i] = n; + total += n; } ca->updated = 1; + + if(unlikely(ca->enabled == CONFIG_ONDEMAND_ONDEMAND && total)) + ca->enabled = CONFIG_ONDEMAND_YES; } } -void cgroup_read_blkio(struct blkio *io) { +static inline void cgroup_read_blkio(struct blkio *io) { static procfile *ff = NULL; - static uint32_t Read_hash = 0; - static uint32_t Write_hash = 0; -/* - static uint32_t Sync_hash = 0; - static uint32_t Async_hash = 0; - static uint32_t Total_hash = 0; -*/ - - if(unlikely(Read_hash == 0)) { - Read_hash = simple_hash("Read"); - Write_hash = simple_hash("Write"); -/* - Sync_hash = simple_hash("Sync"); - Async_hash = simple_hash("Async"); - Total_hash = simple_hash("Total"); -*/ + if(unlikely(io->enabled == CONFIG_ONDEMAND_ONDEMAND && io->delay_counter > 0)) { + io->delay_counter--; + return; } - io->updated = 0; - if(io->filename) { + if(likely(io->filename)) { ff = procfile_reopen(ff, io->filename, NULL, PROCFILE_FLAG_DEFAULT); - if(!ff) return; + if(unlikely(!ff)) { + io->updated = 0; + return; + } ff = procfile_readall(ff); - if(!ff) return; + if(unlikely(!ff)) { + io->updated = 0; + return; + } unsigned long i, lines = procfile_lines(ff); - if(lines < 1) { + if(unlikely(lines < 1)) { error("File '%s' should have 1+ lines.", io->filename); + io->updated = 0; return; } @@ -341,255 +534,142 @@ void cgroup_read_blkio(struct blkio *io) { char *s = procfile_lineword(ff, i, 1); uint32_t hash = simple_hash(s); - if(hash == Read_hash && !strcmp(s, "Read")) - io->Read += strtoull(procfile_lineword(ff, i, 2), NULL, 10); + if(unlikely(hash == Read_hash && !strcmp(s, "Read"))) + io->Read += str2ull(procfile_lineword(ff, i, 2)); - else if(hash == Write_hash && !strcmp(s, "Write")) - io->Write += strtoull(procfile_lineword(ff, i, 2), NULL, 10); + else if(unlikely(hash == Write_hash && !strcmp(s, "Write"))) + io->Write += str2ull(procfile_lineword(ff, i, 2)); /* - else if(hash == Sync_hash && !strcmp(s, "Sync")) - io->Sync += strtoull(procfile_lineword(ff, i, 2), NULL, 10); + else if(unlikely(hash == Sync_hash && !strcmp(s, "Sync"))) + io->Sync += str2ull(procfile_lineword(ff, i, 2)); - else if(hash == Async_hash && !strcmp(s, "Async")) - io->Async += strtoull(procfile_lineword(ff, i, 2), NULL, 10); + else if(unlikely(hash == Async_hash && !strcmp(s, "Async"))) + io->Async += str2ull(procfile_lineword(ff, i, 2)); - else if(hash == Total_hash && !strcmp(s, "Total")) - io->Total += strtoull(procfile_lineword(ff, i, 2), NULL, 10); + else if(unlikely(hash == Total_hash && !strcmp(s, "Total"))) + io->Total += str2ull(procfile_lineword(ff, i, 2)); */ } io->updated = 1; - // fprintf(stderr, "READ '%s': Read: %llu, Write: %llu, Sync: %llu, Async: %llu, Total: %llu\n", io->filename, io->Read, io->Write, io->Sync, io->Async, io->Total); + + if(unlikely(io->enabled == CONFIG_ONDEMAND_ONDEMAND)) { + if(unlikely(io->Read || io->Write)) + io->enabled = CONFIG_ONDEMAND_YES; + else + io->delay_counter = cgroup_recheck_zero_blkio_every_iterations; + } } } -void cgroup_read_memory(struct memory *mem) { +static inline void cgroup_read_memory(struct memory *mem) { static procfile *ff = NULL; - static uint32_t cache_hash = 0; - static uint32_t rss_hash = 0; - static uint32_t rss_huge_hash = 0; - static uint32_t mapped_file_hash = 0; - static uint32_t writeback_hash = 0; - static uint32_t dirty_hash = 0; - static uint32_t swap_hash = 0; - static uint32_t pgpgin_hash = 0; - static uint32_t pgpgout_hash = 0; - static uint32_t pgfault_hash = 0; - static uint32_t pgmajfault_hash = 0; -/* - static uint32_t inactive_anon_hash = 0; - static uint32_t active_anon_hash = 0; - static uint32_t inactive_file_hash = 0; - static uint32_t active_file_hash = 0; - static uint32_t unevictable_hash = 0; - static uint32_t hierarchical_memory_limit_hash = 0; - static uint32_t total_cache_hash = 0; - static uint32_t total_rss_hash = 0; - static uint32_t total_rss_huge_hash = 0; - static uint32_t total_mapped_file_hash = 0; - static uint32_t total_writeback_hash = 0; - static uint32_t total_dirty_hash = 0; - static uint32_t total_swap_hash = 0; - static uint32_t total_pgpgin_hash = 0; - static uint32_t total_pgpgout_hash = 0; - static uint32_t total_pgfault_hash = 0; - static uint32_t total_pgmajfault_hash = 0; - static uint32_t total_inactive_anon_hash = 0; - static uint32_t total_active_anon_hash = 0; - static uint32_t total_inactive_file_hash = 0; - static uint32_t total_active_file_hash = 0; - static uint32_t total_unevictable_hash = 0; -*/ - if(unlikely(cache_hash == 0)) { - cache_hash = simple_hash("cache"); - rss_hash = simple_hash("rss"); - rss_huge_hash = simple_hash("rss_huge"); - mapped_file_hash = simple_hash("mapped_file"); - writeback_hash = simple_hash("writeback"); - dirty_hash = simple_hash("dirty"); - swap_hash = simple_hash("swap"); - pgpgin_hash = simple_hash("pgpgin"); - pgpgout_hash = simple_hash("pgpgout"); - pgfault_hash = simple_hash("pgfault"); - pgmajfault_hash = simple_hash("pgmajfault"); -/* - inactive_anon_hash = simple_hash("inactive_anon"); - active_anon_hash = simple_hash("active_anon"); - inactive_file_hash = simple_hash("inactive_file"); - active_file_hash = simple_hash("active_file"); - unevictable_hash = simple_hash("unevictable"); - hierarchical_memory_limit_hash = simple_hash("hierarchical_memory_limit"); - total_cache_hash = simple_hash("total_cache"); - total_rss_hash = simple_hash("total_rss"); - total_rss_huge_hash = simple_hash("total_rss_huge"); - total_mapped_file_hash = simple_hash("total_mapped_file"); - total_writeback_hash = simple_hash("total_writeback"); - total_dirty_hash = simple_hash("total_dirty"); - total_swap_hash = simple_hash("total_swap"); - total_pgpgin_hash = simple_hash("total_pgpgin"); - total_pgpgout_hash = simple_hash("total_pgpgout"); - total_pgfault_hash = simple_hash("total_pgfault"); - total_pgmajfault_hash = simple_hash("total_pgmajfault"); - total_inactive_anon_hash = simple_hash("total_inactive_anon"); - total_active_anon_hash = simple_hash("total_active_anon"); - total_inactive_file_hash = simple_hash("total_inactive_file"); - total_active_file_hash = simple_hash("total_active_file"); - total_unevictable_hash = simple_hash("total_unevictable"); -*/ - } + // read detailed ram usage + if(likely(mem->filename_detailed)) { + if(unlikely(mem->enabled_detailed == CONFIG_ONDEMAND_ONDEMAND && mem->delay_counter_detailed > 0)) { + mem->delay_counter_detailed--; + goto memory_next; + } - mem->updated = 0; - if(mem->filename) { - ff = procfile_reopen(ff, mem->filename, NULL, PROCFILE_FLAG_DEFAULT); - if(!ff) return; + ff = procfile_reopen(ff, mem->filename_detailed, NULL, PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) { + mem->updated_detailed = 0; + goto memory_next; + } ff = procfile_readall(ff); - if(!ff) return; + if(unlikely(!ff)) { + mem->updated_detailed = 0; + goto memory_next; + } unsigned long i, lines = procfile_lines(ff); - if(lines < 1) { - error("File '%s' should have 1+ lines.", mem->filename); - return; + if(unlikely(lines < 1)) { + error("File '%s' should have 1+ lines.", mem->filename_detailed); + mem->updated_detailed = 0; + goto memory_next; } - for(i = 0; i < lines ; i++) { - char *s = procfile_lineword(ff, i, 0); - uint32_t hash = simple_hash(s); - - if(hash == cache_hash && !strcmp(s, "cache")) - mem->cache = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == rss_hash && !strcmp(s, "rss")) - mem->rss = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == rss_huge_hash && !strcmp(s, "rss_huge")) - mem->rss_huge = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == mapped_file_hash && !strcmp(s, "mapped_file")) - mem->mapped_file = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == writeback_hash && !strcmp(s, "writeback")) - mem->writeback = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == dirty_hash && !strcmp(s, "dirty")) { - mem->dirty = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - mem->has_dirty_swap = 1; - } - - else if(hash == swap_hash && !strcmp(s, "swap")) { - mem->swap = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - mem->has_dirty_swap = 1; - } - - else if(hash == pgpgin_hash && !strcmp(s, "pgpgin")) - mem->pgpgin = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == pgpgout_hash && !strcmp(s, "pgpgout")) - mem->pgpgout = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == pgfault_hash && !strcmp(s, "pgfault")) - mem->pgfault = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == pgmajfault_hash && !strcmp(s, "pgmajfault")) - mem->pgmajfault = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - -/* - else if(hash == inactive_anon_hash && !strcmp(s, "inactive_anon")) - mem->inactive_anon = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == active_anon_hash && !strcmp(s, "active_anon")) - mem->active_anon = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == inactive_file_hash && !strcmp(s, "inactive_file")) - mem->inactive_file = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == active_file_hash && !strcmp(s, "active_file")) - mem->active_file = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == unevictable_hash && !strcmp(s, "unevictable")) - mem->unevictable = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == hierarchical_memory_limit_hash && !strcmp(s, "hierarchical_memory_limit")) - mem->hierarchical_memory_limit = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_cache_hash && !strcmp(s, "total_cache")) - mem->total_cache = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_rss_hash && !strcmp(s, "total_rss")) - mem->total_rss = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_rss_huge_hash && !strcmp(s, "total_rss_huge")) - mem->total_rss_huge = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_mapped_file_hash && !strcmp(s, "total_mapped_file")) - mem->total_mapped_file = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_writeback_hash && !strcmp(s, "total_writeback")) - mem->total_writeback = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_dirty_hash && !strcmp(s, "total_dirty")) - mem->total_dirty = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_swap_hash && !strcmp(s, "total_swap")) - mem->total_swap = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_pgpgin_hash && !strcmp(s, "total_pgpgin")) - mem->total_pgpgin = strtoull(procfile_lineword(ff, i, 1), NULL, 10); - - else if(hash == total_pgpgout_hash && !strcmp(s, "total_pgpgout")) - mem->total_pgpgout = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + if(unlikely(!mem->arl_base)) { + mem->arl_base = arl_create("cgroup/memory", NULL, 60); + + arl_expect(mem->arl_base, "cache", &mem->cache); + arl_expect(mem->arl_base, "rss", &mem->rss); + arl_expect(mem->arl_base, "rss_huge", &mem->rss_huge); + arl_expect(mem->arl_base, "mapped_file", &mem->mapped_file); + arl_expect(mem->arl_base, "writeback", &mem->writeback); + mem->arl_dirty = arl_expect(mem->arl_base, "dirty", &mem->dirty); + mem->arl_swap = arl_expect(mem->arl_base, "swap", &mem->swap); + arl_expect(mem->arl_base, "pgpgin", &mem->pgpgin); + arl_expect(mem->arl_base, "pgpgout", &mem->pgpgout); + arl_expect(mem->arl_base, "pgfault", &mem->pgfault); + arl_expect(mem->arl_base, "pgmajfault", &mem->pgmajfault); + } - else if(hash == total_pgfault_hash && !strcmp(s, "total_pgfault")) - mem->total_pgfault = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + arl_begin(mem->arl_base); - else if(hash == total_pgmajfault_hash && !strcmp(s, "total_pgmajfault")) - mem->total_pgmajfault = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + for(i = 0; i < lines ; i++) { + if(arl_check(mem->arl_base, + procfile_lineword(ff, i, 0), + procfile_lineword(ff, i, 1))) break; + } - else if(hash == total_inactive_anon_hash && !strcmp(s, "total_inactive_anon")) - mem->total_inactive_anon = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + if(unlikely(mem->arl_dirty->flags & ARL_ENTRY_FLAG_FOUND)) + mem->detailed_has_dirty = 1; - else if(hash == total_active_anon_hash && !strcmp(s, "total_active_anon")) - mem->total_active_anon = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + if(unlikely(mem->arl_swap->flags & ARL_ENTRY_FLAG_FOUND)) + mem->detailed_has_swap = 1; - else if(hash == total_inactive_file_hash && !strcmp(s, "total_inactive_file")) - mem->total_inactive_file = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + // fprintf(stderr, "READ: '%s', cache: %llu, rss: %llu, rss_huge: %llu, mapped_file: %llu, writeback: %llu, dirty: %llu, swap: %llu, pgpgin: %llu, pgpgout: %llu, pgfault: %llu, pgmajfault: %llu, inactive_anon: %llu, active_anon: %llu, inactive_file: %llu, active_file: %llu, unevictable: %llu, hierarchical_memory_limit: %llu, total_cache: %llu, total_rss: %llu, total_rss_huge: %llu, total_mapped_file: %llu, total_writeback: %llu, total_dirty: %llu, total_swap: %llu, total_pgpgin: %llu, total_pgpgout: %llu, total_pgfault: %llu, total_pgmajfault: %llu, total_inactive_anon: %llu, total_active_anon: %llu, total_inactive_file: %llu, total_active_file: %llu, total_unevictable: %llu\n", mem->filename, mem->cache, mem->rss, mem->rss_huge, mem->mapped_file, mem->writeback, mem->dirty, mem->swap, mem->pgpgin, mem->pgpgout, mem->pgfault, mem->pgmajfault, mem->inactive_anon, mem->active_anon, mem->inactive_file, mem->active_file, mem->unevictable, mem->hierarchical_memory_limit, mem->total_cache, mem->total_rss, mem->total_rss_huge, mem->total_mapped_file, mem->total_writeback, mem->total_dirty, mem->total_swap, mem->total_pgpgin, mem->total_pgpgout, mem->total_pgfault, mem->total_pgmajfault, mem->total_inactive_anon, mem->total_active_anon, mem->total_inactive_file, mem->total_active_file, mem->total_unevictable); - else if(hash == total_active_file_hash && !strcmp(s, "total_active_file")) - mem->total_active_file = strtoull(procfile_lineword(ff, i, 1), NULL, 10); + mem->updated_detailed = 1; - else if(hash == total_unevictable_hash && !strcmp(s, "total_unevictable")) - mem->total_unevictable = strtoull(procfile_lineword(ff, i, 1), NULL, 10); -*/ + if(unlikely(mem->enabled_detailed == CONFIG_ONDEMAND_ONDEMAND)) { + if(mem->cache || mem->dirty || mem->rss || mem->rss_huge || mem->mapped_file || mem->writeback || mem->swap || mem->pgpgin || mem->pgpgout || mem->pgfault || mem->pgmajfault) + mem->enabled_detailed = CONFIG_ONDEMAND_YES; + else + mem->delay_counter_detailed = cgroup_recheck_zero_mem_detailed_every_iterations; } - - // fprintf(stderr, "READ: '%s', cache: %llu, rss: %llu, rss_huge: %llu, mapped_file: %llu, writeback: %llu, dirty: %llu, swap: %llu, pgpgin: %llu, pgpgout: %llu, pgfault: %llu, pgmajfault: %llu, inactive_anon: %llu, active_anon: %llu, inactive_file: %llu, active_file: %llu, unevictable: %llu, hierarchical_memory_limit: %llu, total_cache: %llu, total_rss: %llu, total_rss_huge: %llu, total_mapped_file: %llu, total_writeback: %llu, total_dirty: %llu, total_swap: %llu, total_pgpgin: %llu, total_pgpgout: %llu, total_pgfault: %llu, total_pgmajfault: %llu, total_inactive_anon: %llu, total_active_anon: %llu, total_inactive_file: %llu, total_active_file: %llu, total_unevictable: %llu\n", mem->filename, mem->cache, mem->rss, mem->rss_huge, mem->mapped_file, mem->writeback, mem->dirty, mem->swap, mem->pgpgin, mem->pgpgout, mem->pgfault, mem->pgmajfault, mem->inactive_anon, mem->active_anon, mem->inactive_file, mem->active_file, mem->unevictable, mem->hierarchical_memory_limit, mem->total_cache, mem->total_rss, mem->total_rss_huge, mem->total_mapped_file, mem->total_writeback, mem->total_dirty, mem->total_swap, mem->total_pgpgin, mem->total_pgpgout, mem->total_pgfault, mem->total_pgmajfault, mem->total_inactive_anon, mem->total_active_anon, mem->total_inactive_file, mem->total_active_file, mem->total_unevictable); - - mem->updated = 1; } - mem->usage_in_bytes_updated = 0; - if(mem->filename_usage_in_bytes) { - if(likely(!read_single_number_file(mem->filename_usage_in_bytes, &mem->usage_in_bytes))) - mem->usage_in_bytes_updated = 1; +memory_next: + + // read usage_in_bytes + if(likely(mem->filename_usage_in_bytes)) { + mem->updated_usage_in_bytes = !read_single_number_file(mem->filename_usage_in_bytes, &mem->usage_in_bytes); + if(unlikely(mem->updated_usage_in_bytes && mem->enabled_usage_in_bytes == CONFIG_ONDEMAND_ONDEMAND && mem->usage_in_bytes)) + mem->enabled_usage_in_bytes = CONFIG_ONDEMAND_YES; } - mem->msw_usage_in_bytes_updated = 0; - if(mem->filename_msw_usage_in_bytes) { - if(likely(!read_single_number_file(mem->filename_msw_usage_in_bytes, &mem->msw_usage_in_bytes))) - mem->msw_usage_in_bytes_updated = 1; + // read msw_usage_in_bytes + if(likely(mem->filename_msw_usage_in_bytes)) { + mem->updated_msw_usage_in_bytes = !read_single_number_file(mem->filename_msw_usage_in_bytes, &mem->msw_usage_in_bytes); + if(unlikely(mem->updated_msw_usage_in_bytes && mem->enabled_msw_usage_in_bytes == CONFIG_ONDEMAND_ONDEMAND && mem->msw_usage_in_bytes)) + mem->enabled_msw_usage_in_bytes = CONFIG_ONDEMAND_YES; } - mem->failcnt_updated = 0; - if(mem->filename_failcnt) { - if(likely(!read_single_number_file(mem->filename_failcnt, &mem->failcnt))) - mem->failcnt_updated = 1; + // read failcnt + if(likely(mem->filename_failcnt)) { + if(unlikely(mem->enabled_failcnt == CONFIG_ONDEMAND_ONDEMAND && mem->delay_counter_failcnt > 0)) { + mem->updated_failcnt = 0; + mem->delay_counter_failcnt--; + } + else { + mem->updated_failcnt = !read_single_number_file(mem->filename_failcnt, &mem->failcnt); + if(unlikely(mem->updated_failcnt && mem->enabled_failcnt == CONFIG_ONDEMAND_ONDEMAND)) { + if(unlikely(!mem->failcnt)) + mem->delay_counter_failcnt = cgroup_recheck_zero_mem_failcnt_every_iterations; + else + mem->enabled_failcnt = CONFIG_ONDEMAND_YES; + } + } } } -void cgroup_read(struct cgroup *cg) { +static inline void cgroup_read(struct cgroup *cg) { debug(D_CGROUP, "reading metrics for cgroups '%s'", cg->id); cgroup_read_cpuacct_stat(&cg->cpuacct_stat); @@ -603,7 +683,7 @@ void cgroup_read(struct cgroup *cg) { cgroup_read_blkio(&cg->io_queued); } -void read_all_cgroups(struct cgroup *root) { +static inline void read_all_cgroups(struct cgroup *root) { debug(D_CGROUP, "reading metrics for all cgroups"); struct cgroup *cg; @@ -618,108 +698,81 @@ void read_all_cgroups(struct cgroup *root) { #define CGROUP_CHARTID_LINE_MAX 1024 -void cgroup_get_chart_id(struct cgroup *cg) { - debug(D_CGROUP, "getting the name of cgroup '%s'", cg->id); +static inline char *cgroup_title_strdupz(const char *s) { + if(!s || !*s) s = "/"; + + if(*s == '/' && s[1] != '\0') s++; + + char *r = strdupz(s); + netdata_fix_chart_name(r); + + return r; +} + +static inline char *cgroup_chart_id_strdupz(const char *s) { + if(!s || !*s) s = "/"; + + if(*s == '/' && s[1] != '\0') s++; + + char *r = strdupz(s); + netdata_fix_chart_id(r); + + return r; +} + +static inline void cgroup_get_chart_name(struct cgroup *cg) { + debug(D_CGROUP, "looking for the name of cgroup '%s' with chart id '%s' and title '%s'", cg->id, cg->chart_id, cg->chart_title); pid_t cgroup_pid; char buffer[CGROUP_CHARTID_LINE_MAX + 1]; - snprintfz(buffer, CGROUP_CHARTID_LINE_MAX, "exec %s '%s'", - config_get("plugin:cgroups", "script to get cgroup names", PLUGINS_DIR "/cgroup-name.sh"), cg->chart_id); + snprintfz(buffer, CGROUP_CHARTID_LINE_MAX, "exec %s '%s'", cgroups_rename_script, cg->chart_id); debug(D_CGROUP, "executing command '%s' for cgroup '%s'", buffer, cg->id); FILE *fp = mypopen(buffer, &cgroup_pid); - if(!fp) { - error("CGROUP: Cannot popen(\"%s\", \"r\").", buffer); - return; - } - debug(D_CGROUP, "reading from command '%s' for cgroup '%s'", buffer, cg->id); - char *s = fgets(buffer, CGROUP_CHARTID_LINE_MAX, fp); - debug(D_CGROUP, "closing command for cgroup '%s'", cg->id); - mypclose(fp, cgroup_pid); - debug(D_CGROUP, "closed command for cgroup '%s'", cg->id); + if(fp) { + // debug(D_CGROUP, "reading from command '%s' for cgroup '%s'", buffer, cg->id); + char *s = fgets(buffer, CGROUP_CHARTID_LINE_MAX, fp); + // debug(D_CGROUP, "closing command for cgroup '%s'", cg->id); + mypclose(fp, cgroup_pid); + // debug(D_CGROUP, "closed command for cgroup '%s'", cg->id); - if(s && *s && *s != '\n') { - debug(D_CGROUP, "cgroup '%s' should be renamed to '%s'", cg->id, s); + if(s && *s && *s != '\n') { + debug(D_CGROUP, "cgroup '%s' should be renamed to '%s'", cg->id, s); - trim(s); + trim(s); - freez(cg->chart_title); - cg->chart_title = strdupz(s); - netdata_fix_chart_name(cg->chart_title); + freez(cg->chart_title); + cg->chart_title = cgroup_title_strdupz(s); - freez(cg->chart_id); - cg->chart_id = strdupz(s); - netdata_fix_chart_id(cg->chart_id); - cg->hash_chart = simple_hash(cg->chart_id); - - debug(D_CGROUP, "cgroup '%s' renamed to '%s' (title: '%s')", cg->id, cg->chart_id, cg->chart_title); + freez(cg->chart_id); + cg->chart_id = cgroup_chart_id_strdupz(s); + cg->hash_chart = simple_hash(cg->chart_id); + } } - else debug(D_CGROUP, "cgroup '%s' is not to be renamed (will be shown as '%s')", cg->id, cg->chart_id); + else + error("CGROUP: Cannot popen(\"%s\", \"r\").", buffer); } -struct cgroup *cgroup_add(const char *id) { - debug(D_CGROUP, "adding cgroup '%s'", id); +static inline struct cgroup *cgroup_add(const char *id) { + if(!id || !*id) id = "/"; + debug(D_CGROUP, "adding to list, cgroup with id '%s'", id); if(cgroup_root_count >= cgroup_root_max) { info("Maximum number of cgroups reached (%d). Not adding cgroup '%s'", cgroup_root_count, id); return NULL; } - int def = cgroup_enable_new_cgroups_detected_at_runtime; - const char *chart_id = id; - if(!*chart_id) { - chart_id = "/"; - - // disable by default the root cgroup - def = 0; - debug(D_CGROUP, "cgroup '%s' is the root container (by default %s)", id, (def)?"enabled":"disabled"); - } - else { - if(*chart_id == '/') chart_id++; - - size_t len = strlen(chart_id); - - // disable by default the parent cgroup - // for known cgroup managers - if(!strcmp(chart_id, "lxc") || - !strcmp(chart_id, "docker") || - !strcmp(chart_id, "libvirt") || - !strcmp(chart_id, "qemu") || - !strcmp(chart_id, "systemd") || - !strcmp(chart_id, "system.slice") || - !strcmp(chart_id, "machine.slice") || - !strcmp(chart_id, "init.scope") || - !strcmp(chart_id, "user") || - !strcmp(chart_id, "system") || - !strcmp(chart_id, "machine") || - // starts with them - (len > 6 && !strncmp(chart_id, "user/", 6)) || - (len > 11 && !strncmp(chart_id, "user.slice/", 11)) || - // ends with them - (len > 5 && !strncmp(&chart_id[len - 5], ".user", 5)) || - (len > 5 && !strncmp(&chart_id[len - 5], ".swap", 5)) || - (len > 6 && !strncmp(&chart_id[len - 6], ".slice", 6)) || - (len > 6 && !strncmp(&chart_id[len - 6], ".mount", 6)) || - (len > 8 && !strncmp(&chart_id[len - 8], ".session", 8)) || - (len > 8 && !strncmp(&chart_id[len - 8], ".service", 8)) || - (len > 10 && !strncmp(&chart_id[len - 10], ".partition", 10)) - ) { - def = 0; - debug(D_CGROUP, "cgroup '%s' is %s (by default)", id, (def)?"enabled":"disabled"); - } - } - + int def = simple_pattern_matches(enabled_cgroup_patterns, id)?cgroup_enable_new_cgroups_detected_at_runtime:0; struct cgroup *cg = callocz(1, sizeof(struct cgroup)); cg->id = strdupz(id); cg->hash = simple_hash(cg->id); - cg->chart_id = strdupz(chart_id); - netdata_fix_chart_id(cg->chart_id); - cg->hash_chart = simple_hash(cg->chart_id); + cg->chart_title = cgroup_title_strdupz(id); - cg->chart_title = strdupz(chart_id); + cg->chart_id = cgroup_chart_id_strdupz(id); + cg->hash_chart = simple_hash(cg->chart_id); if(!cgroup_root) cgroup_root = cg; @@ -732,15 +785,64 @@ struct cgroup *cgroup_add(const char *id) { cgroup_root_count++; - // fix the name by calling the external script - cgroup_get_chart_id(cg); + // fix the chart_id and title by calling the external script + if(simple_pattern_matches(enabled_cgroup_renames, cg->id)) { + + cgroup_get_chart_name(cg); + + debug(D_CGROUP, "cgroup '%s' renamed to '%s' (title: '%s')", cg->id, cg->chart_id, cg->chart_title); + } + else + debug(D_CGROUP, "cgroup '%s' will not be renamed - it matches the list of disabled cgroup renames (will be shown as '%s')", cg->id, cg->chart_id); + + int user_configurable = 1; + + // check if this cgroup should be a systemd service + if(cgroup_enable_systemd_services) { + if(simple_pattern_matches(systemd_services_cgroups, cg->id) || + simple_pattern_matches(systemd_services_cgroups, cg->chart_id)) { + debug(D_CGROUP, "cgroup '%s' with chart id '%s' (title: '%s') matches systemd services cgroups", cg->id, cg->chart_id, cg->chart_title); + + char buffer[CGROUP_CHARTID_LINE_MAX + 1]; + cg->options |= CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE; + + strncpy(buffer, cg->id, CGROUP_CHARTID_LINE_MAX); + char *s = buffer; + + //freez(cg->chart_id); + //cg->chart_id = cgroup_chart_id_strdupz(s); + //cg->hash_chart = simple_hash(cg->chart_id); + + // skip to the last slash + size_t len = strlen(s); + while(len--) if(unlikely(s[len] == '/')) break; + if(len) s = &s[len + 1]; + + // remove extension + len = strlen(s); + while(len--) if(unlikely(s[len] == '.')) break; + if(len) s[len] = '\0'; - debug(D_CGROUP, "adding cgroup '%s' with chart id '%s'", id, chart_id); + freez(cg->chart_title); + cg->chart_title = cgroup_title_strdupz(s); - char option[FILENAME_MAX + 1]; - snprintfz(option, FILENAME_MAX, "enable cgroup %s", cg->chart_title); - cg->enabled = config_get_boolean("plugin:cgroups", option, def); + cg->enabled = 1; + user_configurable = 0; + debug(D_CGROUP, "cgroup '%s' renamed to '%s' (title: '%s')", cg->id, cg->chart_id, cg->chart_title); + } + else + debug(D_CGROUP, "cgroup '%s' with chart id '%s' (title: '%s') does not match systemd services groups", cg->id, cg->chart_id, cg->chart_title); + } + + if(user_configurable) { + // allow the user to enable/disable this individualy + char option[FILENAME_MAX + 1]; + snprintfz(option, FILENAME_MAX, "enable cgroup %s", cg->chart_title); + cg->enabled = (char) config_get_boolean("plugin:cgroups", option, def); + } + + // detect duplicate cgroups if(cg->enabled) { struct cgroup *t; for (t = cgroup_root; t; t = t->next) { @@ -767,36 +869,45 @@ struct cgroup *cgroup_add(const char *id) { } } - debug(D_CGROUP, "Added cgroup '%s' with chart id '%s' and title '%s' as %s (default was %s)", cg->id, cg->chart_id, cg->chart_title, (cg->enabled)?"enabled":"disabled", (def)?"enabled":"disabled"); + debug(D_CGROUP, "ADDED CGROUP: '%s' with chart id '%s' and title '%s' as %s (default was %s)", cg->id, cg->chart_id, cg->chart_title, (cg->enabled)?"enabled":"disabled", (def)?"enabled":"disabled"); return cg; } -void cgroup_free(struct cgroup *cg) { +static inline void cgroup_free(struct cgroup *cg) { debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available"); freez(cg->cpuacct_usage.cpu_percpu); freez(cg->cpuacct_stat.filename); freez(cg->cpuacct_usage.filename); - freez(cg->memory.filename); + + arl_free(cg->memory.arl_base); + freez(cg->memory.filename_detailed); + freez(cg->memory.filename_failcnt); + freez(cg->memory.filename_usage_in_bytes); + freez(cg->memory.filename_msw_usage_in_bytes); + freez(cg->io_service_bytes.filename); freez(cg->io_serviced.filename); + freez(cg->throttle_io_service_bytes.filename); freez(cg->throttle_io_serviced.filename); + freez(cg->io_merged.filename); freez(cg->io_queued.filename); freez(cg->id); freez(cg->chart_id); freez(cg->chart_title); + freez(cg); cgroup_root_count--; } // find if a given cgroup exists -struct cgroup *cgroup_find(const char *id) { +static inline struct cgroup *cgroup_find(const char *id) { debug(D_CGROUP, "searching for cgroup '%s'", id); uint32_t hash = simple_hash(id); @@ -807,7 +918,7 @@ struct cgroup *cgroup_find(const char *id) { break; } - debug(D_CGROUP, "cgroup_find('%s') %s", id, (cg)?"found":"not found"); + debug(D_CGROUP, "cgroup '%s' %s in memory", id, (cg)?"found":"not found"); return cg; } @@ -815,7 +926,7 @@ struct cgroup *cgroup_find(const char *id) { // detect running cgroups // callback for find_file_in_subdirs() -void found_subdir_in_dir(const char *dir) { +static inline void found_subdir_in_dir(const char *dir) { debug(D_CGROUP, "examining cgroup dir '%s'", dir); struct cgroup *cg = cgroup_find(dir); @@ -833,21 +944,24 @@ void found_subdir_in_dir(const char *dir) { return; } } - debug(D_CGROUP, "will add dir '%s' as cgroup", dir); + // debug(D_CGROUP, "will add dir '%s' as cgroup", dir); cg = cgroup_add(dir); } if(cg) cg->available = 1; } -int find_dir_in_subdirs(const char *base, const char *this, void (*callback)(const char *)) { - debug(D_CGROUP, "searching for directories in '%s'", base); +static inline int find_dir_in_subdirs(const char *base, const char *this, void (*callback)(const char *)) { + if(!this) this = base; + debug(D_CGROUP, "searching for directories in '%s' (base '%s')", this?this:"", base); + + size_t dirlen = strlen(this), baselen = strlen(base); int ret = -1; int enabled = -1; - if(!this) this = base; - size_t dirlen = strlen(this), baselen = strlen(base); + const char *relative_path = &this[baselen]; + if(!*relative_path) relative_path = "/"; DIR *dir = opendir(this); if(!dir) { @@ -867,13 +981,13 @@ int find_dir_in_subdirs(const char *base, const char *this, void (*callback)(con )) continue; - debug(D_CGROUP, "examining '%s/%s'", this, de->d_name); - if(de->d_type == DT_DIR) { if(enabled == -1) { const char *r = relative_path; if(*r == '\0') r = "/"; - else if (*r == '/') r++; + + // do not decent in directories we are not interested + int def = simple_pattern_matches(enabled_cgroup_paths, r); // we check for this option here // so that the config will not have settings @@ -881,7 +995,7 @@ int find_dir_in_subdirs(const char *base, const char *this, void (*callback)(con char option[FILENAME_MAX + 1]; snprintfz(option, FILENAME_MAX, "search for cgroups under %s", r); option[FILENAME_MAX] = '\0'; - enabled = config_get_boolean("plugin:cgroups", option, 1); + enabled = config_get_boolean("plugin:cgroups", option, def); } if(enabled) { @@ -900,7 +1014,7 @@ int find_dir_in_subdirs(const char *base, const char *this, void (*callback)(con return ret; } -void mark_all_cgroups_as_not_available() { +static inline void mark_all_cgroups_as_not_available() { debug(D_CGROUP, "marking all cgroups as not available"); struct cgroup *cg; @@ -911,7 +1025,7 @@ void mark_all_cgroups_as_not_available() { } } -void cleanup_all_cgroups() { +static inline void cleanup_all_cgroups() { struct cgroup *cg = cgroup_root, *last = NULL; for(; cg ;) { @@ -948,36 +1062,45 @@ void cleanup_all_cgroups() { } } -void find_all_cgroups() { +static inline void find_all_cgroups() { debug(D_CGROUP, "searching for cgroups"); mark_all_cgroups_as_not_available(); if(cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_usage) { - if (find_dir_in_subdirs(cgroup_cpuacct_base, NULL, found_subdir_in_dir) == -1) { - cgroup_enable_cpuacct_stat = cgroup_enable_cpuacct_usage = 0; - error("disabled cgroup cpu statistics."); + if(find_dir_in_subdirs(cgroup_cpuacct_base, NULL, found_subdir_in_dir) == -1) { + cgroup_enable_cpuacct_stat = + cgroup_enable_cpuacct_usage = CONFIG_ONDEMAND_NO; + error("disabled CGROUP cpu statistics."); } } - if(cgroup_enable_blkio) { - if (find_dir_in_subdirs(cgroup_blkio_base, NULL, found_subdir_in_dir) == -1) { - cgroup_enable_blkio = 0; - error("disabled cgroup blkio statistics."); + if(cgroup_enable_blkio_io || cgroup_enable_blkio_ops || cgroup_enable_blkio_throttle_io || cgroup_enable_blkio_throttle_ops || cgroup_enable_blkio_merged_ops || cgroup_enable_blkio_queued_ops) { + if(find_dir_in_subdirs(cgroup_blkio_base, NULL, found_subdir_in_dir) == -1) { + cgroup_enable_blkio_io = + cgroup_enable_blkio_ops = + cgroup_enable_blkio_throttle_io = + cgroup_enable_blkio_throttle_ops = + cgroup_enable_blkio_merged_ops = + cgroup_enable_blkio_queued_ops = CONFIG_ONDEMAND_NO; + error("disabled CGROUP blkio statistics."); } } - if(cgroup_enable_memory) { + if(cgroup_enable_memory || cgroup_enable_detailed_memory || cgroup_enable_swap || cgroup_enable_memory_failcnt) { if(find_dir_in_subdirs(cgroup_memory_base, NULL, found_subdir_in_dir) == -1) { - cgroup_enable_memory = 0; - error("disabled cgroup memory statistics."); + cgroup_enable_memory = + cgroup_enable_detailed_memory = + cgroup_enable_swap = + cgroup_enable_memory_failcnt = CONFIG_ONDEMAND_NO; + error("disabled CGROUP memory statistics."); } } - if(cgroup_enable_devices) { + if(cgroup_search_in_devices) { if(find_dir_in_subdirs(cgroup_devices_base, NULL, found_subdir_in_dir) == -1) { - cgroup_enable_devices = 0; - error("disabled cgroup devices statistics."); + cgroup_search_in_devices = 0; + error("disabled CGROUP devices statistics."); } } @@ -997,100 +1120,136 @@ void find_all_cgroups() { // check for newly added cgroups // and update the filenames they read char filename[FILENAME_MAX + 1]; - if(cgroup_enable_cpuacct_stat && !cg->cpuacct_stat.filename) { + if(unlikely(cgroup_enable_cpuacct_stat && !cg->cpuacct_stat.filename)) { snprintfz(filename, FILENAME_MAX, "%s%s/cpuacct.stat", cgroup_cpuacct_base, cg->id); - if(stat(filename, &buf) != -1) { + if(likely(stat(filename, &buf) != -1)) { cg->cpuacct_stat.filename = strdupz(filename); + cg->cpuacct_stat.enabled = cgroup_enable_cpuacct_stat; debug(D_CGROUP, "cpuacct.stat filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_stat.filename); } - else debug(D_CGROUP, "cpuacct.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "cpuacct.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename); } - if(cgroup_enable_cpuacct_usage && !cg->cpuacct_usage.filename) { + + if(unlikely(cgroup_enable_cpuacct_usage && !cg->cpuacct_usage.filename && !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))) { snprintfz(filename, FILENAME_MAX, "%s%s/cpuacct.usage_percpu", cgroup_cpuacct_base, cg->id); - if(stat(filename, &buf) != -1) { + if(likely(stat(filename, &buf) != -1)) { cg->cpuacct_usage.filename = strdupz(filename); + cg->cpuacct_usage.enabled = cgroup_enable_cpuacct_usage; debug(D_CGROUP, "cpuacct.usage_percpu filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_usage.filename); } - else debug(D_CGROUP, "cpuacct.usage_percpu file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "cpuacct.usage_percpu file for cgroup '%s': '%s' does not exist.", cg->id, filename); } - if(cgroup_enable_memory && !cg->memory.filename) { + + if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory_without_cache) && !cg->memory.filename_detailed && (cgroup_used_memory_without_cache || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) { snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_memory_base, cg->id); - if(stat(filename, &buf) != -1) { - cg->memory.filename = strdupz(filename); - debug(D_CGROUP, "memory.stat filename for cgroup '%s': '%s'", cg->id, cg->memory.filename); + if(likely(stat(filename, &buf) != -1)) { + cg->memory.filename_detailed = strdupz(filename); + cg->memory.enabled_detailed = (cgroup_enable_detailed_memory == CONFIG_ONDEMAND_YES)?CONFIG_ONDEMAND_YES:CONFIG_ONDEMAND_ONDEMAND; + debug(D_CGROUP, "memory.stat filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_detailed); } - else debug(D_CGROUP, "memory.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "memory.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + if(unlikely(cgroup_enable_memory && !cg->memory.filename_usage_in_bytes)) { snprintfz(filename, FILENAME_MAX, "%s%s/memory.usage_in_bytes", cgroup_memory_base, cg->id); - if(stat(filename, &buf) != -1) { + if(likely(stat(filename, &buf) != -1)) { cg->memory.filename_usage_in_bytes = strdupz(filename); + cg->memory.enabled_usage_in_bytes = cgroup_enable_memory; debug(D_CGROUP, "memory.usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_usage_in_bytes); } - else debug(D_CGROUP, "memory.usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "memory.usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + if(unlikely(cgroup_enable_swap && !cg->memory.filename_msw_usage_in_bytes)) { snprintfz(filename, FILENAME_MAX, "%s%s/memory.msw_usage_in_bytes", cgroup_memory_base, cg->id); - if(stat(filename, &buf) != -1) { + if(likely(stat(filename, &buf) != -1)) { cg->memory.filename_msw_usage_in_bytes = strdupz(filename); + cg->memory.enabled_msw_usage_in_bytes = cgroup_enable_swap; debug(D_CGROUP, "memory.msw_usage_in_bytes filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_msw_usage_in_bytes); } - else debug(D_CGROUP, "memory.msw_usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "memory.msw_usage_in_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + if(unlikely(cgroup_enable_memory_failcnt && !cg->memory.filename_failcnt)) { snprintfz(filename, FILENAME_MAX, "%s%s/memory.failcnt", cgroup_memory_base, cg->id); - if(stat(filename, &buf) != -1) { + if(likely(stat(filename, &buf) != -1)) { cg->memory.filename_failcnt = strdupz(filename); + cg->memory.enabled_failcnt = cgroup_enable_memory_failcnt; debug(D_CGROUP, "memory.failcnt filename for cgroup '%s': '%s'", cg->id, cg->memory.filename_failcnt); } - else debug(D_CGROUP, "memory.failcnt file for cgroup '%s': '%s' does not exist.", cg->id, filename); - } - if(cgroup_enable_blkio) { - if(!cg->io_service_bytes.filename) { - snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes", cgroup_blkio_base, cg->id); - if(stat(filename, &buf) != -1) { - cg->io_service_bytes.filename = strdupz(filename); - debug(D_CGROUP, "io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename); - } - else debug(D_CGROUP, "io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "memory.failcnt file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + + if(unlikely(cgroup_enable_blkio_io && !cg->io_service_bytes.filename)) { + snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes", cgroup_blkio_base, cg->id); + if(likely(stat(filename, &buf) != -1)) { + cg->io_service_bytes.filename = strdupz(filename); + cg->io_service_bytes.enabled = cgroup_enable_blkio_io; + debug(D_CGROUP, "io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename); } - if(!cg->io_serviced.filename) { - snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced", cgroup_blkio_base, cg->id); - if(stat(filename, &buf) != -1) { - cg->io_serviced.filename = strdupz(filename); - debug(D_CGROUP, "io_serviced filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename); - } - else debug(D_CGROUP, "io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + + if(unlikely(cgroup_enable_blkio_ops && !cg->io_serviced.filename)) { + snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced", cgroup_blkio_base, cg->id); + if(likely(stat(filename, &buf) != -1)) { + cg->io_serviced.filename = strdupz(filename); + cg->io_serviced.enabled = cgroup_enable_blkio_ops; + debug(D_CGROUP, "io_serviced filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename); } - if(!cg->throttle_io_service_bytes.filename) { - snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes", cgroup_blkio_base, cg->id); - if(stat(filename, &buf) != -1) { - cg->throttle_io_service_bytes.filename = strdupz(filename); - debug(D_CGROUP, "throttle_io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename); - } - else debug(D_CGROUP, "throttle_io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + + if(unlikely(cgroup_enable_blkio_throttle_io && !cg->throttle_io_service_bytes.filename)) { + snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes", cgroup_blkio_base, cg->id); + if(likely(stat(filename, &buf) != -1)) { + cg->throttle_io_service_bytes.filename = strdupz(filename); + cg->throttle_io_service_bytes.enabled = cgroup_enable_blkio_throttle_io; + debug(D_CGROUP, "throttle_io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename); } - if(!cg->throttle_io_serviced.filename) { - snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced", cgroup_blkio_base, cg->id); - if(stat(filename, &buf) != -1) { - cg->throttle_io_serviced.filename = strdupz(filename); - debug(D_CGROUP, "throttle_io_serviced filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename); - } - else debug(D_CGROUP, "throttle_io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "throttle_io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + + if(unlikely(cgroup_enable_blkio_throttle_ops && !cg->throttle_io_serviced.filename)) { + snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced", cgroup_blkio_base, cg->id); + if(likely(stat(filename, &buf) != -1)) { + cg->throttle_io_serviced.filename = strdupz(filename); + cg->throttle_io_serviced.enabled = cgroup_enable_blkio_throttle_ops; + debug(D_CGROUP, "throttle_io_serviced filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename); } - if(!cg->io_merged.filename) { - snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged", cgroup_blkio_base, cg->id); - if(stat(filename, &buf) != -1) { - cg->io_merged.filename = strdupz(filename); - debug(D_CGROUP, "io_merged filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename); - } - else debug(D_CGROUP, "io_merged file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "throttle_io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + + if(unlikely(cgroup_enable_blkio_merged_ops && !cg->io_merged.filename)) { + snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged", cgroup_blkio_base, cg->id); + if(likely(stat(filename, &buf) != -1)) { + cg->io_merged.filename = strdupz(filename); + cg->io_merged.enabled = cgroup_enable_blkio_merged_ops; + debug(D_CGROUP, "io_merged filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename); } - if(!cg->io_queued.filename) { - snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued", cgroup_blkio_base, cg->id); - if(stat(filename, &buf) != -1) { - cg->io_queued.filename = strdupz(filename); - debug(D_CGROUP, "io_queued filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename); - } - else debug(D_CGROUP, "io_queued file for cgroup '%s': '%s' does not exist.", cg->id, filename); + else + debug(D_CGROUP, "io_merged file for cgroup '%s': '%s' does not exist.", cg->id, filename); + } + + if(unlikely(cgroup_enable_blkio_queued_ops && !cg->io_queued.filename)) { + snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued", cgroup_blkio_base, cg->id); + if(likely(stat(filename, &buf) != -1)) { + cg->io_queued.filename = strdupz(filename); + cg->io_queued.enabled = cgroup_enable_blkio_queued_ops; + debug(D_CGROUP, "io_queued filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename); } + else + debug(D_CGROUP, "io_queued file for cgroup '%s': '%s' does not exist.", cg->id, filename); } } @@ -1103,313 +1262,805 @@ void find_all_cgroups() { #define CHART_TITLE_MAX 300 +void update_services_charts(int update_every, + int do_cpu, + int do_mem_usage, + int do_mem_detailed, + int do_mem_failcnt, + int do_swap_usage, + int do_io, + int do_io_ops, + int do_throttle_io, + int do_throttle_ops, + int do_queued_ops, + int do_merged_ops +) { + static RRDSET + *st_cpu = NULL, + *st_mem_usage = NULL, + *st_mem_failcnt = NULL, + *st_swap_usage = NULL, + + *st_mem_detailed_cache = NULL, + *st_mem_detailed_rss = NULL, + *st_mem_detailed_mapped = NULL, + *st_mem_detailed_writeback = NULL, + *st_mem_detailed_pgfault = NULL, + *st_mem_detailed_pgmajfault = NULL, + *st_mem_detailed_pgpgin = NULL, + *st_mem_detailed_pgpgout = NULL, + + *st_io_read = NULL, + *st_io_serviced_read = NULL, + *st_throttle_io_read = NULL, + *st_throttle_ops_read = NULL, + *st_queued_ops_read = NULL, + *st_merged_ops_read = NULL, + + *st_io_write = NULL, + *st_io_serviced_write = NULL, + *st_throttle_io_write = NULL, + *st_throttle_ops_write = NULL, + *st_queued_ops_write = NULL, + *st_merged_ops_write = NULL; + + // create the charts + + if(likely(do_cpu)) { + if(unlikely(!st_cpu)) { + char title[CHART_TITLE_MAX + 1]; + + st_cpu = rrdset_find_bytype("services", "cpu"); + if(likely(!st_cpu)) { + snprintfz(title, CHART_TITLE_MAX, "Systemd Services CPU utilization (%d%% = %d core%s)", (processors * 100), processors, (processors > 1) ? "s" : ""); + st_cpu = rrdset_create("services", "cpu", NULL, "cpu", "services.cpu", title, "%", CHART_PRIORITY_SYSTEMD_SERVICES, update_every, RRDSET_TYPE_STACKED); + } + } + else + rrdset_next(st_cpu); + } + + if(likely(do_mem_usage)) { + if(unlikely(!st_mem_usage)) { + st_mem_usage = rrdset_find_bytype("services", "mem_usage"); + if(likely(!st_mem_usage)) + st_mem_usage = rrdset_create("services", "mem_usage", NULL, "mem", "services.mem_usage", (cgroup_used_memory_without_cache)?"Systemd Services Used Memory without Cache":"Systemd Services Used Memory", "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 10, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_usage); + } + + if(likely(do_mem_detailed)) { + if(unlikely(!st_mem_detailed_rss)) { + st_mem_detailed_rss = rrdset_find_bytype("services", "mem_rss"); + if(likely(!st_mem_detailed_rss)) + st_mem_detailed_rss = rrdset_create("services", "mem_rss", NULL, "mem", "services.mem_rss", "Systemd Services RSS Memory", "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 20, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_rss); + + if(unlikely(!st_mem_detailed_mapped)) { + st_mem_detailed_mapped = rrdset_find_bytype("services", "mem_mapped"); + if(likely(!st_mem_detailed_mapped)) + st_mem_detailed_mapped = rrdset_create("services", "mem_mapped", NULL, "mem", "services.mem_mapped", "Systemd Services Mapped Memory", "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 30, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_mapped); + + if(unlikely(!st_mem_detailed_cache)) { + st_mem_detailed_cache = rrdset_find_bytype("services", "mem_cache"); + if(likely(!st_mem_detailed_cache)) + st_mem_detailed_cache = rrdset_create("services", "mem_cache", NULL, "mem", "services.mem_cache", "Systemd Services Cache Memory", "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 40, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_cache); + + if(unlikely(!st_mem_detailed_writeback)) { + st_mem_detailed_writeback = rrdset_find_bytype("services", "mem_writeback"); + if(likely(!st_mem_detailed_writeback)) + st_mem_detailed_writeback = rrdset_create("services", "mem_writeback", NULL, "mem", "services.mem_writeback", "Systemd Services Writeback Memory", "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 50, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_writeback); + + if(unlikely(!st_mem_detailed_pgfault)) { + st_mem_detailed_pgfault = rrdset_find_bytype("services", "mem_pgfault"); + if(likely(!st_mem_detailed_pgfault)) + st_mem_detailed_pgfault = rrdset_create("services", "mem_pgfault", NULL, "mem", "services.mem_pgfault", "Systemd Services Memory Minor Page Faults", "MB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 60, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_pgfault); + + if(unlikely(!st_mem_detailed_pgmajfault)) { + st_mem_detailed_pgmajfault = rrdset_find_bytype("services", "mem_pgmajfault"); + if(likely(!st_mem_detailed_pgmajfault)) + st_mem_detailed_pgmajfault = rrdset_create("services", "mem_pgmajfault", NULL, "mem", "services.mem_pgmajfault", "Systemd Services Memory Major Page Faults", "MB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 70, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_pgmajfault); + + if(unlikely(!st_mem_detailed_pgpgin)) { + st_mem_detailed_pgpgin = rrdset_find_bytype("services", "mem_pgpgin"); + if(likely(!st_mem_detailed_pgpgin)) + st_mem_detailed_pgpgin = rrdset_create("services", "mem_pgpgin", NULL, "mem", "services.mem_pgpgin", "Systemd Services Memory Charging Activity", "MB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 80, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_pgpgin); + + if(unlikely(!st_mem_detailed_pgpgout)) { + st_mem_detailed_pgpgout = rrdset_find_bytype("services", "mem_pgpgout"); + if(likely(!st_mem_detailed_pgpgout)) + st_mem_detailed_pgpgout = rrdset_create("services", "mem_pgpgout", NULL, "mem", "services.mem_pgpgout", "Systemd Services Memory Uncharging Activity", "MB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 90, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_detailed_pgpgout); + } + + if(likely(do_mem_failcnt)) { + if(unlikely(!st_mem_failcnt)) { + st_mem_failcnt = rrdset_find_bytype("services", "mem_failcnt"); + if(likely(!st_mem_failcnt)) + st_mem_failcnt = rrdset_create("services", "mem_failcnt", NULL, "mem", "services.mem_failcnt", "Systemd Services Memory Limit Failures", "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 110, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_mem_failcnt); + } + + if(likely(do_swap_usage)) { + if(unlikely(!st_swap_usage)) { + st_swap_usage = rrdset_find_bytype("services", "swap_usage"); + if(likely(!st_swap_usage)) + st_swap_usage = rrdset_create("services", "swap_usage", NULL, "swap", "services.swap_usage", "Systemd Services Swap Memory Used", "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 100, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_swap_usage); + } + + if(likely(do_io)) { + if(unlikely(!st_io_read)) { + st_io_read = rrdset_find_bytype("services", "io_read"); + if(likely(!st_io_read)) + st_io_read = rrdset_create("services", "io_read", NULL, "disk", "services.io_read", "Systemd Services Disk Read Bandwidth", "KB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 120, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_io_read); + + if(unlikely(!st_io_write)) { + st_io_write = rrdset_find_bytype("services", "io_write"); + if(likely(!st_io_write)) + st_io_write = rrdset_create("services", "io_write", NULL, "disk", "services.io_write", "Systemd Services Disk Write Bandwidth", "KB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 130, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_io_write); + } + + if(likely(do_io_ops)) { + if(unlikely(!st_io_serviced_read)) { + st_io_serviced_read = rrdset_find_bytype("services", "io_ops_read"); + if(likely(!st_io_serviced_read)) + st_io_serviced_read = rrdset_create("services", "io_ops_read", NULL, "disk", "services.io_ops_read", "Systemd Services Disk Read Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 140, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_io_serviced_read); + + if(unlikely(!st_io_serviced_write)) { + st_io_serviced_write = rrdset_find_bytype("services", "io_ops_write"); + if(likely(!st_io_serviced_write)) + st_io_serviced_write = rrdset_create("services", "io_ops_write", NULL, "disk", "services.io_ops_write", "Systemd Services Disk Write Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 150, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_io_serviced_write); + } + + if(likely(do_throttle_io)) { + if(unlikely(!st_throttle_io_read)) { + st_throttle_io_read = rrdset_find_bytype("services", "throttle_io_read"); + if(likely(!st_throttle_io_read)) + st_throttle_io_read = rrdset_create("services", "throttle_io_read", NULL, "disk", "services.throttle_io_read", "Systemd Services Throttle Disk Read Bandwidth", "KB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 160, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_throttle_io_read); + + if(unlikely(!st_throttle_io_write)) { + st_throttle_io_write = rrdset_find_bytype("services", "throttle_io_write"); + if(likely(!st_throttle_io_write)) + st_throttle_io_write = rrdset_create("services", "throttle_io_write", NULL, "disk", "services.throttle_io_write", "Systemd Services Throttle Disk Write Bandwidth", "KB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 170, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_throttle_io_write); + } + + if(likely(do_throttle_ops)) { + if(unlikely(!st_throttle_ops_read)) { + st_throttle_ops_read = rrdset_find_bytype("services", "throttle_io_ops_read"); + if(likely(!st_throttle_ops_read)) + st_throttle_ops_read = rrdset_create("services", "throttle_io_ops_read", NULL, "disk", "services.throttle_io_ops_read", "Systemd Services Throttle Disk Read Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 180, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_throttle_ops_read); + + if(unlikely(!st_throttle_ops_write)) { + st_throttle_ops_write = rrdset_find_bytype("services", "throttle_io_ops_write"); + if(likely(!st_throttle_ops_write)) + st_throttle_ops_write = rrdset_create("services", "throttle_io_ops_write", NULL, "disk", "services.throttle_io_ops_write", "Systemd Services Throttle Disk Write Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 190, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_throttle_ops_write); + } + + if(likely(do_queued_ops)) { + if(unlikely(!st_queued_ops_read)) { + st_queued_ops_read = rrdset_find_bytype("services", "queued_io_ops_read"); + if(likely(!st_queued_ops_read)) + st_queued_ops_read = rrdset_create("services", "queued_io_ops_read", NULL, "disk", "services.queued_io_ops_read", "Systemd Services Queued Disk Read Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 200, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_queued_ops_read); + + if(unlikely(!st_queued_ops_write)) { + st_queued_ops_write = rrdset_find_bytype("services", "queued_io_ops_write"); + if(likely(!st_queued_ops_write)) + st_queued_ops_write = rrdset_create("services", "queued_io_ops_write", NULL, "disk", "services.queued_io_ops_write", "Systemd Services Queued Disk Write Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 210, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_queued_ops_write); + } + + if(likely(do_merged_ops)) { + if(unlikely(!st_merged_ops_read)) { + st_merged_ops_read = rrdset_find_bytype("services", "merged_io_ops_read"); + if(likely(!st_merged_ops_read)) + st_merged_ops_read = rrdset_create("services", "merged_io_ops_read", NULL, "disk", "services.merged_io_ops_read", "Systemd Services Merged Disk Read Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 220, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_merged_ops_read); + + if(unlikely(!st_merged_ops_write)) { + st_merged_ops_write = rrdset_find_bytype("services", "merged_io_ops_write"); + if(likely(!st_merged_ops_write)) + st_merged_ops_write = rrdset_create("services", "merged_io_ops_write", NULL, "disk", "services.merged_io_ops_write", "Systemd Services Merged Disk Write Operations", "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 230, update_every, RRDSET_TYPE_STACKED); + } + else + rrdset_next(st_merged_ops_write); + } + + // update the values + struct cgroup *cg; + for(cg = cgroup_root; cg ; cg = cg->next) { + if(unlikely(!cg->available || !cg->enabled || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))) + continue; + + if(likely(do_cpu && cg->cpuacct_stat.updated)) { + if(unlikely(!cg->rd_cpu)) + cg->rd_cpu = rrddim_add(st_cpu, cg->chart_id, cg->chart_title, 100, hz, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_cpu, cg->rd_cpu, cg->cpuacct_stat.user + cg->cpuacct_stat.system); + } + + if(likely(do_mem_usage && cg->memory.updated_usage_in_bytes)) { + if(unlikely(!cg->rd_mem_usage)) + cg->rd_mem_usage = rrddim_add(st_mem_usage, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + + rrddim_set_by_pointer(st_mem_usage, cg->rd_mem_usage, cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.cache:0)); + } + + if(likely(do_mem_detailed && cg->memory.updated_detailed)) { + if(unlikely(!cg->rd_mem_detailed_rss)) + cg->rd_mem_detailed_rss = rrddim_add(st_mem_detailed_rss, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_rss, cg->rd_mem_detailed_rss, cg->memory.rss + cg->memory.rss_huge); + + if(unlikely(!cg->rd_mem_detailed_mapped)) + cg->rd_mem_detailed_mapped = rrddim_add(st_mem_detailed_mapped, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_mapped, cg->rd_mem_detailed_mapped, cg->memory.mapped_file); + + if(unlikely(!cg->rd_mem_detailed_cache)) + cg->rd_mem_detailed_cache = rrddim_add(st_mem_detailed_cache, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_cache, cg->rd_mem_detailed_cache, cg->memory.cache); + + if(unlikely(!cg->rd_mem_detailed_writeback)) + cg->rd_mem_detailed_writeback = rrddim_add(st_mem_detailed_writeback, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_writeback, cg->rd_mem_detailed_writeback, cg->memory.writeback); + + if(unlikely(!cg->rd_mem_detailed_pgfault)) + cg->rd_mem_detailed_pgfault = rrddim_add(st_mem_detailed_pgfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgfault, cg->rd_mem_detailed_pgfault, cg->memory.pgfault); + + if(unlikely(!cg->rd_mem_detailed_pgmajfault)) + cg->rd_mem_detailed_pgmajfault = rrddim_add(st_mem_detailed_pgmajfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgmajfault, cg->rd_mem_detailed_pgmajfault, cg->memory.pgmajfault); + + if(unlikely(!cg->rd_mem_detailed_pgpgin)) + cg->rd_mem_detailed_pgpgin = rrddim_add(st_mem_detailed_pgpgin, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgpgin, cg->rd_mem_detailed_pgpgin, cg->memory.pgpgin); + + if(unlikely(!cg->rd_mem_detailed_pgpgout)) + cg->rd_mem_detailed_pgpgout = rrddim_add(st_mem_detailed_pgpgout, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgpgout, cg->rd_mem_detailed_pgpgout, cg->memory.pgpgout); + } + + if(likely(do_mem_failcnt && cg->memory.updated_failcnt)) { + if(unlikely(!cg->rd_mem_failcnt)) + cg->rd_mem_failcnt = rrddim_add(st_mem_failcnt, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_mem_failcnt, cg->rd_mem_failcnt, cg->memory.failcnt); + } + + if(likely(do_swap_usage && cg->memory.updated_msw_usage_in_bytes)) { + if(unlikely(!cg->rd_swap_usage)) + cg->rd_swap_usage = rrddim_add(st_swap_usage, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + + rrddim_set_by_pointer(st_swap_usage, cg->rd_swap_usage, cg->memory.msw_usage_in_bytes); + } + + if(likely(do_io && cg->io_service_bytes.updated)) { + if(unlikely(!cg->rd_io_service_bytes_read)) + cg->rd_io_service_bytes_read = rrddim_add(st_io_read, cg->chart_id, cg->chart_title, 1, 1024, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_io_read, cg->rd_io_service_bytes_read, cg->io_service_bytes.Read); + + if(unlikely(!cg->rd_io_service_bytes_write)) + cg->rd_io_service_bytes_write = rrddim_add(st_io_write, cg->chart_id, cg->chart_title, 1, 1024, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_io_write, cg->rd_io_service_bytes_write, cg->io_service_bytes.Write); + } + + if(likely(do_io_ops && cg->io_serviced.updated)) { + if(unlikely(!cg->rd_io_serviced_read)) + cg->rd_io_serviced_read = rrddim_add(st_io_serviced_read, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_io_serviced_read, cg->rd_io_serviced_read, cg->io_serviced.Read); + + if(unlikely(!cg->rd_io_serviced_write)) + cg->rd_io_serviced_write = rrddim_add(st_io_serviced_write, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_io_serviced_write, cg->rd_io_serviced_write, cg->io_serviced.Write); + } + + if(likely(do_throttle_io && cg->throttle_io_service_bytes.updated)) { + if(unlikely(!cg->rd_throttle_io_read)) + cg->rd_throttle_io_read = rrddim_add(st_throttle_io_read, cg->chart_id, cg->chart_title, 1, 1024, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_throttle_io_read, cg->rd_throttle_io_read, cg->throttle_io_service_bytes.Read); + + if(unlikely(!cg->rd_throttle_io_write)) + cg->rd_throttle_io_write = rrddim_add(st_throttle_io_write, cg->chart_id, cg->chart_title, 1, 1024, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_throttle_io_write, cg->rd_throttle_io_write, cg->throttle_io_service_bytes.Write); + } + + if(likely(do_throttle_ops && cg->throttle_io_serviced.updated)) { + if(unlikely(!cg->rd_throttle_io_serviced_read)) + cg->rd_throttle_io_serviced_read = rrddim_add(st_throttle_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_throttle_ops_read, cg->rd_throttle_io_serviced_read, cg->throttle_io_serviced.Read); + + if(unlikely(!cg->rd_throttle_io_serviced_write)) + cg->rd_throttle_io_serviced_write = rrddim_add(st_throttle_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_throttle_ops_write, cg->rd_throttle_io_serviced_write, cg->throttle_io_serviced.Write); + } + + if(likely(do_queued_ops && cg->io_queued.updated)) { + if(unlikely(!cg->rd_io_queued_read)) + cg->rd_io_queued_read = rrddim_add(st_queued_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_queued_ops_read, cg->rd_io_queued_read, cg->io_queued.Read); + + if(unlikely(!cg->rd_io_queued_write)) + cg->rd_io_queued_write = rrddim_add(st_queued_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_queued_ops_write, cg->rd_io_queued_write, cg->io_queued.Write); + } + + if(likely(do_merged_ops && cg->io_merged.updated)) { + if(unlikely(!cg->rd_io_merged_read)) + cg->rd_io_merged_read = rrddim_add(st_merged_ops_read, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_merged_ops_read, cg->rd_io_merged_read, cg->io_merged.Read); + + if(unlikely(!cg->rd_io_merged_write)) + cg->rd_io_merged_write = rrddim_add(st_merged_ops_write, cg->chart_id, cg->chart_title, 1, 1, RRDDIM_INCREMENTAL); + + rrddim_set_by_pointer(st_merged_ops_write, cg->rd_io_merged_write, cg->io_merged.Write); + } + } + + // complete the iteration + if(likely(do_cpu)) + rrdset_done(st_cpu); + + if(likely(do_mem_usage)) + rrdset_done(st_mem_usage); + + if(unlikely(do_mem_detailed)) { + rrdset_done(st_mem_detailed_cache); + rrdset_done(st_mem_detailed_rss); + rrdset_done(st_mem_detailed_mapped); + rrdset_done(st_mem_detailed_writeback); + rrdset_done(st_mem_detailed_pgfault); + rrdset_done(st_mem_detailed_pgmajfault); + rrdset_done(st_mem_detailed_pgpgin); + rrdset_done(st_mem_detailed_pgpgout); + } + + if(likely(do_mem_failcnt)) + rrdset_done(st_mem_failcnt); + + if(likely(do_swap_usage)) + rrdset_done(st_swap_usage); + + if(likely(do_io)) { + rrdset_done(st_io_read); + rrdset_done(st_io_write); + } + + if(likely(do_io_ops)) { + rrdset_done(st_io_serviced_read); + rrdset_done(st_io_serviced_write); + } + + if(likely(do_throttle_io)) { + rrdset_done(st_throttle_io_read); + rrdset_done(st_throttle_io_write); + } + + if(likely(do_throttle_ops)) { + rrdset_done(st_throttle_ops_read); + rrdset_done(st_throttle_ops_write); + } + + if(likely(do_queued_ops)) { + rrdset_done(st_queued_ops_read); + rrdset_done(st_queued_ops_write); + } + + if(likely(do_merged_ops)) { + rrdset_done(st_merged_ops_read); + rrdset_done(st_merged_ops_write); + } +} + +static inline char *cgroup_chart_type(char *buffer, const char *id, size_t len) { + if(buffer[0]) return buffer; + + if(id[0] == '\0' || (id[0] == '/' && id[1] == '\0')) + strncpy(buffer, "cgroup_root", len); + else + snprintfz(buffer, len, "cgroup_%s", id); + + netdata_fix_chart_id(buffer); + return buffer; +} + void update_cgroup_charts(int update_every) { debug(D_CGROUP, "updating cgroups charts"); char type[RRD_ID_LENGTH_MAX + 1]; char title[CHART_TITLE_MAX + 1]; - struct cgroup *cg; - RRDSET *st; + int services_do_cpu = 0, + services_do_mem_usage = 0, + services_do_mem_detailed = 0, + services_do_mem_failcnt = 0, + services_do_swap_usage = 0, + services_do_io = 0, + services_do_io_ops = 0, + services_do_throttle_io = 0, + services_do_throttle_ops = 0, + services_do_queued_ops = 0, + services_do_merged_ops = 0; + struct cgroup *cg; for(cg = cgroup_root; cg ; cg = cg->next) { - if(!cg->available || !cg->enabled) + if(unlikely(!cg->available || !cg->enabled)) continue; - if(cg->id[0] == '\0') - strcpy(type, "cgroup_root"); - else if(cg->id[0] == '/') - snprintfz(type, RRD_ID_LENGTH_MAX, "cgroup_%s", cg->chart_id); - else - snprintfz(type, RRD_ID_LENGTH_MAX, "cgroup_%s", cg->chart_id); + if(likely(cgroup_enable_systemd_services && cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)) { + if(cg->cpuacct_stat.updated && cg->cpuacct_stat.enabled == CONFIG_ONDEMAND_YES) services_do_cpu++; - netdata_fix_chart_id(type); + if(cgroup_enable_systemd_services_detailed_memory && cg->memory.updated_detailed && cg->memory.enabled_detailed) services_do_mem_detailed++; + if(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_ONDEMAND_YES) services_do_mem_usage++; + if(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_ONDEMAND_YES) services_do_mem_failcnt++; + if(cg->memory.updated_msw_usage_in_bytes && cg->memory.enabled_msw_usage_in_bytes == CONFIG_ONDEMAND_YES) services_do_swap_usage++; - if(cg->cpuacct_stat.updated) { - st = rrdset_find_bytype(type, "cpu"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) for cgroup %s", (processors * 100), processors, (processors>1)?"s":"", cg->chart_title); - st = rrdset_create(type, "cpu", NULL, "cpu", "cgroup.cpu", title, "%", 40000, update_every, RRDSET_TYPE_STACKED); + if(cg->io_service_bytes.updated && cg->io_service_bytes.enabled == CONFIG_ONDEMAND_YES) services_do_io++; + if(cg->io_serviced.updated && cg->io_serviced.enabled == CONFIG_ONDEMAND_YES) services_do_io_ops++; + if(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.enabled == CONFIG_ONDEMAND_YES) services_do_throttle_io++; + if(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.enabled == CONFIG_ONDEMAND_YES) services_do_throttle_ops++; + if(cg->io_queued.updated && cg->io_queued.enabled == CONFIG_ONDEMAND_YES) services_do_queued_ops++; + if(cg->io_merged.updated && cg->io_merged.enabled == CONFIG_ONDEMAND_YES) services_do_merged_ops++; + continue; + } + + type[0] = '\0'; - rrddim_add(st, "user", NULL, 100, hz, RRDDIM_INCREMENTAL); - rrddim_add(st, "system", NULL, 100, hz, RRDDIM_INCREMENTAL); + if(likely(cg->cpuacct_stat.updated && cg->cpuacct_stat.enabled == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_cpu)) { + cg->st_cpu = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "cpu"); + if(likely(!cg->st_cpu)) { + snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title); + cg->st_cpu = rrdset_create(type, "cpu", NULL, "cpu", "cgroup.cpu", title, "%", CHART_PRIORITY_CONTAINERS, update_every, RRDSET_TYPE_STACKED); + } + rrddim_add(cg->st_cpu, "user", NULL, 100, hz, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_cpu, "system", NULL, 100, hz, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else + rrdset_next(cg->st_cpu); - rrddim_set(st, "user", cg->cpuacct_stat.user); - rrddim_set(st, "system", cg->cpuacct_stat.system); - rrdset_done(st); + rrddim_set(cg->st_cpu, "user", cg->cpuacct_stat.user); + rrddim_set(cg->st_cpu, "system", cg->cpuacct_stat.system); + rrdset_done(cg->st_cpu); } - if(cg->cpuacct_usage.updated) { + if(likely(cg->cpuacct_usage.updated && cg->cpuacct_usage.enabled == CONFIG_ONDEMAND_YES)) { char id[RRD_ID_LENGTH_MAX + 1]; unsigned int i; - st = rrdset_find_bytype(type, "cpu_per_core"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) Per Core for cgroup %s", (processors * 100), processors, (processors>1)?"s":"", cg->chart_title); - st = rrdset_create(type, "cpu_per_core", NULL, "cpu", "cgroup.cpu_per_core", title, "%", 40100, update_every, RRDSET_TYPE_STACKED); - - for(i = 0; i < cg->cpuacct_usage.cpus ;i++) { + if(unlikely(!cg->st_cpu_per_core)) { + cg->st_cpu_per_core = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "cpu_per_core"); + if(likely(!cg->st_cpu_per_core)) { + snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) Per Core for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title); + cg->st_cpu_per_core = rrdset_create(type, "cpu_per_core", NULL, "cpu", "cgroup.cpu_per_core", title, "%", CHART_PRIORITY_CONTAINERS + 100, update_every, RRDSET_TYPE_STACKED); + } + for(i = 0; i < cg->cpuacct_usage.cpus; i++) { snprintfz(id, CHART_TITLE_MAX, "cpu%u", i); - rrddim_add(st, id, NULL, 100, 1000000000, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_cpu_per_core, id, NULL, 100, 1000000000, RRDDIM_INCREMENTAL); } } - else rrdset_next(st); + else + rrdset_next(cg->st_cpu_per_core); for(i = 0; i < cg->cpuacct_usage.cpus ;i++) { snprintfz(id, CHART_TITLE_MAX, "cpu%u", i); - rrddim_set(st, id, cg->cpuacct_usage.cpu_percpu[i]); + rrddim_set(cg->st_cpu_per_core, id, cg->cpuacct_usage.cpu_percpu[i]); } - rrdset_done(st); + rrdset_done(cg->st_cpu_per_core); } - if(cg->memory.updated) { - if(cg->memory.cache + cg->memory.rss + cg->memory.rss_huge + cg->memory.mapped_file > 0) { - st = rrdset_find_bytype(type, "mem"); - if(!st) { + if(likely(cg->memory.updated_detailed && cg->memory.enabled_detailed == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_mem)) { + cg->st_mem = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "mem"); + if(likely(!cg->st_mem)) { snprintfz(title, CHART_TITLE_MAX, "Memory Usage for cgroup %s", cg->chart_title); - st = rrdset_create(type, "mem", NULL, "mem", "cgroup.mem", title, "MB", 40210, update_every, - RRDSET_TYPE_STACKED); - - rrddim_add(st, "cache", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); - rrddim_add(st, "rss", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); - if(cg->memory.has_dirty_swap) - rrddim_add(st, "swap", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); - rrddim_add(st, "rss_huge", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); - rrddim_add(st, "mapped_file", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + cg->st_mem = rrdset_create(type, "mem", NULL, "mem", "cgroup.mem", title, "MB", CHART_PRIORITY_CONTAINERS + 210, update_every, RRDSET_TYPE_STACKED); } - else rrdset_next(st); - - rrddim_set(st, "cache", cg->memory.cache); - rrddim_set(st, "rss", cg->memory.rss); - if(cg->memory.has_dirty_swap) - rrddim_set(st, "swap", cg->memory.swap); - rrddim_set(st, "rss_huge", cg->memory.rss_huge); - rrddim_set(st, "mapped_file", cg->memory.mapped_file); - rrdset_done(st); - } - st = rrdset_find_bytype(type, "writeback"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Writeback Memory for cgroup %s", cg->chart_title); - st = rrdset_create(type, "writeback", NULL, "mem", "cgroup.writeback", title, "MB", 40300, - update_every, RRDSET_TYPE_AREA); + rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(cg->st_mem, "rss", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + if(cg->memory.detailed_has_swap) + rrddim_add(cg->st_mem, "swap", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(cg->st_mem, "rss_huge", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(cg->st_mem, "mapped_file", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + } + else + rrdset_next(cg->st_mem); + + rrddim_set(cg->st_mem, "cache", cg->memory.cache); + rrddim_set(cg->st_mem, "rss", cg->memory.rss); + if(cg->memory.detailed_has_swap) + rrddim_set(cg->st_mem, "swap", cg->memory.swap); + rrddim_set(cg->st_mem, "rss_huge", cg->memory.rss_huge); + rrddim_set(cg->st_mem, "mapped_file", cg->memory.mapped_file); + rrdset_done(cg->st_mem); + + if(unlikely(!cg->st_writeback)) { + cg->st_writeback = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "writeback"); + if(likely(!cg->st_writeback)) { + snprintfz(title, CHART_TITLE_MAX, "Writeback Memory for cgroup %s", cg->chart_title); + cg->st_writeback = rrdset_create(type, "writeback", NULL, "mem", "cgroup.writeback", title, "MB", CHART_PRIORITY_CONTAINERS + 300, update_every, RRDSET_TYPE_AREA); + } - if(cg->memory.has_dirty_swap) - rrddim_add(st, "dirty", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); - rrddim_add(st, "writeback", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + if(cg->memory.detailed_has_dirty) + rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(cg->st_writeback, "writeback", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); } - else rrdset_next(st); + else + rrdset_next(cg->st_writeback); - if(cg->memory.has_dirty_swap) - rrddim_set(st, "dirty", cg->memory.dirty); - rrddim_set(st, "writeback", cg->memory.writeback); - rrdset_done(st); + if(cg->memory.detailed_has_dirty) + rrddim_set(cg->st_writeback, "dirty", cg->memory.dirty); + rrddim_set(cg->st_writeback, "writeback", cg->memory.writeback); + rrdset_done(cg->st_writeback); - if(cg->memory.pgpgin + cg->memory.pgpgout > 0) { - st = rrdset_find_bytype(type, "mem_activity"); - if(!st) { + if(unlikely(!cg->st_mem_activity)) { + cg->st_mem_activity = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "mem_activity"); + if(likely(!cg->st_mem_activity)) { snprintfz(title, CHART_TITLE_MAX, "Memory Activity for cgroup %s", cg->chart_title); - st = rrdset_create(type, "mem_activity", NULL, "mem", "cgroup.mem_activity", title, "MB/s", - 40400, update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "pgpgin", "in", sysconf(_SC_PAGESIZE), 1024 * 1024, RRDDIM_INCREMENTAL); - rrddim_add(st, "pgpgout", "out", -sysconf(_SC_PAGESIZE), 1024 * 1024, RRDDIM_INCREMENTAL); + cg->st_mem_activity = rrdset_create(type, "mem_activity", NULL, "mem", "cgroup.mem_activity", title, "MB/s", CHART_PRIORITY_CONTAINERS + 400, update_every, RRDSET_TYPE_LINE); } - else rrdset_next(st); - - rrddim_set(st, "pgpgin", cg->memory.pgpgin); - rrddim_set(st, "pgpgout", cg->memory.pgpgout); - rrdset_done(st); + rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); } + else + rrdset_next(cg->st_mem_activity); - if(cg->memory.pgfault + cg->memory.pgmajfault > 0) { - st = rrdset_find_bytype(type, "pgfaults"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Memory Page Faults for cgroup %s", cg->chart_title); - st = rrdset_create(type, "pgfaults", NULL, "mem", "cgroup.pgfaults", title, "MB/s", 40500, - update_every, RRDSET_TYPE_LINE); + rrddim_set(cg->st_mem_activity, "pgpgin", cg->memory.pgpgin); + rrddim_set(cg->st_mem_activity, "pgpgout", cg->memory.pgpgout); + rrdset_done(cg->st_mem_activity); - rrddim_add(st, "pgfault", NULL, sysconf(_SC_PAGESIZE), 1024 * 1024, RRDDIM_INCREMENTAL); - rrddim_add(st, "pgmajfault", "swap", -sysconf(_SC_PAGESIZE), 1024 * 1024, RRDDIM_INCREMENTAL); + if(unlikely(!cg->st_pgfaults)) { + cg->st_pgfaults = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "pgfaults"); + if(likely(!cg->st_pgfaults)) { + snprintfz(title, CHART_TITLE_MAX, "Memory Page Faults for cgroup %s", cg->chart_title); + cg->st_pgfaults = rrdset_create(type, "pgfaults", NULL, "mem", "cgroup.pgfaults", title, "MB/s", CHART_PRIORITY_CONTAINERS + 500, update_every, RRDSET_TYPE_LINE); } - else rrdset_next(st); - - rrddim_set(st, "pgfault", cg->memory.pgfault); - rrddim_set(st, "pgmajfault", cg->memory.pgmajfault); - rrdset_done(st); + rrddim_add(cg->st_pgfaults, "pgfault", NULL, system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_pgfaults, "pgmajfault", "swap", -system_page_size, 1024 * 1024, RRDDIM_INCREMENTAL); } - } + else + rrdset_next(cg->st_pgfaults); - if(cg->memory.usage_in_bytes_updated) { - st = rrdset_find_bytype(type, "mem_usage"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Total Memory for cgroup %s", cg->chart_title); - st = rrdset_create(type, "mem_usage", NULL, "mem", "cgroup.mem_usage", title, "MB", 40200, - update_every, RRDSET_TYPE_STACKED); + rrddim_set(cg->st_pgfaults, "pgfault", cg->memory.pgfault); + rrddim_set(cg->st_pgfaults, "pgmajfault", cg->memory.pgmajfault); + rrdset_done(cg->st_pgfaults); + } - rrddim_add(st, "ram", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); - rrddim_add(st, "swap", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + if(likely(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_mem_usage)) { + cg->st_mem_usage = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "mem_usage"); + if(likely(!cg->st_mem_usage)) { + snprintfz(title, CHART_TITLE_MAX, "Used Memory %sfor cgroup %s", (cgroup_used_memory_without_cache && cg->memory.updated_detailed)?"without Cache ":"", cg->chart_title); + cg->st_mem_usage = rrdset_create(type, "mem_usage", NULL, "mem", "cgroup.mem_usage", title, "MB", CHART_PRIORITY_CONTAINERS + 200, update_every, RRDSET_TYPE_STACKED); + } + rrddim_add(cg->st_mem_usage, "ram", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); + rrddim_add(cg->st_mem_usage, "swap", NULL, 1, 1024 * 1024, RRDDIM_ABSOLUTE); } - else rrdset_next(st); + else + rrdset_next(cg->st_mem_usage); - rrddim_set(st, "ram", cg->memory.usage_in_bytes); - rrddim_set(st, "swap", (cg->memory.msw_usage_in_bytes > cg->memory.usage_in_bytes)?cg->memory.msw_usage_in_bytes - cg->memory.usage_in_bytes:0); - rrdset_done(st); + rrddim_set(cg->st_mem_usage, "ram", cg->memory.usage_in_bytes - ((cgroup_used_memory_without_cache)?cg->memory.cache:0)); + rrddim_set(cg->st_mem_usage, "swap", (cg->memory.msw_usage_in_bytes > cg->memory.usage_in_bytes)?cg->memory.msw_usage_in_bytes - cg->memory.usage_in_bytes:0); + rrdset_done(cg->st_mem_usage); } - if(cg->memory.failcnt_updated && cg->memory.failcnt > 0) { - st = rrdset_find_bytype(type, "mem_failcnt"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Memory Limit Failures for cgroup %s", cg->chart_title); - st = rrdset_create(type, "mem_failcnt", NULL, "mem", "cgroup.mem_failcnt", title, "MB", 40250, - update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "failures", NULL, 1, 1, RRDDIM_INCREMENTAL); + if(likely(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_mem_failcnt)) { + cg->st_mem_failcnt = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "mem_failcnt"); + if(likely(!cg->st_mem_failcnt)) { + snprintfz(title, CHART_TITLE_MAX, "Memory Limit Failures for cgroup %s", cg->chart_title); + cg->st_mem_failcnt = rrdset_create(type, "mem_failcnt", NULL, "mem", "cgroup.mem_failcnt", title, "count", CHART_PRIORITY_CONTAINERS + 250, update_every, RRDSET_TYPE_LINE); + } + rrddim_add(cg->st_mem_failcnt, "failures", NULL, 1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else + rrdset_next(cg->st_mem_failcnt); - rrddim_set(st, "failures", cg->memory.failcnt); - rrdset_done(st); + rrddim_set(cg->st_mem_failcnt, "failures", cg->memory.failcnt); + rrdset_done(cg->st_mem_failcnt); } - if(cg->io_service_bytes.updated && cg->io_service_bytes.Read + cg->io_service_bytes.Write > 0) { - st = rrdset_find_bytype(type, "io"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); - st = rrdset_create(type, "io", NULL, "disk", "cgroup.io", title, "KB/s", 41200, - update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "read", NULL, 1, 1024, RRDDIM_INCREMENTAL); - rrddim_add(st, "write", NULL, -1, 1024, RRDDIM_INCREMENTAL); + if(likely(cg->io_service_bytes.updated && cg->io_service_bytes.enabled == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_io)) { + cg->st_io = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "io"); + if(likely(!cg->st_io)) { + snprintfz(title, CHART_TITLE_MAX, "I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); + cg->st_io = rrdset_create(type, "io", NULL, "disk", "cgroup.io", title, "KB/s", CHART_PRIORITY_CONTAINERS + 1200, update_every, RRDSET_TYPE_AREA); + } + rrddim_add(cg->st_io, "read", NULL, 1, 1024, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_io, "write", NULL, -1, 1024, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else + rrdset_next(cg->st_io); - rrddim_set(st, "read", cg->io_service_bytes.Read); - rrddim_set(st, "write", cg->io_service_bytes.Write); - rrdset_done(st); + rrddim_set(cg->st_io, "read", cg->io_service_bytes.Read); + rrddim_set(cg->st_io, "write", cg->io_service_bytes.Write); + rrdset_done(cg->st_io); } - if(cg->io_serviced.updated && cg->io_serviced.Read + cg->io_serviced.Write > 0) { - st = rrdset_find_bytype(type, "serviced_ops"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); - st = rrdset_create(type, "serviced_ops", NULL, "disk", "cgroup.serviced_ops", title, "operations/s", 41200, - update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "read", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "write", NULL, -1, 1, RRDDIM_INCREMENTAL); + if(likely(cg->io_serviced.updated && cg->io_serviced.enabled == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_serviced_ops)) { + cg->st_serviced_ops = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "serviced_ops"); + if(likely(!cg->st_serviced_ops)) { + snprintfz(title, CHART_TITLE_MAX, "Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); + cg->st_serviced_ops = rrdset_create(type, "serviced_ops", NULL, "disk", "cgroup.serviced_ops", title, "operations/s", CHART_PRIORITY_CONTAINERS + 1200, update_every, RRDSET_TYPE_LINE); + } + rrddim_add(cg->st_serviced_ops, "read", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_serviced_ops, "write", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else + rrdset_next(cg->st_serviced_ops); - rrddim_set(st, "read", cg->io_serviced.Read); - rrddim_set(st, "write", cg->io_serviced.Write); - rrdset_done(st); + rrddim_set(cg->st_serviced_ops, "read", cg->io_serviced.Read); + rrddim_set(cg->st_serviced_ops, "write", cg->io_serviced.Write); + rrdset_done(cg->st_serviced_ops); } - if(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.Read + cg->throttle_io_service_bytes.Write > 0) { - st = rrdset_find_bytype(type, "io"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Throttle I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); - st = rrdset_create(type, "io", NULL, "disk", "cgroup.io", title, "KB/s", 41200, - update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "read", NULL, 1, 1024, RRDDIM_INCREMENTAL); - rrddim_add(st, "write", NULL, -1, 1024, RRDDIM_INCREMENTAL); + if(likely(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.enabled == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_throttle_io)) { + cg->st_throttle_io = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "throttle_io"); + if(likely(!cg->st_throttle_io)) { + snprintfz(title, CHART_TITLE_MAX, "Throttle I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); + cg->st_throttle_io = rrdset_create(type, "throttle_io", NULL, "disk", "cgroup.throttle_io", title, "KB/s", CHART_PRIORITY_CONTAINERS + 1200, update_every, RRDSET_TYPE_AREA); + } + rrddim_add(cg->st_throttle_io, "read", NULL, 1, 1024, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_throttle_io, "write", NULL, -1, 1024, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else + rrdset_next(cg->st_throttle_io); - rrddim_set(st, "read", cg->throttle_io_service_bytes.Read); - rrddim_set(st, "write", cg->throttle_io_service_bytes.Write); - rrdset_done(st); + rrddim_set(cg->st_throttle_io, "read", cg->throttle_io_service_bytes.Read); + rrddim_set(cg->st_throttle_io, "write", cg->throttle_io_service_bytes.Write); + rrdset_done(cg->st_throttle_io); } - - if(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.Read + cg->throttle_io_serviced.Write > 0) { - st = rrdset_find_bytype(type, "throttle_serviced_ops"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Throttle Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); - st = rrdset_create(type, "throttle_serviced_ops", NULL, "disk", "cgroup.throttle_serviced_ops", title, "operations/s", 41200, - update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "read", NULL, 1, 1, RRDDIM_INCREMENTAL); - rrddim_add(st, "write", NULL, -1, 1, RRDDIM_INCREMENTAL); + if(likely(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.enabled == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_throttle_serviced_ops)) { + cg->st_throttle_serviced_ops = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "throttle_serviced_ops"); + if(likely(!cg->st_throttle_serviced_ops)) { + snprintfz(title, CHART_TITLE_MAX, "Throttle Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); + cg->st_throttle_serviced_ops = rrdset_create(type, "throttle_serviced_ops", NULL, "disk", "cgroup.throttle_serviced_ops", title, "operations/s", CHART_PRIORITY_CONTAINERS + 1200, update_every, RRDSET_TYPE_LINE); + } + rrddim_add(cg->st_throttle_serviced_ops, "read", NULL, 1, 1, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_throttle_serviced_ops, "write", NULL, -1, 1, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else + rrdset_next(cg->st_throttle_serviced_ops); - rrddim_set(st, "read", cg->throttle_io_serviced.Read); - rrddim_set(st, "write", cg->throttle_io_serviced.Write); - rrdset_done(st); + rrddim_set(cg->st_throttle_serviced_ops, "read", cg->throttle_io_serviced.Read); + rrddim_set(cg->st_throttle_serviced_ops, "write", cg->throttle_io_serviced.Write); + rrdset_done(cg->st_throttle_serviced_ops); } - if(cg->io_queued.updated) { - st = rrdset_find_bytype(type, "queued_ops"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Queued I/O Operations (all disks) for cgroup %s", cg->chart_title); - st = rrdset_create(type, "queued_ops", NULL, "disk", "cgroup.queued_ops", title, "operations", 42000, - update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "read", NULL, 1, 1, RRDDIM_ABSOLUTE); - rrddim_add(st, "write", NULL, -1, 1, RRDDIM_ABSOLUTE); + if(likely(cg->io_queued.updated && cg->io_queued.enabled == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_queued_ops)) { + cg->st_queued_ops = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "queued_ops"); + if(likely(!cg->st_queued_ops)) { + snprintfz(title, CHART_TITLE_MAX, "Queued I/O Operations (all disks) for cgroup %s", cg->chart_title); + cg->st_queued_ops = rrdset_create(type, "queued_ops", NULL, "disk", "cgroup.queued_ops", title, "operations", CHART_PRIORITY_CONTAINERS + 2000, update_every, RRDSET_TYPE_LINE); + } + rrddim_add(cg->st_queued_ops, "read", NULL, 1, 1, RRDDIM_ABSOLUTE); + rrddim_add(cg->st_queued_ops, "write", NULL, -1, 1, RRDDIM_ABSOLUTE); } - else rrdset_next(st); + else + rrdset_next(cg->st_queued_ops); - rrddim_set(st, "read", cg->io_queued.Read); - rrddim_set(st, "write", cg->io_queued.Write); - rrdset_done(st); + rrddim_set(cg->st_queued_ops, "read", cg->io_queued.Read); + rrddim_set(cg->st_queued_ops, "write", cg->io_queued.Write); + rrdset_done(cg->st_queued_ops); } - if(cg->io_merged.updated && cg->io_merged.Read + cg->io_merged.Write > 0) { - st = rrdset_find_bytype(type, "merged_ops"); - if(!st) { - snprintfz(title, CHART_TITLE_MAX, "Merged I/O Operations (all disks) for cgroup %s", cg->chart_title); - st = rrdset_create(type, "merged_ops", NULL, "disk", "cgroup.merged_ops", title, "operations/s", 42100, - update_every, RRDSET_TYPE_LINE); - - rrddim_add(st, "read", NULL, 1, 1024, RRDDIM_INCREMENTAL); - rrddim_add(st, "write", NULL, -1, 1024, RRDDIM_INCREMENTAL); + if(likely(cg->io_merged.updated && cg->io_merged.enabled == CONFIG_ONDEMAND_YES)) { + if(unlikely(!cg->st_merged_ops)) { + cg->st_merged_ops = rrdset_find_bytype(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "merged_ops"); + if(likely(!cg->st_merged_ops)) { + snprintfz(title, CHART_TITLE_MAX, "Merged I/O Operations (all disks) for cgroup %s", cg->chart_title); + cg->st_merged_ops = rrdset_create(type, "merged_ops", NULL, "disk", "cgroup.merged_ops", title, "operations/s", CHART_PRIORITY_CONTAINERS + 2100, update_every, RRDSET_TYPE_LINE); + } + rrddim_add(cg->st_merged_ops, "read", NULL, 1, 1024, RRDDIM_INCREMENTAL); + rrddim_add(cg->st_merged_ops, "write", NULL, -1, 1024, RRDDIM_INCREMENTAL); } - else rrdset_next(st); + else + rrdset_next(cg->st_merged_ops); - rrddim_set(st, "read", cg->io_merged.Read); - rrddim_set(st, "write", cg->io_merged.Write); - rrdset_done(st); + rrddim_set(cg->st_merged_ops, "read", cg->io_merged.Read); + rrddim_set(cg->st_merged_ops, "write", cg->io_merged.Write); + rrdset_done(cg->st_merged_ops); } } + if(likely(cgroup_enable_systemd_services)) + update_services_charts(update_every, + services_do_cpu, + services_do_mem_usage, + services_do_mem_detailed, + services_do_mem_failcnt, + services_do_swap_usage, + services_do_io, + services_do_io_ops, + services_do_throttle_io, + services_do_throttle_ops, + services_do_queued_ops, + services_do_merged_ops + ); + debug(D_CGROUP, "done updating cgroups charts"); } // ---------------------------------------------------------------------------- // cgroups main -int do_sys_fs_cgroup(int update_every, unsigned long long dt) { - (void)dt; - - static int cgroup_global_config_read = 0; - static time_t last_run = 0; - time_t now = time(NULL); - - if(unlikely(!cgroup_global_config_read)) { - read_cgroup_plugin_configuration(); - cgroup_global_config_read = 1; - } - - if(unlikely(cgroup_enable_new_cgroups_detected_at_runtime && now - last_run > cgroup_check_for_new_every)) { - find_all_cgroups(); - last_run = now; - } - - read_all_cgroups(cgroup_root); - update_cgroup_charts(update_every); - - return 0; -} - -void *cgroups_main(void *ptr) -{ - (void)ptr; +void *cgroups_main(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; info("CGROUP Plugin thread created with task id %d", gettid()); @@ -1422,56 +2073,52 @@ void *cgroups_main(void *ptr) struct rusage thread; // when ZERO, attempt to do it - int vdo_sys_fs_cgroup = 0; - int vdo_cpu_netdata = !config_get_boolean("plugin:cgroups", "cgroups plugin resources", 1); - - // keep track of the time each module was called - unsigned long long sutime_sys_fs_cgroup = 0ULL; + int vdo_cpu_netdata = config_get_boolean("plugin:cgroups", "cgroups plugin resource charts", 1); - // the next time we will run - aligned properly - unsigned long long sunext = (time(NULL) - (time(NULL) % rrd_update_every) + rrd_update_every) * 1000000ULL; - unsigned long long sunow; + read_cgroup_plugin_configuration(); RRDSET *stcpu_thread = NULL; + usec_t step = cgroup_update_every * USEC_PER_SEC; + usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_next = 0; for(;;) { - if(unlikely(netdata_exit)) break; - - // delay until it is our time to run - while((sunow = time_usec()) < sunext) - sleep_usec(sunext - sunow); + usec_t now = now_monotonic_usec(); + usec_t next = now - (now % step) + step; - // find the next time we need to run - while(time_usec() > sunext) - sunext += rrd_update_every * 1000000ULL; + while(now < next) { + sleep_usec(next - now); + now = now_monotonic_usec(); + } if(unlikely(netdata_exit)) break; // BEGIN -- the job to be done - if(!vdo_sys_fs_cgroup) { - debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_fs_cgroup()."); - sunow = time_usec(); - vdo_sys_fs_cgroup = do_sys_fs_cgroup(rrd_update_every, (sutime_sys_fs_cgroup > 0)?sunow - sutime_sys_fs_cgroup:0ULL); - sutime_sys_fs_cgroup = sunow; + if(unlikely(now >= find_next)) { + find_all_cgroups(); + find_next = now + find_every; } - if(unlikely(netdata_exit)) break; + + read_all_cgroups(cgroup_root); + update_cgroup_charts(cgroup_update_every); // END -- the job is done // -------------------------------------------------------------------- - if(!vdo_cpu_netdata) { + if(vdo_cpu_netdata) { getrusage(RUSAGE_THREAD, &thread); - if(!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_cgroups_cpu"); - if(!stcpu_thread) { - stcpu_thread = rrdset_create("netdata", "plugin_cgroups_cpu", NULL, "proc.internal", NULL, "NetData CGroups Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, RRDSET_TYPE_STACKED); + if(unlikely(!stcpu_thread)) { + stcpu_thread = rrdset_find("netdata.plugin_cgroups_cpu"); + if(unlikely(!stcpu_thread)) + stcpu_thread = rrdset_create("netdata", "plugin_cgroups_cpu", NULL, "cgroups", NULL, "NetData CGroups Plugin CPU usage", "milliseconds/s", 132000, cgroup_update_every, RRDSET_TYPE_STACKED); rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); } - else rrdset_next(stcpu_thread); + else + rrdset_next(stcpu_thread); rrddim_set(stcpu_thread, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); @@ -1481,6 +2128,7 @@ void *cgroups_main(void *ptr) info("CGROUP thread exiting"); + static_thread->enabled = 0; pthread_exit(NULL); return NULL; } diff --git a/src/sys_kernel_mm_ksm.c b/src/sys_kernel_mm_ksm.c index 8c51be1df..83da74429 100644 --- a/src/sys_kernel_mm_ksm.c +++ b/src/sys_kernel_mm_ksm.c @@ -19,7 +19,7 @@ KSM_NAME_VALUE values[] = { [PAGES_TO_SCAN] = { "/sys/kernel/mm/ksm/pages_to_scan", 0ULL }, }; -int do_sys_kernel_mm_ksm(int update_every, unsigned long long dt) { +int do_sys_kernel_mm_ksm(int update_every, usec_t dt) { static procfile *ff_pages_shared = NULL, *ff_pages_sharing = NULL, *ff_pages_unshared = NULL, *ff_pages_volatile = NULL, *ff_pages_to_scan = NULL; static long page_size = -1; @@ -64,23 +64,23 @@ int do_sys_kernel_mm_ksm(int update_every, unsigned long long dt) { ff_pages_shared = procfile_readall(ff_pages_shared); if(!ff_pages_shared) return 0; // we return 0, so that we will retry to open it next time - pages_shared = strtoull(procfile_lineword(ff_pages_shared, 0, 0), NULL, 10); + pages_shared = str2ull(procfile_lineword(ff_pages_shared, 0, 0)); ff_pages_sharing = procfile_readall(ff_pages_sharing); if(!ff_pages_sharing) return 0; // we return 0, so that we will retry to open it next time - pages_sharing = strtoull(procfile_lineword(ff_pages_sharing, 0, 0), NULL, 10); + pages_sharing = str2ull(procfile_lineword(ff_pages_sharing, 0, 0)); ff_pages_unshared = procfile_readall(ff_pages_unshared); if(!ff_pages_unshared) return 0; // we return 0, so that we will retry to open it next time - pages_unshared = strtoull(procfile_lineword(ff_pages_unshared, 0, 0), NULL, 10); + pages_unshared = str2ull(procfile_lineword(ff_pages_unshared, 0, 0)); ff_pages_volatile = procfile_readall(ff_pages_volatile); if(!ff_pages_volatile) return 0; // we return 0, so that we will retry to open it next time - pages_volatile = strtoull(procfile_lineword(ff_pages_volatile, 0, 0), NULL, 10); + pages_volatile = str2ull(procfile_lineword(ff_pages_volatile, 0, 0)); ff_pages_to_scan = procfile_readall(ff_pages_to_scan); if(!ff_pages_to_scan) return 0; // we return 0, so that we will retry to open it next time - pages_to_scan = strtoull(procfile_lineword(ff_pages_to_scan, 0, 0), NULL, 10); + pages_to_scan = str2ull(procfile_lineword(ff_pages_to_scan, 0, 0)); offered = pages_sharing + pages_shared + pages_unshared + pages_volatile; saved = pages_sharing - pages_shared; diff --git a/src/unit_test.c b/src/unit_test.c index d699707a4..4e2f10c0a 100644 --- a/src/unit_test.c +++ b/src/unit_test.c @@ -18,7 +18,7 @@ int check_storage_number(calculated_number n, int debug) { if(dcdiff < 0) dcdiff = -dcdiff; size_t len = print_calculated_number(buffer, d); - calculated_number p = strtold(buffer, NULL); + calculated_number p = str2l(buffer); calculated_number pdiff = n - p; calculated_number pcdiff = pdiff * 100.0 / n; if(pcdiff < 0) pcdiff = -pcdiff; @@ -901,7 +901,7 @@ int run_test(struct test *test) st->debug = 1; // feed it with the test data - time_t time_now = 0, time_start = time(NULL); + time_t time_now = 0, time_start = now_realtime_sec(); unsigned long c; collected_number last = 0; for(c = 0; c < test->feed_entries; c++) { @@ -915,7 +915,7 @@ int run_test(struct test *test) (float)time_now / 1000000.0, ((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor, (((calculated_number)test->feed[c].value - (calculated_number)last) * (calculated_number)test->multiplier / (calculated_number)test->divisor) / (calculated_number)test->feed[c].microseconds * (calculated_number)1000000); - rrdset_next_usec(st, test->feed[c].microseconds); + rrdset_next_usec_unfiltered(st, test->feed[c].microseconds); } else { fprintf(stderr, " > %s: feeding position %lu\n", test->name, c+1); @@ -975,8 +975,55 @@ int run_test(struct test *test) return errors; } +static int test_variable_renames(void) { + fprintf(stderr, "Creating chart\n"); + RRDSET *st = rrdset_create("chart", "ID", NULL, "family", "context", "Unit Testing", "a value", 1, 1, RRDSET_TYPE_LINE); + fprintf(stderr, "Created chart with id '%s', name '%s'\n", st->id, st->name); + + fprintf(stderr, "Creating dimension DIM1\n"); + RRDDIM *rd1 = rrddim_add(st, "DIM1", NULL, 1, 1, RRDDIM_INCREMENTAL); + fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd1->id, rd1->name); + + fprintf(stderr, "Creating dimension DIM2\n"); + RRDDIM *rd2 = rrddim_add(st, "DIM2", NULL, 1, 1, RRDDIM_INCREMENTAL); + fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd2->id, rd2->name); + + fprintf(stderr, "Renaming chart to CHARTNAME1\n"); + rrdset_set_name(st, "CHARTNAME1"); + fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name); + + fprintf(stderr, "Renaming chart to CHARTNAME2\n"); + rrdset_set_name(st, "CHARTNAME2"); + fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name); + + fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME1\n"); + rrddim_set_name(st, rd1, "DIM1NAME1"); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name); + + fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME2\n"); + rrddim_set_name(st, rd1, "DIM1NAME2"); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name); + + fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME1\n"); + rrddim_set_name(st, rd2, "DIM2NAME1"); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name); + + fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME2\n"); + rrddim_set_name(st, rd2, "DIM2NAME2"); + fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name); + + BUFFER *buf = buffer_create(1); + health_api_v1_chart_variables2json(st, buf); + fprintf(stderr, "%s", buffer_tostring(buf)); + buffer_free(buf); + return 1; +} + int run_all_mockup_tests(void) { + if(!test_variable_renames()) + return 1; + if(run_test(&test1)) return 1; @@ -1028,6 +1075,8 @@ int run_all_mockup_tests(void) if(run_test(&test15)) return 1; + + return 0; } @@ -1073,7 +1122,7 @@ int unit_test(long delay, long shift) fprintf(stderr, "\n\nLOOP = %lu, DELAY = %ld, VALUE = " COLLECTED_NUMBER_FORMAT "\n", c, delay, i); if(c) { - rrdset_next_usec(st, delay); + rrdset_next_usec_unfiltered(st, delay); } if(do_abs) rrddim_set(st, "absolute", i); if(do_inc) rrddim_set(st, "incremental", i); @@ -1081,7 +1130,7 @@ int unit_test(long delay, long shift) if(do_absi) rrddim_set(st, "percentage-of-incremental-row", i); if(!c) { - gettimeofday(&st->last_collected_time, NULL); + now_realtime_timeval(&st->last_collected_time); st->last_collected_time.tv_usec = shift; } diff --git a/src/web_buffer.c b/src/web_buffer.c index 93ba782af..6203db0f7 100644 --- a/src/web_buffer.c +++ b/src/web_buffer.c @@ -113,6 +113,8 @@ void buffer_print_llu(BUFFER *wb, unsigned long long uvalue) void buffer_strcat(BUFFER *wb, const char *txt) { + // buffer_sprintf(wb, "%s", txt); + if(unlikely(!txt || !*txt)) return; buffer_need_bytes(wb, 1); @@ -143,6 +145,26 @@ void buffer_strcat(BUFFER *wb, const char *txt) } } +void buffer_strcat_htmlescape(BUFFER *wb, const char *txt) +{ + char b[2] = { [0] = '\0', [1] = '\0' }; + + while(*txt) { + switch(*txt) { + case '&': buffer_strcat(wb, "&"); break; + case '<': buffer_strcat(wb, "<"); break; + case '>': buffer_strcat(wb, ">"); break; + case '"': buffer_strcat(wb, """); break; + case '/': buffer_strcat(wb, "/"); break; + case '\'': buffer_strcat(wb, "'"); break; + default: { + b[0] = *txt; + buffer_strcat(wb, b); + } + } + txt++; + } +} void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) { diff --git a/src/web_buffer.h b/src/web_buffer.h index ee611209b..8f0d29cd2 100644 --- a/src/web_buffer.h +++ b/src/web_buffer.h @@ -39,6 +39,7 @@ typedef struct web_buffer { #define CT_IMAGE_XICON 19 #define CT_IMAGE_ICNS 20 #define CT_IMAGE_BMP 21 +#define CT_PROMETHEUS 22 #define buffer_cacheable(wb) do { (wb)->options |= WB_CONTENT_CACHEABLE; if((wb)->options & WB_CONTENT_NO_CACHEABLE) (wb)->options &= ~WB_CONTENT_NO_CACHEABLE; } while(0) #define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE) (wb)->options &= ~WB_CONTENT_CACHEABLE; (wb)->expires = 0; } while(0) @@ -61,9 +62,10 @@ extern BUFFER *buffer_create(size_t size); extern void buffer_free(BUFFER *b); extern void buffer_increase(BUFFER *b, size_t free_size_required); -extern void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) __attribute__ (( format (printf, 3, 4))); +extern void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) PRINTFLIKE(3, 4); extern void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args); -extern void buffer_sprintf(BUFFER *wb, const char *fmt, ...) __attribute__ (( format (printf, 2, 3))); +extern void buffer_sprintf(BUFFER *wb, const char *fmt, ...) PRINTFLIKE(2,3); +extern void buffer_strcat_htmlescape(BUFFER *wb, const char *txt); extern void buffer_char_replace(BUFFER *wb, char from, char to); diff --git a/src/web_buffer_svg.c b/src/web_buffer_svg.c index 3e847b5d9..cac365ab1 100644 --- a/src/web_buffer_svg.c +++ b/src/web_buffer_svg.c @@ -290,7 +290,7 @@ static inline int verdana11_width(char *s) { *d = '\0'; w -= VERDANA_KERNING; w += VERDANA_PADDING; - return ceil(w); + return (int)ceil(w); } static inline size_t escape_xmlz(char *dst, const char *src, size_t len) { @@ -470,7 +470,7 @@ static inline void calc_colorz(const char *color, char *final, size_t len, calcu break; } else { - calculated_number v = strtold(value_buffer, NULL); + calculated_number v = str2l(value_buffer); if(comparison == '<' && value < v) break; else if(comparison == '(' && value <= v) break; @@ -525,7 +525,49 @@ void buffer_svg(BUFFER *wb, const char *label, calculated_number value, const ch if(unlikely(isalnum(*units))) separator = " "; - if(unlikely(value_is_null)) + if(unlikely(!strcmp(units, "seconds"))) { + size_t s = (size_t)value; + size_t d = s / 86400; + s = s % 86400; + + size_t h = s / 3600; + s = s % 3600; + + size_t m = s / 60; + s = s % 60; + + if(d) + snprintfz(value_string, VALUE_STRING_SIZE, "%zu %s %02zu:%02zu:%02zu", d, (d == 1)?"day":"days", h, m, s); + else + snprintfz(value_string, VALUE_STRING_SIZE, "%02zu:%02zu:%02zu", h, m, s); + } + + else if(unlikely(!strcmp(units, "minutes"))) { + size_t m = (size_t)value; + size_t d = m / (60 * 24); + m = m % (60 * 24); + + size_t h = m / 60; + m = m % 60; + + if(d) + snprintfz(value_string, VALUE_STRING_SIZE, "%zud %02zuh %02zum", d, h, m); + else + snprintfz(value_string, VALUE_STRING_SIZE, "%zuh %zum", h, m); + } + + else if(unlikely(!strcmp(units, "hours"))) { + size_t h = (size_t)value; + size_t d = h / 24; + h = h % 24; + + if(d) + snprintfz(value_string, VALUE_STRING_SIZE, "%zud %zuh", d, h); + else + snprintfz(value_string, VALUE_STRING_SIZE, "%zuh", h); + } + + else if(unlikely(value_is_null)) strcpy(value_string, "-"); else if(precision < 0) { diff --git a/src/web_client.c b/src/web_client.c index 0cf9eeb6a..4b6ccf646 100644 --- a/src/web_client.c +++ b/src/web_client.c @@ -14,7 +14,7 @@ int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRAT struct web_client *web_clients = NULL; unsigned long long web_clients_count = 0; -inline int web_client_crock_socket(struct web_client *w) { +static inline int web_client_crock_socket(struct web_client *w) { #ifdef TCP_CORK if(likely(!w->tcp_cork && w->ofd != -1)) { w->tcp_cork = 1; @@ -29,7 +29,7 @@ inline int web_client_crock_socket(struct web_client *w) { return 0; } -inline int web_client_uncrock_socket(struct web_client *w) { +static inline int web_client_uncrock_socket(struct web_client *w) { #ifdef TCP_CORK if(likely(w->tcp_cork && w->ofd != -1)) { w->tcp_cork = 0; @@ -121,11 +121,11 @@ struct web_client *web_client_create(int listener) void web_client_reset(struct web_client *w) { web_client_uncrock_socket(w); - debug(D_WEB_CLIENT, "%llu: Reseting client.", w->id); + debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id); if(likely(w->last_url[0])) { struct timeval tv; - gettimeofday(&tv, NULL); + now_realtime_timeval(&tv); size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; size_t sent = size; @@ -136,7 +136,7 @@ void web_client_reset(struct web_client *w) { // -------------------------------------------------------------------- // global statistics - finished_web_request_statistics(usec_dt(&tv, &w->tv_in), + finished_web_request_statistics(dt_usec(&tv, &w->tv_in), w->stats_received_bytes, w->stats_sent_bytes, size, @@ -152,9 +152,9 @@ void web_client_reset(struct web_client *w) { log_access("%llu: (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'", w->id, sent, size, -((size > 0) ? ((size - sent) / (double) size * 100.0) : 0.0), - usec_dt(&w->tv_ready, &w->tv_in) / 1000.0, - usec_dt(&tv, &w->tv_ready) / 1000.0, - usec_dt(&tv, &w->tv_in) / 1000.0, + dt_usec(&w->tv_ready, &w->tv_in) / 1000.0, + dt_usec(&tv, &w->tv_ready) / 1000.0, + dt_usec(&tv, &w->tv_in) / 1000.0, (w->mode == WEB_CLIENT_MODE_FILECOPY) ? "filecopy" : ((w->mode == WEB_CLIENT_MODE_OPTIONS) ? "options" : "data"), w->response.code, @@ -308,7 +308,8 @@ int mysendfile(struct web_client *w, char *filename) for(s = filename; *s ;s++) { if( !isalnum(*s) && *s != '/' && *s != '.' && *s != '-' && *s != '_') { debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename); - buffer_sprintf(w->response.data, "File '%s' cannot be served. Filename contains invalid character '%c'", filename, *s); + buffer_sprintf(w->response.data, "Filename contains invalid characters: "); + buffer_strcat_htmlescape(w->response.data, filename); return 400; } } @@ -316,7 +317,8 @@ int mysendfile(struct web_client *w, char *filename) // if the filename contains a .. refuse to serve it if(strstr(filename, "..") != 0) { debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename); - buffer_sprintf(w->response.data, "File '%s' cannot be served. Relative filenames with '..' in them are not supported.", filename); + buffer_strcat(w->response.data, "Relative filenames are not supported: "); + buffer_strcat_htmlescape(w->response.data, filename); return 400; } @@ -328,21 +330,24 @@ int mysendfile(struct web_client *w, char *filename) struct stat stat; if(lstat(webfilename, &stat) != 0) { debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not found.", w->id, webfilename); - buffer_sprintf(w->response.data, "File '%s' does not exist, or is not accessible.", webfilename); + buffer_strcat(w->response.data, "File does not exist, or is not accessible: "); + buffer_strcat_htmlescape(w->response.data, webfilename); return 404; } // check if the file is owned by expected user if(stat.st_uid != web_files_uid()) { error("%llu: File '%s' is owned by user %u (expected user %u). Access Denied.", w->id, webfilename, stat.st_uid, web_files_uid()); - buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", webfilename); + buffer_strcat(w->response.data, "Access to file is not permitted: "); + buffer_strcat_htmlescape(w->response.data, webfilename); return 403; } // check if the file is owned by expected group if(stat.st_gid != web_files_gid()) { error("%llu: File '%s' is owned by group %u (expected group %u). Access Denied.", w->id, webfilename, stat.st_gid, web_files_gid()); - buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", webfilename); + buffer_strcat(w->response.data, "Access to file is not permitted: "); + buffer_strcat_htmlescape(w->response.data, webfilename); return 403; } @@ -353,7 +358,8 @@ int mysendfile(struct web_client *w, char *filename) if((stat.st_mode & S_IFMT) != S_IFREG) { error("%llu: File '%s' is not a regular file. Access Denied.", w->id, webfilename); - buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", webfilename); + buffer_strcat(w->response.data, "Access to file is not permitted: "); + buffer_strcat_htmlescape(w->response.data, webfilename); return 403; } @@ -365,12 +371,14 @@ int mysendfile(struct web_client *w, char *filename) if(errno == EBUSY || errno == EAGAIN) { error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, webfilename); buffer_sprintf(w->response.header, "Location: /" WEB_PATH_FILE "/%s\r\n", filename); - buffer_sprintf(w->response.data, "The file '%s' is currently busy. Please try again later.", webfilename); + buffer_strcat(w->response.data, "File is currently busy, please try again later: "); + buffer_strcat_htmlescape(w->response.data, webfilename); return 307; } else { error("%llu: Cannot open file '%s'.", w->id, webfilename); - buffer_sprintf(w->response.data, "Cannot open file '%s'.", webfilename); + buffer_strcat(w->response.data, "Cannot open file: "); + buffer_strcat_htmlescape(w->response.data, webfilename); return 404; } } @@ -406,7 +414,11 @@ int mysendfile(struct web_client *w, char *filename) w->wait_send = 0; buffer_flush(w->response.data); w->response.rlen = stat.st_size; +#ifdef __APPLE__ + w->response.data->date = stat.st_mtimespec.tv_sec; +#else w->response.data->date = stat.st_mtim.tv_sec; +#endif /* __APPLE__ */ buffer_cacheable(w->response.data); return 200; @@ -416,7 +428,7 @@ int mysendfile(struct web_client *w, char *filename) #ifdef NETDATA_WITH_ZLIB void web_client_enable_deflate(struct web_client *w, int gzip) { if(unlikely(w->response.zinitialized)) { - error("%llu: Compression has already be initialized for this client.", w->id); + debug(D_DEFLATE, "%llu: Compression has already be initialized for this client.", w->id); return; } @@ -490,7 +502,7 @@ void buffer_data_options2string(BUFFER *wb, uint32_t options) { if(options & RRDR_OPTION_ABSOLUTE) { if(count++) buffer_strcat(wb, " "); - buffer_strcat(wb, "abs"); + buffer_strcat(wb, "absolute"); } if(options & RRDR_OPTION_SECONDS) { @@ -700,17 +712,7 @@ int web_client_api_request_v1_alarm_log(struct web_client *w, char *url) return 200; } -int web_client_api_request_v1_charts(struct web_client *w, char *url) -{ - (void)url; - - buffer_flush(w->response.data); - w->response.data->contenttype = CT_APPLICATION_JSON; - rrd_stats_api_v1_charts(w->response.data); - return 200; -} - -int web_client_api_request_v1_chart(struct web_client *w, char *url) +int web_client_api_request_single_chart(struct web_client *w, char *url, void callback(RRDSET *st, BUFFER *buf)) { int ret = 400; char *chart = NULL; @@ -743,19 +745,83 @@ int web_client_api_request_v1_chart(struct web_client *w, char *url) RRDSET *st = rrdset_find(chart); if(!st) st = rrdset_find_byname(chart); if(!st) { - buffer_sprintf(w->response.data, "Chart '%s' is not found.", chart); + buffer_strcat(w->response.data, "Chart is not found: "); + buffer_strcat_htmlescape(w->response.data, chart); ret = 404; goto cleanup; } w->response.data->contenttype = CT_APPLICATION_JSON; - rrd_stats_api_v1_chart(st, w->response.data); + callback(st, w->response.data); return 200; -cleanup: + cleanup: return ret; } +int web_client_api_request_v1_alarm_variables(struct web_client *w, char *url) +{ + return web_client_api_request_single_chart(w, url, health_api_v1_chart_variables2json); +} + +int web_client_api_request_v1_charts(struct web_client *w, char *url) +{ + (void)url; + + buffer_flush(w->response.data); + w->response.data->contenttype = CT_APPLICATION_JSON; + rrd_stats_api_v1_charts(w->response.data); + return 200; +} + +int web_client_api_request_v1_allmetrics(struct web_client *w, char *url) +{ + int format = ALLMETRICS_SHELL; + + while(url) { + char *value = mystrsep(&url, "?&"); + if (!value || !*value) continue; + + char *name = mystrsep(&value, "="); + if(!name || !*name) continue; + if(!value || !*value) continue; + + if(!strcmp(name, "format")) { + if(!strcmp(value, ALLMETRICS_FORMAT_SHELL)) + format = ALLMETRICS_SHELL; + else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS)) + format = ALLMETRICS_PROMETHEUS; + else + format = 0; + } + } + + buffer_flush(w->response.data); + buffer_no_cacheable(w->response.data); + + switch(format) { + case ALLMETRICS_SHELL: + w->response.data->contenttype = CT_TEXT_PLAIN; + rrd_stats_api_v1_charts_allmetrics_shell(w->response.data); + return 200; + + case ALLMETRICS_PROMETHEUS: + w->response.data->contenttype = CT_PROMETHEUS; + rrd_stats_api_v1_charts_allmetrics_prometheus(w->response.data); + return 200; + + default: + w->response.data->contenttype = CT_TEXT_PLAIN; + buffer_strcat(w->response.data, "Which format? Only '" ALLMETRICS_FORMAT_SHELL "' and '" ALLMETRICS_FORMAT_PROMETHEUS "' is currently supported."); + return 400; + } +} + +int web_client_api_request_v1_chart(struct web_client *w, char *url) +{ + return web_client_api_request_single_chart(w, url, rrd_stats_api_v1_chart); +} + int web_client_api_request_v1_badge(struct web_client *w, char *url) { int ret = 400; buffer_flush(w->response.data); @@ -846,12 +912,12 @@ int web_client_api_request_v1_badge(struct web_client *w, char *url) { } } - long long multiply = (multiply_str && *multiply_str )?atol(multiply_str):1; - long long divide = (divide_str && *divide_str )?atol(divide_str):1; - long long before = (before_str && *before_str )?atol(before_str):0; - long long after = (after_str && *after_str )?atol(after_str):-st->update_every; - int points = (points_str && *points_str )?atoi(points_str):1; - int precision = (precision_str && *precision_str)?atoi(precision_str):-1; + long long multiply = (multiply_str && *multiply_str )?str2l(multiply_str):1; + long long divide = (divide_str && *divide_str )?str2l(divide_str):1; + long long before = (before_str && *before_str )?str2l(before_str):0; + long long after = (after_str && *after_str )?str2l(after_str):-st->update_every; + int points = (points_str && *points_str )?str2i(points_str):1; + int precision = (precision_str && *precision_str)?str2i(precision_str):-1; if(!multiply) multiply = 1; if(!divide) divide = 1; @@ -868,7 +934,7 @@ int web_client_api_request_v1_badge(struct web_client *w, char *url) { } } else { - refresh = atoi(refresh_str); + refresh = str2i(refresh_str); if(refresh < 0) refresh = -refresh; } } @@ -921,7 +987,7 @@ int web_client_api_request_v1_badge(struct web_client *w, char *url) { if (refresh > 0) { buffer_sprintf(w->response.header, "Refresh: %d\r\n", refresh); - w->response.data->expires = time(NULL) + refresh; + w->response.data->expires = now_realtime_sec() + refresh; } else buffer_no_cacheable(w->response.data); @@ -970,7 +1036,7 @@ int web_client_api_request_v1_badge(struct web_client *w, char *url) { ret = 500; // if the collected value is too old, don't calculate its value - if (rrdset_last_entry_t(st) >= (time(NULL) - (st->update_every * st->gap_when_lost_iterations_above))) + if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above))) ret = rrd2value(st, w->response.data, &n, @@ -993,7 +1059,7 @@ int web_client_api_request_v1_badge(struct web_client *w, char *url) { } else if (refresh > 0) { buffer_sprintf(w->response.header, "Refresh: %d\r\n", refresh); - w->response.data->expires = time(NULL) + refresh; + w->response.data->expires = now_realtime_sec() + refresh; } else buffer_no_cacheable(w->response.data); @@ -1120,14 +1186,15 @@ int web_client_api_request_v1_data(struct web_client *w, char *url) RRDSET *st = rrdset_find(chart); if(!st) st = rrdset_find_byname(chart); if(!st) { - buffer_sprintf(w->response.data, "Chart '%s' is not found.", chart); + buffer_strcat(w->response.data, "Chart is not found: "); + buffer_strcat_htmlescape(w->response.data, chart); ret = 404; goto cleanup; } - long long before = (before_str && *before_str)?atol(before_str):0; - long long after = (after_str && *after_str) ?atol(after_str):0; - int points = (points_str && *points_str)?atoi(points_str):0; + long long before = (before_str && *before_str)?str2l(before_str):0; + long long after = (after_str && *after_str) ?str2l(after_str):0; + int points = (points_str && *points_str)?str2i(points_str):0; debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', format '%u', options '0x%08x'" , w->id @@ -1189,8 +1256,6 @@ cleanup: } -#define REGISTRY_VERIFY_COOKIES_GUID "give-me-back-this-cookie-now--please" - int web_client_api_request_v1_registry(struct web_client *w, char *url) { static uint32_t hash_action = 0, hash_access = 0, hash_hello = 0, hash_delete = 0, hash_search = 0, @@ -1215,7 +1280,7 @@ int web_client_api_request_v1_registry(struct web_client *w, char *url) */ } - char person_guid[36 + 1] = ""; + char person_guid[GUID_LEN + 1] = ""; debug(D_WEB_CLIENT, "%llu: API v1 registry with URL '%s'", w->id, url); @@ -1299,100 +1364,50 @@ int web_client_api_request_v1_registry(struct web_client *w, char *url) } if(action == 'A' && (!machine_guid || !machine_url || !url_name)) { + error("Invalid registry request - access requires these parameters: machine ('%s'), url ('%s'), name ('%s')", + machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", url_name?url_name:"UNSET"); buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Invalid registry request - access requires these parameters: machine ('%s'), url ('%s'), name ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", url_name?url_name:"UNSET"); + buffer_strcat(w->response.data, "Invalid registry Access request."); return 400; } else if(action == 'D' && (!machine_guid || !machine_url || !delete_url)) { + error("Invalid registry request - delete requires these parameters: machine ('%s'), url ('%s'), delete_url ('%s')", + machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", delete_url?delete_url:"UNSET"); buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Invalid registry request - delete requires these parameters: machine ('%s'), url ('%s'), delete_url ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", delete_url?delete_url:"UNSET"); + buffer_strcat(w->response.data, "Invalid registry Delete request."); return 400; } else if(action == 'S' && (!machine_guid || !machine_url || !search_machine_guid)) { + error("Invalid registry request - search requires these parameters: machine ('%s'), url ('%s'), for ('%s')", + machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", search_machine_guid?search_machine_guid:"UNSET"); buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Invalid registry request - search requires these parameters: machine ('%s'), url ('%s'), for ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", search_machine_guid?search_machine_guid:"UNSET"); + buffer_strcat(w->response.data, "Invalid registry Search request."); return 400; } else if(action == 'W' && (!machine_guid || !machine_url || !to_person_guid)) { + error("Invalid registry request - switching identity requires these parameters: machine ('%s'), url ('%s'), to ('%s')", + machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", to_person_guid?to_person_guid:"UNSET"); buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Invalid registry request - switching identity requires these parameters: machine ('%s'), url ('%s'), to ('%s')", - machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", to_person_guid?to_person_guid:"UNSET"); + buffer_strcat(w->response.data, "Invalid registry Switch request."); return 400; } switch(action) { case 'A': w->tracking_required = 1; - if(registry_verify_cookies_redirects() > 0 && (!cookie || !person_guid[0])) { - buffer_flush(w->response.data); - registry_set_cookie(w, REGISTRY_VERIFY_COOKIES_GUID); - w->response.data->contenttype = CT_APPLICATION_JSON; - buffer_sprintf(w->response.data, "{ \"status\": \"redirect\", \"registry\": \"%s\" }", registry_to_announce()); - return 200; - -/* - * it seems that web browsers are ignoring 307 (Moved Temporarily) - * under certain conditions, when using CORS - * so this is commented and we use application level redirects instead - * - redirects++; - - if(redirects > registry_verify_cookies_redirects()) { - buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Your browser does not support cookies"); - return 400; - } - - char *encoded_url = url_encode(machine_url); - if(!encoded_url) { - error("%llu: Cannot URL encode string '%s'", w->id, machine_url); - return 500; - } - - char *encoded_name = url_encode(url_name); - if(!encoded_name) { - free(encoded_url); - error("%llu: Cannot URL encode string '%s'", w->id, url_name); - return 500; - } - - char *encoded_guid = url_encode(machine_guid); - if(!encoded_guid) { - free(encoded_url); - free(encoded_name); - error("%llu: Cannot URL encode string '%s'", w->id, machine_guid); - return 500; - } - - buffer_sprintf(w->response.header, "Location: %s/api/v1/registry?action=access&machine=%s&name=%s&url=%s&redirects=%d\r\n", - registry_to_announce(), encoded_guid, encoded_name, encoded_url, redirects); - - free(encoded_guid); - free(encoded_name); - free(encoded_url); - return 307 -*/ - } - - if(unlikely(cookie && person_guid[0] && !strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID))) - person_guid[0] = '\0'; - - return registry_request_access_json(w, person_guid, machine_guid, machine_url, url_name, time(NULL)); + return registry_request_access_json(w, person_guid, machine_guid, machine_url, url_name, now_realtime_sec()); case 'D': w->tracking_required = 1; - return registry_request_delete_json(w, person_guid, machine_guid, machine_url, delete_url, time(NULL)); + return registry_request_delete_json(w, person_guid, machine_guid, machine_url, delete_url, now_realtime_sec()); case 'S': w->tracking_required = 1; - return registry_request_search_json(w, person_guid, machine_guid, machine_url, search_machine_guid, time(NULL)); + return registry_request_search_json(w, person_guid, machine_guid, machine_url, search_machine_guid, now_realtime_sec()); case 'W': w->tracking_required = 1; - return registry_request_switch_json(w, person_guid, machine_guid, machine_url, to_person_guid, time(NULL)); + return registry_request_switch_json(w, person_guid, machine_guid, machine_url, to_person_guid, now_realtime_sec()); case 'H': return registry_request_hello_json(w); @@ -1405,7 +1420,7 @@ int web_client_api_request_v1_registry(struct web_client *w, char *url) } int web_client_api_request_v1(struct web_client *w, char *url) { - static uint32_t hash_data = 0, hash_chart = 0, hash_charts = 0, hash_registry = 0, hash_badge = 0, hash_alarms = 0, hash_alarm_log = 0; + static uint32_t hash_data = 0, hash_chart = 0, hash_charts = 0, hash_registry = 0, hash_badge = 0, hash_alarms = 0, hash_alarm_log = 0, hash_alarm_variables = 0, hash_raw = 0; if(unlikely(hash_data == 0)) { hash_data = simple_hash("data"); @@ -1415,6 +1430,8 @@ int web_client_api_request_v1(struct web_client *w, char *url) { hash_badge = simple_hash("badge.svg"); hash_alarms = simple_hash("alarms"); hash_alarm_log = simple_hash("alarm_log"); + hash_alarm_variables = simple_hash("alarm_variables"); + hash_raw = simple_hash("allmetrics"); } // get the command @@ -1444,15 +1461,22 @@ int web_client_api_request_v1(struct web_client *w, char *url) { else if(hash == hash_alarm_log && !strcmp(tok, "alarm_log")) return web_client_api_request_v1_alarm_log(w, url); + else if(hash == hash_alarm_variables && !strcmp(tok, "alarm_variables")) + return web_client_api_request_v1_alarm_variables(w, url); + + else if(hash == hash_raw && !strcmp(tok, "allmetrics")) + return web_client_api_request_v1_allmetrics(w, url); + else { buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Unsupported v1 API command: %s", tok); + buffer_strcat(w->response.data, "Unsupported v1 API command: "); + buffer_strcat_htmlescape(w->response.data, tok); return 404; } } else { buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "API v1 command?"); + buffer_sprintf(w->response.data, "Which API v1 command?"); return 400; } } @@ -1467,7 +1491,8 @@ int web_client_api_request(struct web_client *w, char *url) return web_client_api_request_v1(w, url); else { buffer_flush(w->response.data); - buffer_sprintf(w->response.data, "Unsupported API version: %s", tok); + buffer_strcat(w->response.data, "Unsupported API version: "); + buffer_strcat_htmlescape(w->response.data, tok); return 404; } } @@ -1480,6 +1505,12 @@ int web_client_api_request(struct web_client *w, char *url) int web_client_api_old_data_request(struct web_client *w, char *url, int datasource_type) { + if(!url || !*url) { + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, "Incomplete request."); + return 400; + } + RRDSET *st = NULL; char *args = strchr(url, '?'); @@ -1519,13 +1550,13 @@ int web_client_api_old_data_request(struct web_client *w, char *url, int datasou if(url) { // parse the lines required tok = mystrsep(&url, "/"); - if(tok) lines = atoi(tok); + if(tok) lines = str2i(tok); if(lines < 1) lines = 1; } if(url) { // parse the group count required tok = mystrsep(&url, "/"); - if(tok && *tok) group_count = atoi(tok); + if(tok && *tok) group_count = str2i(tok); if(group_count < 1) group_count = 1; //if(group_count > save_history / 20) group_count = save_history / 20; } @@ -1542,13 +1573,13 @@ int web_client_api_old_data_request(struct web_client *w, char *url, int datasou if(url) { // parse after time tok = mystrsep(&url, "/"); - if(tok && *tok) after = strtoul(tok, NULL, 10); + if(tok && *tok) after = str2ul(tok); if(after < 0) after = 0; } if(url) { // parse before time tok = mystrsep(&url, "/"); - if(tok && *tok) before = strtoul(tok, NULL, 10); + if(tok && *tok) before = str2ul(tok); if(before < 0) before = 0; } if(url) { @@ -1707,6 +1738,9 @@ const char *web_content_type_to_string(uint8_t contenttype) { case CT_IMAGE_ICNS: return "image/icns"; + case CT_PROMETHEUS: + return "text/plain; version=0.0.4"; + default: case CT_TEXT_PLAIN: return "text/plain; charset=utf-8"; @@ -1917,7 +1951,7 @@ void web_client_process(struct web_client *w) { #endif // start timing us - gettimeofday(&w->tv_in, NULL); + now_realtime_timeval(&w->tv_in); if(unlikely(!hash_api)) { hash_api = simple_hash("api"); @@ -2063,7 +2097,6 @@ void web_client_process(struct web_client *w) { error("web request to exit received."); netdata_cleanup_and_exit(0); - netdata_exit = 1; } else if(hash == hash_debug && strcmp(tok, "debug") == 0) { buffer_flush(w->response.data); @@ -2078,14 +2111,16 @@ void web_client_process(struct web_client *w) { if(!st) st = rrdset_find(tok); if(!st) { code = 404; - buffer_sprintf(w->response.data, "Chart %s is not found.\r\n", tok); + buffer_strcat(w->response.data, "Chart is not found: "); + buffer_strcat_htmlescape(w->response.data, tok); debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok); } else { code = 200; debug_flags |= D_RRD_STATS; st->debug = !st->debug; - buffer_sprintf(w->response.data, "Chart %s has now debug %s.\r\n", tok, st->debug?"enabled":"disabled"); + buffer_sprintf(w->response.data, "Chart has now debug %s: ", st->debug?"enabled":"disabled"); + buffer_strcat_htmlescape(w->response.data, tok); debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, st->debug?"enabled":"disabled"); } } @@ -2127,7 +2162,7 @@ void web_client_process(struct web_client *w) { } } - gettimeofday(&w->tv_ready, NULL); + now_realtime_timeval(&w->tv_ready); w->response.sent = 0; w->response.code = code; @@ -2210,7 +2245,7 @@ void web_client_process(struct web_client *w) { if(w->mode == WEB_CLIENT_MODE_OPTIONS) { buffer_strcat(w->response.header_output, "Access-Control-Allow-Methods: GET, OPTIONS\r\n" - "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie\r\n" + "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie, pragma, cache-control\r\n" "Access-Control-Max-Age: 1209600\r\n" // 86400 * 14 ); } diff --git a/src/web_server.c b/src/web_server.c index cbbe6bb40..8e942a59d 100644 --- a/src/web_server.c +++ b/src/web_server.c @@ -7,6 +7,8 @@ char *listen_fds_names[MAX_LISTEN_FDS] = { [0 ... 99] = NULL }; int listen_port = LISTEN_PORT; int web_server_mode = WEB_SERVER_MODE_MULTI_THREADED; +static int shown_server_socket_error = 0; + #ifdef NETDATA_INTERNAL_CHECKS static void log_allocations(void) { @@ -90,6 +92,7 @@ int create_listen_socket4(const char *ip, int port, int listen_backlog) { sock = socket(AF_INET, SOCK_STREAM, 0); if(sock < 0) { error("IPv4 socket() on ip '%s' port %d failed.", ip, port); + shown_server_socket_error = 1; return -1; } @@ -105,6 +108,7 @@ int create_listen_socket4(const char *ip, int port, int listen_backlog) { int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr); if(ret != 1) { error("Failed to convert IP '%s' to a valid IPv4 address.", ip); + shown_server_socket_error = 1; close(sock); return -1; } @@ -112,12 +116,14 @@ int create_listen_socket4(const char *ip, int port, int listen_backlog) { if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { close(sock); error("IPv4 bind() on ip '%s' port %d failed.", ip, port); + shown_server_socket_error = 1; return -1; } if(listen(sock, listen_backlog) < 0) { close(sock); - fatal("IPv4 listen() on ip '%s' port %d failed.", ip, port); + error("IPv4 listen() on ip '%s' port %d failed.", ip, port); + shown_server_socket_error = 1; return -1; } @@ -135,6 +141,7 @@ int create_listen_socket6(const char *ip, int port, int listen_backlog) { sock = socket(AF_INET6, SOCK_STREAM, 0); if (sock < 0) { error("IPv6 socket() on ip '%s' port %d failed.", ip, port); + shown_server_socket_error = 1; return -1; } @@ -154,6 +161,7 @@ int create_listen_socket6(const char *ip, int port, int listen_backlog) { int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr); if(ret != 1) { error("Failed to convert IP '%s' to a valid IPv6 address.", ip); + shown_server_socket_error = 1; close(sock); return -1; } @@ -163,12 +171,14 @@ int create_listen_socket6(const char *ip, int port, int listen_backlog) { if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { close(sock); error("IPv6 bind() on ip '%s' port %d failed.", ip, port); + shown_server_socket_error = 1; return -1; } if (listen(sock, listen_backlog) < 0) { close(sock); error("IPv6 listen() on ip '%s' port %d failed.", ip, port); + shown_server_socket_error = 1; return -1; } @@ -179,6 +189,7 @@ int create_listen_socket6(const char *ip, int port, int listen_backlog) { static inline int add_listen_socket(int fd, const char *ip, int port) { if(listen_fds_count >= MAX_LISTEN_FDS) { error("Too many listening sockets. Failed to add listening socket at ip '%s' port %d", ip, port); + shown_server_socket_error = 1; close(fd); return -1; } @@ -290,7 +301,7 @@ static inline int bind_to_one(const char *definition, int default_port, int list } if (fd == -1) - error("Cannot bind to ip '%s', port %d", rip, default_port); + error("Cannot bind to ip '%s', port %d", rip, rport); else { add_listen_socket(fd, rip, rport); added++; @@ -303,6 +314,8 @@ static inline int bind_to_one(const char *definition, int default_port, int list } int create_listen_sockets(void) { + shown_server_socket_error = 0; + listen_backlog = (int) config_get_number("global", "http port listen backlog", LISTEN_BACKLOG); if(config_exists("global", "bind socket to IP") && !config_exists("global", "bind to")) @@ -340,6 +353,11 @@ int create_listen_sockets(void) { if(!listen_fds_count) fatal("Cannot listen on any socket. Exiting..."); + else if(shown_server_socket_error) { + size_t i; + for(i = 0; i < listen_fds_count ;i++) + info("Listen socket %s opened.", listen_fds_names[i]); + } return (int)listen_fds_count; } @@ -372,7 +390,7 @@ static inline void cleanup_web_clients(void) { #define CLEANUP_EVERY_EVENTS 100 void *socket_listen_main_multi_threaded(void *ptr) { - (void)ptr; + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; web_server_mode = WEB_SERVER_MODE_MULTI_THREADED; info("Multi-threaded WEB SERVER thread created with task id %d", gettid()); @@ -452,6 +470,10 @@ void *socket_listen_main_multi_threaded(void *ptr) { debug(D_WEB_CLIENT, "LISTENER: exit!"); close_listen_sockets(); + freez(fds); + + static_thread->enabled = 0; + pthread_exit(NULL); return NULL; } @@ -500,7 +522,7 @@ static inline int single_threaded_unlink_client(struct web_client *w, fd_set *if } void *socket_listen_main_single_threaded(void *ptr) { - (void)ptr; + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; web_server_mode = WEB_SERVER_MODE_SINGLE_THREADED; @@ -619,5 +641,8 @@ void *socket_listen_main_single_threaded(void *ptr) { debug(D_WEB_CLIENT, "LISTENER: exit!"); close_listen_sockets(); + + static_thread->enabled = 0; + pthread_exit(NULL); return NULL; } |