diff options
Diffstat (limited to 'daemon')
| -rw-r--r-- | daemon/Makefile.am | 2 | ||||
| -rw-r--r-- | daemon/Makefile.in | 619 | ||||
| -rw-r--r-- | daemon/README.md | 44 | ||||
| -rw-r--r-- | daemon/anonymous-statistics.sh | 89 | ||||
| -rwxr-xr-x | daemon/anonymous-statistics.sh.in | 95 | ||||
| -rw-r--r-- | daemon/buildinfo.c | 229 | ||||
| -rw-r--r-- | daemon/buildinfo.h | 8 | ||||
| -rw-r--r-- | daemon/commands.c | 726 | ||||
| -rw-r--r-- | daemon/commands.h | 81 | ||||
| -rw-r--r-- | daemon/common.c | 5 | ||||
| -rw-r--r-- | daemon/common.h | 21 | ||||
| -rw-r--r-- | daemon/config/README.md | 153 | ||||
| -rw-r--r-- | daemon/daemon.c | 49 | ||||
| -rw-r--r-- | daemon/daemon.h | 4 | ||||
| -rw-r--r-- | daemon/get-kubernetes-labels.sh.in | 18 | ||||
| -rw-r--r-- | daemon/global_statistics.c | 101 | ||||
| -rw-r--r-- | daemon/main.c | 326 | ||||
| -rw-r--r-- | daemon/signals.c | 31 | ||||
| -rw-r--r-- | daemon/signals.h | 1 | ||||
| -rwxr-xr-x | daemon/system-info.sh | 459 | ||||
| -rw-r--r-- | daemon/unit_test.c | 106 | ||||
| -rw-r--r-- | daemon/unit_test.h | 2 |
22 files changed, 2154 insertions, 1015 deletions
diff --git a/daemon/Makefile.am b/daemon/Makefile.am index 6383b559e..d3102f695 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -13,9 +13,11 @@ dist_noinst_DATA = \ README.md \ config/README.md \ anonymous-statistics.sh.in \ + get-kubernetes-labels.sh.in \ $(NULL) dist_plugins_SCRIPTS = \ anonymous-statistics.sh \ system-info.sh \ + get-kubernetes-labels.sh \ $(NULL) diff --git a/daemon/Makefile.in b/daemon/Makefile.in deleted file mode 100644 index 1c63d6045..000000000 --- a/daemon/Makefile.in +++ /dev/null @@ -1,619 +0,0 @@ -# Makefile.in generated by automake 1.15.1 from Makefile.am. -# @configure_input@ - -# Copyright (C) 1994-2017 Free Software Foundation, Inc. - -# This Makefile.in is free software; the Free Software Foundation -# gives unlimited permission to copy and/or distribute it, -# with or without modifications, as long as this notice is preserved. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY, to the extent permitted by law; without -# even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. - -@SET_MAKE@ - -# SPDX-License-Identifier: GPL-3.0-or-later - - -VPATH = @srcdir@ -am__is_gnu_make = { \ - if test -z '$(MAKELEVEL)'; then \ - false; \ - elif test -n '$(MAKE_HOST)'; then \ - true; \ - elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ - true; \ - else \ - false; \ - fi; \ -} -am__make_running_with_option = \ - case $${target_option-} in \ - ?) ;; \ - *) echo "am__make_running_with_option: internal error: invalid" \ - "target option '$${target_option-}' specified" >&2; \ - exit 1;; \ - esac; \ - has_opt=no; \ - sane_makeflags=$$MAKEFLAGS; \ - if $(am__is_gnu_make); then \ - sane_makeflags=$$MFLAGS; \ - else \ - case $$MAKEFLAGS in \ - *\\[\ \ ]*) \ - bs=\\; \ - sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ - | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ - esac; \ - fi; \ - skip_next=no; \ - strip_trailopt () \ - { \ - flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ - }; \ - for flg in $$sane_makeflags; do \ - test $$skip_next = yes && { skip_next=no; continue; }; \ - case $$flg in \ - *=*|--*) continue;; \ - -*I) strip_trailopt 'I'; skip_next=yes;; \ - -*I?*) strip_trailopt 'I';; \ - -*O) strip_trailopt 'O'; skip_next=yes;; \ - -*O?*) strip_trailopt 'O';; \ - -*l) strip_trailopt 'l'; skip_next=yes;; \ - -*l?*) strip_trailopt 'l';; \ - -[dEDm]) skip_next=yes;; \ - -[JT]) skip_next=yes;; \ - esac; \ - case $$flg in \ - *$$target_option*) has_opt=yes; break;; \ - esac; \ - done; \ - test $$has_opt = yes -am__make_dryrun = (target_option=n; $(am__make_running_with_option)) -am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) -pkgdatadir = $(datadir)/@PACKAGE@ -pkgincludedir = $(includedir)/@PACKAGE@ -pkglibdir = $(libdir)/@PACKAGE@ -pkglibexecdir = $(libexecdir)/@PACKAGE@ -am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd -install_sh_DATA = $(install_sh) -c -m 644 -install_sh_PROGRAM = $(install_sh) -c -install_sh_SCRIPT = $(install_sh) -c -INSTALL_HEADER = $(INSTALL_DATA) -transform = $(program_transform_name) -NORMAL_INSTALL = : -PRE_INSTALL = : -POST_INSTALL = : -NORMAL_UNINSTALL = : -PRE_UNINSTALL = : -POST_UNINSTALL = : -build_triplet = @build@ -host_triplet = @host@ -subdir = daemon -ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/build/m4/ax_c___atomic.m4 \ - $(top_srcdir)/build/m4/ax_c__generic.m4 \ - $(top_srcdir)/build/m4/ax_c_lto.m4 \ - $(top_srcdir)/build/m4/ax_c_mallinfo.m4 \ - $(top_srcdir)/build/m4/ax_c_mallopt.m4 \ - $(top_srcdir)/build/m4/ax_check_compile_flag.m4 \ - $(top_srcdir)/build/m4/ax_gcc_func_attribute.m4 \ - $(top_srcdir)/build/m4/ax_pthread.m4 \ - $(top_srcdir)/build/m4/jemalloc.m4 \ - $(top_srcdir)/build/m4/tcmalloc.m4 $(top_srcdir)/configure.ac -am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ - $(ACLOCAL_M4) -DIST_COMMON = $(srcdir)/Makefile.am $(dist_plugins_SCRIPTS) \ - $(dist_noinst_DATA) $(am__DIST_COMMON) -mkinstalldirs = $(install_sh) -d -CONFIG_HEADER = $(top_builddir)/config.h -CONFIG_CLEAN_FILES = -CONFIG_CLEAN_VPATH_FILES = -am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; -am__vpath_adj = case $$p in \ - $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ - *) f=$$p;; \ - esac; -am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; -am__install_max = 40 -am__nobase_strip_setup = \ - srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` -am__nobase_strip = \ - for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" -am__nobase_list = $(am__nobase_strip_setup); \ - for p in $$list; do echo "$$p $$p"; done | \ - sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ - $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ - if (++n[$$2] == $(am__install_max)) \ - { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ - END { for (dir in files) print dir, files[dir] }' -am__base_list = \ - sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ - sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' -am__uninstall_files_from_dir = { \ - test -z "$$files" \ - || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ - || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ - $(am__cd) "$$dir" && rm -f $$files; }; \ - } -am__installdirs = "$(DESTDIR)$(pluginsdir)" -SCRIPTS = $(dist_plugins_SCRIPTS) -AM_V_P = $(am__v_P_@AM_V@) -am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) -am__v_P_0 = false -am__v_P_1 = : -AM_V_GEN = $(am__v_GEN_@AM_V@) -am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) -am__v_GEN_0 = @echo " GEN " $@; -am__v_GEN_1 = -AM_V_at = $(am__v_at_@AM_V@) -am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) -am__v_at_0 = @ -am__v_at_1 = -SOURCES = -DIST_SOURCES = -am__can_run_installinfo = \ - case $$AM_UPDATE_INFO_DIR in \ - n|no|NO) false;; \ - *) (install-info --version) >/dev/null 2>&1;; \ - esac -DATA = $(dist_noinst_DATA) -am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) -am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/build/subst.inc -DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) -ACLOCAL = @ACLOCAL@ -AMTAR = @AMTAR@ -AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ -AUTOCONF = @AUTOCONF@ -AUTOHEADER = @AUTOHEADER@ -AUTOMAKE = @AUTOMAKE@ -AWK = @AWK@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ -CMOCKA_CFLAGS = @CMOCKA_CFLAGS@ -CMOCKA_LIBS = @CMOCKA_LIBS@ -CPP = @CPP@ -CPPFLAGS = @CPPFLAGS@ -CUPSCONFIG = @CUPSCONFIG@ -CXX = @CXX@ -CXXDEPMODE = @CXXDEPMODE@ -CXXFLAGS = @CXXFLAGS@ -CXX_BINARY = @CXX_BINARY@ -CYGPATH_W = @CYGPATH_W@ -DEFS = @DEFS@ -DEPDIR = @DEPDIR@ -ECHO_C = @ECHO_C@ -ECHO_N = @ECHO_N@ -ECHO_T = @ECHO_T@ -EGREP = @EGREP@ -ENABLE_UNITTESTS = @ENABLE_UNITTESTS@ -EXEEXT = @EXEEXT@ -GREP = @GREP@ -INSTALL = @INSTALL@ -INSTALL_DATA = @INSTALL_DATA@ -INSTALL_PROGRAM = @INSTALL_PROGRAM@ -INSTALL_SCRIPT = @INSTALL_SCRIPT@ -INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ -IPMIMONITORING_CFLAGS = @IPMIMONITORING_CFLAGS@ -IPMIMONITORING_LIBS = @IPMIMONITORING_LIBS@ -JSON_CFLAGS = @JSON_CFLAGS@ -JSON_LIBS = @JSON_LIBS@ -LDFLAGS = @LDFLAGS@ -LIBCAP_CFLAGS = @LIBCAP_CFLAGS@ -LIBCAP_LIBS = @LIBCAP_LIBS@ -LIBCRYPTO_CFLAGS = @LIBCRYPTO_CFLAGS@ -LIBCRYPTO_LIBS = @LIBCRYPTO_LIBS@ -LIBCURL_CFLAGS = @LIBCURL_CFLAGS@ -LIBCURL_LIBS = @LIBCURL_LIBS@ -LIBMNL_CFLAGS = @LIBMNL_CFLAGS@ -LIBMNL_LIBS = @LIBMNL_LIBS@ -LIBMONGOC_CFLAGS = @LIBMONGOC_CFLAGS@ -LIBMONGOC_LIBS = @LIBMONGOC_LIBS@ -LIBOBJS = @LIBOBJS@ -LIBS = @LIBS@ -LIBSSL_CFLAGS = @LIBSSL_CFLAGS@ -LIBSSL_LIBS = @LIBSSL_LIBS@ -LTLIBOBJS = @LTLIBOBJS@ -MAINT = @MAINT@ -MAKEINFO = @MAKEINFO@ -MATH_CFLAGS = @MATH_CFLAGS@ -MATH_LIBS = @MATH_LIBS@ -MKDIR_P = @MKDIR_P@ -NFACCT_CFLAGS = @NFACCT_CFLAGS@ -NFACCT_LIBS = @NFACCT_LIBS@ -OBJEXT = @OBJEXT@ -OPTIONAL_CUPS_CFLAGS = @OPTIONAL_CUPS_CFLAGS@ -OPTIONAL_CUPS_LIBS = @OPTIONAL_CUPS_LIBS@ -OPTIONAL_IPMIMONITORING_CFLAGS = @OPTIONAL_IPMIMONITORING_CFLAGS@ -OPTIONAL_IPMIMONITORING_LIBS = @OPTIONAL_IPMIMONITORING_LIBS@ -OPTIONAL_JSONC_LIBS = @OPTIONAL_JSONC_LIBS@ -OPTIONAL_JUDY_LIBS = @OPTIONAL_JUDY_LIBS@ -OPTIONAL_KINESIS_CFLAGS = @OPTIONAL_KINESIS_CFLAGS@ -OPTIONAL_KINESIS_LIBS = @OPTIONAL_KINESIS_LIBS@ -OPTIONAL_LIBCAP_CFLAGS = @OPTIONAL_LIBCAP_CFLAGS@ -OPTIONAL_LIBCAP_LIBS = @OPTIONAL_LIBCAP_LIBS@ -OPTIONAL_LZ4_LIBS = @OPTIONAL_LZ4_LIBS@ -OPTIONAL_MATH_CFLAGS = @OPTIONAL_MATH_CFLAGS@ -OPTIONAL_MATH_LIBS = @OPTIONAL_MATH_LIBS@ -OPTIONAL_MONGOC_CFLAGS = @OPTIONAL_MONGOC_CFLAGS@ -OPTIONAL_MONGOC_LIBS = @OPTIONAL_MONGOC_LIBS@ -OPTIONAL_NFACCT_CFLAGS = @OPTIONAL_NFACCT_CFLAGS@ -OPTIONAL_NFACCT_LIBS = @OPTIONAL_NFACCT_LIBS@ -OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS = @OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS@ -OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS = @OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS@ -OPTIONAL_SSL_LIBS = @OPTIONAL_SSL_LIBS@ -OPTIONAL_UUID_CFLAGS = @OPTIONAL_UUID_CFLAGS@ -OPTIONAL_UUID_LIBS = @OPTIONAL_UUID_LIBS@ -OPTIONAL_UV_LIBS = @OPTIONAL_UV_LIBS@ -OPTIONAL_XENSTAT_CFLAGS = @OPTIONAL_XENSTAT_CFLAGS@ -OPTIONAL_XENSTAT_LIBS = @OPTIONAL_XENSTAT_LIBS@ -OPTIONAL_ZLIB_CFLAGS = @OPTIONAL_ZLIB_CFLAGS@ -OPTIONAL_ZLIB_LIBS = @OPTIONAL_ZLIB_LIBS@ -PACKAGE = @PACKAGE@ -PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_RPM_VERSION = @PACKAGE_RPM_VERSION@ -PACKAGE_STRING = @PACKAGE_STRING@ -PACKAGE_TARNAME = @PACKAGE_TARNAME@ -PACKAGE_URL = @PACKAGE_URL@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -PATH_SEPARATOR = @PATH_SEPARATOR@ -PKG_CONFIG = @PKG_CONFIG@ -PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ -PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ -PROTOBUF_CFLAGS = @PROTOBUF_CFLAGS@ -PROTOBUF_LIBS = @PROTOBUF_LIBS@ -PROTOC = @PROTOC@ -PTHREAD_CC = @PTHREAD_CC@ -PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ -PTHREAD_LIBS = @PTHREAD_LIBS@ -SET_MAKE = @SET_MAKE@ -SHELL = @SHELL@ -SSE_CANDIDATE = @SSE_CANDIDATE@ -STRIP = @STRIP@ -TEST_CFLAGS = @TEST_CFLAGS@ -TEST_LIBS = @TEST_LIBS@ -UUID_CFLAGS = @UUID_CFLAGS@ -UUID_LIBS = @UUID_LIBS@ -VERSION = @VERSION@ -XENLIGHT_CFLAGS = @XENLIGHT_CFLAGS@ -XENLIGHT_LIBS = @XENLIGHT_LIBS@ -YAJL_CFLAGS = @YAJL_CFLAGS@ -YAJL_LIBS = @YAJL_LIBS@ -ZLIB_CFLAGS = @ZLIB_CFLAGS@ -ZLIB_LIBS = @ZLIB_LIBS@ -abs_builddir = @abs_builddir@ -abs_srcdir = @abs_srcdir@ -abs_top_builddir = @abs_top_builddir@ -abs_top_srcdir = @abs_top_srcdir@ -ac_ct_CC = @ac_ct_CC@ -ac_ct_CXX = @ac_ct_CXX@ -am__include = @am__include@ -am__leading_dot = @am__leading_dot@ -am__quote = @am__quote@ -am__tar = @am__tar@ -am__untar = @am__untar@ -ax_pthread_config = @ax_pthread_config@ -bindir = @bindir@ -build = @build@ -build_alias = @build_alias@ -build_cpu = @build_cpu@ -build_os = @build_os@ -build_target = @build_target@ -build_vendor = @build_vendor@ -builddir = @builddir@ -cachedir = @cachedir@ -chartsdir = @chartsdir@ -configdir = @configdir@ -datadir = @datadir@ -datarootdir = @datarootdir@ -docdir = @docdir@ -dvidir = @dvidir@ -exec_prefix = @exec_prefix@ -has_jemalloc = @has_jemalloc@ -has_tcmalloc = @has_tcmalloc@ -host = @host@ -host_alias = @host_alias@ -host_cpu = @host_cpu@ -host_os = @host_os@ -host_vendor = @host_vendor@ -htmldir = @htmldir@ -includedir = @includedir@ -infodir = @infodir@ -install_sh = @install_sh@ -libconfigdir = @libconfigdir@ -libdir = @libdir@ -libexecdir = @libexecdir@ -localedir = @localedir@ -localstatedir = @localstatedir@ -logdir = @logdir@ -mandir = @mandir@ -mkdir_p = @mkdir_p@ -nodedir = @nodedir@ -oldincludedir = @oldincludedir@ -pdfdir = @pdfdir@ -pluginsdir = @pluginsdir@ -prefix = @prefix@ -program_transform_name = @program_transform_name@ -psdir = @psdir@ -pythondir = @pythondir@ -registrydir = @registrydir@ -runstatedir = @runstatedir@ -sbindir = @sbindir@ -sharedstatedir = @sharedstatedir@ -srcdir = @srcdir@ -sysconfdir = @sysconfdir@ -target_alias = @target_alias@ -top_build_prefix = @top_build_prefix@ -top_builddir = @top_builddir@ -top_srcdir = @top_srcdir@ -varlibdir = @varlibdir@ -webdir = @webdir@ -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in -CLEANFILES = \ - anonymous-statistics.sh \ - $(NULL) - -SUFFIXES = .in -dist_noinst_DATA = \ - README.md \ - config/README.md \ - anonymous-statistics.sh.in \ - $(NULL) - -dist_plugins_SCRIPTS = \ - anonymous-statistics.sh \ - system-info.sh \ - $(NULL) - -all: all-am - -.SUFFIXES: -.SUFFIXES: .in -$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(top_srcdir)/build/subst.inc $(am__configure_deps) - @for dep in $?; do \ - case '$(am__configure_deps)' in \ - *$$dep*) \ - ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ - && { if test -f $@; then exit 0; else break; fi; }; \ - exit 1;; \ - esac; \ - done; \ - echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu daemon/Makefile'; \ - $(am__cd) $(top_srcdir) && \ - $(AUTOMAKE) --gnu daemon/Makefile -Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status - @case '$?' in \ - *config.status*) \ - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ - *) \ - echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ - cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ - esac; -$(top_srcdir)/build/subst.inc $(am__empty): - -$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh - -$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(am__aclocal_m4_deps): -install-dist_pluginsSCRIPTS: $(dist_plugins_SCRIPTS) - @$(NORMAL_INSTALL) - @list='$(dist_plugins_SCRIPTS)'; test -n "$(pluginsdir)" || list=; \ - if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(pluginsdir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(pluginsdir)" || exit 1; \ - fi; \ - for p in $$list; do \ - if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ - if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ - done | \ - sed -e 'p;s,.*/,,;n' \ - -e 'h;s|.*|.|' \ - -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ - $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ - { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ - if ($$2 == $$4) { files[d] = files[d] " " $$1; \ - if (++n[d] == $(am__install_max)) { \ - print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ - else { print "f", d "/" $$4, $$1 } } \ - END { for (d in files) print "f", d, files[d] }' | \ - while read type dir files; do \ - if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ - test -z "$$files" || { \ - echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(pluginsdir)$$dir'"; \ - $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(pluginsdir)$$dir" || exit $$?; \ - } \ - ; done - -uninstall-dist_pluginsSCRIPTS: - @$(NORMAL_UNINSTALL) - @list='$(dist_plugins_SCRIPTS)'; test -n "$(pluginsdir)" || exit 0; \ - files=`for p in $$list; do echo "$$p"; done | \ - sed -e 's,.*/,,;$(transform)'`; \ - dir='$(DESTDIR)$(pluginsdir)'; $(am__uninstall_files_from_dir) -tags TAGS: - -ctags CTAGS: - -cscope cscopelist: - - -distdir: $(DISTFILES) - @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - list='$(DISTFILES)'; \ - dist_files=`for file in $$list; do echo $$file; done | \ - sed -e "s|^$$srcdirstrip/||;t" \ - -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ - case $$dist_files in \ - */*) $(MKDIR_P) `echo "$$dist_files" | \ - sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ - sort -u` ;; \ - esac; \ - for file in $$dist_files; do \ - if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ - if test -d $$d/$$file; then \ - dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ - if test -d "$(distdir)/$$file"; then \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ - cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ - else \ - test -f "$(distdir)/$$file" \ - || cp -p $$d/$$file "$(distdir)/$$file" \ - || exit 1; \ - fi; \ - done -check-am: all-am -check: check-am -all-am: Makefile $(SCRIPTS) $(DATA) -installdirs: - for dir in "$(DESTDIR)$(pluginsdir)"; do \ - test -z "$$dir" || $(MKDIR_P) "$$dir"; \ - done -install: install-am -install-exec: install-exec-am -install-data: install-data-am -uninstall: uninstall-am - -install-am: all-am - @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am - -installcheck: installcheck-am -install-strip: - if test -z '$(STRIP)'; then \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - install; \ - else \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ - fi -mostlyclean-generic: - -clean-generic: - -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) - -distclean-generic: - -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) - -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) - -maintainer-clean-generic: - @echo "This command is intended for maintainers to use" - @echo "it deletes files that may require special tools to rebuild." - -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES) -clean: clean-am - -clean-am: clean-generic mostlyclean-am - -distclean: distclean-am - -rm -f Makefile -distclean-am: clean-am distclean-generic - -dvi: dvi-am - -dvi-am: - -html: html-am - -html-am: - -info: info-am - -info-am: - -install-data-am: install-dist_pluginsSCRIPTS - -install-dvi: install-dvi-am - -install-dvi-am: - -install-exec-am: - -install-html: install-html-am - -install-html-am: - -install-info: install-info-am - -install-info-am: - -install-man: - -install-pdf: install-pdf-am - -install-pdf-am: - -install-ps: install-ps-am - -install-ps-am: - -installcheck-am: - -maintainer-clean: maintainer-clean-am - -rm -f Makefile -maintainer-clean-am: distclean-am maintainer-clean-generic - -mostlyclean: mostlyclean-am - -mostlyclean-am: mostlyclean-generic - -pdf: pdf-am - -pdf-am: - -ps: ps-am - -ps-am: - -uninstall-am: uninstall-dist_pluginsSCRIPTS - -.MAKE: install-am install-strip - -.PHONY: all all-am check check-am clean clean-generic cscopelist-am \ - ctags-am distclean distclean-generic distdir dvi dvi-am html \ - html-am info info-am install install-am install-data \ - install-data-am install-dist_pluginsSCRIPTS install-dvi \ - install-dvi-am install-exec install-exec-am install-html \ - install-html-am install-info install-info-am install-man \ - install-pdf install-pdf-am install-ps install-ps-am \ - install-strip installcheck installcheck-am installdirs \ - maintainer-clean maintainer-clean-generic mostlyclean \ - mostlyclean-generic pdf pdf-am ps ps-am tags-am uninstall \ - uninstall-am uninstall-dist_pluginsSCRIPTS - -.PRECIOUS: Makefile - -.in: - if sed \ - -e 's#[@]localstatedir_POST@#$(localstatedir)#g' \ - -e 's#[@]sbindir_POST@#$(sbindir)#g' \ - -e 's#[@]pluginsdir_POST@#$(pluginsdir)#g' \ - -e 's#[@]configdir_POST@#$(configdir)#g' \ - -e 's#[@]libconfigdir_POST@#$(libconfigdir)#g' \ - -e 's#[@]cachedir_POST@#$(cachedir)#g' \ - -e 's#[@]registrydir_POST@#$(registrydir)#g' \ - -e 's#[@]varlibdir_POST@#$(varlibdir)#g' \ - $< > $@.tmp; then \ - mv "$@.tmp" "$@"; \ - else \ - rm -f "$@.tmp"; \ - false; \ - fi - -# Tell versions [3.59,3.63) of GNU make to not export all variables. -# Otherwise a system limit (for SysV at least) may be exceeded. -.NOEXPORT: diff --git a/daemon/README.md b/daemon/README.md index 0d4b0cdbb..ec1f1c7cc 100644 --- a/daemon/README.md +++ b/daemon/README.md @@ -1,12 +1,18 @@ +<!-- +title: "Netdata daemon" +date: 2020-04-29 +custom_edit_url: https://github.com/netdata/netdata/edit/master/daemon/README.md +--> + # Netdata daemon ## Starting netdata - You can start Netdata by executing it with `/usr/sbin/netdata` (the installer will also start it). -- You can stop Netdata by killing it with `killall netdata`. You can stop and start Netdata at any point. Netdata - saves on exit its round robbin database to `/var/cache/netdata` so that it will continue from where it stopped the - last time. +- You can stop Netdata by killing it with `killall netdata`. You can stop and start Netdata at any point. When + exiting, the [database engine](/database/engine/README.md) saves metrics to `/var/cache/netdata/dbengine/` so that + it can continue when started again. Access to the web site, for all graphs, is by default on port `19999`, so go to: @@ -110,15 +116,15 @@ The command line options of the Netdata 1.10.0 version are the following: | '-' '-' '-' '-' real-time performance monitoring, done right! +----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+---> - Copyright (C) 2016-2017, Costa Tsaousis <costa@tsaousis.gr> + Copyright (C) 2016-2020, Netdata, Inc. <info@netdata.cloud> Released under GNU General Public License v3 or later. All rights reserved. - Home Page : https://my-netdata.io + Home Page : https://netdata.cloud Source Code: https://github.com/netdata/netdata - Wiki / Docs: https://github.com/netdata/netdata/wiki + Docs : https://learn.netdata.cloud Support : https://github.com/netdata/netdata/issues - License : https://github.com/netdata/netdata/blob/master/LICENSE + License : https://github.com/netdata/netdata/blob/master/LICENSE.md Twitter : https://twitter.com/linuxnetdata Facebook : https://www.facebook.com/linuxnetdata/ @@ -177,6 +183,8 @@ The command line options of the Netdata 1.10.0 version are the following: -W simple-pattern pattern string Check if string matches pattern and exit. + -W "claim -token=TOKEN -rooms=ROOM1,ROOM2 url=https://app.netdata.cloud" + Claim the agent to the workspace rooms pointed to by TOKEN and ROOM*. Signals netdata handles: @@ -185,6 +193,8 @@ The command line options of the Netdata 1.10.0 version are the following: - USR2 Reload health configuration. ``` +You can send commands during runtime via [netdatacli](/cli/README.md). + ## Log files Netdata uses 3 log files: @@ -305,14 +315,14 @@ You can set Netdata scheduling policy in `netdata.conf`, like this: You can use the following: -| policy | description | -|:----:|:----------| -| `idle` | use CPU only when there is spare - this is lower than nice 19 - it is the default for Netdata and it is so low that Netdata will run in "slow motion" under extreme system load, resulting in short (1-2 seconds) gaps at the charts. | -| `other`<br/>or<br/>`nice` | this is the default policy for all processes under Linux. It provides dynamic priorities based on the `nice` level of each process. Check below for setting this `nice` level for netdata. | +| policy | description | +| :-----------------------: | :---------- | +| `idle` | use CPU only when there is spare - this is lower than nice 19 - it is the default for Netdata and it is so low that Netdata will run in "slow motion" under extreme system load, resulting in short (1-2 seconds) gaps at the charts. | +| `other`<br/>or<br/>`nice` | this is the default policy for all processes under Linux. It provides dynamic priorities based on the `nice` level of each process. Check below for setting this `nice` level for netdata. | | `batch` | This policy is similar to `other` in that it schedules the thread according to its dynamic priority (based on the `nice` value). The difference is that this policy will cause the scheduler to always assume that the thread is CPU-intensive. Consequently, the scheduler will apply a small scheduling penalty with respect to wake-up behavior, so that this thread is mildly disfavored in scheduling decisions. | -| `fifo` | `fifo` can be used only with static priorities higher than 0, which means that when a `fifo` threads becomes runnable, it will always immediately preempt any currently running `other`, `batch`, or `idle` thread. `fifo` is a simple scheduling algorithm without time slicing. | -| `rr` | a simple enhancement of `fifo`. Everything described above for `fifo` also applies to `rr`, except that each thread is allowed to run only for a maximum time quantum. | -| `keep`<br/>or<br/>`none` | do not set scheduling policy, priority or nice level - i.e. keep running with whatever it is set already (e.g. by systemd). | +| `fifo` | `fifo` can be used only with static priorities higher than 0, which means that when a `fifo` threads becomes runnable, it will always immediately preempt any currently running `other`, `batch`, or `idle` thread. `fifo` is a simple scheduling algorithm without time slicing. | +| `rr` | a simple enhancement of `fifo`. Everything described above for `fifo` also applies to `rr`, except that each thread is allowed to run only for a maximum time quantum. | +| `keep`<br/>or<br/>`none` | do not set scheduling policy, priority or nice level - i.e. keep running with whatever it is set already (e.g. by systemd). | For more information see `man sched`. @@ -476,8 +486,8 @@ When you compile Netdata with debugging: 2. a lot of code is added all over netdata, to log debug messages to `/var/log/netdata/debug.log`. However, nothing is printed by default. Netdata allows you to select which sections of Netdata you want to trace. Tracing is activated via the config option `debug flags`. It accepts a hex number, to enable or disable specific sections. You can find - the options supported at [log.h](../libnetdata/log/log.h). They are the `D_*` defines. The value - `0xffffffffffffffff` will enable all possible debug flags. + the options supported at [log.h](https://raw.githubusercontent.com/netdata/netdata/master/libnetdata/log/log.h). + They are the `D_*` defines. The value `0xffffffffffffffff` will enable all possible debug flags. Once Netdata is compiled with debugging and tracing is enabled for a few sections, the file `/var/log/netdata/debug.log` will contain the messages. @@ -504,7 +514,7 @@ section(s) you need to trace. We have made the most to make Netdata crash free. If however, Netdata crashes on your system, it would be very helpful to provide stack traces of the crash. Without them, is will be almost impossible to find the issue (the code base is -quite large to find such an issue by just objerving it). +quite large to find such an issue by just observing it). To provide stack traces, **you need to have Netdata compiled with debugging**. There is no need to enable any tracing (`debug flags`). diff --git a/daemon/anonymous-statistics.sh b/daemon/anonymous-statistics.sh deleted file mode 100644 index 8c5a77f5e..000000000 --- a/daemon/anonymous-statistics.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env sh - -# Valid actions: - -# - FATAL - netdata exited due to a fatal condition -# ACTION_RESULT -- program name and thread tag -# ACTION_DATA -- fmt, args passed to fatal -# - START - netdata started -# ACTION_DATA -- nan -# - EXIT - installation action -# ACTION_DATA -- ret value of - -ACTION="${1}" -ACTION_RESULT="${2}" -ACTION_DATA="${3}" -ACTION_DATA=$(echo "${ACTION_DATA}" | tr '"' "'") - -# ------------------------------------------------------------------------------------------------- -# check opt-out - -if [ -f "/etc/netdata/.opt-out-from-anonymous-statistics" ]; then - exit 0 -fi - -# Shorten version for easier reporting -NETDATA_VERSION=$(echo "${NETDATA_VERSION}" | sed 's/-.*//g' | tr -d 'v') - -# ------------------------------------------------------------------------------------------------- -# send the anonymous statistics to GA -# https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters -if [ -n "$(command -v curl 2>/dev/null)" ]; then - curl -X POST -Ss --max-time 2 \ - --data "v=1" \ - --data "tid=UA-64295674-3" \ - --data "aip=1" \ - --data "ds=shell" \ - --data-urlencode "cid=${NETDATA_REGISTRY_UNIQUE_ID}" \ - --data-urlencode "cs=${NETDATA_REGISTRY_UNIQUE_ID}" \ - --data "t=event" \ - --data "ni=1" \ - --data "an=anonymous-statistics" \ - --data-urlencode "av=${NETDATA_VERSION}" \ - --data-urlencode "ec=${ACTION}" \ - --data-urlencode "ea=${ACTION_RESULT}" \ - --data-urlencode "el=${ACTION_DATA}" \ - --data-urlencode "cd1=${NETDATA_SYSTEM_OS_NAME}" \ - --data-urlencode "cd2=${NETDATA_SYSTEM_OS_ID}" \ - --data-urlencode "cd3=${NETDATA_SYSTEM_OS_ID_LIKE}" \ - --data-urlencode "cd4=${NETDATA_SYSTEM_OS_VERSION}" \ - --data-urlencode "cd5=${NETDATA_SYSTEM_OS_VERSION_ID}" \ - --data-urlencode "cd6=${NETDATA_SYSTEM_OS_DETECTION}" \ - --data-urlencode "cd7=${NETDATA_SYSTEM_KERNEL_NAME}" \ - --data-urlencode "cd8=${NETDATA_SYSTEM_KERNEL_VERSION}" \ - --data-urlencode "cd9=${NETDATA_SYSTEM_ARCHITECTURE}" \ - --data-urlencode "cd10=${NETDATA_SYSTEM_VIRTUALIZATION}" \ - --data-urlencode "cd11=${NETDATA_SYSTEM_VIRT_DETECTION}" \ - --data-urlencode "cd12=${NETDATA_SYSTEM_CONTAINER}" \ - --data-urlencode "cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}" \ - "https://www.google-analytics.com/collect" >/dev/null 2>&1 -else - wget -q -O - --timeout=1 "https://www.google-analytics.com/collect?\ -&v=1\ -&tid=UA-64295674-3\ -&aip=1\ -&ds=shell\ -&cid=${NETDATA_REGISTRY_UNIQUE_ID}\ -&cs=${NETDATA_REGISTRY_UNIQUE_ID}\ -&t=event\ -&ni=1\ -&an=anonymous-statistics\ -&av=${NETDATA_VERSION}\ -&ec=${ACTION}\ -&ea=${ACTION_RESULT}\ -&el=${ACTION_DATA}\ -&cd1=${NETDATA_SYSTEM_OS_NAME}\ -&cd2=${NETDATA_SYSTEM_OS_ID}\ -&cd3=${NETDATA_SYSTEM_OS_ID_LIKE}\ -&cd4=${NETDATA_SYSTEM_OS_VERSION}\ -&cd5=${NETDATA_SYSTEM_OS_VERSION_ID}\ -&cd6=${NETDATA_SYSTEM_OS_DETECTION}\ -&cd7=${NETDATA_SYSTEM_KERNEL_NAME}\ -&cd8=${NETDATA_SYSTEM_KERNEL_VERSION}\ -&cd9=${NETDATA_SYSTEM_ARCHITECTURE}\ -&cd10=${NETDATA_SYSTEM_VIRTUALIZATION}\ -&cd11=${NETDATA_SYSTEM_VIRT_DETECTION}\ -&cd12=${NETDATA_SYSTEM_CONTAINER}\ -&cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}\ -" > /dev/null 2>&1 -fi diff --git a/daemon/anonymous-statistics.sh.in b/daemon/anonymous-statistics.sh.in index f16c85a45..f0d9c10ef 100755 --- a/daemon/anonymous-statistics.sh.in +++ b/daemon/anonymous-statistics.sh.in @@ -7,8 +7,8 @@ # ACTION_DATA -- fmt, args passed to fatal # - START - netdata started # ACTION_DATA -- nan -# - EXIT - installation action -# ACTION_DATA -- ret value of +# - EXIT - installation action +# ACTION_DATA -- ret value of ACTION="${1}" ACTION_RESULT="${2}" @@ -18,8 +18,8 @@ ACTION_DATA=$(echo "${ACTION_DATA}" | tr '"' "'") # ------------------------------------------------------------------------------------------------- # check opt-out -if [ -f "@configdir_POST@/.opt-out-from-anonymous-statistics" ]; then - exit 0 +if [ -f "@configdir_POST@/.opt-out-from-anonymous-statistics" ] || [ ! "${DO_NOT_TRACK:-0}" -eq 0 ] || [ -n "$DO_NOT_TRACK" ]; then + exit 0 fi # Shorten version for easier reporting @@ -28,37 +28,45 @@ NETDATA_VERSION=$(echo "${NETDATA_VERSION}" | sed 's/-.*//g' | tr -d 'v') # ------------------------------------------------------------------------------------------------- # send the anonymous statistics to GA # https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters -if [ -n "$(command -v curl 2>/dev/null)" ]; then - curl -X POST -Ss --max-time 2 \ - --data "v=1" \ - --data "tid=UA-64295674-3" \ - --data "aip=1" \ - --data "ds=shell" \ - --data-urlencode "cid=${NETDATA_REGISTRY_UNIQUE_ID}" \ - --data-urlencode "cs=${NETDATA_REGISTRY_UNIQUE_ID}" \ - --data "t=event" \ - --data "ni=1" \ - --data "an=anonymous-statistics" \ - --data-urlencode "av=${NETDATA_VERSION}" \ - --data-urlencode "ec=${ACTION}" \ - --data-urlencode "ea=${ACTION_RESULT}" \ - --data-urlencode "el=${ACTION_DATA}" \ - --data-urlencode "cd1=${NETDATA_SYSTEM_OS_NAME}" \ - --data-urlencode "cd2=${NETDATA_SYSTEM_OS_ID}" \ - --data-urlencode "cd3=${NETDATA_SYSTEM_OS_ID_LIKE}" \ - --data-urlencode "cd4=${NETDATA_SYSTEM_OS_VERSION}" \ - --data-urlencode "cd5=${NETDATA_SYSTEM_OS_VERSION_ID}" \ - --data-urlencode "cd6=${NETDATA_SYSTEM_OS_DETECTION}" \ - --data-urlencode "cd7=${NETDATA_SYSTEM_KERNEL_NAME}" \ - --data-urlencode "cd8=${NETDATA_SYSTEM_KERNEL_VERSION}" \ - --data-urlencode "cd9=${NETDATA_SYSTEM_ARCHITECTURE}" \ - --data-urlencode "cd10=${NETDATA_SYSTEM_VIRTUALIZATION}" \ - --data-urlencode "cd11=${NETDATA_SYSTEM_VIRT_DETECTION}" \ - --data-urlencode "cd12=${NETDATA_SYSTEM_CONTAINER}" \ - --data-urlencode "cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}" \ - "https://www.google-analytics.com/collect" >/dev/null 2>&1 +# The maximum index for a cd parameter is 20 so we have effectively run out. +if [ -n "$(command -v curl 2> /dev/null)" ]; then + curl -X POST -Ss --max-time 2 \ + --data "v=1" \ + --data "tid=UA-64295674-3" \ + --data "aip=1" \ + --data "ds=shell" \ + --data-urlencode "cid=${NETDATA_REGISTRY_UNIQUE_ID}" \ + --data-urlencode "cs=${NETDATA_REGISTRY_UNIQUE_ID}" \ + --data "t=event" \ + --data "ni=1" \ + --data "an=anonymous-statistics" \ + --data-urlencode "av=${NETDATA_VERSION}" \ + --data-urlencode "ec=${ACTION}" \ + --data-urlencode "ea=${ACTION_RESULT}" \ + --data-urlencode "el=${ACTION_DATA}" \ + --data-urlencode "cd1=${NETDATA_HOST_OS_NAME}" \ + --data-urlencode "cd2=${NETDATA_HOST_OS_ID}" \ + --data-urlencode "cd3=${NETDATA_HOST_OS_ID_LIKE}" \ + --data-urlencode "cd4=${NETDATA_HOST_OS_VERSION}" \ + --data-urlencode "cd5=${NETDATA_HOST_OS_VERSION_ID}" \ + --data-urlencode "cd6=${NETDATA_HOST_OS_DETECTION}" \ + --data-urlencode "cd7=${NETDATA_SYSTEM_KERNEL_NAME}" \ + --data-urlencode "cd8=${NETDATA_SYSTEM_KERNEL_VERSION}" \ + --data-urlencode "cd9=${NETDATA_SYSTEM_ARCHITECTURE}" \ + --data-urlencode "cd10=${NETDATA_SYSTEM_VIRTUALIZATION}" \ + --data-urlencode "cd11=${NETDATA_SYSTEM_VIRT_DETECTION}" \ + --data-urlencode "cd12=${NETDATA_SYSTEM_CONTAINER}" \ + --data-urlencode "cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}" \ + --data-urlencode "cd14=${NETDATA_CONTAINER_OS_NAME}" \ + --data-urlencode "cd15=${NETDATA_CONTAINER_OS_ID}" \ + --data-urlencode "cd16=${NETDATA_CONTAINER_OS_ID_LIKE}" \ + --data-urlencode "cd17=${NETDATA_CONTAINER_OS_VERSION}" \ + --data-urlencode "cd18=${NETDATA_CONTAINER_OS_VERSION_ID}" \ + --data-urlencode "cd19=${NETDATA_CONTAINER_OS_DETECTION}" \ + --data-urlencode "cd20=${NETDATA_HOST_IS_K8S_NODE}" \ + "https://www.google-analytics.com/collect" > /dev/null 2>&1 else - wget -q -O - --timeout=1 "https://www.google-analytics.com/collect?\ + wget -q -O - --timeout=1 "https://www.google-analytics.com/collect?\ &v=1\ &tid=UA-64295674-3\ &aip=1\ @@ -72,12 +80,12 @@ else &ec=${ACTION}\ &ea=${ACTION_RESULT}\ &el=${ACTION_DATA}\ -&cd1=${NETDATA_SYSTEM_OS_NAME}\ -&cd2=${NETDATA_SYSTEM_OS_ID}\ -&cd3=${NETDATA_SYSTEM_OS_ID_LIKE}\ -&cd4=${NETDATA_SYSTEM_OS_VERSION}\ -&cd5=${NETDATA_SYSTEM_OS_VERSION_ID}\ -&cd6=${NETDATA_SYSTEM_OS_DETECTION}\ +&cd1=${NETDATA_HOST_OS_NAME}\ +&cd2=${NETDATA_HOST_OS_ID}\ +&cd3=${NETDATA_HOST_OS_ID_LIKE}\ +&cd4=${NETDATA_HOST_OS_VERSION}\ +&cd5=${NETDATA_HOST_OS_VERSION_ID}\ +&cd6=${NETDATA_HOST_OS_DETECTION}\ &cd7=${NETDATA_SYSTEM_KERNEL_NAME}\ &cd8=${NETDATA_SYSTEM_KERNEL_VERSION}\ &cd9=${NETDATA_SYSTEM_ARCHITECTURE}\ @@ -85,5 +93,12 @@ else &cd11=${NETDATA_SYSTEM_VIRT_DETECTION}\ &cd12=${NETDATA_SYSTEM_CONTAINER}\ &cd13=${NETDATA_SYSTEM_CONTAINER_DETECTION}\ +&cd14=${NETDATA_CONTAINER_OS_NAME} \ +&cd15=${NETDATA_CONTAINER_OS_ID} \ +&cd16=${NETDATA_CONTAINER_OS_ID_LIKE} \ +&cd17=${NETDATA_CONTAINER_OS_VERSION} \ +&cd18=${NETDATA_CONTAINER_OS_VERSION_ID} \ +&cd19=${NETDATA_CONTAINER_OS_DETECTION} \ +&cd20=${NETDATA_HOST_IS_K8S_NODE} \ " > /dev/null 2>&1 fi diff --git a/daemon/buildinfo.c b/daemon/buildinfo.c new file mode 100644 index 000000000..de02a72e1 --- /dev/null +++ b/daemon/buildinfo.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <stdio.h> +#include "./config.h" + +// Optional features + +#ifdef ENABLE_ACLK +#define FEAT_CLOUD "YES" +#else +#ifdef DISABLE_CLOUD +#define FEAT_CLOUD "NO (by user request e.g. '--disable-cloud')" +#else +#define FEAT_CLOUD "NO" +#endif +#endif + +#ifdef ENABLE_DBENGINE +#define FEAT_DBENGINE "YES" +#else +#define FEAT_DBENGINE "NO" +#endif + +#if defined(HAVE_X509_VERIFY_PARAM_set1_host) && HAVE_X509_VERIFY_PARAM_set1_host == 1 +#define FEAT_TLS_HOST_VERIFY "YES" +#else +#define FEAT_TLS_HOST_VERIFY "NO" +#endif + +#ifdef ENABLE_HTTPS +#define FEAT_NATIVE_HTTPS "YES" +#else +#define FEAT_NATIVE_HTTPS "NO" +#endif + +// Optional libraries + +#ifdef ENABLE_JSONC +#define FEAT_JSONC "YES" +#else +#define FEAT_JSONC "NO" +#endif + +#ifdef ENABLE_JEMALLOC +#define FEAT_JEMALLOC "YES" +#else +#define FEAT_JEMALLOC "NO" +#endif + +#ifdef ENABLE_TCMALLOC +#define FEAT_TCMALLOC "YES" +#else +#define FEAT_TCMALLOC "NO" +#endif + +#ifdef HAVE_CAPABILITY +#define FEAT_LIBCAP "YES" +#else +#define FEAT_LIBCAP "NO" +#endif + +#ifdef ACLK_NO_LIBMOSQ +#define FEAT_MOSQUITTO "NO" +#else +#define FEAT_MOSQUITTO "YES" +#endif + +#ifdef ACLK_NO_LWS +#define FEAT_LWS "NO" +#else +#ifdef ENABLE_ACLK +#include <libwebsockets.h> +#endif +#ifdef BUNDLED_LWS +#define FEAT_LWS "YES static" +#else +#define FEAT_LWS "YES shared-lib" +#endif +#endif + +#ifdef NETDATA_WITH_ZLIB +#define FEAT_ZLIB "YES" +#else +#define FEAT_ZLIB "NO" +#endif + +#ifdef STORAGE_WITH_MATH +#define FEAT_LIBM "YES" +#else +#define FEAT_LIBM "NO" +#endif + +#ifdef HAVE_CRYPTO +#define FEAT_CRYPTO "YES" +#else +#define FEAT_CRYPTO "NO" +#endif + +// Optional plugins + +#ifdef ENABLE_APPS_PLUGIN +#define FEAT_APPS_PLUGIN "YES" +#else +#define FEAT_APPS_PLUGIN "NO" +#endif + +#ifdef HAVE_FREEIPMI +#define FEAT_IPMI "YES" +#else +#define FEAT_IPMI "NO" +#endif + +#ifdef HAVE_CUPS +#define FEAT_CUPS "YES" +#else +#define FEAT_CUPS "NO" +#endif + +#ifdef HAVE_LIBMNL +#define FEAT_NFACCT "YES" +#else +#define FEAT_NFACCT "NO" +#endif + +#ifdef HAVE_LIBXENSTAT +#define FEAT_XEN "YES" +#else +#define FEAT_XEN "NO" +#endif + +#ifdef HAVE_XENSTAT_VBD_ERROR +#define FEAT_XEN_VBD_ERROR "YES" +#else +#define FEAT_XEN_VBD_ERROR "NO" +#endif + +#ifdef HAVE_LIBBPF +#define FEAT_EBPF "YES" +#else +#define FEAT_EBPF "NO" +#endif + +#ifdef HAVE_SETNS +#define FEAT_CGROUP_NET "YES" +#else +#define FEAT_CGROUP_NET "NO" +#endif + +#ifdef ENABLE_PERF_PLUGIN +#define FEAT_PERF "YES" +#else +#define FEAT_PERF "NO" +#endif + +#ifdef ENABLE_SLABINFO +#define FEAT_SLABINFO "YES" +#else +#define FEAT_SLABINFO "NO" +#endif + +// Optional Exporters + +#ifdef HAVE_KINESIS +#define FEAT_KINESIS "YES" +#else +#define FEAT_KINESIS "NO" +#endif + +#ifdef ENABLE_EXPORTING_PUBSUB +#define FEAT_PUBSUB "YES" +#else +#define FEAT_PUBSUB "NO" +#endif + +#ifdef HAVE_MONGOC +#define FEAT_MONGO "YES" +#else +#define FEAT_MONGO "NO" +#endif + +#ifdef ENABLE_PROMETHEUS_REMOTE_WRITE +#define FEAT_REMOTE_WRITE "YES" +#else +#define FEAT_REMOTE_WRITE "NO" +#endif + + +void print_build_info(void) { + printf("Configure options: %s\n", CONFIGURE_COMMAND); + + printf("Features:\n"); + printf(" dbengine: %s\n", FEAT_DBENGINE); + printf(" Native HTTPS: %s\n", FEAT_NATIVE_HTTPS); + printf(" Netdata Cloud: %s\n", FEAT_CLOUD); + printf(" TLS Host Verification: %s\n", FEAT_TLS_HOST_VERIFY); + + printf("Libraries:\n"); + printf(" jemalloc: %s\n", FEAT_JEMALLOC); + printf(" JSON-C: %s\n", FEAT_JSONC); + printf(" libcap: %s\n", FEAT_LIBCAP); + printf(" libcrypto: %s\n", FEAT_CRYPTO); + printf(" libm: %s\n", FEAT_LIBM); +#if defined(ENABLE_ACLK) + printf(" LWS: %s v%d.%d.%d\n", FEAT_LWS, LWS_LIBRARY_VERSION_MAJOR, LWS_LIBRARY_VERSION_MINOR, LWS_LIBRARY_VERSION_PATCH); +#else + printf(" LWS: %s\n", FEAT_LWS); +#endif + printf(" mosquitto: %s\n", FEAT_MOSQUITTO); + printf(" tcalloc: %s\n", FEAT_TCMALLOC); + printf(" zlib: %s\n", FEAT_ZLIB); + + printf("Plugins:\n"); + printf(" apps: %s\n", FEAT_APPS_PLUGIN); + printf(" cgroup Network Tracking: %s\n", FEAT_CGROUP_NET); + printf(" CUPS: %s\n", FEAT_CUPS); + printf(" EBPF: %s\n", FEAT_EBPF); + printf(" IPMI: %s\n", FEAT_IPMI); + printf(" NFACCT: %s\n", FEAT_NFACCT); + printf(" perf: %s\n", FEAT_PERF); + printf(" slabinfo: %s\n", FEAT_SLABINFO); + printf(" Xen: %s\n", FEAT_XEN); + printf(" Xen VBD Error Tracking: %s\n", FEAT_XEN_VBD_ERROR); + + printf("Exporters:\n"); + printf(" AWS Kinesis: %s\n", FEAT_KINESIS); + printf(" GCP PubSub: %s\n", FEAT_PUBSUB); + printf(" MongoDB: %s\n", FEAT_MONGO); + printf(" Prometheus Remote Write: %s\n", FEAT_REMOTE_WRITE); +}; diff --git a/daemon/buildinfo.h b/daemon/buildinfo.h new file mode 100644 index 000000000..76912ea67 --- /dev/null +++ b/daemon/buildinfo.h @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_BUILDINFO_H +#define NETDATA_BUILDINFO_H 1 + +extern void print_build_info(void); + +#endif // NETDATA_BUILDINFO_H diff --git a/daemon/commands.c b/daemon/commands.c new file mode 100644 index 000000000..eac392e33 --- /dev/null +++ b/daemon/commands.c @@ -0,0 +1,726 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "common.h" +#include "../database/engine/rrdenginelib.h" + +static uv_thread_t thread; +static uv_loop_t* loop; +static uv_async_t async; +static struct completion completion; +static uv_pipe_t server_pipe; + +char cmd_prefix_by_status[] = { + CMD_PREFIX_INFO, + CMD_PREFIX_ERROR, + CMD_PREFIX_ERROR +}; + +static int command_server_initialized = 0; +static int command_thread_error; +static int command_thread_shutdown; +static unsigned clients = 0; + +struct command_context { + /* embedded client pipe structure at address 0 */ + uv_pipe_t client; + + uv_work_t work; + uv_write_t write_req; + cmd_t idx; + char *args; + char *message; + cmd_status_t status; + char command_string[MAX_COMMAND_LENGTH]; + unsigned command_string_size; +}; + +/* Forward declarations */ +static cmd_status_t cmd_help_execute(char *args, char **message); +static cmd_status_t cmd_reload_health_execute(char *args, char **message); +static cmd_status_t cmd_save_database_execute(char *args, char **message); +static cmd_status_t cmd_reopen_logs_execute(char *args, char **message); +static cmd_status_t cmd_exit_execute(char *args, char **message); +static cmd_status_t cmd_fatal_execute(char *args, char **message); +static cmd_status_t cmd_reload_claiming_state_execute(char *args, char **message); +static cmd_status_t cmd_reload_labels_execute(char *args, char **message); +static cmd_status_t cmd_read_config_execute(char *args, char **message); +static cmd_status_t cmd_write_config_execute(char *args, char **message); +static cmd_status_t cmd_ping_execute(char *args, char **message); + +static command_info_t command_info_array[] = { + {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY}, // show help menu + {"reload-health", cmd_reload_health_execute, CMD_TYPE_ORTHOGONAL}, // reload health configuration + {"save-database", cmd_save_database_execute, CMD_TYPE_ORTHOGONAL}, // save database for memory mode save + {"reopen-logs", cmd_reopen_logs_execute, CMD_TYPE_ORTHOGONAL}, // Close and reopen log files + {"shutdown-agent", cmd_exit_execute, CMD_TYPE_EXCLUSIVE}, // exit cleanly + {"fatal-agent", cmd_fatal_execute, CMD_TYPE_HIGH_PRIORITY}, // exit with fatal error + {"reload-claiming-state", cmd_reload_claiming_state_execute, CMD_TYPE_ORTHOGONAL}, // reload claiming state + {"reload-labels", cmd_reload_labels_execute, CMD_TYPE_ORTHOGONAL}, // reload the labels + {"read-config", cmd_read_config_execute, CMD_TYPE_CONCURRENT}, + {"write-config", cmd_write_config_execute, CMD_TYPE_ORTHOGONAL}, + {"ping", cmd_ping_execute, CMD_TYPE_ORTHOGONAL} +}; + +/* Mutexes for commands of type CMD_TYPE_ORTHOGONAL */ +static uv_mutex_t command_lock_array[CMD_TOTAL_COMMANDS]; +/* Commands of type CMD_TYPE_EXCLUSIVE are writers */ +static uv_rwlock_t exclusive_rwlock; +/* + * Locking order: + * 1. exclusive_rwlock + * 2. command_lock_array[] + */ + +/* Forward declarations */ +static void cmd_lock_exclusive(unsigned index); +static void cmd_lock_orthogonal(unsigned index); +static void cmd_lock_idempotent(unsigned index); +static void cmd_lock_high_priority(unsigned index); + +static command_lock_t *cmd_lock_by_type[] = { + cmd_lock_exclusive, + cmd_lock_orthogonal, + cmd_lock_idempotent, + cmd_lock_high_priority +}; + +/* Forward declarations */ +static void cmd_unlock_exclusive(unsigned index); +static void cmd_unlock_orthogonal(unsigned index); +static void cmd_unlock_idempotent(unsigned index); +static void cmd_unlock_high_priority(unsigned index); + +static command_lock_t *cmd_unlock_by_type[] = { + cmd_unlock_exclusive, + cmd_unlock_orthogonal, + cmd_unlock_idempotent, + cmd_unlock_high_priority +}; + +static cmd_status_t cmd_help_execute(char *args, char **message) +{ + (void)args; + + *message = mallocz(MAX_COMMAND_LENGTH); + strncpyz(*message, + "\nThe commands are (arguments are in brackets):\n" + "help\n" + " Show this help menu.\n" + "reload-health\n" + " Reload health configuration.\n" + "reload-labels\n" + " Reload all labels.\n" + "save-database\n" + " Save internal DB to disk for memory mode save.\n" + "reopen-logs\n" + " Close and reopen log files.\n" + "shutdown-agent\n" + " Cleanup and exit the netdata agent.\n" + "fatal-agent\n" + " Log the state and halt the netdata agent.\n" + "reload-claiming-state\n" + " Reload agent claiming state from disk.\n" + "ping\n" + " Return with 'pong' if agent is alive.\n", + MAX_COMMAND_LENGTH - 1); + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_reload_health_execute(char *args, char **message) +{ + (void)args; + (void)message; + + error_log_limit_unlimited(); + info("COMMAND: Reloading HEALTH configuration."); + health_reload(); + error_log_limit_reset(); + + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_save_database_execute(char *args, char **message) +{ + (void)args; + (void)message; + + error_log_limit_unlimited(); + info("COMMAND: Saving databases."); + rrdhost_save_all(); + info("COMMAND: Databases saved."); + error_log_limit_reset(); + + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_reopen_logs_execute(char *args, char **message) +{ + (void)args; + (void)message; + + error_log_limit_unlimited(); + info("COMMAND: Reopening all log files."); + reopen_all_log_files(); + error_log_limit_reset(); + + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_exit_execute(char *args, char **message) +{ + (void)args; + (void)message; + + error_log_limit_unlimited(); + info("COMMAND: Cleaning up to exit."); + netdata_cleanup_and_exit(0); + exit(0); + + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_fatal_execute(char *args, char **message) +{ + (void)args; + (void)message; + + fatal("COMMAND: netdata now exits."); + + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_reload_claiming_state_execute(char *args, char **message) +{ + (void)args; + (void)message; +#if defined(DISABLE_CLOUD) || !defined(ENABLE_ACLK) + info("The claiming feature has been explicitly disabled"); + *message = strdupz("This agent cannot be claimed, it was built without support for Cloud"); + return CMD_STATUS_FAILURE; +#endif + error_log_limit_unlimited(); + info("COMMAND: Reloading Agent Claiming configuration."); + load_claiming_state(); + registry_update_cloud_base_url(); + rrdpush_claimed_id(localhost); + error_log_limit_reset(); + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_reload_labels_execute(char *args, char **message) +{ + (void)args; + info("COMMAND: reloading host labels."); + reload_host_labels(); + + BUFFER *wb = buffer_create(10); + + rrdhost_rdlock(localhost); + netdata_rwlock_rdlock(&localhost->labels.labels_rwlock); + struct label *l = localhost->labels.head; + while (l != NULL) { + buffer_sprintf(wb,"Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(l->label_source), l->key, l->value); + l = l->next; + } + netdata_rwlock_unlock(&localhost->labels.labels_rwlock); + rrdhost_unlock(localhost); + + (*message)=strdupz(buffer_tostring(wb)); + buffer_free(wb); + + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_read_config_execute(char *args, char **message) +{ + size_t n = strlen(args); + char *separator = strchr(args,'|'); + if (separator == NULL) + return CMD_STATUS_FAILURE; + char *separator2 = strchr(separator + 1,'|'); + if (separator2 == NULL) + return CMD_STATUS_FAILURE; + + char *temp = callocz(n + 1, 1); + strcpy(temp, args); + size_t offset = separator - args; + temp[offset] = 0; + size_t offset2 = separator2 - args; + temp[offset2] = 0; + + const char *conf_file = temp; /* "cloud" is cloud.conf, otherwise netdata.conf */ + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config; + + char *value = appconfig_get(tmp_config, temp + offset + 1, temp + offset2 + 1, NULL); + if (value == NULL) + { + error("Cannot execute read-config conf_file=%s section=%s / key=%s because no value set", conf_file, + temp + offset + 1, temp + offset2 + 1); + freez(temp); + return CMD_STATUS_FAILURE; + } + else + { + (*message) = strdupz(value); + freez(temp); + return CMD_STATUS_SUCCESS; + } + +} + +static cmd_status_t cmd_write_config_execute(char *args, char **message) +{ + UNUSED(message); + info("write-config %s", args); + size_t n = strlen(args); + char *separator = strchr(args,'|'); + if (separator == NULL) + return CMD_STATUS_FAILURE; + char *separator2 = strchr(separator + 1,'|'); + if (separator2 == NULL) + return CMD_STATUS_FAILURE; + char *separator3 = strchr(separator2 + 1,'|'); + if (separator3 == NULL) + return CMD_STATUS_FAILURE; + char *temp = callocz(n + 1, 1); + strcpy(temp, args); + size_t offset = separator - args; + temp[offset] = 0; + size_t offset2 = separator2 - args; + temp[offset2] = 0; + size_t offset3 = separator3 - args; + temp[offset3] = 0; + + const char *conf_file = temp; /* "cloud" is cloud.conf, otherwise netdata.conf */ + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config; + + appconfig_set(tmp_config, temp + offset + 1, temp + offset2 + 1, temp + offset3 + 1); + info("write-config conf_file=%s section=%s key=%s value=%s",conf_file, temp + offset + 1, temp + offset2 + 1, + temp + offset3 + 1); + freez(temp); + return CMD_STATUS_SUCCESS; +} + +static cmd_status_t cmd_ping_execute(char *args, char **message) +{ + (void)args; + + *message = strdupz("pong"); + + return CMD_STATUS_SUCCESS; +} + +static void cmd_lock_exclusive(unsigned index) +{ + (void)index; + + uv_rwlock_wrlock(&exclusive_rwlock); +} + +static void cmd_lock_orthogonal(unsigned index) +{ + uv_rwlock_rdlock(&exclusive_rwlock); + uv_mutex_lock(&command_lock_array[index]); +} + +static void cmd_lock_idempotent(unsigned index) +{ + (void)index; + + uv_rwlock_rdlock(&exclusive_rwlock); +} + +static void cmd_lock_high_priority(unsigned index) +{ + (void)index; +} + +static void cmd_unlock_exclusive(unsigned index) +{ + (void)index; + + uv_rwlock_wrunlock(&exclusive_rwlock); +} + +static void cmd_unlock_orthogonal(unsigned index) +{ + uv_rwlock_rdunlock(&exclusive_rwlock); + uv_mutex_unlock(&command_lock_array[index]); +} + +static void cmd_unlock_idempotent(unsigned index) +{ + (void)index; + + uv_rwlock_rdunlock(&exclusive_rwlock); +} + +static void cmd_unlock_high_priority(unsigned index) +{ + (void)index; +} + +static void pipe_close_cb(uv_handle_t* handle) +{ + /* Also frees command context */ + freez(handle); +} + +static void pipe_write_cb(uv_write_t* req, int status) +{ + (void)status; + uv_pipe_t *client = req->data; + + uv_close((uv_handle_t *)client, pipe_close_cb); + --clients; + info("Command Clients = %u\n", clients); +} + +static inline void add_char_to_command_reply(char *reply_string, unsigned *reply_string_size, char character) +{ + reply_string[(*reply_string_size)++] = character; +} + +static inline void add_string_to_command_reply(char *reply_string, unsigned *reply_string_size, char *str) +{ + unsigned len; + + len = strlen(str); + strncpyz(reply_string + *reply_string_size, str, len); + *reply_string_size += len; +} + +static void send_command_reply(struct command_context *cmd_ctx, cmd_status_t status, char *message) +{ + int ret; + char reply_string[MAX_COMMAND_LENGTH] = {'\0', }; + char exit_status_string[MAX_EXIT_STATUS_LENGTH + 1] = {'\0', }; + unsigned reply_string_size = 0; + uv_buf_t write_buf; + uv_stream_t *client = (uv_stream_t *)(uv_pipe_t *)cmd_ctx; + + snprintfz(exit_status_string, MAX_EXIT_STATUS_LENGTH, "%u", status); + add_char_to_command_reply(reply_string, &reply_string_size, CMD_PREFIX_EXIT_CODE); + add_string_to_command_reply(reply_string, &reply_string_size, exit_status_string); + add_char_to_command_reply(reply_string, &reply_string_size, '\0'); + + if (message) { + add_char_to_command_reply(reply_string, &reply_string_size, cmd_prefix_by_status[status]); + add_string_to_command_reply(reply_string, &reply_string_size, message); + } + + cmd_ctx->write_req.data = client; + write_buf.base = reply_string; + write_buf.len = reply_string_size; + ret = uv_write(&cmd_ctx->write_req, (uv_stream_t *)client, &write_buf, 1, pipe_write_cb); + if (ret) { + error("uv_write(): %s", uv_strerror(ret)); + } + info("COMMAND: Sending reply: \"%s\"", reply_string); +} + +cmd_status_t execute_command(cmd_t idx, char *args, char **message) +{ + cmd_status_t status; + cmd_type_t type = command_info_array[idx].type; + + cmd_lock_by_type[type](idx); + status = command_info_array[idx].func(args, message); + cmd_unlock_by_type[type](idx); + + return status; +} + +static void after_schedule_command(uv_work_t *req, int status) +{ + struct command_context *cmd_ctx = req->data; + + (void)status; + + send_command_reply(cmd_ctx, cmd_ctx->status, cmd_ctx->message); + if (cmd_ctx->message) + freez(cmd_ctx->message); +} + +static void schedule_command(uv_work_t *req) +{ + struct command_context *cmd_ctx = req->data; + + cmd_ctx->status = execute_command(cmd_ctx->idx, cmd_ctx->args, &cmd_ctx->message); +} + +/* This will alter the state of the command_info_array.cmd_str +*/ +static void parse_commands(struct command_context *cmd_ctx) +{ + char *message = NULL, *pos, *lstrip, *rstrip; + cmd_t i; + cmd_status_t status; + + status = CMD_STATUS_FAILURE; + + /* Skip white-space characters */ + for (pos = cmd_ctx->command_string ; isspace(*pos) && ('\0' != *pos) ; ++pos) {;} + for (i = 0 ; i < CMD_TOTAL_COMMANDS ; ++i) { + if (!strncmp(pos, command_info_array[i].cmd_str, strlen(command_info_array[i].cmd_str))) { + if (CMD_EXIT == i) { + /* musl C does not like libuv workqueues calling exit() */ + execute_command(CMD_EXIT, NULL, NULL); + } + for (lstrip=pos + strlen(command_info_array[i].cmd_str); isspace(*lstrip) && ('\0' != *lstrip); ++lstrip) {;} + for (rstrip=lstrip+strlen(lstrip)-1; rstrip>lstrip && isspace(*rstrip); *(rstrip--) = 0 ); + + cmd_ctx->work.data = cmd_ctx; + cmd_ctx->idx = i; + cmd_ctx->args = lstrip; + cmd_ctx->message = NULL; + + fatal_assert(0 == uv_queue_work(loop, &cmd_ctx->work, schedule_command, after_schedule_command)); + break; + } + } + if (CMD_TOTAL_COMMANDS == i) { + /* no command found */ + message = strdupz("Illegal command. Please type \"help\" for instructions."); + send_command_reply(cmd_ctx, status, message); + freez(message); + } +} + +static void pipe_read_cb(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) +{ + struct command_context *cmd_ctx = (struct command_context *)client; + + if (0 == nread) { + info("%s: Zero bytes read by command pipe.", __func__); + } else if (UV_EOF == nread) { + info("EOF found in command pipe."); + parse_commands(cmd_ctx); + } else if (nread < 0) { + error("%s: %s", __func__, uv_strerror(nread)); + } + + if (nread < 0) { /* stop stream due to EOF or error */ + (void)uv_read_stop((uv_stream_t *)client); + } else if (nread) { + size_t to_copy; + + to_copy = MIN(nread, MAX_COMMAND_LENGTH - 1 - cmd_ctx->command_string_size); + memcpy(cmd_ctx->command_string + cmd_ctx->command_string_size, buf->base, to_copy); + cmd_ctx->command_string_size += to_copy; + cmd_ctx->command_string[cmd_ctx->command_string_size] = '\0'; + } + if (buf && buf->len) { + freez(buf->base); + } + + if (nread < 0 && UV_EOF != nread) { + uv_close((uv_handle_t *)client, pipe_close_cb); + --clients; + info("Command Clients = %u\n", clients); + } +} + +static void alloc_cb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) +{ + (void)handle; + + buf->base = mallocz(suggested_size); + buf->len = suggested_size; +} + +static void connection_cb(uv_stream_t *server, int status) +{ + int ret; + uv_pipe_t *client; + struct command_context *cmd_ctx; + fatal_assert(status == 0); + + /* combined allocation of client pipe and command context */ + cmd_ctx = mallocz(sizeof(*cmd_ctx)); + client = (uv_pipe_t *)cmd_ctx; + ret = uv_pipe_init(server->loop, client, 1); + if (ret) { + error("uv_pipe_init(): %s", uv_strerror(ret)); + freez(cmd_ctx); + return; + } + ret = uv_accept(server, (uv_stream_t *)client); + if (ret) { + error("uv_accept(): %s", uv_strerror(ret)); + uv_close((uv_handle_t *)client, pipe_close_cb); + return; + } + + ++clients; + info("Command Clients = %u\n", clients); + /* Start parsing a new command */ + cmd_ctx->command_string_size = 0; + cmd_ctx->command_string[0] = '\0'; + + ret = uv_read_start((uv_stream_t*)client, alloc_cb, pipe_read_cb); + if (ret) { + error("uv_read_start(): %s", uv_strerror(ret)); + uv_close((uv_handle_t *)client, pipe_close_cb); + --clients; + info("Command Clients = %u\n", clients); + return; + } +} + +static void async_cb(uv_async_t *handle) +{ + uv_stop(handle->loop); +} + +static void command_thread(void *arg) +{ + int ret; + uv_fs_t req; + + (void) arg; + loop = mallocz(sizeof(uv_loop_t)); + ret = uv_loop_init(loop); + if (ret) { + error("uv_loop_init(): %s", uv_strerror(ret)); + command_thread_error = ret; + goto error_after_loop_init; + } + loop->data = NULL; + + ret = uv_async_init(loop, &async, async_cb); + if (ret) { + error("uv_async_init(): %s", uv_strerror(ret)); + command_thread_error = ret; + goto error_after_async_init; + } + async.data = NULL; + + ret = uv_pipe_init(loop, &server_pipe, 0); + if (ret) { + error("uv_pipe_init(): %s", uv_strerror(ret)); + command_thread_error = ret; + goto error_after_pipe_init; + } + (void)uv_fs_unlink(loop, &req, PIPENAME, NULL); + uv_fs_req_cleanup(&req); + ret = uv_pipe_bind(&server_pipe, PIPENAME); + if (ret) { + error("uv_pipe_bind(): %s", uv_strerror(ret)); + command_thread_error = ret; + goto error_after_pipe_bind; + } + ret = uv_listen((uv_stream_t *)&server_pipe, SOMAXCONN, connection_cb); + if (ret) { + /* Fallback to backlog of 1 */ + info("uv_listen() failed with backlog = %d, falling back to backlog = 1.", SOMAXCONN); + ret = uv_listen((uv_stream_t *)&server_pipe, 1, connection_cb); + } + if (ret) { + error("uv_listen(): %s", uv_strerror(ret)); + command_thread_error = ret; + goto error_after_uv_listen; + } + + command_thread_error = 0; + command_thread_shutdown = 0; + /* wake up initialization thread */ + complete(&completion); + + while (command_thread_shutdown == 0) { + uv_run(loop, UV_RUN_DEFAULT); + } + /* cleanup operations of the event loop */ + info("Shutting down command event loop."); + uv_close((uv_handle_t *)&async, NULL); + uv_close((uv_handle_t*)&server_pipe, NULL); + uv_run(loop, UV_RUN_DEFAULT); /* flush all libuv handles */ + + info("Shutting down command loop complete."); + fatal_assert(0 == uv_loop_close(loop)); + freez(loop); + + return; + +error_after_uv_listen: +error_after_pipe_bind: + uv_close((uv_handle_t*)&server_pipe, NULL); +error_after_pipe_init: + uv_close((uv_handle_t *)&async, NULL); +error_after_async_init: + uv_run(loop, UV_RUN_DEFAULT); /* flush all libuv handles */ + fatal_assert(0 == uv_loop_close(loop)); +error_after_loop_init: + freez(loop); + + /* wake up initialization thread */ + complete(&completion); +} + +static void sanity_check(void) +{ + /* The size of command_info_array must be CMD_TOTAL_COMMANDS elements */ + BUILD_BUG_ON(CMD_TOTAL_COMMANDS != sizeof(command_info_array) / sizeof(command_info_array[0])); +} + +void commands_init(void) +{ + cmd_t i; + int error; + + sanity_check(); + if (command_server_initialized) + return; + + info("Initializing command server."); + for (i = 0 ; i < CMD_TOTAL_COMMANDS ; ++i) { + fatal_assert(0 == uv_mutex_init(&command_lock_array[i])); + } + fatal_assert(0 == uv_rwlock_init(&exclusive_rwlock)); + + init_completion(&completion); + error = uv_thread_create(&thread, command_thread, NULL); + if (error) { + error("uv_thread_create(): %s", uv_strerror(error)); + goto after_error; + } + /* wait for worker thread to initialize */ + wait_for_completion(&completion); + destroy_completion(&completion); + uv_thread_set_name_np(thread, "DAEMON_COMMAND"); + + if (command_thread_error) { + error = uv_thread_join(&thread); + if (error) { + error("uv_thread_create(): %s", uv_strerror(error)); + } + goto after_error; + } + + command_server_initialized = 1; + return; + +after_error: + error("Failed to initialize command server. The netdata cli tool will be unable to send commands."); +} + +void commands_exit(void) +{ + cmd_t i; + + if (!command_server_initialized) + return; + + command_thread_shutdown = 1; + info("Shutting down command server."); + /* wake up event loop */ + fatal_assert(0 == uv_async_send(&async)); + fatal_assert(0 == uv_thread_join(&thread)); + + for (i = 0 ; i < CMD_TOTAL_COMMANDS ; ++i) { + uv_mutex_destroy(&command_lock_array[i]); + } + uv_rwlock_destroy(&exclusive_rwlock); + info("Command server has stopped."); + command_server_initialized = 0; +} diff --git a/daemon/commands.h b/daemon/commands.h new file mode 100644 index 000000000..bd4aabfe1 --- /dev/null +++ b/daemon/commands.h @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_COMMANDS_H +#define NETDATA_COMMANDS_H 1 + +#ifdef _WIN32 +# define PIPENAME "\\\\?\\pipe\\netdata-cli" +#else +# define PIPENAME "/tmp/netdata-ipc" +#endif + +#define MAX_COMMAND_LENGTH 4096 +#define MAX_EXIT_STATUS_LENGTH 23 /* Can't ever be bigger than "X-18446744073709551616" */ + +typedef enum cmd { + CMD_HELP = 0, + CMD_RELOAD_HEALTH, + CMD_SAVE_DATABASE, + CMD_REOPEN_LOGS, + CMD_EXIT, + CMD_FATAL, + CMD_RELOAD_CLAIMING_STATE, + CMD_RELOAD_LABELS, + CMD_READ_CONFIG, + CMD_WRITE_CONFIG, + CMD_PING, + CMD_TOTAL_COMMANDS +} cmd_t; + +typedef enum cmd_status { + CMD_STATUS_SUCCESS = 0, + CMD_STATUS_FAILURE, + CMD_STATUS_BUSY +} cmd_status_t; + +#define CMD_PREFIX_INFO 'O' /* Following string should go to cli stdout */ +#define CMD_PREFIX_ERROR 'E' /* Following string should go to cli stderr */ +#define CMD_PREFIX_EXIT_CODE 'X' /* Following string is cli integer exit code */ + +typedef enum cmd_type { + /* + * No other command is allowed to run at the same time (except for CMD_TYPE_HIGH_PRIORITY). + */ + CMD_TYPE_EXCLUSIVE = 0, + /* + * Other commands are allowed to run concurrently (except for CMD_TYPE_EXCLUSIVE) but calls to this command are + * serialized. + */ + CMD_TYPE_ORTHOGONAL, + /* + * Other commands are allowed to run concurrently (except for CMD_TYPE_EXCLUSIVE) as are calls to this command. + */ + CMD_TYPE_CONCURRENT, + /* + * Those commands are always allowed to run. + */ + CMD_TYPE_HIGH_PRIORITY +} cmd_type_t; + +/** + * Executes a command and returns the status. + * + * @param args a string that may contain additional parameters to be parsed + * @param message allocate and return a message if need be (up to MAX_COMMAND_LENGTH bytes) + * @return CMD_FAILURE or CMD_SUCCESS + */ +typedef cmd_status_t (command_action_t) (char *args, char **message); + +typedef struct command_info { + char *cmd_str; // the command string + command_action_t *func; // the function that executes the command + cmd_type_t type; // Concurrency control information for the command +} command_info_t; + +typedef void (command_lock_t) (unsigned index); + +cmd_status_t execute_command(cmd_t idx, char *args, char **message); +extern void commands_init(void); +extern void commands_exit(void); + +#endif //NETDATA_COMMANDS_H diff --git a/daemon/common.c b/daemon/common.c index e28f6cf00..45d5fa3fd 100644 --- a/daemon/common.c +++ b/daemon/common.c @@ -10,7 +10,10 @@ char *netdata_configured_primary_plugins_dir = NULL; char *netdata_configured_web_dir = WEB_DIR; char *netdata_configured_cache_dir = CACHE_DIR; char *netdata_configured_varlib_dir = VARLIB_DIR; -char *netdata_configured_home_dir = CACHE_DIR; +char *netdata_configured_lock_dir = NULL; +char *netdata_configured_home_dir = VARLIB_DIR; char *netdata_configured_host_prefix = NULL; char *netdata_configured_timezone = NULL; +int netdata_ready; +int netdata_cloud_setting; diff --git a/daemon/common.h b/daemon/common.h index dfbd6cfef..68af95740 100644 --- a/daemon/common.h +++ b/daemon/common.h @@ -8,7 +8,7 @@ // ---------------------------------------------------------------------------- // shortcuts for the default netdata configuration -#define config_load(filename, overwrite_used) appconfig_load(&netdata_config, filename, overwrite_used) +#define config_load(filename, overwrite_used, section) appconfig_load(&netdata_config, filename, overwrite_used, section) #define config_get(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value) #define config_get_number(section, name, value) appconfig_get_number(&netdata_config, section, name, value) #define config_get_float(section, name, value) appconfig_get_float(&netdata_config, section, name, value) @@ -27,7 +27,6 @@ #define config_generate(buffer, only_changed) appconfig_generate(&netdata_config, buffer, only_changed) - // ---------------------------------------------------------------------------- // netdata include files @@ -51,6 +50,8 @@ // backends for archiving the metrics #include "backends/backends.h" +// the new exporting engine for archiving the metrics +#include "exporting/exporting_engine.h" // the netdata API #include "web/api/web_api_v1.h" @@ -61,10 +62,22 @@ // netdata unit tests #include "unit_test.h" +// netdata agent claiming +#include "claim/claim.h" + +// netdata agent cloud link +#include "aclk/legacy/agent_cloud_link.h" + +// global GUID map functions + +// netdata agent spawn server +#include "spawn/spawn.h" + // the netdata deamon #include "daemon.h" #include "main.h" #include "signals.h" +#include "commands.h" // global netdata daemon variables extern char *netdata_configured_hostname; @@ -75,12 +88,14 @@ extern char *netdata_configured_primary_plugins_dir; extern char *netdata_configured_web_dir; extern char *netdata_configured_cache_dir; extern char *netdata_configured_varlib_dir; +extern char *netdata_configured_lock_dir; extern char *netdata_configured_home_dir; extern char *netdata_configured_host_prefix; extern char *netdata_configured_timezone; extern int netdata_zero_metrics_enabled; extern int netdata_anonymous_statistics_enabled; -int netdata_ready; +extern int netdata_ready; +extern int netdata_cloud_setting; #endif /* NETDATA_COMMON_H */ diff --git a/daemon/config/README.md b/daemon/config/README.md index 72d961b1e..a1e2b04b5 100644 --- a/daemon/config/README.md +++ b/daemon/config/README.md @@ -1,19 +1,26 @@ +<!-- +title: "Daemon configuration" +description: "The Netdata Agent's daemon is installed preconfigured to collect thousands of metrics every second, but is highly configurable for real-world workloads." +custom_edit_url: https://github.com/netdata/netdata/edit/master/daemon/config/README.md +--> + # Daemon configuration <details markdown="1"><summary>The daemon configuration file is read from `/etc/netdata/netdata.conf`.</summary> Depending on your installation method, Netdata will have been installed either directly under `/`, or under `/opt/netdata`. The paths mentioned here and in the documentation in general assume that your installation is under `/`. If it is not, you will find the exact same paths under `/opt/netdata` as well. (i.e. `/etc/netdata` will be `/opt/netdata/etc/netdata`).</details> -This config file **is not needed by default**. Netdata works fine out of the box without it. But it does allow you to adapt the general behavior of Netdata, in great detail. You can find all these settings, with their default values, by accessing the URL `https://netdata.server.hostname:19999/netdata.conf`. For example check the configuration file of [netdata.firehol.org](http://netdata.firehol.org/netdata.conf). HTTP access to this file is limited by default to private IPs, via the [web server access lists](../../web/server/#access-lists). +This config file **is not needed by default**. Netdata works fine out of the box without it. But it does allow you to adapt the general behavior of Netdata, in great detail. You can find all these settings, with their default values, by accessing the URL `https://netdata.server.hostname:19999/netdata.conf`. For example check the configuration file of [netdata.firehol.org](http://netdata.firehol.org/netdata.conf). HTTP access to this file is limited by default to private IPs, via the [web server access lists](/web/server/README.md#access-lists). `netdata.conf` has sections stated with `[section]`. You will see the following sections: -1. `[global]` to [configure](#global-section-options) the [Netdata daemon](../). -2. `[web]` to [configure the web server](../../web/server). -3. `[plugins]` to [configure](#plugins-section-options) which [collectors](../../collectors) to use and PATH settings. -4. `[health]` to [configure](#health-section-options) general settings for [health monitoring](../../health) -5. `[registry]` for the [Netdata registry](../../registry). -6. `[backend]` to set up [streaming and replication](../../streaming) options. -7. `[statsd]` for the general settings of the [stats.d.plugin](../../collectors/statsd.plugin). +1. `[global]` to [configure](#global-section-options) the [Netdata daemon](/daemon/README.md). +2. `[web]` to [configure the web server](/web/server/README.md). +3. `[plugins]` to [configure](#plugins-section-options) which [collectors](/collectors/README.md) to use and PATH + settings. +4. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md) +5. `[registry]` for the [Netdata registry](/registry/README.md). +6. `[backend]` to set up [streaming and replication](/streaming/README.md) options. +7. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md). 8. `[plugin:NAME]` sections for each collector plugin, under the comment [Per plugin configuration](#per-plugin-configuration). 9. `[CHART_NAME]` sections for each chart defined, under the comment [Per chart configuration](#per-chart-configuration). @@ -41,43 +48,47 @@ Please note that your data history will be lost if you have modified `history` p | setting|default|info||| |:-----:|:-----:|:---|---|---| -| process scheduling policy|`keep`|See [Netdata process scheduling policy](../#netdata-process-scheduling-policy)||| +| process scheduling policy|`keep`|See [Netdata process scheduling policy](/daemon/README.md#netdata-process-scheduling-policy)||| | OOM score|`1000`|See [OOM score](../#oom-score)||| -| glibc malloc arena max for plugins|`1`|See [Virtual memory](../#virtual-memory).||| -| glibc malloc arena max for Netdata|`1`|See [Virtual memory](../#virtual-memory).||| +| glibc malloc arena max for plugins|`1`|See [Virtual memory](/daemon/README.md#virtual-memory).||| +| glibc malloc arena max for Netdata|`1`|See [Virtual memory](/daemon/README.md#virtual-memory).||| | hostname|auto-detected|The hostname of the computer running Netdata.||| -| history|`3996`|The number of entries the `netdata` daemon will by default keep in memory for each chart dimension. This setting can also be configured per chart. Check [Memory Requirements](../../database/#database) for more information.||| -| update every|`1`|The frequency in seconds, for data collection. For more information see [Performance](../../docs/Performance.md#performance).||| +| history|`3996`| Used with `memory mode = save/map/ram/alloc`, not the default `memory mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. This setting can also be configured per chart. Check [Memory Requirements](/database/README.md) for more information. ||| +| update every|`1`|The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md).||| | config directory|`/etc/netdata`|The directory configuration files are kept.||| | stock config directory|`/usr/lib/netdata/conf.d`|||| -| log directory|`/var/log/netdata`|The directory in which the [log files](../#log-files) are kept.||| +| log directory|`/var/log/netdata`|The directory in which the [log files](/daemon/README.md#log-files) are kept.||| | web files directory|`/usr/share/netdata/web`|The directory the web static files are kept.||| | cache directory|`/var/cache/netdata`|The directory the memory database will be stored if and when Netdata exits. Netdata will re-read the database when it will start again, to continue from the same point.||| | lib directory|`/var/lib/netdata`|Contains the alarm log and the Netdata instance guid.||| | home directory|`/var/cache/netdata`|Contains the db files for the collected metrics||| | plugins directory|`"/usr/libexec/netdata/plugins.d" "/etc/netdata/custom-plugins.d"`|The directory plugin programs are kept. This setting supports multiple directories, space separated. If any directory path contains spaces, enclose it in single or double quotes.||| -| memory mode|`save`|When set to `save` Netdata will save its round robin database on exit and load it on startup. When set to `map` the cache files will be updated in real time (check `man mmap` - do not set this on systems with heavy load or slow disks - the disks will continuously sync the in-memory database of Netdata). When set to `dbengine` it behaves similarly to `map` but with much better disk and memory efficiency, however, with higher overhead. When set to `ram` the round robin database will be temporary and it will be lost when Netdata exits. `none` disables the database at this host. This also disables health monitoring (there cannot be health monitoring without a database). host access prefix||This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43).| -| memory deduplication (ksm)|`yes`|When set to `yes`, Netdata will offer its in-memory round robin database to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](../../database/#ksm)||| +| memory mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size` and `dbengine disk space`. <br />`save`: Netdata will save its round robin database on exit and load it on startup. <br />`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`). <br />`ram`: The round-robin database will be temporary and it will be lost when Netdata exits. <br />`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. | +| page cache size | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. ||| +| dbengine disk space | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. ||| +| dbengine multihost disk space | 256 | Same functionality as `dbengine disk space`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. ||| +| host access prefix||This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43).| +| memory deduplication (ksm)|`yes`|When set to `yes`, Netdata will offer its in-memory round robin database to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm)||| | TZ environment variable|`:/etc/localtime`|Where to find the timezone||| | timezone|auto-detected|The timezone retrieved from the environment variable||| -| debug flags|`0x0000000000000000`|Bitmap of debug options to enable. For more information check [Tracing Options](../#debugging).||| -| debug log|`/var/log/netdata/debug.log`|The filename to save debug information. This file will not be created if debugging is not enabled. You can also set it to `syslog` to send the debug messages to syslog, or `none` to disable this log. For more information check [Tracing Options](../#debugging).||| +| debug flags|`0x0000000000000000`|Bitmap of debug options to enable. For more information check [Tracing Options](/daemon/README.md#debugging).||| +| debug log|`/var/log/netdata/debug.log`|The filename to save debug information. This file will not be created if debugging is not enabled. You can also set it to `syslog` to send the debug messages to syslog, or `none` to disable this log. For more information check [Tracing Options](/daemon/README.md#debugging).||| | error log|`/var/log/netdata/error.log`|The filename to save error messages for Netdata daemon and all plugins (`stderr` is sent here for all Netdata programs, including the plugins). You can also set it to `syslog` to send the errors to syslog, or `none` to disable this log.||| | access log|`/var/log/netdata/access.log`|The filename to save the log of web clients accessing Netdata charts. You can also set it to `syslog` to send the access log to syslog, or `none` to disable this log.||| -| errors flood protection period|`1200`|UNUSED - Length of period (in sec) during which the number of errors should not exceed the `errors to trigger flood protection`.||| -| errors to trigger flood protection|`200`|UNUSED - Number of errors written to the log in `errors flood protection period` sec before flood protection is activated.||| +| errors flood protection period|`1200`|Length of period (in sec) during which the number of errors should not exceed the `errors to trigger flood protection`.||| +| errors to trigger flood protection|`200`|Number of errors written to the log in `errors flood protection period` sec before flood protection is activated.||| | run as user|`netdata`|The user Netdata will run as.||| | pthread stack size|auto-detected|||| -| cleanup obsolete charts after seconds|`3600`|See [monitoring ephemeral containers](../../collectors/cgroups.plugin/#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions||| +| cleanup obsolete charts after seconds|`3600`|See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions||| | gap when lost iterations above|`1`|||| -| cleanup orphan hosts after seconds|`3600`|How long to wait until automatically removing from the DB a remote Netdata host (slave) that is no longer sending data.||| -| delete obsolete charts files|`yes`|See [monitoring ephemeral containers](../../collectors/cgroups.plugin/#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions||| +| cleanup orphan hosts after seconds|`3600`|How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data.||| +| delete obsolete charts files|`yes`|See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions||| | delete orphan hosts files|`yes`|Set to `no` to disable non-responsive host removal.||| | enable zero metrics|`no`|Set to `yes` to show charts when all their metrics are zero.||| ### [web] section options -Refer to the [web server documentation](../../web/server) +Refer to the [web server documentation](/web/server/README.md) ### [plugins] section options @@ -89,7 +100,7 @@ Additionally, there will be the following options: |:-----:|:-----:|:---| | PATH environment variable|`auto-detected`|| | PYTHONPATH environment variable||Used to set a custom python path| -| enable running new plugins|`yes`|When set to `yes`, Netdata will enable detected plugins, even if they are not configured explicitly. Setting this to `no` will only enable plugins explicitly configirued in this file with a `yes`| +| enable running new plugins|`yes`|When set to `yes`, Netdata will enable detected plugins, even if they are not configured explicitly. Setting this to `no` will only enable plugins explicitly configured in this file with a `yes`| | check for new plugins every|60|The time in seconds to check for new plugins in the plugins directory. This allows having other applications dynamically creating plugins for Netdata.| | checks|`no`|This is a debugging plugin for the internal latency| @@ -97,9 +108,10 @@ Additionally, there will be the following options: This section controls the general behavior of the health monitoring capabilities of Netdata. -Specific alarms are configured in per-collector config files under the `health.d` directory. For more info, see [health monitoring](../../health/#health-monitoring). +Specific alarms are configured in per-collector config files under the `health.d` directory. For more info, see [health +monitoring](/health/README.md). -[Alarm notifications](../../health/notifications/#netdata-alarm-notifications) are configured in `health_alarm_notify.conf`. +[Alarm notifications](/health/notifications/README.md) are configured in `health_alarm_notify.conf`. | setting|default|info| |:-----:|:-----:|:---| @@ -114,35 +126,104 @@ Specific alarms are configured in per-collector config files under the `health.d ### [registry] section options -To understand what this section is and how it should be configured, please refer to the [registry documentation](../../registry). +To understand what this section is and how it should be configured, please refer to the [registry documentation](/registry/README.md). ### [backend] -Refer to the [streaming and replication](../../streaming) documentation. +Refer to the [streaming and replication](/streaming/README.md) documentation. -### Per plugin configuration +## Per-plugin configuration The configuration options for plugins appear in sections following the pattern `[plugin:NAME]`. -#### Internal plugins +### Internal plugins -Most internal plugins will provide additional options. Check [Internal Plugins](../../collectors/) for more information. +Most internal plugins will provide additional options. Check [Internal Plugins](/collectors/README.md) for more +information. Please note, that by default Netdata will enable monitoring metrics for disks, memory, and network only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Use `yes` instead of `auto` in plugin configuration sections to enable these charts permanently. You can also set the `enable zero metrics` option to `yes` in the `[global]` section which enables charts with zero metrics for all internal Netdata plugins. -#### External plugins +### External plugins External plugins will have only 2 options at `netdata.conf`: | setting | default | info | | :-----:|:-----:|:---| -| update every | the value of `[global].update every` setting|The frequency in seconds the plugin should collect values. For more information check [Performance](../../docs/Performance.md#performance).| +| update every | the value of `[global].update every` setting|The frequency in seconds the plugin should collect values. For more information check the [performance guide](/docs/guides/configure/performance.md).| | command options | _empty_ | Additional command line options to pass to the plugin.| External plugins that need additional configuration may support a dedicated file in `/etc/netdata`. Check their documentation. -### Per chart configuration +## Per-chart configuration + +In this area of `netdata.conf` you can find configuration options for individual charts. They appear in sections +following the pattern `[NAME]`. + +Using the settings and values under these sections, you can control all aspects of a specific chart. You can change its +title, make it appear higher in Netdata's [menu](/web/gui/README.md#metrics-menus), tweak its dimensions, and much more. + +To find the name of a given chart, and thus the name of its section in `netdata.conf`, look at the top-left corner of a +chart: + + + +Every per-chart configuration section has several common settings, which are listed in the table just below. Beneath +that is information about lines that begin with `dim`, which affect a chart's dimensions. + +| Setting | Function | +| :---------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `history` | Override the `history` setting in the [[global] options](#global-section-options) for this particular chart. Should be less than or equal to the global `history` setting. | +| `enabled` | A boolean (`yes` or `no`) that explicitly enables or disables the chart in question. | +| `cache directory` | The directory where cache files for this plugin, if needed, are stored. | +| `chart type` | Defines what type of chart to display. It can be `line`, `area`, or `stacked`. If empty or missing, `line` will be used. | +| `type` | Uniquely identify which [metrics menu](/web/gui/README.md#metrics-menus) on the Netdata dashboard this chart should appear under. Some examples include `system` (**System**), `disk` (**Disks**), `net` (**Network Interfaces**), and `netdata` (**Netdata Monitoring**). | +| `family` | Change the chart's [family](/web/README.md#families) from its default. For example, you could force a disk space chart to collect metrics for family `sdb` instead of family `sda`. | +| `units` | Text for the label of the vertical axis of the chart. This means all dimensions should have the same unit of measurement. | +| `context` | Change the default [context](/web/README.md#contexts) of the chart. Changing this setting will affect what metrics and metrics the chart displays, and which alarms are attached to it. | +| `priority` | Define where the chart should appear on the Netdata dashboard. Lower values equal higher priority, so a priority of `1` will place the chart highest, while a priority of `9999999` would place the chart at the bottom of the Netdata dashboard. | +| `name` | The name of the chart that appears in the top-left corner, after the chart's title. You can also use this name when writing [health entities](/health/REFERENCE.md#health-entity-reference). | +| `title` | The text that appears above the chart in the Netdata dashboard. | + +### Dimension settings + +You may notice some settings that begin with `dim` beneath the ones defined in the table above. These settings determine +which dimensions appear on the given chart and how Netdata calculates them. + +Each dimension setting has the following structure: `dim [DIMENSION ID] [OPTION] = [VALUE]`. The available options are `name`, `algorithm`, `multiplier`, and `divisor`. + +| Setting | Function | +| :----------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | The name of the dimension as it will appear in the legend of the chart. If left empty, or is missing, Netdata will use the `[DIMENSION ID]` instead. | +| `algorithm` | Can be `absolute`, `incremental`, `percentage-of-absolute-row`, or `percentage-of-incremental-row`. If this setting is empty, invalid, or missing, Netdata will use `absolute`. See the list beneath this table for descriptions of what each algorithm does. | +| `multiplier` | An integer value by which to multiply the collected value. If empty or missing, Netdata will use `1`. This setting is often used with the value `1024` to convert metabytes to kilobytes, kilobytes to bytes, and so on. | +| `divisor` | An integer value by which to divide the collected value. If empty or missing, Netdata will use `1`. This setting is often used with the value `1024` to convert bytes to kilobytes, kilobytes to megabytes, and so on. | + +Here are the options for the `algorithm` setting: + +- `absolute`: The value is drawn as-is (interpolated to second boundary). +- `incremental`: To be used when the value always increases over time, such as the I/O on a disk. Netdata takes the + difference between the current metric and the past metric to calculate a per-second figure. +- `percentage-of-absolute-row`: The % of this value compared to the total of all dimensions. +- `percentage-of-incremental-row`: The % of this value compared to the incremental total of all dimensions. + +For example, the `system.io` chart has the following default settings: + +```conf + # dim in name = in + # dim in algorithm = incremental + # dim in multiplier = 1 + # dim in divisor = 1 + # dim out name = out + # dim out algorithm = incremental + # dim out multiplier = -1 + # dim out divisor = 1 +``` + +These `dim` settings produce two dimensions, `in` and `out`, both of which use the `incremental` algorithm. By +multiplying the value of `out` by -1, Netdata creates the negative values seen in the following area chart: -In this section you will find a separate subsection for each chart shown on the dashboard. You can control all aspects of a specific chart here. You can understand what each option does by reading [how charts are defined](../../collectors/plugins.d/#chart). If you don't know how to find the name of a chart, you can learn about it [here](../../web/README.md#charts-contexts-families). + [](<>) diff --git a/daemon/daemon.c b/daemon/daemon.c index 4ad082b95..83191109a 100644 --- a/daemon/daemon.c +++ b/daemon/daemon.c @@ -4,6 +4,28 @@ #include <sched.h> char pidfile[FILENAME_MAX + 1] = ""; +char claimingdirectory[FILENAME_MAX + 1]; +char exepath[FILENAME_MAX + 1]; + +void get_netdata_execution_path(void) +{ + int ret; + size_t exepath_size = 0; + struct passwd *passwd = NULL; + char *user = NULL; + + passwd = getpwuid(getuid()); + user = (passwd && passwd->pw_name) ? passwd->pw_name : ""; + + exepath_size = sizeof(exepath) - 1; + ret = uv_exepath(exepath, &exepath_size); + if (0 != ret) { + error("uv_exepath(\"%s\", %u) (user: %s) failed (%s).", exepath, (unsigned)exepath_size, user, + uv_strerror(ret)); + fatal("Cannot start netdata without getting execution path."); + } + exepath[exepath_size] = '\0'; +} static void chown_open_file(int fd, uid_t uid, gid_t gid) { if(fd == -1) return; @@ -36,6 +58,22 @@ void create_needed_dir(const char *dir, uid_t uid, gid_t gid) error("Cannot create directory '%s'", dir); } +void clean_directory(char *dirname) +{ + DIR *dir = opendir(dirname); + if(!dir) return; + + int dir_fd = dirfd(dir); + struct dirent *de = NULL; + + while((de = readdir(dir))) + if(de->d_type == DT_REG) + if (unlinkat(dir_fd, de->d_name, 0)) + error("Cannot delete %s/%s", dirname, de->d_name); + + closedir(dir); +} + int become_user(const char *username, int pid_fd) { int am_i_root = (getuid() == 0)?1:0; @@ -50,6 +88,10 @@ int become_user(const char *username, int pid_fd) { create_needed_dir(netdata_configured_cache_dir, uid, gid); create_needed_dir(netdata_configured_varlib_dir, uid, gid); + create_needed_dir(netdata_configured_lock_dir, uid, gid); + create_needed_dir(claimingdirectory, uid, gid); + + clean_directory(netdata_configured_lock_dir); if(pidfile[0]) { if(chown(pidfile, uid, gid) == -1) @@ -434,6 +476,9 @@ int become_daemon(int dont_fork, const char *user) // never become a problem sched_setscheduler_set(); + // Set claiming directory based on user config directory with correct ownership + snprintfz(claimingdirectory, FILENAME_MAX, "%s/cloud.d", netdata_configured_varlib_dir); + if(user && *user) { if(become_user(user, pidfd) != 0) { error("Cannot become user '%s'. Continuing as we are.", user); @@ -443,6 +488,10 @@ int become_daemon(int dont_fork, const char *user) else { create_needed_dir(netdata_configured_cache_dir, getuid(), getgid()); create_needed_dir(netdata_configured_varlib_dir, getuid(), getgid()); + create_needed_dir(netdata_configured_lock_dir, getuid(), getgid()); + create_needed_dir(claimingdirectory, getuid(), getgid()); + + clean_directory(netdata_configured_lock_dir); } if(pidfd != -1) diff --git a/daemon/daemon.h b/daemon/daemon.h index 5d176341a..bec3df9fc 100644 --- a/daemon/daemon.h +++ b/daemon/daemon.h @@ -10,7 +10,9 @@ extern int become_daemon(int dont_fork, const char *user); extern void netdata_cleanup_and_exit(int i); extern void send_statistics(const char *action, const char *action_result, const char *action_data); -extern char pidfile[]; +extern void get_netdata_execution_path(void); +extern char pidfile[]; +extern char exepath[]; #endif /* NETDATA_DAEMON_H */ diff --git a/daemon/get-kubernetes-labels.sh.in b/daemon/get-kubernetes-labels.sh.in new file mode 100644 index 000000000..805d027b8 --- /dev/null +++ b/daemon/get-kubernetes-labels.sh.in @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Checks if netdata is running in a kubernetes pod and fetches that pod's labels + +if [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ] && [ -n "${MY_POD_NAMESPACE}" ] && [ -n "${MY_POD_NAME}" ]; then + if command -v jq >/dev/null 2>&1; then + KUBE_TOKEN="$(</var/run/secrets/kubernetes.io/serviceaccount/token)" + URL="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$MY_POD_NAMESPACE/pods/$MY_POD_NAME" + curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" "$URL" | + jq -r '.metadata.labels' | grep ':' | tr -d '," ' + exit 0 + else + echo "jq command not available. Please install jq to get host labels for kubernetes pods." + exit 1 + fi +else + exit 0 +fi diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index 5197dcc10..7e7835513 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -537,14 +537,19 @@ void global_statistics_charts(void) { RRDHOST *host; unsigned long long stats_array[RRDENG_NR_STATS] = {0}; unsigned long long local_stats_array[RRDENG_NR_STATS]; - unsigned hosts_with_dbengine = 0, i; + unsigned dbengine_contexts = 0, counted_multihost_db = 0, i; rrd_rdlock(); rrdhost_foreach_read(host) { - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - ++hosts_with_dbengine; + if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) { + if (&multidb_ctx == host->rrdeng_ctx) { + if (counted_multihost_db) + continue; /* Only count multi-host DB once */ + counted_multihost_db = 1; + } + ++dbengine_contexts; /* get localhost's DB engine's statistics */ - rrdeng_get_35_statistics(host->rrdeng_ctx, local_stats_array); + rrdeng_get_37_statistics(host->rrdeng_ctx, local_stats_array); for (i = 0 ; i < RRDENG_NR_STATS ; ++i) { /* aggregate statistics across hosts */ stats_array[i] += local_stats_array[i]; @@ -553,11 +558,13 @@ void global_statistics_charts(void) { } rrd_unlock(); - if (hosts_with_dbengine) { - /* deduplicate global statistics by getting the ones from the last host */ + if (dbengine_contexts) { + /* deduplicate global statistics by getting the ones from the last context */ stats_array[30] = local_stats_array[30]; stats_array[31] = local_stats_array[31]; stats_array[32] = local_stats_array[32]; + stats_array[34] = local_stats_array[34]; + stats_array[36] = local_stats_array[36]; // ---------------------------------------------------------------- @@ -642,7 +649,6 @@ void global_statistics_charts(void) { old_misses = misses; if (hits_delta + misses_delta) { - // allow negative savings ratio = (hits_delta * 100 * 1000) / (hits_delta + misses_delta); } else { ratio = 0; @@ -658,11 +664,10 @@ void global_statistics_charts(void) { static RRDSET *st_pg_cache_pages = NULL; static RRDDIM *rd_descriptors = NULL; static RRDDIM *rd_populated = NULL; - static RRDDIM *rd_committed = NULL; - static RRDDIM *rd_insertions = NULL; - static RRDDIM *rd_deletions = NULL; + static RRDDIM *rd_dirty = NULL; static RRDDIM *rd_backfills = NULL; static RRDDIM *rd_evictions = NULL; + static RRDDIM *rd_used_by_collectors = NULL; if (unlikely(!st_pg_cache_pages)) { st_pg_cache_pages = rrdset_create_localhost( @@ -671,7 +676,7 @@ void global_statistics_charts(void) { , NULL , "dbengine" , NULL - , "NetData DB engine page statistics" + , "NetData dbengine page cache statistics" , "pages" , "netdata" , "stats" @@ -682,28 +687,69 @@ void global_statistics_charts(void) { rd_descriptors = rrddim_add(st_pg_cache_pages, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_committed = rrddim_add(st_pg_cache_pages, "committed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_insertions = rrddim_add(st_pg_cache_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_deletions = rrddim_add(st_pg_cache_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_dirty = rrddim_add(st_pg_cache_pages, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rd_backfills = rrddim_add(st_pg_cache_pages, "backfills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rd_evictions = rrddim_add(st_pg_cache_pages, "evictions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_used_by_collectors = rrddim_add(st_pg_cache_pages, "used_by_collectors", NULL, 1, 1, + RRD_ALGORITHM_ABSOLUTE); } else rrdset_next(st_pg_cache_pages); rrddim_set_by_pointer(st_pg_cache_pages, rd_descriptors, (collected_number)stats_array[27]); rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]); - rrddim_set_by_pointer(st_pg_cache_pages, rd_committed, (collected_number)stats_array[4]); - rrddim_set_by_pointer(st_pg_cache_pages, rd_insertions, (collected_number)stats_array[5]); - rrddim_set_by_pointer(st_pg_cache_pages, rd_deletions, (collected_number)stats_array[6]); + rrddim_set_by_pointer(st_pg_cache_pages, rd_dirty, (collected_number)stats_array[0] + stats_array[4]); rrddim_set_by_pointer(st_pg_cache_pages, rd_backfills, (collected_number)stats_array[9]); rrddim_set_by_pointer(st_pg_cache_pages, rd_evictions, (collected_number)stats_array[10]); + rrddim_set_by_pointer(st_pg_cache_pages, rd_used_by_collectors, (collected_number)stats_array[0]); rrdset_done(st_pg_cache_pages); } // ---------------------------------------------------------------- { + static RRDSET *st_long_term_pages = NULL; + static RRDDIM *rd_total = NULL; + static RRDDIM *rd_insertions = NULL; + static RRDDIM *rd_deletions = NULL; + static RRDDIM *rd_flushing_pressure_deletions = NULL; + + if (unlikely(!st_long_term_pages)) { + st_long_term_pages = rrdset_create_localhost( + "netdata" + , "dbengine_long_term_page_stats" + , NULL + , "dbengine" + , NULL + , "NetData dbengine long-term page statistics" + , "pages" + , "netdata" + , "stats" + , 130505 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_total = rrddim_add(st_long_term_pages, "total", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_insertions = rrddim_add(st_long_term_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_deletions = rrddim_add(st_long_term_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_flushing_pressure_deletions = rrddim_add(st_long_term_pages, "flushing_pressure_deletions", NULL, -1, + 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_long_term_pages); + + rrddim_set_by_pointer(st_long_term_pages, rd_total, (collected_number)stats_array[2]); + rrddim_set_by_pointer(st_long_term_pages, rd_insertions, (collected_number)stats_array[5]); + rrddim_set_by_pointer(st_long_term_pages, rd_deletions, (collected_number)stats_array[6]); + rrddim_set_by_pointer(st_long_term_pages, rd_flushing_pressure_deletions, + (collected_number)stats_array[36]); + rrdset_done(st_long_term_pages); + } + + // ---------------------------------------------------------------- + + { static RRDSET *st_io_stats = NULL; static RRDDIM *rd_reads = NULL; static RRDDIM *rd_writes = NULL; @@ -719,7 +765,7 @@ void global_statistics_charts(void) { , "MiB/s" , "netdata" , "stats" - , 130505 + , 130506 , localhost->rrd_update_every , RRDSET_TYPE_LINE ); @@ -753,7 +799,7 @@ void global_statistics_charts(void) { , "operations/s" , "netdata" , "stats" - , 130506 + , 130507 , localhost->rrd_update_every , RRDSET_TYPE_LINE ); @@ -775,7 +821,7 @@ void global_statistics_charts(void) { static RRDSET *st_errors = NULL; static RRDDIM *rd_fs_errors = NULL; static RRDDIM *rd_io_errors = NULL; - static RRDDIM *rd_flushing_errors = NULL; + static RRDDIM *pg_cache_over_half_dirty_events = NULL; if (unlikely(!st_errors)) { st_errors = rrdset_create_localhost( @@ -788,21 +834,22 @@ void global_statistics_charts(void) { , "errors/s" , "netdata" , "stats" - , 130507 + , 130508 , localhost->rrd_update_every , RRDSET_TYPE_LINE ); - rd_io_errors = rrddim_add(st_errors, "I/O errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_fs_errors = rrddim_add(st_errors, "FS errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); - rd_flushing_errors = rrddim_add(st_errors, "flushing errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_io_errors = rrddim_add(st_errors, "io_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_fs_errors = rrddim_add(st_errors, "fs_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + pg_cache_over_half_dirty_events = rrddim_add(st_errors, "pg_cache_over_half_dirty_events", NULL, 1, 1, + RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(st_errors); rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]); rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]); - rrddim_set_by_pointer(st_errors, rd_flushing_errors, (collected_number)stats_array[34]); + rrddim_set_by_pointer(st_errors, pg_cache_over_half_dirty_events, (collected_number)stats_array[34]); rrdset_done(st_errors); } @@ -824,7 +871,7 @@ void global_statistics_charts(void) { , "descriptors" , "netdata" , "stats" - , 130508 + , 130509 , localhost->rrd_update_every , RRDSET_TYPE_LINE ); @@ -863,7 +910,7 @@ void global_statistics_charts(void) { , "MiB" , "netdata" , "stats" - , 130509 + , 130510 , localhost->rrd_update_every , RRDSET_TYPE_STACKED ); diff --git a/daemon/main.c b/daemon/main.c index de7a43702..7c002ac47 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -1,12 +1,14 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "common.h" +#include "buildinfo.h" int netdata_zero_metrics_enabled; int netdata_anonymous_statistics_enabled; struct config netdata_config = { - .sections = NULL, + .first_section = NULL, + .last_section = NULL, .mutex = NETDATA_MUTEX_INITIALIZER, .index = { .avl_tree = { @@ -35,12 +37,18 @@ void netdata_cleanup_and_exit(int ret) { // exit cleanly // stop everything - info("EXIT: stopping master threads..."); + info("EXIT: stopping static threads..."); cancel_main_threads(); // free the database info("EXIT: freeing database memory..."); +#ifdef ENABLE_DBENGINE + rrdeng_prepare_exit(&multidb_ctx); +#endif rrdhost_free_all(); +#ifdef ENABLE_DBENGINE + rrdeng_exit(&multidb_ctx); +#endif } // unlink the pid @@ -53,7 +61,6 @@ void netdata_cleanup_and_exit(int ret) { #ifdef ENABLE_HTTPS security_clean_openssl(); #endif - info("EXIT: all done - netdata is now exiting - bye bye..."); exit(ret); } @@ -73,8 +80,13 @@ struct netdata_static_thread static_threads[] = { NETDATA_PLUGIN_HOOK_IDLEJITTER NETDATA_PLUGIN_HOOK_STATSD +#ifdef ENABLE_ACLK + NETDATA_ACLK_HOOK +#endif + // common plugins for all systems {"BACKENDS", NULL, NULL, 1, NULL, NULL, backends_main}, + {"EXPORTING", NULL, NULL, 1, NULL, NULL, exporting_main}, {"WEB_SERVER[static1]", NULL, NULL, 0, NULL, NULL, socket_listen_main_static_threaded}, {"STREAM", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread}, @@ -106,6 +118,7 @@ int make_dns_decision(const char *section_name, const char *config_name, const c if(strcmp("heuristic",value)) error("Invalid configuration option '%s' for '%s'/'%s'. Valid options are 'yes', 'no' and 'heuristic'. Proceeding with 'heuristic'", value, section_name, config_name); + return simple_pattern_is_potential_name(p); } @@ -151,9 +164,9 @@ void web_server_config_options(void) "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*" " 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*" " 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*" - " 172.31.*"), NULL, SIMPLE_PATTERN_EXACT); + " 172.31.* UNKNOWN"), NULL, SIMPLE_PATTERN_EXACT); web_allow_netdataconf_dns = - make_dns_decision(CONFIG_SECTION_WEB, "allow netdata.conf by dns", "no", web_allow_mgmt_from); + make_dns_decision(CONFIG_SECTION_WEB, "allow netdata.conf by dns", "no", web_allow_netdataconf_from); web_allow_mgmt_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"), NULL, SIMPLE_PATTERN_EXACT); @@ -226,7 +239,7 @@ void cancel_main_threads() { usec_t max = 5 * USEC_PER_SEC, step = 100000; for (i = 0; static_threads[i].name != NULL ; i++) { if(static_threads[i].enabled == NETDATA_MAIN_THREAD_RUNNING) { - info("EXIT: Stopping master thread: %s", static_threads[i].name); + info("EXIT: Stopping main thread: %s", static_threads[i].name); netdata_thread_cancel(*static_threads[i].thread); found++; } @@ -248,7 +261,7 @@ void cancel_main_threads() { if(found) { for (i = 0; static_threads[i].name != NULL ; i++) { if (static_threads[i].enabled != NETDATA_MAIN_THREAD_EXITED) - error("Master thread %s takes too long to exit. Giving up...", static_threads[i].name); + error("Main thread %s takes too long to exit. Giving up...", static_threads[i].name); } } else @@ -300,13 +313,13 @@ int help(int exitcode) { " | '-' '-' '-' '-' real-time performance monitoring, done right! \n" " +----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+--->\n" "\n" - " Copyright (C) 2016-2017, Costa Tsaousis <costa@tsaousis.gr>\n" + " Copyright (C) 2016-2020, Netdata, Inc. <info@netdata.cloud>\n" " Released under GNU General Public License v3 or later.\n" " All rights reserved.\n" "\n" - " Home Page : https://my-netdata.io\n" + " Home Page : https://netdata.cloud\n" " Source Code: https://github.com/netdata/netdata\n" - " Wiki / Docs: https://github.com/netdata/netdata/wiki\n" + " Docs : https://learn.netdata.cloud\n" " Support : https://github.com/netdata/netdata/issues\n" " License : https://github.com/netdata/netdata/blob/master/LICENSE.md\n" "\n" @@ -337,15 +350,19 @@ int help(int exitcode) { " -W unittest Run internal unittests and exit.\n\n" #ifdef ENABLE_DBENGINE " -W createdataset=N Create a DB engine dataset of N seconds and exit.\n\n" - " -W stresstest=A,B,C,D,E Run a DB engine stress test for A seconds,\n" + " -W stresstest=A,B,C,D,E,F\n" + " Run a DB engine stress test for A seconds,\n" " with B writers and C readers, with a ramp up\n" " time of D seconds for writers, a page cache\n" - " size of E MiB, and exit.\n\n" + " size of E MiB, an optional disk space limit" + " of F MiB and exit.\n\n" #endif " -W set section option value\n" " set netdata.conf option from the command line.\n\n" " -W simple-pattern pattern string\n" " Check if string matches pattern and exit.\n\n" + " -W \"claim -token=TOKEN -rooms=ROOM1,ROOM2\"\n" + " Claim the agent to the workspace rooms pointed to by TOKEN and ROOM*.\n\n" ); fprintf(stream, "\n Signals netdata handles:\n\n" @@ -394,6 +411,9 @@ static void security_init(){ snprintfz(filename, FILENAME_MAX, "%s/ssl/cert.pem",netdata_configured_user_config_dir); security_cert = config_get(CONFIG_SECTION_WEB, "ssl certificate", filename); + tls_version = config_get(CONFIG_SECTION_WEB, "tls version", "1.3"); + tls_ciphers = config_get(CONFIG_SECTION_WEB, "tls ciphers", "none"); + security_openssl_library(); } #endif @@ -421,6 +441,14 @@ static void log_init(void) { setenv("NETDATA_ERRORS_PER_PERIOD", config_get(CONFIG_SECTION_GLOBAL, "errors to trigger flood protection", ""), 1); } +char *initialize_lock_directory_path(char *prefix) +{ + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/lock", prefix); + + return config_get(CONFIG_SECTION_GLOBAL, "lock directory", filename); +} + static void backwards_compatible_config() { // move [global] options to the [web] section config_move(CONFIG_SECTION_GLOBAL, "http port listen backlog", @@ -514,7 +542,10 @@ static void get_netdata_configured_variables() { netdata_configured_web_dir = config_get(CONFIG_SECTION_GLOBAL, "web files directory", netdata_configured_web_dir); netdata_configured_cache_dir = config_get(CONFIG_SECTION_GLOBAL, "cache directory", netdata_configured_cache_dir); netdata_configured_varlib_dir = config_get(CONFIG_SECTION_GLOBAL, "lib directory", netdata_configured_varlib_dir); - netdata_configured_home_dir = config_get(CONFIG_SECTION_GLOBAL, "home directory", netdata_configured_home_dir); + char *env_home=getenv("HOME"); + netdata_configured_home_dir = config_get(CONFIG_SECTION_GLOBAL, "home directory", env_home?env_home:netdata_configured_home_dir); + + netdata_configured_lock_dir = initialize_lock_directory_path(netdata_configured_varlib_dir); { pluginsd_initialize_plugin_directories(); @@ -544,6 +575,13 @@ static void get_netdata_configured_variables() { error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB); default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB; } + + default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", compute_multidb_diskspace()); + if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { + error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb); + default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; + } + #endif // ------------------------------------------------------------------------ @@ -563,6 +601,8 @@ static void get_netdata_configured_variables() { get_system_HZ(); get_system_cpus(); get_system_pid_max(); + + } static void get_system_timezone(void) { @@ -670,10 +710,22 @@ void set_global_environment() { setenv("NETDATA_WEB_DIR" , verify_required_directory(netdata_configured_web_dir), 1); setenv("NETDATA_CACHE_DIR" , verify_required_directory(netdata_configured_cache_dir), 1); setenv("NETDATA_LIB_DIR" , verify_required_directory(netdata_configured_varlib_dir), 1); + setenv("NETDATA_LOCK_DIR" , netdata_configured_lock_dir, 1); setenv("NETDATA_LOG_DIR" , verify_required_directory(netdata_configured_log_dir), 1); setenv("HOME" , verify_required_directory(netdata_configured_home_dir), 1); setenv("NETDATA_HOST_PREFIX" , netdata_configured_host_prefix, 1); + char *default_port = appconfig_get(&netdata_config, CONFIG_SECTION_WEB, "default port", NULL); + int clean = 0; + if (!default_port) { + default_port = strdupz("19999"); + clean = 1; + } + + setenv("NETDATA_LISTEN_PORT" , default_port, 1); + if(clean) + freez(default_port); + get_system_timezone(); // set the path we need @@ -700,20 +752,20 @@ static int load_netdata_conf(char *filename, char overwrite_used) { int ret = 0; if(filename && *filename) { - ret = config_load(filename, overwrite_used); + ret = config_load(filename, overwrite_used, NULL); if(!ret) error("CONFIG: cannot load config file '%s'.", filename); } else { filename = strdupz_path_subpath(netdata_configured_user_config_dir, "netdata.conf"); - ret = config_load(filename, overwrite_used); + ret = config_load(filename, overwrite_used, NULL); if(!ret) { info("CONFIG: cannot load user config '%s'. Will try the stock version.", filename); freez(filename); filename = strdupz_path_subpath(netdata_configured_stock_config_dir, "netdata.conf"); - ret = config_load(filename, overwrite_used); + ret = config_load(filename, overwrite_used, NULL); if(!ret) info("CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename); } @@ -724,6 +776,12 @@ static int load_netdata_conf(char *filename, char overwrite_used) { return ret; } +// coverity[ +tainted_string_sanitize_content : arg-0 ] +static inline void coverity_remove_taint(char *s) +{ + (void)s; +} + int get_system_info(struct rrdhost_system_info *system_info) { char *script; script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("system-info.sh") + 2)); @@ -740,27 +798,27 @@ int get_system_info(struct rrdhost_system_info *system_info) { FILE *fp = mypopen(script, &command_pid); if(fp) { - char buffer[200 + 1]; - while (fgets(buffer, 200, fp) != NULL) { - char *name=buffer; - char *value=buffer; + char line[200 + 1]; + // Removed the double strlens, if the Coverity tainted string warning reappears I'll revert. + // One time init code, but I'm curious about the warning... + while (fgets(line, 200, fp) != NULL) { + char *value=line; while (*value && *value != '=') value++; if (*value=='=') { *value='\0'; value++; - if (strlen(value)>1) { - char *newline = value + strlen(value) - 1; - (*newline) = '\0'; - } - char n[51], v[101]; - snprintfz(n, 50,"%s",name); - snprintfz(v, 100,"%s",value); - if(unlikely(rrdhost_set_system_info_variable(system_info, n, v))) { - info("Unexpected environment variable %s=%s", n, v); + char *end = value; + while (*end && *end != '\n') end++; + *end = '\0'; // Overwrite newline if present + coverity_remove_taint(line); // I/O is controlled result of system_info.sh - not tainted + coverity_remove_taint(value); + + if(unlikely(rrdhost_set_system_info_variable(system_info, line, value))) { + info("Unexpected environment variable %s=%s", line, value); } else { - info("%s=%s", n, v); - setenv(n, v, 1); + info("%s=%s", line, value); + setenv(line, value, 1); } } } @@ -772,6 +830,7 @@ int get_system_info(struct rrdhost_system_info *system_info) { void send_statistics( const char *action, const char *action_result, const char *action_data) { static char *as_script; + if (netdata_anonymous_statistics_enabled == -1) { char *optout_file = mallocz(sizeof(char) * (strlen(netdata_configured_user_config_dir) +strlen(".opt-out-from-anonymous-statistics") + 2)); sprintf(optout_file, "%s/%s", netdata_configured_user_config_dir, ".opt-out-from-anonymous-statistics"); @@ -816,11 +875,41 @@ void set_silencers_filename() { silencers_filename = config_get(CONFIG_SECTION_HEALTH, "silencers file", filename); } +/* Any config setting that can be accessed without a default value i.e. configget(...,...,NULL) *MUST* + be set in this procedure to be called in all the relevant code paths. +*/ +void post_conf_load(char **user) +{ + // -------------------------------------------------------------------- + // get the user we should run + + // IMPORTANT: this is required before web_files_uid() + if(getuid() == 0) { + *user = config_get(CONFIG_SECTION_GLOBAL, "run as user", NETDATA_USER); + } + else { + struct passwd *passwd = getpwuid(getuid()); + *user = config_get(CONFIG_SECTION_GLOBAL, "run as user", (passwd && passwd->pw_name)?passwd->pw_name:""); + } + + // -------------------------------------------------------------------- + // Check if the cloud is enabled +#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK ) + netdata_cloud_setting = 0; +#else + netdata_cloud_setting = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1); +#endif + // This must be set before any point in the code that accesses it. Do not move it from this function. + appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL); +} + int main(int argc, char **argv) { int i; int config_loaded = 0; int dont_fork = 0; size_t default_stacksize; + char *user = NULL; + netdata_ready=0; // set the name for logging @@ -854,6 +943,11 @@ int main(int argc, char **argv) { else i++; } } + if (argc > 1 && strcmp(argv[1], SPAWN_SERVER_COMMAND_LINE_ARGUMENT) == 0) { + // don't run netdata, this is the spawn server + spawn_server(); + exit(0); + } // parse options { @@ -883,6 +977,8 @@ int main(int argc, char **argv) { } else { debug(D_OPTIONS, "Configuration loaded from %s.", optarg); + post_conf_load(&user); + load_cloud_conf(1); config_loaded = 1; } break; @@ -921,6 +1017,7 @@ int main(int argc, char **argv) { { char* stacksize_string = "stacksize="; char* debug_flags_string = "debug_flags="; + char* claim_string = "claim"; #ifdef ENABLE_DBENGINE char* createdataset_string = "createdataset="; char* stresstest_string = "stresstest="; @@ -929,11 +1026,17 @@ int main(int argc, char **argv) { if(strcmp(optarg, "unittest") == 0) { if(unit_test_buffer()) return 1; if(unit_test_str2ld()) return 1; + // No call to load the config file on this code-path + post_conf_load(&user); get_netdata_configured_variables(); default_rrd_update_every = 1; default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; default_health_enabled = 0; - rrd_init("unittest", NULL); + registry_init(); + if(rrd_init("unittest", NULL)) { + fprintf(stderr, "rrd_init failed for unittest\n"); + return 1; + } default_rrdpush_enabled = 0; if(run_all_mockup_tests()) return 1; if(unit_test_storage()) return 1; @@ -953,7 +1056,7 @@ int main(int argc, char **argv) { else if(strncmp(optarg, stresstest_string, strlen(stresstest_string)) == 0) { char *endptr; unsigned test_duration_sec = 0, dset_charts = 0, query_threads = 0, ramp_up_seconds = 0, - page_cache_mb = 0; + page_cache_mb = 0, disk_space_mb = 0; optarg += strlen(stresstest_string); test_duration_sec = (unsigned)strtoul(optarg, &endptr, 0); @@ -965,8 +1068,11 @@ int main(int argc, char **argv) { ramp_up_seconds = (unsigned)strtoul(endptr + 1, &endptr, 0); if (',' == *endptr) page_cache_mb = (unsigned)strtoul(endptr + 1, &endptr, 0); + if (',' == *endptr) + disk_space_mb = (unsigned)strtoul(endptr + 1, &endptr, 0); + dbengine_stress_test(test_duration_sec, dset_charts, query_threads, ramp_up_seconds, - page_cache_mb); + page_cache_mb, disk_space_mb); return 0; } #endif @@ -1052,6 +1158,39 @@ int main(int argc, char **argv) { // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value); } + else if(strcmp(optarg, "set2") == 0) { + if(optind + 4 > argc) { + fprintf(stderr, "%s", "\nUSAGE: -W set 'conf_file' 'section' 'key' 'value'\n\n" + " Overwrites settings of netdata.conf or cloud.conf\n" + "\n" + " These options interact with: -c netdata.conf\n" + " If -c netdata.conf is given on the command line,\n" + " before -W set... the user may overwrite command\n" + " line parameters at netdata.conf\n" + " If -c netdata.conf is given after (or missing)\n" + " -W set... the user cannot overwrite the command line\n" + " parameters." + " conf_file can be \"cloud\" or \"netdata\".\n" + "\n" + ); + return 1; + } + const char *conf_file = argv[optind]; /* "cloud" is cloud.conf, otherwise netdata.conf */ + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config; + const char *section = argv[optind + 1]; + const char *key = argv[optind + 2]; + const char *value = argv[optind + 3]; + optind += 4; + + // set this one as the default + // only if it is not already set in the config file + // so the caller can use -c netdata.conf before or + // after this parameter to prevent or allow overwriting + // variables at netdata.conf + appconfig_set_default(tmp_config, section, key, value); + + // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value); + } else if(strcmp(optarg, "get") == 0) { if(optind + 3 > argc) { fprintf(stderr, "%s", "\nUSAGE: -W get 'section' 'key' 'value'\n\n" @@ -1067,6 +1206,7 @@ int main(int argc, char **argv) { if(!config_loaded) { fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n"); load_netdata_conf(NULL, 0); + post_conf_load(&user); } get_netdata_configured_variables(); @@ -1078,6 +1218,46 @@ int main(int argc, char **argv) { printf("%s\n", value); return 0; } + else if(strcmp(optarg, "get2") == 0) { + if(optind + 4 > argc) { + fprintf(stderr, "%s", "\nUSAGE: -W get2 'conf_file' 'section' 'key' 'value'\n\n" + " Prints settings of netdata.conf or cloud.conf\n" + "\n" + " These options interact with: -c netdata.conf\n" + " -c netdata.conf has to be given before -W get2.\n" + " conf_file can be \"cloud\" or \"netdata\".\n" + "\n" + ); + return 1; + } + + if(!config_loaded) { + fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n"); + load_netdata_conf(NULL, 0); + post_conf_load(&user); + load_cloud_conf(1); + } + + get_netdata_configured_variables(); + + const char *conf_file = argv[optind]; /* "cloud" is cloud.conf, otherwise netdata.conf */ + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config; + const char *section = argv[optind + 1]; + const char *key = argv[optind + 2]; + const char *def = argv[optind + 3]; + const char *value = appconfig_get(tmp_config, section, key, def); + printf("%s\n", value); + return 0; + } + else if(strncmp(optarg, claim_string, strlen(claim_string)) == 0) { + /* will trigger a claiming attempt when the agent is initialized */ + claiming_pending_arguments = optarg + strlen(claim_string); + } + else if(strcmp(optarg, "buildinfo") == 0) { + printf("Version: %s %s\n", program_name, program_version); + print_build_info(); + return 0; + } else { fprintf(stderr, "Unknown -W parameter '%s'\n", optarg); return help(1); @@ -1103,7 +1283,12 @@ int main(int argc, char **argv) { #endif if(!config_loaded) + { load_netdata_conf(NULL, 0); + post_conf_load(&user); + load_cloud_conf(0); + } + // ------------------------------------------------------------------------ // initialize netdata @@ -1118,6 +1303,7 @@ int main(int argc, char **argv) { mallopt(M_ARENA_MAX, 1); #endif test_clock_boottime(); + test_clock_monotonic_coarse(); // prepare configuration environment variables for the plugins @@ -1129,9 +1315,10 @@ int main(int argc, char **argv) { // files using relative filenames if(chdir(netdata_configured_user_config_dir) == -1) fatal("Cannot cd to '%s'", netdata_configured_user_config_dir); - } - char *user = NULL; + // Get execution path before switching user to avoid permission issues + get_netdata_execution_path(); + } { // -------------------------------------------------------------------- @@ -1199,19 +1386,6 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- - // get the user we should run - - // IMPORTANT: this is required before web_files_uid() - if(getuid() == 0) { - user = config_get(CONFIG_SECTION_GLOBAL, "run as user", NETDATA_USER); - } - else { - struct passwd *passwd = getpwuid(getuid()); - user = config_get(CONFIG_SECTION_GLOBAL, "run as user", (passwd && passwd->pw_name)?passwd->pw_name:""); - } - - - // -------------------------------------------------------------------- // create the listening sockets web_client_api_v1_init(); @@ -1254,6 +1428,19 @@ int main(int argc, char **argv) { netdata_threads_init_after_fork((size_t)config_get_number(CONFIG_SECTION_GLOBAL, "pthread stack size", (long)default_stacksize)); + // initialyze internal registry + registry_init(); + // fork the spawn server + spawn_init(); + /* + * Libuv uv_spawn() uses SIGCHLD internally: + * https://github.com/libuv/libuv/blob/cc51217a317e96510fbb284721d5e6bc2af31e33/src/unix/process.c#L485 + * and inadvertently replaces the netdata signal handler which was setup during initialization. + * Thusly, we must explicitly restore the signal handler for SIGCHLD. + * Warning: extreme care is needed when mixing and matching POSIX and libuv. + */ + signals_restore_SIGCHLD(); + // ------------------------------------------------------------------------ // initialize rrd, registry, health, rrdpush, etc. @@ -1261,12 +1448,24 @@ int main(int argc, char **argv) { struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info)); get_system_info(system_info); - rrd_init(netdata_configured_hostname, system_info); + if(rrd_init(netdata_configured_hostname, system_info)) + fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname); + + // ------------------------------------------------------------------------ + // Claim netdata agent to a cloud endpoint + + if (claiming_pending_arguments) + claim_agent(claiming_pending_arguments); + load_claiming_state(); + // ------------------------------------------------------------------------ // enable log flood protection error_log_limit_reset(); + // Load host labels + reload_host_labels(); + // ------------------------------------------------------------------------ // spawn the threads @@ -1285,12 +1484,39 @@ int main(int argc, char **argv) { else debug(D_SYSTEM, "Not starting thread %s.", st->name); } + // ------------------------------------------------------------------------ + // Initialize netdata agent command serving from cli and signals + + commands_init(); + info("netdata initialization completed. Enjoy real-time performance monitoring!"); netdata_ready = 1; send_statistics("START", "-", "-"); // ------------------------------------------------------------------------ + // Report ACLK build failure +#ifndef ENABLE_ACLK + error("This agent doesn't have ACLK."); + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/.aclk_report_sent", netdata_configured_varlib_dir); + if (netdata_anonymous_statistics_enabled > 0 && access(filename, F_OK)) { // -1 -> not initialized + send_statistics("ACLK_DISABLED", "-", "-"); +#ifdef ACLK_NO_LWS + send_statistics("BUILD_FAIL_LWS", "-", "-"); +#endif +#ifdef ACLK_NO_LIBMOSQ + send_statistics("BUILD_FAIL_MOSQ", "-", "-"); +#endif + int fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 444); + if (fd == -1) + error("Cannot create file '%s'. Please fix this.", filename); + else + close(fd); + } +#endif + + // ------------------------------------------------------------------------ // unblock signals signals_unblock(); diff --git a/daemon/signals.c b/daemon/signals.c index 5378b04e5..9e30bf19d 100644 --- a/daemon/signals.c +++ b/daemon/signals.c @@ -9,7 +9,7 @@ typedef enum signal_action { NETDATA_SIGNAL_IGNORE, NETDATA_SIGNAL_EXIT_CLEANLY, NETDATA_SIGNAL_SAVE_DATABASE, - NETDATA_SIGNAL_LOG_ROTATE, + NETDATA_SIGNAL_REOPEN_LOGS, NETDATA_SIGNAL_RELOAD_HEALTH, NETDATA_SIGNAL_FATAL, NETDATA_SIGNAL_CHILD, @@ -25,7 +25,7 @@ static struct { { SIGINT , "SIGINT", 0, NETDATA_SIGNAL_EXIT_CLEANLY }, { SIGQUIT, "SIGQUIT", 0, NETDATA_SIGNAL_EXIT_CLEANLY }, { SIGTERM, "SIGTERM", 0, NETDATA_SIGNAL_EXIT_CLEANLY }, - { SIGHUP, "SIGHUP", 0, NETDATA_SIGNAL_LOG_ROTATE }, + { SIGHUP, "SIGHUP", 0, NETDATA_SIGNAL_REOPEN_LOGS }, { SIGUSR1, "SIGUSR1", 0, NETDATA_SIGNAL_SAVE_DATABASE }, { SIGUSR2, "SIGUSR2", 0, NETDATA_SIGNAL_RELOAD_HEALTH }, { SIGBUS, "SIGBUS", 0, NETDATA_SIGNAL_FATAL }, @@ -111,6 +111,21 @@ void signals_init(void) { } } +void signals_restore_SIGCHLD(void) +{ + struct sigaction sa; + + if (reaper_enabled == 0) + return; + + sa.sa_flags = 0; + sigfillset(&sa.sa_mask); + sa.sa_handler = signal_handler; + + if(sigaction(SIGCHLD, &sa, NULL) == -1) + error("SIGNAL: Failed to change signal handler for: SIGCHLD"); +} + void signals_reset(void) { struct sigaction sa; sigemptyset(&sa.sa_mask); @@ -221,33 +236,35 @@ void signals_handle(void) { case NETDATA_SIGNAL_RELOAD_HEALTH: error_log_limit_unlimited(); info("SIGNAL: Received %s. Reloading HEALTH configuration...", name); - health_reload(); error_log_limit_reset(); + execute_command(CMD_RELOAD_HEALTH, NULL, NULL); break; case NETDATA_SIGNAL_SAVE_DATABASE: error_log_limit_unlimited(); info("SIGNAL: Received %s. Saving databases...", name); - rrdhost_save_all(); - info("Databases saved."); error_log_limit_reset(); + execute_command(CMD_SAVE_DATABASE, NULL, NULL); break; - case NETDATA_SIGNAL_LOG_ROTATE: + case NETDATA_SIGNAL_REOPEN_LOGS: error_log_limit_unlimited(); info("SIGNAL: Received %s. Reopening all log files...", name); - reopen_all_log_files(); error_log_limit_reset(); + execute_command(CMD_REOPEN_LOGS, NULL, NULL); break; case NETDATA_SIGNAL_EXIT_CLEANLY: error_log_limit_unlimited(); info("SIGNAL: Received %s. Cleaning up to exit...", name); + commands_exit(); netdata_cleanup_and_exit(0); exit(0); + break; case NETDATA_SIGNAL_FATAL: fatal("SIGNAL: Received %s. netdata now exits.", name); + break; case NETDATA_SIGNAL_CHILD: debug(D_CHILDS, "SIGNAL: Received %s. Reaping...", name); diff --git a/daemon/signals.h b/daemon/signals.h index e7e64365d..3fa2b0f43 100644 --- a/daemon/signals.h +++ b/daemon/signals.h @@ -6,6 +6,7 @@ extern void signals_init(void); extern void signals_block(void); extern void signals_unblock(void); +extern void signals_restore_SIGCHLD(void); extern void signals_reset(void); extern void signals_handle(void) NORETURN; diff --git a/daemon/system-info.sh b/daemon/system-info.sh index 6fab741fd..80eb82f86 100755 --- a/daemon/system-info.sh +++ b/daemon/system-info.sh @@ -8,111 +8,384 @@ KERNEL_VERSION="$(uname -r)" ARCHITECTURE="$(uname -m)" # ------------------------------------------------------------------------------------------------- +# detect the virtualization and possibly the container technology + +CONTAINER="unknown" +CONT_DETECTION="none" + +if [ -z "${VIRTUALIZATION}" ]; then + VIRTUALIZATION="unknown" + VIRT_DETECTION="none" + + if [ -n "$(command -v systemd-detect-virt 2> /dev/null)" ]; then + VIRTUALIZATION="$(systemd-detect-virt -v)" + VIRT_DETECTION="systemd-detect-virt" + CONTAINER="$(systemd-detect-virt -c)" + CONT_DETECTION="systemd-detect-virt" + else + if grep -q "^flags.*hypervisor" /proc/cpuinfo 2> /dev/null; then + VIRTUALIZATION="hypervisor" + VIRT_DETECTION="/proc/cpuinfo" + elif [ -n "$(command -v dmidecode)" ] && dmidecode -s system-product-name 2> /dev/null | grep -q "VMware\|Virtual\|KVM\|Bochs"; then + VIRTUALIZATION="$(dmidecode -s system-product-name)" + VIRT_DETECTION="dmidecode" + else + VIRTUALIZATION="none" + fi + fi + if [ -z "${VIRTUALIZATION}" ]; then + # Output from the command is outside of spec + VIRTUALIZATION="unknown" + fi +else + # Passed from outside - probably in docker run + VIRT_DETECTION="provided" +fi + +# ------------------------------------------------------------------------------------------------- +# detect containers with heuristics + +if [ "${CONTAINER}" = "unknown" ]; then + if [ -f /proc/1/sched ]; then + IFS='(, ' read -r process _ < /proc/1/sched + if [ "${process}" = "netdata" ]; then + CONTAINER="container" + CONT_DETECTION="process" + fi + fi + # ubuntu and debian supply /bin/running-in-container + # https://www.apt-browse.org/browse/ubuntu/trusty/main/i386/upstart/1.12.1-0ubuntu4/file/bin/running-in-container + if /bin/running-in-container > /dev/null 2>&1; then + CONTAINER="container" + CONT_DETECTION="/bin/running-in-container" + fi + + # lxc sets environment variable 'container' + #shellcheck disable=SC2154 + if [ -n "${container}" ]; then + CONTAINER="lxc" + CONT_DETECTION="containerenv" + fi + + # docker creates /.dockerenv + # http://stackoverflow.com/a/25518345 + if [ -f "/.dockerenv" ]; then + CONTAINER="docker" + CONT_DETECTION="dockerenv" + fi + +fi + +# ------------------------------------------------------------------------------------------------- # detect the operating system -OS_DETECTION="unknown" -NAME="unknown" -VERSION="unknown" -VERSION_ID="unknown" -ID="unknown" -ID_LIKE="unknown" +# Initially assume all OS detection values are for a container, these are moved later if we are bare-metal + +CONTAINER_OS_DETECTION="unknown" +CONTAINER_NAME="unknown" +CONTAINER_VERSION="unknown" +CONTAINER_VERSION_ID="unknown" +CONTAINER_ID="unknown" +CONTAINER_ID_LIKE="unknown" if [ "${KERNEL_NAME}" = "Darwin" ]; then - # Mac OS - OIFS="$IFS" - IFS=$'\n' - set $(sw_vers) > /dev/null - NAME=$(echo $1 | tr "\n\t" ' ' | sed -e 's/ProductName:[ ]*//' -e 's/[ ]*$//') - VERSION=$(echo $2 | tr "\n\t" ' ' | sed -e 's/ProductVersion:[ ]*//' -e 's/[ ]*$//') - ID="mac" - ID_LIKE="mac" - OS_DETECTION="sw_vers" - IFS="$OIFS" + CONTAINER_ID=$(sw_vers -productName) + CONTAINER_ID_LIKE="mac" + CONTAINER_NAME="mac" + CONTAINER_VERSION=$(sw_vers -productVersion) + CONTAINER_OS_DETECTION="sw_vers" +elif [ "${KERNEL_NAME}" = "FreeBSD" ]; then + CONTAINER_ID="FreeBSD" + CONTAINER_ID_LIKE="FreeBSD" + CONTAINER_NAME="FreeBSD" + CONTAINER_OS_DETECTION="uname" + CONTAINER_VERSION=$(uname -r) + KERNEL_VERSION=$(uname -K) +else + if [ -f "/etc/os-release" ]; then + eval "$(grep -E "^(NAME|ID|ID_LIKE|VERSION|VERSION_ID)=" < /etc/os-release | sed 's/^/CONTAINER_/')" + CONTAINER_OS_DETECTION="/etc/os-release" + fi + + # shellcheck disable=SC2153 + if [ "${NAME}" = "unknown" ] || [ "${VERSION}" = "unknown" ] || [ "${ID}" = "unknown" ]; then + if [ -f "/etc/lsb-release" ]; then + if [ "${OS_DETECTION}" = "unknown" ]; then + CONTAINER_OS_DETECTION="/etc/lsb-release" + else + CONTAINER_OS_DETECTION="Mixed" + fi + DISTRIB_ID="unknown" + DISTRIB_RELEASE="unknown" + DISTRIB_CODENAME="unknown" + eval "$(grep -E "^(DISTRIB_ID|DISTRIB_RELEASE|DISTRIB_CODENAME)=" < /etc/lsb-release)" + if [ "${NAME}" = "unknown" ]; then CONTAINER_NAME="${DISTRIB_ID}"; fi + if [ "${VERSION}" = "unknown" ]; then CONTAINER_VERSION="${DISTRIB_RELEASE}"; fi + if [ "${ID}" = "unknown" ]; then CONTAINER_ID="${DISTRIB_CODENAME}"; fi + fi + if [ -n "$(command -v lsb_release 2> /dev/null)" ]; then + if [ "${OS_DETECTION}" = "unknown" ]; then + CONTAINER_OS_DETECTION="lsb_release" + else + CONTAINER_OS_DETECTION="Mixed" + fi + if [ "${NAME}" = "unknown" ]; then CONTAINER_NAME="$(lsb_release -is 2> /dev/null)"; fi + if [ "${VERSION}" = "unknown" ]; then CONTAINER_VERSION="$(lsb_release -rs 2> /dev/null)"; fi + if [ "${ID}" = "unknown" ]; then CONTAINER_ID="$(lsb_release -cs 2> /dev/null)"; fi + fi + fi +fi + +# If Netdata is not running in a container then use the local detection as the host +HOST_OS_DETECTION="unknown" +HOST_NAME="unknown" +HOST_VERSION="unknown" +HOST_VERSION_ID="unknown" +HOST_ID="unknown" +HOST_ID_LIKE="unknown" +if [ "${CONTAINER}" = "unknown" ]; then + for v in NAME ID ID_LIKE VERSION VERSION_ID OS_DETECTION; do + eval "HOST_$v=\$CONTAINER_$v; CONTAINER_$v=none" + done else - if [ -f "/etc/os-release" ]; then - OS_DETECTION="/etc/os-release" - eval "$(grep -E "^(NAME|ID|ID_LIKE|VERSION|VERSION_ID)=" </etc/os-release)" - fi - - if [ "${NAME}" = "unknown" ] || [ "${VERSION}" = "unknown" ] || [ "${ID}" = "unknown" ]; then - if [ -f "/etc/lsb-release" ]; then - if [ "${OS_DETECTION}" = "unknown" ]; then OS_DETECTION="/etc/lsb-release"; else OS_DETECTION="Mixed"; fi - DISTRIB_ID="unknown" - DISTRIB_RELEASE="unknown" - DISTRIB_CODENAME="unknown" - eval "$(grep -E "^(DISTRIB_ID|DISTRIB_RELEASE|DISTRIB_CODENAME)=" </etc/lsb-release)" - if [ "${NAME}" = "unknown" ]; then NAME="${DISTRIB_ID}"; fi - if [ "${VERSION}" = "unknown" ]; then VERSION="${DISTRIB_RELEASE}"; fi - if [ "${ID}" = "unknown" ]; then ID="${DISTRIB_CODENAME}"; fi - fi - if [ -n "$(command -v lsb_release 2>/dev/null)" ]; then - if [ "${OS_DETECTION}" = "unknown" ]; then OS_DETECTION="lsb_release"; else OS_DETECTION="Mixed"; fi - if [ "${NAME}" = "unknown" ]; then NAME="$(lsb_release -is 2>/dev/null)"; fi - if [ "${VERSION}" = "unknown" ]; then VERSION="$(lsb_release -rs 2>/dev/null)"; fi - if [ "${ID}" = "unknown" ]; then ID="$(lsb_release -cs 2>/dev/null)"; fi - fi - fi + # Otherwise try and use a user-supplied bind-mount into the container to resolve the host details + if [ -e "/host/etc/os-release" ]; then + OS_DETECTION="/etc/os-release" + eval "$(grep -E "^(NAME|ID|ID_LIKE|VERSION|VERSION_ID)=" < /host/etc/os-release | sed 's/^/HOST_/')" + HOST_OS_DETECTION="/host/etc/os-release" + fi + if [ "${HOST_NAME}" = "unknown" ] || [ "${HOST_VERSION}" = "unknown" ] || [ "${HOST_ID}" = "unknown" ]; then + if [ -f "/host/etc/lsb-release" ]; then + if [ "${HOST_OS_DETECTION}" = "unknown" ]; then + HOST_OS_DETECTION="/etc/lsb-release" + else + HOST_OS_DETECTION="Mixed" + fi + DISTRIB_ID="unknown" + DISTRIB_RELEASE="unknown" + DISTRIB_CODENAME="unknown" + eval "$(grep -E "^(DISTRIB_ID|DISTRIB_RELEASE|DISTRIB_CODENAME)=" < /etc/lsb-release)" + if [ "${HOST_NAME}" = "unknown" ]; then HOST_NAME="${DISTRIB_ID}"; fi + if [ "${HOST_VERSION}" = "unknown" ]; then HOST_VERSION="${DISTRIB_RELEASE}"; fi + if [ "${HOST_ID}" = "unknown" ]; then HOST_ID="${DISTRIB_CODENAME}"; fi + fi + fi fi # ------------------------------------------------------------------------------------------------- -# detect the virtualization +# Detect information about the CPU -VIRTUALIZATION="unknown" -VIRT_DETECTION="none" -CONTAINER="unknown" -CONT_DETECTION="none" +LCPU_COUNT="unknown" +CPU_MODEL="unknown" +CPU_VENDOR="unknown" +CPU_FREQ="unknown" +CPU_INFO_SOURCE="none" -if [ -n "$(command -v systemd-detect-virt 2>/dev/null)" ]; then - VIRTUALIZATION="$(systemd-detect-virt -v)" - VIRT_DETECTION="systemd-detect-virt" - CONTAINER="$(systemd-detect-virt -c)" - CONT_DETECTION="systemd-detect-virt" +possible_cpu_freq="" +nproc="$(command -v nproc)" +lscpu="$(command -v lscpu)" +lscpu_output="" +dmidecode="$(command -v dmidecode)" +dmidecode_output="" + +if [ -n "${lscpu}" ] && lscpu > /dev/null 2>&1; then + lscpu_output="$(LC_NUMERIC=C ${lscpu} 2> /dev/null)" + CPU_INFO_SOURCE="lscpu" + LCPU_COUNT="$(echo "${lscpu_output}" | grep "^CPU(s):" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + CPU_VENDOR="$(echo "${lscpu_output}" | grep "^Vendor ID:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + CPU_MODEL="$(echo "${lscpu_output}" | grep "^Model name:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + possible_cpu_freq="$(echo "${lscpu_output}" | grep -F "CPU max MHz:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | grep -o '^[0-9]*') MHz" + if [ "${possible_cpu_freq}" = " MHz" ]; then + possible_cpu_freq="$(echo "${lscpu_output}" | grep -F "CPU MHz:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | grep -o '^[0-9]*') MHz" + fi +elif [ -n "${dmidecode}" ] && dmidecode -t processor > /dev/null 2>&1; then + dmidecode_output="$(${dmidecode} -t processor 2> /dev/null)" + CPU_INFO_SOURCE="dmidecode" + LCPU_COUNT="$(echo "${dmidecode_output}" | grep -F "Thread Count:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + CPU_VENDOR="$(echo "${dmidecode_output}" | grep -F "Manufacturer:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + CPU_MODEL="$(echo "${dmidecode_output}" | grep -F "Version:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + possible_cpu_freq="$(echo "${dmidecode_output}" | grep -F "Current Speed:" | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" else - if grep -q "^flags.*hypervisor" /proc/cpuinfo 2>/dev/null; then - VIRTUALIZATION="hypervisor" - VIRT_DETECTION="/proc/cpuinfo" - fi + if [ -n "${nproc}" ]; then + CPU_INFO_SOURCE="nproc" + LCPU_COUNT="$(${nproc})" + elif [ "${KERNEL_NAME}" = FreeBSD ]; then + CPU_INFO_SOURCE="sysctl" + LCPU_COUNT="$(sysctl -n kern.smp.cpus)" + elif [ -d /sys/devices/system/cpu ]; then + CPU_INFO_SOURCE="sysfs" + # This is potentially more accurate than checking `/proc/cpuinfo`. + LCPU_COUNT="$(find /sys/devices/system/cpu -mindepth 1 -maxdepth 1 -type d -name 'cpu*' | grep -cEv 'idle|freq')" + elif [ -r /proc/cpuinfo ]; then + CPU_INFO_SOURCE="procfs" + LCPU_COUNT="$(grep -c ^processor /proc/cpuinfo)" + fi + + # If we have GNU uname, we can use that to get CPU info (probably). + if uname --version 2> /dev/null | grep -qF 'GNU coreutils'; then + CPU_INFO_SOURCE="${CPU_INFO_SOURCE} uname" + CPU_MODEL="$(uname -p)" + CPU_VENDOR="$(uname -i)" + elif [ "${KERNEL_NAME}" = FreeBSD ]; then + if (echo "${CPU_INFO_SOURCE}" | grep -qv sysctl); then + CPU_INFO_SOURCE="${CPU_INFO_SOURCE} sysctl" + fi + + CPU_MODEL="$(sysctl -n hw.model)" + elif [ -r /proc/cpuinfo ]; then + if (echo "${CPU_INFO_SOURCE}" | grep -qv procfs); then + CPU_INFO_SOURCE="${CPU_INFO_SOURCE} procfs" + fi + + CPU_MODEL="$(grep -F "model name" /proc/cpuinfo | head -n 1 | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + CPU_VENDOR="$(grep -F "vendor_id" /proc/cpuinfo | head -n 1 | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + fi fi +if [ -r /sys/devices/system/cpu/cpu0/cpufreq/base_frequency ]; then + if (echo "${CPU_INFO_SOURCE}" | grep -qv sysfs); then + CPU_INFO_SOURCE="${CPU_INFO_SOURCE} sysfs" + fi + + CPU_FREQ="$(cat /sys/devices/system/cpu/cpu0/cpufreq/base_frequency)" +elif [ -n "${possible_cpu_freq}" ]; then + CPU_FREQ="${possible_cpu_freq}" +elif [ -r /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq ]; then + if (echo "${CPU_INFO_SOURCE}" | grep -qv sysfs); then + CPU_INFO_SOURCE="${CPU_INFO_SOURCE} sysfs" + fi + + CPU_FREQ="$(cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq)" +fi + +freq_units="$(echo "${CPU_FREQ}" | cut -f 2 -d ' ')" + +case "${freq_units}" in + GHz) + value="$(echo "${CPU_FREQ}" | cut -f 1 -d ' ')" + CPU_FREQ="$((value * 1000 * 1000 * 1000))" + ;; + MHz) + value="$(echo "${CPU_FREQ}" | cut -f 1 -d ' ')" + CPU_FREQ="$((value * 1000 * 1000))" + ;; + KHz) + value="$(echo "${CPU_FREQ}" | cut -f 1 -d ' ')" + CPU_FREQ="$((value * 1000))" + ;; + *) ;; + +esac + # ------------------------------------------------------------------------------------------------- -# detect containers with heuristics +# Detect the total system RAM -if [ "${CONTAINER}" = "unknown" ]; then - if [ -f /proc/1/sched ] ; then - IFS='(, ' read -r process _ </proc/1/sched - if [ "${process}" = "netdata" ]; then - CONTAINER="container" - CONT_DETECTION="process" - fi - fi - # ubuntu and debian supply /bin/running-in-container - # https://www.apt-browse.org/browse/ubuntu/trusty/main/i386/upstart/1.12.1-0ubuntu4/file/bin/running-in-container - if /bin/running-in-container >/dev/null 2>&1; then - CONTAINER="container" - CONT_DETECTION="/bin/running-in-container" - fi - - # lxc sets environment variable 'container' - #shellcheck disable=SC2154 - if [ -n "${container}" ]; then - CONTAINER="lxc" - CONT_DETECTION="containerenv" - fi - - # docker creates /.dockerenv - # http://stackoverflow.com/a/25518345 - if [ -f "/.dockerenv" ]; then - CONTAINER="docker" - CONT_DETECTION="dockerenv" - fi +TOTAL_RAM="unknown" +RAM_DETECTION="none" + +if [ "${KERNEL_NAME}" = FreeBSD ]; then + RAM_DETECTION="sysctl" + TOTAL_RAM="$(sysctl -n hw.physmem)" +elif [ "${KERNEL_NAME}" = Darwin ]; then + RAM_DETECTION="sysctl" + TOTAL_RAM="$(sysctl -n hw.physmem)" +elif [ -r /proc/meminfo ]; then + RAM_DETECTION="procfs" + TOTAL_RAM="$(grep -F MemTotal /proc/meminfo | cut -f 2 -d ':' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | cut -f 1 -d ' ')" + TOTAL_RAM="$((TOTAL_RAM * 1024))" +fi + +# ------------------------------------------------------------------------------------------------- +# Detect the total system disk space + +DISK_SIZE="unknown" +DISK_DETECTION="none" + +if [ "${KERNEL_NAME}" = "Darwin" ]; then + types='hfs' + + if (lsvfs | grep -q apfs); then + types="${types},apfs" + fi + + if (lsvfs | grep -q ufs); then + types="${types},ufs" + fi + + DISK_DETECTION="df" + DISK_SIZE=$(($(/bin/df -k -t ${types} | tail -n +2 | sed -E 's/\/dev\/disk([[:digit:]]*)s[[:digit:]]*/\/dev\/disk\1/g' | sort -k 1 | awk -F ' ' '{s=$NF;for(i=NF-1;i>=1;i--)s=s FS $i;print s}' | uniq -f 9 | awk '{print $8}' | tr '\n' '+' | rev | cut -f 2- -d '+' | rev) * 1024)) +elif [ "${KERNEL_NAME}" = FreeBSD ]; then + types='ufs' + + if (lsvfs | grep -q zfs); then + types="${types},zfs" + fi + + DISK_DETECTION="df" + total="$(df -t ${types} -c -k | tail -n 1 | awk '{print $2}')" + DISK_SIZE="$((total * 1024))" +else + if [ -d /sys/block ] && [ -r /proc/devices ]; then + dev_major_whitelist='' + + # This is a list of device names used for block storage devices. + # These translate to the prefixs of files in `/dev` indicating the device type. + # They are sorted by lowest used device major number, with dynamically assigned ones at the end. + # We use this to look up device major numbers in `/proc/devices` + device_names='hd sd mfm ad ftl pd nftl dasd intfl mmcblk ub xvd rfd vbd nvme' + + for name in ${device_names}; do + if grep -qE " ${name}\$" /proc/devices; then + dev_major_whitelist="${dev_major_whitelist}:$(grep -E "${name}\$" /proc/devices | sed -e 's/^[[:space:]]*//' | cut -f 1 -d ' ' | tr '\n' ':'):" + fi + done + + DISK_DETECTION="sysfs" + DISK_SIZE="0" + for disk in /sys/block/*; do + if [ -r "${disk}/size" ] \ + && (echo "${dev_major_whitelist}" | grep -q ":$(cut -f 1 -d ':' "${disk}/dev"):") \ + && grep -qv 1 "${disk}/removable"; then + size="$(($(cat "${disk}/size") * 512))" + DISK_SIZE="$((DISK_SIZE + size))" + fi + done + elif df --version 2> /dev/null | grep -qF "GNU coreutils"; then + DISK_DETECTION="df" + DISK_SIZE=$(($(df -x tmpfs -x devtmpfs -x squashfs -l -B1 --output=source,size | tail -n +2 | sort -u -k 1 | awk '{print $2}' | tr '\n' '+' | head -c -1))) + else + DISK_DETECTION="df" + include_fs_types="ext*|btrfs|xfs|jfs|reiser*|zfs" + DISK_SIZE=$(($(df -T -P | tail -n +2 | sort -u -k 1 | grep "${include_fs_types}" | awk '{print $3}' | tr '\n' '+' | head -c -1) * 1024)) + fi fi -echo "NETDATA_SYSTEM_OS_NAME=\"${NAME}\"" -echo "NETDATA_SYSTEM_OS_ID=${ID}" -echo "NETDATA_SYSTEM_OS_ID_LIKE=${ID_LIKE}" -echo "NETDATA_SYSTEM_OS_VERSION=${VERSION}" -echo "NETDATA_SYSTEM_OS_VERSION_ID=${VERSION_ID}" -echo "NETDATA_SYSTEM_OS_DETECTION=${OS_DETECTION}" +# ------------------------------------------------------------------------------------------------- +# Detect whether the node is kubernetes node + +HOST_IS_K8S_NODE="false" + +if [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_SERVICE_PORT}" ]; then + # These env vars are set for every container managed by k8s. + HOST_IS_K8S_NODE="true" +elif pgrep "kubelet"; then + # The kubelet is the primary "node agent" that runs on each node. + HOST_IS_K8S_NODE="true" +fi + +echo "NETDATA_CONTAINER_OS_NAME=${CONTAINER_NAME}" +echo "NETDATA_CONTAINER_OS_ID=${CONTAINER_ID}" +echo "NETDATA_CONTAINER_OS_ID_LIKE=${CONTAINER_ID_LIKE}" +echo "NETDATA_CONTAINER_OS_VERSION=${CONTAINER_VERSION}" +echo "NETDATA_CONTAINER_OS_VERSION_ID=${CONTAINER_VERSION_ID}" +echo "NETDATA_CONTAINER_OS_DETECTION=${CONTAINER_OS_DETECTION}" +echo "NETDATA_HOST_OS_NAME=${HOST_NAME}" +echo "NETDATA_HOST_OS_ID=${HOST_ID}" +echo "NETDATA_HOST_OS_ID_LIKE=${HOST_ID_LIKE}" +echo "NETDATA_HOST_OS_VERSION=${HOST_VERSION}" +echo "NETDATA_HOST_OS_VERSION_ID=${HOST_VERSION_ID}" +echo "NETDATA_HOST_OS_DETECTION=${HOST_OS_DETECTION}" +echo "NETDATA_HOST_IS_K8S_NODE=${HOST_IS_K8S_NODE}" echo "NETDATA_SYSTEM_KERNEL_NAME=${KERNEL_NAME}" echo "NETDATA_SYSTEM_KERNEL_VERSION=${KERNEL_VERSION}" echo "NETDATA_SYSTEM_ARCHITECTURE=${ARCHITECTURE}" @@ -120,4 +393,12 @@ echo "NETDATA_SYSTEM_VIRTUALIZATION=${VIRTUALIZATION}" echo "NETDATA_SYSTEM_VIRT_DETECTION=${VIRT_DETECTION}" echo "NETDATA_SYSTEM_CONTAINER=${CONTAINER}" echo "NETDATA_SYSTEM_CONTAINER_DETECTION=${CONT_DETECTION}" - +echo "NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT=${LCPU_COUNT}" +echo "NETDATA_SYSTEM_CPU_VENDOR=${CPU_VENDOR}" +echo "NETDATA_SYSTEM_CPU_MODEL=${CPU_MODEL}" +echo "NETDATA_SYSTEM_CPU_FREQ=${CPU_FREQ}" +echo "NETDATA_SYSTEM_CPU_DETECTION=${CPU_INFO_SOURCE}" +echo "NETDATA_SYSTEM_TOTAL_RAM=${TOTAL_RAM}" +echo "NETDATA_SYSTEM_RAM_DETECTION=${RAM_DETECTION}" +echo "NETDATA_SYSTEM_TOTAL_DISK_SIZE=${DISK_SIZE}" +echo "NETDATA_SYSTEM_DISK_DETECTION=${DISK_DETECTION}" diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 2e5927321..e6a69e354 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -1491,6 +1491,9 @@ static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number static RRDHOST *dbengine_rrdhost_find_or_create(char *name) { + /* We don't want to drop metrics when generating load, we prefer to block data generation itself */ + rrdeng_drop_metrics_under_page_cache_pressure = 0; + return rrdhost_find_or_create( name , name @@ -1670,7 +1673,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] update_every = REGION_UPDATE_EVERY[current_region]; long points = (time_end - time_start) / update_every - 1; for (i = 0 ; i < CHARTS ; ++i) { - RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL); + RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL); if (!r) { fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name); return ++errors; @@ -1723,7 +1726,7 @@ int test_dbengine(void) default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; - debug(D_RRDHOST, "Initializing localhost with hostname 'unittest-dbengine'"); + fprintf(stderr, "Initializing localhost with hostname 'unittest-dbengine'"); host = dbengine_rrdhost_find_or_create("unittest-dbengine"); if (NULL == host) return 1; @@ -1789,7 +1792,7 @@ int test_dbengine(void) long points = (time_end[REGIONS - 1] - time_start[0]) / update_every - 1; // cover all time regions with RRDR long point_offset = (time_start[current_region] - time_start[0]) / update_every; for (i = 0 ; i < CHARTS ; ++i) { - RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL); + RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL); if (!r) { fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name); ++errors; @@ -1830,9 +1833,10 @@ int test_dbengine(void) } } error_out: - rrdeng_exit(host->rrdeng_ctx); rrd_wrlock(); + rrdeng_prepare_exit(host->rrdeng_ctx); rrdhost_delete_charts(host); + rrdeng_exit(host->rrdeng_ctx); rrd_unlock(); return errors; @@ -1915,6 +1919,9 @@ static void generate_dbengine_chart(void *arg) rrdset_done(st); thread_info->time_max = time_current; } + for (j = 0; j < DSET_DIMS; ++j) { + rrdeng_store_metric_finalize(rd[j]); + } } void generate_dbengine_dataset(unsigned history_seconds) @@ -1935,7 +1942,7 @@ void generate_dbengine_dataset(unsigned history_seconds) default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100; error_log_limit_unlimited(); - debug(D_RRDHOST, "Initializing localhost with hostname 'dbengine-dataset'"); + fprintf(stderr, "Initializing localhost with hostname 'dbengine-dataset'"); host = dbengine_rrdhost_find_or_create("dbengine-dataset"); if (NULL == host) @@ -1986,6 +1993,7 @@ struct dbengine_query_thread { unsigned history_seconds; /* how far back in the past to go */ volatile long done; /* initialize to 0, set to 1 to stop thread */ unsigned long errors, queries_nr, queried_metrics_nr; /* statistics */ + uint8_t delete_old_data; /* if non zero then data are deleted when disk space is exhausted */ struct dbengine_chart_thread *chart_threads[]; /* dset_charts elements */ }; @@ -1995,7 +2003,7 @@ static void query_dbengine_chart(void *arg) struct dbengine_query_thread *thread_info = (struct dbengine_query_thread *)arg; const int DSET_CHARTS = thread_info->dset_charts; const int DSET_DIMS = thread_info->dset_dims; - time_t time_after, time_before, time_min, time_max, duration; + time_t time_after, time_before, time_min, time_approx_min, time_max, duration; int i, j, update_every = 1; RRDSET *st; RRDDIM *rd; @@ -2015,6 +2023,13 @@ static void query_dbengine_chart(void *arg) time_min = thread_info->time_present - thread_info->history_seconds + 1; time_max = thread_info->chart_threads[i]->time_max; + + if (thread_info->delete_old_data) { + /* A time window of twice the disk space is sufficient for compression space savings of up to 50% */ + time_approx_min = time_max - (default_rrdeng_disk_quota_mb * 2 * 1024 * 1024) / + (((uint64_t) DSET_DIMS * DSET_CHARTS) * sizeof(storage_number)); + time_min = MAX(time_min, time_approx_min); + } if (!time_max) { time_before = time_after = time_min; } else { @@ -2030,18 +2045,22 @@ static void query_dbengine_chart(void *arg) expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_EXISTS)); if (unlikely(rd->state->query_ops.is_finished(&handle))) { - fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " - CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, expected); - ++thread_info->errors; + if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " + CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, expected); + ++thread_info->errors; + } break; } n = rd->state->query_ops.next_metric(&handle, &time_retrieved); if (SN_EMPTY_SLOT == n) { - fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " - CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, expected); - ++thread_info->errors; + if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " + CALCULATED_NUMBER_FORMAT ", found data gap, ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, expected); + ++thread_info->errors; + } break; } ++thread_info->queried_metrics_nr; @@ -2049,15 +2068,21 @@ static void query_dbengine_chart(void *arg) same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if (!same) { - fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " - CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, expected, value); - ++thread_info->errors; + if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ + fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, expecting value " + CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT + ", ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, expected, value); + ++thread_info->errors; + } } if (time_retrieved != time_now) { - fprintf(stderr, " DB-engine stresstest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n", - st->name, rd->name, (unsigned long) time_now, (unsigned long) time_retrieved); - ++thread_info->errors; + if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ + fprintf(stderr, + " DB-engine stresstest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n", + st->name, rd->name, (unsigned long) time_now, (unsigned long) time_retrieved); + ++thread_info->errors; + } } } rd->state->query_ops.finalize(&handle); @@ -2065,16 +2090,18 @@ static void query_dbengine_chart(void *arg) } void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsigned QUERY_THREADS, - unsigned RAMP_UP_SECONDS, unsigned PAGE_CACHE_MB) + unsigned RAMP_UP_SECONDS, unsigned PAGE_CACHE_MB, unsigned DISK_SPACE_MB) { const unsigned DSET_DIMS = 128; const uint64_t EXPECTED_COMPRESSION_RATIO = 20; - const unsigned HISTORY_SECONDS = 3600 * 24 * 365; /* 1 year of history */ + const unsigned HISTORY_SECONDS = 3600 * 24 * 365 * 50; /* 50 year of history */ RRDHOST *host = NULL; struct dbengine_chart_thread **chart_threads; struct dbengine_query_thread **query_threads; unsigned i, j; - time_t time_start, time_end; + time_t time_start, test_duration; + + error_log_limit_unlimited(); if (!TEST_DURATION_SEC) TEST_DURATION_SEC = 10; @@ -2087,13 +2114,18 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; default_rrdeng_page_cache_mb = PAGE_CACHE_MB; - // Worst case for uncompressible data - default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * HISTORY_SECONDS) / - (1024 * 1024); - default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100; + if (DISK_SPACE_MB) { + fprintf(stderr, "By setting disk space limit data are allowed to be deleted. " + "Data validation is turned off for this run.\n"); + default_rrdeng_disk_quota_mb = DISK_SPACE_MB; + } else { + // Worst case for uncompressible data + default_rrdeng_disk_quota_mb = + (((uint64_t) DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * HISTORY_SECONDS) / (1024 * 1024); + default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100; + } - error_log_limit_unlimited(); - debug(D_RRDHOST, "Initializing localhost with hostname 'dbengine-stress-test'"); + fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n"); host = dbengine_rrdhost_find_or_create("dbengine-stress-test"); if (NULL == host) @@ -2112,7 +2144,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi "%u MiB of page cache.\n", RAMP_UP_SECONDS, TEST_DURATION_SEC, DSET_CHARTS, QUERY_THREADS, PAGE_CACHE_MB); - time_start = now_realtime_sec(); + time_start = now_realtime_sec() + HISTORY_SECONDS; /* move history to the future */ for (i = 0 ; i < DSET_CHARTS ; ++i) { chart_threads[i]->host = host; chart_threads[i]->chartname = "random"; @@ -2146,6 +2178,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi for (j = 0 ; j < DSET_CHARTS ; ++j) { query_threads[i]->chart_threads[j] = chart_threads[j]; } + query_threads[i]->delete_old_data = DISK_SPACE_MB ? 1 : 0; assert(0 == uv_thread_create(&query_threads[i]->thread, query_dbengine_chart, query_threads[i])); } sleep(TEST_DURATION_SEC); @@ -2162,8 +2195,10 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi for (i = 0 ; i < QUERY_THREADS ; ++i) { assert(0 == uv_thread_join(&query_threads[i]->thread)); } - time_end = now_realtime_sec(); - fprintf(stderr, "\nDB-engine stress test finished in %ld seconds.\n", time_end - time_start); + test_duration = now_realtime_sec() - (time_start - HISTORY_SECONDS); + if (!test_duration) + test_duration = 1; + fprintf(stderr, "\nDB-engine stress test finished in %ld seconds.\n", test_duration); unsigned long stored_metrics_nr = 0; for (i = 0 ; i < DSET_CHARTS ; ++i) { stored_metrics_nr += chart_threads[i]->stored_metrics_nr; @@ -2180,7 +2215,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi fprintf(stderr, "Query starting time is randomly chosen from the beginning of the time-series up to the time of\n" "the latest data point, and ending time from 1 second up to 1 hour after the starting time.\n"); fprintf(stderr, "Performance is %lu written data points/sec and %lu read data points/sec.\n", - stored_metrics_nr / (time_end - time_start), queried_metrics_nr / (time_end - time_start)); + stored_metrics_nr / test_duration, queried_metrics_nr / test_duration); for (i = 0 ; i < DSET_CHARTS ; ++i) { freez(chart_threads[i]); @@ -2190,9 +2225,10 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi freez(query_threads[i]); } freez(query_threads); - rrdeng_exit(host->rrdeng_ctx); rrd_wrlock(); + rrdeng_prepare_exit(host->rrdeng_ctx); rrdhost_delete_charts(host); + rrdeng_exit(host->rrdeng_ctx); rrd_unlock(); } diff --git a/daemon/unit_test.h b/daemon/unit_test.h index 230a70085..79d415be0 100644 --- a/daemon/unit_test.h +++ b/daemon/unit_test.h @@ -12,7 +12,7 @@ extern int unit_test_buffer(void); extern int test_dbengine(void); extern void generate_dbengine_dataset(unsigned history_seconds); extern void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsigned QUERY_THREADS, - unsigned RAMP_UP_SECONDS, unsigned PAGE_CACHE_MB); + unsigned RAMP_UP_SECONDS, unsigned PAGE_CACHE_MB, unsigned DISK_SPACE_MB); #endif |