summaryrefslogtreecommitdiffstats
path: root/database
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--database/Makefile.am8
-rw-r--r--database/Makefile.in464
-rw-r--r--database/README.md206
-rw-r--r--database/rrd.c (renamed from src/rrd.c)4
-rw-r--r--database/rrd.h (renamed from src/rrd.h)227
-rw-r--r--database/rrdcalc.c (renamed from src/rrdcalc.c)17
-rw-r--r--database/rrdcalc.h136
-rw-r--r--database/rrdcalctemplate.c (renamed from src/rrdcalctemplate.c)4
-rw-r--r--database/rrdcalctemplate.h68
-rw-r--r--database/rrddim.c (renamed from src/rrddim.c)20
-rw-r--r--database/rrddimvar.c (renamed from src/rrddimvar.c)27
-rw-r--r--database/rrddimvar.h56
-rw-r--r--database/rrdfamily.c (renamed from src/rrdfamily.c)6
-rw-r--r--database/rrdhost.c (renamed from src/rrdhost.c)26
-rw-r--r--database/rrdset.c (renamed from src/rrdset.c)111
-rw-r--r--database/rrdsetvar.c (renamed from src/rrdsetvar.c)30
-rw-r--r--database/rrdsetvar.h44
-rw-r--r--database/rrdvar.c (renamed from src/rrdvar.c)28
-rw-r--r--database/rrdvar.h66
19 files changed, 1431 insertions, 117 deletions
diff --git a/database/Makefile.am b/database/Makefile.am
new file mode 100644
index 000000000..19554bed8
--- /dev/null
+++ b/database/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/database/Makefile.in b/database/Makefile.in
new file mode 100644
index 000000000..4f5b710c5
--- /dev/null
+++ b/database/Makefile.in
@@ -0,0 +1,464 @@
+# Makefile.in generated by automake 1.14.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2013 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+VPATH = @srcdir@
+am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)'
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = database
+DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
+ $(dist_noinst_DATA)
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/build/m4/ax_c___atomic.m4 \
+ $(top_srcdir)/build/m4/ax_c__generic.m4 \
+ $(top_srcdir)/build/m4/ax_c_lto.m4 \
+ $(top_srcdir)/build/m4/ax_c_mallinfo.m4 \
+ $(top_srcdir)/build/m4/ax_c_mallopt.m4 \
+ $(top_srcdir)/build/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/build/m4/ax_gcc_func_attribute.m4 \
+ $(top_srcdir)/build/m4/ax_pthread.m4 \
+ $(top_srcdir)/build/m4/jemalloc.m4 \
+ $(top_srcdir)/build/m4/tcmalloc.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
+SOURCES =
+DIST_SOURCES =
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
+DATA = $(dist_noinst_DATA)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+IPMIMONITORING_CFLAGS = @IPMIMONITORING_CFLAGS@
+IPMIMONITORING_LIBS = @IPMIMONITORING_LIBS@
+LDFLAGS = @LDFLAGS@
+LIBCAP_CFLAGS = @LIBCAP_CFLAGS@
+LIBCAP_LIBS = @LIBCAP_LIBS@
+LIBMNL_CFLAGS = @LIBMNL_CFLAGS@
+LIBMNL_LIBS = @LIBMNL_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LTLIBOBJS = @LTLIBOBJS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MATH_CFLAGS = @MATH_CFLAGS@
+MATH_LIBS = @MATH_LIBS@
+MKDIR_P = @MKDIR_P@
+NFACCT_CFLAGS = @NFACCT_CFLAGS@
+NFACCT_LIBS = @NFACCT_LIBS@
+OBJEXT = @OBJEXT@
+OPTIONAL_IPMIMONITORING_CFLAGS = @OPTIONAL_IPMIMONITORING_CFLAGS@
+OPTIONAL_IPMIMONITORING_LIBS = @OPTIONAL_IPMIMONITORING_LIBS@
+OPTIONAL_LIBCAP_CFLAGS = @OPTIONAL_LIBCAP_CFLAGS@
+OPTIONAL_LIBCAP_LIBS = @OPTIONAL_LIBCAP_LIBS@
+OPTIONAL_MATH_CLFAGS = @OPTIONAL_MATH_CLFAGS@
+OPTIONAL_MATH_LIBS = @OPTIONAL_MATH_LIBS@
+OPTIONAL_NFACCT_CLFAGS = @OPTIONAL_NFACCT_CLFAGS@
+OPTIONAL_NFACCT_LIBS = @OPTIONAL_NFACCT_LIBS@
+OPTIONAL_UUID_CLFAGS = @OPTIONAL_UUID_CLFAGS@
+OPTIONAL_UUID_LIBS = @OPTIONAL_UUID_LIBS@
+OPTIONAL_ZLIB_CLFAGS = @OPTIONAL_ZLIB_CLFAGS@
+OPTIONAL_ZLIB_LIBS = @OPTIONAL_ZLIB_LIBS@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_RPM_RELEASE = @PACKAGE_RPM_RELEASE@
+PACKAGE_RPM_VERSION = @PACKAGE_RPM_VERSION@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SSE_CANDIDATE = @SSE_CANDIDATE@
+STRIP = @STRIP@
+UUID_CFLAGS = @UUID_CFLAGS@
+UUID_LIBS = @UUID_LIBS@
+VERSION = @VERSION@
+ZLIB_CFLAGS = @ZLIB_CFLAGS@
+ZLIB_LIBS = @ZLIB_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_CC = @ac_ct_CC@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_target = @build_target@
+build_vendor = @build_vendor@
+builddir = @builddir@
+cachedir = @cachedir@
+chartsdir = @chartsdir@
+configdir = @configdir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+has_jemalloc = @has_jemalloc@
+has_tcmalloc = @has_tcmalloc@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libconfigdir = @libconfigdir@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+logdir = @logdir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+nodedir = @nodedir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+pluginsdir = @pluginsdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+pythondir = @pythondir@
+registrydir = @registrydir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+varlibdir = @varlibdir@
+webdir = @webdir@
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
+
+all: all-am
+
+.SUFFIXES:
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu database/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --gnu database/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+tags TAGS:
+
+ctags CTAGS:
+
+cscope cscopelist:
+
+
+distdir: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+check: check-am
+all-am: Makefile $(DATA)
+installdirs:
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+ if test -z '$(STRIP)'; then \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ install; \
+ else \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+ fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+ -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES)
+clean: clean-am
+
+clean-am: clean-generic mostlyclean-am
+
+distclean: distclean-am
+ -rm -f Makefile
+distclean-am: clean-am distclean-generic
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am:
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-generic
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am:
+
+.MAKE: install-am install-strip
+
+.PHONY: all all-am check check-am clean clean-generic cscopelist-am \
+ ctags-am distclean distclean-generic distdir dvi dvi-am html \
+ html-am info info-am install install-am install-data \
+ install-data-am install-dvi install-dvi-am install-exec \
+ install-exec-am install-html install-html-am install-info \
+ install-info-am install-man install-pdf install-pdf-am \
+ install-ps install-ps-am install-strip installcheck \
+ installcheck-am installdirs maintainer-clean \
+ maintainer-clean-generic mostlyclean mostlyclean-generic pdf \
+ pdf-am ps ps-am tags-am uninstall uninstall-am
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/database/README.md b/database/README.md
new file mode 100644
index 000000000..8f5e3a6df
--- /dev/null
+++ b/database/README.md
@@ -0,0 +1,206 @@
+# netdata database
+
+Although `netdata` does all its calculations using `long double`, it stores all values using
+a [custom-made 32-bit number](../libnetdata/storage_number/).
+
+So, for each dimension of a chart, netdata will need: `4 bytes for the value * the entries
+of its history`. It will not store any other data for each value in the time series database.
+Since all its values are stored in a time series with fixed step, the time each value
+corresponds can be calculated at run time, using the position of a value in the round robin database.
+
+The default history is 3.600 entries, thus it will need 14.4KB for each chart dimension.
+If you need 1.000 dimensions, they will occupy just 14.4MB.
+
+Of course, 3.600 entries is a very short history, especially if data collection frequency is set
+to 1 second. You will have just one hour of data.
+
+For a day of data and 1.000 dimensions, you will need: 86.400 seconds * 4 bytes * 1.000
+dimensions = 345MB of RAM.
+
+Currently the only option you have to lower this number is to use
+**[Memory Deduplication - Kernel Same Page Merging - KSM](#ksm)**.
+
+## Memory modes
+
+Currently netdata supports 5 memory modes:
+
+1. `ram`, data are purely in memory. Data are never saved on disk. This mode uses `mmap()` and
+ supports [KSM](#ksm).
+
+2. `save`, (the default) data are only in RAM while netdata runs and are saved to / loaded from
+ disk on netdata restart. It also uses `mmap()` and supports [KSM](#ksm).
+
+3. `map`, data are in memory mapped files. This works like the swap. Keep in mind though, this
+ will have a constant write on your disk. When netdata writes data on its memory, the Linux kernel
+ marks the related memory pages as dirty and automatically starts updating them on disk.
+ Unfortunately we cannot control how frequently this works. The Linux kernel uses exactly the
+ same algorithm it uses for its swap memory. Check below for additional information on running a
+ dedicated central netdata server. This mode uses `mmap()` but does not support [KSM](#ksm).
+
+4. `none`, without a database (collected metrics can only be streamed to another netdata).
+
+5. `alloc`, like `ram` but it uses `calloc()` and does not support [KSM](#ksm). This mode is the
+ fallback for all others except `none`.
+
+You can select the memory mode by editing netdata.conf and setting:
+
+```
+[global]
+ # ram, save (the default, save on exit, load on start), map (swap like)
+ memory mode = save
+
+ # the directory where data are saved
+ cache directory = /var/cache/netdata
+```
+
+## Running netdata in embedded devices
+
+Embedded devices usually have very limited RAM resources available.
+
+There are 2 settings for you to tweak:
+
+1. `update every`, which controls the data collection frequency
+2. `history`, which controls the size of the database in RAM
+
+By default `update every = 1` and `history = 3600`. This gives you an hour of data with per
+second updates.
+
+If you set `update every = 2` and `history = 1800`, you will still have an hour of data, but
+collected once every 2 seconds. This will **cut in half** both CPU and RAM resources consumed
+by netdata. Of course experiment a bit. On very weak devices you might have to use
+`update every = 5` and `history = 720` (still 1 hour of data, but 1/5 of the CPU and RAM resources).
+
+You can also disable [data collection plugins](../collectors) you don't need.
+Disabling such plugins will also free both CPU and RAM resources.
+
+## running a dedicated central netdata server
+
+netdata allows streaming data between netdata nodes. This allows us to have a central netdata
+server that will maintain the entire database for all nodes, and will also run health checks/alarms
+for all nodes.
+
+For this central netdata, memory size can be a problem. Fortunately, netdata supports several
+memory modes. What is interesting for this setup is `memory mode = map`.
+
+In this mode, the database of netdata is stored in memory mapped files. netdata continues to read
+and write the database in memory, but the kernel automatically loads and saves memory pages from/to
+disk.
+
+**We suggest _not_ to use this mode on nodes that run other applications.** There will always be
+dirty memory to be synced and this syncing process may influence the way other applications work.
+This mode however is ideal when we need a central netdata server that would normally need huge
+amounts of memory. Using memory mode `map` we can overcome all memory restrictions.
+
+There are a few kernel options that provide finer control on the way this syncing works. But before
+explaining them, a brief introduction of how netdata database works is needed.
+
+For each chart, netdata maps the following files:
+
+1. `chart/main.db`, this is the file that maintains chart information. Every time data are collected
+ for a chart, this is updated.
+
+2. `chart/dimension_name.db`, this is the file for each dimension. At its beginning there is a
+ header, followed by the round robin database where metrics are stored.
+
+So, every time netdata collects data, the following pages will become dirty:
+
+1. the chart file
+2. the header part of all dimension files
+3. if the collected metrics are stored far enough in the dimension file, another page will
+ become dirty, for each dimension
+
+Each page in Linux is 4KB. So, with 200 charts and 1000 dimensions, there will be 1200 to 2200 4KB
+pages dirty pages every second. Of course 1200 of them will always be dirty (the chart header and
+the dimensions headers) and 1000 will be dirty for about 1000 seconds (4 bytes per metric, 4KB per
+page, so 1000 seconds, or 16 minutes per page).
+
+Hopefully, the Linux kernel does not sync all these data every second. The frequency they are
+synced is controlled by `/proc/sys/vm/dirty_expire_centisecs` or the
+`sysctl` `vm.dirty_expire_centisecs`. The default on most systems is 3000 (30 seconds).
+
+On a busy server centralizing metrics from 20+ servers you will experience this:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/23834750/429ab0dc-0764-11e7-821a-d7908bc881ac.png)
+
+As you can see, there is quite some stress (this is `iowait`) every 30 seconds.
+
+A simple solution is to increase this time to 10 minutes (60000). This is the same system
+with this setting in 10 minutes:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/23834784/d2304f72-0764-11e7-8389-fb830ffd973a.png)
+
+Of course, setting this to 10 minutes means that data on disk might be up to 10 minutes old if you
+get an abnormal shutdown.
+
+There are 2 more options to tweak:
+
+1. `dirty_background_ratio`, by default `10`.
+2. `dirty_ratio`, by default `20`.
+
+These control the amount of memory that should be dirty for disk syncing to be triggered.
+On dedicated netdata servers, you can use: `80` and `90` respectively, so that all RAM is given
+to netdata.
+
+With these settings, you can expect a little `iowait` spike once every 10 minutes and in case
+of system crash, data on disk will be up to 10 minutes old.
+
+![image](https://cloud.githubusercontent.com/assets/2662304/23835030/ba4bf506-0768-11e7-9bc6-3b23e080c69f.png)
+
+To have these settings automatically applied on boot, create the file `/etc/sysctl.d/netdata-memory.conf` with these contents:
+
+```
+vm.dirty_expire_centisecs = 60000
+vm.dirty_background_ratio = 80
+vm.dirty_ratio = 90
+vm.dirty_writeback_centisecs = 0
+```
+
+## KSM
+
+Netdata offers all its round robin database to kernel for deduplication.
+
+In the past KSM has been criticized for consuming a lot of CPU resources.
+Although this is true when KSM is used for deduplicating certain applications, it is not true with
+netdata, since the netdata memory is written very infrequently (if you have 24 hours of metrics in
+netdata, each byte at the in-memory database will be updated just once per day).
+
+KSM is a solution that will provide 60+% memory savings to netdata.
+
+#### Enable KSM in kernel
+
+You need to run a kernel compiled with:
+
+```sh
+CONFIG_KSM=y
+```
+
+When KSM is enabled at the kernel is just available for the user to enable it.
+
+So, if you build a kernel with `CONFIG_KSM=y` you will just get a few files in `/sys/kernel/mm/ksm`. Nothing else happens. There is no performance penalty (apart I guess from the memory this code occupies into the kernel).
+
+The files that `CONFIG_KSM=y` offers include:
+
+- `/sys/kernel/mm/ksm/run` by default `0`. You have to set this to `1` for the kernel to spawn `ksmd`.
+- `/sys/kernel/mm/ksm/sleep_millisecs`, by default `20`. The frequency ksmd should evaluate memory for deduplication.
+- `/sys/kernel/mm/ksm/pages_to_scan`, by default `100`. The amount of pages ksmd will evaluate on each run.
+
+So, by default `ksmd` is just disabled. It will not harm performance and the user/admin can control the CPU resources he/she is willing `ksmd` to use.
+
+#### Run `ksmd` kernel daemon
+
+To activate / run `ksmd` you need to run:
+
+```sh
+echo 1 >/sys/kernel/mm/ksm/run
+echo 1000 >/sys/kernel/mm/ksm/sleep_millisecs
+```
+
+With these settings ksmd does not even appear in the running process list (it will run once per second and evaluate 100 pages for de-duplication).
+
+Put the above lines in your boot sequence (`/etc/rc.local` or equivalent) to have `ksmd` run at boot.
+
+## Monitoring Kernel Memory de-duplication performance
+
+Netdata will create charts for kernel memory de-duplication performance, like this:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/11998786/eb23ae54-aab6-11e5-94d4-e848e8a5c56a.png)
diff --git a/src/rrd.c b/database/rrd.c
index 3be2d8e41..119efa62e 100644
--- a/src/rrd.c
+++ b/database/rrd.c
@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
#define NETDATA_RRD_INTERNALS 1
-#include "common.h"
+
+#include "rrd.h"
// ----------------------------------------------------------------------------
// globals
diff --git a/src/rrd.h b/database/rrd.h
index d17daacd2..19eb100cd 100644
--- a/src/rrd.h
+++ b/database/rrd.h
@@ -1,6 +1,27 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
#ifndef NETDATA_RRD_H
#define NETDATA_RRD_H 1
+// forward typedefs
+typedef struct rrdhost RRDHOST;
+typedef struct rrddim RRDDIM;
+typedef struct rrdset RRDSET;
+typedef struct rrdvar RRDVAR;
+typedef struct rrdsetvar RRDSETVAR;
+typedef struct rrddimvar RRDDIMVAR;
+typedef struct rrdcalc RRDCALC;
+typedef struct rrdcalctemplate RRDCALCTEMPLATE;
+typedef struct alarm_entry ALARM_ENTRY;
+
+#include "../daemon/common.h"
+#include "web/api/queries/query.h"
+#include "rrdvar.h"
+#include "rrdsetvar.h"
+#include "rrddimvar.h"
+#include "rrdcalc.h"
+#include "rrdcalctemplate.h"
+
#define UPDATE_EVERY 1
#define UPDATE_EVERY_MAX 3600
@@ -19,8 +40,6 @@ extern int gap_when_lost_iterations_above;
typedef long long total_number;
#define TOTAL_NUMBER_FORMAT "%lld"
-typedef struct rrdhost RRDHOST;
-
// ----------------------------------------------------------------------------
// chart types
@@ -103,8 +122,8 @@ typedef struct rrdfamily RRDFAMILY;
typedef enum rrddim_flags {
RRDDIM_FLAG_NONE = 0,
- RRDDIM_FLAG_HIDDEN = 1 << 0, // this dimension will not be offered to callers
- RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = 1 << 1 // do not offer RESET or OVERFLOW info to callers
+ RRDDIM_FLAG_HIDDEN = (1 << 0), // this dimension will not be offered to callers
+ RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1) // do not offer RESET or OVERFLOW info to callers
} RRDDIM_FLAGS;
#ifdef HAVE_C___ATOMIC
@@ -159,8 +178,8 @@ struct rrddim {
size_t collections_counter; // the number of times we added values to this rrdim
size_t unused[10];
- int updated:1; // 1 when the dimension has been updated since the last processing
- int exposed:1; // 1 when set what have sent this dimension to the central netdata
+ unsigned int updated:1; // 1 when the dimension has been updated since the last processing
+ unsigned int exposed:1; // 1 when set what have sent this dimension to the central netdata
struct timeval last_collected_time; // when was this dimension last updated
// this is actual date time we updated the last_collected_value
@@ -202,7 +221,6 @@ struct rrddim {
storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER
};
-typedef struct rrddim RRDDIM;
// ----------------------------------------------------------------------------
// these loop macros make sure the linked list is accessed with the right lock
@@ -222,18 +240,21 @@ typedef struct rrddim RRDDIM;
// and may lead to missing information.
typedef enum rrdset_flags {
- RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart
- RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another
+ RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart
+ RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another
// (the master data set should be the one that has the same family and is not detail)
- RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart
- RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete
- RRDSET_FLAG_BACKEND_SEND = 1 << 4, // if set, this chart should be sent to backends
- RRDSET_FLAG_BACKEND_IGNORE = 1 << 5, // if set, this chart should not be sent to backends
- RRDSET_FLAG_EXPOSED_UPSTREAM = 1 << 6, // if set, we have sent this chart to netdata master (streaming)
- RRDSET_FLAG_STORE_FIRST = 1 << 7, // if set, do not eliminate the first collection during interpolation
- RRDSET_FLAG_HETEROGENEOUS = 1 << 8, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
- RRDSET_FLAG_HOMEGENEOUS_CHECK= 1 << 9, // if set, the chart should be checked to determine if the dimensions as homogeneous
- RRDSET_FLAG_HIDDEN = 1 << 10, // if set, do not show this chart on the dashboard, but use it for backends
+ RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart
+ RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete
+ RRDSET_FLAG_BACKEND_SEND = 1 << 4, // if set, this chart should be sent to backends
+ RRDSET_FLAG_BACKEND_IGNORE = 1 << 5, // if set, this chart should not be sent to backends
+ RRDSET_FLAG_UPSTREAM_SEND = 1 << 6, // if set, this chart should be sent upstream (streaming)
+ RRDSET_FLAG_UPSTREAM_IGNORE = 1 << 7, // if set, this chart should not be sent upstream (streaming)
+ RRDSET_FLAG_UPSTREAM_EXPOSED = 1 << 8, // if set, we have sent this chart definition to netdata master (streaming)
+ RRDSET_FLAG_STORE_FIRST = 1 << 9, // if set, do not eliminate the first collection during interpolation
+ RRDSET_FLAG_HETEROGENEOUS = 1 << 10, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
+ RRDSET_FLAG_HOMEGENEOUS_CHECK = 1 << 11, // if set, the chart should be checked to determine if the dimensions as homogeneous
+ RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for backends
+ RRDSET_FLAG_SYNC_CLOCK = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
} RRDSET_FLAGS;
#ifdef HAVE_C___ATOMIC
@@ -245,6 +266,7 @@ typedef enum rrdset_flags {
#define rrdset_flag_set(st, flag) (st)->flags |= (flag)
#define rrdset_flag_clear(st, flag) (st)->flags &= ~(flag)
#endif
+#define rrdset_flag_check_noatomic(st, flag) ((st)->flags & (flag))
struct rrdset {
// ------------------------------------------------------------------------
@@ -282,7 +304,7 @@ struct rrdset {
long current_entry; // the entry that is currently being updated
// it goes around in a round-robin fashion
- uint32_t flags; // configuration flags
+ RRDSET_FLAGS flags; // configuration flags
int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored
// netdata will interpolate values for gaps lower than this
@@ -353,7 +375,6 @@ struct rrdset {
RRDDIM *dimensions; // the actual data for every dimension
};
-typedef struct rrdset RRDSET;
#define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock))
#define rrdset_wrlock(st) netdata_rwlock_wrlock(&((st)->rrdset_rwlock))
@@ -401,6 +422,65 @@ typedef enum rrdhost_flags {
#define rrdset_debug(st, fmt, args...) debug_dummy()
#endif
+// ----------------------------------------------------------------------------
+// Health data
+
+struct alarm_entry {
+ uint32_t unique_id;
+ uint32_t alarm_id;
+ uint32_t alarm_event_id;
+
+ time_t when;
+ time_t duration;
+ time_t non_clear_duration;
+
+ char *name;
+ uint32_t hash_name;
+
+ char *chart;
+ uint32_t hash_chart;
+
+ char *family;
+
+ char *exec;
+ char *recipient;
+ time_t exec_run_timestamp;
+ int exec_code;
+
+ char *source;
+ char *units;
+ char *info;
+
+ calculated_number old_value;
+ calculated_number new_value;
+
+ char *old_value_string;
+ char *new_value_string;
+
+ RRDCALC_STATUS old_status;
+ RRDCALC_STATUS new_status;
+
+ uint32_t flags;
+
+ int delay;
+ time_t delay_up_to_timestamp;
+
+ uint32_t updated_by_id;
+ uint32_t updates_id;
+
+ struct alarm_entry *next;
+};
+
+
+typedef struct alarm_log {
+ uint32_t next_log_id;
+ uint32_t next_alarm_id;
+ unsigned int count;
+ unsigned int max;
+ ALARM_ENTRY *alarms;
+ netdata_rwlock_t alarm_log_rwlock;
+} ALARM_LOG;
+
// ----------------------------------------------------------------------------
// RRD HOST
@@ -423,7 +503,7 @@ struct rrdhost {
const char *tags; // tags for this host
const char *timezone; // the timezone of the host
- uint32_t flags; // flags about this RRDHOST
+ RRDHOST_FLAGS flags; // flags about this RRDHOST
int rrd_update_every; // the update frequency of the host
long rrd_history_entries; // the number of history entries for the host's charts
@@ -438,20 +518,22 @@ struct rrdhost {
// ------------------------------------------------------------------------
// streaming of data to remote hosts - rrdpush
- int rrdpush_send_enabled:1; // 1 when this host sends metrics to another netdata
+ unsigned int rrdpush_send_enabled:1; // 1 when this host sends metrics to another netdata
char *rrdpush_send_destination; // where to send metrics to
char *rrdpush_send_api_key; // the api key at the receiving netdata
// the following are state information for the threading
// streaming metrics from this netdata to an upstream netdata
- volatile int rrdpush_sender_spawn:1; // 1 when the sender thread has been spawn
+ volatile unsigned int rrdpush_sender_spawn:1; // 1 when the sender thread has been spawn
netdata_thread_t rrdpush_sender_thread; // the sender thread
- volatile int rrdpush_sender_connected:1; // 1 when the sender is ready to push metrics
+ volatile unsigned int rrdpush_sender_connected:1; // 1 when the sender is ready to push metrics
int rrdpush_sender_socket; // the fd of the socket to the remote host, or -1
- volatile int rrdpush_sender_error_shown:1; // 1 when we have logged a communication error
- volatile int rrdpush_sender_join:1; // 1 when we have to join the sending thread
+ volatile unsigned int rrdpush_sender_error_shown:1; // 1 when we have logged a communication error
+ volatile unsigned int rrdpush_sender_join:1; // 1 when we have to join the sending thread
+
+ SIMPLE_PATTERN *rrdpush_send_charts_matching; // pattern to match the charts to be sent
// metrics may be collected asynchronously
// these synchronize all the threads willing the write to our sending buffer
@@ -471,7 +553,7 @@ struct rrdhost {
// ------------------------------------------------------------------------
// health monitoring options
- int health_enabled:1; // 1 when this host has health enabled
+ unsigned int health_enabled:1; // 1 when this host has health enabled
time_t health_delay_up_to; // a timestamp to delay alarms processing up to
char *health_default_exec; // the full path of the alarms notifications program
char *health_default_recipient; // the default recipient for all alarms
@@ -564,10 +646,11 @@ extern RRDHOST *rrdhost_find_or_create(
, int update_every
, long history
, RRD_MEMORY_MODE mode
- , int health_enabled
- , int rrdpush_enabled
+ , unsigned int health_enabled
+ , unsigned int rrdpush_enabled
, char *rrdpush_destination
, char *rrdpush_api_key
+ , char *rrdpush_send_charts_matching
);
#if defined(NETDATA_INTERNAL_CHECKS) && defined(NETDATA_VERIFY_LOCKS)
@@ -666,28 +749,84 @@ extern void rrdset_isnot_obsolete(RRDSET *st);
#define rrdset_first_entry_t(st) ((time_t)(rrdset_last_entry_t(st) - rrdset_duration(st)))
// get the last slot updated in the round robin database
-#define rrdset_last_slot(st) ((unsigned long)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
+#define rrdset_last_slot(st) ((size_t)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
// get the first / oldest slot updated in the round robin database
-#define rrdset_first_slot(st) ((unsigned long)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) ))
+// #define rrdset_first_slot(st) ((size_t)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) ))
+
+// return the slot that has the oldest value
+
+static inline size_t rrdset_first_slot(RRDSET *st) {
+ if(st->counter >= (size_t)st->entries) {
+ // the database has been rotated at least once
+ // the oldest entry is the one that will be next
+ // overwritten by data collection
+ return (size_t)st->current_entry;
+ }
+
+ // we do not have rotated the db yet
+ // so 0 is the first entry
+ return 0;
+}
// get the slot of the round robin database, for the given timestamp (t)
// it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database
-#define rrdset_time2slot(st, t) ( \
- ( (time_t)(t) >= rrdset_last_entry_t(st)) ? ( rrdset_last_slot(st) ) : \
- ( ((time_t)(t) <= rrdset_first_entry_t(st)) ? rrdset_first_slot(st) : \
- ( (rrdset_last_slot(st) >= (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) ? \
- (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) : \
- (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) + (unsigned long)(st)->entries ) \
- )))
+static inline size_t rrdset_time2slot(RRDSET *st, time_t t) {
+ size_t ret = 0;
+
+ if(t >= rrdset_last_entry_t(st)) {
+ // the requested time is after the last entry we have
+ ret = rrdset_last_slot(st);
+ }
+ else {
+ if(t <= rrdset_first_entry_t(st)) {
+ // the requested time is before the first entry we have
+ ret = rrdset_first_slot(st);
+ }
+ else {
+ if(rrdset_last_slot(st) >= ((rrdset_last_entry_t(st) - t) / (size_t)(st->update_every)))
+ ret = rrdset_last_slot(st) - ((rrdset_last_entry_t(st) - t) / (size_t)(st->update_every));
+ else
+ ret = rrdset_last_slot(st) - ((rrdset_last_entry_t(st) - t) / (size_t)(st->update_every)) + (unsigned long)st->entries;
+ }
+ }
+
+ if(unlikely(ret >= (size_t)st->entries)) {
+ error("INTERNAL ERROR: rrdset_time2slot() on %s returns values outside entries", st->name);
+ ret = (size_t)(st->entries - 1);
+ }
+
+ return ret;
+}
// get the timestamp of a specific slot in the round robin database
-#define rrdset_slot2time(st, slot) ( rrdset_last_entry_t(st) - \
- ((unsigned long)(st)->update_every * ( \
- ( (unsigned long)(slot) > rrdset_last_slot(st)) ? \
- ( (rrdset_last_slot(st) - (unsigned long)(slot) + (unsigned long)(st)->entries) ) : \
- ( (rrdset_last_slot(st) - (unsigned long)(slot)) )) \
- ))
+static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
+ time_t ret;
+
+ if(slot >= (size_t)st->entries) {
+ error("INTERNAL ERROR: caller of rrdset_slot2time() gives invalid slot %zu", slot);
+ slot = (size_t)st->entries - 1;
+ }
+
+ if(slot > rrdset_last_slot(st)) {
+ ret = rrdset_last_entry_t(st) - (size_t)st->update_every * (rrdset_last_slot(st) - slot + (size_t)st->entries);
+ }
+ else {
+ ret = rrdset_last_entry_t(st) - (size_t)st->update_every;
+ }
+
+ if(unlikely(ret < rrdset_first_entry_t(st))) {
+ error("INTERNAL ERROR: rrdset_slot2time() on %s returns time too far in the past", st->name);
+ ret = rrdset_first_entry_t(st);
+ }
+
+ if(unlikely(ret > rrdset_last_entry_t(st))) {
+ error("INTERNAL ERROR: rrdset_slot2time() on %s returns time into the future", st->name);
+ ret = rrdset_last_entry_t(st);
+ }
+
+ return ret;
+}
// ----------------------------------------------------------------------------
// RRD DIMENSION functions
diff --git a/src/rrdcalc.c b/database/rrdcalc.c
index 4e41539e2..7f6a896b6 100644
--- a/src/rrdcalc.c
+++ b/database/rrdcalc.c
@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+#include "rrd.h"
// ----------------------------------------------------------------------------
// RRDCALC management
@@ -64,15 +66,18 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
st->red = rc->red;
}
- rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
- rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
+ rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value);
+ rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value);
char fullname[RRDVAR_MAX_LENGTH + 1];
snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);
- rc->hostid = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
+ rc->hostid = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value);
snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name);
- rc->hostname = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
+ rc->hostname = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value);
+
+ if(rc->hostid && !rc->hostname)
+ rc->hostid->options |= RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR;
if(!rc->units) rc->units = strdupz(st->units);
@@ -358,7 +363,7 @@ inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *c
(rc->recipient)?rc->recipient:"DEFAULT",
rc->green,
rc->red,
- rc->group,
+ (int)rc->group,
rc->after,
rc->before,
rc->options,
diff --git a/database/rrdcalc.h b/database/rrdcalc.h
new file mode 100644
index 000000000..0c7cd0aa1
--- /dev/null
+++ b/database/rrdcalc.h
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rrd.h"
+
+#ifndef NETDATA_RRDCALC_H
+#define NETDATA_RRDCALC_H 1
+
+// calculated variables (defined in health configuration)
+// These aggregate time-series data at fixed intervals
+// (defined in their update_every member below)
+// They increase the overhead of netdata.
+//
+// These calculations are allocated and linked (->next)
+// under RRDHOST.
+// Then are also linked to RRDSET (of course only when the
+// chart is found, via ->rrdset_next and ->rrdset_prev).
+// This double-linked list is maintained sorted at all times
+// having as RRDSET.calculations the RRDCALC to be processed
+// next.
+
+#define RRDCALC_FLAG_DB_ERROR 0x00000001
+#define RRDCALC_FLAG_DB_NAN 0x00000002
+/* #define RRDCALC_FLAG_DB_STALE 0x00000004 */
+#define RRDCALC_FLAG_CALC_ERROR 0x00000008
+#define RRDCALC_FLAG_WARN_ERROR 0x00000010
+#define RRDCALC_FLAG_CRIT_ERROR 0x00000020
+#define RRDCALC_FLAG_RUNNABLE 0x00000040
+#define RRDCALC_FLAG_NO_CLEAR_NOTIFICATION 0x80000000
+
+struct rrdcalc {
+ uint32_t id; // the unique id of this alarm
+ uint32_t next_event_id; // the next event id that will be used for this alarm
+
+ char *name; // the name of this alarm
+ uint32_t hash;
+
+ char *exec; // the command to execute when this alarm switches state
+ char *recipient; // the recipient of the alarm (the first parameter to exec)
+
+ char *chart; // the chart id this should be linked to
+ uint32_t hash_chart;
+
+ char *source; // the source of this alarm
+ char *units; // the units of the alarm
+ char *info; // a short description of the alarm
+
+ int update_every; // update frequency for the alarm
+
+ // the red and green threshold of this alarm (to be set to the chart)
+ calculated_number green;
+ calculated_number red;
+
+ // ------------------------------------------------------------------------
+ // database lookup settings
+
+ char *dimensions; // the chart dimensions
+ RRDR_GROUPING group; // grouping method: average, max, etc.
+ int before; // ending point in time-series
+ int after; // starting point in time-series
+ uint32_t options; // calculation options
+
+ // ------------------------------------------------------------------------
+ // expressions related to the alarm
+
+ EVAL_EXPRESSION *calculation; // expression to calculate the value of the alarm
+ EVAL_EXPRESSION *warning; // expression to check the warning condition
+ EVAL_EXPRESSION *critical; // expression to check the critical condition
+
+ // ------------------------------------------------------------------------
+ // notification delay settings
+
+ int delay_up_duration; // duration to delay notifications when alarm raises
+ int delay_down_duration; // duration to delay notifications when alarm lowers
+ int delay_max_duration; // the absolute max delay to apply to this alarm
+ float delay_multiplier; // multiplier for all delays when alarms switch status
+ // while now < delay_up_to
+
+ // ------------------------------------------------------------------------
+ // runtime information
+
+ RRDCALC_STATUS status; // the current status of the alarm
+
+ calculated_number value; // the current value of the alarm
+ calculated_number old_value; // the previous value of the alarm
+
+ uint32_t rrdcalc_flags; // check RRDCALC_FLAG_*
+
+ time_t last_updated; // the last update timestamp of the alarm
+ time_t next_update; // the next update timestamp of the alarm
+ time_t last_status_change; // the timestamp of the last time this alarm changed status
+
+ time_t db_after; // the first timestamp evaluated by the db lookup
+ time_t db_before; // the last timestamp evaluated by the db lookup
+
+ time_t delay_up_to_timestamp; // the timestamp up to which we should delay notifications
+ int delay_up_current; // the current up notification delay duration
+ int delay_down_current; // the current down notification delay duration
+ int delay_last; // the last delay we used
+
+ // ------------------------------------------------------------------------
+ // variables this alarm exposes to the rest of the alarms
+
+ RRDVAR *local;
+ RRDVAR *family;
+ RRDVAR *hostid;
+ RRDVAR *hostname;
+
+ // ------------------------------------------------------------------------
+ // the chart this alarm it is linked to
+
+ struct rrdset *rrdset;
+
+ // linking of this alarm on its chart
+ struct rrdcalc *rrdset_next;
+ struct rrdcalc *rrdset_prev;
+
+ struct rrdcalc *next;
+};
+
+#define RRDCALC_HAS_DB_LOOKUP(rc) ((rc)->after)
+
+extern void rrdsetcalc_link_matching(RRDSET *st);
+extern void rrdsetcalc_unlink(RRDCALC *rc);
+extern RRDCALC *rrdcalc_find(RRDSET *st, const char *name);
+
+extern const char *rrdcalc_status2string(RRDCALC_STATUS status);
+
+extern void rrdcalc_free(RRDCALC *rc);
+extern void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc);
+
+extern int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name);
+extern uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id);
+extern RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart);
+extern void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc);
+
+#endif //NETDATA_RRDCALC_H
diff --git a/src/rrdcalctemplate.c b/database/rrdcalctemplate.c
index 75a7002b3..ba7e7ec94 100644
--- a/src/rrdcalctemplate.c
+++ b/database/rrdcalctemplate.c
@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+#include "rrd.h"
// ----------------------------------------------------------------------------
// RRDCALCTEMPLATE management
diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h
new file mode 100644
index 000000000..2235ecaa1
--- /dev/null
+++ b/database/rrdcalctemplate.h
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RRDCALCTEMPLATE_H
+#define NETDATA_RRDCALCTEMPLATE_H 1
+
+#include "rrd.h"
+
+// RRDCALCTEMPLATE
+// these are to be applied to charts found dynamically
+// based on their context.
+struct rrdcalctemplate {
+ char *name;
+ uint32_t hash_name;
+
+ char *exec;
+ char *recipient;
+
+ char *context;
+ uint32_t hash_context;
+
+ char *family_match;
+ SIMPLE_PATTERN *family_pattern;
+
+ char *source; // the source of this alarm
+ char *units; // the units of the alarm
+ char *info; // a short description of the alarm
+
+ int update_every; // update frequency for the alarm
+
+ // the red and green threshold of this alarm (to be set to the chart)
+ calculated_number green;
+ calculated_number red;
+
+ // ------------------------------------------------------------------------
+ // database lookup settings
+
+ char *dimensions; // the chart dimensions
+ RRDR_GROUPING group; // grouping method: average, max, etc.
+ int before; // ending point in time-series
+ int after; // starting point in time-series
+ uint32_t options; // calculation options
+
+ // ------------------------------------------------------------------------
+ // notification delay settings
+
+ int delay_up_duration; // duration to delay notifications when alarm raises
+ int delay_down_duration; // duration to delay notifications when alarm lowers
+ int delay_max_duration; // the absolute max delay to apply to this alarm
+ float delay_multiplier; // multiplier for all delays when alarms switch status
+
+ // ------------------------------------------------------------------------
+ // expressions related to the alarm
+
+ EVAL_EXPRESSION *calculation;
+ EVAL_EXPRESSION *warning;
+ EVAL_EXPRESSION *critical;
+
+ struct rrdcalctemplate *next;
+};
+
+#define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after)
+
+extern void rrdcalctemplate_link_matching(RRDSET *st);
+
+extern void rrdcalctemplate_free(RRDCALCTEMPLATE *rt);
+extern void rrdcalctemplate_unlink_and_free(RRDHOST *host, RRDCALCTEMPLATE *rt);
+
+#endif //NETDATA_RRDCALCTEMPLATE_H
diff --git a/src/rrddim.c b/database/rrddim.c
index a54c6452f..95e485106 100644
--- a/src/rrddim.c
+++ b/database/rrddim.c
@@ -1,5 +1,7 @@
-#define NETDATA_RRD_INTERNALS 1
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#define NETDATA_RRD_INTERNALS
+#include "rrd.h"
// ----------------------------------------------------------------------------
// RRDDIM index
@@ -47,6 +49,7 @@ inline int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name) {
rd->hash_name = simple_hash(rd->name);
rrddimvar_rename_all(rd);
rd->exposed = 0;
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
return 1;
}
@@ -58,6 +61,7 @@ inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm)
rd->algorithm = algorithm;
rd->exposed = 0;
rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
return 1;
}
@@ -69,6 +73,7 @@ inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip
rd->multiplier = multiplier;
rd->exposed = 0;
rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
return 1;
}
@@ -80,6 +85,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
rd->divisor = divisor;
rd->exposed = 0;
rrdset_flag_set(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
return 1;
}
@@ -87,6 +93,11 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
// RRDDIM create a dimension
RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) {
+ rrdset_wrlock(st);
+
+ rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+
RRDDIM *rd = rrddim_find(st, id);
if(unlikely(rd)) {
debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>");
@@ -96,6 +107,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rrddim_set_multiplier(st, rd, multiplier);
rrddim_set_divisor(st, rd, divisor);
+ rrdset_unlock(st);
return rd;
}
@@ -236,7 +248,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rd->rrdset = st;
// append this dimension
- rrdset_wrlock(st);
if(!st->dimensions)
st->dimensions = rd;
else {
@@ -268,11 +279,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT);
}
- rrdset_unlock(st);
-
if(unlikely(rrddim_index_add(st, rd) != rd))
error("RRDDIM: INTERNAL ERROR: attempt to index duplicate dimension '%s' on chart '%s'", rd->id, st->id);
+ rrdset_unlock(st);
return(rd);
}
diff --git a/src/rrddimvar.c b/database/rrddimvar.c
index 28a3e7fa6..3c2ed75e5 100644
--- a/src/rrddimvar.c
+++ b/database/rrddimvar.c
@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+#include "rrd.h"
// ----------------------------------------------------------------------------
// RRDDIMVAR management
@@ -117,8 +119,8 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {
// - $id
// - $name
- rs->var_local_id = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_id, rs->type, rs->value);
- rs->var_local_name = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_name, rs->type, rs->value);
+ rs->var_local_id = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
+ rs->var_local_name = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
// FAMILY VARIABLES FOR THIS DIMENSION
// -----------------------------------
@@ -129,10 +131,10 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {
// - $chart-context.id
// - $chart-context.name
- rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, rs->value);
- rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, rs->value);
- rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, rs->value);
- rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, rs->value);
+ rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
+ rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
+ rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
+ rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
// HOST VARIABLES FOR THIS DIMENSION
// -----------------------------------
@@ -143,14 +145,15 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {
// - $chart-name.id
// - $chart-name.name
- rs->var_host_chartidid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidid, rs->type, rs->value);
- rs->var_host_chartidname = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidname, rs->type, rs->value);
- rs->var_host_chartnameid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnameid, rs->type, rs->value);
- rs->var_host_chartnamename = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnamename, rs->type, rs->value);
+ rs->var_host_chartidid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
+ rs->var_host_chartidname = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
+ rs->var_host_chartnameid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnameid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
+ rs->var_host_chartnamename = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnamename, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);
}
RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options) {
RRDSET *st = rd->rrdset;
+ (void)st;
debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:"");
@@ -177,6 +180,8 @@ RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, co
void rrddimvar_rename_all(RRDDIM *rd) {
RRDSET *st = rd->rrdset;
+ (void)st;
+
debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
RRDDIMVAR *rs, *next = rd->variables;
diff --git a/database/rrddimvar.h b/database/rrddimvar.h
new file mode 100644
index 000000000..3494824be
--- /dev/null
+++ b/database/rrddimvar.h
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RRDDIMVAR_H
+#define NETDATA_RRDDIMVAR_H 1
+
+#include "rrd.h"
+
+// variables linked to individual dimensions
+// We link variables to point the values that are already
+// calculated / processed by the normal data collection process
+// This means, there will be no speed penalty for using
+// these variables
+struct rrddimvar {
+ char *prefix;
+ char *suffix;
+
+ char *key_id; // dimension id
+ char *key_name; // dimension name
+ char *key_contextid; // context + dimension id
+ char *key_contextname; // context + dimension name
+ char *key_fullidid; // chart type.chart id + dimension id
+ char *key_fullidname; // chart type.chart id + dimension name
+ char *key_fullnameid; // chart type.chart name + dimension id
+ char *key_fullnamename; // chart type.chart name + dimension name
+
+ RRDVAR_TYPE type;
+ void *value;
+
+ RRDVAR_OPTIONS options;
+
+ RRDVAR *var_local_id;
+ RRDVAR *var_local_name;
+
+ RRDVAR *var_family_id;
+ RRDVAR *var_family_name;
+ RRDVAR *var_family_contextid;
+ RRDVAR *var_family_contextname;
+
+ RRDVAR *var_host_chartidid;
+ RRDVAR *var_host_chartidname;
+ RRDVAR *var_host_chartnameid;
+ RRDVAR *var_host_chartnamename;
+
+ struct rrddim *rrddim;
+
+ struct rrddimvar *next;
+};
+
+
+extern void rrddimvar_rename_all(RRDDIM *rd);
+extern RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options);
+extern void rrddimvar_free(RRDDIMVAR *rs);
+
+
+
+#endif //NETDATA_RRDDIMVAR_H
diff --git a/src/rrdfamily.c b/database/rrdfamily.c
index 905ae480b..f75f0adc3 100644
--- a/src/rrdfamily.c
+++ b/database/rrdfamily.c
@@ -1,5 +1,7 @@
-#define NETDATA_RRD_INTERNALS 1
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#define NETDATA_RRD_INTERNALS
+#include "rrd.h"
// ----------------------------------------------------------------------------
// RRDFAMILY index
diff --git a/src/rrdhost.c b/database/rrdhost.c
index e62e61ae8..43aa2daa2 100644
--- a/src/rrdhost.c
+++ b/database/rrdhost.c
@@ -1,5 +1,7 @@
-#define NETDATA_RRD_INTERNALS 1
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#define NETDATA_RRD_INTERNALS
+#include "rrd.h"
RRDHOST *localhost = NULL;
size_t rrd_hosts_available = 0;
@@ -116,10 +118,11 @@ RRDHOST *rrdhost_create(const char *hostname,
int update_every,
long entries,
RRD_MEMORY_MODE memory_mode,
- int health_enabled,
- int rrdpush_enabled,
+ unsigned int health_enabled,
+ unsigned int rrdpush_enabled,
char *rrdpush_destination,
char *rrdpush_api_key,
+ char *rrdpush_send_charts_matching,
int is_localhost
) {
debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
@@ -132,9 +135,10 @@ RRDHOST *rrdhost_create(const char *hostname,
host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
host->rrd_memory_mode = memory_mode;
host->health_enabled = (memory_mode == RRD_MEMORY_MODE_NONE)? 0 : health_enabled;
- host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key);
+ host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key) ? 1 : 0;
host->rrdpush_send_destination = (host->rrdpush_send_enabled)?strdupz(rrdpush_destination):NULL;
host->rrdpush_send_api_key = (host->rrdpush_send_enabled)?strdupz(rrdpush_api_key):NULL;
+ host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL, SIMPLE_PATTERN_EXACT);
host->rrdpush_sender_pipe[0] = -1;
host->rrdpush_sender_pipe[1] = -1;
@@ -227,7 +231,7 @@ RRDHOST *rrdhost_create(const char *hostname,
snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_plugins_dir);
host->health_default_exec = strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
- host->health_default_recipient = strdup("root");
+ host->health_default_recipient = strdupz("root");
// ------------------------------------------------------------------------
@@ -238,7 +242,7 @@ RRDHOST *rrdhost_create(const char *hostname,
health_alarm_log_open(host);
rrdhost_wrlock(host);
- health_readdir(host, health_config_dir());
+ health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
rrdhost_unlock(host);
}
@@ -323,10 +327,11 @@ RRDHOST *rrdhost_find_or_create(
, int update_every
, long history
, RRD_MEMORY_MODE mode
- , int health_enabled
- , int rrdpush_enabled
+ , unsigned int health_enabled
+ , unsigned int rrdpush_enabled
, char *rrdpush_destination
, char *rrdpush_api_key
+ , char *rrdpush_send_charts_matching
) {
debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid);
@@ -349,6 +354,7 @@ RRDHOST *rrdhost_find_or_create(
, rrdpush_enabled
, rrdpush_destination
, rrdpush_api_key
+ , rrdpush_send_charts_matching
, 0
);
}
@@ -461,6 +467,7 @@ void rrd_init(char *hostname) {
, default_rrdpush_enabled
, default_rrdpush_destination
, default_rrdpush_api_key
+ , default_rrdpush_send_charts_matching
, 1
);
rrd_unlock();
@@ -574,6 +581,7 @@ void rrdhost_free(RRDHOST *host) {
freez(host->health_log_filename);
freez(host->hostname);
freez(host->registry_hostname);
+ simple_pattern_free(host->rrdpush_send_charts_matching);
rrdhost_unlock(host);
netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock);
netdata_rwlock_destroy(&host->rrdhost_rwlock);
diff --git a/src/rrdset.c b/database/rrdset.c
index bbd0ae728..3f5ba73b6 100644
--- a/src/rrdset.c
+++ b/database/rrdset.c
@@ -1,5 +1,7 @@
-#define NETDATA_RRD_INTERNALS 1
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#define NETDATA_RRD_INTERNALS
+#include "rrd.h"
void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
debug(D_RRD_CALLS, "Checking read lock on chart '%s'", st->id);
@@ -172,27 +174,30 @@ int rrdset_set_name(RRDSET *st, const char *name) {
if(unlikely(rrdset_index_add_name(host, st) != st))
error("RRDSET: INTERNAL ERROR: attempted to index duplicate chart name '%s'", st->name);
+ rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND);
+ rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+
return 1;
}
inline void rrdset_is_obsolete(RRDSET *st) {
- RRDHOST *host = st->rrdhost;
-
if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE);
- rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
// the chart will not get more updates (data collection)
// so, we have to push its definition now
- if(unlikely(host->rrdpush_send_enabled))
- rrdset_push_chart_definition(st);
+ rrdset_push_chart_definition_now(st);
}
}
inline void rrdset_isnot_obsolete(RRDSET *st) {
if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
- rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
// the chart will be pushed upstream automatically
// due to data collection
@@ -201,6 +206,8 @@ inline void rrdset_isnot_obsolete(RRDSET *st) {
inline void rrdset_update_heterogeneous_flag(RRDSET *st) {
RRDHOST *host = st->rrdhost;
+ (void)host;
+
RRDDIM *rd;
rrdset_flag_clear(st, RRDSET_FLAG_HOMEGENEOUS_CHECK);
@@ -476,13 +483,19 @@ RRDSET *rrdset_create_custom(
snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
RRDSET *st = rrdset_find_on_create(host, fullid);
- if(st) return st;
+ if(st) {
+ rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+ return st;
+ }
rrdhost_wrlock(host);
st = rrdset_find_on_create(host, fullid);
if(st) {
rrdhost_unlock(host);
+ rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
return st;
}
@@ -581,8 +594,8 @@ RRDSET *rrdset_create_custom(
memset(st, 0, size);
}
else if(st->last_updated.tv_sec > now + update_every) {
- error("File %s refers to the future. Clearing it.", fullfilename);
- memset(st, 0, size);
+ error("File %s refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st->last_updated.tv_sec - now));
+ st->last_updated.tv_sec = now;
}
// make sure the database is aligned
@@ -603,10 +616,10 @@ RRDSET *rrdset_create_custom(
st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
}
- st->plugin_name = plugin?strdup(plugin):NULL;
- st->module_name = module?strdup(module):NULL;
+ st->plugin_name = plugin?strdupz(plugin):NULL;
+ st->module_name = module?strdupz(module):NULL;
- st->config_section = strdup(config_section);
+ st->config_section = strdupz(config_section);
st->rrdhost = host;
st->memsize = size;
st->entries = entries;
@@ -644,7 +657,12 @@ RRDSET *rrdset_create_custom(
rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
rrdset_flag_clear(st, RRDSET_FLAG_DEBUG);
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
- rrdset_flag_clear(st, RRDSET_FLAG_EXPOSED_UPSTREAM);
+ rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND);
+ rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);
+ rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+ rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
// if(!strcmp(st->id, "disk_util.dm-0")) {
// st->debug = 1;
@@ -710,7 +728,7 @@ RRDSET *rrdset_create_custom(
// RRDSET - data collection iteration control
inline void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds) {
- if(unlikely(!st->last_collected_time.tv_sec || !microseconds)) {
+ if(unlikely(!st->last_collected_time.tv_sec || !microseconds || (rrdset_flag_check_noatomic(st, RRDSET_FLAG_SYNC_CLOCK)))) {
// call the full next_usec() function
rrdset_next_usec(st, microseconds);
return;
@@ -723,13 +741,36 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
struct timeval now;
now_realtime_timeval(&now);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ char *discard_reason = NULL;
+ usec_t discarded = microseconds;
+ #endif
+
+ if(unlikely(rrdset_flag_check_noatomic(st, RRDSET_FLAG_SYNC_CLOCK))) {
+ // the chart needs to be re-synced to current time
+ rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK);
+
+ // discard the microseconds supplied
+ microseconds = 0;
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(!discard_reason) discard_reason = "SYNC CLOCK FLAG";
+ #endif
+ }
+
if(unlikely(!st->last_collected_time.tv_sec)) {
// the first entry
microseconds = st->update_every * USEC_PER_SEC;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(!discard_reason) discard_reason = "FIRST DATA COLLECTION";
+ #endif
}
else if(unlikely(!microseconds)) {
// no dt given by the plugin
microseconds = dt_usec(&now, &st->last_collected_time);
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(!discard_reason) discard_reason = "NO USEC GIVEN BY COLLECTOR";
+ #endif
}
else {
// microseconds has the time since the last collection
@@ -748,18 +789,52 @@ inline void rrdset_next_usec(RRDSET *st, usec_t microseconds) {
last_updated_time_align(st);
microseconds = st->update_every * USEC_PER_SEC;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(!discard_reason) discard_reason = "COLLECTION TIME IN FUTURE";
+ #endif
}
- else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 10 * USEC_PER_SEC))) {
+ else if(unlikely((usec_t)since_last_usec > (usec_t)(st->update_every * 5 * USEC_PER_SEC))) {
// oops! the database is too far behind
info("RRD database for chart '%s' on host '%s' is %0.5" LONG_DOUBLE_MODIFIER " secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", st->id, st->rrdhost->hostname, (LONG_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);
microseconds = (usec_t)since_last_usec;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(!discard_reason) discard_reason = "COLLECTION TIME TOO FAR IN THE PAST";
+ #endif
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(since_last_usec > 0 && (susec_t)microseconds < since_last_usec) {
+ static __thread susec_t min_delta = USEC_PER_SEC * 3600, permanent_min_delta = 0;
+ static __thread time_t last_t = 0;
+
+ // the first time initialize it so that it will make the check later
+ if(last_t == 0) last_t = now.tv_sec + 60;
+
+ susec_t delta = since_last_usec - (susec_t)microseconds;
+ if(delta < min_delta) min_delta = delta;
+
+ if(now.tv_sec >= last_t + 60) {
+ last_t = now.tv_sec;
+
+ if(min_delta > permanent_min_delta) {
+ info("MINIMUM MICROSECONDS DELTA of thread %d increased from %lld to %lld (+%lld)", gettid(), permanent_min_delta, min_delta, min_delta - permanent_min_delta);
+ permanent_min_delta = min_delta;
+ }
+
+ min_delta = USEC_PER_SEC * 3600;
+ }
}
+#endif
}
#ifdef NETDATA_INTERNAL_CHECKS
debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds);
rrdset_debug(st, "NEXT: %llu microseconds", microseconds);
+
+ if(discarded && discarded != microseconds)
+ info("host '%s', chart '%s': discarded data collection time of %llu usec, replaced with %llu usec, reason: '%s'", st->rrdhost->hostname, st->id, discarded, microseconds, discard_reason?discard_reason:"UNDEFINED");
+
#endif
st->usec_since_last_update = microseconds;
@@ -857,6 +932,8 @@ static inline size_t rrdset_done_interpolate(
size_t stored_entries = 0; // the number of entries we have stored in the db, during this call to rrdset_done()
usec_t first_ut = last_stored_ut, last_ut = 0;
+ (void)first_ut;
+
ssize_t iterations = (ssize_t)((now_collect_ut - last_stored_ut) / (update_every_ut));
if((now_collect_ut % (update_every_ut)) == 0) iterations++;
diff --git a/src/rrdsetvar.c b/database/rrdsetvar.c
index aec57efa9..1bb883f0b 100644
--- a/src/rrdsetvar.c
+++ b/database/rrdsetvar.c
@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+#include "rrd.h"
// ----------------------------------------------------------------------------
// RRDSETVAR management
@@ -43,6 +45,10 @@ static inline void rrdsetvar_create_variables(RRDSETVAR *rs) {
RRDSET *st = rs->rrdset;
RRDHOST *host = st->rrdhost;
+ RRDVAR_OPTIONS options = rs->options;
+ if(rs->options & RRDVAR_OPTION_ALLOCATED)
+ options &= ~ RRDVAR_OPTION_ALLOCATED;
+
// ------------------------------------------------------------------------
// free the old ones (if any)
@@ -60,17 +66,17 @@ static inline void rrdsetvar_create_variables(RRDSETVAR *rs) {
// ------------------------------------------------------------------------
// CHART
- rs->var_local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->variable, rs->type, rs->value);
+ rs->var_local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->variable, rs->type, options, rs->value);
// ------------------------------------------------------------------------
// FAMILY
- rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, rs->value);
- rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, rs->value);
+ rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);
+ rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);
// ------------------------------------------------------------------------
// HOST
- rs->var_host = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullid, rs->type, rs->value);
- rs->var_host_name = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullname, rs->type, rs->value);
+ rs->var_host = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);
+ rs->var_host_name = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);
}
RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options) {
@@ -143,7 +149,7 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
for(rs = st->variables; rs ; rs = rs->next) {
if(hash == rs->hash && strcmp(n, rs->variable) == 0) {
rrdset_unlock(st);
- if(rs->options & RRDVAR_OPTION_ALLOCATED) {
+ if(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) {
free(n);
return rs;
}
@@ -160,16 +166,16 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)
calculated_number *v = mallocz(sizeof(calculated_number));
*v = NAN;
- rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED);
+ rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR);
rrdset_unlock(st);
- free(n);
+ freez(n);
return rs;
}
void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value) {
- if(unlikely(!(rs->options & RRDVAR_OPTION_ALLOCATED))) {
- error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);
+ if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) {
+ error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);
}
else {
calculated_number *v = rs->value;
@@ -177,7 +183,7 @@ void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value)
*v = value;
// mark the chart to be sent upstream
- rrdset_flag_clear(rs->rrdset, RRDSET_FLAG_EXPOSED_UPSTREAM);
+ rrdset_flag_clear(rs->rrdset, RRDSET_FLAG_UPSTREAM_EXPOSED);
}
}
}
diff --git a/database/rrdsetvar.h b/database/rrdsetvar.h
new file mode 100644
index 000000000..34a26d2f0
--- /dev/null
+++ b/database/rrdsetvar.h
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RRDSETVAR_H
+#define NETDATA_RRDSETVAR_H 1
+
+#include "rrd.h"
+
+// variables linked to charts
+// We link variables to point to the values that are already
+// calculated / processed by the normal data collection process
+// This means, there will be no speed penalty for using
+// these variables
+
+struct rrdsetvar {
+ char *variable; // variable name
+ uint32_t hash; // variable name hash
+
+ char *key_fullid; // chart type.chart id.variable
+ char *key_fullname; // chart type.chart name.variable
+
+ RRDVAR_TYPE type;
+ void *value;
+
+ RRDVAR_OPTIONS options;
+
+ RRDVAR *var_local;
+ RRDVAR *var_family;
+ RRDVAR *var_host;
+ RRDVAR *var_family_name;
+ RRDVAR *var_host_name;
+
+ struct rrdset *rrdset;
+
+ struct rrdsetvar *next;
+};
+
+extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name);
+extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, calculated_number value);
+
+extern void rrdsetvar_rename_all(RRDSET *st);
+extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options);
+extern void rrdsetvar_free(RRDSETVAR *rs);
+
+#endif //NETDATA_RRDSETVAR_H
diff --git a/src/rrdvar.c b/database/rrdvar.c
index 6936c36f1..951a38cac 100644
--- a/src/rrdvar.c
+++ b/database/rrdvar.c
@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+#include "rrd.h"
// ----------------------------------------------------------------------------
// RRDVAR management
@@ -59,14 +61,14 @@ inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) {
error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rv->name, host->hostname);
}
- if(rv->type == RRDVAR_TYPE_CALCULATED_ALLOCATED)
+ if(rv->options & RRDVAR_OPTION_ALLOCATED)
freez(rv->value);
freez(rv->name);
freez(rv);
}
-inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, void *value) {
+inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value) {
char *variable = strdupz(name);
rrdvar_fix_name(variable);
uint32_t hash = simple_hash(variable);
@@ -79,7 +81,9 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, c
rv->name = variable;
rv->hash = hash;
rv->type = type;
+ rv->options = options;
rv->value = value;
+ rv->last_updated = now_realtime_sec();
RRDVAR *ret = rrdvar_index_add(tree, rv);
if(unlikely(ret != rv)) {
@@ -106,7 +110,7 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, c
}
void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock) {
- // FIXME: this is not bullet proof - avl should support some means to destroy it
+ // This is not bullet proof - avl should support some means to destroy it
// with a callback for each item already in the index
RRDVAR *rv, *last = NULL;
@@ -123,7 +127,7 @@ void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock) {
// ----------------------------------------------------------------------------
// CUSTOM HOST VARIABLES
-inline int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data) {
+inline int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void * /*rrdvar*/, void * /*data*/), void *data) {
return avl_traverse_lock(&host->rrdvar_root_index, callback, data);
}
@@ -131,7 +135,7 @@ static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *t
calculated_number *v = callocz(1, sizeof(calculated_number));
*v = NAN;
- RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED_ALLOCATED, v);
+ RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v);
if(unlikely(!rv)) {
free(v);
debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", name);
@@ -140,6 +144,7 @@ static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *t
rrdvar_fix_name(variable);
uint32_t hash = simple_hash(variable);
+ // find the existing one to return it
rv = rrdvar_index_find(tree_lock, variable, hash);
freez(variable);
@@ -153,25 +158,30 @@ RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) {
}
void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value) {
- if(rv->type != RRDVAR_TYPE_CALCULATED_ALLOCATED)
+ if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED))
error("requested to set variable '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rv->name, value);
else {
calculated_number *v = rv->value;
if(*v != value) {
*v = value;
+ rv->last_updated = now_realtime_sec();
+
// if the host is streaming, send this variable upstream immediately
rrdpush_sender_send_this_host_variable_now(host, rv);
}
}
}
+int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR * /*rv*/, void * /*data*/), void *data) {
+ return avl_traverse_lock(&host->rrdvar_root_index, (int (*)(void *, void *))callback, data);
+}
+
// ----------------------------------------------------------------------------
// RRDVAR lookup
-static calculated_number rrdvar2number(RRDVAR *rv) {
+calculated_number rrdvar2number(RRDVAR *rv) {
switch(rv->type) {
- case RRDVAR_TYPE_CALCULATED_ALLOCATED:
case RRDVAR_TYPE_CALCULATED: {
calculated_number *n = (calculated_number *)rv->value;
return *n;
diff --git a/database/rrdvar.h b/database/rrdvar.h
new file mode 100644
index 000000000..6d1461b2a
--- /dev/null
+++ b/database/rrdvar.h
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RRDVAR_H
+#define NETDATA_RRDVAR_H 1
+
+#include "libnetdata/libnetdata.h"
+
+extern int rrdvar_compare(void *a, void *b);
+
+typedef enum rrdvar_type {
+ RRDVAR_TYPE_CALCULATED = 1,
+ RRDVAR_TYPE_TIME_T = 2,
+ RRDVAR_TYPE_COLLECTED = 3,
+ RRDVAR_TYPE_TOTAL = 4,
+ RRDVAR_TYPE_INT = 5
+} RRDVAR_TYPE;
+
+typedef enum rrdvar_options {
+ RRDVAR_OPTION_DEFAULT = 0,
+ RRDVAR_OPTION_ALLOCATED = (1 << 0), // the value ptr is allocated (not a reference)
+ RRDVAR_OPTION_CUSTOM_HOST_VAR = (1 << 1), // this is a custom host variable, not associated with a dimension
+ RRDVAR_OPTION_CUSTOM_CHART_VAR = (1 << 2), // this is a custom chart variable, not associated with a dimension
+ RRDVAR_OPTION_RRDCALC_LOCAL_VAR = (1 << 3), // this is a an alarm variable, attached to a chart
+ RRDVAR_OPTION_RRDCALC_FAMILY_VAR = (1 << 4), // this is a an alarm variable, attached to a family
+ RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR = (1 << 5), // this is a an alarm variable, attached to the host, using the chart id
+ RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR = (1 << 6), // this is a an alarm variable, attached to the host, using the chart name
+} RRDVAR_OPTIONS;
+
+// the variables as stored in the variables indexes
+// there are 3 indexes:
+// 1. at each chart (RRDSET.rrdvar_root_index)
+// 2. at each context (RRDFAMILY.rrdvar_root_index)
+// 3. at each host (RRDHOST.rrdvar_root_index)
+struct rrdvar {
+ avl avl;
+
+ char *name;
+ uint32_t hash;
+
+ RRDVAR_TYPE type;
+ RRDVAR_OPTIONS options;
+
+ void *value;
+
+ time_t last_updated;
+};
+
+#define RRDVAR_MAX_LENGTH 1024
+
+extern int rrdvar_fix_name(char *variable);
+
+#include "rrd.h"
+
+extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name);
+extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value);
+extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data);
+extern void rrdvar_free_remaining_variables(RRDHOST *host, avl_tree_lock *tree_lock);
+
+extern int rrdvar_callback_for_all_host_variables(RRDHOST *host, int (*callback)(void *rrdvar, void *data), void *data);
+
+extern calculated_number rrdvar2number(RRDVAR *rv);
+
+extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value);
+extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv);
+
+#endif //NETDATA_RRDVAR_H