summaryrefslogtreecommitdiffstats
path: root/health
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2018-11-07 12:22:44 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2018-11-07 12:22:44 +0000
commit1e6c93250172946eeb38e94a92a1fd12c9d3011e (patch)
tree8ca5e16dfc7ad6b3bf2738ca0a48408a950f8f7e /health
parentUpdate watch file (diff)
downloadnetdata-1e6c93250172946eeb38e94a92a1fd12c9d3011e.tar.xz
netdata-1e6c93250172946eeb38e94a92a1fd12c9d3011e.zip
Merging upstream version 1.11.0+dfsg.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--health/Makefile.am82
-rw-r--r--health/Makefile.in800
-rw-r--r--health/README.md657
-rw-r--r--health/health.c (renamed from src/health.c)35
-rw-r--r--health/health.d/adaptec_raid.conf24
-rw-r--r--health/health.d/apache.conf (renamed from conf.d/health.d/apache.conf)0
-rw-r--r--health/health.d/apcupsd.conf40
-rw-r--r--health/health.d/backend.conf (renamed from conf.d/health.d/backend.conf)0
-rw-r--r--health/health.d/bcache.conf22
-rw-r--r--health/health.d/beanstalkd.conf (renamed from conf.d/health.d/beanstalkd.conf)0
-rw-r--r--health/health.d/bind_rndc.conf (renamed from conf.d/health.d/bind_rndc.conf)0
-rw-r--r--health/health.d/boinc.conf62
-rw-r--r--health/health.d/btrfs.conf (renamed from conf.d/health.d/btrfs.conf)0
-rw-r--r--health/health.d/ceph.conf (renamed from conf.d/health.d/ceph.conf)0
-rw-r--r--health/health.d/couchdb.conf (renamed from conf.d/health.d/couchdb.conf)0
-rw-r--r--health/health.d/cpu.conf (renamed from conf.d/health.d/cpu.conf)0
-rw-r--r--health/health.d/disks.conf (renamed from conf.d/health.d/disks.conf)0
-rw-r--r--health/health.d/dockerd.conf8
-rw-r--r--health/health.d/elasticsearch.conf (renamed from conf.d/health.d/elasticsearch.conf)0
-rw-r--r--health/health.d/entropy.conf (renamed from conf.d/health.d/entropy.conf)0
-rw-r--r--health/health.d/fping.conf (renamed from conf.d/health.d/fping.conf)0
-rw-r--r--health/health.d/fronius.conf (renamed from conf.d/health.d/fronius.conf)0
-rw-r--r--health/health.d/haproxy.conf (renamed from conf.d/health.d/haproxy.conf)0
-rw-r--r--health/health.d/httpcheck.conf (renamed from conf.d/health.d/httpcheck.conf)0
-rw-r--r--health/health.d/ipc.conf (renamed from conf.d/health.d/ipc.conf)4
-rw-r--r--health/health.d/ipfs.conf (renamed from conf.d/health.d/ipfs.conf)0
-rw-r--r--health/health.d/ipmi.conf (renamed from conf.d/health.d/ipmi.conf)0
-rw-r--r--health/health.d/isc_dhcpd.conf (renamed from conf.d/health.d/isc_dhcpd.conf)0
-rw-r--r--health/health.d/lighttpd.conf (renamed from conf.d/health.d/lighttpd.conf)0
-rw-r--r--health/health.d/linux_power_supply.conf12
-rw-r--r--health/health.d/load.conf56
-rw-r--r--health/health.d/mdstat.conf (renamed from conf.d/health.d/mdstat.conf)23
-rw-r--r--health/health.d/megacli.conf48
-rw-r--r--health/health.d/memcached.conf (renamed from conf.d/health.d/memcached.conf)0
-rw-r--r--health/health.d/memory.conf (renamed from conf.d/health.d/memory.conf)0
-rw-r--r--health/health.d/mongodb.conf (renamed from conf.d/health.d/mongodb.conf)0
-rw-r--r--health/health.d/mysql.conf (renamed from conf.d/health.d/mysql.conf)15
-rw-r--r--health/health.d/named.conf (renamed from conf.d/health.d/named.conf)0
-rw-r--r--health/health.d/net.conf (renamed from conf.d/health.d/net.conf)37
-rw-r--r--health/health.d/netfilter.conf (renamed from conf.d/health.d/netfilter.conf)2
-rw-r--r--health/health.d/nginx.conf (renamed from conf.d/health.d/nginx.conf)0
-rw-r--r--health/health.d/nginx_plus.conf (renamed from conf.d/health.d/nginx_plus.conf)0
-rw-r--r--health/health.d/portcheck.conf (renamed from conf.d/health.d/portcheck.conf)0
-rw-r--r--health/health.d/postgres.conf (renamed from conf.d/health.d/postgres.conf)0
-rw-r--r--health/health.d/qos.conf (renamed from conf.d/health.d/qos.conf)0
-rw-r--r--health/health.d/ram.conf (renamed from conf.d/health.d/ram.conf)6
-rw-r--r--health/health.d/redis.conf (renamed from conf.d/health.d/redis.conf)0
-rw-r--r--health/health.d/retroshare.conf (renamed from conf.d/health.d/retroshare.conf)0
-rw-r--r--health/health.d/softnet.conf (renamed from conf.d/health.d/softnet.conf)0
-rw-r--r--health/health.d/squid.conf (renamed from conf.d/health.d/squid.conf)0
-rw-r--r--health/health.d/stiebeleltron.conf (renamed from conf.d/health.d/stiebeleltron.conf)0
-rw-r--r--health/health.d/swap.conf (renamed from conf.d/health.d/swap.conf)0
-rw-r--r--health/health.d/tcp_conn.conf (renamed from conf.d/health.d/tcp_conn.conf)0
-rw-r--r--health/health.d/tcp_listen.conf82
-rw-r--r--health/health.d/tcp_mem.conf (renamed from conf.d/health.d/tcp_mem.conf)0
-rw-r--r--health/health.d/tcp_orphans.conf (renamed from conf.d/health.d/tcp_orphans.conf)0
-rw-r--r--health/health.d/tcp_resets.conf (renamed from conf.d/health.d/tcp_resets.conf)0
-rw-r--r--health/health.d/udp_errors.conf (renamed from conf.d/health.d/udp_errors.conf)4
-rw-r--r--health/health.d/varnish.conf (renamed from conf.d/health.d/varnish.conf)0
-rw-r--r--health/health.d/web_log.conf (renamed from conf.d/health.d/web_log.conf)0
-rw-r--r--health/health.d/zfs.conf (renamed from conf.d/health.d/zfs.conf)0
-rw-r--r--health/health.h76
-rw-r--r--health/health_config.c (renamed from src/health_config.c)256
-rw-r--r--health/health_json.c (renamed from src/health_json.c)5
-rw-r--r--health/health_log.c (renamed from src/health_log.c)5
-rw-r--r--health/notifications/Makefile.am45
-rw-r--r--health/notifications/Makefile.in754
-rw-r--r--health/notifications/README.md60
-rwxr-xr-xhealth/notifications/alarm-email.sh (renamed from plugins.d/alarm-email.sh)1
-rw-r--r--[-rwxr-xr-x]health/notifications/alarm-notify.sh (renamed from plugins.d/alarm-notify.sh)521
-rwxr-xr-xhealth/notifications/alarm-notify.sh.in2378
-rwxr-xr-xhealth/notifications/alarm-test.sh (renamed from plugins.d/alarm-test.sh)4
-rw-r--r--health/notifications/alerta/Makefile.inc12
-rw-r--r--health/notifications/alerta/README.md236
-rw-r--r--health/notifications/awssns/Makefile.inc12
-rw-r--r--health/notifications/awssns/README.md31
-rw-r--r--health/notifications/discord/Makefile.inc12
-rw-r--r--health/notifications/discord/README.md44
-rw-r--r--health/notifications/email/Makefile.inc12
-rw-r--r--health/notifications/email/README.md31
-rw-r--r--health/notifications/flock/Makefile.inc12
-rw-r--r--health/notifications/flock/README.md31
-rwxr-xr-xhealth/notifications/health_alarm_notify.conf (renamed from conf.d/health_alarm_notify.conf)253
-rw-r--r--health/notifications/health_email_recipients.conf (renamed from conf.d/health_email_recipients.conf)0
-rw-r--r--health/notifications/irc/Makefile.inc12
-rw-r--r--health/notifications/irc/README.md73
-rw-r--r--health/notifications/kavenegar/Makefile.inc12
-rw-r--r--health/notifications/kavenegar/README.md39
-rw-r--r--health/notifications/messagebird/Makefile.inc12
-rw-r--r--health/notifications/messagebird/README.md38
-rw-r--r--health/notifications/pagerduty/Makefile.inc12
-rw-r--r--health/notifications/pagerduty/README.md34
-rw-r--r--health/notifications/pushbullet/Makefile.inc12
-rw-r--r--health/notifications/pushbullet/README.md42
-rw-r--r--health/notifications/pushover/Makefile.inc12
-rw-r--r--health/notifications/pushover/README.md17
-rw-r--r--health/notifications/rocketchat/Makefile.inc12
-rw-r--r--health/notifications/rocketchat/README.md46
-rw-r--r--health/notifications/slack/Makefile.inc12
-rw-r--r--health/notifications/slack/README.md45
-rw-r--r--health/notifications/syslog/Makefile.inc12
-rw-r--r--health/notifications/syslog/README.md23
-rw-r--r--health/notifications/telegram/Makefile.inc12
-rw-r--r--health/notifications/telegram/README.md19
-rw-r--r--health/notifications/twilio/Makefile.inc12
-rw-r--r--health/notifications/twilio/README.md40
-rw-r--r--health/notifications/web/Makefile.inc12
-rw-r--r--health/notifications/web/README.md6
108 files changed, 7162 insertions, 214 deletions
diff --git a/health/Makefile.am b/health/Makefile.am
new file mode 100644
index 00000000..829a41b3
--- /dev/null
+++ b/health/Makefile.am
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+SUBDIRS = \
+ notifications \
+ $(NULL)
+
+CLEANFILES = \
+ $(NULL)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
+
+userhealthconfigdir=$(configdir)/health.d
+dist_userhealthconfig_DATA = \
+ $(top_srcdir)/installer/.keep \
+ $(NULL)
+
+healthconfigdir=$(libconfigdir)/health.d
+dist_healthconfig_DATA = \
+ $(top_srcdir)/installer/.keep \
+ health.d/adaptec_raid.conf \
+ health.d/apache.conf \
+ health.d/apcupsd.conf \
+ health.d/backend.conf \
+ health.d/bcache.conf \
+ health.d/beanstalkd.conf \
+ health.d/bind_rndc.conf \
+ health.d/boinc.conf \
+ health.d/btrfs.conf \
+ health.d/ceph.conf \
+ health.d/cpu.conf \
+ health.d/couchdb.conf \
+ health.d/disks.conf \
+ health.d/dockerd.conf \
+ health.d/elasticsearch.conf \
+ health.d/entropy.conf \
+ health.d/fping.conf \
+ health.d/fronius.conf \
+ health.d/haproxy.conf \
+ health.d/httpcheck.conf \
+ health.d/ipc.conf \
+ health.d/ipfs.conf \
+ health.d/ipmi.conf \
+ health.d/isc_dhcpd.conf \
+ health.d/lighttpd.conf \
+ health.d/linux_power_supply.conf \
+ health.d/load.conf \
+ health.d/mdstat.conf \
+ health.d/megacli.conf \
+ health.d/memcached.conf \
+ health.d/memory.conf \
+ health.d/mongodb.conf \
+ health.d/mysql.conf \
+ health.d/named.conf \
+ health.d/net.conf \
+ health.d/netfilter.conf \
+ health.d/nginx.conf \
+ health.d/nginx_plus.conf \
+ health.d/portcheck.conf \
+ health.d/postgres.conf \
+ health.d/qos.conf \
+ health.d/ram.conf \
+ health.d/redis.conf \
+ health.d/retroshare.conf \
+ health.d/softnet.conf \
+ health.d/squid.conf \
+ health.d/stiebeleltron.conf \
+ health.d/swap.conf \
+ health.d/tcp_conn.conf \
+ health.d/tcp_listen.conf \
+ health.d/tcp_mem.conf \
+ health.d/tcp_orphans.conf \
+ health.d/tcp_resets.conf \
+ health.d/udp_errors.conf \
+ health.d/varnish.conf \
+ health.d/web_log.conf \
+ health.d/zfs.conf \
+ $(NULL)
diff --git a/health/Makefile.in b/health/Makefile.in
new file mode 100644
index 00000000..811f7f23
--- /dev/null
+++ b/health/Makefile.in
@@ -0,0 +1,800 @@
+# Makefile.in generated by automake 1.14.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2013 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+VPATH = @srcdir@
+am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)'
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = health
+DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
+ $(dist_healthconfig_DATA) $(dist_noinst_DATA) \
+ $(dist_userhealthconfig_DATA)
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/build/m4/ax_c___atomic.m4 \
+ $(top_srcdir)/build/m4/ax_c__generic.m4 \
+ $(top_srcdir)/build/m4/ax_c_lto.m4 \
+ $(top_srcdir)/build/m4/ax_c_mallinfo.m4 \
+ $(top_srcdir)/build/m4/ax_c_mallopt.m4 \
+ $(top_srcdir)/build/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/build/m4/ax_gcc_func_attribute.m4 \
+ $(top_srcdir)/build/m4/ax_pthread.m4 \
+ $(top_srcdir)/build/m4/jemalloc.m4 \
+ $(top_srcdir)/build/m4/tcmalloc.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
+SOURCES =
+DIST_SOURCES =
+RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \
+ ctags-recursive dvi-recursive html-recursive info-recursive \
+ install-data-recursive install-dvi-recursive \
+ install-exec-recursive install-html-recursive \
+ install-info-recursive install-pdf-recursive \
+ install-ps-recursive install-recursive installcheck-recursive \
+ installdirs-recursive pdf-recursive ps-recursive \
+ tags-recursive uninstall-recursive
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+ test -z "$$files" \
+ || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+ $(am__cd) "$$dir" && rm -f $$files; }; \
+ }
+am__installdirs = "$(DESTDIR)$(healthconfigdir)" \
+ "$(DESTDIR)$(userhealthconfigdir)"
+DATA = $(dist_healthconfig_DATA) $(dist_noinst_DATA) \
+ $(dist_userhealthconfig_DATA)
+RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \
+ distclean-recursive maintainer-clean-recursive
+am__recursive_targets = \
+ $(RECURSIVE_TARGETS) \
+ $(RECURSIVE_CLEAN_TARGETS) \
+ $(am__extra_recursive_targets)
+AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \
+ distdir
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates. Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+ BEGIN { nonempty = 0; } \
+ { items[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique. This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+ list='$(am__tagged_files)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+DIST_SUBDIRS = $(SUBDIRS)
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+am__relativize = \
+ dir0=`pwd`; \
+ sed_first='s,^\([^/]*\)/.*$$,\1,'; \
+ sed_rest='s,^[^/]*/*,,'; \
+ sed_last='s,^.*/\([^/]*\)$$,\1,'; \
+ sed_butlast='s,/*[^/]*$$,,'; \
+ while test -n "$$dir1"; do \
+ first=`echo "$$dir1" | sed -e "$$sed_first"`; \
+ if test "$$first" != "."; then \
+ if test "$$first" = ".."; then \
+ dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \
+ dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \
+ else \
+ first2=`echo "$$dir2" | sed -e "$$sed_first"`; \
+ if test "$$first2" = "$$first"; then \
+ dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \
+ else \
+ dir2="../$$dir2"; \
+ fi; \
+ dir0="$$dir0"/"$$first"; \
+ fi; \
+ fi; \
+ dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \
+ done; \
+ reldir="$$dir2"
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+IPMIMONITORING_CFLAGS = @IPMIMONITORING_CFLAGS@
+IPMIMONITORING_LIBS = @IPMIMONITORING_LIBS@
+LDFLAGS = @LDFLAGS@
+LIBCAP_CFLAGS = @LIBCAP_CFLAGS@
+LIBCAP_LIBS = @LIBCAP_LIBS@
+LIBMNL_CFLAGS = @LIBMNL_CFLAGS@
+LIBMNL_LIBS = @LIBMNL_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LTLIBOBJS = @LTLIBOBJS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MATH_CFLAGS = @MATH_CFLAGS@
+MATH_LIBS = @MATH_LIBS@
+MKDIR_P = @MKDIR_P@
+NFACCT_CFLAGS = @NFACCT_CFLAGS@
+NFACCT_LIBS = @NFACCT_LIBS@
+OBJEXT = @OBJEXT@
+OPTIONAL_IPMIMONITORING_CFLAGS = @OPTIONAL_IPMIMONITORING_CFLAGS@
+OPTIONAL_IPMIMONITORING_LIBS = @OPTIONAL_IPMIMONITORING_LIBS@
+OPTIONAL_LIBCAP_CFLAGS = @OPTIONAL_LIBCAP_CFLAGS@
+OPTIONAL_LIBCAP_LIBS = @OPTIONAL_LIBCAP_LIBS@
+OPTIONAL_MATH_CLFAGS = @OPTIONAL_MATH_CLFAGS@
+OPTIONAL_MATH_LIBS = @OPTIONAL_MATH_LIBS@
+OPTIONAL_NFACCT_CLFAGS = @OPTIONAL_NFACCT_CLFAGS@
+OPTIONAL_NFACCT_LIBS = @OPTIONAL_NFACCT_LIBS@
+OPTIONAL_UUID_CLFAGS = @OPTIONAL_UUID_CLFAGS@
+OPTIONAL_UUID_LIBS = @OPTIONAL_UUID_LIBS@
+OPTIONAL_ZLIB_CLFAGS = @OPTIONAL_ZLIB_CLFAGS@
+OPTIONAL_ZLIB_LIBS = @OPTIONAL_ZLIB_LIBS@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_RPM_RELEASE = @PACKAGE_RPM_RELEASE@
+PACKAGE_RPM_VERSION = @PACKAGE_RPM_VERSION@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SSE_CANDIDATE = @SSE_CANDIDATE@
+STRIP = @STRIP@
+UUID_CFLAGS = @UUID_CFLAGS@
+UUID_LIBS = @UUID_LIBS@
+VERSION = @VERSION@
+ZLIB_CFLAGS = @ZLIB_CFLAGS@
+ZLIB_LIBS = @ZLIB_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_CC = @ac_ct_CC@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_target = @build_target@
+build_vendor = @build_vendor@
+builddir = @builddir@
+cachedir = @cachedir@
+chartsdir = @chartsdir@
+configdir = @configdir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+has_jemalloc = @has_jemalloc@
+has_tcmalloc = @has_tcmalloc@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libconfigdir = @libconfigdir@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+logdir = @logdir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+nodedir = @nodedir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+pluginsdir = @pluginsdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+pythondir = @pythondir@
+registrydir = @registrydir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+varlibdir = @varlibdir@
+webdir = @webdir@
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+SUBDIRS = \
+ notifications \
+ $(NULL)
+
+CLEANFILES = \
+ $(NULL)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
+
+userhealthconfigdir = $(configdir)/health.d
+dist_userhealthconfig_DATA = \
+ $(top_srcdir)/installer/.keep \
+ $(NULL)
+
+healthconfigdir = $(libconfigdir)/health.d
+dist_healthconfig_DATA = \
+ $(top_srcdir)/installer/.keep \
+ health.d/adaptec_raid.conf \
+ health.d/apache.conf \
+ health.d/apcupsd.conf \
+ health.d/backend.conf \
+ health.d/bcache.conf \
+ health.d/beanstalkd.conf \
+ health.d/bind_rndc.conf \
+ health.d/boinc.conf \
+ health.d/btrfs.conf \
+ health.d/ceph.conf \
+ health.d/cpu.conf \
+ health.d/couchdb.conf \
+ health.d/disks.conf \
+ health.d/dockerd.conf \
+ health.d/elasticsearch.conf \
+ health.d/entropy.conf \
+ health.d/fping.conf \
+ health.d/fronius.conf \
+ health.d/haproxy.conf \
+ health.d/httpcheck.conf \
+ health.d/ipc.conf \
+ health.d/ipfs.conf \
+ health.d/ipmi.conf \
+ health.d/isc_dhcpd.conf \
+ health.d/lighttpd.conf \
+ health.d/linux_power_supply.conf \
+ health.d/load.conf \
+ health.d/mdstat.conf \
+ health.d/megacli.conf \
+ health.d/memcached.conf \
+ health.d/memory.conf \
+ health.d/mongodb.conf \
+ health.d/mysql.conf \
+ health.d/named.conf \
+ health.d/net.conf \
+ health.d/netfilter.conf \
+ health.d/nginx.conf \
+ health.d/nginx_plus.conf \
+ health.d/portcheck.conf \
+ health.d/postgres.conf \
+ health.d/qos.conf \
+ health.d/ram.conf \
+ health.d/redis.conf \
+ health.d/retroshare.conf \
+ health.d/softnet.conf \
+ health.d/squid.conf \
+ health.d/stiebeleltron.conf \
+ health.d/swap.conf \
+ health.d/tcp_conn.conf \
+ health.d/tcp_listen.conf \
+ health.d/tcp_mem.conf \
+ health.d/tcp_orphans.conf \
+ health.d/tcp_resets.conf \
+ health.d/udp_errors.conf \
+ health.d/varnish.conf \
+ health.d/web_log.conf \
+ health.d/zfs.conf \
+ $(NULL)
+
+all: all-recursive
+
+.SUFFIXES:
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu health/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --gnu health/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+install-dist_healthconfigDATA: $(dist_healthconfig_DATA)
+ @$(NORMAL_INSTALL)
+ @list='$(dist_healthconfig_DATA)'; test -n "$(healthconfigdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(healthconfigdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(healthconfigdir)" || exit 1; \
+ fi; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(healthconfigdir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(healthconfigdir)" || exit $$?; \
+ done
+
+uninstall-dist_healthconfigDATA:
+ @$(NORMAL_UNINSTALL)
+ @list='$(dist_healthconfig_DATA)'; test -n "$(healthconfigdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ dir='$(DESTDIR)$(healthconfigdir)'; $(am__uninstall_files_from_dir)
+install-dist_userhealthconfigDATA: $(dist_userhealthconfig_DATA)
+ @$(NORMAL_INSTALL)
+ @list='$(dist_userhealthconfig_DATA)'; test -n "$(userhealthconfigdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(userhealthconfigdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(userhealthconfigdir)" || exit 1; \
+ fi; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(userhealthconfigdir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(userhealthconfigdir)" || exit $$?; \
+ done
+
+uninstall-dist_userhealthconfigDATA:
+ @$(NORMAL_UNINSTALL)
+ @list='$(dist_userhealthconfig_DATA)'; test -n "$(userhealthconfigdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ dir='$(DESTDIR)$(userhealthconfigdir)'; $(am__uninstall_files_from_dir)
+
+# This directory's subdirectories are mostly independent; you can cd
+# into them and run 'make' without going through this Makefile.
+# To change the values of 'make' variables: instead of editing Makefiles,
+# (1) if the variable is set in 'config.status', edit 'config.status'
+# (which will cause the Makefiles to be regenerated when you run 'make');
+# (2) otherwise, pass the desired values on the 'make' command line.
+$(am__recursive_targets):
+ @fail=; \
+ if $(am__make_keepgoing); then \
+ failcom='fail=yes'; \
+ else \
+ failcom='exit 1'; \
+ fi; \
+ dot_seen=no; \
+ target=`echo $@ | sed s/-recursive//`; \
+ case "$@" in \
+ distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \
+ *) list='$(SUBDIRS)' ;; \
+ esac; \
+ for subdir in $$list; do \
+ echo "Making $$target in $$subdir"; \
+ if test "$$subdir" = "."; then \
+ dot_seen=yes; \
+ local_target="$$target-am"; \
+ else \
+ local_target="$$target"; \
+ fi; \
+ ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \
+ || eval $$failcom; \
+ done; \
+ if test "$$dot_seen" = "no"; then \
+ $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \
+ fi; test -z "$$fail"
+
+ID: $(am__tagged_files)
+ $(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-recursive
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ set x; \
+ here=`pwd`; \
+ if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \
+ include_option=--etags-include; \
+ empty_fix=.; \
+ else \
+ include_option=--include; \
+ empty_fix=; \
+ fi; \
+ list='$(SUBDIRS)'; for subdir in $$list; do \
+ if test "$$subdir" = .; then :; else \
+ test ! -f $$subdir/TAGS || \
+ set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \
+ fi; \
+ done; \
+ $(am__define_uniq_tagged_files); \
+ shift; \
+ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+ test -n "$$unique" || unique=$$empty_fix; \
+ if test $$# -gt 0; then \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ "$$@" $$unique; \
+ else \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ $$unique; \
+ fi; \
+ fi
+ctags: ctags-recursive
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ $(am__define_uniq_tagged_files); \
+ test -z "$(CTAGS_ARGS)$$unique" \
+ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+ $$unique
+
+GTAGS:
+ here=`$(am__cd) $(top_builddir) && pwd` \
+ && $(am__cd) $(top_srcdir) \
+ && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-recursive
+
+cscopelist-am: $(am__tagged_files)
+ list='$(am__tagged_files)'; \
+ case "$(srcdir)" in \
+ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+ *) sdir=$(subdir)/$(srcdir) ;; \
+ esac; \
+ for i in $$list; do \
+ if test -f "$$i"; then \
+ echo "$(subdir)/$$i"; \
+ else \
+ echo "$$sdir/$$i"; \
+ fi; \
+ done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+ -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+ @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \
+ if test "$$subdir" = .; then :; else \
+ $(am__make_dryrun) \
+ || test -d "$(distdir)/$$subdir" \
+ || $(MKDIR_P) "$(distdir)/$$subdir" \
+ || exit 1; \
+ dir1=$$subdir; dir2="$(distdir)/$$subdir"; \
+ $(am__relativize); \
+ new_distdir=$$reldir; \
+ dir1=$$subdir; dir2="$(top_distdir)"; \
+ $(am__relativize); \
+ new_top_distdir=$$reldir; \
+ echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \
+ echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \
+ ($(am__cd) $$subdir && \
+ $(MAKE) $(AM_MAKEFLAGS) \
+ top_distdir="$$new_top_distdir" \
+ distdir="$$new_distdir" \
+ am__remove_distdir=: \
+ am__skip_length_check=: \
+ am__skip_mode_fix=: \
+ distdir) \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+check: check-recursive
+all-am: Makefile $(DATA)
+installdirs: installdirs-recursive
+installdirs-am:
+ for dir in "$(DESTDIR)$(healthconfigdir)" "$(DESTDIR)$(userhealthconfigdir)"; do \
+ test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+ done
+install: install-recursive
+install-exec: install-exec-recursive
+install-data: install-data-recursive
+uninstall: uninstall-recursive
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-recursive
+install-strip:
+ if test -z '$(STRIP)'; then \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ install; \
+ else \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+ fi
+mostlyclean-generic:
+
+clean-generic:
+ -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+ -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES)
+clean: clean-recursive
+
+clean-am: clean-generic mostlyclean-am
+
+distclean: distclean-recursive
+ -rm -f Makefile
+distclean-am: clean-am distclean-generic distclean-tags
+
+dvi: dvi-recursive
+
+dvi-am:
+
+html: html-recursive
+
+html-am:
+
+info: info-recursive
+
+info-am:
+
+install-data-am: install-dist_healthconfigDATA \
+ install-dist_userhealthconfigDATA
+
+install-dvi: install-dvi-recursive
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-recursive
+
+install-html-am:
+
+install-info: install-info-recursive
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-recursive
+
+install-pdf-am:
+
+install-ps: install-ps-recursive
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-recursive
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-recursive
+
+mostlyclean-am: mostlyclean-generic
+
+pdf: pdf-recursive
+
+pdf-am:
+
+ps: ps-recursive
+
+ps-am:
+
+uninstall-am: uninstall-dist_healthconfigDATA \
+ uninstall-dist_userhealthconfigDATA
+
+.MAKE: $(am__recursive_targets) install-am install-strip
+
+.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am check \
+ check-am clean clean-generic cscopelist-am ctags ctags-am \
+ distclean distclean-generic distclean-tags distdir dvi dvi-am \
+ html html-am info info-am install install-am install-data \
+ install-data-am install-dist_healthconfigDATA \
+ install-dist_userhealthconfigDATA install-dvi install-dvi-am \
+ install-exec install-exec-am install-html install-html-am \
+ install-info install-info-am install-man install-pdf \
+ install-pdf-am install-ps install-ps-am install-strip \
+ installcheck installcheck-am installdirs installdirs-am \
+ maintainer-clean maintainer-clean-generic mostlyclean \
+ mostlyclean-generic pdf pdf-am ps ps-am tags tags-am uninstall \
+ uninstall-am uninstall-dist_healthconfigDATA \
+ uninstall-dist_userhealthconfigDATA
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/health/README.md b/health/README.md
new file mode 100644
index 00000000..597bd3c3
--- /dev/null
+++ b/health/README.md
@@ -0,0 +1,657 @@
+
+# Health monitoring
+
+Each netdata node runs an independent thread evaluating health monitoring checks.
+This thread has lock free access to the database, so that it can operate as a watchdog.
+
+Health checks (alarms) are attached to netdata charts, allowing netdata to automatically
+activate an alarm as soon as a chart is created. This is very important for
+netdata, since many charts are dynamically created during runtime (for example, the
+chart tracking network interface packet drops, is automatically created on the first
+packet dropped).
+
+Netdata also supports alarm **templates**, so that an alarm can be attached to all
+the charts of the same context (i.e. all network interfaces, or all disks, or all mysql servers, etc.)
+
+Each alarm can execute a single query to the database using statistical algorithms against past data,
+but alarms can be combined. So, if you need 2 queries in the database, you can combine
+2 alarms together (both will run a query to the database, and the results can be combined).
+
+Each alarm has unlimited access to all the metrics collected. So, a single alarm can
+use expressions combining the latest value of any number of metrics.
+
+## Health configuration reference
+
+Stock netdata health configuration is in `/usr/lib/netdata/conf.d/health.d`.
+These files can be overwritten by copying them and editing them in `/etc/netdata/health.d`
+(run `/etc/netdata/edit-config` to edit them).
+
+In `/etc/netdata/health.d` you can also put any number of files (in any number of sub-directories)
+with a suffix `.conf` to have them processed by netdata.
+
+Health configuration can be reloaded at any time, without restarting netdata.
+Just send netdata the SIGUSR2 signal, like this:
+
+```sh
+killall -USR2 netdata
+```
+
+### Entities in the health files
+
+There are 2 entities:
+
+1. **alarms**, which are attached to specific charts, and
+
+2. **templates**, which define rules that should be applied to all charts having a
+ specific `context`. You can use this feature to apply **alarms** to all disks,
+ all network interfaces, all mysql databases, all nginx web servers, etc.
+
+Both of these entities have exactly the same format and feature set.
+The only difference is the label `alarm` or `template`.
+
+netdata supports overriding **templates** with **alarms**.
+For example, when a template is defined for a set of charts, an alarm with exactly the
+same name attached to the same chart the template matches, will have higher precedence
+(i.e. netdata will use the alarm on this chart and prevent the template from being applied
+to it).
+
+### The format
+
+The following lines are parsed.
+
+#### alarm line `alarm` or `template`
+
+This line starts an alarm or alarm template.
+
+```
+alarm: NAME
+```
+
+or
+
+```
+template: NAME
+```
+
+This line has to be first on each alarm or template.
+`NAME` is anything you would like to name it (the only symbols allowed are `.` and `_`).
+
+---
+
+#### alarm line `on`
+
+This line defines the data the alarm should be attached to.
+
+For alarms:
+
+```
+on: CHART
+```
+
+For `CHART` you can use a chart `id` or `name` of the chart, as shown on the dashboard.
+
+For alarm templates:
+
+```
+on: CONTEXT
+```
+
+`CONTEXT` is the template of a chart. For example the charts `mysql_local.net` and
+`mysql_server2.net` have the same context: `mysql.net`. So, you can use this to apply
+alarms to all `mysql.net` charts.
+
+To find the `CONTEXT` of a chart hover over its date, above the legend. A tooltip will
+appear with this format `plugin:nodule, context`. For example, the bandwidth chart of
+a network interface says:
+
+```
+proc:/proc/dev/dev, net.net
+```
+
+So, `plugin = proc`, `module = /proc/net/dev` and `context = net.net`.
+
+---
+
+#### alarm line `os`
+
+This alarm or template will be used only if the O/S of the host loading it, matches this
+pattern list. The value is a space separated list of simple patterns (use `*` as wildcard,
+prefix with `!` for a negative match, order is important).
+
+```
+os: linux freebsd macos
+```
+
+---
+
+#### alarm line `hosts`
+
+This alarm or template will be used only if the hostname of the host loading it, matches
+this pattern list. The value is a space separated list of simple patterns (use `*` as wildcard,
+prefix with `!` for a negative match, order is important).
+
+```
+hosts: server1 server2 database* !redis3 redis*
+```
+
+The above says: use this alarm on all hosts named `server1`, `server2`, `database*`, and
+all `redis*` except `redis3`.
+
+This is useful when you centralize metrics from multiple hosts, to one netdata.
+
+---
+
+#### alarm line `families`
+
+This line is only used in alarm templates. It filters the charts. So, if you need to create
+an alarm template for a few of a kind of chart (a few of your disks, or a few of your network
+interfaces, or a few your mysql servers, etc), you can create an alarm template that would
+normally be applied to all of them, and filter them by family.
+
+The format is:
+
+```
+families: SIMPLE PATTERN LIST
+```
+
+Simple patterns list is a lists of space separated patterns. Use ` * ` as wildcard and ` ! `
+for a negative match. Processing is left to right, and on the first hit (positive or negative),
+processing stops.
+
+So. `families: *` means, match anything, while `families: !bad*pattern* *` means anything
+except `bad*pattern*` (where `*` is a wildcard to match any sequence of characters).
+
+The family of a chart is usually the submenu of the netdata dashboard it appears.
+
+---
+
+#### alarm line `lookup`
+
+This lines makes a database lookup to find a value. This result of this lookup is available as `$this`.
+
+The format is:
+
+```
+lookup: METHOD AFTER [at BEFORE] [every DURATION] [OPTIONS] [of DIMENSIONS]
+```
+
+Everything is the same with [badges](../web/api/badges/). In short:
+
+- `METHOD` is one of `average`, `min`, `max`, `sum`, `incremental-sum`.
+ This is required.
+
+- `AFTER` is a relative number of seconds, but it also accepts a single letter for changing
+ the units, like `-1s` = 1 second in the past, `-1m` = 1 minute in the past, `-1h` = 1 hour
+ in the past, `-1d` = 1 day in the past. You need a negative number (i.e. how far in the past
+ to look for the value). **This is required**.
+
+- `at BEFORE` is by default 0 and is not required. Using this you can define the end of the
+ lookup. So data will be evaluated between `AFTER` and `BEFORE`.
+
+- `every DURATION` sets the updated frequency of the lookup (supports single letter units as
+ above too).
+
+- `OPTIONS` is a space separated list of `percentage`, `absolute`, `min2max`, `unaligned`,
+ `match-ids`, `match-names`. Check the badges documentation for more info.
+
+- `of DIMENSIONS` is optional and has to be the last parameter. Dimensions have to be separated
+ by `,` or `|`. The space characters found in dimensions will be kept as-is (a few dimensions
+ have spaces in their names). This accepts netdata simple patterns and the `match-ids` and
+ `match-names` options affect the searches for dimensions.
+
+The result of the lookup will be available as `$this` and `$NAME` in expressions.
+The timestamps of the timeframe evaluated by the database lookup is available as variables
+`$after` and `$before` (both are unix timestamps).
+
+---
+
+#### alarm line `calc`
+
+This expression is evaluated just after the `lookup` (if any). Its purpose is to apply some
+calculation before using the value looked up from the db.
+
+You can also have an expression without a lookup, using other variables that are available.
+
+The result of the calculation will be available as `$this` in warning and critical expressions
+(overwriting the `lookup` one).
+
+Format:
+
+```
+calc: EXPRESSION
+```
+
+Check [Expressions](#expressions) for more information.
+
+---
+
+#### alarm line `every`
+
+Sets the update frequency of this alarm. This is the same to the `every DURATION` given
+in the `lookup` lines.
+
+Format:
+
+```
+every: DURATION
+```
+
+`DURATION` accepts `s` for seconds, `m` is minutes, `h` for hours, `d` for days.
+
+---
+
+#### alarm lines `green` and `red`
+
+Set the green and red thresholds of a chart. Both are available as `$green` and `$red` in
+expressions. If multiple alarms define different thresholds, the ones defined by the first
+alarm will be used. These will eventually visualized on the dashboard, so only one set of
+them is allowed. If you need multiple sets of them in different alarms, use absolute numbers
+instead of `$red` and `$green`.
+
+Format:
+
+```
+green: NUMBER
+red: NUMBER
+```
+
+---
+
+#### alarm lines `warn` and `crit`
+
+These expressions should evaluate to true or false (alternatively non-zero or zero).
+They trigger the alarm. Both are optional.
+
+Format:
+
+```
+warn: EXPRESSION
+crit: EXPRESSION
+```
+Check [Expressions](#expressions) for more information.
+
+---
+
+#### alarm line `to`
+
+This will be the first parameter of the script to be executed when the alarm switches status.
+Its meaning is left up to the `exec` script.
+
+The default `exec` script, `alarm-notify.sh`, uses this field as a space separated list of roles,
+which are then consulted to find the exact recipients per notification method.
+
+Format:
+
+```
+to: ROLE1 ROLE2 ROLE3 ...
+```
+
+---
+
+#### alarm line `exec`
+
+The script that will be executed when the alarm changes status.
+
+Format:
+
+```
+exec: SCRIPT
+```
+
+The default `SCRIPT` is netdata's `alarm-notify.sh`, which supports all the notifications
+methods netdata supports, including custom hooks.
+
+---
+
+#### alarm line `delay`
+
+This is used to provide optional hysteresis settings for the notifications, to defend
+against notification floods. These settings do not affect the actual alarm - only the time
+the `exec` script is executed.
+
+Format:
+
+```
+delay: [[[up U] [down D] multiplier M] max X]
+```
+
+- `up U` defines the delay to be applied to a notification for an alarm that raised its status
+ (i.e. CLEAR to WARNING, CLEAR to CRITICAL, WARNING to CRITICAL). For example, `up 10s`, the
+ notification for this event will be sent 10 seconds after the actual event. This is used in
+ hope the alarm will get back to its previous state within the duration given. The default `U`
+ is zero.
+
+- `down D` defines the delay to be applied to a notification for an alarm that moves to lower
+ state (i.e. CRITICAL to WARNING, CRITICAL to CLEAR, WARNING to CLEAR). For example, `down 1m`
+ will delay the notification by 1 minute. This is used to prevent notifications for flapping
+ alarms. The default `D` is zero.
+
+- `mutliplier M` multiplies `U` and `D` when an alarm changes state, while a notification is
+ delayed. The default multiplier is `1.0`.
+
+- `max X` defines the maximum absolute notification delay an alarm may get. The default `X`
+ is `max(U * M, D * M)` (i.e. the max duration of `U` or `D` multiplied once with `M`).
+
+ Example:
+
+ `delay: up 10s down 15m multiplier 2 max 1h`
+
+ The time is `00:00:00` and the status of the alarm is CLEAR.
+
+ time of event|new status|delay|notification will be sent|why
+ -------------|----------|:---:|-------------------------|---
+ 00:00:01 | WARNING | `up 10s` | 00:00:11 |first state switch
+ 00:00:05 | CLEAR | `down 15m x2`| 00:30:05 |the alarm changes state while a notification is delayed, so it was multiplied
+ 00:00:06 | WARNING | `up 10s x2 x2` | 00:00:26 |multiplied twice
+ 00:00:07|CLEAR|`down 15m x2 x2 x2`|00:45:07|multiplied 3 times.
+
+ So:
+ - `U` and `D` are multiplied by `M` every time the alarm changes state (any state, not just
+ their matching one) and a delay is in place.
+ - All are reset to their defaults when the alarm switches state without a delay in place.
+
+---
+
+### Expressions
+
+netdata has an internal [infix expression parser](../libnetdata/eval).
+This parses expressions and creates an internal structure that allows fast execution of them.
+
+These operators are supported `+`, `-`, `*`, `/`, `<`, `<=`, `<>`, `!=`, `>`, `>=`, `&&`, `||`,
+`!`, `AND`, `OR`, `NOT`. Boolean operators result in either `1` (true) or `0` (false).
+
+The conditional evaluation operator `?` is supported too. Using this operator IF-THEN-ELSE
+conditional statements can be specified. The format is: `(condition) ? (true expression) :
+(false expression)`. So, netdata will first evaluate the `condition` and based on the result
+will either evaluate `true expression` or `false expression`.
+Example: `($this > 0) ? ($avail * 2) : ($used / 2)`.
+Nested such expressions are also supported (i.e. `true expression` and `false expression` can
+contain conditional evaluations).
+
+Expressions also support the `abs()` function.
+
+Expressions can have variables. Variables start with `$`. Check below for more information.
+
+There are two special values you can use:
+
+ - `nan`, for example `$this != nan` will check if the variable `this` is available.
+ A variable can be `nan` if the database lookup failed. All calculations (i.e. addition,
+ multiplication, etc) with a `nan` result in a `nan`.
+
+ - `inf`, for example `$this != inf` will check if `this` is not infinite. A value or
+ variable can be infinite if divided by zero. All calculations (i.e. addition,
+ multiplication, etc) with a `inf` result in a `inf`.
+
+---
+
+### Special use of the conditional operator
+
+A common (but not necessarily obvious) use of the conditional evaluation operator is
+to provide [hysteresis](https://en.wikipedia.org/wiki/Hysteresis) around the critical
+or warning thresholds. This usage helps to avoid bogus messages resulting from small
+variations in the value when it is varying regularly but staying close to the threshold
+value, without needing to delay sending messages at all.
+
+An example of such usage from the default CPU usage alarms bundled with netdata is:
+
+```
+warn: $this > (($status >= $WARNING) ? (75) : (85))
+crit: $this > (($status == $CRITICAL) ? (85) : (95))
+```
+
+The above say:
+* If the alarm is currently a warning, then the threshold for being considered a warning
+ is 75, otherwise it's 85.
+
+* If the alarm is currently critical, then the threshold for being considered critical
+ is 85, otherwise it's 95.
+
+Which in turn, results in the following behavior:
+* While the value is rising, it will trigger a warning when it exceeds 85, and a critical
+ alert when it exceeds 95.
+
+* While the value is falling, it will return to a warning state when it goes below 85,
+ and a normal state when it goes below 75.
+
+* If the value is constantly varying between 80 and 90, then it will trigger a warning the
+ first time it goes above 85, but will remain a warning until it goes below 75 (or goes above 85).
+
+* If the value is constantly varying between 90 and 100, then it will trigger a critical alert
+ the first time it goes above 95, but will remain a critical alert goes below 85 (at which
+ point it will return to being a warning).
+
+---
+
+### Variables
+
+netdata supports 3 new internal indexes for variables that will be used in health monitoring:
+
+ - **chart local variables**. All the dimensions of the chart are exposed as local variables.
+ All chart alarms names are exposed as variables too.
+
+ Charts also define a few special variables:
+
+ - `$last_collected_t` is the unix timestamp of the last data collection
+ - `$collected_total_raw` is the sum of all the dimensions (their last collected values)
+ - `$update_every` is the update frequency of the chart
+ - `$green` and `$red` the threshold defined in alarms (these are per chart - the charts
+ inherits them from the the first alarm that defined them)
+
+ Chart dimensions define their last calculated (i.e. interpolated) value, exactly as
+ shown on the charts, but also a variable with their name and suffix `_raw` that resolves
+ to the last collected value - as collected and another with suffix `_last_collected_t`
+ that resolves to unix timestamp the dimension was last collected (there may be dimensions
+ that fail to be collected while others continue normally).
+
+ - **family variables**. Families are used to group charts together. For example all `eth0`
+ charts, have `family = eth0`. This index includes all local variables, but if there are
+ overlapping variables, only the first are exposed.
+
+ - **host variables**. All the dimensions of all charts, including all alarms, in fullname.
+ Fullname is `CHART.VARIABLE`, where `CHART` is either the chart id or the chart name (both
+ are supported).
+
+ - **special variables*** are:
+
+ - `this`, which is resolved to the value of the current alarm.
+
+ - `status`, which is resolved to the current status of the alarm (the current = the last
+ status, i.e. before the current database lookup and the evaluation of the `calc` line).
+ This values can be compared with `$REMOVED`, `$UNINITIALIZED`, `$UNDEFINED`, `$CLEAR`,
+ `$WARNING`, `$CRITICAL`. These values are incremental, ie. `$status > $CLEAL` works as
+ expected.
+
+ - `now`, which is resolved to current unix timestamp.
+
+You can find all the variables that can be used for a given chart, using
+`http://your.netdata.ip:19999/api/v1/alarm_variables?chart=NAME`.
+This will dump all the indexes from the chart's perspective.
+Example: [variables for the `system.cpu` chart of the registry](https://registry.my-netdata.io/api/v1/alarm_variables?chart=system.cpu).
+
+## Alarm Statuses
+
+Alarms can have the following statuses:
+
+ - `REMOVED` - the alarm has been deleted (this happens when a SIGUSR2 is sent to netdata
+ to reload health configuration)
+
+ - `UNINITIALIZED` - the alarm is not initialized yet
+
+ - `UNDEFINED` - the alarm failed to be calculated (i.e. the database lookup failed,
+ a division by zero occurred, etc)
+
+ - `CLEAR` - the alarm is not armed / raised (i.e. is OK)
+
+ - `WARNING` - the warning expression resulted in true or non-zero
+
+ - `CRITICAL` - the critical expression resulted in true or non-zero
+
+The external script will be called for all status changes.
+
+## Examples
+
+
+Check the **[health.d directory](health.d)** for all alarms shipped with netdata.
+
+Here are a few examples:
+
+### Example 1
+
+A simple check if an apache server is alive:
+
+```
+template: apache_last_collected_secs
+ on: apache.requests
+ calc: $now - $last_collected_t
+ every: 10s
+ warn: $this > ( 5 * $update_every)
+ crit: $this > (10 * $update_every)
+```
+
+The above checks that netdata is able to collect data from apache. In detail:
+
+```
+template: apache_last_collected_secs
+```
+
+The above defines a **template** named `apache_last_collected_secs`.
+The name is important since `$apache_last_collected_secs` resolves to the `calc` line.
+So, try to give something descriptive.
+
+```
+ on: apache.requests
+```
+
+The above applies the **template** to all charts that have `context = apache.requests`
+(i.e. all your apache servers).
+
+```
+ calc: $now - $last_collected_t
+```
+
+- `$now` is a standard variable that resolves to the current timestamp.
+
+- `$last_collected_t` is the last data collection timestamp of the chart.
+ So this calculation gives the number of seconds passed since the last data collection.
+
+```
+ every: 10s
+```
+
+The alarm will be evaluated every 10 seconds.
+
+```
+ warn: $this > ( 5 * $update_every)
+ crit: $this > (10 * $update_every)
+```
+
+If these result in non-zero or true, they trigger the alarm.
+
+- `$this` refers to the value of this alarm (i.e. the result of the `calc` line.
+ We could also use `$apache_last_collected_secs`.
+
+`$update_every` is the update frequency of the chart, in seconds.
+
+So, the warning condition checks if we have not collected data from apache for 5
+iterations and the critical condition checks for 10 iterations.
+
+### Example 2
+
+Check if any of the disks is critically low on disk space:
+
+```
+template: disk_full_percent
+ on: disk.space
+ calc: $used * 100 / ($avail + $used)
+ every: 1m
+ warn: $this > 80
+ crit: $this > 95
+```
+
+`$used` and `$avail` are the `used` and `avail` chart dimensions as shown on the dashboard.
+
+So, the `calc` line finds the percentage of used space. `$this` resolves to this percentage.
+
+### Example 3
+
+Predict if any disk will run out of space in the near future.
+
+We do this in 2 steps:
+
+Calculate the disk fill rate:
+
+```
+ template: disk_fill_rate
+ on: disk.space
+ lookup: max -1s at -30m unaligned of avail
+ calc: ($this - $avail) / (30 * 60)
+ every: 15s
+```
+
+In the `calc` line: `$this` is the result of the `lookup` line (i.e. the free space 30 minutes
+ago) and `$avail` is the current disk free space. So the `calc` line will either have a positive
+number of GB/second if the disk if filling up, or a negative number of GB/second if the disk is
+freeing up space.
+
+There is no `warn` or `crit` lines here. So, this template will just do the calculation and
+nothing more.
+
+Predict the hours after which the disk will run out of space:
+
+```
+ template: disk_full_after_hours
+ on: disk.space
+ calc: $avail / $disk_fill_rate / 3600
+ every: 10s
+ warn: $this > 0 and $this < 48
+ crit: $this > 0 and $this < 24
+```
+
+The `calc` line estimates the time in hours, we will run out of disk space. Of course, only
+positive values are interesting for this check, so the warning and critical conditions check
+for positive values and that we have enough free space for 48 and 24 hours respectively.
+
+Once this alarm triggers we will receive an email like this:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/17839993/87872b32-6802-11e6-8e08-b2e4afef93bb.png)
+
+### Example 4
+
+Check if any network interface is dropping packets:
+
+```
+template: 30min_packet_drops
+ on: net.drops
+ lookup: sum -30m unaligned absolute
+ every: 10s
+ crit: $this > 0
+```
+
+The `lookup` line will calculate the sum of the all dropped packets in the last 30 minutes.
+
+The `crit` line will issue a critical alarm if even a single packet has been dropped.
+
+Note that the drops chart does not exist if a network interface has never dropped a single packet.
+When netdata detects a dropped packet, it will add the chart and it will automatically attach this
+alarm to it.
+
+## Troubleshooting
+
+You can compile netdata with [debugging](../daemon#debugging) and then set in `netdata.conf`:
+
+```
+[global]
+ debug flags = 0x0000000000800000
+```
+
+Then check your `/var/log/netdata/debug.log`. It will show you how it works.
+Important: this will generate a lot of output in debug.log.
+
+You can find the context of charts by looking up the chart in either
+`http://your.netdata:19999/netdata.conf` or `http://your.netdata:19999/api/v1/charts`.
+
+You can find how netdata interpreted the expressions by examining the alarm at
+`http://your.netdata:19999/api/v1/alarms?all`. For each expression, netdata will return the
+expression as given in its config file, and the same expression with additional parentheses
+added to indicate the evaluation flow of the expression.
+
diff --git a/src/health.c b/health/health.c
index 04e04f08..ae0c464b 100644
--- a/src/health.c
+++ b/health/health.c
@@ -1,21 +1,28 @@
-#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
-int default_health_enabled = 1;
+#include "health.h"
+
+unsigned int default_health_enabled = 1;
// ----------------------------------------------------------------------------
// health initialization
-inline char *health_config_dir(void) {
+inline char *health_user_config_dir(void) {
char buffer[FILENAME_MAX + 1];
- snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_config_dir);
+ snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_user_config_dir);
return config_get(CONFIG_SECTION_HEALTH, "health configuration directory", buffer);
}
+inline char *health_stock_config_dir(void) {
+ char buffer[FILENAME_MAX + 1];
+ snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_stock_config_dir);
+ return config_get(CONFIG_SECTION_HEALTH, "stock health configuration directory", buffer);
+}
+
void health_init(void) {
debug(D_HEALTH, "Health configuration initializing");
- if(!(default_health_enabled = config_get_boolean(CONFIG_SECTION_HEALTH, "enabled", 1))) {
+ if(!(default_health_enabled = (unsigned int)config_get_boolean(CONFIG_SECTION_HEALTH, "enabled", default_health_enabled))) {
debug(D_HEALTH, "Health is disabled.");
return;
}
@@ -28,7 +35,8 @@ void health_reload_host(RRDHOST *host) {
if(unlikely(!host->health_enabled))
return;
- char *path = health_config_dir();
+ char *user_path = health_user_config_dir();
+ char *stock_path = health_stock_config_dir();
// free all running alarms
rrdhost_wrlock(host);
@@ -59,7 +67,7 @@ void health_reload_host(RRDHOST *host) {
// load the new alarms
rrdhost_wrlock(host);
- health_readdir(host, path);
+ health_readdir(host, user_path, stock_path, NULL);
// link the loaded alarms to their charts
rrdset_foreach_write(st, host) {
@@ -178,9 +186,9 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
error("HEALTH: Cannot popen(\"%s\", \"r\").", command_to_run);
goto done;
}
- debug(D_HEALTH, "HEALTH reading from command");
- char *s = fgets(command_to_run, FILENAME_MAX, fp);
- (void)s;
+ debug(D_HEALTH, "HEALTH reading from command (discarding command's output)");
+ char buffer[100 + 1];
+ while(fgets(buffer, 100, fp) != NULL) ;
ae->exec_code = mypclose(fp, command_pid);
debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
@@ -521,6 +529,11 @@ void *health_main(void *ptr) {
);
rc->value = rc->calculation->result;
+
+ if(rc->local) rc->local->last_updated = now;
+ if(rc->family) rc->family->last_updated = now;
+ if(rc->hostid) rc->hostid->last_updated = now;
+ if(rc->hostname) rc->hostname->last_updated = now;
}
}
}
diff --git a/health/health.d/adaptec_raid.conf b/health/health.d/adaptec_raid.conf
new file mode 100644
index 00000000..a1301ce8
--- /dev/null
+++ b/health/health.d/adaptec_raid.conf
@@ -0,0 +1,24 @@
+
+# logical device status check
+
+template: adapter_raid_ld_status
+ on: adapter_raid.ld_status
+ lookup: max -5s
+ units: bool
+ every: 10s
+ crit: $this > 0
+ delay: down 5m multiplier 1.5 max 1h
+ info: at least 1 logical device is failed or degraded
+ to: sysadmin
+
+# physical device state check
+
+template: adapter_raid_pd_state
+ on: adapter_raid.pd_state
+ lookup: max -5s
+ units: bool
+ every: 10s
+ crit: $this > 0
+ delay: down 5m multiplier 1.5 max 1h
+ info: at least 1 physical device is not in online state
+ to: sysadmin
diff --git a/conf.d/health.d/apache.conf b/health/health.d/apache.conf
index 0c98b877..0c98b877 100644
--- a/conf.d/health.d/apache.conf
+++ b/health/health.d/apache.conf
diff --git a/health/health.d/apcupsd.conf b/health/health.d/apcupsd.conf
new file mode 100644
index 00000000..4f86037b
--- /dev/null
+++ b/health/health.d/apcupsd.conf
@@ -0,0 +1,40 @@
+# you can disable an alarm notification by setting the 'to' line to: silent
+
+template: 10min_ups_load
+ on: apcupsd.load
+ os: *
+ hosts: *
+ lookup: average -10m unaligned of percentage
+ units: %
+ every: 1m
+ warn: $this > (($status >= $WARNING) ? (70) : (80))
+ crit: $this > (($status == $CRITICAL) ? (85) : (95))
+ delay: down 10m multiplier 1.5 max 1h
+ info: average UPS load for the last 10 minutes
+ to: sitemgr
+
+# Discussion in https://github.com/netdata/netdata/pull/3928:
+# Fire the alarm as soon as it's going on battery (99% charge) and clear only when full.
+template: ups_charge
+ on: apcupsd.charge
+ os: *
+ hosts: *
+ lookup: average -60s unaligned of charge
+ units: %
+ every: 60s
+ warn: $this < 100
+ crit: $this < (($status == $CRITICAL) ? (60) : (50))
+ delay: down 10m multiplier 1.5 max 1h
+ info: current UPS charge, averaged over the last 60 seconds to reduce measurement errors
+ to: sitemgr
+
+template: apcupsd_last_collected_secs
+ on: apcupsd.load
+ calc: $now - $last_collected_t
+ every: 10s
+ units: seconds ago
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
+ delay: down 5m multiplier 1.5 max 1h
+ info: number of seconds since the last successful data collection
+ to: sitemgr
diff --git a/conf.d/health.d/backend.conf b/health/health.d/backend.conf
index 7af100d8..7af100d8 100644
--- a/conf.d/health.d/backend.conf
+++ b/health/health.d/backend.conf
diff --git a/health/health.d/bcache.conf b/health/health.d/bcache.conf
new file mode 100644
index 00000000..f0da9ac5
--- /dev/null
+++ b/health/health.d/bcache.conf
@@ -0,0 +1,22 @@
+
+template: bcache_cache_errors
+ on: disk.bcache_cache_read_races
+ lookup: sum -10m unaligned absolute
+ units: errors
+ every: 1m
+ warn: $this > 0
+ crit: $this > ( ($status >= $CRITICAL) ? (0) : (10) )
+ delay: down 1h multiplier 1.5 max 2h
+ info: the number of times bcache had issues using the cache, during the last 10 mins (this usually means your SSD cache is failing)
+ to: sysadmin
+
+template: bcache_cache_dirty
+ on: disk.bcache_cache_alloc
+ calc: $dirty + $metadata + $undefined
+ units: %
+ every: 1m
+ warn: $this > ( ($status >= $WARNING ) ? ( 70 ) : ( 90 ) )
+ crit: $this > ( ($status >= $CRITICAL) ? ( 90 ) : ( 95 ) )
+ delay: up 1m down 1h multiplier 1.5 max 2h
+ info: the percentage of cache space used for dirty and metadata (this usually means your SSD cache is too small)
+ to: sysadmin
diff --git a/conf.d/health.d/beanstalkd.conf b/health/health.d/beanstalkd.conf
index 30dc2732..30dc2732 100644
--- a/conf.d/health.d/beanstalkd.conf
+++ b/health/health.d/beanstalkd.conf
diff --git a/conf.d/health.d/bind_rndc.conf b/health/health.d/bind_rndc.conf
index 4145e77c..4145e77c 100644
--- a/conf.d/health.d/bind_rndc.conf
+++ b/health/health.d/bind_rndc.conf
diff --git a/health/health.d/boinc.conf b/health/health.d/boinc.conf
new file mode 100644
index 00000000..43c588db
--- /dev/null
+++ b/health/health.d/boinc.conf
@@ -0,0 +1,62 @@
+# Alarms for various BOINC issues.
+
+# Warn on any compute errors encountered.
+template: boinc_compute_errors
+ on: boinc.states
+ os: *
+ hosts: *
+families: *
+ lookup: average -10m unaligned of comperror
+ units: tasks
+ every: 1m
+ warn: $this > 0
+ crit: $this > 1
+ delay: up 1m down 5m multiplier 1.5 max 1h
+ info: the total number of compute errors over the past 10 minutes
+ to: sysadmin
+
+# Warn on lots of upload errors
+template: boinc_upload_errors
+ on: boinc.states
+ os: *
+ hosts: *
+families: *
+ lookup: average -10m unaligned of upload_failed
+ units: tasks
+ every: 1m
+ warn: $this > 0
+ crit: $this > 1
+ delay: up 1m down 5m multiplier 1.5 max 1h
+ info: the average number of failed uploads over the past 10 minutes
+ to: sysadmin
+
+# Warn on the task queue being empty
+template: boinc_total_tasks
+ on: boinc.tasks
+ os: *
+ hosts: *
+families: *
+ lookup: average -10m unaligned of total
+ units: tasks
+ every: 1m
+ warn: $this < 1
+ crit: $this < 0.1
+ delay: up 5m down 10m multiplier 1.5 max 1h
+ info: the total number of locally available tasks
+ to: sysadmin
+
+# Warn on no active tasks with a non-empty queue
+template: boinc_active_tasks
+ on: boinc.tasks
+ os: *
+ hosts: *
+families: *
+ lookup: average -10m unaligned of active
+ calc: ($boinc_total_tasks >= 1) ? ($this) : (inf)
+ units: tasks
+ every: 1m
+ warn: $this < 1
+ crit: $this < 0.1
+ delay: up 5m down 10m multiplier 1.5 max 1h
+ info: the total number of active tasks
+ to: sysadmin
diff --git a/conf.d/health.d/btrfs.conf b/health/health.d/btrfs.conf
index b27aa544..b27aa544 100644
--- a/conf.d/health.d/btrfs.conf
+++ b/health/health.d/btrfs.conf
diff --git a/conf.d/health.d/ceph.conf b/health/health.d/ceph.conf
index de16f7b6..de16f7b6 100644
--- a/conf.d/health.d/ceph.conf
+++ b/health/health.d/ceph.conf
diff --git a/conf.d/health.d/couchdb.conf b/health/health.d/couchdb.conf
index 4a289528..4a289528 100644
--- a/conf.d/health.d/couchdb.conf
+++ b/health/health.d/couchdb.conf
diff --git a/conf.d/health.d/cpu.conf b/health/health.d/cpu.conf
index fa818985..fa818985 100644
--- a/conf.d/health.d/cpu.conf
+++ b/health/health.d/cpu.conf
diff --git a/conf.d/health.d/disks.conf b/health/health.d/disks.conf
index 26f85848..26f85848 100644
--- a/conf.d/health.d/disks.conf
+++ b/health/health.d/disks.conf
diff --git a/health/health.d/dockerd.conf b/health/health.d/dockerd.conf
new file mode 100644
index 00000000..729906cd
--- /dev/null
+++ b/health/health.d/dockerd.conf
@@ -0,0 +1,8 @@
+template: docker_unhealthy_containers
+ on: docker.unhealthy_containers
+ units: unhealthy containers
+ every: 10s
+ lookup: average -10s
+ crit: $this > 0
+ info: number of unhealthy containers
+ to: sysadmin
diff --git a/conf.d/health.d/elasticsearch.conf b/health/health.d/elasticsearch.conf
index dffd4096..dffd4096 100644
--- a/conf.d/health.d/elasticsearch.conf
+++ b/health/health.d/elasticsearch.conf
diff --git a/conf.d/health.d/entropy.conf b/health/health.d/entropy.conf
index 66d44ec1..66d44ec1 100644
--- a/conf.d/health.d/entropy.conf
+++ b/health/health.d/entropy.conf
diff --git a/conf.d/health.d/fping.conf b/health/health.d/fping.conf
index 43658fef..43658fef 100644
--- a/conf.d/health.d/fping.conf
+++ b/health/health.d/fping.conf
diff --git a/conf.d/health.d/fronius.conf b/health/health.d/fronius.conf
index cdf6c8fc..cdf6c8fc 100644
--- a/conf.d/health.d/fronius.conf
+++ b/health/health.d/fronius.conf
diff --git a/conf.d/health.d/haproxy.conf b/health/health.d/haproxy.conf
index e49c70d4..e49c70d4 100644
--- a/conf.d/health.d/haproxy.conf
+++ b/health/health.d/haproxy.conf
diff --git a/conf.d/health.d/httpcheck.conf b/health/health.d/httpcheck.conf
index 0ddf35ea..0ddf35ea 100644
--- a/conf.d/health.d/httpcheck.conf
+++ b/health/health.d/httpcheck.conf
diff --git a/conf.d/health.d/ipc.conf b/health/health.d/ipc.conf
index 03cf264d..989d6e91 100644
--- a/conf.d/health.d/ipc.conf
+++ b/health/health.d/ipc.conf
@@ -5,7 +5,7 @@
on: system.ipc_semaphores
os: linux
hosts: *
- calc: $semaphores * 100 / $ipc.semaphores.max
+ calc: $semaphores * 100 / $ipc_semaphores_max
units: %
every: 10s
warn: $this > (($status >= $WARNING) ? (70) : (80))
@@ -18,7 +18,7 @@
on: system.ipc_semaphore_arrays
os: linux
hosts: *
- calc: $arrays * 100 / $ipc.semaphores.arrays.max
+ calc: $arrays * 100 / $ipc_semaphores_arrays_max
units: %
every: 10s
warn: $this > (($status >= $WARNING) ? (70) : (80))
diff --git a/conf.d/health.d/ipfs.conf b/health/health.d/ipfs.conf
index 3f77572d..3f77572d 100644
--- a/conf.d/health.d/ipfs.conf
+++ b/health/health.d/ipfs.conf
diff --git a/conf.d/health.d/ipmi.conf b/health/health.d/ipmi.conf
index c2558196..c2558196 100644
--- a/conf.d/health.d/ipmi.conf
+++ b/health/health.d/ipmi.conf
diff --git a/conf.d/health.d/isc_dhcpd.conf b/health/health.d/isc_dhcpd.conf
index 8054656f..8054656f 100644
--- a/conf.d/health.d/isc_dhcpd.conf
+++ b/health/health.d/isc_dhcpd.conf
diff --git a/conf.d/health.d/lighttpd.conf b/health/health.d/lighttpd.conf
index 915907a4..915907a4 100644
--- a/conf.d/health.d/lighttpd.conf
+++ b/health/health.d/lighttpd.conf
diff --git a/health/health.d/linux_power_supply.conf b/health/health.d/linux_power_supply.conf
new file mode 100644
index 00000000..27a172a1
--- /dev/null
+++ b/health/health.d/linux_power_supply.conf
@@ -0,0 +1,12 @@
+# Alert on low battery capacity.
+
+template: linux_power_supply_capacity
+ on: power_supply.capacity
+ calc: $capacity
+ units: %
+ every: 10s
+ warn: $this < 10
+ crit: $this < 5
+ delay: up 0 down 5m multiplier 1.2 max 1h
+ info: the percentage remaining capacity of the power supply
+ to: sysadmin
diff --git a/health/health.d/load.conf b/health/health.d/load.conf
new file mode 100644
index 00000000..ee0c54b8
--- /dev/null
+++ b/health/health.d/load.conf
@@ -0,0 +1,56 @@
+
+# you can disable an alarm notification by setting the 'to' line to: silent
+
+# Calculate the base trigger point for the load average alarms.
+# This is the maximum number of CPU's in the system over the past 1
+# minute, with a special case for a single CPU of setting the trigger at 2.
+ alarm: load_trigger
+ on: system.load
+ os: linux
+ hosts: *
+ calc: ($active_processors == nan or $active_processors == inf or $active_processors < 2) ? ( 2 ) : ( $active_processors )
+ units: cpus
+ every: 1m
+ info: trigger point for load average alarms
+
+# Send alarms if the load average is unusually high.
+# These intentionally _do not_ calculate the average over the sampled
+# time period because the values being checked already are averages.
+ alarm: load_average_15
+ on: system.load
+ os: linux
+ hosts: *
+ lookup: max -1m unaligned of load15
+ units: load
+ every: 1m
+ warn: $this > (($status >= $WARNING) ? (1.75 * $load_trigger) : (2 * $load_trigger))
+ crit: $this > (($status == $CRITICAL) ? (3.5 * $load_trigger) : (4 * $load_trigger))
+ delay: down 15m multiplier 1.5 max 1h
+ info: fifteen-minute load average
+ to: sysadmin
+
+ alarm: load_average_5
+ on: system.load
+ os: linux
+ hosts: *
+ lookup: max -1m unaligned of load5
+ units: load
+ every: 1m
+ warn: $this > (($status >= $WARNING) ? (3.5 * $load_trigger) : (4 * $load_trigger))
+ crit: $this > (($status == $CRITICAL) ? (7 * $load_trigger) : (8 * $load_trigger))
+ delay: down 15m multiplier 1.5 max 1h
+ info: five-minute load average
+ to: sysadmin
+
+ alarm: load_average_1
+ on: system.load
+ os: linux
+ hosts: *
+ lookup: max -1m unaligned of load1
+ units: load
+ every: 1m
+ warn: $this > (($status >= $WARNING) ? (7 * $load_trigger) : (8 * $load_trigger))
+ crit: $this > (($status == $CRITICAL) ? (14 * $load_trigger) : (16 * $load_trigger))
+ delay: down 15m multiplier 1.5 max 1h
+ info: one-minute load average
+ to: sysadmin
diff --git a/conf.d/health.d/mdstat.conf b/health/health.d/mdstat.conf
index c9e7d20d..0f5f2837 100644
--- a/conf.d/health.d/mdstat.conf
+++ b/health/health.d/mdstat.conf
@@ -1,3 +1,13 @@
+template: mdstat_last_collected
+ on: md.disks
+ calc: $now - $last_collected_t
+ units: seconds ago
+ every: 10s
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
+ info: number of seconds since the last successful data collection
+ to: sysadmin
+
template: mdstat_disks
on: md.disks
units: failed devices
@@ -7,12 +17,11 @@ template: mdstat_disks
info: Array is degraded!
to: sysadmin
-template: mdstat_last_collected
- on: md.disks
- calc: $now - $last_collected_t
- units: seconds ago
+template: mdstat_mismatch_cnt
+ on: md.mismatch_cnt
+ units: unsynchronized blocks
+ calc: $count
every: 10s
- warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
- crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
- info: number of seconds since the last successful data collection
+ crit: $this > 0
+ info: Mismatch count!
to: sysadmin
diff --git a/health/health.d/megacli.conf b/health/health.d/megacli.conf
new file mode 100644
index 00000000..1881a7be
--- /dev/null
+++ b/health/health.d/megacli.conf
@@ -0,0 +1,48 @@
+ alarm: adapter_state
+ on: megacli.adapter_degraded
+ units: is degraded
+ lookup: sum -10s
+ every: 10s
+ crit: $this > 0
+ info: adapter state
+ to: sysadmin
+
+ template: bbu_relative_charge
+ on: megacli.bbu_relative_charge
+ units: percent
+ lookup: average -10s
+ every: 10s
+ warn: $this <= (($status >= $WARNING) ? (85) : (80))
+ crit: $this <= (($status == $CRITICAL) ? (50) : (40))
+ info: BBU relative state of charge
+ to: sysadmin
+
+ template: bbu_cycle_count
+ on: megacli.bbu_cycle_count
+ units: cycle count
+ lookup: average -10s
+ every: 10s
+ warn: $this >= 100
+ crit: $this >= 500
+ info: BBU cycle count
+ to: sysadmin
+
+ alarm: pd_media_errors
+ on: megacli.pd_media_error
+ units: media errors
+ lookup: sum -10s
+ every: 10s
+ warn: $this > 0
+ delay: down 1m multiplier 2 max 10m
+ info: physical drive media errors
+ to: sysadmin
+
+ alarm: pd_predictive_failures
+ on: megacli.pd_predictive_failure
+ units: predictive failures
+ lookup: sum -10s
+ every: 10s
+ warn: $this > 0
+ delay: down 1m multiplier 2 max 10m
+ info: physical drive predictive failures
+ to: sysadmin
diff --git a/conf.d/health.d/memcached.conf b/health/health.d/memcached.conf
index d248ef57..d248ef57 100644
--- a/conf.d/health.d/memcached.conf
+++ b/health/health.d/memcached.conf
diff --git a/conf.d/health.d/memory.conf b/health/health.d/memory.conf
index 4a0e6e52..4a0e6e52 100644
--- a/conf.d/health.d/memory.conf
+++ b/health/health.d/memory.conf
diff --git a/conf.d/health.d/mongodb.conf b/health/health.d/mongodb.conf
index a80cb311..a80cb311 100644
--- a/conf.d/health.d/mongodb.conf
+++ b/health/health.d/mongodb.conf
diff --git a/conf.d/health.d/mysql.conf b/health/health.d/mysql.conf
index 1eeb993f..39c40191 100644
--- a/conf.d/health.d/mysql.conf
+++ b/health/health.d/mysql.conf
@@ -60,6 +60,21 @@ template: mysql_10s_waited_locks_ratio
# -----------------------------------------------------------------------------
+# connections
+
+template: mysql_connections
+ on: mysql.connections_active
+ calc: $active * 100 / $limit
+ units: %
+ every: 10s
+ warn: $this > (($status >= $WARNING) ? (60) : (70))
+ crit: $this > (($status == $CRITICAL) ? (80) : (90))
+ delay: down 15m multiplier 1.5 max 1h
+ info: the ratio of current active connections vs the maximum possible number of connections
+ to: dba
+
+
+# -----------------------------------------------------------------------------
# replication
template: mysql_replication
diff --git a/conf.d/health.d/named.conf b/health/health.d/named.conf
index 4fc65c8e..4fc65c8e 100644
--- a/conf.d/health.d/named.conf
+++ b/health/health.d/named.conf
diff --git a/conf.d/health.d/net.conf b/health/health.d/net.conf
index 22a88927..489016dd 100644
--- a/conf.d/health.d/net.conf
+++ b/health/health.d/net.conf
@@ -2,6 +2,39 @@
# you can disable an alarm notification by setting the 'to' line to: silent
# -----------------------------------------------------------------------------
+# net traffic overflow
+
+ template: 1m_received_traffic_overflow
+ on: net.net
+ os: linux
+ hosts: *
+ families: *
+ lookup: average -1m unaligned absolute of received
+ calc: ($nic_speed_max > 0) ? ($this * 100 / ($nic_speed_max * 1000)) : ( nan )
+ units: %
+ every: 10s
+ warn: $this > (($status >= $WARNING) ? (80) : (85))
+ crit: $this > (($status == $CRITICAL) ? (85) : (90))
+ delay: down 1m multiplier 1.5 max 1h
+ info: interface received bandwidth usage over net device speed max
+ to: sysadmin
+
+ template: 1m_sent_traffic_overflow
+ on: net.net
+ os: linux
+ hosts: *
+ families: *
+ lookup: average -1m unaligned absolute of sent
+ calc: ($nic_speed_max > 0) ? ($this * 100 / ($nic_speed_max * 1000)) : ( nan )
+ units: %
+ every: 10s
+ warn: $this > (($status >= $WARNING) ? (80) : (85))
+ crit: $this > (($status == $CRITICAL) ? (85) : (90))
+ delay: down 1m multiplier 1.5 max 1h
+ info: interface sent bandwidth usage over net device speed max
+ to: sysadmin
+
+# -----------------------------------------------------------------------------
# dropped packets
# check if an interface is dropping packets
@@ -101,7 +134,7 @@ template: 1m_received_packets_rate
os: linux freebsd
hosts: *
families: *
- lookup: average -1m of received
+ lookup: average -1m unaligned of received
units: packets
every: 10s
info: the average number of packets received during the last minute
@@ -111,7 +144,7 @@ template: 10s_received_packets_storm
os: linux freebsd
hosts: *
families: *
- lookup: average -10s of received
+ lookup: average -10s unaligned of received
calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate))
every: 10s
units: %
diff --git a/conf.d/health.d/netfilter.conf b/health/health.d/netfilter.conf
index fa1732b3..1d07752c 100644
--- a/conf.d/health.d/netfilter.conf
+++ b/health/health.d/netfilter.conf
@@ -19,7 +19,7 @@
os: linux
hosts: *
lookup: max -10s unaligned of connections
- calc: $this * 100 / $netfilter.conntrack.max
+ calc: $this * 100 / $netfilter_conntrack_max
units: %
every: 10s
warn: $this > (($status >= $WARNING) ? (70) : (80))
diff --git a/conf.d/health.d/nginx.conf b/health/health.d/nginx.conf
index a686c3d9..a686c3d9 100644
--- a/conf.d/health.d/nginx.conf
+++ b/health/health.d/nginx.conf
diff --git a/conf.d/health.d/nginx_plus.conf b/health/health.d/nginx_plus.conf
index 5a171a76..5a171a76 100644
--- a/conf.d/health.d/nginx_plus.conf
+++ b/health/health.d/nginx_plus.conf
diff --git a/conf.d/health.d/portcheck.conf b/health/health.d/portcheck.conf
index f42b63d3..f42b63d3 100644
--- a/conf.d/health.d/portcheck.conf
+++ b/health/health.d/portcheck.conf
diff --git a/conf.d/health.d/postgres.conf b/health/health.d/postgres.conf
index 4e0583b8..4e0583b8 100644
--- a/conf.d/health.d/postgres.conf
+++ b/health/health.d/postgres.conf
diff --git a/conf.d/health.d/qos.conf b/health/health.d/qos.conf
index 7290d15f..7290d15f 100644
--- a/conf.d/health.d/qos.conf
+++ b/health/health.d/qos.conf
diff --git a/conf.d/health.d/ram.conf b/health/health.d/ram.conf
index b6dc5f94..4e437322 100644
--- a/conf.d/health.d/ram.conf
+++ b/health/health.d/ram.conf
@@ -3,7 +3,7 @@
alarm: used_ram_to_ignore
on: system.ram
- os: linux
+ os: linux freebsd
hosts: *
calc: ($zfs.arc_size.arcsz = nan)?(0):($zfs.arc_size.arcsz)
every: 10s
@@ -41,7 +41,7 @@ alarm: ram_in_use
on: system.ram
os: freebsd
hosts: *
- calc: (($active + $wired) - $used_ram_to_ignore) * 100 / (($active + $wired) - $used_ram_to_ignore + $cached + $free)
+ calc: ($active + $wired + $laundry + $buffers - $used_ram_to_ignore) * 100 / ($active + $wired + $laundry + $buffers - $used_ram_to_ignore + $cache + $free + $inactive)
units: %
every: 10s
warn: $this > (($status >= $WARNING) ? (80) : (90))
@@ -54,7 +54,7 @@ delay: down 15m multiplier 1.5 max 1h
on: system.ram
os: freebsd
hosts: *
- calc: ($free + $inactive + $used_ram_to_ignore) * 100 / ($free + $active + $inactive + $wired + $cache + $buffers)
+ calc: ($free + $inactive + $used_ram_to_ignore) * 100 / ($free + $active + $inactive + $wired + $cache + $laundry + $buffers)
units: %
every: 10s
warn: $this < (($status >= $WARNING) ? ( 5) : (10))
diff --git a/conf.d/health.d/redis.conf b/health/health.d/redis.conf
index c08a884a..c08a884a 100644
--- a/conf.d/health.d/redis.conf
+++ b/health/health.d/redis.conf
diff --git a/conf.d/health.d/retroshare.conf b/health/health.d/retroshare.conf
index 2344b60e..2344b60e 100644
--- a/conf.d/health.d/retroshare.conf
+++ b/health/health.d/retroshare.conf
diff --git a/conf.d/health.d/softnet.conf b/health/health.d/softnet.conf
index 77c804bf..77c804bf 100644
--- a/conf.d/health.d/softnet.conf
+++ b/health/health.d/softnet.conf
diff --git a/conf.d/health.d/squid.conf b/health/health.d/squid.conf
index 06cc9678..06cc9678 100644
--- a/conf.d/health.d/squid.conf
+++ b/health/health.d/squid.conf
diff --git a/conf.d/health.d/stiebeleltron.conf b/health/health.d/stiebeleltron.conf
index e0361eb2..e0361eb2 100644
--- a/conf.d/health.d/stiebeleltron.conf
+++ b/health/health.d/stiebeleltron.conf
diff --git a/conf.d/health.d/swap.conf b/health/health.d/swap.conf
index f920b080..f920b080 100644
--- a/conf.d/health.d/swap.conf
+++ b/health/health.d/swap.conf
diff --git a/conf.d/health.d/tcp_conn.conf b/health/health.d/tcp_conn.conf
index 7aa9a980..7aa9a980 100644
--- a/conf.d/health.d/tcp_conn.conf
+++ b/health/health.d/tcp_conn.conf
diff --git a/health/health.d/tcp_listen.conf b/health/health.d/tcp_listen.conf
new file mode 100644
index 00000000..552930ab
--- /dev/null
+++ b/health/health.d/tcp_listen.conf
@@ -0,0 +1,82 @@
+#
+# There are two queues involved when incoming TCP connections are handled
+# (both at the kernel):
+#
+# SYN queue
+# The SYN queue tracks TCP handshakes until connections are fully established.
+# It overflows when too many incoming TCP connection requests hang in the
+# half-open state and the server is not configured to fall back to SYN cookies.
+# Overflows are usually caused by SYN flood DoS attacks (i.e. someone sends
+# lots of SYN packets and never completes the handshakes).
+#
+# Accept queue
+# The accept queue holds fully established TCP connections waiting to be handled
+# by the listening application. It overflows when the server application fails
+# to accept new connections at the rate they are coming in.
+#
+#
+# -----------------------------------------------------------------------------
+# tcp accept queue (at the kernel)
+
+ alarm: 1m_tcp_accept_queue_overflows
+ on: ip.tcp_accept_queue
+ os: linux
+ hosts: *
+ lookup: sum -60s unaligned absolute of ListenOverflows
+ units: overflows
+ every: 10s
+ crit: $this > 0
+ delay: up 0 down 5m multiplier 1.5 max 1h
+ info: the number of times the TCP accept queue of the kernel overflown, during the last minute
+ to: sysadmin
+
+# THIS IS TOO GENERIC
+# CHECK: https://github.com/netdata/netdata/issues/3234#issuecomment-423935842
+ alarm: 1m_tcp_accept_queue_drops
+ on: ip.tcp_accept_queue
+ os: linux
+ hosts: *
+ lookup: sum -60s unaligned absolute of ListenDrops
+ units: drops
+ every: 10s
+# warn: $this > 0
+ crit: $this > (($status == $CRITICAL) ? (0) : (150))
+ delay: up 0 down 5m multiplier 1.5 max 1h
+ info: the number of times the TCP accept queue of the kernel dropped packets, during the last minute (includes bogus packets received)
+ to: sysadmin
+
+
+# -----------------------------------------------------------------------------
+# tcp SYN queue (at the kernel)
+
+# When the SYN queue is full, either TcpExtTCPReqQFullDoCookies or
+# TcpExtTCPReqQFullDrop is incremented, depending on whether SYN cookies are
+# enabled or not. In both cases this probably indicates a SYN flood attack,
+# so i guess a notification should be sent.
+
+ alarm: 1m_tcp_syn_queue_drops
+ on: ip.tcp_syn_queue
+ os: linux
+ hosts: *
+ lookup: sum -60s unaligned absolute of TCPReqQFullDrop
+ units: drops
+ every: 10s
+ warn: $this > 0
+ crit: $this > (($status == $CRITICAL) ? (0) : (60))
+ delay: up 0 down 5m multiplier 1.5 max 1h
+ info: the number of times the TCP SYN queue of the kernel was full and dropped packets, during the last minute
+ to: sysadmin
+
+ alarm: 1m_tcp_syn_queue_cookies
+ on: ip.tcp_syn_queue
+ os: linux
+ hosts: *
+ lookup: sum -60s unaligned absolute of TCPReqQFullDoCookies
+ units: cookies
+ every: 10s
+ warn: $this > 0
+ crit: $this > (($status == $CRITICAL) ? (0) : (60))
+ delay: up 0 down 5m multiplier 1.5 max 1h
+ info: the number of times the TCP SYN queue of the kernel was full and sent SYN cookies, during the last minute
+ to: sysadmin
+
diff --git a/conf.d/health.d/tcp_mem.conf b/health/health.d/tcp_mem.conf
index 6927d576..6927d576 100644
--- a/conf.d/health.d/tcp_mem.conf
+++ b/health/health.d/tcp_mem.conf
diff --git a/conf.d/health.d/tcp_orphans.conf b/health/health.d/tcp_orphans.conf
index 280d6590..280d6590 100644
--- a/conf.d/health.d/tcp_orphans.conf
+++ b/health/health.d/tcp_orphans.conf
diff --git a/conf.d/health.d/tcp_resets.conf b/health/health.d/tcp_resets.conf
index 91dad3c6..91dad3c6 100644
--- a/conf.d/health.d/tcp_resets.conf
+++ b/health/health.d/tcp_resets.conf
diff --git a/conf.d/health.d/udp_errors.conf b/health/health.d/udp_errors.conf
index 382b3965..5140228f 100644
--- a/conf.d/health.d/udp_errors.conf
+++ b/health/health.d/udp_errors.conf
@@ -27,7 +27,7 @@
units: errors
every: 10s
warn: $this > 0
- crit: $this > 100
+ crit: $this > (($status == $CRITICAL) ? (0) : (100))
info: number of UDP receive buffer errors during the last minute
delay: up 0 down 60m multiplier 1.2 max 2h
to: sysadmin
@@ -43,7 +43,7 @@
units: errors
every: 10s
warn: $this > 0
- crit: $this > 100
+ crit: $this > (($status == $CRITICAL) ? (0) : (100))
info: number of UDP send buffer errors during the last minute
delay: up 0 down 60m multiplier 1.2 max 2h
to: sysadmin
diff --git a/conf.d/health.d/varnish.conf b/health/health.d/varnish.conf
index cca7446b..cca7446b 100644
--- a/conf.d/health.d/varnish.conf
+++ b/health/health.d/varnish.conf
diff --git a/conf.d/health.d/web_log.conf b/health/health.d/web_log.conf
index d8be88b4..d8be88b4 100644
--- a/conf.d/health.d/web_log.conf
+++ b/health/health.d/web_log.conf
diff --git a/conf.d/health.d/zfs.conf b/health/health.d/zfs.conf
index af73824e..af73824e 100644
--- a/conf.d/health.d/zfs.conf
+++ b/health/health.d/zfs.conf
diff --git a/health/health.h b/health/health.h
new file mode 100644
index 00000000..ff7a4d9b
--- /dev/null
+++ b/health/health.h
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_HEALTH_H
+#define NETDATA_HEALTH_H 1
+
+#include "../daemon/common.h"
+
+#define NETDATA_PLUGIN_HOOK_HEALTH \
+ { \
+ .name = "HEALTH", \
+ .config_section = NULL, \
+ .config_name = NULL, \
+ .enabled = 1, \
+ .thread = NULL, \
+ .init_routine = NULL, \
+ .start_routine = health_main \
+ },
+
+extern unsigned int default_health_enabled;
+
+#define HEALTH_ENTRY_FLAG_PROCESSED 0x00000001
+#define HEALTH_ENTRY_FLAG_UPDATED 0x00000002
+#define HEALTH_ENTRY_FLAG_EXEC_RUN 0x00000004
+#define HEALTH_ENTRY_FLAG_EXEC_FAILED 0x00000008
+#define HEALTH_ENTRY_FLAG_SAVED 0x10000000
+#define HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION 0x80000000
+
+extern void health_init(void);
+extern void *health_main(void *ptr);
+
+extern void health_reload(void);
+
+extern int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result);
+extern void health_alarms2json(RRDHOST *host, BUFFER *wb, int all);
+extern void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after);
+
+void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf);
+
+extern int health_alarm_log_open(RRDHOST *host);
+extern void health_alarm_log_close(RRDHOST *host);
+extern void health_log_rotate(RRDHOST *host);
+extern void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
+extern ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename);
+extern void health_alarm_log_load(RRDHOST *host);
+
+extern void health_alarm_log(
+ RRDHOST *host,
+ uint32_t alarm_id,
+ uint32_t alarm_event_id,
+ time_t when,
+ const char *name,
+ const char *chart,
+ const char *family,
+ const char *exec,
+ const char *recipient,
+ time_t duration,
+ calculated_number old_value,
+ calculated_number new_value,
+ RRDCALC_STATUS old_status,
+ RRDCALC_STATUS new_status,
+ const char *source,
+ const char *units,
+ const char *info,
+ int delay,
+ uint32_t flags
+);
+
+extern void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath);
+extern char *health_user_config_dir(void);
+extern char *health_stock_config_dir(void);
+extern void health_reload_host(RRDHOST *host);
+extern void health_alarm_log_free(RRDHOST *host);
+
+extern void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae);
+
+#endif //NETDATA_HEALTH_H
diff --git a/src/health_config.c b/health/health_config.c
index a25ee722..d4af9776 100644
--- a/src/health_config.c
+++ b/health/health_config.c
@@ -1,5 +1,6 @@
-#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "health.h"
#define HEALTH_CONF_MAX_LINE 4096
@@ -52,7 +53,7 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
(rc->recipient)?rc->recipient:"DEFAULT",
rc->green,
rc->red,
- rc->group,
+ (int)rc->group,
rc->after,
rc->before,
rc->options,
@@ -106,7 +107,7 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL
(rt->recipient)?rt->recipient:"DEFAULT",
rt->green,
rt->red,
- rt->group,
+ (int)rt->group,
rt->after,
rt->before,
rt->options,
@@ -176,7 +177,7 @@ static inline int health_parse_duration(char *string, int *result) {
}
static inline int health_parse_delay(
- size_t line, const char *path, const char *file, char *string,
+ size_t line, const char *filename, char *string,
int *delay_up_duration,
int *delay_down_duration,
int *delay_max_duration,
@@ -202,36 +203,36 @@ static inline int health_parse_delay(
if(!strcasecmp(key, "up")) {
if (!health_parse_duration(value, delay_up_duration)) {
- error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
- line, path, file, value, key);
+ error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",
+ line, filename, value, key);
}
else given_up = 1;
}
else if(!strcasecmp(key, "down")) {
if (!health_parse_duration(value, delay_down_duration)) {
- error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
- line, path, file, value, key);
+ error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",
+ line, filename, value, key);
}
else given_down = 1;
}
else if(!strcasecmp(key, "multiplier")) {
*delay_multiplier = strtof(value, NULL);
if(isnan(*delay_multiplier) || isinf(*delay_multiplier) || islessequal(*delay_multiplier, 0)) {
- error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
- line, path, file, value, key);
+ error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",
+ line, filename, value, key);
}
else given_multiplier = 1;
}
else if(!strcasecmp(key, "max")) {
if (!health_parse_duration(value, delay_max_duration)) {
- error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
- line, path, file, value, key);
+ error("Health configuration at line %zu of file '%s': invalid value '%s' for '%s' keyword",
+ line, filename, value, key);
}
else given_max = 1;
}
else {
- error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
- line, path, file, key);
+ error("Health configuration at line %zu of file '%s': unknown keyword '%s'",
+ line, filename, key);
}
}
@@ -285,11 +286,11 @@ static inline uint32_t health_parse_options(const char *s) {
}
static inline int health_parse_db_lookup(
- size_t line, const char *path, const char *file, char *string,
- int *group_method, int *after, int *before, int *every,
+ size_t line, const char *filename, char *string,
+ RRDR_GROUPING *group_method, int *after, int *before, int *every,
uint32_t *options, char **dimensions
) {
- debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
+ debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s: %s", line, filename, string);
if(*dimensions) freez(*dimensions);
*dimensions = NULL;
@@ -305,14 +306,14 @@ static inline int health_parse_db_lookup(
while(*s && !isspace(*s)) s++;
while(*s && isspace(*s)) *s++ = '\0';
if(!*s) {
- error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
- line, path, file, key);
+ error("Health configuration invalid chart calculation at line %zu of file '%s': expected group method followed by the 'after' time, but got '%s'",
+ line, filename, key);
return 0;
}
- if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
- error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
- line, path, file, key);
+ if((*group_method = web_client_api_request_v1_data_group(key, RRDR_GROUPING_UNDEFINED)) == RRDR_GROUPING_UNDEFINED) {
+ error("Health configuration at line %zu of file '%s': invalid group method '%s'",
+ line, filename, key);
return 0;
}
@@ -322,8 +323,8 @@ static inline int health_parse_db_lookup(
while(*s && isspace(*s)) *s++ = '\0';
if(!health_parse_duration(key, after)) {
- error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
- line, path, file, key);
+ error("Health configuration at line %zu of file '%s': invalid duration '%s' after group method",
+ line, filename, key);
return 0;
}
@@ -343,8 +344,8 @@ static inline int health_parse_db_lookup(
while(*s && isspace(*s)) *s++ = '\0';
if (!health_parse_duration(value, before)) {
- error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
- line, path, file, value, key);
+ error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword",
+ line, filename, value, key);
}
}
else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
@@ -353,8 +354,8 @@ static inline int health_parse_db_lookup(
while(*s && isspace(*s)) *s++ = '\0';
if (!health_parse_duration(value, every)) {
- error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
- line, path, file, value, key);
+ error("Health configuration at line %zu of file '%s': invalid duration '%s' for '%s' keyword",
+ line, filename, value, key);
}
}
else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
@@ -384,17 +385,17 @@ static inline int health_parse_db_lookup(
break;
}
else {
- error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
- line, path, file, key);
+ error("Health configuration at line %zu of file '%s': unknown keyword '%s'",
+ line, filename, key);
}
}
return 1;
}
-static inline char *health_source_file(size_t line, const char *path, const char *filename) {
+static inline char *health_source_file(size_t line, const char *file) {
char buffer[FILENAME_MAX + 1];
- snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
+ snprintfz(buffer, FILENAME_MAX, "%zu@%s", line, file);
return strdupz(buffer);
}
@@ -405,8 +406,10 @@ static inline void strip_quotes(char *s) {
}
}
-int health_readfile(RRDHOST *host, const char *path, const char *filename) {
- debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
+static int health_readfile(const char *filename, void *data) {
+ RRDHOST *host = (RRDHOST *)data;
+
+ debug(D_HEALTH, "Health configuration reading file '%s'", filename);
static uint32_t
hash_alarm = 0,
@@ -453,10 +456,9 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
}
- snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
- FILE *fp = fopen(buffer, "r");
+ FILE *fp = fopen(filename, "r");
if(!fp) {
- error("Health configuration cannot read file '%s'.", buffer);
+ error("Health configuration cannot read file '%s'.", filename);
return 0;
}
@@ -479,7 +481,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
if(append < HEALTH_CONF_MAX_LINE)
continue;
else {
- error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
+ error("Health configuration has too long muli-line at line %zu of file '%s'.", line, filename);
}
}
append = 0;
@@ -487,7 +489,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
char *key = s;
while(*s && *s != ':') s++;
if(!*s) {
- error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
+ error("Health configuration has invalid line %zu of file '%s'. It does not contain a ':'. Ignoring it.", line, filename);
continue;
}
*s = '\0';
@@ -498,12 +500,12 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
value = trim_all(value);
if(!key) {
- error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
+ error("Health configuration has invalid line %zu of file '%s'. Keyword is empty. Ignoring it.", line, filename);
continue;
}
if(!value) {
- error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
+ error("Health configuration has invalid line %zu of file '%s'. value is empty. Ignoring it.", line, filename);
continue;
}
@@ -524,7 +526,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
rc->next_event_id = 1;
rc->name = strdupz(value);
rc->hash = simple_hash(rc->name);
- rc->source = health_source_file(line, path, filename);
+ rc->source = health_source_file(line, filename);
rc->green = NAN;
rc->red = NAN;
rc->value = NAN;
@@ -550,7 +552,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
rt = callocz(1, sizeof(RRDCALCTEMPLATE));
rt->name = strdupz(value);
rt->hash_name = simple_hash(rt->name);
- rt->source = health_source_file(line, path, filename);
+ rt->source = health_source_file(line, filename);
rt->green = NAN;
rt->red = NAN;
rt->delay_multiplier = 1.0;
@@ -566,10 +568,10 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
if(!simple_pattern_matches(os_pattern, host->os)) {
if(rc)
- debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s/%s: host O/S does not match '%s'", host->hostname, rc->name, line, path, filename, os_match);
+ debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: host O/S does not match '%s'", host->hostname, rc->name, line, filename, os_match);
if(rt)
- debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s/%s: host O/S does not match '%s'", host->hostname, rt->name, line, path, filename, os_match);
+ debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s: host O/S does not match '%s'", host->hostname, rt->name, line, filename, os_match);
ignore_this = 1;
}
@@ -582,10 +584,10 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
if(!simple_pattern_matches(host_pattern, host->hostname)) {
if(rc)
- debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s/%s: hostname does not match '%s'", host->hostname, rc->name, line, path, filename, host_match);
+ debug(D_HEALTH, "HEALTH on '%s' ignoring alarm '%s' defined at %zu@%s: hostname does not match '%s'", host->hostname, rc->name, line, filename, host_match);
if(rt)
- debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s/%s: hostname does not match '%s'", host->hostname, rt->name, line, path, filename, host_match);
+ debug(D_HEALTH, "HEALTH on '%s' ignoring template '%s' defined at %zu@%s: hostname does not match '%s'", host->hostname, rt->name, line, filename, host_match);
ignore_this = 1;
}
@@ -596,8 +598,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
if(rc->chart) {
if(strcmp(rc->chart, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->chart, value, value);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rc->name, key, rc->chart, value, value);
freez(rc->chart);
}
@@ -605,29 +607,29 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
rc->hash_chart = simple_hash(rc->chart);
}
else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
- health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
+ health_parse_db_lookup(line, filename, value, &rc->group, &rc->after, &rc->before,
&rc->update_every,
&rc->options, &rc->dimensions);
}
else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
if(!health_parse_duration(value, &rc->update_every))
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
- line, path, filename, rc->name, key, value);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
+ line, filename, rc->name, key, value);
}
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
char *e;
rc->green = str2ld(value, &e);
if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rc->name, key, e);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
+ line, filename, rc->name, key, e);
}
}
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
char *e;
rc->red = str2ld(value, &e);
if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rc->name, key, e);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
+ line, filename, rc->name, key, e);
}
}
else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
@@ -635,8 +637,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
int error = 0;
rc->calculation = expression_parse(value, &failed_at, &error);
if(!rc->calculation) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
+ line, filename, rc->name, key, value, expression_strerror(error), failed_at);
}
}
else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
@@ -644,8 +646,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
int error = 0;
rc->warning = expression_parse(value, &failed_at, &error);
if(!rc->warning) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
+ line, filename, rc->name, key, value, expression_strerror(error), failed_at);
}
}
else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
@@ -653,15 +655,15 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
int error = 0;
rc->critical = expression_parse(value, &failed_at, &error);
if(!rc->critical) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
+ line, filename, rc->name, key, value, expression_strerror(error), failed_at);
}
}
else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
if(rc->exec) {
if(strcmp(rc->exec, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->exec, value, value);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rc->name, key, rc->exec, value, value);
freez(rc->exec);
}
@@ -670,8 +672,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
if(rc->recipient) {
if(strcmp(rc->recipient, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->recipient, value, value);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rc->name, key, rc->recipient, value, value);
freez(rc->recipient);
}
@@ -680,8 +682,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
if(rc->units) {
if(strcmp(rc->units, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->units, value, value);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rc->name, key, rc->units, value, value);
freez(rc->units);
}
@@ -691,8 +693,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
if(rc->info) {
if(strcmp(rc->info, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->info, value, value);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rc->name, key, rc->info, value, value);
freez(rc->info);
}
@@ -700,22 +702,22 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
strip_quotes(rc->info);
}
else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
- health_parse_delay(line, path, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
+ health_parse_delay(line, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
}
else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
rc->options |= health_parse_options(value);
}
else {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
- line, path, filename, rc->name, key);
+ error("Health configuration at line %zu of file '%s' for alarm '%s' has unknown key '%s'.",
+ line, filename, rc->name, key);
}
}
else if(rt) {
if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
if(rt->context) {
if(strcmp(rt->context, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->context, value, value);
+ error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rt->name, key, rt->context, value, value);
freez(rt->context);
}
@@ -730,28 +732,28 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
rt->family_pattern = simple_pattern_create(rt->family_match, NULL, SIMPLE_PATTERN_EXACT);
}
else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
- health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
+ health_parse_db_lookup(line, filename, value, &rt->group, &rt->after, &rt->before,
&rt->update_every, &rt->options, &rt->dimensions);
}
else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
if(!health_parse_duration(value, &rt->update_every))
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
- line, path, filename, rt->name, key, value);
+ error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
+ line, filename, rt->name, key, value);
}
else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
char *e;
rt->green = str2ld(value, &e);
if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rt->name, key, e);
+ error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
+ line, filename, rt->name, key, e);
}
}
else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
char *e;
rt->red = str2ld(value, &e);
if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rt->name, key, e);
+ error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
+ line, filename, rt->name, key, e);
}
}
else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
@@ -759,8 +761,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
int error = 0;
rt->calculation = expression_parse(value, &failed_at, &error);
if(!rt->calculation) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
+ error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
+ line, filename, rt->name, key, value, expression_strerror(error), failed_at);
}
}
else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
@@ -768,8 +770,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
int error = 0;
rt->warning = expression_parse(value, &failed_at, &error);
if(!rt->warning) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
+ error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
+ line, filename, rt->name, key, value, expression_strerror(error), failed_at);
}
}
else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
@@ -777,15 +779,15 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
int error = 0;
rt->critical = expression_parse(value, &failed_at, &error);
if(!rt->critical) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
+ error("Health configuration at line %zu of file '%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
+ line, filename, rt->name, key, value, expression_strerror(error), failed_at);
}
}
else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
if(rt->exec) {
if(strcmp(rt->exec, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->exec, value, value);
+ error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rt->name, key, rt->exec, value, value);
freez(rt->exec);
}
@@ -794,8 +796,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
if(rt->recipient) {
if(strcmp(rt->recipient, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->recipient, value, value);
+ error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rt->name, key, rt->recipient, value, value);
freez(rt->recipient);
}
@@ -804,8 +806,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
if(rt->units) {
if(strcmp(rt->units, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->units, value, value);
+ error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rt->name, key, rt->units, value, value);
freez(rt->units);
}
@@ -815,8 +817,8 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
if(rt->info) {
if(strcmp(rt->info, value) != 0)
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->info, value, value);
+ error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
+ line, filename, rt->name, key, rt->info, value, value);
freez(rt->info);
}
@@ -824,19 +826,19 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
strip_quotes(rt->info);
}
else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
- health_parse_delay(line, path, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
+ health_parse_delay(line, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
}
else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
rt->options |= health_parse_options(value);
}
else {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
- line, path, filename, rt->name, key);
+ error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",
+ line, filename, rt->name, key);
}
}
else {
- error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
- line, path, filename, key);
+ error("Health configuration at line %zu of file '%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
+ line, filename, key);
}
}
@@ -850,51 +852,7 @@ int health_readfile(RRDHOST *host, const char *path, const char *filename) {
return 1;
}
-void health_readdir(RRDHOST *host, const char *path) {
- if(!host->health_enabled) return;
-
- size_t pathlen = strlen(path);
-
- debug(D_HEALTH, "Health configuration reading directory '%s'", path);
-
- DIR *dir = opendir(path);
- if (!dir) {
- error("Health configuration cannot open directory '%s'.", path);
- return;
- }
-
- struct dirent *de = NULL;
- while ((de = readdir(dir))) {
- size_t len = strlen(de->d_name);
-
- if(de->d_type == DT_DIR
- && (
- (de->d_name[0] == '.' && de->d_name[1] == '\0')
- || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
- )) {
- debug(D_HEALTH, "Ignoring directory '%s'", de->d_name);
- continue;
- }
-
- else if(de->d_type == DT_DIR) {
- char *s = mallocz(pathlen + strlen(de->d_name) + 2);
- strcpy(s, path);
- strcat(s, "/");
- strcat(s, de->d_name);
- health_readdir(host, s);
- freez(s);
- continue;
- }
-
- else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
- len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
- health_readfile(host, path, de->d_name);
- }
-
- else debug(D_HEALTH, "Ignoring file '%s'", de->d_name);
- }
-
- closedir(dir);
+void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath) {
+ if(unlikely(!host->health_enabled)) return;
+ recursive_config_double_dir_load(user_path, stock_path, subpath, health_readfile, (void *) host, 0);
}
-
-
diff --git a/src/health_json.c b/health/health_json.c
index aba7425d..a049dc1b 100644
--- a/src/health_json.c
+++ b/health/health_json.c
@@ -1,5 +1,6 @@
-#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "health.h"
static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
if(value && *value) {
diff --git a/src/health_log.c b/health/health_log.c
index a44fbadb..dd51be2a 100644
--- a/src/health_log.c
+++ b/health/health_log.c
@@ -1,5 +1,6 @@
-#define NETDATA_HEALTH_INTERNALS
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "health.h"
// ----------------------------------------------------------------------------
// health alarm log load/save
diff --git a/health/notifications/Makefile.am b/health/notifications/Makefile.am
new file mode 100644
index 00000000..a5b88f0d
--- /dev/null
+++ b/health/notifications/Makefile.am
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+CLEANFILES = \
+ alarm-notify.sh \
+ $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_libconfig_DATA = \
+ health_alarm_notify.conf \
+ health_email_recipients.conf \
+ $(NULL)
+
+dist_plugins_SCRIPTS = \
+ alarm-notify.sh \
+ alarm-email.sh \
+ alarm-test.sh \
+ $(NULL)
+
+dist_noinst_DATA = \
+ alarm-notify.sh.in \
+ README.md \
+ $(NULL)
+
+include alerta/Makefile.inc
+include awssns/Makefile.inc
+include discord/Makefile.inc
+include email/Makefile.inc
+include flock/Makefile.inc
+include irc/Makefile.inc
+include kavenegar/Makefile.inc
+include messagebird/Makefile.inc
+include pagerduty/Makefile.inc
+include pushbullet/Makefile.inc
+include pushover/Makefile.inc
+include rocketchat/Makefile.inc
+include slack/Makefile.inc
+include syslog/Makefile.inc
+include telegram/Makefile.inc
+include twilio/Makefile.inc
+include web/Makefile.inc
diff --git a/health/notifications/Makefile.in b/health/notifications/Makefile.in
new file mode 100644
index 00000000..05a5814f
--- /dev/null
+++ b/health/notifications/Makefile.in
@@ -0,0 +1,754 @@
+# Makefile.in generated by automake 1.14.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2013 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+
+VPATH = @srcdir@
+am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)'
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+DIST_COMMON = $(top_srcdir)/build/subst.inc \
+ $(srcdir)/alerta/Makefile.inc $(srcdir)/awssns/Makefile.inc \
+ $(srcdir)/discord/Makefile.inc $(srcdir)/email/Makefile.inc \
+ $(srcdir)/flock/Makefile.inc $(srcdir)/irc/Makefile.inc \
+ $(srcdir)/kavenegar/Makefile.inc \
+ $(srcdir)/messagebird/Makefile.inc \
+ $(srcdir)/pagerduty/Makefile.inc \
+ $(srcdir)/pushbullet/Makefile.inc \
+ $(srcdir)/pushover/Makefile.inc \
+ $(srcdir)/rocketchat/Makefile.inc $(srcdir)/slack/Makefile.inc \
+ $(srcdir)/syslog/Makefile.inc $(srcdir)/telegram/Makefile.inc \
+ $(srcdir)/twilio/Makefile.inc $(srcdir)/web/Makefile.inc \
+ $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
+ $(dist_plugins_SCRIPTS) $(dist_libconfig_DATA) \
+ $(dist_noinst_DATA)
+subdir = health/notifications
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/build/m4/ax_c___atomic.m4 \
+ $(top_srcdir)/build/m4/ax_c__generic.m4 \
+ $(top_srcdir)/build/m4/ax_c_lto.m4 \
+ $(top_srcdir)/build/m4/ax_c_mallinfo.m4 \
+ $(top_srcdir)/build/m4/ax_c_mallopt.m4 \
+ $(top_srcdir)/build/m4/ax_check_compile_flag.m4 \
+ $(top_srcdir)/build/m4/ax_gcc_func_attribute.m4 \
+ $(top_srcdir)/build/m4/ax_pthread.m4 \
+ $(top_srcdir)/build/m4/jemalloc.m4 \
+ $(top_srcdir)/build/m4/tcmalloc.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+ test -z "$$files" \
+ || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+ $(am__cd) "$$dir" && rm -f $$files; }; \
+ }
+am__installdirs = "$(DESTDIR)$(pluginsdir)" \
+ "$(DESTDIR)$(libconfigdir)"
+SCRIPTS = $(dist_plugins_SCRIPTS)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
+SOURCES =
+DIST_SOURCES =
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
+DATA = $(dist_libconfig_DATA) $(dist_noinst_DATA)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+IPMIMONITORING_CFLAGS = @IPMIMONITORING_CFLAGS@
+IPMIMONITORING_LIBS = @IPMIMONITORING_LIBS@
+LDFLAGS = @LDFLAGS@
+LIBCAP_CFLAGS = @LIBCAP_CFLAGS@
+LIBCAP_LIBS = @LIBCAP_LIBS@
+LIBMNL_CFLAGS = @LIBMNL_CFLAGS@
+LIBMNL_LIBS = @LIBMNL_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LTLIBOBJS = @LTLIBOBJS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MATH_CFLAGS = @MATH_CFLAGS@
+MATH_LIBS = @MATH_LIBS@
+MKDIR_P = @MKDIR_P@
+NFACCT_CFLAGS = @NFACCT_CFLAGS@
+NFACCT_LIBS = @NFACCT_LIBS@
+OBJEXT = @OBJEXT@
+OPTIONAL_IPMIMONITORING_CFLAGS = @OPTIONAL_IPMIMONITORING_CFLAGS@
+OPTIONAL_IPMIMONITORING_LIBS = @OPTIONAL_IPMIMONITORING_LIBS@
+OPTIONAL_LIBCAP_CFLAGS = @OPTIONAL_LIBCAP_CFLAGS@
+OPTIONAL_LIBCAP_LIBS = @OPTIONAL_LIBCAP_LIBS@
+OPTIONAL_MATH_CLFAGS = @OPTIONAL_MATH_CLFAGS@
+OPTIONAL_MATH_LIBS = @OPTIONAL_MATH_LIBS@
+OPTIONAL_NFACCT_CLFAGS = @OPTIONAL_NFACCT_CLFAGS@
+OPTIONAL_NFACCT_LIBS = @OPTIONAL_NFACCT_LIBS@
+OPTIONAL_UUID_CLFAGS = @OPTIONAL_UUID_CLFAGS@
+OPTIONAL_UUID_LIBS = @OPTIONAL_UUID_LIBS@
+OPTIONAL_ZLIB_CLFAGS = @OPTIONAL_ZLIB_CLFAGS@
+OPTIONAL_ZLIB_LIBS = @OPTIONAL_ZLIB_LIBS@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_RPM_RELEASE = @PACKAGE_RPM_RELEASE@
+PACKAGE_RPM_VERSION = @PACKAGE_RPM_VERSION@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SSE_CANDIDATE = @SSE_CANDIDATE@
+STRIP = @STRIP@
+UUID_CFLAGS = @UUID_CFLAGS@
+UUID_LIBS = @UUID_LIBS@
+VERSION = @VERSION@
+ZLIB_CFLAGS = @ZLIB_CFLAGS@
+ZLIB_LIBS = @ZLIB_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_CC = @ac_ct_CC@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_target = @build_target@
+build_vendor = @build_vendor@
+builddir = @builddir@
+cachedir = @cachedir@
+chartsdir = @chartsdir@
+configdir = @configdir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+has_jemalloc = @has_jemalloc@
+has_tcmalloc = @has_tcmalloc@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libconfigdir = @libconfigdir@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+logdir = @logdir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+nodedir = @nodedir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+pluginsdir = @pluginsdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+pythondir = @pythondir@
+registrydir = @registrydir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+varlibdir = @varlibdir@
+webdir = @webdir@
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+CLEANFILES = \
+ alarm-notify.sh \
+ $(NULL)
+
+SUFFIXES = .in
+dist_libconfig_DATA = \
+ health_alarm_notify.conf \
+ health_email_recipients.conf \
+ $(NULL)
+
+dist_plugins_SCRIPTS = \
+ alarm-notify.sh \
+ alarm-email.sh \
+ alarm-test.sh \
+ $(NULL)
+
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+
+# install these files
+dist_noinst_DATA = alarm-notify.sh.in README.md $(NULL) \
+ alerta/README.md alerta/Makefile.inc $(NULL) awssns/README.md \
+ awssns/Makefile.inc $(NULL) discord/README.md \
+ discord/Makefile.inc $(NULL) email/README.md \
+ email/Makefile.inc $(NULL) flock/README.md flock/Makefile.inc \
+ $(NULL) irc/README.md irc/Makefile.inc $(NULL) \
+ kavenegar/README.md kavenegar/Makefile.inc $(NULL) \
+ messagebird/README.md messagebird/Makefile.inc $(NULL) \
+ pagerduty/README.md pagerduty/Makefile.inc $(NULL) \
+ pushbullet/README.md pushbullet/Makefile.inc $(NULL) \
+ pushover/README.md pushover/Makefile.inc $(NULL) \
+ rocketchat/README.md rocketchat/Makefile.inc $(NULL) \
+ slack/README.md slack/Makefile.inc $(NULL) syslog/README.md \
+ syslog/Makefile.inc $(NULL) telegram/README.md \
+ telegram/Makefile.inc $(NULL) twilio/README.md \
+ twilio/Makefile.inc $(NULL) web/README.md web/Makefile.inc \
+ $(NULL)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .in
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(top_srcdir)/build/subst.inc $(srcdir)/alerta/Makefile.inc $(srcdir)/awssns/Makefile.inc $(srcdir)/discord/Makefile.inc $(srcdir)/email/Makefile.inc $(srcdir)/flock/Makefile.inc $(srcdir)/irc/Makefile.inc $(srcdir)/kavenegar/Makefile.inc $(srcdir)/messagebird/Makefile.inc $(srcdir)/pagerduty/Makefile.inc $(srcdir)/pushbullet/Makefile.inc $(srcdir)/pushover/Makefile.inc $(srcdir)/rocketchat/Makefile.inc $(srcdir)/slack/Makefile.inc $(srcdir)/syslog/Makefile.inc $(srcdir)/telegram/Makefile.inc $(srcdir)/twilio/Makefile.inc $(srcdir)/web/Makefile.inc $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu health/notifications/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --gnu health/notifications/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+ esac;
+$(top_srcdir)/build/subst.inc $(srcdir)/alerta/Makefile.inc $(srcdir)/awssns/Makefile.inc $(srcdir)/discord/Makefile.inc $(srcdir)/email/Makefile.inc $(srcdir)/flock/Makefile.inc $(srcdir)/irc/Makefile.inc $(srcdir)/kavenegar/Makefile.inc $(srcdir)/messagebird/Makefile.inc $(srcdir)/pagerduty/Makefile.inc $(srcdir)/pushbullet/Makefile.inc $(srcdir)/pushover/Makefile.inc $(srcdir)/rocketchat/Makefile.inc $(srcdir)/slack/Makefile.inc $(srcdir)/syslog/Makefile.inc $(srcdir)/telegram/Makefile.inc $(srcdir)/twilio/Makefile.inc $(srcdir)/web/Makefile.inc:
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+install-dist_pluginsSCRIPTS: $(dist_plugins_SCRIPTS)
+ @$(NORMAL_INSTALL)
+ @list='$(dist_plugins_SCRIPTS)'; test -n "$(pluginsdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(pluginsdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(pluginsdir)" || exit 1; \
+ fi; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \
+ done | \
+ sed -e 'p;s,.*/,,;n' \
+ -e 'h;s|.*|.|' \
+ -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \
+ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \
+ { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+ if ($$2 == $$4) { files[d] = files[d] " " $$1; \
+ if (++n[d] == $(am__install_max)) { \
+ print "f", d, files[d]; n[d] = 0; files[d] = "" } } \
+ else { print "f", d "/" $$4, $$1 } } \
+ END { for (d in files) print "f", d, files[d] }' | \
+ while read type dir files; do \
+ if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+ test -z "$$files" || { \
+ echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(pluginsdir)$$dir'"; \
+ $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(pluginsdir)$$dir" || exit $$?; \
+ } \
+ ; done
+
+uninstall-dist_pluginsSCRIPTS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(dist_plugins_SCRIPTS)'; test -n "$(pluginsdir)" || exit 0; \
+ files=`for p in $$list; do echo "$$p"; done | \
+ sed -e 's,.*/,,;$(transform)'`; \
+ dir='$(DESTDIR)$(pluginsdir)'; $(am__uninstall_files_from_dir)
+install-dist_libconfigDATA: $(dist_libconfig_DATA)
+ @$(NORMAL_INSTALL)
+ @list='$(dist_libconfig_DATA)'; test -n "$(libconfigdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(libconfigdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(libconfigdir)" || exit 1; \
+ fi; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(libconfigdir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(libconfigdir)" || exit $$?; \
+ done
+
+uninstall-dist_libconfigDATA:
+ @$(NORMAL_UNINSTALL)
+ @list='$(dist_libconfig_DATA)'; test -n "$(libconfigdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ dir='$(DESTDIR)$(libconfigdir)'; $(am__uninstall_files_from_dir)
+tags TAGS:
+
+ctags CTAGS:
+
+cscope cscopelist:
+
+
+distdir: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+check: check-am
+all-am: Makefile $(SCRIPTS) $(DATA)
+installdirs:
+ for dir in "$(DESTDIR)$(pluginsdir)" "$(DESTDIR)$(libconfigdir)"; do \
+ test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+ done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+ if test -z '$(STRIP)'; then \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ install; \
+ else \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+ fi
+mostlyclean-generic:
+
+clean-generic:
+ -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+ -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES)
+clean: clean-am
+
+clean-am: clean-generic mostlyclean-am
+
+distclean: distclean-am
+ -rm -f Makefile
+distclean-am: clean-am distclean-generic
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-dist_libconfigDATA \
+ install-dist_pluginsSCRIPTS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-generic
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-dist_libconfigDATA \
+ uninstall-dist_pluginsSCRIPTS
+
+.MAKE: install-am install-strip
+
+.PHONY: all all-am check check-am clean clean-generic cscopelist-am \
+ ctags-am distclean distclean-generic distdir dvi dvi-am html \
+ html-am info info-am install install-am install-data \
+ install-data-am install-dist_libconfigDATA \
+ install-dist_pluginsSCRIPTS install-dvi install-dvi-am \
+ install-exec install-exec-am install-html install-html-am \
+ install-info install-info-am install-man install-pdf \
+ install-pdf-am install-ps install-ps-am install-strip \
+ installcheck installcheck-am installdirs maintainer-clean \
+ maintainer-clean-generic mostlyclean mostlyclean-generic pdf \
+ pdf-am ps ps-am tags-am uninstall uninstall-am \
+ uninstall-dist_libconfigDATA uninstall-dist_pluginsSCRIPTS
+
+.in:
+ if sed \
+ -e 's#[@]localstatedir_POST@#$(localstatedir)#g' \
+ -e 's#[@]sbindir_POST@#$(sbindir)#g' \
+ -e 's#[@]sysconfdir_POST@#$(sysconfdir)#g' \
+ -e 's#[@]pythondir_POST@#$(pythondir)#g' \
+ -e 's#[@]configdir_POST@#$(configdir)#g' \
+ -e 's#[@]libconfigdir_POST@#$(libconfigdir)#g' \
+ -e 's#[@]cachedir_POST@#$(cachedir)#g' \
+ $< > $@.tmp; then \
+ mv "$@.tmp" "$@"; \
+ else \
+ rm -f "$@.tmp"; \
+ false; \
+ fi
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/health/notifications/README.md b/health/notifications/README.md
new file mode 100644
index 00000000..c06638ad
--- /dev/null
+++ b/health/notifications/README.md
@@ -0,0 +1,60 @@
+# Netdata alarm notifications
+
+The `exec` line in health configuration defines an external script that will be called once
+the alarm is triggered. The default script is **[alarm-notify.sh](alarm-notify.sh.in)**.
+
+You can change the default script globally by editing `/etc/netdata/netdata.conf`.
+
+`alarm-notify.sh` is capable of sending notifications:
+
+- to multiple recipients
+- using multiple notification methods
+- filtering severity per recipient
+
+It uses **roles**. For example `sysadmin`, `webmaster`, `dba`, etc.
+
+Each alarm is assigned to one or more roles, using the `to` line of the alarm configuration.
+Then `alarm-notify.sh` uses its own configuration file `/etc/netdata/health_alarm_notify.conf`
+the default is [here](health_alarm_notify.conf)
+(to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`)
+to find the destination address of the notification for each method.
+
+Each role may have one or more destinations.
+
+So, for example the `sysadmin` role may send:
+
+1. emails to admin1@example.com and admin2@example.com
+2. pushover.net notifications to USERTOKENS `A`, `B` and `C`.
+3. pushbullet.com push notifications to admin1@example.com and admin2@example.com
+4. messages to slack.com channel `#alarms` and `#systems`.
+5. messages to Discord channels `#alarms` and `#systems`.
+
+## Configuration
+
+Edit [`/etc/netdata/health_alarm_notify.conf`](health_alarm_notify.conf)
+by running `/etc/netdata/edit-config health_alarm_notify.conf`:
+
+- settings per notification method:
+
+ all notification methods except email, require some configuration
+ (i.e. API keys, tokens, destination rooms, channels, etc).
+
+2. **recipients** per **role** per **notification method**
+
+## Testing Notifications
+
+You can run the following command by hand, to test alarms configuration:
+
+```sh
+# become user netdata
+su -s /bin/bash netdata
+
+# enable debugging info on the console
+export NETDATA_ALARM_NOTIFY_DEBUG=1
+
+# send test alarms to sysadmin
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test
+
+# send test alarms to any role
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test "ROLE"
+```
diff --git a/plugins.d/alarm-email.sh b/health/notifications/alarm-email.sh
index df083c65..69c4c3f8 100755
--- a/plugins.d/alarm-email.sh
+++ b/health/notifications/alarm-email.sh
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
+# SPDX-License-Identifier: GPL-3.0-or-later
# OBSOLETE - REPLACED WITH
# alarm-notify.sh
diff --git a/plugins.d/alarm-notify.sh b/health/notifications/alarm-notify.sh
index 3e23a164..33a59590 100755..100644
--- a/plugins.d/alarm-notify.sh
+++ b/health/notifications/alarm-notify.sh
@@ -3,7 +3,7 @@
# netdata
# real-time performance and health monitoring, done right!
# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
-# GPL v3+
+# SPDX-License-Identifier: GPL-3.0-or-later
#
# Script to send alarm notifications for netdata
#
@@ -26,11 +26,15 @@
# - pagerduty.com notifications by Jim Cooley @jimcooley #1373
# - messagebird.com notifications by @tech_no_logical #1453
# - hipchat notifications by @ktsaou #1561
+# - fleep notifications by @Ferroin
# - custom notifications by @ktsaou
+# - syslog messages by @Ferroin
+# - Microsoft Team notification by @tioumen
# -----------------------------------------------------------------------------
# testing notifications
+
if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ]
then
if [ "${2}" = "test" ]
@@ -44,6 +48,7 @@ then
id=1
last="CLEAR"
+ test_res=0
for x in "WARNING" "CRITICAL" "CLEAR"
do
echo >&2
@@ -53,6 +58,7 @@ then
if [ $? -ne 0 ]
then
echo >&2 "# FAILED"
+ test_res=1
else
echo >&2 "# OK"
fi
@@ -61,7 +67,7 @@ then
id=$((id + 1))
done
- exit 1
+ exit $test_res
fi
export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
@@ -153,9 +159,10 @@ custom_sender() {
# -----------------------------------------------------------------------------
# defaults to allow running this script by hand
-[ -z "${NETDATA_CONFIG_DIR}" ] && NETDATA_CONFIG_DIR="$(dirname "${0}")/../../../../etc/netdata"
-[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="$(dirname "${0}")/../../../../var/cache/netdata"
-[ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
+[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="/usr/local/etc/netdata"
+[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="/usr/local/lib/netdata/conf.d"
+[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="/usr/local/var/cache/netdata"
+[ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
# -----------------------------------------------------------------------------
# parse command line parameters
@@ -214,7 +221,7 @@ fi
images_base_url="https://registry.my-netdata.io"
# curl options to use
-curl_options=
+curl_options=""
# needed commands
# if empty they will be searched in the system path
@@ -223,6 +230,7 @@ sendmail=
# enable / disable features
SEND_SLACK="YES"
+SEND_MSTEAM="YES"
SEND_ALERTA="YES"
SEND_FLOCK="YES"
SEND_DISCORD="YES"
@@ -236,7 +244,10 @@ SEND_EMAIL="YES"
SEND_PUSHBULLET="YES"
SEND_KAFKA="YES"
SEND_PD="YES"
+SEND_FLEEP="YES"
SEND_IRC="YES"
+SEND_AWSSNS="YES"
+SEND_SYSLOG="NO"
SEND_CUSTOM="YES"
# slack configs
@@ -244,6 +255,16 @@ SLACK_WEBHOOK_URL=
DEFAULT_RECIPIENT_SLACK=
declare -A role_recipients_slack=()
+# Microsoft Team configs
+MSTEAM_WEBHOOK_URL=
+DEFAULT_RECIPIENT_MSTEAM=
+declare -A role_recipients_msteam=()
+
+# rocketchat configs
+ROCKETCHAT_WEBHOOK_URL=
+DEFAULT_RECIPIENT_ROCKETCHAT=
+declare -A role_recipients_rocketchat=()
+
# alerta configs
ALERTA_WEBHOOK_URL=
ALERTA_API_KEY=
@@ -310,6 +331,20 @@ PD_SERVICE_KEY=
DEFAULT_RECIPIENT_PD=
declare -A role_recipients_pd=()
+# fleep.io configs
+FLEEP_SENDER="${host}"
+DEFAULT_RECIPIENT_FLEEP=
+declare -A role_recipients_fleep=()
+
+# Amazon SNS configs
+DEFAULT_RECIPIENT_AWSSNS=
+AWSSNS_MESSAGE_FORMAT=
+declare -A role_recipients_awssns=()
+
+# syslog configs
+SYSLOG_FACILITY=
+declare -A role_recipients_syslog=()
+
# custom configs
DEFAULT_RECIPIENT_CUSTOM=
declare -A role_recipients_custom=()
@@ -318,6 +353,7 @@ declare -A role_recipients_custom=()
EMAIL_SENDER=
DEFAULT_RECIPIENT_EMAIL="root"
EMAIL_CHARSET=$(locale charmap 2>/dev/null)
+EMAIL_THREADING=
declare -A role_recipients_email=()
# irc configs
@@ -327,14 +363,20 @@ DEFAULT_RECIPIENT_IRC=
IRC_NETWORK=
declare -A role_recipients_irc=()
-# load the user configuration
-# this will overwrite the variables above
-if [ -f "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf" ]
- then
- source "${NETDATA_CONFIG_DIR}/health_alarm_notify.conf"
-else
- error "Cannot find file ${NETDATA_CONFIG_DIR}/health_alarm_notify.conf. Using internal defaults."
-fi
+# load the stock and user configuration files
+# these will overwrite the variables above
+
+for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"
+do
+ if [ -f "${CONFIG}" ]
+ then
+ debug "Loading config file '${CONFIG}'..."
+ source "${CONFIG}"
+ [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'."
+ else
+ warning "Cannot find file '${CONFIG}'."
+ fi
+done
# If we didn't autodetect the character set for e-mail and it wasn't
# set by the user, we need to set it to a reasonable default. UTF-8
@@ -405,6 +447,8 @@ filter_recipient_by_criticality() {
# find the recipients' addresses per method
declare -A arr_slack=()
+declare -A arr_msteam=()
+declare -A arr_rocketchat=()
declare -A arr_alerta=()
declare -A arr_flock=()
declare -A arr_discord=()
@@ -418,7 +462,10 @@ declare -A arr_email=()
declare -A arr_custom=()
declare -A arr_messagebird=()
declare -A arr_kavenegar=()
+declare -A arr_fleep=()
declare -A arr_irc=()
+declare -A arr_syslog=()
+declare -A arr_awssns=()
# netdata may call us with multiple roles, and roles may have multiple but
# overlapping recipients - so, here we find the unique recipients.
@@ -500,6 +547,22 @@ do
[ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
done
+ # Microsoft Team
+ a="${role_recipients_msteam[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MSTEAM}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality msteam "${r}" && arr_msteam[${r/|*/}]="1"
+ done
+
+ # rocketchat
+ a="${role_recipients_rocketchat[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality rocketchat "${r}" && arr_rocketchat[${r/|*/}]="1"
+ done
+
# alerta
a="${role_recipients_alerta[${x}]}"
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ALERTA}"
@@ -531,7 +594,15 @@ do
do
[ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1"
done
-
+
+ # fleep.io
+ a="${role_recipients_fleep[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLEEP}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality fleep "${r}" && arr_fleep[${r/|*/}]="1"
+ done
+
# irc
a="${role_recipients_irc[${x}]}"
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_IRC}"
@@ -540,6 +611,22 @@ do
[ "${r}" != "disabled" ] && filter_recipient_by_criticality irc "${r}" && arr_irc[${r/|*/}]="1"
done
+ # amazon sns
+ a="${role_recipients_awssns[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_AWSSNS}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality awssns "${r}" && arr_awssns[${r/|*/}]="1"
+ done
+
+ # syslog
+ a="${role_recipients_syslog[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SYSLOG}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality syslog "${r}" && arr_syslog[${r/|*/}]="1"
+ done
+
# custom
a="${role_recipients_custom[${x}]}"
[ -z "${a}" ] && a="${DEFAULT_RECIPIENT_CUSTOM}"
@@ -554,6 +641,14 @@ done
to_slack="${!arr_slack[*]}"
[ -z "${to_slack}" ] && SEND_SLACK="NO"
+# build the list of Microsoft team recipients (channels)
+to_msteam="${!arr_msteam[*]}"
+[ -z "${to_msteam}" ] && SEND_MSTEAM="NO"
+
+# build the list of rocketchat recipients (channels)
+to_rocketchat="${!arr_rocketchat[*]}"
+[ -z "${to_rocketchat}" ] && SEND_ROCKETCHAT="NO"
+
# build the list of alerta recipients (channels)
to_alerta="${!arr_alerta[*]}"
[ -z "${to_alerta}" ] && SEND_ALERTA="NO"
@@ -598,6 +693,10 @@ to_telegram="${!arr_telegram[*]}"
to_pd="${!arr_pd[*]}"
[ -z "${to_pd}" ] && SEND_PD="NO"
+# build the list of fleep recipients (conversation webhooks)
+to_fleep="${!arr_fleep[*]}"
+[ -z "${to_fleep}" ] && SEND_FLEEP="NO"
+
# build the list of custom recipients
to_custom="${!arr_custom[*]}"
[ -z "${to_custom}" ] && SEND_CUSTOM="NO"
@@ -615,12 +714,23 @@ done
to_irc="${!arr_irc[*]}"
[ -z "${to_irc}" ] && SEND_IRC="NO"
+# build the list of awssns recipients (facilities, servers, and prefixes)
+to_awssns="${!arr_awssns[*]}"
+[ -z "${to_awssns}" ] && SEND_AWSSNS="NO"
+
+# build the list of syslog recipients (facilities, servers, and prefixes)
+to_syslog="${!arr_syslog[*]}"
+[ -z "${to_syslog}" ] && SEND_SYSLOG="NO"
+
# -----------------------------------------------------------------------------
# verify the delivery methods supported
# check slack
[ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
+# check rocketchat
+[ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
+
# check alerta
[ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
@@ -657,6 +767,9 @@ to_irc="${!arr_irc[*]}"
# check irc
[ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
+# check fleep
+[ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO"
+
# check pagerduty.com
# if we need pd-send, check for the pd-send command
# https://www.pagerduty.com/docs/guides/agent-install-guide/
@@ -674,6 +787,7 @@ fi
if [ \( \
"${SEND_PUSHOVER}" = "YES" \
-o "${SEND_SLACK}" = "YES" \
+ -o "${SEND_ROCKETCHAT}" = "YES" \
-o "${SEND_ALERTA}" = "YES" \
-o "${SEND_FLOCK}" = "YES" \
-o "${SEND_DISCORD}" = "YES" \
@@ -684,7 +798,9 @@ if [ \( \
-o "${SEND_TELEGRAM}" = "YES" \
-o "${SEND_PUSHBULLET}" = "YES" \
-o "${SEND_KAFKA}" = "YES" \
+ -o "${SEND_FLEEP}" = "YES" \
-o "${SEND_CUSTOM}" = "YES" \
+ -o "${SEND_MSTEAM}" = "YES" \
\) -a -z "${curl}" ]
then
curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
@@ -695,6 +811,8 @@ if [ \( \
SEND_PUSHBULLET="NO"
SEND_TELEGRAM="NO"
SEND_SLACK="NO"
+ SEND_MSTEAM="NO"
+ SEND_ROCKETCHAT="NO"
SEND_ALERTA="NO"
SEND_FLOCK="NO"
SEND_DISCORD="NO"
@@ -703,6 +821,7 @@ if [ \( \
SEND_MESSAGEBIRD="NO"
SEND_KAVENEGAR="NO"
SEND_KAFKA="NO"
+ SEND_FLEEP="NO"
SEND_CUSTOM="NO"
fi
fi
@@ -718,11 +837,34 @@ if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
fi
fi
+# if we need logger, check for the logger command
+if [ "${SEND_SYSLOG}" = "YES" -a -z "${logger}" ]
+ then
+ logger="$(which logger 2>/dev/null || command -v logger 2>/dev/null)"
+ if [ -z "${logger}" ]
+ then
+ debug "Cannot find logger command in the system path. Disabling syslog notifications."
+ SEND_SYSLOG="NO"
+ fi
+fi
+
+# if we need aws, check for the aws command
+if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ]
+ then
+ aws="$(which aws 2>/dev/null || command -v aws 2>/dev/null)"
+ if [ -z "${aws}" ]
+ then
+ debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
+ SEND_AWSSNS="NO"
+ fi
+fi
+
# check that we have at least a method enabled
if [ "${SEND_EMAIL}" != "YES" \
-a "${SEND_PUSHOVER}" != "YES" \
-a "${SEND_TELEGRAM}" != "YES" \
-a "${SEND_SLACK}" != "YES" \
+ -a "${SEND_ROCKETCHAT}" != "YES" \
-a "${SEND_ALERTA}" != "YES" \
-a "${SEND_FLOCK}" != "YES" \
-a "${SEND_DISCORD}" != "YES" \
@@ -733,8 +875,12 @@ if [ "${SEND_EMAIL}" != "YES" \
-a "${SEND_PUSHBULLET}" != "YES" \
-a "${SEND_KAFKA}" != "YES" \
-a "${SEND_PD}" != "YES" \
+ -a "${SEND_FLEEP}" != "YES" \
-a "${SEND_CUSTOM}" != "YES" \
-a "${SEND_IRC}" != "YES" \
+ -a "${SEND_AWSSNS}" != "YES" \
+ -a "${SEND_SYSLOG}" != "YES" \
+ -a "${SEND_MSTEAM}" != "YES" \
]
then
fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
@@ -743,8 +889,18 @@ fi
# -----------------------------------------------------------------------------
# get the date the alarm happened
-date="$(date --date=@${when} 2>/dev/null)"
-[ -z "${date}" ] && date="$(date 2>/dev/null)"
+date=$(date --date=@${when} "${date_format}" 2>/dev/null)
+[ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
+[ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
+[ -z "${date}" ] && date=$(date 2>/dev/null)
+
+# ----------------------------------------------------------------------------
+# prepare some extra headers if we've been asked to thread e-mails
+if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" == "YES" ] ; then
+ email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\nReferences: <${chart}-${name}@${host}>"
+else
+ email_thread_headers=
+fi
# -----------------------------------------------------------------------------
# function to URL encode a string
@@ -830,39 +986,42 @@ duration4human() {
# email sender
send_email() {
- local ret= opts=
+ local ret= opts=() sender_email="${EMAIL_SENDER}" sender_name=
if [ "${SEND_EMAIL}" = "YES" ]
then
if [ ! -z "${EMAIL_SENDER}" ]
then
- if [[ "${EMAIL_SENDER}" =~ \".*\"\ \<.*\> ]]
- then
- # the name includes single quotes
- opts=" -f $(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1) -F $(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
- elif [[ "${EMAIL_SENDER}" =~ \'.*\'\ \<.*\> ]]
+ if [[ "${EMAIL_SENDER}" =~ ^\".*\"\ \<.*\>$ ]]
then
# the name includes double quotes
- opts=" -f $(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1) -F $(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
- elif [[ "${EMAIL_SENDER}" =~ .*\ \<.*\> ]]
+ sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+ sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
+ elif [[ "${EMAIL_SENDER}" =~ ^\'.*\'\ \<.*\>$ ]]
+ then
+ # the name includes single quotes
+ sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+ sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
+ elif [[ "${EMAIL_SENDER}" =~ ^.*\ \<.*\>$ ]]
then
# the name does not have any quotes
- opts=" -f $(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1) -F '$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)'"
- else
- # no name at all
- opts=" -f ${EMAIL_SENDER}"
+ sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+ sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
fi
fi
+ [ ! -z "${sender_email}" ] && opts+=(-f "${sender_email}")
+ [ ! -z "${sender_name}" ] && opts+=(-F "${sender_name}")
+
if [ "${debug}" = "1" ]
then
echo >&2 "--- BEGIN sendmail command ---"
- printf >&2 "%q " "${sendmail}" -t ${opts}
+ printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
echo >&2
echo >&2 "--- END sendmail command ---"
fi
- "${sendmail}" -t ${opts}
+ "${sendmail}" -t "${opts[@]}"
ret=$?
if [ ${ret} -eq 0 ]
@@ -1242,6 +1401,65 @@ send_telegram() {
}
# -----------------------------------------------------------------------------
+# Microsoft Team sender
+
+send_msteam() {
+
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+ [ "${SEND_MSTEAM}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}";;
+ CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}";;
+ CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}";;
+ *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}";;
+ esac
+
+ for channel in ${channels}
+ do
+ ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
+ ## Online designer : https://acdesignerbeta.azurewebsites.net/
+ payload="$(cat <<EOF
+ {
+ "@context": "http://schema.org/extensions",
+ "@type": "MessageCard",
+ "themeColor": "${color}",
+ "title": "$icon Alert ${status} from netdata for ${host}",
+ "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
+ "potentialAction": [
+ {
+ "@type": "OpenUri",
+ "name": "Netdata",
+ "targets": [
+ { "os": "default", "uri": "${goto_url}" }
+ ]
+ }
+ ]
+ }
+EOF
+ )"
+
+ # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
+ webhook="${webhook//CHANNEL/${channel}}"
+
+ httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
+
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+
# slack sender
send_slack() {
@@ -1305,6 +1523,71 @@ EOF
return 1
}
+
+# -----------------------------------------------------------------------------
+# rocketchat sender
+
+send_rocketchat() {
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+ [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) color="warning" ;;
+ CRITICAL) color="danger" ;;
+ CLEAR) color="good" ;;
+ *) color="#777777" ;;
+ esac
+
+ for channel in ${channels}
+ do
+ payload="$(cat <<EOF
+ {
+ "channel": "#${channel}",
+ "alias": "netdata on ${host}",
+ "avatar": "${images_base_url}/images/seo-performance-128.png",
+ "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
+ "attachments": [
+ {
+ "color": "${color}",
+ "title": "${alarm}",
+ "title_link": "${goto_url}",
+ "text": "${info}",
+ "fields": [
+ {
+ "title": "${chart}",
+ "short": true,
+ "value": "chart"
+ },
+ {
+ "title": "${family}",
+ "short": true,
+ "value": "family"
+ }
+ ],
+ "thumb_url": "${image}",
+ "ts": "${when}"
+ }
+ ]
+ }
+EOF
+ )"
+
+ httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
# -----------------------------------------------------------------------------
# alerta sender
@@ -1471,6 +1754,36 @@ EOF
}
# -----------------------------------------------------------------------------
+# fleep sender
+
+send_fleep() {
+ local httpcode sent=0 webhooks="${1}" data message
+ if [ "${SEND_FLEEP}" = "YES" ] ; then
+ message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
+
+ for hook in "${webhooks}" ; do
+ data="{ "
+ data="${data} 'message': '${message}', "
+ data="${data} 'user': '${FLEEP_SENDER}' "
+ data="${data} }"
+
+ httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
+
+ if [ "${httpcode}" = "200" ] ; then
+ info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
# irc sender
send_irc() {
@@ -1510,6 +1823,106 @@ send_irc() {
return 1
}
+# -----------------------------------------------------------------------------
+# Amazon SNS sender
+
+send_awssns() {
+ local targets="${1}" message='' sent=0 region=''
+ local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+ [ "${SEND_AWSSNS}" = "YES" ] || return 1
+
+ message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
+
+ for target in ${targets} ; do
+ # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
+ region="$(echo ${target} | cut -f 4 -d ':')"
+ ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null
+ if [ $? = 0 ]; then
+ info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# syslog sender
+
+send_syslog() {
+ local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
+ local priority='' message='' host='' port='' prefix=''
+ local temp1='' temp2=''
+
+ [ "${SEND_SYSLOG}" = "YES" ] || return 1
+
+ if [ "${status}" = "CRITICAL" ] ; then
+ level='crit'
+ elif [ "${status}" = "WARNING" ] ; then
+ level='warning'
+ fi
+
+ for target in ${targets} ; do
+ priority="${facility}.${level}"
+ message=''
+ host=''
+ port=''
+ prefix=''
+ temp1=''
+ temp2=''
+
+ prefix=$(echo ${target} | cut -d '/' -f 2)
+ temp1=$(echo ${target} | cut -d '/' -f 1)
+
+ if [ ${prefix} != ${temp1} ] ; then
+ if (echo ${temp1} | grep -q '@' ) ; then
+ temp2=$(echo ${temp1} | cut -d '@' -f 1)
+ host=$(echo ${temp1} | cut -d '@' -f 2)
+
+ if [ ${temp2} != ${host} ] ; then
+ priority=${temp2}
+ fi
+
+ port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
+
+ if ( echo ${host} | grep -E -q '\[.*\]' ) ; then
+ if ( echo ${port} | grep -q ']' ) ; then
+ port=''
+ else
+ host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
+ fi
+ else
+ if [ ${port} = ${host} ] ; then
+ port=''
+ else
+ host=$(echo ${host} | cut -d ':' -f 1)
+ fi
+ fi
+ else
+ priority=${temp1}
+ fi
+ fi
+
+ message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+ if [ ${host} ] ; then
+ logger_options="${logger_options} -n ${host}"
+ if [ ${port} ] ; then
+ logger_options="${logger_options} -P ${port}"
+ fi
+ fi
+
+ ${logger} -p ${priority} ${logger_options} "${message}"
+ done
+
+ return $?
+}
+
# -----------------------------------------------------------------------------
# prepare the content of the notification
@@ -1611,6 +2024,24 @@ send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
SENT_SLACK=$?
# -----------------------------------------------------------------------------
+# send the Microsoft notification
+
+# Microsoft team aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
+SENT_MSTEAM=$?
+
+# -----------------------------------------------------------------------------
+# send the rocketchat notification
+
+# rocketchat aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
+SENT_ROCKETCHAT=$?
+
+# -----------------------------------------------------------------------------
# send the alerta notification
# alerta aggregates posts from the same username
@@ -1725,6 +2156,12 @@ send_pd "${to_pd}"
SENT_PD=$?
# -----------------------------------------------------------------------------
+# send the fleep message
+
+send_fleep "${to_fleep}"
+SENT_FLEEP=$?
+
+# -----------------------------------------------------------------------------
# send the irc message
send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
@@ -1769,6 +2206,22 @@ SENT_HIPCHAT=$?
# -----------------------------------------------------------------------------
+# send the Amazon SNS message
+
+send_awssns ${to_awssns}
+
+SENT_AWSSNS=$?
+
+
+# -----------------------------------------------------------------------------
+# send the syslog message
+
+send_syslog ${to_syslog}
+
+SENT_SYSLOG=$?
+
+
+# -----------------------------------------------------------------------------
# send the email
send_email <<EOF
@@ -1776,6 +2229,7 @@ To: ${to_email}
Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
MIME-Version: 1.0
Content-Type: multipart/alternative; boundary="multipart-boundary"
+${email_thread_headers}
This is a MIME-encoded multipart message
@@ -1897,6 +2351,8 @@ if [ ${SENT_EMAIL} -eq 0 \
-o ${SENT_PUSHOVER} -eq 0 \
-o ${SENT_TELEGRAM} -eq 0 \
-o ${SENT_SLACK} -eq 0 \
+ -o ${SENT_MSTEAM} -eq 0 \
+ -o ${SENT_ROCKETCHAT} -eq 0 \
-o ${SENT_ALERTA} -eq 0 \
-o ${SENT_FLOCK} -eq 0 \
-o ${SENT_DISCORD} -eq 0 \
@@ -1907,8 +2363,11 @@ if [ ${SENT_EMAIL} -eq 0 \
-o ${SENT_PUSHBULLET} -eq 0 \
-o ${SENT_KAFKA} -eq 0 \
-o ${SENT_PD} -eq 0 \
+ -o ${SENT_FLEEP} -eq 0 \
-o ${SENT_IRC} -eq 0 \
+ -o ${SENT_AWSSNS} -eq 0 \
-o ${SENT_CUSTOM} -eq 0 \
+ -o ${SENT_SYSLOG} -eq 0 \
]
then
# we did send something
diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in
new file mode 100755
index 00000000..4aef3a52
--- /dev/null
+++ b/health/notifications/alarm-notify.sh.in
@@ -0,0 +1,2378 @@
+#!/usr/bin/env bash
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Script to send alarm notifications for netdata
+#
+# Features:
+# - multiple notification methods
+# - multiple roles per alarm
+# - multiple recipients per role
+# - severity filtering per recipient
+#
+# Supported notification methods:
+# - emails by @ktsaou
+# - slack.com notifications by @ktsaou
+# - alerta.io notifications by @kattunga
+# - discordapp.com notifications by @lowfive
+# - pushover.net notifications by @ktsaou
+# - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
+# - telegram.org notifications by @hashworks #1002
+# - twilio.com notifications by Levi Blaney @shadycuz #1211
+# - kafka notifications by @ktsaou #1342
+# - pagerduty.com notifications by Jim Cooley @jimcooley #1373
+# - messagebird.com notifications by @tech_no_logical #1453
+# - hipchat notifications by @ktsaou #1561
+# - fleep notifications by @Ferroin
+# - custom notifications by @ktsaou
+# - syslog messages by @Ferroin
+# - Microsoft Team notification by @tioumen
+
+# -----------------------------------------------------------------------------
+# testing notifications
+
+
+if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ]
+then
+ if [ "${2}" = "test" ]
+ then
+ recipient="${1}"
+ else
+ recipient="${2}"
+ fi
+
+ [ -z "${recipient}" ] && recipient="sysadmin"
+
+ id=1
+ last="CLEAR"
+ test_res=0
+ for x in "WARNING" "CRITICAL" "CLEAR"
+ do
+ echo >&2
+ echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
+
+ "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value"
+ if [ $? -ne 0 ]
+ then
+ echo >&2 "# FAILED"
+ test_res=1
+ else
+ echo >&2 "# OK"
+ fi
+
+ last="${x}"
+ id=$((id + 1))
+ done
+
+ exit $test_res
+fi
+
+export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
+export LC_ALL=C
+
+# -----------------------------------------------------------------------------
+
+PROGRAM_NAME="$(basename "${0}")"
+
+logdate() {
+ date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+ local status="${1}"
+ shift
+
+ echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
+
+}
+
+warning() {
+ log WARNING "${@}"
+}
+
+error() {
+ log ERROR "${@}"
+}
+
+info() {
+ log INFO "${@}"
+}
+
+fatal() {
+ log FATAL "${@}"
+ exit 1
+}
+
+debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
+debug() {
+ [ "${debug}" = "1" ] && log DEBUG "${@}"
+}
+
+docurl() {
+ if [ -z "${curl}" ]
+ then
+ error "\${curl} is unset."
+ return 1
+ fi
+
+ if [ "${debug}" = "1" ]
+ then
+ echo >&2 "--- BEGIN curl command ---"
+ printf >&2 "%q " ${curl} "${@}"
+ echo >&2
+ echo >&2 "--- END curl command ---"
+
+ local out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
+ local code=$(${curl} ${curl_options} --write-out %{http_code} --output "${out}" --silent --show-error "${@}")
+ local ret=$?
+ echo >&2 "--- BEGIN received response ---"
+ cat >&2 "${out}"
+ echo >&2
+ echo >&2 "--- END received response ---"
+ echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
+ rm "${out}"
+ echo "${code}"
+ return ${ret}
+ fi
+
+ ${curl} ${curl_options} --write-out %{http_code} --output /dev/null --silent --show-error "${@}"
+ return $?
+}
+
+# -----------------------------------------------------------------------------
+# this is to be overwritten by the config file
+
+custom_sender() {
+ info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
+}
+
+
+# -----------------------------------------------------------------------------
+
+# check for BASH v4+ (required for associative arrays)
+[ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \
+ fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
+
+# -----------------------------------------------------------------------------
+# defaults to allow running this script by hand
+
+[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
+[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
+[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
+[ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
+
+# -----------------------------------------------------------------------------
+# parse command line parameters
+
+roles="${1}" # the roles that should be notified for this event
+host="${2}" # the host generated this event
+unique_id="${3}" # the unique id of this event
+alarm_id="${4}" # the unique id of the alarm that generated this event
+event_id="${5}" # the incremental id of the event, for this alarm id
+when="${6}" # the timestamp this event occurred
+name="${7}" # the name of the alarm, as given in netdata health.d entries
+chart="${8}" # the name of the chart (type.id)
+family="${9}" # the family of the chart
+status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+value="${12}" # the current value of the alarm
+old_value="${13}" # the previous value of the alarm
+src="${14}" # the line number and file the alarm has been configured
+duration="${15}" # the duration in seconds of the previous alarm state
+non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
+units="${17}" # the units of the value
+info="${18}" # a short description of the alarm
+value_string="${19}" # friendly value (with units)
+old_value_string="${20}" # friendly old value (with units)
+
+# -----------------------------------------------------------------------------
+# find a suitable hostname to use, if netdata did not supply a hostname
+
+this_host=$(hostname -s 2>/dev/null)
+[ -z "${host}" ] && host="${this_host}"
+
+# -----------------------------------------------------------------------------
+# screen statuses we don't need to send a notification
+
+# don't do anything if this is not WARNING, CRITICAL or CLEAR
+if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ]
+then
+ info "not sending notification for ${status} of '${host}.${chart}.${name}'"
+ exit 1
+fi
+
+# don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
+if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ]
+then
+ info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
+ exit 1
+fi
+
+# -----------------------------------------------------------------------------
+# load configuration
+
+# By default fetch images from the global public registry.
+# This is required by default, since all notification methods need to download
+# images via the Internet, and private registries might not be reachable.
+# This can be overwritten at the configuration file.
+images_base_url="https://registry.my-netdata.io"
+
+# curl options to use
+curl_options=""
+
+# needed commands
+# if empty they will be searched in the system path
+curl=
+sendmail=
+
+# enable / disable features
+SEND_SLACK="YES"
+SEND_MSTEAM="YES"
+SEND_ALERTA="YES"
+SEND_FLOCK="YES"
+SEND_DISCORD="YES"
+SEND_PUSHOVER="YES"
+SEND_TWILIO="YES"
+SEND_HIPCHAT="YES"
+SEND_MESSAGEBIRD="YES"
+SEND_KAVENEGAR="YES"
+SEND_TELEGRAM="YES"
+SEND_EMAIL="YES"
+SEND_PUSHBULLET="YES"
+SEND_KAFKA="YES"
+SEND_PD="YES"
+SEND_FLEEP="YES"
+SEND_IRC="YES"
+SEND_AWSSNS="YES"
+SEND_SYSLOG="NO"
+SEND_CUSTOM="YES"
+
+# slack configs
+SLACK_WEBHOOK_URL=
+DEFAULT_RECIPIENT_SLACK=
+declare -A role_recipients_slack=()
+
+# Microsoft Team configs
+MSTEAM_WEBHOOK_URL=
+DEFAULT_RECIPIENT_MSTEAM=
+declare -A role_recipients_msteam=()
+
+# rocketchat configs
+ROCKETCHAT_WEBHOOK_URL=
+DEFAULT_RECIPIENT_ROCKETCHAT=
+declare -A role_recipients_rocketchat=()
+
+# alerta configs
+ALERTA_WEBHOOK_URL=
+ALERTA_API_KEY=
+DEFAULT_RECIPIENT_ALERTA=
+declare -A role_recipients_alerta=()
+
+# flock configs
+FLOCK_WEBHOOK_URL=
+DEFAULT_RECIPIENT_FLOCK=
+declare -A role_recipients_flock=()
+
+# discord configs
+DISCORD_WEBHOOK_URL=
+DEFAULT_RECIPIENT_DISCORD=
+declare -A role_recipients_discord=()
+
+# pushover configs
+PUSHOVER_APP_TOKEN=
+DEFAULT_RECIPIENT_PUSHOVER=
+declare -A role_recipients_pushover=()
+
+# pushbullet configs
+PUSHBULLET_ACCESS_TOKEN=
+PUSHBULLET_SOURCE_DEVICE=
+DEFAULT_RECIPIENT_PUSHBULLET=
+declare -A role_recipients_pushbullet=()
+
+# twilio configs
+TWILIO_ACCOUNT_SID=
+TWILIO_ACCOUNT_TOKEN=
+TWILIO_NUMBER=
+DEFAULT_RECIPIENT_TWILIO=
+declare -A role_recipients_twilio=()
+
+# hipchat configs
+HIPCHAT_SERVER=
+HIPCHAT_AUTH_TOKEN=
+DEFAULT_RECIPIENT_HIPCHAT=
+declare -A role_recipients_hipchat=()
+
+# messagebird configs
+MESSAGEBIRD_ACCESS_KEY=
+MESSAGEBIRD_NUMBER=
+DEFAULT_RECIPIENT_MESSAGEBIRD=
+declare -A role_recipients_messagebird=()
+
+# kavenegar configs
+KAVENEGAR_API_KEY=""
+KAVENEGAR_SENDER=""
+DEFAULT_RECIPIENT_KAVENEGAR=()
+declare -A role_recipients_kavenegar=""
+
+# telegram configs
+TELEGRAM_BOT_TOKEN=
+DEFAULT_RECIPIENT_TELEGRAM=
+declare -A role_recipients_telegram=()
+
+# kafka configs
+KAFKA_URL=
+KAFKA_SENDER_IP=
+
+# pagerduty.com configs
+PD_SERVICE_KEY=
+DEFAULT_RECIPIENT_PD=
+declare -A role_recipients_pd=()
+
+# fleep.io configs
+FLEEP_SENDER="${host}"
+DEFAULT_RECIPIENT_FLEEP=
+declare -A role_recipients_fleep=()
+
+# Amazon SNS configs
+DEFAULT_RECIPIENT_AWSSNS=
+AWSSNS_MESSAGE_FORMAT=
+declare -A role_recipients_awssns=()
+
+# syslog configs
+SYSLOG_FACILITY=
+declare -A role_recipients_syslog=()
+
+# custom configs
+DEFAULT_RECIPIENT_CUSTOM=
+declare -A role_recipients_custom=()
+
+# email configs
+EMAIL_SENDER=
+DEFAULT_RECIPIENT_EMAIL="root"
+EMAIL_CHARSET=$(locale charmap 2>/dev/null)
+EMAIL_THREADING=
+declare -A role_recipients_email=()
+
+# irc configs
+IRC_NICKNAME=
+IRC_REALNAME=
+DEFAULT_RECIPIENT_IRC=
+IRC_NETWORK=
+declare -A role_recipients_irc=()
+
+# load the stock and user configuration files
+# these will overwrite the variables above
+
+for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"
+do
+ if [ -f "${CONFIG}" ]
+ then
+ debug "Loading config file '${CONFIG}'..."
+ source "${CONFIG}"
+ [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'."
+ else
+ warning "Cannot find file '${CONFIG}'."
+ fi
+done
+
+# If we didn't autodetect the character set for e-mail and it wasn't
+# set by the user, we need to set it to a reasonable default. UTF-8
+# should be correct for almost all modern UNIX systems.
+if [ -z ${EMAIL_CHARSET} ]
+ then
+ EMAIL_CHARSET="UTF-8"
+fi
+
+# -----------------------------------------------------------------------------
+# filter a recipient based on alarm event severity
+
+filter_recipient_by_criticality() {
+ local method="${1}" x="${2}" r s
+ shift
+
+ r="${x/|*/}" # the recipient
+ s="${x/*|/}" # the severity required for notifying this recipient
+
+ # no severity filtering for this person
+ [ "${r}" = "${s}" ] && return 0
+
+ # the severity is invalid
+ s="${s^^}"
+ if [ "${s}" != "CRITICAL" ]
+ then
+ error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
+ return 0
+ fi
+
+ # create the status tracking directory for this user
+ [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \
+ mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
+
+ case "${status}" in
+ CRITICAL)
+ # make sure he will get future notifications for this alarm too
+ touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
+ return 0
+ ;;
+
+ WARNING)
+ if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
+ then
+ # we do not remove the file, so that he will get future notifications of this alarm
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
+ return 0
+ fi
+ ;;
+
+ *)
+ if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]
+ then
+ # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
+ rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
+ return 0
+ fi
+ ;;
+ esac
+
+ debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# find the recipients' addresses per method
+
+declare -A arr_slack=()
+declare -A arr_msteam=()
+declare -A arr_rocketchat=()
+declare -A arr_alerta=()
+declare -A arr_flock=()
+declare -A arr_discord=()
+declare -A arr_pushover=()
+declare -A arr_pushbullet=()
+declare -A arr_twilio=()
+declare -A arr_hipchat=()
+declare -A arr_telegram=()
+declare -A arr_pd=()
+declare -A arr_email=()
+declare -A arr_custom=()
+declare -A arr_messagebird=()
+declare -A arr_kavenegar=()
+declare -A arr_fleep=()
+declare -A arr_irc=()
+declare -A arr_syslog=()
+declare -A arr_awssns=()
+
+# netdata may call us with multiple roles, and roles may have multiple but
+# overlapping recipients - so, here we find the unique recipients.
+for x in ${roles//,/ }
+do
+ # the roles 'silent' and 'disabled' mean:
+ # don't send a notification for this role
+ [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue
+
+ # email
+ a="${role_recipients_email[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1"
+ done
+
+ # pushover
+ a="${role_recipients_pushover[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1"
+ done
+
+ # pushbullet
+ a="${role_recipients_pushbullet[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1"
+ done
+
+ # twilio
+ a="${role_recipients_twilio[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TWILIO}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1"
+ done
+
+ # hipchat
+ a="${role_recipients_hipchat[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_HIPCHAT}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality hipchat "${r}" && arr_hipchat[${r/|*/}]="1"
+ done
+
+ # messagebird
+ a="${role_recipients_messagebird[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality messagebird "${r}" && arr_messagebird[${r/|*/}]="1"
+ done
+
+ # kavenegar
+ a="${role_recipients_kavenegar[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_KAVENEGAR}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality kavenegar "${r}" && arr_kavenegar[${r/|*/}]="1"
+ done
+
+ # telegram
+ a="${role_recipients_telegram[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1"
+ done
+
+ # slack
+ a="${role_recipients_slack[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
+ done
+
+ # Microsoft Team
+ a="${role_recipients_msteam[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MSTEAM}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality msteam "${r}" && arr_msteam[${r/|*/}]="1"
+ done
+
+ # rocketchat
+ a="${role_recipients_rocketchat[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality rocketchat "${r}" && arr_rocketchat[${r/|*/}]="1"
+ done
+
+ # alerta
+ a="${role_recipients_alerta[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ALERTA}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality alerta "${r}" && arr_alerta[${r/|*/}]="1"
+ done
+
+ # flock
+ a="${role_recipients_flock[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLOCK}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality flock "${r}" && arr_flock[${r/|*/}]="1"
+ done
+
+ # discord
+ a="${role_recipients_discord[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_DISCORD}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality discord "${r}" && arr_discord[${r/|*/}]="1"
+ done
+
+ # pagerduty.com
+ a="${role_recipients_pd[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PD}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1"
+ done
+
+ # fleep.io
+ a="${role_recipients_fleep[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLEEP}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality fleep "${r}" && arr_fleep[${r/|*/}]="1"
+ done
+
+ # irc
+ a="${role_recipients_irc[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_IRC}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality irc "${r}" && arr_irc[${r/|*/}]="1"
+ done
+
+ # amazon sns
+ a="${role_recipients_awssns[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_AWSSNS}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality awssns "${r}" && arr_awssns[${r/|*/}]="1"
+ done
+
+ # syslog
+ a="${role_recipients_syslog[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SYSLOG}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality syslog "${r}" && arr_syslog[${r/|*/}]="1"
+ done
+
+ # custom
+ a="${role_recipients_custom[${x}]}"
+ [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_CUSTOM}"
+ for r in ${a//,/ }
+ do
+ [ "${r}" != "disabled" ] && filter_recipient_by_criticality custom "${r}" && arr_custom[${r/|*/}]="1"
+ done
+
+done
+
+# build the list of slack recipients (channels)
+to_slack="${!arr_slack[*]}"
+[ -z "${to_slack}" ] && SEND_SLACK="NO"
+
+# build the list of Microsoft team recipients (channels)
+to_msteam="${!arr_msteam[*]}"
+[ -z "${to_msteam}" ] && SEND_MSTEAM="NO"
+
+# build the list of rocketchat recipients (channels)
+to_rocketchat="${!arr_rocketchat[*]}"
+[ -z "${to_rocketchat}" ] && SEND_ROCKETCHAT="NO"
+
+# build the list of alerta recipients (channels)
+to_alerta="${!arr_alerta[*]}"
+[ -z "${to_alerta}" ] && SEND_ALERTA="NO"
+
+# build the list of flock recipients (channels)
+to_flock="${!arr_flock[*]}"
+[ -z "${to_flock}" ] && SEND_FLOCK="NO"
+
+# build the list of discord recipients (channels)
+to_discord="${!arr_discord[*]}"
+[ -z "${to_discord}" ] && SEND_DISCORD="NO"
+
+# build the list of pushover recipients (user tokens)
+to_pushover="${!arr_pushover[*]}"
+[ -z "${to_pushover}" ] && SEND_PUSHOVER="NO"
+
+# build the list of pushbulet recipients (user tokens)
+to_pushbullet="${!arr_pushbullet[*]}"
+[ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO"
+
+# build the list of twilio recipients (phone numbers)
+to_twilio="${!arr_twilio[*]}"
+[ -z "${to_twilio}" ] && SEND_TWILIO="NO"
+
+# build the list of hipchat recipients (rooms)
+to_hipchat="${!arr_hipchat[*]}"
+[ -z "${to_hipchat}" ] && SEND_HIPCHAT="NO"
+
+# build the list of messagebird recipients (phone numbers)
+to_messagebird="${!arr_messagebird[*]}"
+[ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO"
+
+# build the list of kavenegar recipients (phone numbers)
+to_kavenegar="${!arr_kavenegar[*]}"
+[ -z "${to_kavenegar}" ] && SEND_KAVENEGAR="NO"
+
+# check array of telegram recipients (chat ids)
+to_telegram="${!arr_telegram[*]}"
+[ -z "${to_telegram}" ] && SEND_TELEGRAM="NO"
+
+# build the list of pagerduty recipients (service keys)
+to_pd="${!arr_pd[*]}"
+[ -z "${to_pd}" ] && SEND_PD="NO"
+
+# build the list of fleep recipients (conversation webhooks)
+to_fleep="${!arr_fleep[*]}"
+[ -z "${to_fleep}" ] && SEND_FLEEP="NO"
+
+# build the list of custom recipients
+to_custom="${!arr_custom[*]}"
+[ -z "${to_custom}" ] && SEND_CUSTOM="NO"
+
+# build the list of email recipients (email addresses)
+to_email=
+for x in "${!arr_email[@]}"
+do
+ [ ! -z "${to_email}" ] && to_email="${to_email}, "
+ to_email="${to_email}${x}"
+done
+[ -z "${to_email}" ] && SEND_EMAIL="NO"
+
+# build the list of irc recipients (channels)
+to_irc="${!arr_irc[*]}"
+[ -z "${to_irc}" ] && SEND_IRC="NO"
+
+# build the list of awssns recipients (facilities, servers, and prefixes)
+to_awssns="${!arr_awssns[*]}"
+[ -z "${to_awssns}" ] && SEND_AWSSNS="NO"
+
+# build the list of syslog recipients (facilities, servers, and prefixes)
+to_syslog="${!arr_syslog[*]}"
+[ -z "${to_syslog}" ] && SEND_SYSLOG="NO"
+
+# -----------------------------------------------------------------------------
+# verify the delivery methods supported
+
+# check slack
+[ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
+
+# check rocketchat
+[ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
+
+# check alerta
+[ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
+
+# check flock
+[ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
+
+# check discord
+[ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
+
+# check pushover
+[ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
+
+# check pushbullet
+[ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
+
+# check twilio
+[ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO"
+
+# check hipchat
+[ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
+
+# check messagebird
+[ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO"
+
+# check kavenegar
+[ -z "${KAVENEGAR_API_KEY}" -o -z "${KAVENEGAR_SENDER}" ] && SEND_KAVENEGAR="NO"
+
+# check telegram
+[ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
+
+# check kafka
+[ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO"
+
+# check irc
+[ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
+
+# check fleep
+[ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO"
+
+# check pagerduty.com
+# if we need pd-send, check for the pd-send command
+# https://www.pagerduty.com/docs/guides/agent-install-guide/
+if [ "${SEND_PD}" = "YES" ]
+ then
+ pd_send="$(which pd-send 2>/dev/null || command -v pd-send 2>/dev/null)"
+ if [ -z "${pd_send}" ]
+ then
+ error "Cannot find pd-send command in the system path. Disabling pagerduty.com notifications."
+ SEND_PD="NO"
+ fi
+fi
+
+# if we need curl, check for the curl command
+if [ \( \
+ "${SEND_PUSHOVER}" = "YES" \
+ -o "${SEND_SLACK}" = "YES" \
+ -o "${SEND_ROCKETCHAT}" = "YES" \
+ -o "${SEND_ALERTA}" = "YES" \
+ -o "${SEND_FLOCK}" = "YES" \
+ -o "${SEND_DISCORD}" = "YES" \
+ -o "${SEND_HIPCHAT}" = "YES" \
+ -o "${SEND_TWILIO}" = "YES" \
+ -o "${SEND_MESSAGEBIRD}" = "YES" \
+ -o "${SEND_KAVENEGAR}" = "YES" \
+ -o "${SEND_TELEGRAM}" = "YES" \
+ -o "${SEND_PUSHBULLET}" = "YES" \
+ -o "${SEND_KAFKA}" = "YES" \
+ -o "${SEND_FLEEP}" = "YES" \
+ -o "${SEND_CUSTOM}" = "YES" \
+ -o "${SEND_MSTEAM}" = "YES" \
+ \) -a -z "${curl}" ]
+ then
+ curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)"
+ if [ -z "${curl}" ]
+ then
+ error "Cannot find curl command in the system path. Disabling all curl based notifications."
+ SEND_PUSHOVER="NO"
+ SEND_PUSHBULLET="NO"
+ SEND_TELEGRAM="NO"
+ SEND_SLACK="NO"
+ SEND_MSTEAM="NO"
+ SEND_ROCKETCHAT="NO"
+ SEND_ALERTA="NO"
+ SEND_FLOCK="NO"
+ SEND_DISCORD="NO"
+ SEND_TWILIO="NO"
+ SEND_HIPCHAT="NO"
+ SEND_MESSAGEBIRD="NO"
+ SEND_KAVENEGAR="NO"
+ SEND_KAFKA="NO"
+ SEND_FLEEP="NO"
+ SEND_CUSTOM="NO"
+ fi
+fi
+
+# if we need sendmail, check for the sendmail command
+if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ]
+ then
+ sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)"
+ if [ -z "${sendmail}" ]
+ then
+ debug "Cannot find sendmail command in the system path. Disabling email notifications."
+ SEND_EMAIL="NO"
+ fi
+fi
+
+# if we need logger, check for the logger command
+if [ "${SEND_SYSLOG}" = "YES" -a -z "${logger}" ]
+ then
+ logger="$(which logger 2>/dev/null || command -v logger 2>/dev/null)"
+ if [ -z "${logger}" ]
+ then
+ debug "Cannot find logger command in the system path. Disabling syslog notifications."
+ SEND_SYSLOG="NO"
+ fi
+fi
+
+# if we need aws, check for the aws command
+if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ]
+ then
+ aws="$(which aws 2>/dev/null || command -v aws 2>/dev/null)"
+ if [ -z "${aws}" ]
+ then
+ debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications."
+ SEND_AWSSNS="NO"
+ fi
+fi
+
+# check that we have at least a method enabled
+if [ "${SEND_EMAIL}" != "YES" \
+ -a "${SEND_PUSHOVER}" != "YES" \
+ -a "${SEND_TELEGRAM}" != "YES" \
+ -a "${SEND_SLACK}" != "YES" \
+ -a "${SEND_ROCKETCHAT}" != "YES" \
+ -a "${SEND_ALERTA}" != "YES" \
+ -a "${SEND_FLOCK}" != "YES" \
+ -a "${SEND_DISCORD}" != "YES" \
+ -a "${SEND_TWILIO}" != "YES" \
+ -a "${SEND_HIPCHAT}" != "YES" \
+ -a "${SEND_MESSAGEBIRD}" != "YES" \
+ -a "${SEND_KAVENEGAR}" != "YES" \
+ -a "${SEND_PUSHBULLET}" != "YES" \
+ -a "${SEND_KAFKA}" != "YES" \
+ -a "${SEND_PD}" != "YES" \
+ -a "${SEND_FLEEP}" != "YES" \
+ -a "${SEND_CUSTOM}" != "YES" \
+ -a "${SEND_IRC}" != "YES" \
+ -a "${SEND_AWSSNS}" != "YES" \
+ -a "${SEND_SYSLOG}" != "YES" \
+ -a "${SEND_MSTEAM}" != "YES" \
+ ]
+ then
+ fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
+fi
+
+# -----------------------------------------------------------------------------
+# get the date the alarm happened
+
+date=$(date --date=@${when} "${date_format}" 2>/dev/null)
+[ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
+[ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
+[ -z "${date}" ] && date=$(date 2>/dev/null)
+
+# ----------------------------------------------------------------------------
+# prepare some extra headers if we've been asked to thread e-mails
+if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" == "YES" ] ; then
+ email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\nReferences: <${chart}-${name}@${host}>"
+else
+ email_thread_headers=
+fi
+
+# -----------------------------------------------------------------------------
+# function to URL encode a string
+
+urlencode() {
+ local string="${1}" strlen encoded pos c o
+
+ strlen=${#string}
+ for (( pos=0 ; pos<strlen ; pos++ ))
+ do
+ c=${string:${pos}:1}
+ case "${c}" in
+ [-_.~a-zA-Z0-9])
+ o="${c}"
+ ;;
+
+ *)
+ printf -v o '%%%02x' "'${c}"
+ ;;
+ esac
+ encoded+="${o}"
+ done
+
+ REPLY="${encoded}"
+ echo "${REPLY}"
+}
+
+# -----------------------------------------------------------------------------
+# function to convert a duration in seconds, to a human readable duration
+# using DAYS, MINUTES, SECONDS
+
+duration4human() {
+ local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
+ d=$(( s / 86400 ))
+ s=$(( s - (d * 86400) ))
+ h=$(( s / 3600 ))
+ s=$(( s - (h * 3600) ))
+ m=$(( s / 60 ))
+ s=$(( s - (m * 60) ))
+
+ if [ ${d} -gt 0 ]
+ then
+ [ ${m} -ge 30 ] && h=$(( h + 1 ))
+ [ ${d} -gt 1 ] && ds="days"
+ [ ${h} -gt 1 ] && hs="hours"
+ if [ ${h} -gt 0 ]
+ then
+ ret="${d} ${ds} and ${h} ${hs}"
+ else
+ ret="${d} ${ds}"
+ fi
+ elif [ ${h} -gt 0 ]
+ then
+ [ ${s} -ge 30 ] && m=$(( m + 1 ))
+ [ ${h} -gt 1 ] && hs="hours"
+ [ ${m} -gt 1 ] && ms="minutes"
+ if [ ${m} -gt 0 ]
+ then
+ ret="${h} ${hs} and ${m} ${ms}"
+ else
+ ret="${h} ${hs}"
+ fi
+ elif [ ${m} -gt 0 ]
+ then
+ [ ${m} -gt 1 ] && ms="minutes"
+ [ ${s} -gt 1 ] && ss="seconds"
+ if [ ${s} -gt 0 ]
+ then
+ ret="${m} ${ms} and ${s} ${ss}"
+ else
+ ret="${m} ${ms}"
+ fi
+ else
+ [ ${s} -gt 1 ] && ss="seconds"
+ ret="${s} ${ss}"
+ fi
+
+ REPLY="${ret}"
+ echo "${REPLY}"
+}
+
+# -----------------------------------------------------------------------------
+# email sender
+
+send_email() {
+ local ret= opts=() sender_email="${EMAIL_SENDER}" sender_name=
+ if [ "${SEND_EMAIL}" = "YES" ]
+ then
+
+ if [ ! -z "${EMAIL_SENDER}" ]
+ then
+ if [[ "${EMAIL_SENDER}" =~ ^\".*\"\ \<.*\>$ ]]
+ then
+ # the name includes double quotes
+ sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+ sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
+ elif [[ "${EMAIL_SENDER}" =~ ^\'.*\'\ \<.*\>$ ]]
+ then
+ # the name includes single quotes
+ sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+ sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
+ elif [[ "${EMAIL_SENDER}" =~ ^.*\ \<.*\>$ ]]
+ then
+ # the name does not have any quotes
+ sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+ sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
+ fi
+ fi
+
+ [ ! -z "${sender_email}" ] && opts+=(-f "${sender_email}")
+ [ ! -z "${sender_name}" ] && opts+=(-F "${sender_name}")
+
+ if [ "${debug}" = "1" ]
+ then
+ echo >&2 "--- BEGIN sendmail command ---"
+ printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
+ echo >&2
+ echo >&2 "--- END sendmail command ---"
+ fi
+
+ "${sendmail}" -t "${opts[@]}"
+ ret=$?
+
+ if [ ${ret} -eq 0 ]
+ then
+ info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
+ return 0
+ else
+ error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
+ return 1
+ fi
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# pushover sender
+
+send_pushover() {
+ local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
+
+ if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ]
+ then
+
+ # https://pushover.net/api
+ priority=-2
+ case "${status}" in
+ CLEAR) priority=-1;; # low priority: no sound or vibration
+ WARNING) priority=0;; # normal priority: respect quiet hours
+ CRITICAL) priority=1;; # high priority: bypass quiet hours
+ *) priority=-2;; # lowest priority: no notification at all
+ esac
+
+ for user in ${usertokens}
+ do
+ httpcode=$(docurl \
+ --form-string "token=${apptoken}" \
+ --form-string "user=${user}" \
+ --form-string "html=1" \
+ --form-string "title=${title}" \
+ --form-string "message=${message}" \
+ --form-string "timestamp=${when}" \
+ --form-string "url=${url}" \
+ --form-string "url_title=Open netdata dashboard to view the alarm" \
+ --form-string "priority=${priority}" \
+ https://api.pushover.net/1/messages.json)
+
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# pushbullet sender
+
+send_pushbullet() {
+ local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
+ if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
+ then
+ #https://docs.pushbullet.com/#create-push
+ for user in ${recipients}
+ do
+ httpcode=$(docurl \
+ --header 'Access-Token: '${userapikey}'' \
+ --header 'Content-Type: application/json' \
+ --data-binary @<(cat <<EOF
+ {"title": "${title}",
+ "type": "link",
+ "email": "${user}",
+ "body": "$( echo -n ${message})",
+ "url": "${url}",
+ "source_device_iden": "${source_device}"}
+EOF
+ ) "https://api.pushbullet.com/v2/pushes" -X POST)
+
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# kafka sender
+
+send_kafka() {
+ local httpcode sent=0
+ if [ "${SEND_KAFKA}" = "YES" ]
+ then
+ httpcode=$(docurl -X POST \
+ --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
+ "${KAFKA_URL}")
+
+ if [ "${httpcode}" = "204" ]
+ then
+ info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}."
+ fi
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# pagerduty.com sender
+
+send_pd() {
+ local recipients="${1}" sent=0
+ unset t
+ case ${status} in
+ CLEAR) t='resolve';;
+ WARNING) t='trigger';;
+ CRITICAL) t='trigger';;
+ esac
+
+ if [ ${SEND_PD} = "YES" -a ! -z "${t}" ]
+ then
+ for PD_SERVICE_KEY in ${recipients}
+ do
+ d="${status} ${name} = ${value_string} - ${host}, ${family}"
+ ${pd_send} -k ${PD_SERVICE_KEY} \
+ -t ${t} \
+ -d "${d}" \
+ -i ${host}:${chart}:${name} \
+ -f 'info'="${info}" \
+ -f 'value_w_units'="${value_string}" \
+ -f 'when'="${when}" \
+ -f 'duration'="${duration}" \
+ -f 'roles'="${roles}" \
+ -f 'host'="${host}" \
+ -f 'unique_id'="${unique_id}" \
+ -f 'alarm_id'="${alarm_id}" \
+ -f 'event_id'="${event_id}" \
+ -f 'name'="${name}" \
+ -f 'chart'="${chart}" \
+ -f 'family'="${family}" \
+ -f 'status'="${status}" \
+ -f 'old_status'="${old_status}" \
+ -f 'value'="${value}" \
+ -f 'old_value'="${old_value}" \
+ -f 'src'="${src}" \
+ -f 'non_clear_duration'="${non_clear_duration}" \
+ -f 'units'="${units}"
+ retval=$?
+ if [ ${retval} -eq 0 ]
+ then
+ info "sent pagerduty.com notification for host ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}....: ${d}"
+ sent=$((sent + 1))
+ else
+ error "failed to send pagerduty.com notification for ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}.... (error code ${retval}): ${d}"
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# twilio sender
+
+send_twilio() {
+ local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
+ if [ "${SEND_TWILIO}" = "YES" -a ! -z "${accountsid}" -a ! -z "${accounttoken}" -a ! -z "${twilionumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
+ then
+ #https://www.twilio.com/packages/labs/code/bash/twilio-sms
+ for user in ${recipients}
+ do
+ httpcode=$(docurl -X POST \
+ --data-urlencode "From=${twilionumber}" \
+ --data-urlencode "To=${user}" \
+ --data-urlencode "Body=${title} ${message}" \
+ -u "${accountsid}:${accounttoken}" \
+ "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
+
+ if [ "${httpcode}" = "201" ]
+ then
+ info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+
+# -----------------------------------------------------------------------------
+# hipchat sender
+
+send_hipchat() {
+ local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color sender msg_format notify
+
+ # remove <small></small> from the message
+ message="${message//<small>/}"
+ message="${message//<\/small>/}"
+
+ if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${HIPCHAT_SERVER}" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ]
+ then
+ # A label to be shown in addition to the sender's name
+ # Valid length range: 0 - 64.
+ sender="netdata"
+
+ # Valid values: html, text.
+ # Defaults to 'html'.
+ msg_format="html"
+
+ # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
+ case "${status}" in
+ WARNING) color="yellow" ;;
+ CRITICAL) color="red" ;;
+ CLEAR) color="green" ;;
+ *) color="gray" ;;
+ esac
+
+ # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
+ # Each recipient's notification preferences are taken into account.
+ # Defaults to false.
+ notify="true"
+
+ for room in ${recipients}
+ do
+ httpcode=$(docurl -X POST \
+ -H "Content-type: application/json" \
+ -H "Authorization: Bearer ${authtoken}" \
+ -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
+ "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
+
+ if [ "${httpcode}" = "204" ]
+ then
+ info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+
+# -----------------------------------------------------------------------------
+# messagebird sender
+
+send_messagebird() {
+ local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
+ if [ "${SEND_MESSAGEBIRD}" = "YES" -a ! -z "${accesskey}" -a ! -z "${messagebirdnumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
+ then
+ #https://developers.messagebird.com/docs/messaging
+ for user in ${recipients}
+ do
+ httpcode=$(docurl -X POST \
+ --data-urlencode "originator=${messagebirdnumber}" \
+ --data-urlencode "recipients=${user}" \
+ --data-urlencode "body=${title} ${message}" \
+ --data-urlencode "datacoding=auto" \
+ -H "Authorization: AccessKey ${accesskey}" \
+ "https://rest.messagebird.com/messages")
+
+ if [ "${httpcode}" = "201" ]
+ then
+ info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# kavenegar sender
+
+send_kavenegar() {
+ local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
+ if [ "${SEND_KAVENEGAR}" = "YES" -a ! -z "${API_KEY}" -a ! -z "${kavenegarsender}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ]
+ then
+ # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
+ for user in ${recipients}
+ do
+ httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
+ --data-urlencode "sender=${kavenegarsender}" \
+ --data-urlencode "receptor=${user}" \
+ --data-urlencode "message=${title} ${message}")
+
+ if [ "${httpcode}" = "201" ]
+ then
+ info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# telegram sender
+
+send_telegram() {
+ local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
+
+ if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
+
+ case "${status}" in
+ WARNING) emoji="⚠️" ;;
+ CRITICAL) emoji="πŸ”΄" ;;
+ CLEAR) emoji="βœ…" ;;
+ *) emoji="βšͺ️" ;;
+ esac
+
+ if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ];
+ then
+ for chatid in ${chatids}
+ do
+ # https://core.telegram.org/bots/api#sendmessage
+ httpcode=$(docurl ${disableNotification} \
+ --data-urlencode "parse_mode=HTML" \
+ --data-urlencode "disable_web_page_preview=true" \
+ --data-urlencode "text=${emoji} ${message}" \
+ "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
+
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
+ sent=$((sent + 1))
+ elif [ "${httpcode}" = "401" ]
+ then
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
+ else
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# Microsoft Team sender
+
+send_msteam() {
+
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+ [ "${SEND_MSTEAM}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}";;
+ CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}";;
+ CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}";;
+ *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}";;
+ esac
+
+ for channel in ${channels}
+ do
+ ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
+ ## Online designer : https://acdesignerbeta.azurewebsites.net/
+ payload="$(cat <<EOF
+ {
+ "@context": "http://schema.org/extensions",
+ "@type": "MessageCard",
+ "themeColor": "${color}",
+ "title": "$icon Alert ${status} from netdata for ${host}",
+ "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
+ "potentialAction": [
+ {
+ "@type": "OpenUri",
+ "name": "Netdata",
+ "targets": [
+ { "os": "default", "uri": "${goto_url}" }
+ ]
+ }
+ ]
+ }
+EOF
+ )"
+
+ # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
+ webhook="${webhook//CHANNEL/${channel}}"
+
+ httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
+
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+
+# slack sender
+
+send_slack() {
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+ [ "${SEND_SLACK}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) color="warning" ;;
+ CRITICAL) color="danger" ;;
+ CLEAR) color="good" ;;
+ *) color="#777777" ;;
+ esac
+
+ for channel in ${channels}
+ do
+ payload="$(cat <<EOF
+ {
+ "channel": "#${channel}",
+ "username": "netdata on ${host}",
+ "icon_url": "${images_base_url}/images/seo-performance-128.png",
+ "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
+ "attachments": [
+ {
+ "fallback": "${alarm} - ${chart} (${family}) - ${info}",
+ "color": "${color}",
+ "title": "${alarm}",
+ "title_link": "${goto_url}",
+ "text": "${info}",
+ "fields": [
+ {
+ "title": "${chart}",
+ "short": true
+ },
+ {
+ "title": "${family}",
+ "short": true
+ }
+ ],
+ "thumb_url": "${image}",
+ "footer": "by <${goto_url}|${this_host}>",
+ "ts": ${when}
+ }
+ ]
+ }
+EOF
+ )"
+
+ httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+
+# -----------------------------------------------------------------------------
+# rocketchat sender
+
+send_rocketchat() {
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+ [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) color="warning" ;;
+ CRITICAL) color="danger" ;;
+ CLEAR) color="good" ;;
+ *) color="#777777" ;;
+ esac
+
+ for channel in ${channels}
+ do
+ payload="$(cat <<EOF
+ {
+ "channel": "#${channel}",
+ "alias": "netdata on ${host}",
+ "avatar": "${images_base_url}/images/seo-performance-128.png",
+ "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
+ "attachments": [
+ {
+ "color": "${color}",
+ "title": "${alarm}",
+ "title_link": "${goto_url}",
+ "text": "${info}",
+ "fields": [
+ {
+ "title": "${chart}",
+ "short": true,
+ "value": "chart"
+ },
+ {
+ "title": "${family}",
+ "short": true,
+ "value": "family"
+ }
+ ],
+ "thumb_url": "${image}",
+ "ts": "${when}"
+ }
+ ]
+ }
+EOF
+ )"
+
+ httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# alerta sender
+
+send_alerta() {
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel severity content
+
+ [ "${SEND_ALERTA}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) severity="warning" ;;
+ CRITICAL) severity="critical" ;;
+ CLEAR) severity="cleared" ;;
+ *) severity="unknown" ;;
+ esac
+
+ info=$( echo -n ${info})
+
+ # the "event" property must be unique and repetible between states to let alerta do automatic correlation using severity value
+ for channel in ${channels}
+ do
+ content="{"
+ content="$content \"environment\": \"${channel}\","
+ content="$content \"service\": [\"${host}\"],"
+ content="$content \"resource\": \"${host}\","
+ content="$content \"event\": \"${name}.${chart} (${family})\","
+ content="$content \"severity\": \"${severity}\","
+ content="$content \"value\": \"${alarm}\","
+ content="$content \"text\": \"${info}\""
+ content="$content }"
+
+
+ httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: Key $ALERTA_API_KEY" -d "$content" )
+
+ if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]]
+ then
+ info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# flock sender
+
+send_flock() {
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+ [ "${SEND_FLOCK}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) color="warning" ;;
+ CRITICAL) color="danger" ;;
+ CLEAR) color="good" ;;
+ *) color="#777777" ;;
+ esac
+
+ for channel in ${channels}
+ do
+ httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
+ \"sendAs\": {
+ \"name\" : \"netdata on ${host}\",
+ \"profileImage\" : \"${images_base_url}/images/seo-performance-128.png\"
+ },
+ \"text\": \"${host} *${status_message}*\",
+ \"timestamp\": \"${when}\",
+ \"attachments\": [
+ {
+ \"description\": \"${chart} (${family}) - ${info}\",
+ \"color\": \"${color}\",
+ \"title\": \"${alarm}\",
+ \"url\": \"${goto_url}\",
+ \"text\": \"${info}\",
+ \"views\": {
+ \"image\": {
+ \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
+ \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
+ \"filename\": \"${image}\"
+ }
+ }
+ }
+ ]
+ }" )
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# discord sender
+
+send_discord() {
+ local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
+
+ [ "${SEND_DISCORD}" != "YES" ] && return 1
+
+ case "${status}" in
+ WARNING) color="warning" ;;
+ CRITICAL) color="danger" ;;
+ CLEAR) color="good" ;;
+ *) color="#777777" ;;
+ esac
+
+ for channel in ${channels}
+ do
+ username="netdata on ${host}"
+ [ ${#username} -gt 32 ] && username="${username:0:29}..."
+
+ payload="$(cat <<EOF
+ {
+ "channel": "#${channel}",
+ "username": "${username}",
+ "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
+ "icon_url": "${images_base_url}/images/seo-performance-128.png",
+ "attachments": [
+ {
+ "color": "${color}",
+ "title": "${alarm}",
+ "title_link": "${goto_url}",
+ "text": "${info}",
+ "fields": [
+ {
+ "title": "${chart}",
+ "value": "${family}"
+ }
+ ],
+ "thumb_url": "${image}",
+ "footer_icon": "${images_base_url}/images/seo-performance-128.png",
+ "footer": "${this_host}",
+ "ts": ${when}
+ }
+ ]
+ }
+EOF
+ )"
+
+ httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
+ if [ "${httpcode}" = "200" ]
+ then
+ info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# fleep sender
+
+send_fleep() {
+ local httpcode sent=0 webhooks="${1}" data message
+ if [ "${SEND_FLEEP}" = "YES" ] ; then
+ message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
+
+ for hook in "${webhooks}" ; do
+ data="{ "
+ data="${data} 'message': '${message}', "
+ data="${data} 'user': '${FLEEP_SENDER}' "
+ data="${data} }"
+
+ httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
+
+ if [ "${httpcode}" = "200" ] ; then
+ info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}."
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+ fi
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# irc sender
+
+send_irc() {
+ local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
+
+ if [ "${SEND_IRC}" = "YES" -a ! -z "${NICKNAME}" -a ! -z "${REALNAME}" -a ! -z "${CHANNELS}" -a ! -z "${NETWORK}" -a ! -z "${SERVERNAME}" ]
+ then
+ case "${status}" in
+ WARNING) color="warning" ;;
+ CRITICAL) color="danger" ;;
+ CLEAR) color="good" ;;
+ *) color="#777777" ;;
+ esac
+
+ for CHANNEL in ${CHANNELS}
+ do
+ error=0
+ send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\nNICK ${NICKNAME}\nJOIN ${CHANNEL}\nPRIVMSG ${CHANNEL} :${MESSAGE}\nQUIT\n" \ | nc ${NETWORK} 6667)
+ reply_codes=$(echo ${send_alarm} | cut -d ' ' -f 2 | grep -o '[0-9]*')
+ for code in ${reply_codes}
+ do
+ [ "${code}" -ge 400 -a "${code}" -le 599 ] && error=1 && break
+ done
+
+ if [ "${error}" -eq 0 ]
+ then
+ info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
+ fi
+ done
+ fi
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# Amazon SNS sender
+
+send_awssns() {
+ local targets="${1}" message='' sent=0 region=''
+ local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+ [ "${SEND_AWSSNS}" = "YES" ] || return 1
+
+ message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
+
+ for target in ${targets} ; do
+ # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly.
+ region="$(echo ${target} | cut -f 4 -d ':')"
+ ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null
+ if [ $? = 0 ]; then
+ info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
+ sent=$((sent + 1))
+ else
+ error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
+ fi
+ done
+
+ [ ${sent} -gt 0 ] && return 0
+
+ return 1
+}
+
+# -----------------------------------------------------------------------------
+# syslog sender
+
+send_syslog() {
+ local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
+ local priority='' message='' host='' port='' prefix=''
+ local temp1='' temp2=''
+
+ [ "${SEND_SYSLOG}" = "YES" ] || return 1
+
+ if [ "${status}" = "CRITICAL" ] ; then
+ level='crit'
+ elif [ "${status}" = "WARNING" ] ; then
+ level='warning'
+ fi
+
+ for target in ${targets} ; do
+ priority="${facility}.${level}"
+ message=''
+ host=''
+ port=''
+ prefix=''
+ temp1=''
+ temp2=''
+
+ prefix=$(echo ${target} | cut -d '/' -f 2)
+ temp1=$(echo ${target} | cut -d '/' -f 1)
+
+ if [ ${prefix} != ${temp1} ] ; then
+ if (echo ${temp1} | grep -q '@' ) ; then
+ temp2=$(echo ${temp1} | cut -d '@' -f 1)
+ host=$(echo ${temp1} | cut -d '@' -f 2)
+
+ if [ ${temp2} != ${host} ] ; then
+ priority=${temp2}
+ fi
+
+ port=$(echo ${host} | rev | cut -d ':' -f 1 | rev)
+
+ if ( echo ${host} | grep -E -q '\[.*\]' ) ; then
+ if ( echo ${port} | grep -q ']' ) ; then
+ port=''
+ else
+ host=$(echo ${host} | rev | cut -d ':' -f 2- | rev)
+ fi
+ else
+ if [ ${port} = ${host} ] ; then
+ port=''
+ else
+ host=$(echo ${host} | cut -d ':' -f 1)
+ fi
+ fi
+ else
+ priority=${temp1}
+ fi
+ fi
+
+ message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+ if [ ${host} ] ; then
+ logger_options="${logger_options} -n ${host}"
+ if [ ${port} ] ; then
+ logger_options="${logger_options} -P ${port}"
+ fi
+ fi
+
+ ${logger} -p ${priority} ${logger_options} "${message}"
+ done
+
+ return $?
+}
+
+
+# -----------------------------------------------------------------------------
+# prepare the content of the notification
+
+# the url to send the user on click
+urlencode "${host}" >/dev/null; url_host="${REPLY}"
+urlencode "${chart}" >/dev/null; url_chart="${REPLY}"
+urlencode "${family}" >/dev/null; url_family="${REPLY}"
+urlencode "${name}" >/dev/null; url_name="${REPLY}"
+goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}"
+
+# the severity of the alarm
+severity="${status}"
+
+# the time the alarm was raised
+duration4human ${duration} >/dev/null; duration_txt="${REPLY}"
+duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}"
+raised_for="(was ${old_status,,} for ${duration_txt})"
+
+# the key status message
+status_message="status unknown"
+
+# the color of the alarm
+color="grey"
+
+# the alarm value
+alarm="${name//_/ } = ${value_string}"
+
+# the image of the alarm
+image="${images_base_url}/images/seo-performance-128.png"
+
+# prepare the title based on status
+case "${status}" in
+ CRITICAL)
+ image="${images_base_url}/images/alert-128-red.png"
+ status_message="is critical"
+ color="#ca414b"
+ ;;
+
+ WARNING)
+ image="${images_base_url}/images/alert-128-orange.png"
+ status_message="needs attention"
+ color="#ffc107"
+ ;;
+
+ CLEAR)
+ image="${images_base_url}/images/check-mark-2-128-green.png"
+ status_message="recovered"
+ color="#77ca6d"
+ ;;
+esac
+
+if [ "${status}" = "CLEAR" ]
+then
+ severity="Recovered from ${old_status}"
+ if [ ${non_clear_duration} -gt ${duration} ]
+ then
+ raised_for="(alarm was raised for ${non_clear_duration_txt})"
+ fi
+
+ # don't show the value when the status is CLEAR
+ # for certain alarms, this value might not have any meaning
+ alarm="${name//_/ } ${raised_for}"
+
+elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
+then
+ severity="Escalated to ${status}"
+ if [ ${non_clear_duration} -gt ${duration} ]
+ then
+ raised_for="(alarm is raised for ${non_clear_duration_txt})"
+ fi
+
+elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
+then
+ severity="Demoted to ${status}"
+ if [ ${non_clear_duration} -gt ${duration} ]
+ then
+ raised_for="(alarm is raised for ${non_clear_duration_txt})"
+ fi
+
+else
+ raised_for=
+fi
+
+# prepare HTML versions of elements
+info_html=
+[ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>"
+
+raised_for_html=
+[ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
+
+# -----------------------------------------------------------------------------
+# send the slack notification
+
+# slack aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
+SENT_SLACK=$?
+
+# -----------------------------------------------------------------------------
+# send the Microsoft notification
+
+# Microsoft team aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
+SENT_MSTEAM=$?
+
+# -----------------------------------------------------------------------------
+# send the rocketchat notification
+
+# rocketchat aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
+SENT_ROCKETCHAT=$?
+
+# -----------------------------------------------------------------------------
+# send the alerta notification
+
+# alerta aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
+SENT_ALERTA=$?
+
+# -----------------------------------------------------------------------------
+# send the flock notification
+
+# flock aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
+SENT_FLOCK=$?
+
+# -----------------------------------------------------------------------------
+# send the discord notification
+
+# discord aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
+SENT_DISCORD=$?
+
+# -----------------------------------------------------------------------------
+# send the pushover notification
+
+send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
+<font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
+<small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
+<small><b>${family}</b><br/>Family<br/>&nbsp;</small>
+<small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
+<small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
+<a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
+<small><small>The source of this alarm is line ${src}</small></small>
+"
+
+SENT_PUSHOVER=$?
+
+# -----------------------------------------------------------------------------
+# send the pushbullet notification
+
+send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\n
+Severity: ${severity}\n
+Chart: ${chart}\n
+Family: ${family}\n
+$(date -d @${when})\n
+The source of this alarm is line ${src}"
+
+SENT_PUSHBULLET=$?
+
+# -----------------------------------------------------------------------------
+# send the twilio SMS
+
+send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_TWILIO=$?
+
+# -----------------------------------------------------------------------------
+# send the messagebird SMS
+
+send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_MESSAGEBIRD=$?
+
+
+# -----------------------------------------------------------------------------
+# send the kavenegar SMS
+
+send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_KAVENEGAR=$?
+
+
+# -----------------------------------------------------------------------------
+# send the telegram.org message
+
+# https://core.telegram.org/bots/api#formatting-options
+send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
+${chart} (${family})
+<a href=\"${goto_url}\">${alarm}</a>
+<i>${info}</i>"
+
+SENT_TELEGRAM=$?
+
+
+# -----------------------------------------------------------------------------
+# send the kafka message
+
+send_kafka
+SENT_KAFKA=$?
+
+
+# -----------------------------------------------------------------------------
+# send the pagerduty.com message
+
+send_pd "${to_pd}"
+SENT_PD=$?
+
+# -----------------------------------------------------------------------------
+# send the fleep message
+
+send_fleep "${to_fleep}"
+SENT_FLEEP=$?
+
+# -----------------------------------------------------------------------------
+# send the irc message
+
+send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_IRC=$?
+
+# -----------------------------------------------------------------------------
+# send the custom message
+
+send_custom() {
+ # is it enabled?
+ [ "${SEND_CUSTOM}" != "YES" ] && return 1
+
+ # do we have any sender?
+ [ -z "${1}" ] && return 1
+
+ # call the custom_sender function
+ custom_sender "${@}"
+}
+
+send_custom "${to_custom}"
+SENT_CUSTOM=$?
+
+
+# -----------------------------------------------------------------------------
+# send hipchat message
+
+send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
+${host} ${status_message}<br/> \
+<b>${alarm}</b> ${info_html}<br/> \
+<b>${chart}</b> (family <b>${family}</b>)<br/> \
+<b>${date}${raised_for_html}</b><br/> \
+<a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
+(source of alarm ${src}) \
+"
+
+SENT_HIPCHAT=$?
+
+
+# -----------------------------------------------------------------------------
+# send the Amazon SNS message
+
+send_awssns ${to_awssns}
+
+SENT_AWSSNS=$?
+
+
+# -----------------------------------------------------------------------------
+# send the syslog message
+
+send_syslog ${to_syslog}
+
+SENT_SYSLOG=$?
+
+
+# -----------------------------------------------------------------------------
+# send the email
+
+send_email <<EOF
+To: ${to_email}
+Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="multipart-boundary"
+${email_thread_headers}
+
+This is a MIME-encoded multipart message
+
+--multipart-boundary
+Content-Type: text/plain; encoding=${EMAIL_CHARSET}
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+${host} ${status_message}
+
+${alarm} ${info}
+${raised_for}
+
+Chart : ${chart}
+Family : ${family}
+Severity: ${severity}
+URL : ${goto_url}
+Source : ${src}
+Date : ${date}
+Notification generated on ${this_host}
+
+--multipart-boundary
+Content-Type: text/html; encoding=${EMAIL_CHARSET}
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
+<body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
+<table>
+ <tbody>
+ <tr>
+ <td style="vertical-align: top;" valign="top"></td>
+ <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
+ <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
+ <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
+ <tbody>
+ <tr>
+ <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
+ <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
+ </td>
+ </tr>
+ <tr>
+ <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
+ <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
+ </td>
+ </tr>
+ <tr>
+ <td style="vertical-align: top;" valign="top">
+ <div style="margin: 0; padding: 20px; max-width: 700px;">
+ <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
+ <tbody>
+ <tr>
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
+ <span>${chart}</span>
+ <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
+ </td>
+ </tr>
+ <tr style="margin: 0; padding: 0;">
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+ <span><b>${alarm}</b>${info_html}</span>
+ <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
+ </td>
+ </tr>
+ <tr>
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+ <span>${family}</span>
+ <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
+ </td>
+ </tr>
+ <tr style="margin: 0; padding: 0;">
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+ <span>${severity}</span>
+ <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
+ </td>
+ </tr>
+ <tr style="margin: 0; padding: 0;">
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
+ <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
+ </td>
+ </tr>
+ <tr style="margin: 0; padding: 0;">
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
+ <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
+ </td>
+ </tr>
+ <tr style="text-align: center; margin: 0; padding: 0;">
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
+ </td>
+ </tr>
+ <tr style="text-align: center; margin: 0; padding: 0;">
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
+ <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${this_host}</code>.
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </td>
+ </tr>
+ </tbody>
+</table>
+</body>
+</html>
+--multipart-boundary--
+EOF
+
+SENT_EMAIL=$?
+
+# -----------------------------------------------------------------------------
+# let netdata know
+
+if [ ${SENT_EMAIL} -eq 0 \
+ -o ${SENT_PUSHOVER} -eq 0 \
+ -o ${SENT_TELEGRAM} -eq 0 \
+ -o ${SENT_SLACK} -eq 0 \
+ -o ${SENT_MSTEAM} -eq 0 \
+ -o ${SENT_ROCKETCHAT} -eq 0 \
+ -o ${SENT_ALERTA} -eq 0 \
+ -o ${SENT_FLOCK} -eq 0 \
+ -o ${SENT_DISCORD} -eq 0 \
+ -o ${SENT_TWILIO} -eq 0 \
+ -o ${SENT_HIPCHAT} -eq 0 \
+ -o ${SENT_MESSAGEBIRD} -eq 0 \
+ -o ${SENT_KAVENEGAR} -eq 0 \
+ -o ${SENT_PUSHBULLET} -eq 0 \
+ -o ${SENT_KAFKA} -eq 0 \
+ -o ${SENT_PD} -eq 0 \
+ -o ${SENT_FLEEP} -eq 0 \
+ -o ${SENT_IRC} -eq 0 \
+ -o ${SENT_AWSSNS} -eq 0 \
+ -o ${SENT_CUSTOM} -eq 0 \
+ -o ${SENT_SYSLOG} -eq 0 \
+ ]
+ then
+ # we did send something
+ exit 0
+fi
+
+# we did not send anything
+exit 1
diff --git a/plugins.d/alarm-test.sh b/health/notifications/alarm-test.sh
index 9df5361a..828aa756 100755
--- a/plugins.d/alarm-test.sh
+++ b/health/notifications/alarm-test.sh
@@ -3,10 +3,10 @@
# netdata
# real-time performance and health monitoring, done right!
# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
-# GPL v3+
+# SPDX-License-Identifier: GPL-3.0-or-later
#
# Script to test alarm notifications for netdata
dir="$(dirname "${0}")"
-${dir}/alarm-notify.sh test "${1}"
+"${dir}/alarm-notify.sh" test "${1}"
exit $?
diff --git a/health/notifications/alerta/Makefile.inc b/health/notifications/alerta/Makefile.inc
new file mode 100644
index 00000000..32fa0894
--- /dev/null
+++ b/health/notifications/alerta/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ alerta/README.md \
+ alerta/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/alerta/README.md b/health/notifications/alerta/README.md
new file mode 100644
index 00000000..bbed23ba
--- /dev/null
+++ b/health/notifications/alerta/README.md
@@ -0,0 +1,236 @@
+# alerta.io notifications
+
+The alerta monitoring system is a tool used to consolidate and de-duplicate alerts from multiple sources for quick β€˜at-a-glance’ visualisation. With just one system you can monitor alerts from many other monitoring tools on a single screen.
+
+![](http://docs.alerta.io/en/latest/_images/alerta-screen-shot-3.png)
+
+When receiving alerts from multiple sources you can quickly become overwhelmed. With Alerta any alert with the same environment and resource is considered a duplicate if it has the same severity. If it has a different severity it is correlated so that you only see the most recent one. Awesome.
+
+main site http://www.alerta.io
+
+We can send Netadata alarms to Alerta so yo can see in one place alerts coming from many Netdata hosts or also from a multihost Netadata configuration.\
+The big advantage over other notifications method is that you have in a main view all active alarms with only las state, but you can also search history.
+
+## Setting up an Alerta server with Ubuntu 16.04
+
+Here we will set a basic Alerta server to test it with Netdata alerts.\
+More advanced configurations are out os scope of this tutorial.
+
+source: http://alerta.readthedocs.io/en/latest/gettingstarted/tutorial-1-deploy-alerta.html
+
+I recommend to set up the server in a separated server, VM or container.\
+If you have other Nginx or Apache server in your organization, I recommend to proxy to this new server.
+
+Set us as root for easiest working
+```
+sudo su
+cd
+```
+
+Install Mongodb https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
+```
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
+echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list
+apt-get update
+apt-get install -y mongodb-org
+systemctl enable mongod
+systemctl start mongod
+systemctl status mongod
+```
+
+Install Nginx and Alerta uwsgi
+```
+apt-get install -y python-pip python-dev nginx
+pip install alerta-server uwsgi
+```
+
+Install web console
+```
+cd /var/www/html
+mkdir alerta
+cd alerta
+wget -q -O - https://github.com/alerta/angular-alerta-webui/tarball/master | tar zxf -
+mv alerta*/app/* .
+cd
+```
+## Services configuration
+
+Create a wsgi python file
+```
+nano /var/www/wsgi.py
+```
+fill with
+```
+from alerta import app
+```
+Create uWsgi configuration file
+```
+nano /etc/uwsgi.ini
+```
+fill with
+```
+[uwsgi]
+chdir = /var/www
+mount = /alerta/api=wsgi.py
+callable = app
+manage-script-name = true
+
+master = true
+processes = 5
+logger = syslog:alertad
+
+socket = /tmp/uwsgi.sock
+chmod-socket = 664
+uid = www-data
+gid = www-data
+vacuum = true
+
+die-on-term = true
+```
+Create a systemd configuration file
+```
+nano /etc/systemd/system/uwsgi.service
+```
+fill with
+```
+[Unit]
+Description=uWSGI service
+
+[Service]
+ExecStart=/usr/local/bin/uwsgi --ini /etc/uwsgi.ini
+
+[Install]
+WantedBy=multi-user.target
+```
+enable service
+```
+systemctl start uwsgi
+systemctl status uwsgi
+systemctl enable uwsgi
+```
+Configure nginx to serve Alerta as a uWsgi application on /alerta/api
+```
+nano /etc/nginx/sites-enabled/default
+```
+fill with
+```
+server {
+ listen 80 default_server;
+ listen [::]:80 default_server;
+
+ location /alerta/api { try_files $uri @alerta/api; }
+ location @alerta/api {
+ include uwsgi_params;
+ uwsgi_pass unix:/tmp/uwsgi.sock;
+ proxy_set_header Host $host:$server_port;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+
+ location / {
+ root /var/www/html;
+ }
+}
+```
+restart nginx
+```
+service nginx restart
+```
+## Config web console
+```
+nano /var/www/html/config.js
+```
+fill with
+```
+'use strict';
+
+angular.module('config', [])
+ .constant('config', {
+ 'endpoint' : "/alerta/api",
+ 'provider' : "basic",
+ 'colors' : {},
+ 'severity' : {},
+ 'audio' : {}
+ });
+```
+
+## Config Alerta server
+
+source: http://alerta.readthedocs.io/en/latest/configuration.html
+
+Create a random string to use as SECRET_KEY
+```
+cat /dev/urandom | tr -dc A-Za-z0-9_\!\@\#\$\%\^\&\*\(\)-+= | head -c 32 && echo
+```
+will output something like
+```
+0pv8Bw7VKfW6avDAz_TqzYPme_fYV%7g
+```
+Edit alertad.conf
+```
+nano /etc/alertad.conf
+```
+fill with (take care about all single quotes)
+```
+BASE_URL='/alerta/api'
+AUTH_REQUIRED=True
+SECRET_KEY='0pv8Bw7VKfW6avDAz_TqzYPme_fYV%7g'
+ADMIN_USERS=['<here put you email for future login>']
+```
+
+restart
+```
+systemctl restart uwsgi
+```
+
+* go to console to http://yourserver/alerta/
+* go to Login -> Create an account
+* use your email for login so and administrative account will be created
+
+## create an API KEY
+
+You need an API KEY to send messages from any source.\
+To create an API KEY go to Configuration -> Api Keys\
+Then create a API KEY with write permisions.
+
+## configure Netdata to send alarms to Alerta
+
+On your system run:
+
+```
+/etc/netdata/edit-config health_alarm_notify.conf
+```
+
+and set
+
+```
+# enable/disable sending alerta notifications
+SEND_ALERTA="YES"
+
+# here set your alerta server API url
+# this is the API url you defined when installed Alerta server,
+# it is the same for all users. Do not include last slash.
+ALERTA_WEBHOOK_URL="http://yourserver/alerta/api"
+
+# Login with an administrative user to you Alerta server and create an API KEY
+# with write permissions.
+ALERTA_API_KEY="you last created API KEY"
+
+# you can define environments in /etc/alertad.conf option ALLOWED_ENVIRONMENTS
+# standard environments are Production and Development
+# if a role's recipients are not configured, a notification will be send to
+# this Environment (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_ALERTA="Production"
+```
+
+## Test alarms
+
+We can test alarms with standard
+```
+sudo su -s /bin/bash netdata
+/opt/netdata/netdata-plugins/plugins.d/alarm-notify.sh test
+exit
+```
+But the problem is that Netdata will send 3 alarms, and because last alarm is "CLEAR" you will not se them in main Alerta page, you need to select to see "closed" alarma in top-right lookup.
+
+A little change in alarm-notify.sh that let us test each state one by one will be useful. \ No newline at end of file
diff --git a/health/notifications/awssns/Makefile.inc b/health/notifications/awssns/Makefile.inc
new file mode 100644
index 00000000..3d8e58ff
--- /dev/null
+++ b/health/notifications/awssns/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ awssns/README.md \
+ awssns/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/awssns/README.md b/health/notifications/awssns/README.md
new file mode 100644
index 00000000..d040d2d2
--- /dev/null
+++ b/health/notifications/awssns/README.md
@@ -0,0 +1,31 @@
+# Amazon SNS notifications
+
+As part of it's AWS suite, Amazon provides a notification broker service called 'Simple Notification Service' or SNS. Amazon SNS works kind of similarly to Netdata's own notification system, allowing dispatch of a single notification to multiple subscribers of different types. Among other things, SNS supports sending notifications to:
+
+* Email addresses.
+* Mobile Phones via SMS.
+* HTTP or HTTPS web hooks.
+* AWS Lambda functions.
+* AWS SQS queues.
+* Mobile applications via push notifications.
+
+To get this working, you will need:
+
+* The Amazon Web Services CLI tools. Most distributions provide these with the package name `awscli`.
+* An actual home directory for the user you run Netdata as, instead of just using `/` as a home directory. Setup of this is distribution specific. `/var/lib/netdata` is the recommended directory (because the permissions will already be correct) if you are using a dedicated user (which is how most distributions work).
+* An Amazon SNS topic to send notifications to with one or more subscribers. The [Getting Started](https://docs.aws.amazon.com/sns/latest/dg/GettingStarted.html) section of the Amazon SNS documentation covers the basics of how to set this up. Make note of the Topic ARN when you create the topic.
+* While not mandatory, it is highly recommended to create a dedicated IAM user on your account for netdata to send notifications. This user needs to have programmatic access, and should only allow access to SNS. If you're really paranoid, you can create one for each system or group of systems.
+
+Once you have all the above, run the follwing command as the user netdata runs under:
+
+ aws configure
+
+THis will prompt you for the access key and secret key for accessing Amazon SNS (as well as the default region and output format, but you can leave those blank because we don't use them).
+
+Once that's done, you're ready to go and can specify the desired topic ARN as a recipient.
+
+Notes:
+
+ * Netdata's native email notification support is far better in almost all respects than it's support through Amazon SNS. If you want email notifications, use the native support, not SNS.
+ * If you need to change the notification format for SNS notifications, you can do so by specifying the format in `AWSSNS_MESSAGE_FORMAT` in the configuration. This variable supports all the same vairiables you can use in custom notifications.
+ * While Amazon SNS supports sending differently formatted messages for different delivery methods, netdata does not currently support this functionality. \ No newline at end of file
diff --git a/health/notifications/discord/Makefile.inc b/health/notifications/discord/Makefile.inc
new file mode 100644
index 00000000..03d0339a
--- /dev/null
+++ b/health/notifications/discord/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ discord/README.md \
+ discord/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/discord/README.md b/health/notifications/discord/README.md
new file mode 100644
index 00000000..5889ea09
--- /dev/null
+++ b/health/notifications/discord/README.md
@@ -0,0 +1,44 @@
+# Discordapp.com notifications
+
+This is what you will get:
+
+![image](https://cloud.githubusercontent.com/assets/7321975/22215935/b49ede7e-e162-11e6-98d0-ae8541e6b92e.png)
+
+You need:
+
+1. The **incoming webhook URL** as given by Discord. Create a webhook by following the official [Discord documentation](https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks). You can use the same on all your netdata servers (or you can have multiple if you like - your decision).
+2. One or more Discord channels to post the messages to.
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# sending discord notifications
+
+# note: multiple recipients can be given like this:
+# "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending discord notifications
+SEND_DISCORD="YES"
+
+# Create a webhook by following the official documentation -
+# https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks
+DISCORD_WEBHOOK_URL="https://discordapp.com/api/webhooks/XXXXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+# if a role's recipients are not configured, a notification will be send to
+# this discord channel (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_DISCORD="alarms"
+
+```
+
+You can define multiple channels like this: `alarms systems`.
+You can give different channels per **role** using these (at the same file):
+
+```
+role_recipients_discord[sysadmin]="systems"
+role_recipients_discord[dba]="databases systems"
+role_recipients_discord[webmaster]="marketing development"
+```
+
+The keywords `systems`, `databases`, `marketing`, `development` are discordapp.com channels (they should already exist within your discord server).
diff --git a/health/notifications/email/Makefile.inc b/health/notifications/email/Makefile.inc
new file mode 100644
index 00000000..62dd18a6
--- /dev/null
+++ b/health/notifications/email/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ email/README.md \
+ email/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/email/README.md b/health/notifications/email/README.md
new file mode 100644
index 00000000..979790ad
--- /dev/null
+++ b/health/notifications/email/README.md
@@ -0,0 +1,31 @@
+# email notifications
+
+You need a working `sendmail` command for email alerts to work. Almost all MTAs provide a `sendmail` interface.
+
+netdata sends all emails as user `netdata`, so make sure your `sendmail` works for local users.
+
+email notifications look like this:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/18407294/e9218c68-7714-11e6-8739-e4dd8a498252.png)
+
+## configuration
+
+To edit `health_alarm_notify.conf` on your system run `/etc/netdata/edit-config health_alarm_notify.conf`.
+
+You can configure recipients in [`/etc/netdata/health_alarm_notify.conf`](https://github.com/netdata/netdata/blob/99d44b7d0c4e006b11318a28ba4a7e7d3f9b3bae/conf.d/health_alarm_notify.conf#L101).
+
+You can also configure per role recipients [in the same file, a few lines below](https://github.com/netdata/netdata/blob/99d44b7d0c4e006b11318a28ba4a7e7d3f9b3bae/conf.d/health_alarm_notify.conf#L313).
+
+Changes to this file do not require netdata restart.
+
+You can test your configuration by issuing the commands:
+
+```sh
+# become user netdata
+sudo su -s /bin/bash netdata
+
+# send a test alarm
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test [ROLE]
+```
+
+Where `[ROLE]` is the role you want to test. The default (if you don't give a `[ROLE]`) is `sysadmin`. \ No newline at end of file
diff --git a/health/notifications/flock/Makefile.inc b/health/notifications/flock/Makefile.inc
new file mode 100644
index 00000000..fbff309a
--- /dev/null
+++ b/health/notifications/flock/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ flock/README.md \
+ flock/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/flock/README.md b/health/notifications/flock/README.md
new file mode 100644
index 00000000..33a545ea
--- /dev/null
+++ b/health/notifications/flock/README.md
@@ -0,0 +1,31 @@
+# flock.com notifications
+
+This is what you will get:
+
+
+![Flock](https://i.imgur.com/ok9bRzw.png)
+
+You need:
+
+The **incoming webhook URL** as given by flock.com. You can use the same on all your netdata servers (or you can have multiple if you like - your decision).
+
+Get them here: https://admin.flock.com/webhooks
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# sending flock notifications
+
+# enable/disable sending pushover notifications
+SEND_FLOCK="YES"
+
+# Login to flock.com and create an incoming webhook.
+# You need only one for all your netdata servers.
+# Without it, netdata cannot send flock notifications.
+FLOCK_WEBHOOK_URL="https://api.flock.com/hooks/sendMessage/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+# if a role recipient is not configured, no notification will be sent
+DEFAULT_RECIPIENT_FLOCK="alarms"
+
+``` \ No newline at end of file
diff --git a/conf.d/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf
index 0a95931e..9e72aac4 100755
--- a/conf.d/health_alarm_notify.conf
+++ b/health/notifications/health_alarm_notify.conf
@@ -14,7 +14,10 @@
# - sms messages to your cell phone or any sms enabled device (twilio.com)
# - sms messages to your cell phone or any sms enabled device (messagebird.com)
# - notifications to users on pagerduty.com
+# - notifications to Amazon SNS topics (aws.amazon.com)
# - messages to your irc channel on your selected network
+# - messages to a local or remote syslog daemon
+# - message to Microsoft Team (thru webhook)
#
# The 'to' line given at netdata alarms defines a *role*, so that many
# people can be notified for each role.
@@ -44,6 +47,23 @@
#------------------------------------------------------------------------------
+# date handling
+#
+# You can configure netdata alerts to send dates in any format you want.
+# This uses standard `date` command format strings. See `man date` for
+# more info on what you can put in here. Note that this has to start with a '+', otherwise it won't work.
+#
+# For ISO 8601 dates, use '+%FT%T%z'
+# For RFC 5322 dates, use '+%a, %d %b %Y %H:%M:%S %z'
+# For RFC 3339 dates, use '+%F %T%:z'
+# For RFC 1123 dates, use '+%a, %d %b %Y %H:%M:%S %Z'
+# For RFC 1036 dates, use '+%A, %d-%b-%y %H:%M:%S %Z'
+# For a reasonably local date and time (in that order), use '+%x %X'
+# For the old default behavior (compatible with ANSI C's asctime() function), leave this empty.
+date_format=''
+
+
+#------------------------------------------------------------------------------
# external commands
# The full path to the sendmail command.
@@ -61,6 +81,16 @@ curl=""
# If not found, irc notifications will be silently disabled.
nc=""
+# The full path of the logger command.
+# If empty, the system $PATH will be searched for it.
+# If not found, syslog notifications will be silently disabled.
+logger=""
+
+# The full path of the aws command.
+# If empty, the system $PATH will be searched for it.
+# If not found, Amazon SNS notifications will be silently disabled.
+aws=""
+
#------------------------------------------------------------------------------
# extra options for external commands
#
@@ -74,6 +104,10 @@ nc=""
# of potentially sensitive information.
#curl_options="--insecure"
+# Extra options to pass to logger. You shouldn't have to specify anything
+# here in most cases.
+#logger_options=""
+
#------------------------------------------------------------------------------
# NOTE ABOUT RECIPIENTS
#
@@ -144,6 +178,14 @@ DEFAULT_RECIPIENT_EMAIL="root"
# autodetected from the environment.
#EMAIL_CHARSET="UTF-8"
+# You can also have netdata add headers to the message that will
+# cause most e-mail clients to treat all notifications for a given
+# chart+alarm+host combination as a single thread. This can help
+# simplify tracking of alarms, as it provides an easy wway for scripts
+# to corelate messages and also will cause most clients to group all the
+# messages together. THis is off by default.
+#EMAIL_THREADING="YES"
+
#------------------------------------------------------------------------------
# pushover (pushover.net) global notification options
@@ -297,6 +339,60 @@ SLACK_WEBHOOK_URL=""
# roles):
DEFAULT_RECIPIENT_SLACK=""
+#------------------------------------------------------------------------------
+# Microsoft Team (office.com) global notification options
+# More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
+# Online designer : https://acdesignerbeta.azurewebsites.net/
+# multiple recipients can be given like this:
+# "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending team notifications
+SEND_MSTEAM="YES"
+
+# if a role's recipients are not configured, a notification will be send to
+# this slack channel (empty = do not send a notification for unconfigured
+# roles):
+# For team the channel name is encoded in the URI after ....IncomingWebhook/___/.....
+# This value will be replaced in the webhook value to publish to several channels in a same Team.
+# In order to get it working properly, you have to replace the value between [] ....IncomingWebhook/[___]/..... by "CHANNEL" string.
+DEFAULT_RECIPIENT_MSTEAM=""
+# Based on the way MS Teams is working, put the differents channels here like : "CHANNEL1 CHANNEL2 ..."
+# AT LEAST ONE CHANNEL IS MANDATORY
+MSTEAM_WEBHOOK_URL=""
+
+# Define the default color scheme for alert to MS Team - icon and color
+# Icons - go to https://emojipedia.org/bomb/
+MSTEAM_ICON_DEFAULT="β™‘"
+MSTEAM_ICON_CLEAR="πŸ’š"
+MSTEAM_ICON_WARNING="⚠️"
+MSTEAM_ICON_CRITICAL="πŸ”₯"
+
+# Colors
+MSTEAM_COLOR_DEFAULT="0076D7"
+MSTEAM_COLOR_CLEAR="65A677"
+MSTEAM_COLOR_WARNING="FFA500"
+MSTEAM_COLOR_CRITICAL="D93F3C"
+
+
+#------------------------------------------------------------------------------
+# rocketchat (rocket.chat) global notification options
+
+# multiple recipients can be given like this:
+# "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending rocketchat notifications
+SEND_ROCKETCHAT="YES"
+
+# Login to rocket.chat and create an incoming webhook. You need only one for all
+# your netdata servers (or you can have one for each of your netdata).
+# Without it, netdata cannot send rocketchat notifications.
+ROCKETCHAT_WEBHOOK_URL=""
+
+# if a role's recipients are not configured, a notification will be send to
+# this rocketchat channel (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_ROCKETCHAT=""
+
#------------------------------------------------------------------------------
# alerta (alerta.io) global notification options
@@ -413,6 +509,28 @@ DEFAULT_RECIPIENT_PD=""
#------------------------------------------------------------------------------
+# fleep notification options
+#
+# To send fleep.io notifications, you will need a webhook for the
+# conversation you want to send to.
+
+# Fleep recipients are specified as the last part of the webhook URL.
+# So, for a webhook URL of: https://fleep.io/hook/IJONmBuuSlWlkb_ttqyXJg, the
+# recipient name would be: 'IJONmBuuSlWlkb_ttqyXJg'.
+
+# enable/disable sending fleep notifications
+SEND_FLEEP="YES"
+
+# if a role's recipients are not configured, a notification will not be sent.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_FLEEP=""
+
+# The user name to label the messages with. If this is unset,
+# the hostname of the system the notification is for will be used.
+FLEEP_SENDER=""
+
+
+#------------------------------------------------------------------------------
# irc notification options
#
# irc notifications require only the nc utility to be installed.
@@ -441,6 +559,94 @@ IRC_REALNAME=""
#------------------------------------------------------------------------------
+# syslog notifications
+#
+# syslog notifications only need you to have a working logger command, which
+# should be the case on pretty much any Linux system.
+
+# enable/disable sending syslog notifications
+# NOTE: make sure you have everything else configured the way you want
+# it _before_ turning this on.
+SEND_SYSLOG="NO"
+
+# A note on log levels and facilities:
+#
+# The traditional UNIX syslog mechanism has the concept of both log
+# levels and facilities. A log level indicates the relaitve severity of
+# the message, while a facility specifies a generic source for the message
+# (for example, the `mail` facility is where sendmail and postfix log
+# their messages). All major syslog daemons have the ability to filter
+# messages based on both log level and facility, and can often also make
+# routing decisions for messages based on both factors.
+#
+# On Linux, the eight log levels in decreasing order of severity are:
+# emerg, alert, crit, err, warning, notice, info, debug
+#
+# By default, warnings will be logged at the warning level, critical
+# alerts at the crit level, and clear notifications at the invo level.
+#
+# And the 19 facilities you can log to are:
+# auth, authpriv, cron, daemon, ftp, lpr, mail, news, syslog, user,
+# uucp, local0, local1, local2, local3, local4, local5, local6, and local7
+#
+# By default, netdata alerts will be logged to the local6 facility.
+#
+# Depending on your distribution, this means that either all your
+# netdata alerts will by default end up in the main system log (usually
+# /var/log/messages), or they won't be logged to a file at all.
+# Neither of these are likely to be what you actually want, but any
+# configuration to change that needs to happen in the syslog daemon
+# configuration, not here.
+
+# This controls which facility is used by defalt for logging. Defaults
+# to local6.
+SYSLOG_FACILITY=''
+
+# If a role's recipients are not configured, use the following.
+# (empty = do not send a notification for unconfigured roles)
+#
+# The recipient format for syslog uses the following format:
+# [[facility.level][@host[:port]]/]prefix
+#
+# `prefix` gets appended to the front of all log messages generated for
+# that recipient. The prefix is mandatory.
+# 'host' and 'port' can be used to specify a remote syslog server to
+# send messages to. Leave these out if you want messages to be delivered
+# locally. 'host' can be either a hostname or an IP address.
+# IPv6 addresses must have square around them.
+# 'facility' and 'level' are used to override the default logging facility
+# set above and the log level. If one is specified, both must be present.
+#
+# For example, to send messages with a 'netdata' prefix to a syslog
+# daemon listening on port 514 on 'loghost' using the daemon facility and
+# notice log level:
+# DEFAULT_RECIPIENT_SYSLOG='daemon.notice@loghost:514/netdata'
+#
+DEFAULT_RECIPIENT_SYSLOG="netdata"
+
+#------------------------------------------------------------------------------
+# Amazon SNS notifications
+#
+# This method requires potentially complex manual configuration. See the
+# netdata wiki for information on what is needed.
+
+# enable/disable sending Amazon SNS notifications
+SEND_AWSSNS="YES"
+
+# Specify a template for the Amazon SNS notifications. This supports
+# the same set of variables that are usable in the `custom_sender()`
+# function in the custom notification configuration below.
+#
+AWSSNS_MESSAGE_FORMAT="${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+# If a role's recipients are not configured, use the following.
+# (empty = do not send a notification for unconfigured roles)
+#
+# Recipients for AWS SNS notifications are specified as topic ARN's.
+#
+DEFAULT_RECIPIENT_AWSSNS=""
+
+#------------------------------------------------------------------------------
# custom notifications
#
@@ -535,10 +741,18 @@ role_recipients_kavenegar[sysadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"
role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"
+role_recipients_fleep[sysadmin]="${DEFAULT_RECIPIENT_FLEEP}"
+
role_recipients_irc[sysadmin]="${DEFAULT_RECIPIENT_IRC}"
+role_recipients_syslog[sysadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_awssns[sysadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
+
role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+role_recipients_msteam[sysadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
+
# -----------------------------------------------------------------------------
# DNS related alarms
@@ -568,10 +782,18 @@ role_recipients_kavenegar[domainadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"
role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
+role_recipients_fleep[domainadmin]="${DEFAULT_RECIPIENT_FLEEP}"
+
role_recipients_irc[domainadmin]="${DEFAULT_RECIPIENT_IRC}"
+role_recipients_syslog[domainadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_awssns[domainadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
+
role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+role_recipients_msteam[domainadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
+
# -----------------------------------------------------------------------------
# database servers alarms
# mysql, redis, memcached, postgres, etc
@@ -602,10 +824,18 @@ role_recipients_kavenegar[dba]="${DEFAULT_RECIPIENT_KAVENEGAR}"
role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
+role_recipients_fleep[dba]="${DEFAULT_RECIPIENT_FLEEP}"
+
role_recipients_irc[dba]="${DEFAULT_RECIPIENT_IRC}"
+role_recipients_syslog[dba]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_awssns[dba]="${DEFAULT_RECIPIENT_AWSSNS}"
+
role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"
+role_recipients_msteam[dba]="${DEFAULT_RECIPIENT_MSTEAM}"
+
# -----------------------------------------------------------------------------
# web servers alarms
# apache, nginx, lighttpd, etc
@@ -636,10 +866,18 @@ role_recipients_kavenegar[webmaster]="${DEFAULT_RECIPIENT_KAVENEGAR}"
role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
+role_recipients_fleep[webmaster]="${DEFAULT_RECIPIENT_FLEEP}"
+
role_recipients_irc[webmaster]="${DEFAULT_RECIPIENT_IRC}"
+role_recipients_syslog[webmaster]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_awssns[webmaster]="${DEFAULT_RECIPIENT_AWSSNS}"
+
role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"
+role_recipients_msteam[webmaster]="${DEFAULT_RECIPIENT_MSTEAM}"
+
# -----------------------------------------------------------------------------
# proxy servers alarms
# squid, etc
@@ -670,10 +908,18 @@ role_recipients_kavenegar[proxyadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"
role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"
+role_recipients_fleep[proxyadmin]="${DEFAULT_RECIPIENT_FLEEP}"
+
role_recipients_irc[proxyadmin]="${DEFAULT_RECIPIENT_IRC}"
+role_recipients_syslog[proxyadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_awssns[porxyadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
+
role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+role_recipients_msteam[proxyadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
+
# -----------------------------------------------------------------------------
# peripheral devices
# UPS, photovoltaics, etc
@@ -704,5 +950,12 @@ role_recipients_kavenegar[sitemgr]="${DEFAULT_RECIPIENT_KAVENEGAR}"
role_recipients_pd[sitemgr]="${DEFAULT_RECIPIENT_PD}"
+role_recipients_fleep[sitemgr]="${DEFAULT_RECIPIENT_FLEEP}"
+
+role_recipients_syslog[sitemgr]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_awssns[sitemgr]="${DEFAULT_RECIPIENT_AWSSNS}"
+
role_recipients_custom[sitemgr]="${DEFAULT_RECIPIENT_CUSTOM}"
+role_recipients_msteam[sitemgr]="${DEFAULT_RECIPIENT_MSTEAM}"
diff --git a/conf.d/health_email_recipients.conf b/health/notifications/health_email_recipients.conf
index f56c6c64..f56c6c64 100644
--- a/conf.d/health_email_recipients.conf
+++ b/health/notifications/health_email_recipients.conf
diff --git a/health/notifications/irc/Makefile.inc b/health/notifications/irc/Makefile.inc
new file mode 100644
index 00000000..23be7210
--- /dev/null
+++ b/health/notifications/irc/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ irc/README.md \
+ irc/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/irc/README.md b/health/notifications/irc/README.md
new file mode 100644
index 00000000..ece879fe
--- /dev/null
+++ b/health/notifications/irc/README.md
@@ -0,0 +1,73 @@
+# IRC notifications
+
+This is what you will get:
+
+IRCCloud web client:
+![image](https://user-images.githubusercontent.com/31221999/36793487-3735673e-1ca6-11e8-8880-d1d8b6cd3bc0.png)
+
+Irssi terminal client:
+![image](https://user-images.githubusercontent.com/31221999/36793486-3713ada6-1ca6-11e8-8c12-70d956ad801e.png)
+
+
+You need:
+1. The `nc` utility. If you do not set the path, netdata will search for it in your system `$PATH`.
+
+Set the path for `nc` in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# external commands
+#
+# The full path of the nc command.
+# If empty, the system $PATH will be searched for it.
+# If not found, irc notifications will be silently disabled.
+nc="/usr/bin/nc"
+
+```
+
+2. Ξ‘n `IRC_NETWORK` to which your preffered channels belong to.
+3. One or more channels ( `DEFAULT_RECIPIENT_IRC` ) to post the messages to.
+4. An `IRC_NICKNAME` and an `IRC_REALNAME` to identify in IRC.
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# irc notification options
+#
+# irc notifications require only the nc utility to be installed.
+
+# multiple recipients can be given like this:
+# "<irc_channel_1> <irc_channel_2> ..."
+
+# enable/disable sending irc notifications
+SEND_IRC="YES"
+
+# if a role's recipients are not configured, a notification will not be sent.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_IRC="#system-alarms"
+
+# The irc network to which the recipients belong. It must be the full network.
+IRC_NETWORK="irc.freenode.net"
+
+# The irc nickname which is required to send the notification. It must not be
+# an already registered name as the connection's MODE is defined as a 'guest'.
+IRC_NICKNAME="netdata-alarm-user"
+
+# The irc realname which is required in order to make the connection and is an
+# extra identifier.
+IRC_REALNAME="netdata-user"
+
+```
+
+You can define multiple channels like this: `#system-alarms #networking-alarms`.
+You can also filter the notifications like this: `#system-alarms|critical`.
+You can give different channels per **role** using these (at the same file):
+
+```
+role_recipients_irc[sysadmin]="#user-alarms #networking-alarms #system-alarms"
+role_recipients_irc[dba]="#databases-alarms"
+role_recipients_irc[webmaster]="#networking-alarms"
+```
+
+The keywords `#user-alarms`, `#networking-alarms`, `#system-alarms`, `#databases-alarms` are irc channels which belong to the specified IRC network. \ No newline at end of file
diff --git a/health/notifications/kavenegar/Makefile.inc b/health/notifications/kavenegar/Makefile.inc
new file mode 100644
index 00000000..6a17c348
--- /dev/null
+++ b/health/notifications/kavenegar/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ kavenegar/README.md \
+ kavenegar/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/kavenegar/README.md b/health/notifications/kavenegar/README.md
new file mode 100644
index 00000000..e59ad4d4
--- /dev/null
+++ b/health/notifications/kavenegar/README.md
@@ -0,0 +1,39 @@
+# Kavenegar notifications
+
+[Kavenegar](https://www.kavenegar.com/) as service for software developers, based in Iran, provides send and receive SMS, calling voice by using its APIs.
+
+Will look like this on your Android device:
+
+![image](https://cloud.githubusercontent.com/assets/17090999/20034652/620b6100-a39b-11e6-96af-4f83b8e830e2.png)
+
+You will need:
+
+1. Signup and Login to kavenegar.com
+2. Get your APIKEY and Sender from http://panel.kavenegar.com/client/setting/account
+3. Fill in KAVENEGAR_API_KEY="" KAVENEGAR_SENDER=""
+4. Add the recipient phone numbers to DEFAULT_RECIPIENT_KAVENEGAR=""
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# Kavenegar (kavenegar.com) SMS options
+
+# multiple recipients can be given like this:
+# "09155555555 09177777777"
+
+# enable/disable sending kavenegar SMS
+SEND_KAVENEGAR="YES"
+
+# to get an access key, after selecting and purchasing your desired service
+# at http://kavenegar.com/pricing.html
+# login to your account, go to your dashboard and my account are
+# https://panel.kavenegar.com/Client/setting/account from API Key
+# copy your api key. You can generate new API Key too.
+# You can find and select kevenegar sender number from this place.
+
+# Without an API key, netdata cannot send KAVENEGAR text messages.
+KAVENEGAR_API_KEY=""
+KAVENEGAR_SENDER=""
+DEFAULT_RECIPIENT_KAVENEGAR=""
+``` \ No newline at end of file
diff --git a/health/notifications/messagebird/Makefile.inc b/health/notifications/messagebird/Makefile.inc
new file mode 100644
index 00000000..8132fecc
--- /dev/null
+++ b/health/notifications/messagebird/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ messagebird/README.md \
+ messagebird/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/messagebird/README.md b/health/notifications/messagebird/README.md
new file mode 100644
index 00000000..e09ba5d3
--- /dev/null
+++ b/health/notifications/messagebird/README.md
@@ -0,0 +1,38 @@
+
+Will look like this on your Android device:
+
+![image](https://cloud.githubusercontent.com/assets/17090999/20034652/620b6100-a39b-11e6-96af-4f83b8e830e2.png)
+
+You will need:
+
+1. Signup and Login to messagebird.com
+2. Pick an SMS capable number after sign up to get some free credits
+3. Go to <https://www.messagebird.com/app/settings/developers/access>
+4. Create a new access key under 'API ACCESS (REST)' (you will want a live key)
+3. Fill in MESSAGEBIRD_ACCESS_KEY="XXXXXXXX" MESSAGEBIRD_NUMBER="+XXXXXXXXXXX"
+4. Add the recipient phone numbers to DEFAULT_RECIPIENT_MESSAGEBIRD="+XXXXXXXXXXX"
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# Messagebird (messagebird.com) SMS options
+
+# multiple recipients can be given like this:
+# "+15555555555 +17777777777"
+
+# enable/disable sending messagebird SMS
+SEND_MESSAGEBIRD="YES"
+
+# to get an access key, create a free account at https://www.messagebird.com
+# verify and activate the account (no CC info needed)
+# login to your account and enter your phonenumber to get some free credits
+# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)'
+# click 'Add access key' and fill in data (you want a live key to send SMS)
+
+# Without an access key, netdata cannot send Messagebird text messages.
+MESSAGEBIRD_ACCESS_KEY="XXXXXXXX"
+MESSAGEBIRD_NUMBER="XXXXXXX"
+DEFAULT_RECIPIENT_MESSAGEBIRD="XXXXXXX"
+
+```
diff --git a/health/notifications/pagerduty/Makefile.inc b/health/notifications/pagerduty/Makefile.inc
new file mode 100644
index 00000000..6012d20e
--- /dev/null
+++ b/health/notifications/pagerduty/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ pagerduty/README.md \
+ pagerduty/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/pagerduty/README.md b/health/notifications/pagerduty/README.md
new file mode 100644
index 00000000..e35051fd
--- /dev/null
+++ b/health/notifications/pagerduty/README.md
@@ -0,0 +1,34 @@
+
+[PagerDuty](https://www.pagerduty.com/company/) is the enterprise incident resolution service that integrates with ITOps and DevOps monitoring stacks to improve operational reliability and agility. From enriching and aggregating events to correlating them into incidents, PagerDuty streamlines the incident management process by reducing alert noise and resolution times.
+
+Here is an example of a PagerDuty dashboard with netdata notifications:
+
+![PagerDuty dashboard with netdata notifications](https://cloud.githubusercontent.com/assets/19278582/21233877/b466a08a-c2a5-11e6-8d66-ee6eed43818f.png)
+
+To have netdata send notifications to PagerDuty, you'll first need to set up a PagerDuty `Generic API` service and install the PagerDuty agent on the host running netdata. See the following guide for details:
+
+https://www.pagerduty.com/docs/guides/agent-install-guide/
+
+During the setup of the `Generic API` PagerDuty service, you'll obtain a `pagerduty service key`. Keep this **service key** handy.
+
+Once the PagerDuty agent is installed on your host and can send notifications from your host to your `Generic API` service on PagerDuty, add the **service key** to `DEFAULT_RECIPIENT_PD` in `health_alarm_notify.conf`:
+
+```
+#------------------------------------------------------------------------------
+# pagerduty.com notification options
+#
+# pagerduty.com notifications require the pagerduty agent to be installed and
+# a "Generic API" pagerduty service.
+# https://www.pagerduty.com/docs/guides/agent-install-guide/
+
+# multiple recipients can be given like this:
+# "<pd_service_key_1> <pd_service_key_2> ..."
+
+# enable/disable sending pagerduty notifications
+SEND_PD="YES"
+
+# if a role's recipients are not configured, a notification will be sent to
+# the "General API" pagerduty.com service that uses this service key.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_PD="<service key>"
+```
diff --git a/health/notifications/pushbullet/Makefile.inc b/health/notifications/pushbullet/Makefile.inc
new file mode 100644
index 00000000..693a0ffb
--- /dev/null
+++ b/health/notifications/pushbullet/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ pushbullet/README.md \
+ pushbullet/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/pushbullet/README.md b/health/notifications/pushbullet/README.md
new file mode 100644
index 00000000..827d9301
--- /dev/null
+++ b/health/notifications/pushbullet/README.md
@@ -0,0 +1,42 @@
+$ PushBullet notifications
+
+Will look like this on your browser:
+![image](https://cloud.githubusercontent.com/assets/4300670/19109636/278b1c0c-8aee-11e6-8a09-7fc94fdbfec8.png)
+
+And like this on your Android device:
+
+
+![image](https://cloud.githubusercontent.com/assets/4300670/19109635/278a1dde-8aee-11e6-9984-0bc87a13312d.png)
+
+You will need:
+
+1. Signup and Login to pushbullet.com
+2. Get your Access Token, go to https://www.pushbullet.com/#settings/account and create a new one
+3. Fill in the PUSHBULLET_ACCESS_TOKEN with that value
+4. Add the recipient emails to DEFAULT_RECIPIENT_PUSHBULLET
+!!PLEASE NOTE THAT IF THE RECIPIENT DOES NOT HAVE A PUSHBULLET ACCOUNT, PUSHBULLET SERVICE WILL SEND AN EMAIL!!
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# pushbullet (pushbullet.com) push notification options
+
+# multiple recipients can be given like this:
+# "user1@email.com user2@mail.com"
+
+# enable/disable sending pushbullet notifications
+SEND_PUSHBULLET="YES"
+
+# Signup and Login to pushbullet.com
+# To get your Access Token, go to https://www.pushbullet.com/#settings/account
+# And create a new access token
+# Then just set the recipients emails
+# Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does
+# not have a pushbullet account, the pushbullet service will send an email
+# to that address instead
+
+# Without an access token, netdata cannot send pushbullet notifications.
+PUSHBULLET_ACCESS_TOKEN="o.Sometokenhere"
+DEFAULT_RECIPIENT_PUSHBULLET="admin1@example.com admin3@somemail.com"
+``` \ No newline at end of file
diff --git a/health/notifications/pushover/Makefile.inc b/health/notifications/pushover/Makefile.inc
new file mode 100644
index 00000000..926ac7c3
--- /dev/null
+++ b/health/notifications/pushover/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ pushover/README.md \
+ pushover/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/pushover/README.md b/health/notifications/pushover/README.md
new file mode 100644
index 00000000..28498169
--- /dev/null
+++ b/health/notifications/pushover/README.md
@@ -0,0 +1,17 @@
+
+# PushOver notifications
+
+pushover.net allows you to receive push notifications on your mobile phone. The service seems free for up to 7.500 messages per month.
+
+netdata will send warning messages with priority `0` and critical messages with priority `1`. pushover.net allows you to select do-not-disturb hours. The way this is configured, critical notifications will ring and vibrate your phone, even during the do-not-disturb-hours. All other notifications will be delivered silently.
+
+You need:
+
+1. APP TOKEN. You can use the same on all your netdata servers.
+2. USER TOKEN for each user you are going to send notifications to. This is the actual recipient of the notification.
+
+The configuration is like above (slack messages).
+
+pushover.net notifications look like this:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/18407319/839c10c4-7715-11e6-92c0-12f8215128d3.png) \ No newline at end of file
diff --git a/health/notifications/rocketchat/Makefile.inc b/health/notifications/rocketchat/Makefile.inc
new file mode 100644
index 00000000..a6fc5d57
--- /dev/null
+++ b/health/notifications/rocketchat/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ rocketchat/README.md \
+ rocketchat/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/rocketchat/README.md b/health/notifications/rocketchat/README.md
new file mode 100644
index 00000000..70c69867
--- /dev/null
+++ b/health/notifications/rocketchat/README.md
@@ -0,0 +1,46 @@
+# Rocket.Chat notifications
+
+This is what you will get:
+![Netdata on RocketChat](https://i.imgur.com/Zu4t3j3.png)
+You need:
+
+1. The **incoming webhook URL** as given by RocketChat. You can use the same on all your netdata servers (or you can have multiple if you like - your decision).
+2. One or more channels to post the messages to.
+
+Get them here: https://rocket.chat/docs/administrator-guides/integrations/index.html#how-to-create-a-new-incoming-webhook
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# rocketchat (rocket.chat) global notification options
+
+# multiple recipients can be given like this:
+# "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending rocketchat notifications
+SEND_ROCKETCHAT="YES"
+
+# Login to rocket.chat and create an incoming webhook. You need only one for all
+# your netdata servers (or you can have one for each of your netdata).
+# Without it, netdata cannot send rocketchat notifications.
+ROCKETCHAT_WEBHOOK_URL="<your_incoming_webhook_url>"
+
+# if a role's recipients are not configured, a notification will be send to
+# this rocketchat channel (empty = do not send a notification for unconfigured
+# roles).
+DEFAULT_RECIPIENT_ROCKETCHAT="monitoring_alarms"
+
+```
+
+You can define multiple channels like this: `alarms systems`.
+You can give different channels per **role** using these (at the same file):
+
+```
+role_recipients_rocketchat[sysadmin]="systems"
+role_recipients_rocketchat[dba]="databases systems"
+role_recipients_rocketchat[webmaster]="marketing development"
+```
+
+The keywords `systems`, `databases`, `marketing`, `development` are RocketChat channels (they should already exist).
+Both public and private channels can be used, even if they differ from the channel configured in yout RocketChat incomming webhook.
diff --git a/health/notifications/slack/Makefile.inc b/health/notifications/slack/Makefile.inc
new file mode 100644
index 00000000..955a8c7a
--- /dev/null
+++ b/health/notifications/slack/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ slack/README.md \
+ slack/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/slack/README.md b/health/notifications/slack/README.md
new file mode 100644
index 00000000..45be4519
--- /dev/null
+++ b/health/notifications/slack/README.md
@@ -0,0 +1,45 @@
+# Slack.com notifications
+
+This is what you will get:
+![image](https://cloud.githubusercontent.com/assets/2662304/18407116/bbd0fee6-7710-11e6-81cf-58c0defaee2b.png)
+
+You need:
+
+1. The **incoming webhook URL** as given by slack.com. You can use the same on all your netdata servers (or you can have multiple if you like - your decision).
+2. One or more channels to post the messages to.
+
+Get them here: https://api.slack.com/incoming-webhooks
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# sending slack notifications
+
+# note: multiple recipients can be given like this:
+# "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending pushover notifications
+SEND_SLACK="YES"
+
+# Login to slack.com and create an incoming webhook.
+# You need only one for all your netdata servers.
+# Without it, netdata cannot send slack notifications.
+SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXXXXXXX/XXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+# if a role recipient is not configured, a notification will be send to
+# this slack channel:
+DEFAULT_RECIPIENT_SLACK="alarms"
+
+```
+
+You can define multiple channels like this: `alarms systems`.
+You can give different channels per **role** using these (at the same file):
+
+```
+role_recipients_slack[sysadmin]="systems"
+role_recipients_slack[dba]="databases systems"
+role_recipients_slack[webmaster]="marketing development"
+```
+
+The keywords `systems`, `databases`, `marketing`, `development` are slack.com channels (they should already exist in slack).
diff --git a/health/notifications/syslog/Makefile.inc b/health/notifications/syslog/Makefile.inc
new file mode 100644
index 00000000..1792b9d9
--- /dev/null
+++ b/health/notifications/syslog/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ syslog/README.md \
+ syslog/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/syslog/README.md b/health/notifications/syslog/README.md
new file mode 100644
index 00000000..fcc2466a
--- /dev/null
+++ b/health/notifications/syslog/README.md
@@ -0,0 +1,23 @@
+# syslog notifications
+
+You need a working `logger` command for this to work. This is the case on pretty much every Linux system in existence, and most BSD systems.
+
+Logged messages will look like this:
+
+ netdata WARNING on hostname at Tue Apr 3 09:00:00 EDT 2018: disk_space._ out of disk space time = 5h
+
+## configuration
+
+System log targets are configured as recipients in [`/etc/netdata/health_alarm_notify.conf`](https://github.com/netdata/netdata/blob/36bedc044584dea791fd29455bdcd287c3306cb2/conf.d/health_alarm_notify.conf#L534) (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`).
+
+You can als configure per-role targets in the same file a bit further down.
+
+Targets are defined as follows:
+
+ [[facility.level][@host[:port]]/]prefix
+
+`prefix` defines what the log messages are prefixed with. By default, all lines are prefixed with 'netdata'.
+
+The `facility` and `level` are the standard syslog facility and level options, for more info on them see your local `logger` and `syslog` documentation. By default, netdata will log to the `local6` facility, with a log level dependent on the type of message (`crit` for CRITICAL, `warning` for WARNING, and `info` for everything else).
+
+You can configure sending directly to remote log servers by specifying a host (and optionally a port). However, this has a somewhat high overhead, so it is much preferred to use your local syslog daemon to handle the forwarding of messages to remote systems (pretty much all of them allow at least simple forwarding, and most of the really popular ones support complex queueing and routing of messages to remote log servers).
diff --git a/health/notifications/telegram/Makefile.inc b/health/notifications/telegram/Makefile.inc
new file mode 100644
index 00000000..003996ba
--- /dev/null
+++ b/health/notifications/telegram/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ telegram/README.md \
+ telegram/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/telegram/README.md b/health/notifications/telegram/README.md
new file mode 100644
index 00000000..cd52fe19
--- /dev/null
+++ b/health/notifications/telegram/README.md
@@ -0,0 +1,19 @@
+# Telegram.org notifications
+
+[Telegram](https://telegram.org/) is a messaging app with a focus on speed and security, it’s super-fast, simple and free. You can use Telegram on all your devices at the same time β€” your messages sync seamlessly across any number of your phones, tablets or computers.
+
+With Telegram, you can send messages, photos, videos and files of any type (doc, zip, mp3, etc), as well as create groups for up to 30,000 people or channels for broadcasting to unlimited audiences. You can write to your phone contacts and find people by their usernames. As a result, Telegram is like SMS and email combined β€” and can take care of all your personal or business messaging needs.
+
+netdata will send warning messages without vibration.
+
+You need:
+
+1. A bot token. To get one, contact the [@BotFather](https://t.me/BotFather) bot and send the command `/newbot`. Follow the instructions.
+2. A chat id for every chat you want to send messages to. Contact the [@myidbot](https://t.me/myidbot) bot and send the command `/getid` to get your personal chat id or invite him into a group and issue the same command to get the group chat id.
+3. Start a conversation with your bot or invite him into a group you want to sent messages to.
+
+See slack for configuration.
+
+Telegram messages look like this:
+
+![image](https://fb.hash.works/ytl/preview.jpg)
diff --git a/health/notifications/twilio/Makefile.inc b/health/notifications/twilio/Makefile.inc
new file mode 100644
index 00000000..5bd00a21
--- /dev/null
+++ b/health/notifications/twilio/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ twilio/README.md \
+ twilio/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/twilio/README.md b/health/notifications/twilio/README.md
new file mode 100644
index 00000000..ab3dd3c0
--- /dev/null
+++ b/health/notifications/twilio/README.md
@@ -0,0 +1,40 @@
+# Twilio.com notifications
+
+Will look like this on your Android device:
+
+![image](https://cloud.githubusercontent.com/assets/17090999/20034652/620b6100-a39b-11e6-96af-4f83b8e830e2.png)
+
+You will need:
+
+1. Signup and Login to twilio.com
+2. Pick an SMS capable number during sign up.
+3. Get your SID, and Token from <https://www.twilio.com/console>
+3. Fill in TWILIO_ACCOUNT_SID="XXXXXXXX" TWILIO_ACCOUNT_TOKEN="XXXXXXXXX" TWILIO_NUMBER="+XXXXXXXXXXX"
+4. Add the recipient phone numbers to DEFAULT_RECIPIENT_TWILIO="+XXXXXXXXXXX"
+
+!!PLEASE NOTE THAT IF YOUR ACCOUNT IS A TRIAL ACCOUNT YOU WILL ONLY BE ABLE TO SEND NOTIFICATIONS TO THE NUMBER YOU SIGNED UP WITH
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# Twilio (twilio.com) SMS options
+
+# multiple recipients can be given like this:
+# "+15555555555 +17777777777"
+
+# enable/disable sending twilio SMS
+SEND_TWILIO="YES"
+
+# Signup for free trial and select a SMS capable Twilio Number
+# To get your Account SID and Token, go to https://www.twilio.com/console
+# Place your sid, token and number below.
+# Then just set the recipients' phone numbers.
+# The trial account is only allowed to use the number specified when set up.
+
+# Without an account sid and token, netdata cannot send Twilio text messages.
+TWILIO_ACCOUNT_SID="xxxxxxxxx"
+TWILIO_ACCOUNT_TOKEN="xxxxxxxxxx"
+TWILIO_NUMBER="xxxxxxxxxxx"
+DEFAULT_RECIPIENT_TWILIO="+15555555555"
+```
diff --git a/health/notifications/web/Makefile.inc b/health/notifications/web/Makefile.inc
new file mode 100644
index 00000000..89082431
--- /dev/null
+++ b/health/notifications/web/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+ web/README.md \
+ web/Makefile.inc \
+ $(NULL)
+
diff --git a/health/notifications/web/README.md b/health/notifications/web/README.md
new file mode 100644
index 00000000..ba7dac1f
--- /dev/null
+++ b/health/notifications/web/README.md
@@ -0,0 +1,6 @@
+# Dashboard notifications
+
+The netdata dashboard shows HTML notifications, when it is open.
+
+Such web notifications look like this:
+![image](https://cloud.githubusercontent.com/assets/2662304/18407279/82bac6a6-7714-11e6-847e-c2e84eeacbfb.png)