diff options
Diffstat (limited to '')
36 files changed, 1106 insertions, 4837 deletions
diff --git a/health/.keep b/health/.keep new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/health/.keep diff --git a/health/Makefile.am b/health/Makefile.am index 829a41b3b..40592a964 100644 --- a/health/Makefile.am +++ b/health/Makefile.am @@ -16,12 +16,11 @@ dist_noinst_DATA = \ userhealthconfigdir=$(configdir)/health.d dist_userhealthconfig_DATA = \ - $(top_srcdir)/installer/.keep \ + .keep \ $(NULL) healthconfigdir=$(libconfigdir)/health.d dist_healthconfig_DATA = \ - $(top_srcdir)/installer/.keep \ health.d/adaptec_raid.conf \ health.d/apache.conf \ health.d/apcupsd.conf \ diff --git a/health/Makefile.in b/health/Makefile.in deleted file mode 100644 index 811f7f239..000000000 --- a/health/Makefile.in +++ /dev/null @@ -1,800 +0,0 @@ -# Makefile.in generated by automake 1.14.1 from Makefile.am. -# @configure_input@ - -# Copyright (C) 1994-2013 Free Software Foundation, Inc. - -# This Makefile.in is free software; the Free Software Foundation -# gives unlimited permission to copy and/or distribute it, -# with or without modifications, as long as this notice is preserved. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY, to the extent permitted by law; without -# even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. - -@SET_MAKE@ - -# SPDX-License-Identifier: GPL-3.0-or-later - -VPATH = @srcdir@ -am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)' -am__make_running_with_option = \ - case $${target_option-} in \ - ?) ;; \ - *) echo "am__make_running_with_option: internal error: invalid" \ - "target option '$${target_option-}' specified" >&2; \ - exit 1;; \ - esac; \ - has_opt=no; \ - sane_makeflags=$$MAKEFLAGS; \ - if $(am__is_gnu_make); then \ - sane_makeflags=$$MFLAGS; \ - else \ - case $$MAKEFLAGS in \ - *\\[\ \ ]*) \ - bs=\\; \ - sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ - | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ - esac; \ - fi; \ - skip_next=no; \ - strip_trailopt () \ - { \ - flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ - }; \ - for flg in $$sane_makeflags; do \ - test $$skip_next = yes && { skip_next=no; continue; }; \ - case $$flg in \ - *=*|--*) continue;; \ - -*I) strip_trailopt 'I'; skip_next=yes;; \ - -*I?*) strip_trailopt 'I';; \ - -*O) strip_trailopt 'O'; skip_next=yes;; \ - -*O?*) strip_trailopt 'O';; \ - -*l) strip_trailopt 'l'; skip_next=yes;; \ - -*l?*) strip_trailopt 'l';; \ - -[dEDm]) skip_next=yes;; \ - -[JT]) skip_next=yes;; \ - esac; \ - case $$flg in \ - *$$target_option*) has_opt=yes; break;; \ - esac; \ - done; \ - test $$has_opt = yes -am__make_dryrun = (target_option=n; $(am__make_running_with_option)) -am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) -pkgdatadir = $(datadir)/@PACKAGE@ -pkgincludedir = $(includedir)/@PACKAGE@ -pkglibdir = $(libdir)/@PACKAGE@ -pkglibexecdir = $(libexecdir)/@PACKAGE@ -am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd -install_sh_DATA = $(install_sh) -c -m 644 -install_sh_PROGRAM = $(install_sh) -c -install_sh_SCRIPT = $(install_sh) -c -INSTALL_HEADER = $(INSTALL_DATA) -transform = $(program_transform_name) -NORMAL_INSTALL = : -PRE_INSTALL = : -POST_INSTALL = : -NORMAL_UNINSTALL = : -PRE_UNINSTALL = : -POST_UNINSTALL = : -build_triplet = @build@ -host_triplet = @host@ -subdir = health -DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ - $(dist_healthconfig_DATA) $(dist_noinst_DATA) \ - $(dist_userhealthconfig_DATA) -ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/build/m4/ax_c___atomic.m4 \ - $(top_srcdir)/build/m4/ax_c__generic.m4 \ - $(top_srcdir)/build/m4/ax_c_lto.m4 \ - $(top_srcdir)/build/m4/ax_c_mallinfo.m4 \ - $(top_srcdir)/build/m4/ax_c_mallopt.m4 \ - $(top_srcdir)/build/m4/ax_check_compile_flag.m4 \ - $(top_srcdir)/build/m4/ax_gcc_func_attribute.m4 \ - $(top_srcdir)/build/m4/ax_pthread.m4 \ - $(top_srcdir)/build/m4/jemalloc.m4 \ - $(top_srcdir)/build/m4/tcmalloc.m4 $(top_srcdir)/configure.ac -am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ - $(ACLOCAL_M4) -mkinstalldirs = $(install_sh) -d -CONFIG_HEADER = $(top_builddir)/config.h -CONFIG_CLEAN_FILES = -CONFIG_CLEAN_VPATH_FILES = -AM_V_P = $(am__v_P_@AM_V@) -am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) -am__v_P_0 = false -am__v_P_1 = : -AM_V_GEN = $(am__v_GEN_@AM_V@) -am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) -am__v_GEN_0 = @echo " GEN " $@; -am__v_GEN_1 = -AM_V_at = $(am__v_at_@AM_V@) -am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) -am__v_at_0 = @ -am__v_at_1 = -SOURCES = -DIST_SOURCES = -RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \ - ctags-recursive dvi-recursive html-recursive info-recursive \ - install-data-recursive install-dvi-recursive \ - install-exec-recursive install-html-recursive \ - install-info-recursive install-pdf-recursive \ - install-ps-recursive install-recursive installcheck-recursive \ - installdirs-recursive pdf-recursive ps-recursive \ - tags-recursive uninstall-recursive -am__can_run_installinfo = \ - case $$AM_UPDATE_INFO_DIR in \ - n|no|NO) false;; \ - *) (install-info --version) >/dev/null 2>&1;; \ - esac -am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; -am__vpath_adj = case $$p in \ - $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ - *) f=$$p;; \ - esac; -am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; -am__install_max = 40 -am__nobase_strip_setup = \ - srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` -am__nobase_strip = \ - for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" -am__nobase_list = $(am__nobase_strip_setup); \ - for p in $$list; do echo "$$p $$p"; done | \ - sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ - $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ - if (++n[$$2] == $(am__install_max)) \ - { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ - END { for (dir in files) print dir, files[dir] }' -am__base_list = \ - sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ - sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' -am__uninstall_files_from_dir = { \ - test -z "$$files" \ - || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ - || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ - $(am__cd) "$$dir" && rm -f $$files; }; \ - } -am__installdirs = "$(DESTDIR)$(healthconfigdir)" \ - "$(DESTDIR)$(userhealthconfigdir)" -DATA = $(dist_healthconfig_DATA) $(dist_noinst_DATA) \ - $(dist_userhealthconfig_DATA) -RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ - distclean-recursive maintainer-clean-recursive -am__recursive_targets = \ - $(RECURSIVE_TARGETS) \ - $(RECURSIVE_CLEAN_TARGETS) \ - $(am__extra_recursive_targets) -AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \ - distdir -am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) -# Read a list of newline-separated strings from the standard input, -# and print each of them once, without duplicates. Input order is -# *not* preserved. -am__uniquify_input = $(AWK) '\ - BEGIN { nonempty = 0; } \ - { items[$$0] = 1; nonempty = 1; } \ - END { if (nonempty) { for (i in items) print i; }; } \ -' -# Make sure the list of sources is unique. This is necessary because, -# e.g., the same source file might be shared among _SOURCES variables -# for different programs/libraries. -am__define_uniq_tagged_files = \ - list='$(am__tagged_files)'; \ - unique=`for i in $$list; do \ - if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ - done | $(am__uniquify_input)` -ETAGS = etags -CTAGS = ctags -DIST_SUBDIRS = $(SUBDIRS) -DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) -am__relativize = \ - dir0=`pwd`; \ - sed_first='s,^\([^/]*\)/.*$$,\1,'; \ - sed_rest='s,^[^/]*/*,,'; \ - sed_last='s,^.*/\([^/]*\)$$,\1,'; \ - sed_butlast='s,/*[^/]*$$,,'; \ - while test -n "$$dir1"; do \ - first=`echo "$$dir1" | sed -e "$$sed_first"`; \ - if test "$$first" != "."; then \ - if test "$$first" = ".."; then \ - dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \ - dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \ - else \ - first2=`echo "$$dir2" | sed -e "$$sed_first"`; \ - if test "$$first2" = "$$first"; then \ - dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \ - else \ - dir2="../$$dir2"; \ - fi; \ - dir0="$$dir0"/"$$first"; \ - fi; \ - fi; \ - dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \ - done; \ - reldir="$$dir2" -ACLOCAL = @ACLOCAL@ -AMTAR = @AMTAR@ -AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ -AUTOCONF = @AUTOCONF@ -AUTOHEADER = @AUTOHEADER@ -AUTOMAKE = @AUTOMAKE@ -AWK = @AWK@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ -CPP = @CPP@ -CPPFLAGS = @CPPFLAGS@ -CYGPATH_W = @CYGPATH_W@ -DEFS = @DEFS@ -DEPDIR = @DEPDIR@ -ECHO_C = @ECHO_C@ -ECHO_N = @ECHO_N@ -ECHO_T = @ECHO_T@ -EGREP = @EGREP@ -EXEEXT = @EXEEXT@ -GREP = @GREP@ -INSTALL = @INSTALL@ -INSTALL_DATA = @INSTALL_DATA@ -INSTALL_PROGRAM = @INSTALL_PROGRAM@ -INSTALL_SCRIPT = @INSTALL_SCRIPT@ -INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ -IPMIMONITORING_CFLAGS = @IPMIMONITORING_CFLAGS@ -IPMIMONITORING_LIBS = @IPMIMONITORING_LIBS@ -LDFLAGS = @LDFLAGS@ -LIBCAP_CFLAGS = @LIBCAP_CFLAGS@ -LIBCAP_LIBS = @LIBCAP_LIBS@ -LIBMNL_CFLAGS = @LIBMNL_CFLAGS@ -LIBMNL_LIBS = @LIBMNL_LIBS@ -LIBOBJS = @LIBOBJS@ -LIBS = @LIBS@ -LTLIBOBJS = @LTLIBOBJS@ -MAINT = @MAINT@ -MAKEINFO = @MAKEINFO@ -MATH_CFLAGS = @MATH_CFLAGS@ -MATH_LIBS = @MATH_LIBS@ -MKDIR_P = @MKDIR_P@ -NFACCT_CFLAGS = @NFACCT_CFLAGS@ -NFACCT_LIBS = @NFACCT_LIBS@ -OBJEXT = @OBJEXT@ -OPTIONAL_IPMIMONITORING_CFLAGS = @OPTIONAL_IPMIMONITORING_CFLAGS@ -OPTIONAL_IPMIMONITORING_LIBS = @OPTIONAL_IPMIMONITORING_LIBS@ -OPTIONAL_LIBCAP_CFLAGS = @OPTIONAL_LIBCAP_CFLAGS@ -OPTIONAL_LIBCAP_LIBS = @OPTIONAL_LIBCAP_LIBS@ -OPTIONAL_MATH_CLFAGS = @OPTIONAL_MATH_CLFAGS@ -OPTIONAL_MATH_LIBS = @OPTIONAL_MATH_LIBS@ -OPTIONAL_NFACCT_CLFAGS = @OPTIONAL_NFACCT_CLFAGS@ -OPTIONAL_NFACCT_LIBS = @OPTIONAL_NFACCT_LIBS@ -OPTIONAL_UUID_CLFAGS = @OPTIONAL_UUID_CLFAGS@ -OPTIONAL_UUID_LIBS = @OPTIONAL_UUID_LIBS@ -OPTIONAL_ZLIB_CLFAGS = @OPTIONAL_ZLIB_CLFAGS@ -OPTIONAL_ZLIB_LIBS = @OPTIONAL_ZLIB_LIBS@ -PACKAGE = @PACKAGE@ -PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_RPM_RELEASE = @PACKAGE_RPM_RELEASE@ -PACKAGE_RPM_VERSION = @PACKAGE_RPM_VERSION@ -PACKAGE_STRING = @PACKAGE_STRING@ -PACKAGE_TARNAME = @PACKAGE_TARNAME@ -PACKAGE_URL = @PACKAGE_URL@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -PATH_SEPARATOR = @PATH_SEPARATOR@ -PKG_CONFIG = @PKG_CONFIG@ -PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ -PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ -PTHREAD_CC = @PTHREAD_CC@ -PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ -PTHREAD_LIBS = @PTHREAD_LIBS@ -SET_MAKE = @SET_MAKE@ -SHELL = @SHELL@ -SSE_CANDIDATE = @SSE_CANDIDATE@ -STRIP = @STRIP@ -UUID_CFLAGS = @UUID_CFLAGS@ -UUID_LIBS = @UUID_LIBS@ -VERSION = @VERSION@ -ZLIB_CFLAGS = @ZLIB_CFLAGS@ -ZLIB_LIBS = @ZLIB_LIBS@ -abs_builddir = @abs_builddir@ -abs_srcdir = @abs_srcdir@ -abs_top_builddir = @abs_top_builddir@ -abs_top_srcdir = @abs_top_srcdir@ -ac_ct_CC = @ac_ct_CC@ -am__include = @am__include@ -am__leading_dot = @am__leading_dot@ -am__quote = @am__quote@ -am__tar = @am__tar@ -am__untar = @am__untar@ -ax_pthread_config = @ax_pthread_config@ -bindir = @bindir@ -build = @build@ -build_alias = @build_alias@ -build_cpu = @build_cpu@ -build_os = @build_os@ -build_target = @build_target@ -build_vendor = @build_vendor@ -builddir = @builddir@ -cachedir = @cachedir@ -chartsdir = @chartsdir@ -configdir = @configdir@ -datadir = @datadir@ -datarootdir = @datarootdir@ -docdir = @docdir@ -dvidir = @dvidir@ -exec_prefix = @exec_prefix@ -has_jemalloc = @has_jemalloc@ -has_tcmalloc = @has_tcmalloc@ -host = @host@ -host_alias = @host_alias@ -host_cpu = @host_cpu@ -host_os = @host_os@ -host_vendor = @host_vendor@ -htmldir = @htmldir@ -includedir = @includedir@ -infodir = @infodir@ -install_sh = @install_sh@ -libconfigdir = @libconfigdir@ -libdir = @libdir@ -libexecdir = @libexecdir@ -localedir = @localedir@ -localstatedir = @localstatedir@ -logdir = @logdir@ -mandir = @mandir@ -mkdir_p = @mkdir_p@ -nodedir = @nodedir@ -oldincludedir = @oldincludedir@ -pdfdir = @pdfdir@ -pluginsdir = @pluginsdir@ -prefix = @prefix@ -program_transform_name = @program_transform_name@ -psdir = @psdir@ -pythondir = @pythondir@ -registrydir = @registrydir@ -sbindir = @sbindir@ -sharedstatedir = @sharedstatedir@ -srcdir = @srcdir@ -sysconfdir = @sysconfdir@ -target_alias = @target_alias@ -top_build_prefix = @top_build_prefix@ -top_builddir = @top_builddir@ -top_srcdir = @top_srcdir@ -varlibdir = @varlibdir@ -webdir = @webdir@ -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in -SUBDIRS = \ - notifications \ - $(NULL) - -CLEANFILES = \ - $(NULL) - -dist_noinst_DATA = \ - README.md \ - $(NULL) - -userhealthconfigdir = $(configdir)/health.d -dist_userhealthconfig_DATA = \ - $(top_srcdir)/installer/.keep \ - $(NULL) - -healthconfigdir = $(libconfigdir)/health.d -dist_healthconfig_DATA = \ - $(top_srcdir)/installer/.keep \ - health.d/adaptec_raid.conf \ - health.d/apache.conf \ - health.d/apcupsd.conf \ - health.d/backend.conf \ - health.d/bcache.conf \ - health.d/beanstalkd.conf \ - health.d/bind_rndc.conf \ - health.d/boinc.conf \ - health.d/btrfs.conf \ - health.d/ceph.conf \ - health.d/cpu.conf \ - health.d/couchdb.conf \ - health.d/disks.conf \ - health.d/dockerd.conf \ - health.d/elasticsearch.conf \ - health.d/entropy.conf \ - health.d/fping.conf \ - health.d/fronius.conf \ - health.d/haproxy.conf \ - health.d/httpcheck.conf \ - health.d/ipc.conf \ - health.d/ipfs.conf \ - health.d/ipmi.conf \ - health.d/isc_dhcpd.conf \ - health.d/lighttpd.conf \ - health.d/linux_power_supply.conf \ - health.d/load.conf \ - health.d/mdstat.conf \ - health.d/megacli.conf \ - health.d/memcached.conf \ - health.d/memory.conf \ - health.d/mongodb.conf \ - health.d/mysql.conf \ - health.d/named.conf \ - health.d/net.conf \ - health.d/netfilter.conf \ - health.d/nginx.conf \ - health.d/nginx_plus.conf \ - health.d/portcheck.conf \ - health.d/postgres.conf \ - health.d/qos.conf \ - health.d/ram.conf \ - health.d/redis.conf \ - health.d/retroshare.conf \ - health.d/softnet.conf \ - health.d/squid.conf \ - health.d/stiebeleltron.conf \ - health.d/swap.conf \ - health.d/tcp_conn.conf \ - health.d/tcp_listen.conf \ - health.d/tcp_mem.conf \ - health.d/tcp_orphans.conf \ - health.d/tcp_resets.conf \ - health.d/udp_errors.conf \ - health.d/varnish.conf \ - health.d/web_log.conf \ - health.d/zfs.conf \ - $(NULL) - -all: all-recursive - -.SUFFIXES: -$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) - @for dep in $?; do \ - case '$(am__configure_deps)' in \ - *$$dep*) \ - ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ - && { if test -f $@; then exit 0; else break; fi; }; \ - exit 1;; \ - esac; \ - done; \ - echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu health/Makefile'; \ - $(am__cd) $(top_srcdir) && \ - $(AUTOMAKE) --gnu health/Makefile -.PRECIOUS: Makefile -Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status - @case '$?' in \ - *config.status*) \ - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ - *) \ - echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ - cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ - esac; - -$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh - -$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(am__aclocal_m4_deps): -install-dist_healthconfigDATA: $(dist_healthconfig_DATA) - @$(NORMAL_INSTALL) - @list='$(dist_healthconfig_DATA)'; test -n "$(healthconfigdir)" || list=; \ - if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(healthconfigdir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(healthconfigdir)" || exit 1; \ - fi; \ - for p in $$list; do \ - if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ - echo "$$d$$p"; \ - done | $(am__base_list) | \ - while read files; do \ - echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(healthconfigdir)'"; \ - $(INSTALL_DATA) $$files "$(DESTDIR)$(healthconfigdir)" || exit $$?; \ - done - -uninstall-dist_healthconfigDATA: - @$(NORMAL_UNINSTALL) - @list='$(dist_healthconfig_DATA)'; test -n "$(healthconfigdir)" || list=; \ - files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ - dir='$(DESTDIR)$(healthconfigdir)'; $(am__uninstall_files_from_dir) -install-dist_userhealthconfigDATA: $(dist_userhealthconfig_DATA) - @$(NORMAL_INSTALL) - @list='$(dist_userhealthconfig_DATA)'; test -n "$(userhealthconfigdir)" || list=; \ - if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(userhealthconfigdir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(userhealthconfigdir)" || exit 1; \ - fi; \ - for p in $$list; do \ - if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ - echo "$$d$$p"; \ - done | $(am__base_list) | \ - while read files; do \ - echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(userhealthconfigdir)'"; \ - $(INSTALL_DATA) $$files "$(DESTDIR)$(userhealthconfigdir)" || exit $$?; \ - done - -uninstall-dist_userhealthconfigDATA: - @$(NORMAL_UNINSTALL) - @list='$(dist_userhealthconfig_DATA)'; test -n "$(userhealthconfigdir)" || list=; \ - files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ - dir='$(DESTDIR)$(userhealthconfigdir)'; $(am__uninstall_files_from_dir) - -# This directory's subdirectories are mostly independent; you can cd -# into them and run 'make' without going through this Makefile. -# To change the values of 'make' variables: instead of editing Makefiles, -# (1) if the variable is set in 'config.status', edit 'config.status' -# (which will cause the Makefiles to be regenerated when you run 'make'); -# (2) otherwise, pass the desired values on the 'make' command line. -$(am__recursive_targets): - @fail=; \ - if $(am__make_keepgoing); then \ - failcom='fail=yes'; \ - else \ - failcom='exit 1'; \ - fi; \ - dot_seen=no; \ - target=`echo $@ | sed s/-recursive//`; \ - case "$@" in \ - distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \ - *) list='$(SUBDIRS)' ;; \ - esac; \ - for subdir in $$list; do \ - echo "Making $$target in $$subdir"; \ - if test "$$subdir" = "."; then \ - dot_seen=yes; \ - local_target="$$target-am"; \ - else \ - local_target="$$target"; \ - fi; \ - ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ - || eval $$failcom; \ - done; \ - if test "$$dot_seen" = "no"; then \ - $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \ - fi; test -z "$$fail" - -ID: $(am__tagged_files) - $(am__define_uniq_tagged_files); mkid -fID $$unique -tags: tags-recursive -TAGS: tags - -tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) - set x; \ - here=`pwd`; \ - if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \ - include_option=--etags-include; \ - empty_fix=.; \ - else \ - include_option=--include; \ - empty_fix=; \ - fi; \ - list='$(SUBDIRS)'; for subdir in $$list; do \ - if test "$$subdir" = .; then :; else \ - test ! -f $$subdir/TAGS || \ - set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \ - fi; \ - done; \ - $(am__define_uniq_tagged_files); \ - shift; \ - if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ - test -n "$$unique" || unique=$$empty_fix; \ - if test $$# -gt 0; then \ - $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ - "$$@" $$unique; \ - else \ - $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ - $$unique; \ - fi; \ - fi -ctags: ctags-recursive - -CTAGS: ctags -ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) - $(am__define_uniq_tagged_files); \ - test -z "$(CTAGS_ARGS)$$unique" \ - || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ - $$unique - -GTAGS: - here=`$(am__cd) $(top_builddir) && pwd` \ - && $(am__cd) $(top_srcdir) \ - && gtags -i $(GTAGS_ARGS) "$$here" -cscopelist: cscopelist-recursive - -cscopelist-am: $(am__tagged_files) - list='$(am__tagged_files)'; \ - case "$(srcdir)" in \ - [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ - *) sdir=$(subdir)/$(srcdir) ;; \ - esac; \ - for i in $$list; do \ - if test -f "$$i"; then \ - echo "$(subdir)/$$i"; \ - else \ - echo "$$sdir/$$i"; \ - fi; \ - done >> $(top_builddir)/cscope.files - -distclean-tags: - -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags - -distdir: $(DISTFILES) - @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - list='$(DISTFILES)'; \ - dist_files=`for file in $$list; do echo $$file; done | \ - sed -e "s|^$$srcdirstrip/||;t" \ - -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ - case $$dist_files in \ - */*) $(MKDIR_P) `echo "$$dist_files" | \ - sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ - sort -u` ;; \ - esac; \ - for file in $$dist_files; do \ - if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ - if test -d $$d/$$file; then \ - dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ - if test -d "$(distdir)/$$file"; then \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ - cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ - else \ - test -f "$(distdir)/$$file" \ - || cp -p $$d/$$file "$(distdir)/$$file" \ - || exit 1; \ - fi; \ - done - @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ - if test "$$subdir" = .; then :; else \ - $(am__make_dryrun) \ - || test -d "$(distdir)/$$subdir" \ - || $(MKDIR_P) "$(distdir)/$$subdir" \ - || exit 1; \ - dir1=$$subdir; dir2="$(distdir)/$$subdir"; \ - $(am__relativize); \ - new_distdir=$$reldir; \ - dir1=$$subdir; dir2="$(top_distdir)"; \ - $(am__relativize); \ - new_top_distdir=$$reldir; \ - echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \ - echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \ - ($(am__cd) $$subdir && \ - $(MAKE) $(AM_MAKEFLAGS) \ - top_distdir="$$new_top_distdir" \ - distdir="$$new_distdir" \ - am__remove_distdir=: \ - am__skip_length_check=: \ - am__skip_mode_fix=: \ - distdir) \ - || exit 1; \ - fi; \ - done -check-am: all-am -check: check-recursive -all-am: Makefile $(DATA) -installdirs: installdirs-recursive -installdirs-am: - for dir in "$(DESTDIR)$(healthconfigdir)" "$(DESTDIR)$(userhealthconfigdir)"; do \ - test -z "$$dir" || $(MKDIR_P) "$$dir"; \ - done -install: install-recursive -install-exec: install-exec-recursive -install-data: install-data-recursive -uninstall: uninstall-recursive - -install-am: all-am - @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am - -installcheck: installcheck-recursive -install-strip: - if test -z '$(STRIP)'; then \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - install; \ - else \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ - fi -mostlyclean-generic: - -clean-generic: - -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) - -distclean-generic: - -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) - -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) - -maintainer-clean-generic: - @echo "This command is intended for maintainers to use" - @echo "it deletes files that may require special tools to rebuild." - -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES) -clean: clean-recursive - -clean-am: clean-generic mostlyclean-am - -distclean: distclean-recursive - -rm -f Makefile -distclean-am: clean-am distclean-generic distclean-tags - -dvi: dvi-recursive - -dvi-am: - -html: html-recursive - -html-am: - -info: info-recursive - -info-am: - -install-data-am: install-dist_healthconfigDATA \ - install-dist_userhealthconfigDATA - -install-dvi: install-dvi-recursive - -install-dvi-am: - -install-exec-am: - -install-html: install-html-recursive - -install-html-am: - -install-info: install-info-recursive - -install-info-am: - -install-man: - -install-pdf: install-pdf-recursive - -install-pdf-am: - -install-ps: install-ps-recursive - -install-ps-am: - -installcheck-am: - -maintainer-clean: maintainer-clean-recursive - -rm -f Makefile -maintainer-clean-am: distclean-am maintainer-clean-generic - -mostlyclean: mostlyclean-recursive - -mostlyclean-am: mostlyclean-generic - -pdf: pdf-recursive - -pdf-am: - -ps: ps-recursive - -ps-am: - -uninstall-am: uninstall-dist_healthconfigDATA \ - uninstall-dist_userhealthconfigDATA - -.MAKE: $(am__recursive_targets) install-am install-strip - -.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am check \ - check-am clean clean-generic cscopelist-am ctags ctags-am \ - distclean distclean-generic distclean-tags distdir dvi dvi-am \ - html html-am info info-am install install-am install-data \ - install-data-am install-dist_healthconfigDATA \ - install-dist_userhealthconfigDATA install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-pdf \ - install-pdf-am install-ps install-ps-am install-strip \ - installcheck installcheck-am installdirs installdirs-am \ - maintainer-clean maintainer-clean-generic mostlyclean \ - mostlyclean-generic pdf pdf-am ps ps-am tags tags-am uninstall \ - uninstall-am uninstall-dist_healthconfigDATA \ - uninstall-dist_userhealthconfigDATA - - -# Tell versions [3.59,3.63) of GNU make to not export all variables. -# Otherwise a system limit (for SysV at least) may be exceeded. -.NOEXPORT: diff --git a/health/README.md b/health/README.md index 5d68d752a..54f6a3e1f 100644 --- a/health/README.md +++ b/health/README.md @@ -9,8 +9,8 @@ netdata, since many charts are dynamically created during runtime (for example, chart tracking network interface packet drops, is automatically created on the first packet dropped). -Netdata also supports alarm **templates**, so that an alarm can be attached to all -the charts of the same context (i.e. all network interfaces, or all disks, or all mysql servers, etc.) +Netdata also supports alarm **templates**, so that an alarm can be attached to all the charts of the same context (i.e. all network interfaces, or all disks, or all mysql servers, etc.). + Each alarm can execute a single query to the database using statistical algorithms against past data, but alarms can be combined. So, if you need 2 queries in the database, you can combine @@ -145,7 +145,7 @@ This is useful when you centralize metrics from multiple hosts, to one netdata. This line is only used in alarm templates. It filters the charts. So, if you need to create an alarm template for a few of a kind of chart (a few of your disks, or a few of your network interfaces, or a few your mysql servers, etc), you can create an alarm template that would -normally be applied to all of them, and filter them by family. +normally be applied to all of them, and filter them by [family](../docs/Charts.md#families). The format is: @@ -153,20 +153,13 @@ The format is: families: SIMPLE PATTERN LIST ``` -Simple patterns list is a lists of space separated patterns. Use ` * ` as wildcard and ` ! ` -for a negative match. Processing is left to right, and on the first hit (positive or negative), -processing stops. - -So. `families: *` means, match anything, while `families: !bad*pattern* *` means anything -except `bad*pattern*` (where `*` is a wildcard to match any sequence of characters). - -The family of a chart is usually the submenu of the netdata dashboard it appears. +The simple pattern syntax and operation is explained in [simple patterns](../libnetdata/simple_pattern/). --- #### Alarm line `lookup` -This lines makes a database lookup to find a value. This result of this lookup is available as `$this`. +This line makes a database lookup to find a value. This result of this lookup is available as `$this`. The format is: @@ -349,6 +342,16 @@ delay: [[[up U] [down D] multiplier M] max X] their matching one) and a delay is in place. - All are reset to their defaults when the alarm switches state without a delay in place. +#### Alarm line `option` + +The only possible value for the `option` line is + +``` +option: no-clear-notification +``` + +For some alarms we need compare two time-frames, to detect anomalies. For example, `health.d/httpcheck.conf` has an alarm template called `web_service_slow` that compares the average http call response time over the last 3 minutes, compared to the average over the last hour. It triggers a warning alarm when the average of the last 3 minutes is twice the average of the last hour. In such cases, it is easy to trigger the alarm, but difficult to tell when the alarm is cleared. As time passes, the newest window moves into the older, so the average response time of the last hour will keep increasing. Eventually, the comparison will find the averages in the two time-frames close enough to clear the alarm. However, the issue was not resolved, it's just a matter of the newer data "polluting" the old. For such alarms, it's a good idea to tell Netdata to not clear the notification, by using the `no-clear-notification` option. + --- ### Expressions @@ -419,10 +422,19 @@ Which in turn, results in the following behavior: ### Variables -netdata supports 3 new internal indexes for variables that will be used in health monitoring: +You can find all the variables that can be used for a given chart, using +`http://your.netdata.ip:19999/api/v1/alarm_variables?chart=CHART_NAME` +Example: [variables for the `system.cpu` chart of the registry](https://registry.my-netdata.io/api/v1/alarm_variables?chart=system.cpu). + +_Hint: If you don't know how to find the CHART_NAME, you can read about it [here](../docs/Charts.md#charts)._ - - **chart local variables**. All the dimensions of the chart are exposed as local variables. - All chart alarms names are exposed as variables too. + +Netdata supports 3 internal indexes for variables that will be used in health monitoring. +<details markdown="1"><summary>The variables below can be used in both chart alarms and context templates.</summary> +Although the `alarm_variables` link shows you variables for a particular chart, the same variables can also be used in templates for charts belonging to the same [context](../docs/Charts.md#contexts). The reason is that all charts of a given contexts are essentially identical, with the only difference being the [family](../docs/Charts.md#families) that identifies a particular hardware or software instance. Charts and templates do not apply to specific families anyway, unless if you explicitly limit an alarm with the [alarm line `families`](#alarm-line-families). +</details> + + - **chart local variables**. All the dimensions of the chart are exposed as local variables. The value of $this for the other configured alarms of the chart also appears, under the name of each configured alarm. Charts also define a few special variables: @@ -448,20 +460,15 @@ netdata supports 3 new internal indexes for variables that will be used in healt - **special variables*** are: - - `this`, which is resolved to the value of the current alarm. + - `$this`, which is resolved to the value of the current alarm. - - `status`, which is resolved to the current status of the alarm (the current = the last + - `$status`, which is resolved to the current status of the alarm (the current = the last status, i.e. before the current database lookup and the evaluation of the `calc` line). This values can be compared with `$REMOVED`, `$UNINITIALIZED`, `$UNDEFINED`, `$CLEAR`, - `$WARNING`, `$CRITICAL`. These values are incremental, ie. `$status > $CLEAL` works as + `$WARNING`, `$CRITICAL`. These values are incremental, ie. `$status > $CLEAR` works as expected. - - `now`, which is resolved to current unix timestamp. - -You can find all the variables that can be used for a given chart, using -`http://your.netdata.ip:19999/api/v1/alarm_variables?chart=NAME`. -This will dump all the indexes from the chart's perspective. -Example: [variables for the `system.cpu` chart of the registry](https://registry.my-netdata.io/api/v1/alarm_variables?chart=system.cpu). + - `$now`, which is resolved to current unix timestamp. ## Alarm Statuses @@ -646,3 +653,11 @@ You can find the context of charts by looking up the chart in either You can find how netdata interpreted the expressions by examining the alarm at `http://your.netdata:19999/api/v1/alarms?all`. For each expression, netdata will return the expression as given in its config file, and the same expression with additional parentheses added to indicate the evaluation flow of the expression. +## Disabling health checks or silencing notifications at runtime + +The health checks can be controlled at runtime via the [health management api](../web/api/health/#health-management-api). + +[]() + + + diff --git a/health/health.c b/health/health.c index ae0c464b1..f92a1ba6b 100644 --- a/health/health.c +++ b/health/health.c @@ -2,6 +2,12 @@ #include "health.h" +struct health_cmdapi_thread_status { + int status; + ; + struct rusage rusage; +}; + unsigned int default_health_enabled = 1; // ---------------------------------------------------------------------------- @@ -147,13 +153,41 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { } } + // Check if alarm notifications are silenced + if (ae->flags & HEALTH_ENTRY_FLAG_SILENCED) { + info("Health not sending notification for alarm '%s.%s' status %s (command API has disabled notifications)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status)); + goto done; + } + static char command_to_run[ALARM_EXEC_COMMAND_LENGTH + 1]; pid_t command_pid; const char *exec = (ae->exec) ? ae->exec : host->health_default_exec; const char *recipient = (ae->recipient) ? ae->recipient : host->health_default_recipient; - snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" CALCULATED_NUMBER_FORMAT_ZERO "' '" CALCULATED_NUMBER_FORMAT_ZERO "' '%s' '%u' '%u' '%s' '%s' '%s' '%s'", + int n_warn=0, n_crit=0; + RRDCALC *rc; + EVAL_EXPRESSION *expr=NULL; + + for(rc = host->alarms; rc ; rc = rc->next) { + if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) + continue; + + if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) { + n_warn++; + if (ae->alarm_id == rc->id) + expr=rc->warning; + } else if (unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) { + n_crit++; + if (ae->alarm_id == rc->id) + expr=rc->critical; + } else if (unlikely(rc->status == RRDCALC_STATUS_CLEAR)) { + if (ae->alarm_id == rc->id) + expr=rc->warning; + } + } + + snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" CALCULATED_NUMBER_FORMAT_ZERO "' '" CALCULATED_NUMBER_FORMAT_ZERO "' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d'", exec, recipient, host->registry_hostname, @@ -162,7 +196,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { ae->alarm_event_id, (unsigned long)ae->when, ae->name, - ae->chart?ae->chart:"NOCAHRT", + ae->chart?ae->chart:"NOCHART", ae->family?ae->family:"NOFAMILY", rrdcalc_status2string(ae->new_status), rrdcalc_status2string(ae->old_status), @@ -174,7 +208,11 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) { ae->units?ae->units:"", ae->info?ae->info:"", ae->new_value_string, - ae->old_value_string + ae->old_value_string, + (expr && expr->source)?expr->source:"NOSOURCE", + (expr && expr->error_msg)?buffer_tostring(expr->error_msg):"NOERRMSG", + n_warn, + n_crit ); ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN; @@ -355,6 +393,67 @@ static void health_main_cleanup(void *ptr) { static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } +SILENCE_TYPE check_silenced(RRDCALC *rc, char* host, SILENCERS *silencers) { + SILENCER *s; + debug(D_HEALTH, "Checking if alarm was silenced via the command API. Alarm info name:%s context:%s chart:%s host:%s family:%s", + rc->name, (rc->rrdset)?rc->rrdset->context:"", rc->chart, host, (rc->rrdset)?rc->rrdset->family:""); + + for (s = silencers->silencers; s!=NULL; s=s->next){ + if ( + (!s->alarms_pattern || (rc->name && s->alarms_pattern && simple_pattern_matches(s->alarms_pattern,rc->name))) && + (!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches(s->contexts_pattern,rc->rrdset->context))) && + (!s->hosts_pattern || (host && s->hosts_pattern && simple_pattern_matches(s->hosts_pattern,host))) && + (!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches(s->charts_pattern,rc->chart))) && + (!s->families_pattern || (rc->rrdset && rc->rrdset->family && s->families_pattern && simple_pattern_matches(s->families_pattern,rc->rrdset->family))) + ) { + debug(D_HEALTH, "Alarm matches command API silence entry %s:%s:%s:%s:%s", s->alarms,s->charts, s->contexts, s->hosts, s->families); + if (unlikely(silencers->stype == STYPE_NONE)) { + debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rc->name); + } else { + debug(D_HEALTH, "Alarm %s via the command API - name:%s context:%s chart:%s host:%s family:%s" + , (silencers->stype==STYPE_DISABLE_ALARMS)?"Disabled":"Silenced" + , rc->name + , (rc->rrdset)?rc->rrdset->context:"" + , rc->chart + , host + , (rc->rrdset)?rc->rrdset->family:"" + ); + } + return silencers->stype; + } + } + return STYPE_NONE; +} + +int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) { + uint32_t rrdcalc_flags_old = rc->rrdcalc_flags; + // Clear the flags + rc->rrdcalc_flags &= ~(RRDCALC_FLAG_DISABLED | RRDCALC_FLAG_SILENCED); + if (unlikely(silencers->all_alarms)) { + if (silencers->stype == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED; + else if (silencers->stype == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED; + } else { + SILENCE_TYPE st = check_silenced(rc, host->hostname, silencers); + if (st == STYPE_DISABLE_ALARMS) rc->rrdcalc_flags |= RRDCALC_FLAG_DISABLED; + else if (st == STYPE_SILENCE_NOTIFICATIONS) rc->rrdcalc_flags |= RRDCALC_FLAG_SILENCED; + } + + if (rrdcalc_flags_old != rc->rrdcalc_flags) { + info("Alarm silencing changed for host '%s' alarm '%s': Disabled %s->%s Silenced %s->%s", + host->hostname, + rc->name, + (rrdcalc_flags_old & RRDCALC_FLAG_DISABLED)?"true":"false", + (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false", + (rrdcalc_flags_old & RRDCALC_FLAG_SILENCED)?"true":"false", + (rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)?"true":"false" + ); + } + if (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED) + return 1; + else + return 0; +} + void *health_main(void *ptr) { netdata_thread_cleanup_push(health_main_cleanup, ptr); @@ -365,371 +464,338 @@ void *health_main(void *ptr) { time_t hibernation_delay = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60); unsigned int loop = 0; - while(!netdata_exit) { - loop++; - debug(D_HEALTH, "Health monitoring iteration no %u started", loop); - - int runnable = 0, apply_hibernation_delay = 0; - time_t next_run = now + min_run_every; - RRDCALC *rc; - - if(unlikely(check_if_resumed_from_suspention())) { - apply_hibernation_delay = 1; - - info("Postponing alarm checks for %ld seconds, because it seems that the system was just resumed from suspension." - , hibernation_delay - ); - } - - rrd_rdlock(); - - RRDHOST *host; - rrdhost_foreach_read(host) { - if(unlikely(!host->health_enabled)) - continue; - - if(unlikely(apply_hibernation_delay)) { - info("Postponing health checks for %ld seconds, on host '%s'." - , hibernation_delay - , host->hostname - ); + silencers = mallocz(sizeof(SILENCERS)); + silencers->all_alarms=0; + silencers->stype=STYPE_NONE; + silencers->silencers=NULL; - host->health_delay_up_to = now + hibernation_delay; - } - - if(unlikely(host->health_delay_up_to)) { - if(unlikely(now < host->health_delay_up_to)) - continue; - - info("Resuming health checks on host '%s'.", host->hostname); - host->health_delay_up_to = 0; - } - - rrdhost_rdlock(host); - - // the first loop is to lookup values from the db - for(rc = host->alarms; rc; rc = rc->next) { - if(unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) { - if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)) - rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE; - continue; - } - - runnable++; - rc->old_value = rc->value; - rc->rrdcalc_flags |= RRDCALC_FLAG_RUNNABLE; - - // ------------------------------------------------------------ - // if there is database lookup, do it - - if(unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) { - /* time_t old_db_timestamp = rc->db_before; */ - int value_is_null = 0; - - int ret = rrdset2value_api_v1(rc->rrdset - , NULL - , &rc->value - , rc->dimensions - , 1 - , rc->after - , rc->before - , rc->group - , 0 - , rc->options - , &rc->db_after - , &rc->db_before - , &value_is_null - ); - - if(unlikely(ret != 200)) { - // database lookup failed - rc->value = NAN; - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR; - - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': database lookup returned error %d" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , ret - ); - } - else - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR; - - /* - RRDCALC_FLAG_DB_STALE not currently used - if (unlikely(old_db_timestamp == rc->db_before)) { - // database is stale - - debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name); - - if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) { - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE; - error("Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name); - } - } - else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE)) - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE; - */ - - if(unlikely(value_is_null)) { - // collected value is null - rc->value = NAN; - rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN; - - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - ); - } - else - rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN; - - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': database lookup gave value " CALCULATED_NUMBER_FORMAT - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , rc->value - ); - } - - // ------------------------------------------------------------ - // if there is calculation expression, run it - - if(unlikely(rc->calculation)) { - if(unlikely(!expression_evaluate(rc->calculation))) { - // calculation failed - rc->value = NAN; - rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR; - - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , rc->calculation->parsed_as - , buffer_tostring(rc->calculation->error_msg) - ); - } - else { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR; - - debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value " CALCULATED_NUMBER_FORMAT ": %s (source: %s)" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , rc->calculation->parsed_as - , rc->calculation->result - , buffer_tostring(rc->calculation->error_msg) - , rc->source - ); - - rc->value = rc->calculation->result; - - if(rc->local) rc->local->last_updated = now; - if(rc->family) rc->family->last_updated = now; - if(rc->hostid) rc->hostid->last_updated = now; - if(rc->hostname) rc->hostname->last_updated = now; - } - } - } - rrdhost_unlock(host); - - if(unlikely(runnable && !netdata_exit)) { - rrdhost_rdlock(host); - - for(rc = host->alarms; rc; rc = rc->next) { - if(unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))) - continue; - - RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED; - RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED; - - // -------------------------------------------------------- - // check the warning expression - - if(likely(rc->warning)) { - if(unlikely(!expression_evaluate(rc->warning))) { - // calculation failed - rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR; - - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , buffer_tostring(rc->warning->error_msg) - ); - } - else { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR; - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': warning expression gave value " CALCULATED_NUMBER_FORMAT ": %s (source: %s)" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , rc->warning->result - , buffer_tostring(rc->warning->error_msg) - , rc->source - ); - warning_status = rrdcalc_value2status(rc->warning->result); - } - } - - // -------------------------------------------------------- - // check the critical expression - - if(likely(rc->critical)) { - if(unlikely(!expression_evaluate(rc->critical))) { - // calculation failed - rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR; - - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , buffer_tostring(rc->critical->error_msg) - ); - } - else { - rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR; - debug(D_HEALTH - , "Health on host '%s', alarm '%s.%s': critical expression gave value " CALCULATED_NUMBER_FORMAT ": %s (source: %s)" - , host->hostname - , rc->chart ? rc->chart : "NOCHART" - , rc->name - , rc->critical->result - , buffer_tostring(rc->critical->error_msg) - , rc->source - ); - critical_status = rrdcalc_value2status(rc->critical->result); - } - } - - // -------------------------------------------------------- - // decide the final alarm status - - RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED; - - switch(warning_status) { - case RRDCALC_STATUS_CLEAR: - status = RRDCALC_STATUS_CLEAR; - break; - - case RRDCALC_STATUS_RAISED: - status = RRDCALC_STATUS_WARNING; - break; - - default: - break; - } - - switch(critical_status) { - case RRDCALC_STATUS_CLEAR: - if(status == RRDCALC_STATUS_UNDEFINED) - status = RRDCALC_STATUS_CLEAR; - break; - - case RRDCALC_STATUS_RAISED: - status = RRDCALC_STATUS_CRITICAL; - break; - - default: - break; - } - - // -------------------------------------------------------- - // check if the new status and the old differ - - if(status != rc->status) { - int delay = 0; - - // apply trigger hysteresis - - if(now > rc->delay_up_to_timestamp) { - rc->delay_up_current = rc->delay_up_duration; - rc->delay_down_current = rc->delay_down_duration; - rc->delay_last = 0; - rc->delay_up_to_timestamp = 0; - } - else { - rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier); - if(rc->delay_up_current > rc->delay_max_duration) - rc->delay_up_current = rc->delay_max_duration; - - rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier); - if(rc->delay_down_current > rc->delay_max_duration) - rc->delay_down_current = rc->delay_max_duration; - } - - if(status > rc->status) - delay = rc->delay_up_current; - else - delay = rc->delay_down_current; - - // COMMENTED: because we do need to send raising alarms - // if(now + delay < rc->delay_up_to_timestamp) - // delay = (int)(rc->delay_up_to_timestamp - now); - - rc->delay_last = delay; - rc->delay_up_to_timestamp = now + delay; - - // add the alarm into the log - - health_alarm_log( - host - , rc->id - , rc->next_event_id++ - , now - , rc->name - , rc->rrdset->id - , rc->rrdset->family - , rc->exec - , rc->recipient - , now - rc->last_status_change - , rc->old_value - , rc->value - , rc->status - , status - , rc->source - , rc->units - , rc->info - , rc->delay_last - , (rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION) ? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0 - ); - - rc->last_status_change = now; - rc->status = status; - } - - rc->last_updated = now; - rc->next_update = now + rc->update_every; + while(!netdata_exit) { + loop++; + debug(D_HEALTH, "Health monitoring iteration no %u started", loop); - if(next_run > rc->next_update) - next_run = rc->next_update; - } + int runnable = 0, apply_hibernation_delay = 0; + time_t next_run = now + min_run_every; + RRDCALC *rc; + + if (unlikely(check_if_resumed_from_suspention())) { + apply_hibernation_delay = 1; + + info("Postponing alarm checks for %ld seconds, because it seems that the system was just resumed from suspension.", + hibernation_delay + ); + } + + if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) { + static int logged=0; + if (!logged) { + info("Skipping health checks, because all alarms are disabled via a %s command.", + HEALTH_CMDAPI_CMD_DISABLEALL); + logged = 1; + } + } + + rrd_rdlock(); + + RRDHOST *host; + rrdhost_foreach_read(host) { + if (unlikely(!host->health_enabled)) + continue; + + if (unlikely(apply_hibernation_delay)) { + + info("Postponing health checks for %ld seconds, on host '%s'.", hibernation_delay, host->hostname + ); + + host->health_delay_up_to = now + hibernation_delay; + } + + if (unlikely(host->health_delay_up_to)) { + if (unlikely(now < host->health_delay_up_to)) + continue; + + info("Resuming health checks on host '%s'.", host->hostname); + host->health_delay_up_to = 0; + } + + rrdhost_rdlock(host); + + // the first loop is to lookup values from the db + for (rc = host->alarms; rc; rc = rc->next) { + + if (update_disabled_silenced(host, rc)) + continue; + + if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) { + if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)) + rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE; + continue; + } + + runnable++; + rc->old_value = rc->value; + rc->rrdcalc_flags |= RRDCALC_FLAG_RUNNABLE; + + // ------------------------------------------------------------ + // if there is database lookup, do it + + if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) { + /* time_t old_db_timestamp = rc->db_before; */ + int value_is_null = 0; + + int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1, rc->after, + rc->before, rc->group, 0, rc->options, &rc->db_after, + &rc->db_before, &value_is_null + ); + + if (unlikely(ret != 200)) { + // database lookup failed + rc->value = NAN; + rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR; + + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d", + host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, ret + ); + } else + rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR; + + /* - RRDCALC_FLAG_DB_STALE not currently used + if (unlikely(old_db_timestamp == rc->db_before)) { + // database is stale + + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name); + + if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) { + rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE; + error("Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name); + } + } + else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE)) + rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE; + */ + + if (unlikely(value_is_null)) { + // collected value is null + rc->value = NAN; + rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN; + + debug(D_HEALTH, + "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)", + host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name + ); + } else + rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN; + + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " + CALCULATED_NUMBER_FORMAT, host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + rc->value + ); + } + + // ------------------------------------------------------------ + // if there is calculation expression, run it + + if (unlikely(rc->calculation)) { + if (unlikely(!expression_evaluate(rc->calculation))) { + // calculation failed + rc->value = NAN; + rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR; + + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s", + host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg) + ); + } else { + rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR; + + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value " + CALCULATED_NUMBER_FORMAT + ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + rc->calculation->parsed_as, rc->calculation->result, + buffer_tostring(rc->calculation->error_msg), rc->source + ); + + rc->value = rc->calculation->result; + + if (rc->local) rc->local->last_updated = now; + if (rc->family) rc->family->last_updated = now; + if (rc->hostid) rc->hostid->last_updated = now; + if (rc->hostname) rc->hostname->last_updated = now; + } + } + } + + rrdhost_unlock(host); + + if (unlikely(runnable && !netdata_exit)) { + rrdhost_rdlock(host); + + for (rc = host->alarms; rc; rc = rc->next) { + if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE))) + continue; + + if (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED) { + continue; + } + RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED; + RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED; + + // -------------------------------------------------------- + // check the warning expression + + if (likely(rc->warning)) { + if (unlikely(!expression_evaluate(rc->warning))) { + // calculation failed + rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR; + + debug(D_HEALTH, + "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s", + host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + buffer_tostring(rc->warning->error_msg) + ); + } else { + rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR; + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value " + CALCULATED_NUMBER_FORMAT + ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", + rc->name, rc->warning->result, buffer_tostring(rc->warning->error_msg), rc->source + ); + warning_status = rrdcalc_value2status(rc->warning->result); + } + } + + // -------------------------------------------------------- + // check the critical expression + + if (likely(rc->critical)) { + if (unlikely(!expression_evaluate(rc->critical))) { + // calculation failed + rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR; + + debug(D_HEALTH, + "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s", + host->hostname, rc->chart ? rc->chart : "NOCHART", rc->name, + buffer_tostring(rc->critical->error_msg) + ); + } else { + rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR; + debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value " + CALCULATED_NUMBER_FORMAT + ": %s (source: %s)", host->hostname, rc->chart ? rc->chart : "NOCHART", + rc->name, rc->critical->result, buffer_tostring(rc->critical->error_msg), + rc->source + ); + critical_status = rrdcalc_value2status(rc->critical->result); + } + } + + // -------------------------------------------------------- + // decide the final alarm status + + RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED; + + switch (warning_status) { + case RRDCALC_STATUS_CLEAR: + status = RRDCALC_STATUS_CLEAR; + break; + + case RRDCALC_STATUS_RAISED: + status = RRDCALC_STATUS_WARNING; + break; + + default: + break; + } + + switch (critical_status) { + case RRDCALC_STATUS_CLEAR: + if (status == RRDCALC_STATUS_UNDEFINED) + status = RRDCALC_STATUS_CLEAR; + break; + + case RRDCALC_STATUS_RAISED: + status = RRDCALC_STATUS_CRITICAL; + break; + + default: + break; + } + + // -------------------------------------------------------- + // check if the new status and the old differ + + if (status != rc->status) { + int delay = 0; + + // apply trigger hysteresis + + if (now > rc->delay_up_to_timestamp) { + rc->delay_up_current = rc->delay_up_duration; + rc->delay_down_current = rc->delay_down_duration; + rc->delay_last = 0; + rc->delay_up_to_timestamp = 0; + } else { + rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier); + if (rc->delay_up_current > rc->delay_max_duration) + rc->delay_up_current = rc->delay_max_duration; + + rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier); + if (rc->delay_down_current > rc->delay_max_duration) + rc->delay_down_current = rc->delay_max_duration; + } + + if (status > rc->status) + delay = rc->delay_up_current; + else + delay = rc->delay_down_current; + + // COMMENTED: because we do need to send raising alarms + // if(now + delay < rc->delay_up_to_timestamp) + // delay = (int)(rc->delay_up_to_timestamp - now); + + rc->delay_last = delay; + rc->delay_up_to_timestamp = now + delay; + + health_alarm_log( + host, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, + rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, + rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info, + rc->delay_last, + ( + ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) | + ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) + ) + + ); + + rc->last_status_change = now; + rc->status = status; + } + + rc->last_updated = now; + rc->next_update = now + rc->update_every; + + if (next_run > rc->next_update) + next_run = rc->next_update; + } + + rrdhost_unlock(host); + } - rrdhost_unlock(host); - } + if (unlikely(netdata_exit)) + break; - if(unlikely(netdata_exit)) - break; + // execute notifications + // and cleanup + health_alarm_log_process(host); - // execute notifications - // and cleanup - health_alarm_log_process(host); + if (unlikely(netdata_exit)) + break; - if(unlikely(netdata_exit)) - break; + } /* rrdhost_foreach */ - } /* rrdhost_foreach */ + rrd_unlock(); - rrd_unlock(); if(unlikely(netdata_exit)) break; diff --git a/health/health.d/linux_power_supply.conf b/health/health.d/linux_power_supply.conf index 27a172a14..745d2c3dd 100644 --- a/health/health.d/linux_power_supply.conf +++ b/health/health.d/linux_power_supply.conf @@ -1,7 +1,7 @@ # Alert on low battery capacity. template: linux_power_supply_capacity - on: power_supply.capacity + on: powersupply.capacity calc: $capacity units: % every: 10s diff --git a/health/health.d/mdstat.conf b/health/health.d/mdstat.conf index 0f5f2837e..a53ec7a56 100644 --- a/health/health.d/mdstat.conf +++ b/health/health.d/mdstat.conf @@ -25,3 +25,13 @@ template: mdstat_mismatch_cnt crit: $this > 0 info: Mismatch count! to: sysadmin + +template: mdstat_nonredundant_last_collected + on: md.nonredundant + calc: $now - $last_collected_t + units: seconds ago + every: 10s + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + info: number of seconds since the last successful data collection + to: sysadmin
\ No newline at end of file diff --git a/health/health.d/web_log.conf b/health/health.d/web_log.conf index d8be88b47..031adc2ea 100644 --- a/health/health.d/web_log.conf +++ b/health/health.d/web_log.conf @@ -85,6 +85,36 @@ families: * info: the ratio of HTTP internal server errors (5xx), over the last minute to: webmaster +# unmatched lines + +# the following alarms trigger only when there are enough data. +# we assume there are enough data when: +# +# $1m_total_requests > 120 +# +# i.e. when there are at least 120 requests during the last minute + +template: 1m_total_requests + on: web_log.response_codes +families: * + lookup: sum -1m unaligned + calc: ($this == 0)?(1):($this) + units: requests + every: 10s + info: the sum of all HTTP requests over the last minute + +template: 1m_unmatched +on: web_log.response_codes +families: * + lookup: sum -1m unaligned of unmatched + calc: $this * 100 / $1m_total_requests + units: % + every: 10s + warn: ($1m_total_requests > 120) ? ($this > 1) : ( 0 ) + crit: ($1m_total_requests > 120) ? ($this > 5) : ( 0 ) + delay: up 1m down 5m multiplier 1.5 max 1h + info: the ratio of unmatched lines, over the last minute + to: webmaster # ----------------------------------------------------------------------------- # web slow diff --git a/health/health.h b/health/health.h index ff7a4d9bf..ff10fd6d7 100644 --- a/health/health.h +++ b/health/health.h @@ -22,9 +22,74 @@ extern unsigned int default_health_enabled; #define HEALTH_ENTRY_FLAG_UPDATED 0x00000002 #define HEALTH_ENTRY_FLAG_EXEC_RUN 0x00000004 #define HEALTH_ENTRY_FLAG_EXEC_FAILED 0x00000008 +#define HEALTH_ENTRY_FLAG_SILENCED 0x00000008 + #define HEALTH_ENTRY_FLAG_SAVED 0x10000000 #define HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION 0x80000000 +#ifndef HEALTH_LISTEN_PORT +#define HEALTH_LISTEN_PORT 19998 +#endif + +#ifndef HEALTH_LISTEN_BACKLOG +#define HEALTH_LISTEN_BACKLOG 4096 +#endif + +#define HEALTH_ALARM_KEY "alarm" +#define HEALTH_TEMPLATE_KEY "template" +#define HEALTH_ON_KEY "on" +#define HEALTH_CONTEXT_KEY "context" +#define HEALTH_CHART_KEY "chart" +#define HEALTH_HOST_KEY "hosts" +#define HEALTH_OS_KEY "os" +#define HEALTH_FAMILIES_KEY "families" +#define HEALTH_LOOKUP_KEY "lookup" +#define HEALTH_CALC_KEY "calc" +#define HEALTH_EVERY_KEY "every" +#define HEALTH_GREEN_KEY "green" +#define HEALTH_RED_KEY "red" +#define HEALTH_WARN_KEY "warn" +#define HEALTH_CRIT_KEY "crit" +#define HEALTH_EXEC_KEY "exec" +#define HEALTH_RECIPIENT_KEY "to" +#define HEALTH_UNITS_KEY "units" +#define HEALTH_INFO_KEY "info" +#define HEALTH_DELAY_KEY "delay" +#define HEALTH_OPTIONS_KEY "options" + +typedef struct silencer { + char *alarms; + SIMPLE_PATTERN *alarms_pattern; + + char *hosts; + SIMPLE_PATTERN *hosts_pattern; + + char *contexts; + SIMPLE_PATTERN *contexts_pattern; + + char *charts; + SIMPLE_PATTERN *charts_pattern; + + char *families; + SIMPLE_PATTERN *families_pattern; + + struct silencer *next; +} SILENCER; + +typedef enum silence_type { + STYPE_NONE, + STYPE_DISABLE_ALARMS, + STYPE_SILENCE_NOTIFICATIONS +} SILENCE_TYPE; + +typedef struct silencers { + int all_alarms; + SILENCE_TYPE stype; + SILENCER *silencers; +} SILENCERS; + +SILENCERS *silencers; + extern void health_init(void); extern void *health_main(void *ptr); @@ -62,8 +127,7 @@ extern void health_alarm_log( const char *units, const char *info, int delay, - uint32_t flags -); + uint32_t flags); extern void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath); extern char *health_user_config_dir(void); @@ -73,4 +137,6 @@ extern void health_alarm_log_free(RRDHOST *host); extern void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae); +extern void *health_cmdapi_thread(void *ptr); + #endif //NETDATA_HEALTH_H diff --git a/health/health_config.c b/health/health_config.c index d4cf78d97..35fde90bc 100644 --- a/health/health_config.c +++ b/health/health_config.c @@ -853,6 +853,9 @@ static int health_readfile(const char *filename, void *data) { } void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath) { - if(unlikely(!host->health_enabled)) return; + if(unlikely(!host->health_enabled)) { + debug(D_HEALTH, "CONFIG health is not enabled for host '%s'", host->hostname); + return; + } recursive_config_double_dir_load(user_path, stock_path, subpath, health_readfile, (void *) host, 0); } diff --git a/health/health_json.c b/health/health_json.c index a049dc1b2..781132447 100644 --- a/health/health_json.c +++ b/health/health_json.c @@ -43,6 +43,7 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R "\t\t\"updates_id\": %u,\n" "\t\t\"value_string\": \"%s\",\n" "\t\t\"old_value_string\": \"%s\",\n" + "\t\t\"silenced\": \"%s\",\n" , host->hostname , ae->unique_id , ae->alarm_id @@ -70,6 +71,7 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R , ae->updates_id , ae->new_value_string , ae->old_value_string + , (ae->flags & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false" ); health_string2json(wb, "\t\t", "info", ae->info?ae->info:"", ",\n"); @@ -120,6 +122,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC "\t\t\t\"chart\": \"%s\",\n" "\t\t\t\"family\": \"%s\",\n" "\t\t\t\"active\": %s,\n" + "\t\t\t\"disabled\": %s,\n" + "\t\t\t\"silenced\": %s,\n" "\t\t\t\"exec\": \"%s\",\n" "\t\t\t\"recipient\": \"%s\",\n" "\t\t\t\"source\": \"%s\",\n" @@ -143,6 +147,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC , rc->chart , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:"" , (rc->rrdset)?"true":"false" + , (rc->rrdcalc_flags & RRDCALC_FLAG_DISABLED)?"true":"false" + , (rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)?"true":"false" , rc->exec?rc->exec:host->health_default_exec , rc->recipient?rc->recipient:host->health_default_recipient , rc->source diff --git a/health/health_log.c b/health/health_log.c index dd51be2af..009e42673 100644 --- a/health/health_log.c +++ b/health/health_log.c @@ -396,7 +396,6 @@ inline void health_alarm_log( ae->duration = duration; ae->delay = delay; ae->delay_up_to_timestamp = when + delay; - ae->flags |= flags; if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL) diff --git a/health/notifications/Makefile.in b/health/notifications/Makefile.in deleted file mode 100644 index 05a5814fc..000000000 --- a/health/notifications/Makefile.in +++ /dev/null @@ -1,754 +0,0 @@ -# Makefile.in generated by automake 1.14.1 from Makefile.am. -# @configure_input@ - -# Copyright (C) 1994-2013 Free Software Foundation, Inc. - -# This Makefile.in is free software; the Free Software Foundation -# gives unlimited permission to copy and/or distribute it, -# with or without modifications, as long as this notice is preserved. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY, to the extent permitted by law; without -# even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. - -@SET_MAKE@ - -# SPDX-License-Identifier: GPL-3.0-or-later - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - -# SPDX-License-Identifier: GPL-3.0-or-later - -# THIS IS NOT A COMPLETE Makefile -# IT IS INCLUDED BY ITS PARENT'S Makefile.am -# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT - - -VPATH = @srcdir@ -am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)' -am__make_running_with_option = \ - case $${target_option-} in \ - ?) ;; \ - *) echo "am__make_running_with_option: internal error: invalid" \ - "target option '$${target_option-}' specified" >&2; \ - exit 1;; \ - esac; \ - has_opt=no; \ - sane_makeflags=$$MAKEFLAGS; \ - if $(am__is_gnu_make); then \ - sane_makeflags=$$MFLAGS; \ - else \ - case $$MAKEFLAGS in \ - *\\[\ \ ]*) \ - bs=\\; \ - sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ - | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ - esac; \ - fi; \ - skip_next=no; \ - strip_trailopt () \ - { \ - flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ - }; \ - for flg in $$sane_makeflags; do \ - test $$skip_next = yes && { skip_next=no; continue; }; \ - case $$flg in \ - *=*|--*) continue;; \ - -*I) strip_trailopt 'I'; skip_next=yes;; \ - -*I?*) strip_trailopt 'I';; \ - -*O) strip_trailopt 'O'; skip_next=yes;; \ - -*O?*) strip_trailopt 'O';; \ - -*l) strip_trailopt 'l'; skip_next=yes;; \ - -*l?*) strip_trailopt 'l';; \ - -[dEDm]) skip_next=yes;; \ - -[JT]) skip_next=yes;; \ - esac; \ - case $$flg in \ - *$$target_option*) has_opt=yes; break;; \ - esac; \ - done; \ - test $$has_opt = yes -am__make_dryrun = (target_option=n; $(am__make_running_with_option)) -am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) -pkgdatadir = $(datadir)/@PACKAGE@ -pkgincludedir = $(includedir)/@PACKAGE@ -pkglibdir = $(libdir)/@PACKAGE@ -pkglibexecdir = $(libexecdir)/@PACKAGE@ -am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd -install_sh_DATA = $(install_sh) -c -m 644 -install_sh_PROGRAM = $(install_sh) -c -install_sh_SCRIPT = $(install_sh) -c -INSTALL_HEADER = $(INSTALL_DATA) -transform = $(program_transform_name) -NORMAL_INSTALL = : -PRE_INSTALL = : -POST_INSTALL = : -NORMAL_UNINSTALL = : -PRE_UNINSTALL = : -POST_UNINSTALL = : -build_triplet = @build@ -host_triplet = @host@ -DIST_COMMON = $(top_srcdir)/build/subst.inc \ - $(srcdir)/alerta/Makefile.inc $(srcdir)/awssns/Makefile.inc \ - $(srcdir)/discord/Makefile.inc $(srcdir)/email/Makefile.inc \ - $(srcdir)/flock/Makefile.inc $(srcdir)/irc/Makefile.inc \ - $(srcdir)/kavenegar/Makefile.inc \ - $(srcdir)/messagebird/Makefile.inc \ - $(srcdir)/pagerduty/Makefile.inc \ - $(srcdir)/pushbullet/Makefile.inc \ - $(srcdir)/pushover/Makefile.inc \ - $(srcdir)/rocketchat/Makefile.inc $(srcdir)/slack/Makefile.inc \ - $(srcdir)/syslog/Makefile.inc $(srcdir)/telegram/Makefile.inc \ - $(srcdir)/twilio/Makefile.inc $(srcdir)/web/Makefile.inc \ - $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ - $(dist_plugins_SCRIPTS) $(dist_libconfig_DATA) \ - $(dist_noinst_DATA) -subdir = health/notifications -ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/build/m4/ax_c___atomic.m4 \ - $(top_srcdir)/build/m4/ax_c__generic.m4 \ - $(top_srcdir)/build/m4/ax_c_lto.m4 \ - $(top_srcdir)/build/m4/ax_c_mallinfo.m4 \ - $(top_srcdir)/build/m4/ax_c_mallopt.m4 \ - $(top_srcdir)/build/m4/ax_check_compile_flag.m4 \ - $(top_srcdir)/build/m4/ax_gcc_func_attribute.m4 \ - $(top_srcdir)/build/m4/ax_pthread.m4 \ - $(top_srcdir)/build/m4/jemalloc.m4 \ - $(top_srcdir)/build/m4/tcmalloc.m4 $(top_srcdir)/configure.ac -am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ - $(ACLOCAL_M4) -mkinstalldirs = $(install_sh) -d -CONFIG_HEADER = $(top_builddir)/config.h -CONFIG_CLEAN_FILES = -CONFIG_CLEAN_VPATH_FILES = -am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; -am__vpath_adj = case $$p in \ - $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ - *) f=$$p;; \ - esac; -am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; -am__install_max = 40 -am__nobase_strip_setup = \ - srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` -am__nobase_strip = \ - for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" -am__nobase_list = $(am__nobase_strip_setup); \ - for p in $$list; do echo "$$p $$p"; done | \ - sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ - $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ - if (++n[$$2] == $(am__install_max)) \ - { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ - END { for (dir in files) print dir, files[dir] }' -am__base_list = \ - sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ - sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' -am__uninstall_files_from_dir = { \ - test -z "$$files" \ - || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ - || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ - $(am__cd) "$$dir" && rm -f $$files; }; \ - } -am__installdirs = "$(DESTDIR)$(pluginsdir)" \ - "$(DESTDIR)$(libconfigdir)" -SCRIPTS = $(dist_plugins_SCRIPTS) -AM_V_P = $(am__v_P_@AM_V@) -am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) -am__v_P_0 = false -am__v_P_1 = : -AM_V_GEN = $(am__v_GEN_@AM_V@) -am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) -am__v_GEN_0 = @echo " GEN " $@; -am__v_GEN_1 = -AM_V_at = $(am__v_at_@AM_V@) -am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) -am__v_at_0 = @ -am__v_at_1 = -SOURCES = -DIST_SOURCES = -am__can_run_installinfo = \ - case $$AM_UPDATE_INFO_DIR in \ - n|no|NO) false;; \ - *) (install-info --version) >/dev/null 2>&1;; \ - esac -DATA = $(dist_libconfig_DATA) $(dist_noinst_DATA) -am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) -DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) -ACLOCAL = @ACLOCAL@ -AMTAR = @AMTAR@ -AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ -AUTOCONF = @AUTOCONF@ -AUTOHEADER = @AUTOHEADER@ -AUTOMAKE = @AUTOMAKE@ -AWK = @AWK@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ -CPP = @CPP@ -CPPFLAGS = @CPPFLAGS@ -CYGPATH_W = @CYGPATH_W@ -DEFS = @DEFS@ -DEPDIR = @DEPDIR@ -ECHO_C = @ECHO_C@ -ECHO_N = @ECHO_N@ -ECHO_T = @ECHO_T@ -EGREP = @EGREP@ -EXEEXT = @EXEEXT@ -GREP = @GREP@ -INSTALL = @INSTALL@ -INSTALL_DATA = @INSTALL_DATA@ -INSTALL_PROGRAM = @INSTALL_PROGRAM@ -INSTALL_SCRIPT = @INSTALL_SCRIPT@ -INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ -IPMIMONITORING_CFLAGS = @IPMIMONITORING_CFLAGS@ -IPMIMONITORING_LIBS = @IPMIMONITORING_LIBS@ -LDFLAGS = @LDFLAGS@ -LIBCAP_CFLAGS = @LIBCAP_CFLAGS@ -LIBCAP_LIBS = @LIBCAP_LIBS@ -LIBMNL_CFLAGS = @LIBMNL_CFLAGS@ -LIBMNL_LIBS = @LIBMNL_LIBS@ -LIBOBJS = @LIBOBJS@ -LIBS = @LIBS@ -LTLIBOBJS = @LTLIBOBJS@ -MAINT = @MAINT@ -MAKEINFO = @MAKEINFO@ -MATH_CFLAGS = @MATH_CFLAGS@ -MATH_LIBS = @MATH_LIBS@ -MKDIR_P = @MKDIR_P@ -NFACCT_CFLAGS = @NFACCT_CFLAGS@ -NFACCT_LIBS = @NFACCT_LIBS@ -OBJEXT = @OBJEXT@ -OPTIONAL_IPMIMONITORING_CFLAGS = @OPTIONAL_IPMIMONITORING_CFLAGS@ -OPTIONAL_IPMIMONITORING_LIBS = @OPTIONAL_IPMIMONITORING_LIBS@ -OPTIONAL_LIBCAP_CFLAGS = @OPTIONAL_LIBCAP_CFLAGS@ -OPTIONAL_LIBCAP_LIBS = @OPTIONAL_LIBCAP_LIBS@ -OPTIONAL_MATH_CLFAGS = @OPTIONAL_MATH_CLFAGS@ -OPTIONAL_MATH_LIBS = @OPTIONAL_MATH_LIBS@ -OPTIONAL_NFACCT_CLFAGS = @OPTIONAL_NFACCT_CLFAGS@ -OPTIONAL_NFACCT_LIBS = @OPTIONAL_NFACCT_LIBS@ -OPTIONAL_UUID_CLFAGS = @OPTIONAL_UUID_CLFAGS@ -OPTIONAL_UUID_LIBS = @OPTIONAL_UUID_LIBS@ -OPTIONAL_ZLIB_CLFAGS = @OPTIONAL_ZLIB_CLFAGS@ -OPTIONAL_ZLIB_LIBS = @OPTIONAL_ZLIB_LIBS@ -PACKAGE = @PACKAGE@ -PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_RPM_RELEASE = @PACKAGE_RPM_RELEASE@ -PACKAGE_RPM_VERSION = @PACKAGE_RPM_VERSION@ -PACKAGE_STRING = @PACKAGE_STRING@ -PACKAGE_TARNAME = @PACKAGE_TARNAME@ -PACKAGE_URL = @PACKAGE_URL@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -PATH_SEPARATOR = @PATH_SEPARATOR@ -PKG_CONFIG = @PKG_CONFIG@ -PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ -PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ -PTHREAD_CC = @PTHREAD_CC@ -PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ -PTHREAD_LIBS = @PTHREAD_LIBS@ -SET_MAKE = @SET_MAKE@ -SHELL = @SHELL@ -SSE_CANDIDATE = @SSE_CANDIDATE@ -STRIP = @STRIP@ -UUID_CFLAGS = @UUID_CFLAGS@ -UUID_LIBS = @UUID_LIBS@ -VERSION = @VERSION@ -ZLIB_CFLAGS = @ZLIB_CFLAGS@ -ZLIB_LIBS = @ZLIB_LIBS@ -abs_builddir = @abs_builddir@ -abs_srcdir = @abs_srcdir@ -abs_top_builddir = @abs_top_builddir@ -abs_top_srcdir = @abs_top_srcdir@ -ac_ct_CC = @ac_ct_CC@ -am__include = @am__include@ -am__leading_dot = @am__leading_dot@ -am__quote = @am__quote@ -am__tar = @am__tar@ -am__untar = @am__untar@ -ax_pthread_config = @ax_pthread_config@ -bindir = @bindir@ -build = @build@ -build_alias = @build_alias@ -build_cpu = @build_cpu@ -build_os = @build_os@ -build_target = @build_target@ -build_vendor = @build_vendor@ -builddir = @builddir@ -cachedir = @cachedir@ -chartsdir = @chartsdir@ -configdir = @configdir@ -datadir = @datadir@ -datarootdir = @datarootdir@ -docdir = @docdir@ -dvidir = @dvidir@ -exec_prefix = @exec_prefix@ -has_jemalloc = @has_jemalloc@ -has_tcmalloc = @has_tcmalloc@ -host = @host@ -host_alias = @host_alias@ -host_cpu = @host_cpu@ -host_os = @host_os@ -host_vendor = @host_vendor@ -htmldir = @htmldir@ -includedir = @includedir@ -infodir = @infodir@ -install_sh = @install_sh@ -libconfigdir = @libconfigdir@ -libdir = @libdir@ -libexecdir = @libexecdir@ -localedir = @localedir@ -localstatedir = @localstatedir@ -logdir = @logdir@ -mandir = @mandir@ -mkdir_p = @mkdir_p@ -nodedir = @nodedir@ -oldincludedir = @oldincludedir@ -pdfdir = @pdfdir@ -pluginsdir = @pluginsdir@ -prefix = @prefix@ -program_transform_name = @program_transform_name@ -psdir = @psdir@ -pythondir = @pythondir@ -registrydir = @registrydir@ -sbindir = @sbindir@ -sharedstatedir = @sharedstatedir@ -srcdir = @srcdir@ -sysconfdir = @sysconfdir@ -target_alias = @target_alias@ -top_build_prefix = @top_build_prefix@ -top_builddir = @top_builddir@ -top_srcdir = @top_srcdir@ -varlibdir = @varlibdir@ -webdir = @webdir@ -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in -CLEANFILES = \ - alarm-notify.sh \ - $(NULL) - -SUFFIXES = .in -dist_libconfig_DATA = \ - health_alarm_notify.conf \ - health_email_recipients.conf \ - $(NULL) - -dist_plugins_SCRIPTS = \ - alarm-notify.sh \ - alarm-email.sh \ - alarm-test.sh \ - $(NULL) - - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files - -# install these files -dist_noinst_DATA = alarm-notify.sh.in README.md $(NULL) \ - alerta/README.md alerta/Makefile.inc $(NULL) awssns/README.md \ - awssns/Makefile.inc $(NULL) discord/README.md \ - discord/Makefile.inc $(NULL) email/README.md \ - email/Makefile.inc $(NULL) flock/README.md flock/Makefile.inc \ - $(NULL) irc/README.md irc/Makefile.inc $(NULL) \ - kavenegar/README.md kavenegar/Makefile.inc $(NULL) \ - messagebird/README.md messagebird/Makefile.inc $(NULL) \ - pagerduty/README.md pagerduty/Makefile.inc $(NULL) \ - pushbullet/README.md pushbullet/Makefile.inc $(NULL) \ - pushover/README.md pushover/Makefile.inc $(NULL) \ - rocketchat/README.md rocketchat/Makefile.inc $(NULL) \ - slack/README.md slack/Makefile.inc $(NULL) syslog/README.md \ - syslog/Makefile.inc $(NULL) telegram/README.md \ - telegram/Makefile.inc $(NULL) twilio/README.md \ - twilio/Makefile.inc $(NULL) web/README.md web/Makefile.inc \ - $(NULL) -all: all-am - -.SUFFIXES: -.SUFFIXES: .in -$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(top_srcdir)/build/subst.inc $(srcdir)/alerta/Makefile.inc $(srcdir)/awssns/Makefile.inc $(srcdir)/discord/Makefile.inc $(srcdir)/email/Makefile.inc $(srcdir)/flock/Makefile.inc $(srcdir)/irc/Makefile.inc $(srcdir)/kavenegar/Makefile.inc $(srcdir)/messagebird/Makefile.inc $(srcdir)/pagerduty/Makefile.inc $(srcdir)/pushbullet/Makefile.inc $(srcdir)/pushover/Makefile.inc $(srcdir)/rocketchat/Makefile.inc $(srcdir)/slack/Makefile.inc $(srcdir)/syslog/Makefile.inc $(srcdir)/telegram/Makefile.inc $(srcdir)/twilio/Makefile.inc $(srcdir)/web/Makefile.inc $(am__configure_deps) - @for dep in $?; do \ - case '$(am__configure_deps)' in \ - *$$dep*) \ - ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ - && { if test -f $@; then exit 0; else break; fi; }; \ - exit 1;; \ - esac; \ - done; \ - echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu health/notifications/Makefile'; \ - $(am__cd) $(top_srcdir) && \ - $(AUTOMAKE) --gnu health/notifications/Makefile -.PRECIOUS: Makefile -Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status - @case '$?' in \ - *config.status*) \ - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ - *) \ - echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ - cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ - esac; -$(top_srcdir)/build/subst.inc $(srcdir)/alerta/Makefile.inc $(srcdir)/awssns/Makefile.inc $(srcdir)/discord/Makefile.inc $(srcdir)/email/Makefile.inc $(srcdir)/flock/Makefile.inc $(srcdir)/irc/Makefile.inc $(srcdir)/kavenegar/Makefile.inc $(srcdir)/messagebird/Makefile.inc $(srcdir)/pagerduty/Makefile.inc $(srcdir)/pushbullet/Makefile.inc $(srcdir)/pushover/Makefile.inc $(srcdir)/rocketchat/Makefile.inc $(srcdir)/slack/Makefile.inc $(srcdir)/syslog/Makefile.inc $(srcdir)/telegram/Makefile.inc $(srcdir)/twilio/Makefile.inc $(srcdir)/web/Makefile.inc: - -$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh - -$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(am__aclocal_m4_deps): -install-dist_pluginsSCRIPTS: $(dist_plugins_SCRIPTS) - @$(NORMAL_INSTALL) - @list='$(dist_plugins_SCRIPTS)'; test -n "$(pluginsdir)" || list=; \ - if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(pluginsdir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(pluginsdir)" || exit 1; \ - fi; \ - for p in $$list; do \ - if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ - if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ - done | \ - sed -e 'p;s,.*/,,;n' \ - -e 'h;s|.*|.|' \ - -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ - $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ - { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ - if ($$2 == $$4) { files[d] = files[d] " " $$1; \ - if (++n[d] == $(am__install_max)) { \ - print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ - else { print "f", d "/" $$4, $$1 } } \ - END { for (d in files) print "f", d, files[d] }' | \ - while read type dir files; do \ - if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ - test -z "$$files" || { \ - echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(pluginsdir)$$dir'"; \ - $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(pluginsdir)$$dir" || exit $$?; \ - } \ - ; done - -uninstall-dist_pluginsSCRIPTS: - @$(NORMAL_UNINSTALL) - @list='$(dist_plugins_SCRIPTS)'; test -n "$(pluginsdir)" || exit 0; \ - files=`for p in $$list; do echo "$$p"; done | \ - sed -e 's,.*/,,;$(transform)'`; \ - dir='$(DESTDIR)$(pluginsdir)'; $(am__uninstall_files_from_dir) -install-dist_libconfigDATA: $(dist_libconfig_DATA) - @$(NORMAL_INSTALL) - @list='$(dist_libconfig_DATA)'; test -n "$(libconfigdir)" || list=; \ - if test -n "$$list"; then \ - echo " $(MKDIR_P) '$(DESTDIR)$(libconfigdir)'"; \ - $(MKDIR_P) "$(DESTDIR)$(libconfigdir)" || exit 1; \ - fi; \ - for p in $$list; do \ - if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ - echo "$$d$$p"; \ - done | $(am__base_list) | \ - while read files; do \ - echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(libconfigdir)'"; \ - $(INSTALL_DATA) $$files "$(DESTDIR)$(libconfigdir)" || exit $$?; \ - done - -uninstall-dist_libconfigDATA: - @$(NORMAL_UNINSTALL) - @list='$(dist_libconfig_DATA)'; test -n "$(libconfigdir)" || list=; \ - files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ - dir='$(DESTDIR)$(libconfigdir)'; $(am__uninstall_files_from_dir) -tags TAGS: - -ctags CTAGS: - -cscope cscopelist: - - -distdir: $(DISTFILES) - @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - list='$(DISTFILES)'; \ - dist_files=`for file in $$list; do echo $$file; done | \ - sed -e "s|^$$srcdirstrip/||;t" \ - -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ - case $$dist_files in \ - */*) $(MKDIR_P) `echo "$$dist_files" | \ - sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ - sort -u` ;; \ - esac; \ - for file in $$dist_files; do \ - if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ - if test -d $$d/$$file; then \ - dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ - if test -d "$(distdir)/$$file"; then \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ - cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ - else \ - test -f "$(distdir)/$$file" \ - || cp -p $$d/$$file "$(distdir)/$$file" \ - || exit 1; \ - fi; \ - done -check-am: all-am -check: check-am -all-am: Makefile $(SCRIPTS) $(DATA) -installdirs: - for dir in "$(DESTDIR)$(pluginsdir)" "$(DESTDIR)$(libconfigdir)"; do \ - test -z "$$dir" || $(MKDIR_P) "$$dir"; \ - done -install: install-am -install-exec: install-exec-am -install-data: install-data-am -uninstall: uninstall-am - -install-am: all-am - @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am - -installcheck: installcheck-am -install-strip: - if test -z '$(STRIP)'; then \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - install; \ - else \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ - fi -mostlyclean-generic: - -clean-generic: - -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) - -distclean-generic: - -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) - -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) - -maintainer-clean-generic: - @echo "This command is intended for maintainers to use" - @echo "it deletes files that may require special tools to rebuild." - -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES) -clean: clean-am - -clean-am: clean-generic mostlyclean-am - -distclean: distclean-am - -rm -f Makefile -distclean-am: clean-am distclean-generic - -dvi: dvi-am - -dvi-am: - -html: html-am - -html-am: - -info: info-am - -info-am: - -install-data-am: install-dist_libconfigDATA \ - install-dist_pluginsSCRIPTS - -install-dvi: install-dvi-am - -install-dvi-am: - -install-exec-am: - -install-html: install-html-am - -install-html-am: - -install-info: install-info-am - -install-info-am: - -install-man: - -install-pdf: install-pdf-am - -install-pdf-am: - -install-ps: install-ps-am - -install-ps-am: - -installcheck-am: - -maintainer-clean: maintainer-clean-am - -rm -f Makefile -maintainer-clean-am: distclean-am maintainer-clean-generic - -mostlyclean: mostlyclean-am - -mostlyclean-am: mostlyclean-generic - -pdf: pdf-am - -pdf-am: - -ps: ps-am - -ps-am: - -uninstall-am: uninstall-dist_libconfigDATA \ - uninstall-dist_pluginsSCRIPTS - -.MAKE: install-am install-strip - -.PHONY: all all-am check check-am clean clean-generic cscopelist-am \ - ctags-am distclean distclean-generic distdir dvi dvi-am html \ - html-am info info-am install install-am install-data \ - install-data-am install-dist_libconfigDATA \ - install-dist_pluginsSCRIPTS install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-pdf \ - install-pdf-am install-ps install-ps-am install-strip \ - installcheck installcheck-am installdirs maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-generic pdf \ - pdf-am ps ps-am tags-am uninstall uninstall-am \ - uninstall-dist_libconfigDATA uninstall-dist_pluginsSCRIPTS - -.in: - if sed \ - -e 's#[@]localstatedir_POST@#$(localstatedir)#g' \ - -e 's#[@]sbindir_POST@#$(sbindir)#g' \ - -e 's#[@]sysconfdir_POST@#$(sysconfdir)#g' \ - -e 's#[@]pythondir_POST@#$(pythondir)#g' \ - -e 's#[@]configdir_POST@#$(configdir)#g' \ - -e 's#[@]libconfigdir_POST@#$(libconfigdir)#g' \ - -e 's#[@]cachedir_POST@#$(cachedir)#g' \ - $< > $@.tmp; then \ - mv "$@.tmp" "$@"; \ - else \ - rm -f "$@.tmp"; \ - false; \ - fi - -# Tell versions [3.59,3.63) of GNU make to not export all variables. -# Otherwise a system limit (for SysV at least) may be exceeded. -.NOEXPORT: diff --git a/health/notifications/README.md b/health/notifications/README.md index c06638ade..5b7b43406 100644 --- a/health/notifications/README.md +++ b/health/notifications/README.md @@ -58,3 +58,9 @@ export NETDATA_ALARM_NOTIFY_DEBUG=1 # send test alarms to any role /usr/libexec/netdata/plugins.d/alarm-notify.sh test "ROLE" ``` +If you need to dig even deeper, you can trace the execution with `bash -x`. Note that in test mode, alarm-notify.sh calls itself with many more arguments. So first do + ```sh + bash -x /usr/libexec/netdata/plugins.d/alarm-notify.sh test + ``` + Then look in the output for the alarm-notify.sh calls and run the one you want to trace with `bash -x`. +[]() diff --git a/health/notifications/alarm-notify.sh b/health/notifications/alarm-notify.sh deleted file mode 100644 index 3331dcd94..000000000 --- a/health/notifications/alarm-notify.sh +++ /dev/null @@ -1,2407 +0,0 @@ -#!/usr/bin/env bash - -# netdata -# real-time performance and health monitoring, done right! -# (C) 2017 Costa Tsaousis <costa@tsaousis.gr> -# SPDX-License-Identifier: GPL-3.0-or-later -# -# Script to send alarm notifications for netdata -# -# Features: -# - multiple notification methods -# - multiple roles per alarm -# - multiple recipients per role -# - severity filtering per recipient -# -# Supported notification methods: -# - emails by @ktsaou -# - slack.com notifications by @ktsaou -# - alerta.io notifications by @kattunga -# - discordapp.com notifications by @lowfive -# - pushover.net notifications by @ktsaou -# - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070 -# - telegram.org notifications by @hashworks #1002 -# - twilio.com notifications by Levi Blaney @shadycuz #1211 -# - kafka notifications by @ktsaou #1342 -# - pagerduty.com notifications by Jim Cooley @jimcooley #1373 -# - messagebird.com notifications by @tech_no_logical #1453 -# - hipchat notifications by @ktsaou #1561 -# - fleep notifications by @Ferroin -# - custom notifications by @ktsaou -# - syslog messages by @Ferroin -# - Microsoft Team notification by @tioumen - -# ----------------------------------------------------------------------------- -# testing notifications - - -if [ \( "${1}" = "test" -o "${2}" = "test" \) -a "${#}" -le 2 ] -then - if [ "${2}" = "test" ] - then - recipient="${1}" - else - recipient="${2}" - fi - - [ -z "${recipient}" ] && recipient="sysadmin" - - id=1 - last="CLEAR" - test_res=0 - for x in "WARNING" "CRITICAL" "CLEAR" - do - echo >&2 - echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}" - - "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" - if [ $? -ne 0 ] - then - echo >&2 "# FAILED" - test_res=1 - else - echo >&2 "# OK" - fi - - last="${x}" - id=$((id + 1)) - done - - exit $test_res -fi - -export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin" -export LC_ALL=C - -# ----------------------------------------------------------------------------- - -PROGRAM_NAME="$(basename "${0}")" - -logdate() { - date "+%Y-%m-%d %H:%M:%S" -} - -log() { - local status="${1}" - shift - - echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}" - -} - -warning() { - log WARNING "${@}" -} - -error() { - log ERROR "${@}" -} - -info() { - log INFO "${@}" -} - -fatal() { - log FATAL "${@}" - exit 1 -} - -debug=${NETDATA_ALARM_NOTIFY_DEBUG-0} -debug() { - [ "${debug}" = "1" ] && log DEBUG "${@}" -} - -docurl() { - if [ -z "${curl}" ] - then - error "\${curl} is unset." - return 1 - fi - - if [ "${debug}" = "1" ] - then - echo >&2 "--- BEGIN curl command ---" - printf >&2 "%q " ${curl} "${@}" - echo >&2 - echo >&2 "--- END curl command ---" - - local out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX) - local code=$(${curl} ${curl_options} --write-out %{http_code} --output "${out}" --silent --show-error "${@}") - local ret=$? - echo >&2 "--- BEGIN received response ---" - cat >&2 "${out}" - echo >&2 - echo >&2 "--- END received response ---" - echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}" - rm "${out}" - echo "${code}" - return ${ret} - fi - - ${curl} ${curl_options} --write-out %{http_code} --output /dev/null --silent --show-error "${@}" - return $? -} - -# ----------------------------------------------------------------------------- -# this is to be overwritten by the config file - -custom_sender() { - info "not sending custom notification for ${status} of '${host}.${chart}.${name}'" -} - - -# ----------------------------------------------------------------------------- - -# check for BASH v4+ (required for associative arrays) -[ $(( ${BASH_VERSINFO[0]} )) -lt 4 ] && \ - fatal "BASH version 4 or later is required (this is ${BASH_VERSION})." - -# ----------------------------------------------------------------------------- -# defaults to allow running this script by hand - -[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="/usr/local/etc/netdata" -[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="/usr/local/lib/netdata/conf.d" -[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="/usr/local/var/cache/netdata" -[ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io" - -# ----------------------------------------------------------------------------- -# parse command line parameters - -roles="${1}" # the roles that should be notified for this event -host="${2}" # the host generated this event -unique_id="${3}" # the unique id of this event -alarm_id="${4}" # the unique id of the alarm that generated this event -event_id="${5}" # the incremental id of the event, for this alarm id -when="${6}" # the timestamp this event occurred -name="${7}" # the name of the alarm, as given in netdata health.d entries -chart="${8}" # the name of the chart (type.id) -family="${9}" # the family of the chart -status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL -old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL -value="${12}" # the current value of the alarm -old_value="${13}" # the previous value of the alarm -src="${14}" # the line number and file the alarm has been configured -duration="${15}" # the duration in seconds of the previous alarm state -non_clear_duration="${16}" # the total duration in seconds this is/was non-clear -units="${17}" # the units of the value -info="${18}" # a short description of the alarm -value_string="${19}" # friendly value (with units) -old_value_string="${20}" # friendly old value (with units) - -# ----------------------------------------------------------------------------- -# find a suitable hostname to use, if netdata did not supply a hostname - -this_host=$(hostname -s 2>/dev/null) -[ -z "${host}" ] && host="${this_host}" - -# ----------------------------------------------------------------------------- -# screen statuses we don't need to send a notification - -# don't do anything if this is not WARNING, CRITICAL or CLEAR -if [ "${status}" != "WARNING" -a "${status}" != "CRITICAL" -a "${status}" != "CLEAR" ] -then - info "not sending notification for ${status} of '${host}.${chart}.${name}'" - exit 1 -fi - -# don't do anything if this is CLEAR, but it was not WARNING or CRITICAL -if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ] -then - info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})" - exit 1 -fi - -# ----------------------------------------------------------------------------- -# load configuration - -# By default fetch images from the global public registry. -# This is required by default, since all notification methods need to download -# images via the Internet, and private registries might not be reachable. -# This can be overwritten at the configuration file. -images_base_url="https://registry.my-netdata.io" - -# curl options to use -curl_options="" - -# needed commands -# if empty they will be searched in the system path -curl= -sendmail= - -# enable / disable features -SEND_SLACK="YES" -SEND_MSTEAM="YES" -SEND_ALERTA="YES" -SEND_FLOCK="YES" -SEND_DISCORD="YES" -SEND_PUSHOVER="YES" -SEND_TWILIO="YES" -SEND_HIPCHAT="YES" -SEND_MESSAGEBIRD="YES" -SEND_KAVENEGAR="YES" -SEND_TELEGRAM="YES" -SEND_EMAIL="YES" -SEND_PUSHBULLET="YES" -SEND_KAFKA="YES" -SEND_PD="YES" -SEND_FLEEP="YES" -SEND_IRC="YES" -SEND_AWSSNS="YES" -SEND_SYSLOG="NO" -SEND_CUSTOM="YES" - -# slack configs -SLACK_WEBHOOK_URL= -DEFAULT_RECIPIENT_SLACK= -declare -A role_recipients_slack=() - -# Microsoft Team configs -MSTEAM_WEBHOOK_URL= -DEFAULT_RECIPIENT_MSTEAM= -declare -A role_recipients_msteam=() - -# rocketchat configs -ROCKETCHAT_WEBHOOK_URL= -DEFAULT_RECIPIENT_ROCKETCHAT= -declare -A role_recipients_rocketchat=() - -# alerta configs -ALERTA_WEBHOOK_URL= -ALERTA_API_KEY= -DEFAULT_RECIPIENT_ALERTA= -declare -A role_recipients_alerta=() - -# flock configs -FLOCK_WEBHOOK_URL= -DEFAULT_RECIPIENT_FLOCK= -declare -A role_recipients_flock=() - -# discord configs -DISCORD_WEBHOOK_URL= -DEFAULT_RECIPIENT_DISCORD= -declare -A role_recipients_discord=() - -# pushover configs -PUSHOVER_APP_TOKEN= -DEFAULT_RECIPIENT_PUSHOVER= -declare -A role_recipients_pushover=() - -# pushbullet configs -PUSHBULLET_ACCESS_TOKEN= -PUSHBULLET_SOURCE_DEVICE= -DEFAULT_RECIPIENT_PUSHBULLET= -declare -A role_recipients_pushbullet=() - -# twilio configs -TWILIO_ACCOUNT_SID= -TWILIO_ACCOUNT_TOKEN= -TWILIO_NUMBER= -DEFAULT_RECIPIENT_TWILIO= -declare -A role_recipients_twilio=() - -# hipchat configs -HIPCHAT_SERVER= -HIPCHAT_AUTH_TOKEN= -DEFAULT_RECIPIENT_HIPCHAT= -declare -A role_recipients_hipchat=() - -# messagebird configs -MESSAGEBIRD_ACCESS_KEY= -MESSAGEBIRD_NUMBER= -DEFAULT_RECIPIENT_MESSAGEBIRD= -declare -A role_recipients_messagebird=() - -# kavenegar configs -KAVENEGAR_API_KEY="" -KAVENEGAR_SENDER="" -DEFAULT_RECIPIENT_KAVENEGAR=() -declare -A role_recipients_kavenegar="" - -# telegram configs -TELEGRAM_BOT_TOKEN= -DEFAULT_RECIPIENT_TELEGRAM= -declare -A role_recipients_telegram=() - -# kafka configs -KAFKA_URL= -KAFKA_SENDER_IP= - -# pagerduty.com configs -PD_SERVICE_KEY= -DEFAULT_RECIPIENT_PD= -declare -A role_recipients_pd=() - -# fleep.io configs -FLEEP_SENDER="${host}" -DEFAULT_RECIPIENT_FLEEP= -declare -A role_recipients_fleep=() - -# Amazon SNS configs -DEFAULT_RECIPIENT_AWSSNS= -AWSSNS_MESSAGE_FORMAT= -declare -A role_recipients_awssns=() - -# syslog configs -SYSLOG_FACILITY= -declare -A role_recipients_syslog=() - -# custom configs -DEFAULT_RECIPIENT_CUSTOM= -declare -A role_recipients_custom=() - -# email configs -EMAIL_SENDER= -DEFAULT_RECIPIENT_EMAIL="root" -EMAIL_CHARSET=$(locale charmap 2>/dev/null) -EMAIL_THREADING= -declare -A role_recipients_email=() - -# irc configs -IRC_NICKNAME= -IRC_REALNAME= -DEFAULT_RECIPIENT_IRC= -IRC_NETWORK= -declare -A role_recipients_irc=() - -# load the stock and user configuration files -# these will overwrite the variables above - -for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf" -do - if [ -f "${CONFIG}" ] - then - debug "Loading config file '${CONFIG}'..." - source "${CONFIG}" - [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'." - else - warning "Cannot find file '${CONFIG}'." - fi -done - -# If we didn't autodetect the character set for e-mail and it wasn't -# set by the user, we need to set it to a reasonable default. UTF-8 -# should be correct for almost all modern UNIX systems. -if [ -z ${EMAIL_CHARSET} ] - then - EMAIL_CHARSET="UTF-8" -fi - -# ----------------------------------------------------------------------------- -# filter a recipient based on alarm event severity - -filter_recipient_by_criticality() { - local method="${1}" x="${2}" r s - shift - - r="${x/|*/}" # the recipient - s="${x/*|/}" # the severity required for notifying this recipient - - # no severity filtering for this person - [ "${r}" = "${s}" ] && return 0 - - # the severity is invalid - s="${s^^}" - if [ "${s}" != "CRITICAL" ] - then - error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported." - return 0 - fi - - # create the status tracking directory for this user - [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] && \ - mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" - - case "${status}" in - CRITICAL) - # make sure he will get future notifications for this alarm too - touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" - debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)" - return 0 - ;; - - WARNING) - if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ] - then - # we do not remove the file, so that he will get future notifications of this alarm - debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)" - return 0 - fi - ;; - - *) - if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ] - then - # remove the file, so that he will only receive notifications for CRITICAL states for this alarm - rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" - debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)" - return 0 - fi - ;; - esac - - debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification" - return 1 -} - -# ----------------------------------------------------------------------------- -# find the recipients' addresses per method - -declare -A arr_slack=() -declare -A arr_msteam=() -declare -A arr_rocketchat=() -declare -A arr_alerta=() -declare -A arr_flock=() -declare -A arr_discord=() -declare -A arr_pushover=() -declare -A arr_pushbullet=() -declare -A arr_twilio=() -declare -A arr_hipchat=() -declare -A arr_telegram=() -declare -A arr_pd=() -declare -A arr_email=() -declare -A arr_custom=() -declare -A arr_messagebird=() -declare -A arr_kavenegar=() -declare -A arr_fleep=() -declare -A arr_irc=() -declare -A arr_syslog=() -declare -A arr_awssns=() - -# netdata may call us with multiple roles, and roles may have multiple but -# overlapping recipients - so, here we find the unique recipients. -for x in ${roles//,/ } -do - # the roles 'silent' and 'disabled' mean: - # don't send a notification for this role - [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue - - # email - a="${role_recipients_email[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1" - done - - # pushover - a="${role_recipients_pushover[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1" - done - - # pushbullet - a="${role_recipients_pushbullet[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1" - done - - # twilio - a="${role_recipients_twilio[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TWILIO}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1" - done - - # hipchat - a="${role_recipients_hipchat[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_HIPCHAT}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality hipchat "${r}" && arr_hipchat[${r/|*/}]="1" - done - - # messagebird - a="${role_recipients_messagebird[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MESSAGEBIRD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality messagebird "${r}" && arr_messagebird[${r/|*/}]="1" - done - - # kavenegar - a="${role_recipients_kavenegar[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_KAVENEGAR}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality kavenegar "${r}" && arr_kavenegar[${r/|*/}]="1" - done - - # telegram - a="${role_recipients_telegram[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1" - done - - # slack - a="${role_recipients_slack[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1" - done - - # Microsoft Team - a="${role_recipients_msteam[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MSTEAM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality msteam "${r}" && arr_msteam[${r/|*/}]="1" - done - - # rocketchat - a="${role_recipients_rocketchat[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ROCKETCHAT}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality rocketchat "${r}" && arr_rocketchat[${r/|*/}]="1" - done - - # alerta - a="${role_recipients_alerta[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ALERTA}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality alerta "${r}" && arr_alerta[${r/|*/}]="1" - done - - # flock - a="${role_recipients_flock[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLOCK}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality flock "${r}" && arr_flock[${r/|*/}]="1" - done - - # discord - a="${role_recipients_discord[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_DISCORD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality discord "${r}" && arr_discord[${r/|*/}]="1" - done - - # pagerduty.com - a="${role_recipients_pd[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1" - done - - # fleep.io - a="${role_recipients_fleep[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLEEP}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality fleep "${r}" && arr_fleep[${r/|*/}]="1" - done - - # irc - a="${role_recipients_irc[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_IRC}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality irc "${r}" && arr_irc[${r/|*/}]="1" - done - - # amazon sns - a="${role_recipients_awssns[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_AWSSNS}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality awssns "${r}" && arr_awssns[${r/|*/}]="1" - done - - # syslog - a="${role_recipients_syslog[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SYSLOG}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality syslog "${r}" && arr_syslog[${r/|*/}]="1" - done - - # custom - a="${role_recipients_custom[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_CUSTOM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality custom "${r}" && arr_custom[${r/|*/}]="1" - done - -done - -# build the list of slack recipients (channels) -to_slack="${!arr_slack[*]}" -[ -z "${to_slack}" ] && SEND_SLACK="NO" - -# build the list of Microsoft team recipients (channels) -to_msteam="${!arr_msteam[*]}" -[ -z "${to_msteam}" ] && SEND_MSTEAM="NO" - -# build the list of rocketchat recipients (channels) -to_rocketchat="${!arr_rocketchat[*]}" -[ -z "${to_rocketchat}" ] && SEND_ROCKETCHAT="NO" - -# build the list of alerta recipients (channels) -to_alerta="${!arr_alerta[*]}" -[ -z "${to_alerta}" ] && SEND_ALERTA="NO" - -# build the list of flock recipients (channels) -to_flock="${!arr_flock[*]}" -[ -z "${to_flock}" ] && SEND_FLOCK="NO" - -# build the list of discord recipients (channels) -to_discord="${!arr_discord[*]}" -[ -z "${to_discord}" ] && SEND_DISCORD="NO" - -# build the list of pushover recipients (user tokens) -to_pushover="${!arr_pushover[*]}" -[ -z "${to_pushover}" ] && SEND_PUSHOVER="NO" - -# build the list of pushbulet recipients (user tokens) -to_pushbullet="${!arr_pushbullet[*]}" -[ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO" - -# build the list of twilio recipients (phone numbers) -to_twilio="${!arr_twilio[*]}" -[ -z "${to_twilio}" ] && SEND_TWILIO="NO" - -# build the list of hipchat recipients (rooms) -to_hipchat="${!arr_hipchat[*]}" -[ -z "${to_hipchat}" ] && SEND_HIPCHAT="NO" - -# build the list of messagebird recipients (phone numbers) -to_messagebird="${!arr_messagebird[*]}" -[ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO" - -# build the list of kavenegar recipients (phone numbers) -to_kavenegar="${!arr_kavenegar[*]}" -[ -z "${to_kavenegar}" ] && SEND_KAVENEGAR="NO" - -# check array of telegram recipients (chat ids) -to_telegram="${!arr_telegram[*]}" -[ -z "${to_telegram}" ] && SEND_TELEGRAM="NO" - -# build the list of pagerduty recipients (service keys) -to_pd="${!arr_pd[*]}" -[ -z "${to_pd}" ] && SEND_PD="NO" - -# build the list of fleep recipients (conversation webhooks) -to_fleep="${!arr_fleep[*]}" -[ -z "${to_fleep}" ] && SEND_FLEEP="NO" - -# build the list of custom recipients -to_custom="${!arr_custom[*]}" -[ -z "${to_custom}" ] && SEND_CUSTOM="NO" - -# build the list of email recipients (email addresses) -to_email= -for x in "${!arr_email[@]}" -do - [ ! -z "${to_email}" ] && to_email="${to_email}, " - to_email="${to_email}${x}" -done -[ -z "${to_email}" ] && SEND_EMAIL="NO" - -# build the list of irc recipients (channels) -to_irc="${!arr_irc[*]}" -[ -z "${to_irc}" ] && SEND_IRC="NO" - -# build the list of awssns recipients (facilities, servers, and prefixes) -to_awssns="${!arr_awssns[*]}" -[ -z "${to_awssns}" ] && SEND_AWSSNS="NO" - -# build the list of syslog recipients (facilities, servers, and prefixes) -to_syslog="${!arr_syslog[*]}" -[ -z "${to_syslog}" ] && SEND_SYSLOG="NO" - -# ----------------------------------------------------------------------------- -# verify the delivery methods supported - -# check slack -[ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO" - -# check rocketchat -[ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO" - -# check alerta -[ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO" - -# check flock -[ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO" - -# check discord -[ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO" - -# check pushover -[ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO" - -# check pushbullet -[ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO" - -# check twilio -[ -z "${TWILIO_ACCOUNT_TOKEN}" -o -z "${TWILIO_ACCOUNT_SID}" -o -z "${TWILIO_NUMBER}" ] && SEND_TWILIO="NO" - -# check hipchat -[ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO" - -# check messagebird -[ -z "${MESSAGEBIRD_ACCESS_KEY}" -o -z "${MESSAGEBIRD_NUMBER}" ] && SEND_MESSAGEBIRD="NO" - -# check kavenegar -[ -z "${KAVENEGAR_API_KEY}" -o -z "${KAVENEGAR_SENDER}" ] && SEND_KAVENEGAR="NO" - -# check telegram -[ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO" - -# check kafka -[ -z "${KAFKA_URL}" -o -z "${KAFKA_SENDER_IP}" ] && SEND_KAFKA="NO" - -# check irc -[ -z "${IRC_NETWORK}" ] && SEND_IRC="NO" - -# check fleep -[ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO" - -# check pagerduty.com -# if we need pd-send, check for the pd-send command -# https://www.pagerduty.com/docs/guides/agent-install-guide/ -if [ "${SEND_PD}" = "YES" ] - then - pd_send="$(which pd-send 2>/dev/null || command -v pd-send 2>/dev/null)" - if [ -z "${pd_send}" ] - then - error "Cannot find pd-send command in the system path. Disabling pagerduty.com notifications." - SEND_PD="NO" - fi -fi - -# if we need curl, check for the curl command -if [ \( \ - "${SEND_PUSHOVER}" = "YES" \ - -o "${SEND_SLACK}" = "YES" \ - -o "${SEND_ROCKETCHAT}" = "YES" \ - -o "${SEND_ALERTA}" = "YES" \ - -o "${SEND_FLOCK}" = "YES" \ - -o "${SEND_DISCORD}" = "YES" \ - -o "${SEND_HIPCHAT}" = "YES" \ - -o "${SEND_TWILIO}" = "YES" \ - -o "${SEND_MESSAGEBIRD}" = "YES" \ - -o "${SEND_KAVENEGAR}" = "YES" \ - -o "${SEND_TELEGRAM}" = "YES" \ - -o "${SEND_PUSHBULLET}" = "YES" \ - -o "${SEND_KAFKA}" = "YES" \ - -o "${SEND_FLEEP}" = "YES" \ - -o "${SEND_CUSTOM}" = "YES" \ - -o "${SEND_MSTEAM}" = "YES" \ - \) -a -z "${curl}" ] - then - curl="$(which curl 2>/dev/null || command -v curl 2>/dev/null)" - if [ -z "${curl}" ] - then - error "Cannot find curl command in the system path. Disabling all curl based notifications." - SEND_PUSHOVER="NO" - SEND_PUSHBULLET="NO" - SEND_TELEGRAM="NO" - SEND_SLACK="NO" - SEND_MSTEAM="NO" - SEND_ROCKETCHAT="NO" - SEND_ALERTA="NO" - SEND_FLOCK="NO" - SEND_DISCORD="NO" - SEND_TWILIO="NO" - SEND_HIPCHAT="NO" - SEND_MESSAGEBIRD="NO" - SEND_KAVENEGAR="NO" - SEND_KAFKA="NO" - SEND_FLEEP="NO" - SEND_CUSTOM="NO" - fi -fi - -# if we need sendmail, check for the sendmail command -if [ "${SEND_EMAIL}" = "YES" -a -z "${sendmail}" ] - then - sendmail="$(which sendmail 2>/dev/null || command -v sendmail 2>/dev/null)" - if [ -z "${sendmail}" ] - then - debug "Cannot find sendmail command in the system path. Disabling email notifications." - SEND_EMAIL="NO" - fi -fi - -# if we need logger, check for the logger command -if [ "${SEND_SYSLOG}" = "YES" -a -z "${logger}" ] - then - logger="$(which logger 2>/dev/null || command -v logger 2>/dev/null)" - if [ -z "${logger}" ] - then - debug "Cannot find logger command in the system path. Disabling syslog notifications." - SEND_SYSLOG="NO" - fi -fi - -# if we need aws, check for the aws command -if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ] - then - aws="$(which aws 2>/dev/null || command -v aws 2>/dev/null)" - if [ -z "${aws}" ] - then - debug "Cannot find aws command in the system path. Disabling Amazon SNS notifications." - SEND_AWSSNS="NO" - fi -fi - -# check that we have at least a method enabled -if [ "${SEND_EMAIL}" != "YES" \ - -a "${SEND_PUSHOVER}" != "YES" \ - -a "${SEND_TELEGRAM}" != "YES" \ - -a "${SEND_SLACK}" != "YES" \ - -a "${SEND_ROCKETCHAT}" != "YES" \ - -a "${SEND_ALERTA}" != "YES" \ - -a "${SEND_FLOCK}" != "YES" \ - -a "${SEND_DISCORD}" != "YES" \ - -a "${SEND_TWILIO}" != "YES" \ - -a "${SEND_HIPCHAT}" != "YES" \ - -a "${SEND_MESSAGEBIRD}" != "YES" \ - -a "${SEND_KAVENEGAR}" != "YES" \ - -a "${SEND_PUSHBULLET}" != "YES" \ - -a "${SEND_KAFKA}" != "YES" \ - -a "${SEND_PD}" != "YES" \ - -a "${SEND_FLEEP}" != "YES" \ - -a "${SEND_CUSTOM}" != "YES" \ - -a "${SEND_IRC}" != "YES" \ - -a "${SEND_AWSSNS}" != "YES" \ - -a "${SEND_SYSLOG}" != "YES" \ - -a "${SEND_MSTEAM}" != "YES" \ - ] - then - fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'." -fi - -# ----------------------------------------------------------------------------- -# get the date the alarm happened - -date=$(date --date=@${when} "${date_format}" 2>/dev/null) -[ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null) -[ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null) -[ -z "${date}" ] && date=$(date 2>/dev/null) - -# ---------------------------------------------------------------------------- -# prepare some extra headers if we've been asked to thread e-mails -if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" != "NO" ] ; then - email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\nReferences: <${chart}-${name}@${host}>" -else - email_thread_headers= -fi - -# ----------------------------------------------------------------------------- -# function to URL encode a string - -urlencode() { - local string="${1}" strlen encoded pos c o - - strlen=${#string} - for (( pos=0 ; pos<strlen ; pos++ )) - do - c=${string:${pos}:1} - case "${c}" in - [-_.~a-zA-Z0-9]) - o="${c}" - ;; - - *) - printf -v o '%%%02x' "'${c}" - ;; - esac - encoded+="${o}" - done - - REPLY="${encoded}" - echo "${REPLY}" -} - -# ----------------------------------------------------------------------------- -# function to convert a duration in seconds, to a human readable duration -# using DAYS, MINUTES, SECONDS - -duration4human() { - local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret - d=$(( s / 86400 )) - s=$(( s - (d * 86400) )) - h=$(( s / 3600 )) - s=$(( s - (h * 3600) )) - m=$(( s / 60 )) - s=$(( s - (m * 60) )) - - if [ ${d} -gt 0 ] - then - [ ${m} -ge 30 ] && h=$(( h + 1 )) - [ ${d} -gt 1 ] && ds="days" - [ ${h} -gt 1 ] && hs="hours" - if [ ${h} -gt 0 ] - then - ret="${d} ${ds} and ${h} ${hs}" - else - ret="${d} ${ds}" - fi - elif [ ${h} -gt 0 ] - then - [ ${s} -ge 30 ] && m=$(( m + 1 )) - [ ${h} -gt 1 ] && hs="hours" - [ ${m} -gt 1 ] && ms="minutes" - if [ ${m} -gt 0 ] - then - ret="${h} ${hs} and ${m} ${ms}" - else - ret="${h} ${hs}" - fi - elif [ ${m} -gt 0 ] - then - [ ${m} -gt 1 ] && ms="minutes" - [ ${s} -gt 1 ] && ss="seconds" - if [ ${s} -gt 0 ] - then - ret="${m} ${ms} and ${s} ${ss}" - else - ret="${m} ${ms}" - fi - else - [ ${s} -gt 1 ] && ss="seconds" - ret="${s} ${ss}" - fi - - REPLY="${ret}" - echo "${REPLY}" -} - -# ----------------------------------------------------------------------------- -# email sender - -send_email() { - local ret= opts=() sender_email="${EMAIL_SENDER}" sender_name= - if [ "${SEND_EMAIL}" = "YES" ] - then - - if [ ! -z "${EMAIL_SENDER}" ] - then - if [[ "${EMAIL_SENDER}" =~ ^\".*\"\ \<.*\>$ ]] - then - # the name includes double quotes - sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)" - sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)" - elif [[ "${EMAIL_SENDER}" =~ ^\'.*\'\ \<.*\>$ ]] - then - # the name includes single quotes - sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)" - sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)" - elif [[ "${EMAIL_SENDER}" =~ ^.*\ \<.*\>$ ]] - then - # the name does not have any quotes - sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)" - sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)" - fi - fi - - [ ! -z "${sender_email}" ] && opts+=(-f "${sender_email}") - [ ! -z "${sender_name}" ] && opts+=(-F "${sender_name}") - - if [ "${debug}" = "1" ] - then - echo >&2 "--- BEGIN sendmail command ---" - printf >&2 "%q " "${sendmail}" -t "${opts[@]}" - echo >&2 - echo >&2 "--- END sendmail command ---" - fi - - "${sendmail}" -t "${opts[@]}" - ret=$? - - if [ ${ret} -eq 0 ] - then - info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'" - return 0 - else - error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}." - return 1 - fi - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# pushover sender - -send_pushover() { - local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority - - if [ "${SEND_PUSHOVER}" = "YES" -a ! -z "${apptoken}" -a ! -z "${usertokens}" -a ! -z "${title}" -a ! -z "${message}" ] - then - - # https://pushover.net/api - priority=-2 - case "${status}" in - CLEAR) priority=-1;; # low priority: no sound or vibration - WARNING) priority=0;; # normal priority: respect quiet hours - CRITICAL) priority=1;; # high priority: bypass quiet hours - *) priority=-2;; # lowest priority: no notification at all - esac - - for user in ${usertokens} - do - httpcode=$(docurl \ - --form-string "token=${apptoken}" \ - --form-string "user=${user}" \ - --form-string "html=1" \ - --form-string "title=${title}" \ - --form-string "message=${message}" \ - --form-string "timestamp=${when}" \ - --form-string "url=${url}" \ - --form-string "url_title=Open netdata dashboard to view the alarm" \ - --form-string "priority=${priority}" \ - https://api.pushover.net/1/messages.json) - - if [ "${httpcode}" = "200" ] - then - info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'" - sent=$((sent + 1)) - else - error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# pushbullet sender - -send_pushbullet() { - local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user - if [ "${SEND_PUSHBULLET}" = "YES" -a ! -z "${userapikey}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ] - then - #https://docs.pushbullet.com/#create-push - for user in ${recipients} - do - httpcode=$(docurl \ - --header 'Access-Token: '${userapikey}'' \ - --header 'Content-Type: application/json' \ - --data-binary @<(cat <<EOF - {"title": "${title}", - "type": "link", - "email": "${user}", - "body": "$( echo -n ${message})", - "url": "${url}", - "source_device_iden": "${source_device}"} -EOF - ) "https://api.pushbullet.com/v2/pushes" -X POST) - - if [ "${httpcode}" = "200" ] - then - info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'" - sent=$((sent + 1)) - else - error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# kafka sender - -send_kafka() { - local httpcode sent=0 - if [ "${SEND_KAFKA}" = "YES" ] - then - httpcode=$(docurl -X POST \ - --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \ - "${KAFKA_URL}") - - if [ "${httpcode}" = "204" ] - then - info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'" - sent=$((sent + 1)) - else - error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP error code ${httpcode}." - fi - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# pagerduty.com sender - -send_pd() { - local recipients="${1}" sent=0 - unset t - case ${status} in - CLEAR) t='resolve';; - WARNING) t='trigger';; - CRITICAL) t='trigger';; - esac - - if [ ${SEND_PD} = "YES" -a ! -z "${t}" ] - then - for PD_SERVICE_KEY in ${recipients} - do - d="${status} ${name} = ${value_string} - ${host}, ${family}" - ${pd_send} -k ${PD_SERVICE_KEY} \ - -t ${t} \ - -d "${d}" \ - -i ${host}:${chart}:${name} \ - -f 'info'="${info}" \ - -f 'value_w_units'="${value_string}" \ - -f 'when'="${when}" \ - -f 'duration'="${duration}" \ - -f 'roles'="${roles}" \ - -f 'host'="${host}" \ - -f 'unique_id'="${unique_id}" \ - -f 'alarm_id'="${alarm_id}" \ - -f 'event_id'="${event_id}" \ - -f 'name'="${name}" \ - -f 'chart'="${chart}" \ - -f 'family'="${family}" \ - -f 'status'="${status}" \ - -f 'old_status'="${old_status}" \ - -f 'value'="${value}" \ - -f 'old_value'="${old_value}" \ - -f 'src'="${src}" \ - -f 'non_clear_duration'="${non_clear_duration}" \ - -f 'units'="${units}" - retval=$? - if [ ${retval} -eq 0 ] - then - info "sent pagerduty.com notification for host ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}....: ${d}" - sent=$((sent + 1)) - else - error "failed to send pagerduty.com notification for ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}.... (error code ${retval}): ${d}" - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# twilio sender - -send_twilio() { - local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user - if [ "${SEND_TWILIO}" = "YES" -a ! -z "${accountsid}" -a ! -z "${accounttoken}" -a ! -z "${twilionumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ] - then - #https://www.twilio.com/packages/labs/code/bash/twilio-sms - for user in ${recipients} - do - httpcode=$(docurl -X POST \ - --data-urlencode "From=${twilionumber}" \ - --data-urlencode "To=${user}" \ - --data-urlencode "Body=${title} ${message}" \ - -u "${accountsid}:${accounttoken}" \ - "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json") - - if [ "${httpcode}" = "201" ] - then - info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'" - sent=$((sent + 1)) - else - error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - - -# ----------------------------------------------------------------------------- -# hipchat sender - -send_hipchat() { - local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color sender msg_format notify - - # remove <small></small> from the message - message="${message//<small>/}" - message="${message//<\/small>/}" - - if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${HIPCHAT_SERVER}" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ] - then - # A label to be shown in addition to the sender's name - # Valid length range: 0 - 64. - sender="netdata" - - # Valid values: html, text. - # Defaults to 'html'. - msg_format="html" - - # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'. - case "${status}" in - WARNING) color="yellow" ;; - CRITICAL) color="red" ;; - CLEAR) color="green" ;; - *) color="gray" ;; - esac - - # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc). - # Each recipient's notification preferences are taken into account. - # Defaults to false. - notify="true" - - for room in ${recipients} - do - httpcode=$(docurl -X POST \ - -H "Content-type: application/json" \ - -H "Authorization: Bearer ${authtoken}" \ - -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \ - "https://${HIPCHAT_SERVER}/v2/room/${room}/notification") - - if [ "${httpcode}" = "204" ] - then - info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'" - sent=$((sent + 1)) - else - error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - - -# ----------------------------------------------------------------------------- -# messagebird sender - -send_messagebird() { - local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user - if [ "${SEND_MESSAGEBIRD}" = "YES" -a ! -z "${accesskey}" -a ! -z "${messagebirdnumber}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ] - then - #https://developers.messagebird.com/docs/messaging - for user in ${recipients} - do - httpcode=$(docurl -X POST \ - --data-urlencode "originator=${messagebirdnumber}" \ - --data-urlencode "recipients=${user}" \ - --data-urlencode "body=${title} ${message}" \ - --data-urlencode "datacoding=auto" \ - -H "Authorization: AccessKey ${accesskey}" \ - "https://rest.messagebird.com/messages") - - if [ "${httpcode}" = "201" ] - then - info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'" - sent=$((sent + 1)) - else - error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# kavenegar sender - -send_kavenegar() { - local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user - if [ "${SEND_KAVENEGAR}" = "YES" -a ! -z "${API_KEY}" -a ! -z "${kavenegarsender}" -a ! -z "${recipients}" -a ! -z "${message}" -a ! -z "${title}" ] - then - # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json - for user in ${recipients} - do - httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \ - --data-urlencode "sender=${kavenegarsender}" \ - --data-urlencode "receptor=${user}" \ - --data-urlencode "message=${title} ${message}") - - if [ "${httpcode}" = "201" ] - then - info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'" - sent=$((sent + 1)) - else - error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# telegram sender - -send_telegram() { - local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification="" - - if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi - - case "${status}" in - WARNING) emoji="⚠️" ;; - CRITICAL) emoji="🔴" ;; - CLEAR) emoji="✅" ;; - *) emoji="⚪️" ;; - esac - - if [ "${SEND_TELEGRAM}" = "YES" -a ! -z "${bottoken}" -a ! -z "${chatids}" -a ! -z "${message}" ]; - then - for chatid in ${chatids} - do - # https://core.telegram.org/bots/api#sendmessage - httpcode=$(docurl ${disableNotification} \ - --data-urlencode "parse_mode=HTML" \ - --data-urlencode "disable_web_page_preview=true" \ - --data-urlencode "text=${emoji} ${message}" \ - "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}") - - if [ "${httpcode}" = "200" ] - then - info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'" - sent=$((sent + 1)) - elif [ "${httpcode}" = "401" ] - then - error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token." - else - error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# Microsoft Team sender - -send_msteam() { - - local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload - - [ "${SEND_MSTEAM}" != "YES" ] && return 1 - - case "${status}" in - WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}";; - CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}";; - CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}";; - *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}";; - esac - - for channel in ${channels} - do - ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference - ## Online designer : https://acdesignerbeta.azurewebsites.net/ - payload="$(cat <<EOF - { - "@context": "http://schema.org/extensions", - "@type": "MessageCard", - "themeColor": "${color}", - "title": "$icon Alert ${status} from netdata for ${host}", - "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*", - "potentialAction": [ - { - "@type": "OpenUri", - "name": "Netdata", - "targets": [ - { "os": "default", "uri": "${goto_url}" } - ] - } - ] - } -EOF - )" - - # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file. - webhook="${webhook//CHANNEL/${channel}}" - - httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}") - - if [ "${httpcode}" = "200" ] - then - info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'" - sent=$((sent + 1)) - else - error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - - -# slack sender - -send_slack() { - local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload - - [ "${SEND_SLACK}" != "YES" ] && return 1 - - case "${status}" in - WARNING) color="warning" ;; - CRITICAL) color="danger" ;; - CLEAR) color="good" ;; - *) color="#777777" ;; - esac - - for channel in ${channels} - do - payload="$(cat <<EOF - { - "channel": "#${channel}", - "username": "netdata on ${host}", - "icon_url": "${images_base_url}/images/banner-icon-144x144.png", - "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*", - "attachments": [ - { - "fallback": "${alarm} - ${chart} (${family}) - ${info}", - "color": "${color}", - "title": "${alarm}", - "title_link": "${goto_url}", - "text": "${info}", - "fields": [ - { - "title": "${chart}", - "short": true - }, - { - "title": "${family}", - "short": true - } - ], - "thumb_url": "${image}", - "footer": "by <${goto_url}|${this_host}>", - "ts": ${when} - } - ] - } -EOF - )" - - httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}") - if [ "${httpcode}" = "200" ] - then - info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" - sent=$((sent + 1)) - else - error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - - -# ----------------------------------------------------------------------------- -# rocketchat sender - -send_rocketchat() { - local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload - - [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1 - - case "${status}" in - WARNING) color="warning" ;; - CRITICAL) color="danger" ;; - CLEAR) color="good" ;; - *) color="#777777" ;; - esac - - for channel in ${channels} - do - payload="$(cat <<EOF - { - "channel": "#${channel}", - "alias": "netdata on ${host}", - "avatar": "${images_base_url}/images/banner-icon-144x144.png", - "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*", - "attachments": [ - { - "color": "${color}", - "title": "${alarm}", - "title_link": "${goto_url}", - "text": "${info}", - "fields": [ - { - "title": "${chart}", - "short": true, - "value": "chart" - }, - { - "title": "${family}", - "short": true, - "value": "family" - } - ], - "thumb_url": "${image}", - "ts": "${when}" - } - ] - } -EOF - )" - - httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}") - if [ "${httpcode}" = "200" ] - then - info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" - sent=$((sent + 1)) - else - error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - -# ----------------------------------------------------------------------------- -# alerta sender - -send_alerta() { - local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth - - [ "${SEND_ALERTA}" != "YES" ] && return 1 - - case "${status}" in - CRITICAL) severity="critical" ;; - WARNING) severity="warning" ;; - CLEAR) severity="cleared" ;; - *) severity="indeterminate" ;; - esac - - if [[ "${chart}" == httpcheck* ]] - then - resource=$chart - event=$name - else - resource="${host}:${family}" - event="${chart}.${name}" - fi - - for channel in ${channels} - do - payload="$(cat <<EOF - { - "resource": "${resource}", - "event": "${event}", - "environment": "${channel}", - "severity": "${severity}", - "service": ["Netdata"], - "group": "Performance", - "value": "${value_string}", - "text": "${info}", - "tags": ["alarm_id:${alarm_id}"], - "attributes": { - "roles": "${roles}", - "name": "${name}", - "chart": "${chart}", - "family": "${family}", - "source": "${src}", - "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>" - }, - "origin": "netdata/${this_host}", - "type": "netdataAlarm", - "rawData": "${BASH_ARGV[@]}" - } -EOF - )" - - if [[ -n "${ALERTA_API_KEY}" ]] - then - auth="Key ${ALERTA_API_KEY}" - fi - - httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}") - - if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]] - then - info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" - sent=$((sent + 1)) - elif [[ "${httpcode}" = "202" ]] - then - info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" - else - error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - -# ----------------------------------------------------------------------------- -# flock sender - -send_flock() { - local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload - - [ "${SEND_FLOCK}" != "YES" ] && return 1 - - case "${status}" in - WARNING) color="warning" ;; - CRITICAL) color="danger" ;; - CLEAR) color="good" ;; - *) color="#777777" ;; - esac - - for channel in ${channels} - do - httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{ - \"sendAs\": { - \"name\" : \"netdata on ${host}\", - \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\" - }, - \"text\": \"${host} *${status_message}*\", - \"timestamp\": \"${when}\", - \"attachments\": [ - { - \"description\": \"${chart} (${family}) - ${info}\", - \"color\": \"${color}\", - \"title\": \"${alarm}\", - \"url\": \"${goto_url}\", - \"text\": \"${info}\", - \"views\": { - \"image\": { - \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 }, - \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 }, - \"filename\": \"${image}\" - } - } - } - ] - }" ) - if [ "${httpcode}" = "200" ] - then - info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" - sent=$((sent + 1)) - else - error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - -# ----------------------------------------------------------------------------- -# discord sender - -send_discord() { - local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username - - [ "${SEND_DISCORD}" != "YES" ] && return 1 - - case "${status}" in - WARNING) color="warning" ;; - CRITICAL) color="danger" ;; - CLEAR) color="good" ;; - *) color="#777777" ;; - esac - - for channel in ${channels} - do - username="netdata on ${host}" - [ ${#username} -gt 32 ] && username="${username:0:29}..." - - payload="$(cat <<EOF - { - "channel": "#${channel}", - "username": "${username}", - "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*", - "icon_url": "${images_base_url}/images/banner-icon-144x144.png", - "attachments": [ - { - "color": "${color}", - "title": "${alarm}", - "title_link": "${goto_url}", - "text": "${info}", - "fields": [ - { - "title": "${chart}", - "value": "${family}" - } - ], - "thumb_url": "${image}", - "footer_icon": "${images_base_url}/images/banner-icon-144x144.png", - "footer": "${this_host}", - "ts": ${when} - } - ] - } -EOF - )" - - httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}") - if [ "${httpcode}" = "200" ] - then - info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" - sent=$((sent + 1)) - else - error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - -# ----------------------------------------------------------------------------- -# fleep sender - -send_fleep() { - local httpcode sent=0 webhooks="${1}" data message - if [ "${SEND_FLEEP}" = "YES" ] ; then - message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}" - - for hook in "${webhooks}" ; do - data="{ " - data="${data} 'message': '${message}', " - data="${data} 'user': '${FLEEP_SENDER}' " - data="${data} }" - - httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}") - - if [ "${httpcode}" = "200" ] ; then - info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'" - sent=$((sent + 1)) - else - error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP error code ${httpcode}." - fi - done - - [ ${sent} -gt 0 ] && return 0 - fi - - return 1 -} - -# ----------------------------------------------------------------------------- -# irc sender - -send_irc() { - local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error - - if [ "${SEND_IRC}" = "YES" -a ! -z "${NICKNAME}" -a ! -z "${REALNAME}" -a ! -z "${CHANNELS}" -a ! -z "${NETWORK}" -a ! -z "${SERVERNAME}" ] - then - case "${status}" in - WARNING) color="warning" ;; - CRITICAL) color="danger" ;; - CLEAR) color="good" ;; - *) color="#777777" ;; - esac - - for CHANNEL in ${CHANNELS} - do - error=0 - send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\nNICK ${NICKNAME}\nJOIN ${CHANNEL}\nPRIVMSG ${CHANNEL} :${MESSAGE}\nQUIT\n" \ | nc ${NETWORK} 6667) - reply_codes=$(echo ${send_alarm} | cut -d ' ' -f 2 | grep -o '[0-9]*') - for code in ${reply_codes} - do - [ "${code}" -ge 400 -a "${code}" -le 599 ] && error=1 && break - done - - if [ "${error}" -eq 0 ] - then - info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'" - sent=$((sent + 1)) - else - error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}." - fi - done - fi - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - -# ----------------------------------------------------------------------------- -# Amazon SNS sender - -send_awssns() { - local targets="${1}" message='' sent=0 region='' - local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}" - - [ "${SEND_AWSSNS}" = "YES" ] || return 1 - - message=${AWSSNS_MESSAGE_FORMAT:-${default_format}} - - for target in ${targets} ; do - # Extract the region from the target ARN. We need to explicitly specify the region so that it matches up correctly. - region="$(echo ${target} | cut -f 4 -d ':')" - ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null - if [ $? = 0 ]; then - info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" - sent=$((sent + 1)) - else - error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" - fi - done - - [ ${sent} -gt 0 ] && return 0 - - return 1 -} - -# ----------------------------------------------------------------------------- -# syslog sender - -send_syslog() { - local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}" - local priority='' message='' host='' port='' prefix='' - local temp1='' temp2='' - - [ "${SEND_SYSLOG}" = "YES" ] || return 1 - - if [ "${status}" = "CRITICAL" ] ; then - level='crit' - elif [ "${status}" = "WARNING" ] ; then - level='warning' - fi - - for target in ${targets} ; do - priority="${facility}.${level}" - message='' - host='' - port='' - prefix='' - temp1='' - temp2='' - - prefix=$(echo ${target} | cut -d '/' -f 2) - temp1=$(echo ${target} | cut -d '/' -f 1) - - if [ ${prefix} != ${temp1} ] ; then - if (echo ${temp1} | grep -q '@' ) ; then - temp2=$(echo ${temp1} | cut -d '@' -f 1) - host=$(echo ${temp1} | cut -d '@' -f 2) - - if [ ${temp2} != ${host} ] ; then - priority=${temp2} - fi - - port=$(echo ${host} | rev | cut -d ':' -f 1 | rev) - - if ( echo ${host} | grep -E -q '\[.*\]' ) ; then - if ( echo ${port} | grep -q ']' ) ; then - port='' - else - host=$(echo ${host} | rev | cut -d ':' -f 2- | rev) - fi - else - if [ ${port} = ${host} ] ; then - port='' - else - host=$(echo ${host} | cut -d ':' -f 1) - fi - fi - else - priority=${temp1} - fi - fi - - message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}" - - if [ ${host} ] ; then - logger_options="${logger_options} -n ${host}" - if [ ${port} ] ; then - logger_options="${logger_options} -P ${port}" - fi - fi - - ${logger} -p ${priority} ${logger_options} "${message}" - done - - return $? -} - - -# ----------------------------------------------------------------------------- -# prepare the content of the notification - -# the url to send the user on click -urlencode "${host}" >/dev/null; url_host="${REPLY}" -urlencode "${chart}" >/dev/null; url_chart="${REPLY}" -urlencode "${family}" >/dev/null; url_family="${REPLY}" -urlencode "${name}" >/dev/null; url_name="${REPLY}" -goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}" - -# the severity of the alarm -severity="${status}" - -# the time the alarm was raised -duration4human ${duration} >/dev/null; duration_txt="${REPLY}" -duration4human ${non_clear_duration} >/dev/null; non_clear_duration_txt="${REPLY}" -raised_for="(was ${old_status,,} for ${duration_txt})" - -# the key status message -status_message="status unknown" - -# the color of the alarm -color="grey" - -# the alarm value -alarm="${name//_/ } = ${value_string}" - -# the image of the alarm -image="${images_base_url}/images/banner-icon-144x144.png" - -# prepare the title based on status -case "${status}" in - CRITICAL) - image="${images_base_url}/images/alert-128-red.png" - status_message="is critical" - color="#ca414b" - ;; - - WARNING) - image="${images_base_url}/images/alert-128-orange.png" - status_message="needs attention" - color="#ffc107" - ;; - - CLEAR) - image="${images_base_url}/images/check-mark-2-128-green.png" - status_message="recovered" - color="#77ca6d" - ;; -esac - -if [ "${status}" = "CLEAR" ] -then - severity="Recovered from ${old_status}" - if [ ${non_clear_duration} -gt ${duration} ] - then - raised_for="(alarm was raised for ${non_clear_duration_txt})" - fi - - # don't show the value when the status is CLEAR - # for certain alarms, this value might not have any meaning - alarm="${name//_/ } ${raised_for}" - -elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ] -then - severity="Escalated to ${status}" - if [ ${non_clear_duration} -gt ${duration} ] - then - raised_for="(alarm is raised for ${non_clear_duration_txt})" - fi - -elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ] -then - severity="Demoted to ${status}" - if [ ${non_clear_duration} -gt ${duration} ] - then - raised_for="(alarm is raised for ${non_clear_duration_txt})" - fi - -else - raised_for= -fi - -# prepare HTML versions of elements -info_html= -[ ! -z "${info}" ] && info_html=" <small><br/>${info}</small>" - -raised_for_html= -[ ! -z "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>" - -# ----------------------------------------------------------------------------- -# send the slack notification - -# slack aggregates posts from the same username -# so we use "${host} ${status}" as the bot username, to make them diff - -send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}" -SENT_SLACK=$? - -# ----------------------------------------------------------------------------- -# send the Microsoft notification - -# Microsoft team aggregates posts from the same username -# so we use "${host} ${status}" as the bot username, to make them diff - -send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}" -SENT_MSTEAM=$? - -# ----------------------------------------------------------------------------- -# send the rocketchat notification - -# rocketchat aggregates posts from the same username -# so we use "${host} ${status}" as the bot username, to make them diff - -send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}" -SENT_ROCKETCHAT=$? - -# ----------------------------------------------------------------------------- -# send the alerta notification - -# alerta aggregates posts from the same username -# so we use "${host} ${status}" as the bot username, to make them diff - -send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}" -SENT_ALERTA=$? - -# ----------------------------------------------------------------------------- -# send the flock notification - -# flock aggregates posts from the same username -# so we use "${host} ${status}" as the bot username, to make them diff - -send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}" -SENT_FLOCK=$? - -# ----------------------------------------------------------------------------- -# send the discord notification - -# discord aggregates posts from the same username -# so we use "${host} ${status}" as the bot username, to make them diff - -send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}" -SENT_DISCORD=$? - -# ----------------------------------------------------------------------------- -# send the pushover notification - -send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" " -<font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/> -<small><b>${chart}</b><br/>Chart<br/> </small> -<small><b>${family}</b><br/>Family<br/> </small> -<small><b>${severity}</b><br/>Severity<br/> </small> -<small><b>${date}${raised_for_html}</b><br/>Time<br/> </small> -<a href=\"${goto_url}\">View Netdata</a><br/> -<small><small>The source of this alarm is line ${src}</small></small> -" - -SENT_PUSHOVER=$? - -# ----------------------------------------------------------------------------- -# send the pushbullet notification - -send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\n -Severity: ${severity}\n -Chart: ${chart}\n -Family: ${family}\n -$(date -d @${when})\n -The source of this alarm is line ${src}" - -SENT_PUSHBULLET=$? - -# ----------------------------------------------------------------------------- -# send the twilio SMS - -send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} -Severity: ${severity} -Chart: ${chart} -Family: ${family} -${info}" - -SENT_TWILIO=$? - -# ----------------------------------------------------------------------------- -# send the messagebird SMS - -send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} -Severity: ${severity} -Chart: ${chart} -Family: ${family} -${info}" - -SENT_MESSAGEBIRD=$? - - -# ----------------------------------------------------------------------------- -# send the kavenegar SMS - -send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} -Severity: ${severity} -Chart: ${chart} -Family: ${family} -${info}" - -SENT_KAVENEGAR=$? - - -# ----------------------------------------------------------------------------- -# send the telegram.org message - -# https://core.telegram.org/bots/api#formatting-options -send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b> -${chart} (${family}) -<a href=\"${goto_url}\">${alarm}</a> -<i>${info}</i>" - -SENT_TELEGRAM=$? - - -# ----------------------------------------------------------------------------- -# send the kafka message - -send_kafka -SENT_KAFKA=$? - - -# ----------------------------------------------------------------------------- -# send the pagerduty.com message - -send_pd "${to_pd}" -SENT_PD=$? - -# ----------------------------------------------------------------------------- -# send the fleep message - -send_fleep "${to_fleep}" -SENT_FLEEP=$? - -# ----------------------------------------------------------------------------- -# send the irc message - -send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm} -Severity: ${severity} -Chart: ${chart} -Family: ${family} -${info}" - -SENT_IRC=$? - -# ----------------------------------------------------------------------------- -# send the custom message - -send_custom() { - # is it enabled? - [ "${SEND_CUSTOM}" != "YES" ] && return 1 - - # do we have any sender? - [ -z "${1}" ] && return 1 - - # call the custom_sender function - custom_sender "${@}" -} - -send_custom "${to_custom}" -SENT_CUSTOM=$? - - -# ----------------------------------------------------------------------------- -# send hipchat message - -send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \ -${host} ${status_message}<br/> \ -<b>${alarm}</b> ${info_html}<br/> \ -<b>${chart}</b> (family <b>${family}</b>)<br/> \ -<b>${date}${raised_for_html}</b><br/> \ -<a href=\\\"${goto_url}\\\">View netdata dashboard</a> \ -(source of alarm ${src}) \ -" - -SENT_HIPCHAT=$? - - -# ----------------------------------------------------------------------------- -# send the Amazon SNS message - -send_awssns ${to_awssns} - -SENT_AWSSNS=$? - - -# ----------------------------------------------------------------------------- -# send the syslog message - -send_syslog ${to_syslog} - -SENT_SYSLOG=$? - - -# ----------------------------------------------------------------------------- -# send the email - -send_email <<EOF -To: ${to_email} -Subject: ${host} ${status_message} - ${name//_/ } - ${chart} -MIME-Version: 1.0 -Content-Type: multipart/alternative; boundary="multipart-boundary" -${email_thread_headers} - -This is a MIME-encoded multipart message - ---multipart-boundary -Content-Type: text/plain; encoding=${EMAIL_CHARSET} -Content-Disposition: inline -Content-Transfer-Encoding: 8bit - -${host} ${status_message} - -${alarm} ${info} -${raised_for} - -Chart : ${chart} -Family : ${family} -Severity: ${severity} -URL : ${goto_url} -Source : ${src} -Date : ${date} -Notification generated on ${this_host} - ---multipart-boundary -Content-Type: text/html; encoding=${EMAIL_CHARSET} -Content-Disposition: inline -Content-Transfer-Encoding: 8bit - -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;"> -<body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;"> -<table> - <tbody> - <tr> - <td style="vertical-align: top;" valign="top"></td> - <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top"> - <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;"> - <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;"> - <tbody> - <tr> - <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;"> - <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div> - </td> - </tr> - <tr> - <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top"> - <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1> - </td> - </tr> - <tr> - <td style="vertical-align: top;" valign="top"> - <div style="margin: 0; padding: 20px; max-width: 700px;"> - <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px"> - <tbody> - <tr> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top"> - <span>${chart}</span> - <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span> - </td> - </tr> - <tr style="margin: 0; padding: 0;"> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> - <span><b>${alarm}</b>${info_html}</span> - <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span> - </td> - </tr> - <tr> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> - <span>${family}</span> - <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span> - </td> - </tr> - <tr style="margin: 0; padding: 0;"> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> - <span>${severity}</span> - <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span> - </td> - </tr> - <tr style="margin: 0; padding: 0;"> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span> - <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span> - </td> - </tr> - <tr style="margin: 0; padding: 0;"> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;"> - <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a> - </td> - </tr> - <tr style="text-align: center; margin: 0; padding: 0;"> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs) - </td> - </tr> - <tr style="text-align: center; margin: 0; padding: 0;"> - <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by - <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${this_host}</code>. - </td> - </tr> - </tbody> - </table> - </div> - </td> - </tr> - </tbody> - </table> - </div> - </td> - </tr> - </tbody> -</table> -</body> -</html> ---multipart-boundary-- -EOF - -SENT_EMAIL=$? - -# ----------------------------------------------------------------------------- -# let netdata know - -if [ ${SENT_EMAIL} -eq 0 \ - -o ${SENT_PUSHOVER} -eq 0 \ - -o ${SENT_TELEGRAM} -eq 0 \ - -o ${SENT_SLACK} -eq 0 \ - -o ${SENT_MSTEAM} -eq 0 \ - -o ${SENT_ROCKETCHAT} -eq 0 \ - -o ${SENT_ALERTA} -eq 0 \ - -o ${SENT_FLOCK} -eq 0 \ - -o ${SENT_DISCORD} -eq 0 \ - -o ${SENT_TWILIO} -eq 0 \ - -o ${SENT_HIPCHAT} -eq 0 \ - -o ${SENT_MESSAGEBIRD} -eq 0 \ - -o ${SENT_KAVENEGAR} -eq 0 \ - -o ${SENT_PUSHBULLET} -eq 0 \ - -o ${SENT_KAFKA} -eq 0 \ - -o ${SENT_PD} -eq 0 \ - -o ${SENT_FLEEP} -eq 0 \ - -o ${SENT_IRC} -eq 0 \ - -o ${SENT_AWSSNS} -eq 0 \ - -o ${SENT_CUSTOM} -eq 0 \ - -o ${SENT_SYSLOG} -eq 0 \ - ] - then - # we did send something - exit 0 -fi - -# we did not send anything -exit 1 diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in index ea8223097..dd3cda917 100755 --- a/health/notifications/alarm-notify.sh.in +++ b/health/notifications/alarm-notify.sh.in @@ -27,6 +27,7 @@ # - messagebird.com notifications by @tech_no_logical #1453 # - hipchat notifications by @ktsaou #1561 # - fleep notifications by @Ferroin +# - prowlapp.com notifications by @Ferroin # - custom notifications by @ktsaou # - syslog messages by @Ferroin # - Microsoft Team notification by @tioumen @@ -54,7 +55,7 @@ then echo >&2 echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}" - "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" + "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0 if [ $? -ne 0 ] then echo >&2 "# FAILED" @@ -143,6 +144,31 @@ docurl() { } # ----------------------------------------------------------------------------- +# List of all the notification mechanisms we support. +# Used in a couple of places to write more compact code. + +method_names=" +email +pushover +pushbullet +telegram +slack +alerta +flock +discord +hipchat +twilio +messagebird +pd +fleep +syslog +custom +msteam +kavenegar +prowl +" + +# ----------------------------------------------------------------------------- # this is to be overwritten by the config file custom_sender() { @@ -167,32 +193,51 @@ custom_sender() { # ----------------------------------------------------------------------------- # parse command line parameters -roles="${1}" # the roles that should be notified for this event -host="${2}" # the host generated this event -unique_id="${3}" # the unique id of this event -alarm_id="${4}" # the unique id of the alarm that generated this event -event_id="${5}" # the incremental id of the event, for this alarm id -when="${6}" # the timestamp this event occurred -name="${7}" # the name of the alarm, as given in netdata health.d entries -chart="${8}" # the name of the chart (type.id) -family="${9}" # the family of the chart -status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL -old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL -value="${12}" # the current value of the alarm -old_value="${13}" # the previous value of the alarm -src="${14}" # the line number and file the alarm has been configured -duration="${15}" # the duration in seconds of the previous alarm state -non_clear_duration="${16}" # the total duration in seconds this is/was non-clear -units="${17}" # the units of the value -info="${18}" # a short description of the alarm -value_string="${19}" # friendly value (with units) -old_value_string="${20}" # friendly old value (with units) +if [ ${1} = "unittest" ] ; then + unittest=1 # enable unit testing mode + roles="${2}" # the role that should be used for unit testing + cfgfile="${3}" # the location of the config file to use for unit testing + status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL +else + roles="${1}" # the roles that should be notified for this event + args_host="${2}" # the host generated this event + unique_id="${3}" # the unique id of this event + alarm_id="${4}" # the unique id of the alarm that generated this event + event_id="${5}" # the incremental id of the event, for this alarm id + when="${6}" # the timestamp this event occurred + name="${7}" # the name of the alarm, as given in netdata health.d entries + chart="${8}" # the name of the chart (type.id) + family="${9}" # the family of the chart + status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + value="${12}" # the current value of the alarm + old_value="${13}" # the previous value of the alarm + src="${14}" # the line number and file the alarm has been configured + duration="${15}" # the duration in seconds of the previous alarm state + non_clear_duration="${16}" # the total duration in seconds this is/was non-clear + units="${17}" # the units of the value + info="${18}" # a short description of the alarm + value_string="${19}" # friendly value (with units) + old_value_string="${20}" # friendly old value (with units) + calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm + calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation + total_warnings="${23}" # Total number of alarms in WARNING state + total_critical="${24}" # Total number of alarms in CRITICAL state +fi + # ----------------------------------------------------------------------------- # find a suitable hostname to use, if netdata did not supply a hostname -this_host=$(hostname -s 2>/dev/null) -[ -z "${host}" ] && host="${this_host}" +if [ -z ${args_host} ] + then + this_host=$(hostname -s 2>/dev/null) + host="${this_host}" + args_host="${this_host}" +else + host="${args_host}" +fi # ----------------------------------------------------------------------------- # screen statuses we don't need to send a notification @@ -205,7 +250,7 @@ then fi # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL -if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ] +if [ "${clear_alarm_always}" != "YES" -a "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ] then info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})" exit 1 @@ -223,118 +268,80 @@ images_base_url="https://registry.my-netdata.io" # curl options to use curl_options="" +# hostname handling +use_fqdn="NO" + # needed commands # if empty they will be searched in the system path curl= sendmail= # enable / disable features -SEND_SLACK="YES" -SEND_MSTEAM="YES" -SEND_ALERTA="YES" -SEND_FLOCK="YES" -SEND_DISCORD="YES" -SEND_PUSHOVER="YES" -SEND_TWILIO="YES" -SEND_HIPCHAT="YES" -SEND_MESSAGEBIRD="YES" -SEND_KAVENEGAR="YES" -SEND_TELEGRAM="YES" -SEND_EMAIL="YES" -SEND_PUSHBULLET="YES" -SEND_KAFKA="YES" -SEND_PD="YES" -SEND_FLEEP="YES" -SEND_IRC="YES" -SEND_AWSSNS="YES" -SEND_SYSLOG="NO" -SEND_CUSTOM="YES" +for method_name in ${method_names^^} ; do + declare SEND_${method_name}="YES" + declare DEFAULT_RECIPIENT_${method_name} +done + +for method_name in ${method_names} ; do + declare -A role_recipients_${method_name} +done # slack configs SLACK_WEBHOOK_URL= -DEFAULT_RECIPIENT_SLACK= -declare -A role_recipients_slack=() # Microsoft Team configs MSTEAM_WEBHOOK_URL= -DEFAULT_RECIPIENT_MSTEAM= -declare -A role_recipients_msteam=() # rocketchat configs ROCKETCHAT_WEBHOOK_URL= -DEFAULT_RECIPIENT_ROCKETCHAT= -declare -A role_recipients_rocketchat=() # alerta configs ALERTA_WEBHOOK_URL= ALERTA_API_KEY= -DEFAULT_RECIPIENT_ALERTA= -declare -A role_recipients_alerta=() # flock configs FLOCK_WEBHOOK_URL= -DEFAULT_RECIPIENT_FLOCK= -declare -A role_recipients_flock=() # discord configs DISCORD_WEBHOOK_URL= -DEFAULT_RECIPIENT_DISCORD= -declare -A role_recipients_discord=() # pushover configs PUSHOVER_APP_TOKEN= -DEFAULT_RECIPIENT_PUSHOVER= -declare -A role_recipients_pushover=() # pushbullet configs PUSHBULLET_ACCESS_TOKEN= PUSHBULLET_SOURCE_DEVICE= -DEFAULT_RECIPIENT_PUSHBULLET= -declare -A role_recipients_pushbullet=() # twilio configs TWILIO_ACCOUNT_SID= TWILIO_ACCOUNT_TOKEN= TWILIO_NUMBER= -DEFAULT_RECIPIENT_TWILIO= -declare -A role_recipients_twilio=() # hipchat configs HIPCHAT_SERVER= HIPCHAT_AUTH_TOKEN= -DEFAULT_RECIPIENT_HIPCHAT= -declare -A role_recipients_hipchat=() # messagebird configs MESSAGEBIRD_ACCESS_KEY= MESSAGEBIRD_NUMBER= -DEFAULT_RECIPIENT_MESSAGEBIRD= -declare -A role_recipients_messagebird=() # kavenegar configs -KAVENEGAR_API_KEY="" -KAVENEGAR_SENDER="" -DEFAULT_RECIPIENT_KAVENEGAR=() -declare -A role_recipients_kavenegar="" +KAVENEGAR_API_KEY= +KAVENEGAR_SENDER= # telegram configs TELEGRAM_BOT_TOKEN= -DEFAULT_RECIPIENT_TELEGRAM= -declare -A role_recipients_telegram=() # kafka configs +SEND_KAFKA="YES" KAFKA_URL= KAFKA_SENDER_IP= # pagerduty.com configs PD_SERVICE_KEY= -DEFAULT_RECIPIENT_PD= -declare -A role_recipients_pd=() # fleep.io configs FLEEP_SENDER="${host}" -DEFAULT_RECIPIENT_FLEEP= -declare -A role_recipients_fleep=() # Amazon SNS configs DEFAULT_RECIPIENT_AWSSNS= @@ -343,40 +350,38 @@ declare -A role_recipients_awssns=() # syslog configs SYSLOG_FACILITY= -declare -A role_recipients_syslog=() - -# custom configs -DEFAULT_RECIPIENT_CUSTOM= -declare -A role_recipients_custom=() # email configs EMAIL_SENDER= -DEFAULT_RECIPIENT_EMAIL="root" EMAIL_CHARSET=$(locale charmap 2>/dev/null) EMAIL_THREADING= -declare -A role_recipients_email=() +DEFAULT_RECIPIENT_EMAIL="root" # irc configs IRC_NICKNAME= IRC_REALNAME= -DEFAULT_RECIPIENT_IRC= IRC_NETWORK= -declare -A role_recipients_irc=() # load the stock and user configuration files # these will overwrite the variables above -for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf" -do - if [ -f "${CONFIG}" ] - then - debug "Loading config file '${CONFIG}'..." - source "${CONFIG}" - [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'." - else - warning "Cannot find file '${CONFIG}'." - fi -done +if [ ${unittest} ] ; + then + source "${cfgfile}" + [ $? -ne 0 ] && error "Failed to load requested config file." && exit 1 +else + for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf" + do + if [ -f "${CONFIG}" ] + then + debug "Loading config file '${CONFIG}'..." + source "${CONFIG}" + [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'." + else + warning "Cannot find file '${CONFIG}'." + fi + done +fi # If we didn't autodetect the character set for e-mail and it wasn't # set by the user, we need to set it to a reasonable default. UTF-8 @@ -386,6 +391,14 @@ if [ -z ${EMAIL_CHARSET} ] EMAIL_CHARSET="UTF-8" fi +# If we've been asked to use FQDN's for the URL's in the alarm, do so, +# unless we're sending an alarm for a slave system which we can't get the +# FQDN of easily. +if [ "${use_fqdn}" = "YES" -a "${host}" = "$(hostname -s 2>/dev/null)" ] + then + host="$(hostname -f 2>/dev/null)" +fi + # ----------------------------------------------------------------------------- # filter a recipient based on alarm event severity @@ -444,285 +457,6 @@ filter_recipient_by_criticality() { } # ----------------------------------------------------------------------------- -# find the recipients' addresses per method - -declare -A arr_slack=() -declare -A arr_msteam=() -declare -A arr_rocketchat=() -declare -A arr_alerta=() -declare -A arr_flock=() -declare -A arr_discord=() -declare -A arr_pushover=() -declare -A arr_pushbullet=() -declare -A arr_twilio=() -declare -A arr_hipchat=() -declare -A arr_telegram=() -declare -A arr_pd=() -declare -A arr_email=() -declare -A arr_custom=() -declare -A arr_messagebird=() -declare -A arr_kavenegar=() -declare -A arr_fleep=() -declare -A arr_irc=() -declare -A arr_syslog=() -declare -A arr_awssns=() - -# netdata may call us with multiple roles, and roles may have multiple but -# overlapping recipients - so, here we find the unique recipients. -for x in ${roles//,/ } -do - # the roles 'silent' and 'disabled' mean: - # don't send a notification for this role - [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue - - # email - a="${role_recipients_email[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1" - done - - # pushover - a="${role_recipients_pushover[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1" - done - - # pushbullet - a="${role_recipients_pushbullet[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1" - done - - # twilio - a="${role_recipients_twilio[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TWILIO}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1" - done - - # hipchat - a="${role_recipients_hipchat[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_HIPCHAT}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality hipchat "${r}" && arr_hipchat[${r/|*/}]="1" - done - - # messagebird - a="${role_recipients_messagebird[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MESSAGEBIRD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality messagebird "${r}" && arr_messagebird[${r/|*/}]="1" - done - - # kavenegar - a="${role_recipients_kavenegar[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_KAVENEGAR}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality kavenegar "${r}" && arr_kavenegar[${r/|*/}]="1" - done - - # telegram - a="${role_recipients_telegram[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1" - done - - # slack - a="${role_recipients_slack[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1" - done - - # Microsoft Team - a="${role_recipients_msteam[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MSTEAM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality msteam "${r}" && arr_msteam[${r/|*/}]="1" - done - - # rocketchat - a="${role_recipients_rocketchat[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ROCKETCHAT}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality rocketchat "${r}" && arr_rocketchat[${r/|*/}]="1" - done - - # alerta - a="${role_recipients_alerta[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ALERTA}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality alerta "${r}" && arr_alerta[${r/|*/}]="1" - done - - # flock - a="${role_recipients_flock[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLOCK}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality flock "${r}" && arr_flock[${r/|*/}]="1" - done - - # discord - a="${role_recipients_discord[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_DISCORD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality discord "${r}" && arr_discord[${r/|*/}]="1" - done - - # pagerduty.com - a="${role_recipients_pd[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1" - done - - # fleep.io - a="${role_recipients_fleep[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLEEP}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality fleep "${r}" && arr_fleep[${r/|*/}]="1" - done - - # irc - a="${role_recipients_irc[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_IRC}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality irc "${r}" && arr_irc[${r/|*/}]="1" - done - - # amazon sns - a="${role_recipients_awssns[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_AWSSNS}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality awssns "${r}" && arr_awssns[${r/|*/}]="1" - done - - # syslog - a="${role_recipients_syslog[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SYSLOG}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality syslog "${r}" && arr_syslog[${r/|*/}]="1" - done - - # custom - a="${role_recipients_custom[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_CUSTOM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality custom "${r}" && arr_custom[${r/|*/}]="1" - done - -done - -# build the list of slack recipients (channels) -to_slack="${!arr_slack[*]}" -[ -z "${to_slack}" ] && SEND_SLACK="NO" - -# build the list of Microsoft team recipients (channels) -to_msteam="${!arr_msteam[*]}" -[ -z "${to_msteam}" ] && SEND_MSTEAM="NO" - -# build the list of rocketchat recipients (channels) -to_rocketchat="${!arr_rocketchat[*]}" -[ -z "${to_rocketchat}" ] && SEND_ROCKETCHAT="NO" - -# build the list of alerta recipients (channels) -to_alerta="${!arr_alerta[*]}" -[ -z "${to_alerta}" ] && SEND_ALERTA="NO" - -# build the list of flock recipients (channels) -to_flock="${!arr_flock[*]}" -[ -z "${to_flock}" ] && SEND_FLOCK="NO" - -# build the list of discord recipients (channels) -to_discord="${!arr_discord[*]}" -[ -z "${to_discord}" ] && SEND_DISCORD="NO" - -# build the list of pushover recipients (user tokens) -to_pushover="${!arr_pushover[*]}" -[ -z "${to_pushover}" ] && SEND_PUSHOVER="NO" - -# build the list of pushbulet recipients (user tokens) -to_pushbullet="${!arr_pushbullet[*]}" -[ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO" - -# build the list of twilio recipients (phone numbers) -to_twilio="${!arr_twilio[*]}" -[ -z "${to_twilio}" ] && SEND_TWILIO="NO" - -# build the list of hipchat recipients (rooms) -to_hipchat="${!arr_hipchat[*]}" -[ -z "${to_hipchat}" ] && SEND_HIPCHAT="NO" - -# build the list of messagebird recipients (phone numbers) -to_messagebird="${!arr_messagebird[*]}" -[ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO" - -# build the list of kavenegar recipients (phone numbers) -to_kavenegar="${!arr_kavenegar[*]}" -[ -z "${to_kavenegar}" ] && SEND_KAVENEGAR="NO" - -# check array of telegram recipients (chat ids) -to_telegram="${!arr_telegram[*]}" -[ -z "${to_telegram}" ] && SEND_TELEGRAM="NO" - -# build the list of pagerduty recipients (service keys) -to_pd="${!arr_pd[*]}" -[ -z "${to_pd}" ] && SEND_PD="NO" - -# build the list of fleep recipients (conversation webhooks) -to_fleep="${!arr_fleep[*]}" -[ -z "${to_fleep}" ] && SEND_FLEEP="NO" - -# build the list of custom recipients -to_custom="${!arr_custom[*]}" -[ -z "${to_custom}" ] && SEND_CUSTOM="NO" - -# build the list of email recipients (email addresses) -to_email= -for x in "${!arr_email[@]}" -do - [ ! -z "${to_email}" ] && to_email="${to_email}, " - to_email="${to_email}${x}" -done -[ -z "${to_email}" ] && SEND_EMAIL="NO" - -# build the list of irc recipients (channels) -to_irc="${!arr_irc[*]}" -[ -z "${to_irc}" ] && SEND_IRC="NO" - -# build the list of awssns recipients (facilities, servers, and prefixes) -to_awssns="${!arr_awssns[*]}" -[ -z "${to_awssns}" ] && SEND_AWSSNS="NO" - -# build the list of syslog recipients (facilities, servers, and prefixes) -to_syslog="${!arr_syslog[*]}" -[ -z "${to_syslog}" ] && SEND_SYSLOG="NO" - -# ----------------------------------------------------------------------------- # verify the delivery methods supported # check slack @@ -770,25 +504,13 @@ to_syslog="${!arr_syslog[*]}" # check fleep [ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO" -# check pagerduty.com -# if we need pd-send, check for the pd-send command -# https://www.pagerduty.com/docs/guides/agent-install-guide/ -if [ "${SEND_PD}" = "YES" ] - then - pd_send="$(which pd-send 2>/dev/null || command -v pd-send 2>/dev/null)" - if [ -z "${pd_send}" ] - then - error "Cannot find pd-send command in the system path. Disabling pagerduty.com notifications." - SEND_PD="NO" - fi -fi - # if we need curl, check for the curl command if [ \( \ "${SEND_PUSHOVER}" = "YES" \ -o "${SEND_SLACK}" = "YES" \ -o "${SEND_ROCKETCHAT}" = "YES" \ -o "${SEND_ALERTA}" = "YES" \ + -o "${SEND_PD}" = "YES" \ -o "${SEND_FLOCK}" = "YES" \ -o "${SEND_DISCORD}" = "YES" \ -o "${SEND_HIPCHAT}" = "YES" \ @@ -799,6 +521,7 @@ if [ \( \ -o "${SEND_PUSHBULLET}" = "YES" \ -o "${SEND_KAFKA}" = "YES" \ -o "${SEND_FLEEP}" = "YES" \ + -o "${SEND_PROWL}" = "YES" \ -o "${SEND_CUSTOM}" = "YES" \ -o "${SEND_MSTEAM}" = "YES" \ \) -a -z "${curl}" ] @@ -814,6 +537,7 @@ if [ \( \ SEND_MSTEAM="NO" SEND_ROCKETCHAT="NO" SEND_ALERTA="NO" + SEND_PD="NO" SEND_FLOCK="NO" SEND_DISCORD="NO" SEND_TWILIO="NO" @@ -822,6 +546,7 @@ if [ \( \ SEND_KAVENEGAR="NO" SEND_KAFKA="NO" SEND_FLEEP="NO" + SEND_PROWL="NO" SEND_CUSTOM="NO" fi fi @@ -859,6 +584,68 @@ if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ] fi fi +# ----------------------------------------------------------------------------- +# find the recipients' addresses per method + +# netdata may call us with multiple roles, and roles may have multiple but +# overlapping recipients - so, here we find the unique recipients. +for method_name in ${method_names} ; do + send_var="SEND_${method_name^^}" + if [ ${!send_var} = "NO" ] ; then + continue + fi + + declare -A arr_var=() + + for x in ${roles//,/ } ; do + # the roles 'silent' and 'disabled' mean: + # don't send a notification for this role + [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue + + role_recipients="role_recipients_${method_name}[$x]" + default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}" + + a="${!role_recipients}" + [ -z "${a}" ] && a="${!default_recipient_var}" + for r in ${a//,/ } ; do + [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1" + done + done + + # build the list of recipients + to_var="to_${method_name}" + declare to_${method_name}="${!arr_var[*]}" + + [ -z "${!to_var}" ] && declare ${send_var}="NO" +done + +# ----------------------------------------------------------------------------- +# handle fixup of the email recipient list. + +fix_to_email() { + to_email= + while [ ! -z "${1}" ] + do + [ ! -z "${to_email}" ] && to_email="${to_email}, " + to_email="${to_email}${1}" + shift 1 + done +} + +# ${to_email} without quotes here +fix_to_email ${to_email} + +# ----------------------------------------------------------------------------- +# handle output if we're running in unit test mode +if [ ${unittest} ] ; then + for method_name in ${method_names} ; do + to_var="to_${method_name}" + echo "results: ${method_name}: ${!to_var}" + done + exit 0 +fi + +# ----------------------------------------------------------------------------- # check that we have at least a method enabled if [ "${SEND_EMAIL}" != "YES" \ -a "${SEND_PUSHOVER}" != "YES" \ @@ -879,6 +666,7 @@ if [ "${SEND_EMAIL}" != "YES" \ -a "${SEND_CUSTOM}" != "YES" \ -a "${SEND_IRC}" != "YES" \ -a "${SEND_AWSSNS}" != "YES" \ + -a "${SEND_PROWL}" != "YES" \ -a "${SEND_SYSLOG}" != "YES" \ -a "${SEND_MSTEAM}" != "YES" \ ] @@ -1126,7 +914,7 @@ EOF # kafka sender send_kafka() { - local httpcode sent=0 + local httpcode sent=0 if [ "${SEND_KAFKA}" = "YES" ] then httpcode=$(docurl -X POST \ @@ -1163,38 +951,42 @@ send_pd() { then for PD_SERVICE_KEY in ${recipients} do - d="${status} ${name} = ${value_string} - ${host}, ${family}" - ${pd_send} -k ${PD_SERVICE_KEY} \ - -t ${t} \ - -d "${d}" \ - -i ${host}:${chart}:${name} \ - -f 'info'="${info}" \ - -f 'value_w_units'="${value_string}" \ - -f 'when'="${when}" \ - -f 'duration'="${duration}" \ - -f 'roles'="${roles}" \ - -f 'host'="${host}" \ - -f 'unique_id'="${unique_id}" \ - -f 'alarm_id'="${alarm_id}" \ - -f 'event_id'="${event_id}" \ - -f 'name'="${name}" \ - -f 'chart'="${chart}" \ - -f 'family'="${family}" \ - -f 'status'="${status}" \ - -f 'old_status'="${old_status}" \ - -f 'value'="${value}" \ - -f 'old_value'="${old_value}" \ - -f 'src'="${src}" \ - -f 'non_clear_duration'="${non_clear_duration}" \ - -f 'units'="${units}" - retval=$? - if [ ${retval} -eq 0 ] - then - info "sent pagerduty.com notification for host ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}....: ${d}" - sent=$((sent + 1)) - else - error "failed to send pagerduty.com notification for ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}.... (error code ${retval}): ${d}" - fi + d="${status} ${name} = ${value_string} - ${host}, ${family}" + payload="$(cat << EOF + { + "service_key": "${PD_SERVICE_KEY}", + "event_type": "${t}", + "incident_key" : "${alarm_id}", + "description": "${d}", + "details": { + "value_w_units": "${value_string}", + "when": "${when}", + "duration" : "${duration}", + "roles": "${roles}", + "alarm_id" : "${alarm_id}", + "name" : "${name}", + "chart" : "${chart}", + "family" : "${family}", + "status" : "${status}", + "old_status" : "${old_status}", + "value" : "${value}", + "old_value" : "${old_value}", + "src" : "${src}", + "non_clear_duration" : "${non_clear_duration}", + "units" : "${units}", + "info" : "${info}" + } + } +EOF + )" + httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json") + if [ "${httpcode}" = "200" ] + then + info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'" + sent=$((sent + 1)) + else + error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}." + fi done [ ${sent} -gt 0 ] && return 0 @@ -1249,7 +1041,7 @@ send_hipchat() { if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${HIPCHAT_SERVER}" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ] then # A label to be shown in addition to the sender's name - # Valid length range: 0 - 64. + # Valid length range: 0 - 64. sender="netdata" # Valid values: html, text. @@ -1276,7 +1068,7 @@ send_hipchat() { -H "Authorization: Bearer ${authtoken}" \ -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \ "https://${HIPCHAT_SERVER}/v2/room/${room}/notification") - + if [ "${httpcode}" = "204" ] then info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'" @@ -1341,7 +1133,7 @@ send_kavenegar() { --data-urlencode "receptor=${user}" \ --data-urlencode "message=${title} ${message}") - if [ "${httpcode}" = "201" ] + if [ "${httpcode}" = "200" ] then info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'" sent=$((sent + 1)) @@ -1363,7 +1155,7 @@ send_telegram() { local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification="" if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi - + case "${status}" in WARNING) emoji="⚠️" ;; CRITICAL) emoji="🔴" ;; @@ -1476,9 +1268,21 @@ send_slack() { for channel in ${channels} do + # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user. + if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ] ; then channel="#$channel"; fi + + # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks. + if [ "${channel}" = "#" ] ; then + ch="" + chstr="without specifying a channel" + else + ch="\"channel\": \"${channel}\"," + chstr="to '${channel}'" + fi + payload="$(cat <<EOF { - "channel": "#${channel}", + $ch "username": "netdata on ${host}", "icon_url": "${images_base_url}/images/banner-icon-144x144.png", "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*", @@ -1500,7 +1304,7 @@ send_slack() { } ], "thumb_url": "${image}", - "footer": "by <${goto_url}|${this_host}>", + "footer": "by <${goto_url}|${host}>", "ts": ${when} } ] @@ -1511,10 +1315,10 @@ EOF httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}") if [ "${httpcode}" = "200" ] then - info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" + info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}" sent=$((sent + 1)) else - error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." + error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}." fi done @@ -1633,7 +1437,7 @@ send_alerta() { "source": "${src}", "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>" }, - "origin": "netdata/${this_host}", + "origin": "netdata/${host}", "type": "netdataAlarm", "rawData": "${BASH_ARGV[@]}" } @@ -1759,7 +1563,7 @@ send_discord() { ], "thumb_url": "${image}", "footer_icon": "${images_base_url}/images/banner-icon-144x144.png", - "footer": "${this_host}", + "footer": "${host}", "ts": ${when} } ] @@ -1813,11 +1617,55 @@ send_fleep() { } # ----------------------------------------------------------------------------- +# Prowl sender + +send_prowl() { + local httpcode sent=0 data message keys prio=0 alarm_url event + if [ "${SEND_PROWL}" = "YES" ] ; then + message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")" + message="description=${message}" + keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")" + keys="apikey=${keys}" + app="application=Netdata" + + case "${status}" in + CRITICAL) + prio=2 + ;; + WARNING) + prio=1 + ;; + esac + pri="priority=${pri}" + + alarm_url="$(urlencode ${goto_url})" + alarm_url="url=${alarm_url}" + event="$(urlencode "${host} ${status_message}")" + event="event=${event}" + + data="${keys}&${pri}&${alarm_url}&${app}&${event}&${message}" + + httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add") + + if [ "${httpcode}" = "200" ] ; then + info "sent prowl data for: ${host} ${chart}.${name} is ${status}" + sent=1 + else + error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}." + fi + + [ ${sent} -gt 0 ] && return 0 + fi + + return 1 +} + +# ----------------------------------------------------------------------------- # irc sender send_irc() { local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error - + if [ "${SEND_IRC}" = "YES" -a ! -z "${NICKNAME}" -a ! -z "${REALNAME}" -a ! -z "${CHANNELS}" -a ! -z "${NETWORK}" -a ! -z "${SERVERNAME}" ] then case "${status}" in @@ -1842,13 +1690,13 @@ send_irc() { info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'" sent=$((sent + 1)) else - error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}." + error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}." fi done fi - + [ ${sent} -gt 0 ] && return 0 - + return 1 } @@ -1871,7 +1719,7 @@ send_awssns() { info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" sent=$((sent + 1)) else - error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" + error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" fi done @@ -1957,11 +1805,30 @@ send_syslog() { # prepare the content of the notification # the url to send the user on click -urlencode "${host}" >/dev/null; url_host="${REPLY}" +urlencode "${args_host}" >/dev/null; url_host="${REPLY}" urlencode "${chart}" >/dev/null; url_chart="${REPLY}" urlencode "${family}" >/dev/null; url_family="${REPLY}" urlencode "${name}" >/dev/null; url_name="${REPLY}" -goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}" + +redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}" +GOTOCLOUD=0 + +if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ] ; then + if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ] ; then + if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then + NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")" + fi + fi + if [ ! -z "${NETDATA_REGISTRY_UNIQUE_ID}" ] ; then + GOTOCLOUD=1 + fi +fi + +if [ ${GOTOCLOUD} -eq 0 ] ; then + goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}" +else + goto_url="https://netdata.cloud/alarms/redirect?agentID=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}" +fi # the severity of the alarm severity="${status}" @@ -2127,7 +1994,7 @@ SENT_PUSHBULLET=$? # ----------------------------------------------------------------------------- # send the twilio SMS -send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} +send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2138,7 +2005,7 @@ SENT_TWILIO=$? # ----------------------------------------------------------------------------- # send the messagebird SMS -send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} +send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2150,7 +2017,7 @@ SENT_MESSAGEBIRD=$? # ----------------------------------------------------------------------------- # send the kavenegar SMS -send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} +send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2191,9 +2058,15 @@ send_fleep "${to_fleep}" SENT_FLEEP=$? # ----------------------------------------------------------------------------- +# send the Prowl message + +send_prowl "${to_prowl}" +SENT_PROWL=$? + +# ----------------------------------------------------------------------------- # send the irc message -send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm} +send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2278,7 +2151,12 @@ Severity: ${severity} URL : ${goto_url} Source : ${src} Date : ${date} -Notification generated on ${this_host} +Notification generated on ${host} + +Evaluated Expression : ${calc_expression} +Expression Variables : ${calc_param_values} + +The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised. --multipart-boundary Content-Type: text/html; encoding=${EMAIL_CHARSET} @@ -2341,6 +2219,24 @@ Content-Transfer-Encoding: 8bit </td> </tr> <tr style="margin: 0; padding: 0;"> + <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> + <span>${calc_expression}</span> + <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span> + </td> + </tr> + <tr style="margin: 0; padding: 0;"> + <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> + <span>${calc_param_values}</span> + <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span> + </td> + </tr> + <tr style="margin: 0; padding: 0;"> + <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> + The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised. + </td> + </tr> + + <tr style="margin: 0; padding: 0;"> <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;"> <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a> </td> @@ -2351,7 +2247,7 @@ Content-Transfer-Encoding: 8bit </tr> <tr style="text-align: center; margin: 0; padding: 0;"> <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by - <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${this_host}</code>. + <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>. </td> </tr> </tbody> @@ -2393,6 +2289,7 @@ if [ ${SENT_EMAIL} -eq 0 \ -o ${SENT_KAFKA} -eq 0 \ -o ${SENT_PD} -eq 0 \ -o ${SENT_FLEEP} -eq 0 \ + -o ${SENT_PROWL} -eq 0 \ -o ${SENT_IRC} -eq 0 \ -o ${SENT_AWSSNS} -eq 0 \ -o ${SENT_CUSTOM} -eq 0 \ diff --git a/health/notifications/alerta/README.md b/health/notifications/alerta/README.md index cf43621ff..2826fe773 100644 --- a/health/notifications/alerta/README.md +++ b/health/notifications/alerta/README.md @@ -1,4 +1,4 @@ -# alerta.io notifications +# alerta.io The [Alerta](https://alerta.io) monitoring system is a tool used to consolidate and de-duplicate alerts from multiple sources for quick @@ -73,8 +73,10 @@ We can test alarms using the standard approach: $ /opt/netdata/netdata-plugins/plugins.d/alarm-notify.sh test Note: Netdata will send 3 alarms, and because last alarm is "CLEAR" -you will not se them in main Alerta page, you need to select to see +you will not see them in main Alerta page, you need to select to see "closed" alarma in top-right lookup. A little change in `alarm-notify.sh` that let us test each state one by one will be useful. For more information see [https://docs.alerta.io](https://docs.alerta.io) + +[]() diff --git a/health/notifications/awssns/README.md b/health/notifications/awssns/README.md index d040d2d29..5205d4cb4 100644 --- a/health/notifications/awssns/README.md +++ b/health/notifications/awssns/README.md @@ -1,4 +1,4 @@ -# Amazon SNS notifications +# Amazon SNS As part of it's AWS suite, Amazon provides a notification broker service called 'Simple Notification Service' or SNS. Amazon SNS works kind of similarly to Netdata's own notification system, allowing dispatch of a single notification to multiple subscribers of different types. Among other things, SNS supports sending notifications to: @@ -28,4 +28,6 @@ Notes: * Netdata's native email notification support is far better in almost all respects than it's support through Amazon SNS. If you want email notifications, use the native support, not SNS. * If you need to change the notification format for SNS notifications, you can do so by specifying the format in `AWSSNS_MESSAGE_FORMAT` in the configuration. This variable supports all the same vairiables you can use in custom notifications. - * While Amazon SNS supports sending differently formatted messages for different delivery methods, netdata does not currently support this functionality.
\ No newline at end of file + * While Amazon SNS supports sending differently formatted messages for different delivery methods, netdata does not currently support this functionality. + +[]() diff --git a/health/notifications/discord/README.md b/health/notifications/discord/README.md index 5889ea099..7694fef4b 100644 --- a/health/notifications/discord/README.md +++ b/health/notifications/discord/README.md @@ -1,4 +1,4 @@ -# Discordapp.com notifications +# Discordapp.com This is what you will get: @@ -42,3 +42,5 @@ role_recipients_discord[webmaster]="marketing development" ``` The keywords `systems`, `databases`, `marketing`, `development` are discordapp.com channels (they should already exist within your discord server). + +[]() diff --git a/health/notifications/email/README.md b/health/notifications/email/README.md index 979790ad6..163839b6b 100644 --- a/health/notifications/email/README.md +++ b/health/notifications/email/README.md @@ -1,4 +1,4 @@ -# email notifications +# email You need a working `sendmail` command for email alerts to work. Almost all MTAs provide a `sendmail` interface. @@ -28,4 +28,6 @@ sudo su -s /bin/bash netdata /usr/libexec/netdata/plugins.d/alarm-notify.sh test [ROLE] ``` -Where `[ROLE]` is the role you want to test. The default (if you don't give a `[ROLE]`) is `sysadmin`.
\ No newline at end of file +Where `[ROLE]` is the role you want to test. The default (if you don't give a `[ROLE]`) is `sysadmin`. + +[]() diff --git a/health/notifications/flock/README.md b/health/notifications/flock/README.md index 33a545eac..0d679ce6b 100644 --- a/health/notifications/flock/README.md +++ b/health/notifications/flock/README.md @@ -1,4 +1,4 @@ -# flock.com notifications +# flock.com This is what you will get: @@ -28,4 +28,6 @@ FLOCK_WEBHOOK_URL="https://api.flock.com/hooks/sendMessage/XXXXXXXXXXXXXXXXXXXXX # if a role recipient is not configured, no notification will be sent DEFAULT_RECIPIENT_FLOCK="alarms" -```
\ No newline at end of file +``` + +[]() diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf index a997765a6..b96cf5793 100755 --- a/health/notifications/health_alarm_notify.conf +++ b/health/notifications/health_alarm_notify.conf @@ -14,6 +14,7 @@ # - sms messages to your cell phone or any sms enabled device (twilio.com) # - sms messages to your cell phone or any sms enabled device (messagebird.com) # - notifications to users on pagerduty.com +# - push notifications to iOS devices (via prowlapp.com) # - notifications to Amazon SNS topics (aws.amazon.com) # - messages to your irc channel on your selected network # - messages to a local or remote syslog daemon @@ -64,6 +65,24 @@ date_format='' #------------------------------------------------------------------------------ +# hostname handling +# +# By default, Netdata will use the simple hostname for the system (the +# hostname with everything after the first `.` removed) when displaying +# the hostname in alert notifications. If you prefer, you can uncomment +# the line below to have Netdata instead use the host's fully qualified +# domain name. +# +# This does not report correct FQDN's for slave systems for which this +# sytem is a master. +# +# Additionally, if the system host name is overridden in /etc/netdata.conf +# with the `hostname` option, that name will be used unconditionally +# instead of this. +#use_fqdn='YES' + + +#------------------------------------------------------------------------------ # external commands # The full path to the sendmail command. @@ -109,6 +128,14 @@ aws="" #logger_options="" #------------------------------------------------------------------------------ +# extra options + +# By default don't do anything if this is CLEAR, but it was not WARNING or CRITICAL. +# You can send it always if your system makes deduplication for alarms. +#clear_alarm_always='YES' + +# +#------------------------------------------------------------------------------ # NOTE ABOUT RECIPIENTS # # When you define recipients (all types): @@ -268,7 +295,7 @@ SEND_MESSAGEBIRD="YES" # to get an access key, create a free account at https://www.messagebird.com # verify and activate the account (no CC info needed) # login to your account and enter your phonenumber to get some free credits -# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)' +# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)' # click 'Add access key' and fill in data (you want a live key to send SMS) # Without an access key, netdata cannot send Messagebird text messages. @@ -324,7 +351,7 @@ DEFAULT_RECIPIENT_TELEGRAM="" # slack (slack.com) global notification options # multiple recipients can be given like this: -# "CHANNEL1 CHANNEL2 ..." +# "RECIPIENT1 RECIPIENT2 ..." # enable/disable sending slack notifications SEND_SLACK="YES" @@ -335,9 +362,11 @@ SEND_SLACK="YES" # Get yours from: https://api.slack.com/incoming-webhooks SLACK_WEBHOOK_URL="" -# if a role's recipients are not configured, a notification will be send to -# this slack channel (empty = do not send a notification for unconfigured -# roles): +# if a role's recipients are not configured, a notification will be send to: +# - A slack channel (syntax: '#channel' or 'channel') +# - A slack user (syntax: '@user') +# - The channel or user defined in slack for the webhook (syntax: '#') +# empty = do not send a notification for unconfigured roles DEFAULT_RECIPIENT_SLACK="" #------------------------------------------------------------------------------ @@ -405,7 +434,7 @@ DEFAULT_RECIPIENT_ROCKETCHAT="" SEND_ALERTA="YES" # here set your alerta server API url -# this is the API url you defined when installed Alerta server, +# this is the API url you defined when installed Alerta server, # it is the same for all users. Do not include last slash. # ALERTA_WEBHOOK_URL="https://<server>/alerta/api" ALERTA_WEBHOOK_URL="" @@ -493,9 +522,9 @@ KAFKA_SENDER_IP="" #------------------------------------------------------------------------------ # pagerduty.com notification options # -# pagerduty.com notifications require the pagerduty agent to be installed and -# a "Generic API" pagerduty service. -# https://www.pagerduty.com/docs/guides/agent-install-guide/ +# pagerduty.com notifications require a "Generic API" (Events v1) +# pagerduty service. +# https://support.pagerduty.com/docs/services-and-integrations # multiple recipients can be given like this: # "<pd_service_key_1> <pd_service_key_2> ..." @@ -534,7 +563,7 @@ FLEEP_SENDER="" #------------------------------------------------------------------------------ # irc notification options # -# irc notifications require only the nc utility to be installed. +# irc notifications require only the nc utility to be installed. # multiple recipients can be given like this: # "<irc_channel_1> <irc_channel_2> ..." @@ -550,7 +579,7 @@ DEFAULT_RECIPIENT_IRC="" # e.g. "irc.freenode.net" IRC_NETWORK="" -# The irc nickname which is required to send the notification. It must not be +# The irc nickname which is required to send the notification. It must not be # an already registered name as the connection's MODE is defined as a 'guest'. IRC_NICKNAME="" @@ -626,6 +655,22 @@ SYSLOG_FACILITY='' DEFAULT_RECIPIENT_SYSLOG="netdata" #------------------------------------------------------------------------------ +# iOS Push Notifications + +# enable/disable sending iOS push notifications +SEND_PROWL="YES" + +# If a role's recipients are not configured, use the following, +# (empty = do not send a notiication for unconfigured roles) +# +# Recipients for iOS push notifications are Prowl API keys. +# +# A recipient may also consist of multiple Prowl API keys separated by +# commas, in which case notifications will be simultaneously sent for all +# of those API keys. +DEFAULT_RECIPIENT_PROWL="" + +#------------------------------------------------------------------------------ # Amazon SNS notifications # # This method requires potentially complex manual configuration. See the @@ -748,6 +793,8 @@ role_recipients_irc[sysadmin]="${DEFAULT_RECIPIENT_IRC}" role_recipients_syslog[sysadmin]="${DEFAULT_RECIPIENT_SYSLOG}" +role_recipients_prowl[sysadming]="${DEFAULT_RECIPIENT_PROWL}" + role_recipients_awssns[sysadmin]="${DEFAULT_RECIPIENT_AWSSNS}" role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -789,6 +836,8 @@ role_recipients_irc[domainadmin]="${DEFAULT_RECIPIENT_IRC}" role_recipients_syslog[domainadmin]="${DEFAULT_RECIPIENT_SYSLOG}" +role_recipients_prowl[domainadmin]="${DEFAULT_RECIPIENT_PROWL}" + role_recipients_awssns[domainadmin]="${DEFAULT_RECIPIENT_AWSSNS}" role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -831,6 +880,8 @@ role_recipients_irc[dba]="${DEFAULT_RECIPIENT_IRC}" role_recipients_syslog[dba]="${DEFAULT_RECIPIENT_SYSLOG}" +role_recipients_prowl[dba]="${DEFAULT_RECIPIENT_PROWL}" + role_recipients_awssns[dba]="${DEFAULT_RECIPIENT_AWSSNS}" role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -873,6 +924,8 @@ role_recipients_irc[webmaster]="${DEFAULT_RECIPIENT_IRC}" role_recipients_syslog[webmaster]="${DEFAULT_RECIPIENT_SYSLOG}" +role_recipients_prowl[webmaster]="${DEFAULT_RECIPIENT_PROWL}" + role_recipients_awssns[webmaster]="${DEFAULT_RECIPIENT_AWSSNS}" role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -915,6 +968,8 @@ role_recipients_irc[proxyadmin]="${DEFAULT_RECIPIENT_IRC}" role_recipients_syslog[proxyadmin]="${DEFAULT_RECIPIENT_SYSLOG}" +role_recipients_prowl[proxyadmin]="${DEFAULT_RECIPIENT_PROWL}" + role_recipients_awssns[porxyadmin]="${DEFAULT_RECIPIENT_AWSSNS}" role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}" @@ -955,6 +1010,8 @@ role_recipients_fleep[sitemgr]="${DEFAULT_RECIPIENT_FLEEP}" role_recipients_syslog[sitemgr]="${DEFAULT_RECIPIENT_SYSLOG}" +role_recipients_prowl[sitemgr]="${DEFAULT_RECIPIENT_PROWL}" + role_recipients_awssns[sitemgr]="${DEFAULT_RECIPIENT_AWSSNS}" role_recipients_custom[sitemgr]="${DEFAULT_RECIPIENT_CUSTOM}" diff --git a/health/notifications/irc/README.md b/health/notifications/irc/README.md index ece879fe6..9ea86e92d 100644 --- a/health/notifications/irc/README.md +++ b/health/notifications/irc/README.md @@ -1,4 +1,4 @@ -# IRC notifications +# IRC This is what you will get: @@ -70,4 +70,6 @@ role_recipients_irc[dba]="#databases-alarms" role_recipients_irc[webmaster]="#networking-alarms" ``` -The keywords `#user-alarms`, `#networking-alarms`, `#system-alarms`, `#databases-alarms` are irc channels which belong to the specified IRC network.
\ No newline at end of file +The keywords `#user-alarms`, `#networking-alarms`, `#system-alarms`, `#databases-alarms` are irc channels which belong to the specified IRC network. + +[]() diff --git a/health/notifications/kavenegar/README.md b/health/notifications/kavenegar/README.md index e59ad4d4f..d833eef82 100644 --- a/health/notifications/kavenegar/README.md +++ b/health/notifications/kavenegar/README.md @@ -1,4 +1,4 @@ -# Kavenegar notifications +# Kavenegar [Kavenegar](https://www.kavenegar.com/) as service for software developers, based in Iran, provides send and receive SMS, calling voice by using its APIs. @@ -36,4 +36,6 @@ SEND_KAVENEGAR="YES" KAVENEGAR_API_KEY="" KAVENEGAR_SENDER="" DEFAULT_RECIPIENT_KAVENEGAR="" -```
\ No newline at end of file +``` + +[]() diff --git a/health/notifications/messagebird/README.md b/health/notifications/messagebird/README.md index e09ba5d38..cdb3e8dc1 100644 --- a/health/notifications/messagebird/README.md +++ b/health/notifications/messagebird/README.md @@ -1,5 +1,6 @@ +# Messagebird -Will look like this on your Android device: +The messagebird notifications will look like this on your Android device:  @@ -36,3 +37,5 @@ MESSAGEBIRD_NUMBER="XXXXXXX" DEFAULT_RECIPIENT_MESSAGEBIRD="XXXXXXX" ``` + +[]() diff --git a/health/notifications/pagerduty/README.md b/health/notifications/pagerduty/README.md index e35051fd4..884b97923 100644 --- a/health/notifications/pagerduty/README.md +++ b/health/notifications/pagerduty/README.md @@ -1,3 +1,4 @@ +# PagerDuty [PagerDuty](https://www.pagerduty.com/company/) is the enterprise incident resolution service that integrates with ITOps and DevOps monitoring stacks to improve operational reliability and agility. From enriching and aggregating events to correlating them into incidents, PagerDuty streamlines the incident management process by reducing alert noise and resolution times. @@ -32,3 +33,5 @@ SEND_PD="YES" # (empty = do not send a notification for unconfigured roles): DEFAULT_RECIPIENT_PD="<service key>" ``` + +[]() diff --git a/health/notifications/prowl/Makefile.inc b/health/notifications/prowl/Makefile.inc new file mode 100644 index 000000000..08e4c2e54 --- /dev/null +++ b/health/notifications/prowl/Makefile.inc @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# THIS IS NOT A COMPLETE Makefile +# IT IS INCLUDED BY ITS PARENT'S Makefile.am +# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT + +# install these files +dist_noinst_DATA += \ + prowl/README.md \ + prowl/Makefile.inc \ + $(NULL) + diff --git a/health/notifications/prowl/README.md b/health/notifications/prowl/README.md new file mode 100644 index 000000000..1f060edcf --- /dev/null +++ b/health/notifications/prowl/README.md @@ -0,0 +1,22 @@ +# prowl + +(Prowl)[1] is a push notification service for iOS devices. Netdata +supprots delivering notifications to iOS devices through Prowl. + +Because of how Netdata integrates with Prowl, there is a hard limit of +at most 1000 notifications per hour (starting from the first notification +sent). Any alerts beyond the first thousand in an hour will be dropped. + +Warning messages will be sent with the 'High' priority, critical messages +will be sent with the 'Emergency' priority, and all other messages will +be sent with the normal priority. Opening the notification's associated +URL will take you to the Netdata dashboard of the system that issued +the alert, directly to the chart that it triggered on. + +## configuration + +To use this, you will need a Prowl API key, which can be rquested through +the Prowl website after registering. + +Once you have an API key, simply specify that as a recipient for Prowl +notifications. diff --git a/health/notifications/pushbullet/README.md b/health/notifications/pushbullet/README.md index 827d9301d..42b343e45 100644 --- a/health/notifications/pushbullet/README.md +++ b/health/notifications/pushbullet/README.md @@ -1,4 +1,4 @@ -$ PushBullet notifications +# PushBullet Will look like this on your browser:  @@ -39,4 +39,6 @@ SEND_PUSHBULLET="YES" # Without an access token, netdata cannot send pushbullet notifications. PUSHBULLET_ACCESS_TOKEN="o.Sometokenhere" DEFAULT_RECIPIENT_PUSHBULLET="admin1@example.com admin3@somemail.com" -```
\ No newline at end of file +``` + +[]() diff --git a/health/notifications/pushover/README.md b/health/notifications/pushover/README.md index 284981695..1debf5dcd 100644 --- a/health/notifications/pushover/README.md +++ b/health/notifications/pushover/README.md @@ -1,5 +1,4 @@ - -# PushOver notifications +# PushOver pushover.net allows you to receive push notifications on your mobile phone. The service seems free for up to 7.500 messages per month. @@ -14,4 +13,6 @@ The configuration is like above (slack messages). pushover.net notifications look like this: -
\ No newline at end of file + + +[]() diff --git a/health/notifications/rocketchat/README.md b/health/notifications/rocketchat/README.md index 70c698672..f05e73f08 100644 --- a/health/notifications/rocketchat/README.md +++ b/health/notifications/rocketchat/README.md @@ -1,4 +1,4 @@ -# Rocket.Chat notifications +# Rocket.Chat This is what you will get:  @@ -44,3 +44,5 @@ role_recipients_rocketchat[webmaster]="marketing development" The keywords `systems`, `databases`, `marketing`, `development` are RocketChat channels (they should already exist). Both public and private channels can be used, even if they differ from the channel configured in yout RocketChat incomming webhook. + +[]() diff --git a/health/notifications/slack/README.md b/health/notifications/slack/README.md index 45be45197..6e578282e 100644 --- a/health/notifications/slack/README.md +++ b/health/notifications/slack/README.md @@ -1,4 +1,4 @@ -# Slack.com notifications +# Slack This is what you will get:  @@ -17,7 +17,7 @@ Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system r # sending slack notifications # note: multiple recipients can be given like this: -# "CHANNEL1 CHANNEL2 ..." +# "RECIPIENT1 RECIPIENT2 ..." # enable/disable sending pushover notifications SEND_SLACK="YES" @@ -27,14 +27,23 @@ SEND_SLACK="YES" # Without it, netdata cannot send slack notifications. SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXXXXXXX/XXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" -# if a role recipient is not configured, a notification will be send to -# this slack channel: +# if a role's recipients are not configured, a notification will be send to: +# - A slack channel (syntax: '#channel' or 'channel') +# - A slack user (syntax: '@user') +# - The channel or user defined in slack for the webhook (syntax: '#') +# empty = do not send a notification for unconfigured roles DEFAULT_RECIPIENT_SLACK="alarms" ``` -You can define multiple channels like this: `alarms systems`. -You can give different channels per **role** using these (at the same file): +You can define multiple recipients like this: `# #alarms systems @myuser`. +This example will send the alarm to: +- The recipient defined in slack for the webhook (not known to netdata) +- The channel 'alarms' +- The channel 'systems' +- The user @myuser + +You can give different recipients per **role** using these (at the same file): ``` role_recipients_slack[sysadmin]="systems" @@ -42,4 +51,4 @@ role_recipients_slack[dba]="databases systems" role_recipients_slack[webmaster]="marketing development" ``` -The keywords `systems`, `databases`, `marketing`, `development` are slack.com channels (they should already exist in slack). +[]() diff --git a/health/notifications/syslog/README.md b/health/notifications/syslog/README.md index fcc2466a6..597db0cd2 100644 --- a/health/notifications/syslog/README.md +++ b/health/notifications/syslog/README.md @@ -1,4 +1,4 @@ -# syslog notifications +# Syslog You need a working `logger` command for this to work. This is the case on pretty much every Linux system in existence, and most BSD systems. @@ -21,3 +21,5 @@ Targets are defined as follows: The `facility` and `level` are the standard syslog facility and level options, for more info on them see your local `logger` and `syslog` documentation. By default, netdata will log to the `local6` facility, with a log level dependent on the type of message (`crit` for CRITICAL, `warning` for WARNING, and `info` for everything else). You can configure sending directly to remote log servers by specifying a host (and optionally a port). However, this has a somewhat high overhead, so it is much preferred to use your local syslog daemon to handle the forwarding of messages to remote systems (pretty much all of them allow at least simple forwarding, and most of the really popular ones support complex queueing and routing of messages to remote log servers). + +[]() diff --git a/health/notifications/telegram/README.md b/health/notifications/telegram/README.md index cd52fe194..9d652542c 100644 --- a/health/notifications/telegram/README.md +++ b/health/notifications/telegram/README.md @@ -1,4 +1,4 @@ -# Telegram.org notifications +# Telegram [Telegram](https://telegram.org/) is a messaging app with a focus on speed and security, it’s super-fast, simple and free. You can use Telegram on all your devices at the same time — your messages sync seamlessly across any number of your phones, tablets or computers. @@ -17,3 +17,5 @@ See slack for configuration. Telegram messages look like this:  + +[]() diff --git a/health/notifications/twilio/README.md b/health/notifications/twilio/README.md index ab3dd3c04..743f54e3c 100644 --- a/health/notifications/twilio/README.md +++ b/health/notifications/twilio/README.md @@ -1,4 +1,4 @@ -# Twilio.com notifications +# Twilio Will look like this on your Android device: @@ -38,3 +38,5 @@ TWILIO_ACCOUNT_TOKEN="xxxxxxxxxx" TWILIO_NUMBER="xxxxxxxxxxx" DEFAULT_RECIPIENT_TWILIO="+15555555555" ``` + +[]() diff --git a/health/notifications/web/README.md b/health/notifications/web/README.md index ba7dac1fe..0aac9419b 100644 --- a/health/notifications/web/README.md +++ b/health/notifications/web/README.md @@ -1,6 +1,8 @@ -# Dashboard notifications +# Dashboard The netdata dashboard shows HTML notifications, when it is open. Such web notifications look like this:  + +[]() |