Adding upstream version 1.29.3.upstream/1.29.3upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

57 files changed, 5734 insertions, 0 deletions

diff --git a/health/notifications/Makefile.am b/health/notifications/Makefile.am
new file mode 100644
index 0000000..e6b4213
--- /dev/null
+++ b/health/notifications/Makefile.am
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+CLEANFILES = \
+    alarm-notify.sh \
+    $(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
+
+dist_libconfig_DATA = \
+    health_alarm_notify.conf \
+    health_email_recipients.conf \
+    $(NULL)
+
+dist_plugins_SCRIPTS = \
+    alarm-notify.sh \
+    alarm-email.sh \
+    alarm-test.sh \
+    $(NULL)
+
+dist_noinst_DATA = \
+    alarm-notify.sh.in \
+    README.md \
+    $(NULL)
+
+include alerta/Makefile.inc
+include awssns/Makefile.inc
+include discord/Makefile.inc
+include email/Makefile.inc
+include flock/Makefile.inc
+include hangouts/Makefile.inc
+include irc/Makefile.inc
+include kavenegar/Makefile.inc
+include messagebird/Makefile.inc
+include opsgenie/Makefile.inc
+include pagerduty/Makefile.inc
+include pushbullet/Makefile.inc
+include pushover/Makefile.inc
+include rocketchat/Makefile.inc
+include slack/Makefile.inc
+include smstools3/Makefile.inc
+include stackpulse/Makefile.inc
+include syslog/Makefile.inc
+include telegram/Makefile.inc
+include twilio/Makefile.inc
+include web/Makefile.inc
+include matrix/Makefile.inc
+include custom/Makefile.inc
diff --git a/health/notifications/README.md b/health/notifications/README.md
new file mode 100644
index 0000000..5354198
--- /dev/null
+++ b/health/notifications/README.md
@@ -0,0 +1,86 @@
+<!--
+title: "Alarm notifications"
+description: "Reference documentation for Netdata's alarm notification feature, which supports dozens of endpoints, user roles, and more."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/README.md
+-->
+
+# Alarm notifications
+
+The `exec` line in health configuration defines an external script that will be called once
+the alarm is triggered. The default script is `alarm-notify.sh`.
+
+You can change the default script globally by editing `/etc/netdata/netdata.conf`.
+
+`alarm-notify.sh` is capable of sending notifications:
+
+-   to multiple recipients
+-   using multiple notification methods
+-   filtering severity per recipient
+
+It uses **roles**. For example `sysadmin`, `webmaster`, `dba`, etc.
+
+Each alarm is assigned to one or more roles, using the `to` line of the alarm configuration. Then `alarm-notify.sh` uses
+its own configuration file `/etc/netdata/health_alarm_notify.conf`. To edit it on your system, run
+`/etc/netdata/edit-config health_alarm_notify.conf` and find the destination address of the notification for each
+method.
+
+Each role may have one or more destinations.
+
+So, for example the `sysadmin` role may send:
+
+1.  emails to admin1@example.com and admin2@example.com
+2.  pushover.net notifications to USERTOKENS `A`, `B` and `C`.
+3.  pushbullet.com push notifications to admin1@example.com and admin2@example.com
+4.  messages to slack.com channel `#alarms` and `#systems`.
+5.  messages to Discord channels `#alarms` and `#systems`.
+
+## Configuration
+
+Edit `/etc/netdata/health_alarm_notify.conf` by running `/etc/netdata/edit-config health_alarm_notify.conf`:
+
+-   settings per notification method:
+
+     all notification methods except email, require some configuration
+     (i.e. API keys, tokens, destination rooms, channels, etc).
+
+-  **recipients** per **role** per **notification method**
+
+```sh
+grep sysadmin /etc/netdata/health_alarm_notify.conf
+
+role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}"
+role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"
+role_recipients_pushbullet[sysadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
+role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"
+...
+```
+
+## Testing Notifications
+
+You can run the following command by hand, to test alarms configuration:
+
+```sh
+# become user netdata
+su -s /bin/bash netdata
+
+# enable debugging info on the console
+export NETDATA_ALARM_NOTIFY_DEBUG=1
+
+# send test alarms to sysadmin
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test
+
+# send test alarms to any role
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test "ROLE"
+```
+
+Note that in versions before 1.16, the plugins.d directory may be installed in a different location in certain OSs (e.g. under `/usr/lib/netdata`). You can always find the location of the alarm-notify.sh script in `netdata.conf`.
+
+If you need to dig even deeper, you can trace the execution with `bash -x`. Note that in test mode, alarm-notify.sh calls itself with many more arguments. So first do
+
+```sh
+bash -x /usr/libexec/netdata/plugins.d/alarm-notify.sh test
+```
+
+ Then look in the output for the alarm-notify.sh calls and run the one you want to trace with `bash -x`. 
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/alarm-email.sh b/health/notifications/alarm-email.sh
new file mode 100755
index 0000000..69c4c3f
--- /dev/null
+++ b/health/notifications/alarm-email.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# OBSOLETE - REPLACED WITH
+# alarm-notify.sh
+
+${0/alarm-email.sh/alarm-notify.sh} "${@}"
diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in
new file mode 100755
index 0000000..3bf8db5
--- /dev/null
+++ b/health/notifications/alarm-notify.sh.in
@@ -0,0 +1,2759 @@
+#!/usr/bin/env bash
+#shellcheck source=/dev/null disable=SC2086,SC2154
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Script to send alarm notifications for netdata
+#
+# Features:
+#  - multiple notification methods
+#  - multiple roles per alarm
+#  - multiple recipients per role
+#  - severity filtering per recipient
+#
+# Supported notification methods:
+#  - emails by @ktsaou
+#  - slack.com notifications by @ktsaou
+#  - alerta.io notifications by @kattunga
+#  - discordapp.com notifications by @lowfive
+#  - pushover.net notifications by @ktsaou
+#  - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
+#  - telegram.org notifications by @hashworks #1002
+#  - twilio.com notifications by Levi Blaney @shadycuz #1211
+#  - kafka notifications by @ktsaou #1342
+#  - pagerduty.com notifications by Jim Cooley @jimcooley #1373
+#  - messagebird.com notifications by @tech_no_logical #1453
+#  - hipchat notifications by @ktsaou #1561
+#  - fleep notifications by @Ferroin
+#  - prowlapp.com notifications by @Ferroin
+#  - irc notifications by @manosf
+#  - custom notifications by @ktsaou
+#  - syslog messages by @Ferroin
+#  - Microsoft Team notification by @tioumen
+#  - RocketChat notifications by @Hermsi1337 #3777
+#  - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
+#  - Dynatrace Event by @illumine
+#  - Stackpulse Event by @thiagoftsm
+#  - Opsgenie by @thiaoftsm #9858
+
+# -----------------------------------------------------------------------------
+# testing notifications
+
+if { [ "${1}" = "test" ] || [ "${2}" = "test" ]; } && [ "${#}" -le 2 ]; then
+  if [ "${2}" = "test" ]; then
+    recipient="${1}"
+  else
+    recipient="${2}"
+  fi
+
+  [ -z "${recipient}" ] && recipient="sysadmin"
+
+  id=1
+  last="CLEAR"
+  test_res=0
+  for x in "WARNING" "CRITICAL" "CLEAR"; do
+    echo >&2
+    echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
+
+    "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
+    #shellcheck disable=SC2181
+    if [ $? -ne 0 ]; then
+      echo >&2 "# FAILED"
+      test_res=1
+    else
+      echo >&2 "# OK"
+    fi
+
+    last="${x}"
+    id=$((id + 1))
+  done
+
+  exit $test_res
+fi
+
+export PATH="${PATH}:/sbin:/usr/sbin:/usr/local/sbin"
+export LC_ALL=C
+
+# -----------------------------------------------------------------------------
+
+PROGRAM_NAME="$(basename "${0}")"
+
+logdate() {
+  date "+%Y-%m-%d %H:%M:%S"
+}
+
+log() {
+  local status="${1}"
+  shift
+
+  echo >&2 "$(logdate): ${PROGRAM_NAME}: ${status}: ${*}"
+
+}
+
+warning() {
+  log WARNING "${@}"
+}
+
+error() {
+  log ERROR "${@}"
+}
+
+info() {
+  log INFO "${@}"
+}
+
+fatal() {
+  log FATAL "${@}"
+  exit 1
+}
+
+debug=${NETDATA_ALARM_NOTIFY_DEBUG-0}
+debug() {
+  [ "${debug}" = "1" ] && log DEBUG "${@}"
+}
+
+docurl() {
+  if [ -z "${curl}" ]; then
+    error "${curl} is unset."
+    return 1
+  fi
+
+  if [ "${debug}" = "1" ]; then
+    echo >&2 "--- BEGIN curl command ---"
+    printf >&2 "%q " ${curl} "${@}"
+    echo >&2
+    echo >&2 "--- END curl command ---"
+
+    local out code ret
+    out=$(mktemp /tmp/netdata-health-alarm-notify-XXXXXXXX)
+    code=$(${curl} ${curl_options} --write-out "%{http_code}" --output "${out}" --silent --show-error "${@}")
+    ret=$?
+    echo >&2 "--- BEGIN received response ---"
+    cat >&2 "${out}"
+    echo >&2
+    echo >&2 "--- END received response ---"
+    echo >&2 "RECEIVED HTTP RESPONSE CODE: ${code}"
+    rm "${out}"
+    echo "${code}"
+    return ${ret}
+  fi
+
+  ${curl} ${curl_options} --write-out "%{http_code}" --output /dev/null --silent --show-error "${@}"
+  return $?
+}
+
+# -----------------------------------------------------------------------------
+# List of all the notification mechanisms we support.
+# Used in a couple of places to write more compact code.
+
+method_names="
+email
+pushover
+pushbullet
+telegram
+slack
+alerta
+flock
+discord
+hipchat
+twilio
+messagebird
+pd
+fleep
+syslog
+custom
+msteam
+kavenegar
+prowl
+irc
+awssns
+rocketchat
+sms
+hangouts
+dynatrace
+matrix
+"
+
+# -----------------------------------------------------------------------------
+# this is to be overwritten by the config file
+
+custom_sender() {
+  info "not sending custom notification for ${status} of '${host}.${chart}.${name}'"
+}
+
+# -----------------------------------------------------------------------------
+
+# check for BASH v4+ (required for associative arrays)
+if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
+  fatal "BASH version 4 or later is required (this is ${BASH_VERSION})."
+fi
+
+# -----------------------------------------------------------------------------
+# defaults to allow running this script by hand
+
+[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
+[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
+[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
+[ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
+[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud"
+
+# -----------------------------------------------------------------------------
+# parse command line parameters
+
+if [[ ${1} = "unittest" ]]; then
+  unittest=1        # enable unit testing mode
+  roles="${2}"      # the role that should be used for unit testing
+  cfgfile="${3}"    # the location of the config file to use for unit testing
+  status="${4}"     # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+  old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+else
+  roles="${1}"               # the roles that should be notified for this event
+  args_host="${2}"           # the host generated this event
+  unique_id="${3}"           # the unique id of this event
+  alarm_id="${4}"            # the unique id of the alarm that generated this event
+  event_id="${5}"            # the incremental id of the event, for this alarm id
+  when="${6}"                # the timestamp this event occurred
+  name="${7}"                # the name of the alarm, as given in netdata health.d entries
+  chart="${8}"               # the name of the chart (type.id)
+  family="${9}"              # the family of the chart
+  status="${10}"             # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+  old_status="${11}"         # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+  value="${12}"              # the current value of the alarm
+  old_value="${13}"          # the previous value of the alarm
+  src="${14}"                # the line number and file the alarm has been configured
+  duration="${15}"           # the duration in seconds of the previous alarm state
+  non_clear_duration="${16}" # the total duration in seconds this is/was non-clear
+  units="${17}"              # the units of the value
+  info="${18}"               # a short description of the alarm
+  value_string="${19}"       # friendly value (with units)
+  # shellcheck disable=SC2034
+  # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
+  old_value_string="${20}"   # friendly old value (with units), previously named "old_value_string"
+  calc_expression="${21}"    # contains the expression that was evaluated to trigger the alarm
+  calc_param_values="${22}"  # the values of the parameters in the expression, at the time of the evaluation
+  total_warnings="${23}"     # Total number of alarms in WARNING state
+  total_critical="${24}"     # Total number of alarms in CRITICAL state
+fi
+
+# -----------------------------------------------------------------------------
+# find a suitable hostname to use, if netdata did not supply a hostname
+
+if [ -z ${args_host} ]; then
+  this_host=$(hostname -s 2>/dev/null)
+  host="${this_host}"
+  args_host="${this_host}"
+else
+  host="${args_host}"
+fi
+
+# -----------------------------------------------------------------------------
+# screen statuses we don't need to send a notification
+
+# don't do anything if this is not WARNING, CRITICAL or CLEAR
+if [ "${status}" != "WARNING" ] && [ "${status}" != "CRITICAL" ] && [ "${status}" != "CLEAR" ]; then
+  info "not sending notification for ${status} of '${host}.${chart}.${name}'"
+  exit 1
+fi
+
+# don't do anything if this is CLEAR, but it was not WARNING or CRITICAL
+if [ "${clear_alarm_always}" != "YES" ] && [ "${old_status}" != "WARNING" ] && [ "${old_status}" != "CRITICAL" ] && [ "${status}" = "CLEAR" ]; then
+  info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})"
+  exit 1
+fi
+
+# -----------------------------------------------------------------------------
+# load configuration
+
+# By default fetch images from the global public registry.
+# This is required by default, since all notification methods need to download
+# images via the Internet, and private registries might not be reachable.
+# This can be overwritten at the configuration file.
+images_base_url="https://registry.my-netdata.io"
+
+# curl options to use
+curl_options=""
+
+# hostname handling
+use_fqdn="NO"
+
+# needed commands
+# if empty they will be searched in the system path
+curl=
+sendmail=
+
+# enable / disable features
+for method_name in ${method_names^^}; do
+  declare SEND_${method_name}="YES"
+  declare DEFAULT_RECIPIENT_${method_name}
+done
+
+for method_name in ${method_names}; do
+  declare -A role_recipients_${method_name}
+done
+
+# slack configs
+SLACK_WEBHOOK_URL=
+
+# Microsoft Team configs
+MSTEAM_WEBHOOK_URL=
+
+# rocketchat configs
+ROCKETCHAT_WEBHOOK_URL=
+
+# alerta configs
+ALERTA_WEBHOOK_URL=
+ALERTA_API_KEY=
+
+# flock configs
+FLOCK_WEBHOOK_URL=
+
+# discord configs
+DISCORD_WEBHOOK_URL=
+
+# pushover configs
+PUSHOVER_APP_TOKEN=
+
+# pushbullet configs
+PUSHBULLET_ACCESS_TOKEN=
+PUSHBULLET_SOURCE_DEVICE=
+
+# twilio configs
+TWILIO_ACCOUNT_SID=
+TWILIO_ACCOUNT_TOKEN=
+TWILIO_NUMBER=
+
+# hipchat configs
+HIPCHAT_SERVER=
+HIPCHAT_AUTH_TOKEN=
+
+# messagebird configs
+MESSAGEBIRD_ACCESS_KEY=
+MESSAGEBIRD_NUMBER=
+
+# kavenegar configs
+KAVENEGAR_API_KEY=
+KAVENEGAR_SENDER=
+
+# telegram configs
+TELEGRAM_BOT_TOKEN=
+
+# kafka configs
+SEND_KAFKA="YES"
+KAFKA_URL=
+KAFKA_SENDER_IP=
+
+# pagerduty.com configs
+PD_SERVICE_KEY=
+USE_PD_VERSION=
+
+# fleep.io configs
+FLEEP_SENDER="${host}"
+
+# Amazon SNS configs
+AWSSNS_MESSAGE_FORMAT=
+
+# Matrix configs
+MATRIX_HOMESERVER=
+MATRIX_ACCESSTOKEN=
+
+# syslog configs
+SYSLOG_FACILITY=
+
+# email configs
+EMAIL_SENDER=
+EMAIL_CHARSET=$(locale charmap 2>/dev/null)
+EMAIL_THREADING=
+EMAIL_PLAINTEXT_ONLY=
+
+# irc configs
+IRC_NICKNAME=
+IRC_REALNAME=
+IRC_NETWORK=
+
+# hangouts configs
+declare -A HANGOUTS_WEBHOOK_URI
+declare -A HANGOUTS_WEBHOOK_THREAD
+
+# dynatrace configs
+DYNATRACE_SPACE=
+DYNATRACE_SERVER=
+DYNATRACE_TOKEN=
+DYNATRACE_TAG_VALUE=
+DYNATRACE_ANNOTATION_TYPE=
+DYNATRACE_EVENT=
+SEND_DYNATRACE=
+
+# stackpulse configs
+STACKPULSE_WEBHOOK=
+
+# opsgenie configs
+OPSGENIE_API_KEY=
+
+# load the stock and user configuration files
+# these will overwrite the variables above
+
+if [ ${unittest} ]; then
+  if source "${cfgfile}"; then
+    error "Failed to load requested config file."
+    exit 1
+  fi
+else
+  for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf"; do
+    if [ -f "${CONFIG}" ]; then
+      debug "Loading config file '${CONFIG}'..."
+      source "${CONFIG}" || error "Failed to load config file '${CONFIG}'."
+    else
+      warning "Cannot find file '${CONFIG}'."
+    fi
+  done
+fi
+
+OPSGENIE_API_URL=${OPSGENIE_API_URL:-"https://api.opsgenie.com"}
+
+# If we didn't autodetect the character set for e-mail and it wasn't
+# set by the user, we need to set it to a reasonable default.  UTF-8
+# should be correct for almost all modern UNIX systems.
+if [ -z ${EMAIL_CHARSET} ]; then
+  EMAIL_CHARSET="UTF-8"
+fi
+
+# If we've been asked to use FQDN's for the URL's in the alarm, do so,
+# unless we're sending an alarm for a child system which we can't get the
+# FQDN of easily.
+if [ "${use_fqdn}" = "YES" ] && [ "${host}" = "$(hostname -s 2>/dev/null)" ]; then
+  host="$(hostname -f 2>/dev/null)"
+fi
+
+# -----------------------------------------------------------------------------
+# filter a recipient based on alarm event severity
+
+filter_recipient_by_criticality() {
+  local method="${1}" x="${2}" r s
+  shift
+
+  r="${x/|*/}" # the recipient
+  s="${x/*|/}" # the severity required for notifying this recipient
+
+  # no severity filtering for this person
+  [ "${r}" = "${s}" ] && return 0
+
+  # the severity is invalid
+  s="${s^^}"
+  if [ "${s}" != "CRITICAL" ]; then
+    error "SEVERITY FILTERING for ${x} VIA ${method}: invalid severity '${s,,}', only 'critical' is supported."
+    return 0
+  fi
+
+  # create the status tracking directory for this user
+  [ ! -d "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}" ] &&
+    mkdir -p "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}"
+
+  case "${status}" in
+  CRITICAL)
+    # make sure he will get future notifications for this alarm too
+    touch "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
+    debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: the alarm is CRITICAL (will now receive next status change)"
+    return 0
+    ;;
+
+  WARNING)
+    if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
+      # we do not remove the file, so that he will get future notifications of this alarm
+      debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm in the past (will still receive next status change)"
+      return 0
+    fi
+    ;;
+
+  *)
+    if [ -f "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}" ]; then
+      # remove the file, so that he will only receive notifications for CRITICAL states for this alarm
+      rm "${NETDATA_CACHE_DIR}/alarm-notify/${method}/${r}/${alarm_id}"
+      debug "SEVERITY FILTERING for ${x} VIA ${method}: ALLOW: recipient has been notified for this alarm (will only receive CRITICAL notifications from now on)"
+      return 0
+    fi
+    ;;
+  esac
+
+  debug "SEVERITY FILTERING for ${x} VIA ${method}: BLOCK: recipient should not receive this notification"
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# verify the delivery methods supported
+
+# check slack
+[ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
+
+# check rocketchat
+[ -z "${ROCKETCHAT_WEBHOOK_URL}" ] && SEND_ROCKETCHAT="NO"
+
+# check alerta
+[ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
+
+# check flock
+[ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
+
+# check discord
+[ -z "${DISCORD_WEBHOOK_URL}" ] && SEND_DISCORD="NO"
+
+# check pushover
+[ -z "${PUSHOVER_APP_TOKEN}" ] && SEND_PUSHOVER="NO"
+
+# check pushbullet
+[ -z "${PUSHBULLET_ACCESS_TOKEN}" ] && SEND_PUSHBULLET="NO"
+
+# check twilio
+{ [ -z "${TWILIO_ACCOUNT_TOKEN}" ] || [ -z "${TWILIO_ACCOUNT_SID}" ] || [ -z "${TWILIO_NUMBER}" ]; } && SEND_TWILIO="NO"
+
+# check hipchat
+[ -z "${HIPCHAT_AUTH_TOKEN}" ] && SEND_HIPCHAT="NO"
+
+# check messagebird
+{ [ -z "${MESSAGEBIRD_ACCESS_KEY}" ] || [ -z "${MESSAGEBIRD_NUMBER}" ]; } && SEND_MESSAGEBIRD="NO"
+
+# check kavenegar
+{ [ -z "${KAVENEGAR_API_KEY}" ] || [ -z "${KAVENEGAR_SENDER}" ]; } && SEND_KAVENEGAR="NO"
+
+# check telegram
+[ -z "${TELEGRAM_BOT_TOKEN}" ] && SEND_TELEGRAM="NO"
+
+# check kafka
+{ [ -z "${KAFKA_URL}" ] || [ -z "${KAFKA_SENDER_IP}" ]; } && SEND_KAFKA="NO"
+
+# check irc
+[ -z "${IRC_NETWORK}" ] && SEND_IRC="NO"
+
+# check hangouts
+[ ${#HANGOUTS_WEBHOOK_URI[@]} -eq 0 ] && SEND_HANGOUTS="NO"
+
+# check fleep
+#shellcheck disable=SC2153
+{ [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
+
+# check dynatrace
+{ [ -z "${DYNATRACE_SPACE}" ] ||
+  [ -z "${DYNATRACE_SERVER}" ] ||
+  [ -z "${DYNATRACE_TOKEN}" ] ||
+  [ -z "${DYNATRACE_TAG_VALUE}" ] ||
+  [ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
+
+# check opsgenie
+[ -z "${OPSGENIE_API_KEY}" ] && SEND_OPSGENIE="NO"
+
+# check matrix
+{ [ -z "${MATRIX_HOMESERVER}" ] || [ -z "${MATRIX_ACCESSTOKEN}" ]; } && SEND_MATRIX="NO"
+
+# check stackpulse
+[ -z "${STACKPULSE_WEBHOOK}" ] && SEND_STACKPULSE="NO"
+
+if [ "${SEND_PUSHOVER}" = "YES" ] ||
+  [ "${SEND_SLACK}" = "YES" ] ||
+  [ "${SEND_ROCKETCHAT}" = "YES" ] ||
+  [ "${SEND_ALERTA}" = "YES" ] ||
+  [ "${SEND_PD}" = "YES" ] ||
+  [ "${SEND_FLOCK}" = "YES" ] ||
+  [ "${SEND_DISCORD}" = "YES" ] ||
+  [ "${SEND_HIPCHAT}" = "YES" ] ||
+  [ "${SEND_TWILIO}" = "YES" ] ||
+  [ "${SEND_MESSAGEBIRD}" = "YES" ] ||
+  [ "${SEND_KAVENEGAR}" = "YES" ] ||
+  [ "${SEND_TELEGRAM}" = "YES" ] ||
+  [ "${SEND_PUSHBULLET}" = "YES" ] ||
+  [ "${SEND_KAFKA}" = "YES" ] ||
+  [ "${SEND_FLEEP}" = "YES" ] ||
+  [ "${SEND_PROWL}" = "YES" ] ||
+  [ "${SEND_HANGOUTS}" = "YES" ] ||
+  [ "${SEND_MATRIX}" = "YES" ] ||
+  [ "${SEND_CUSTOM}" = "YES" ] ||
+  [ "${SEND_MSTEAM}" = "YES" ] ||
+  [ "${SEND_DYNATRACE}" = "YES" ] ||
+  [ "${SEND_STACKPULSE}" = "YES" ] ||
+  [ "${SEND_OPSGENIE}" = "YES" ]; then
+  # if we need curl, check for the curl command
+  if [ -z "${curl}" ]; then
+    curl="$(command -v curl 2>/dev/null)"
+  fi
+  if [ -z "${curl}" ]; then
+    error "Cannot find curl command in the system path. Disabling all curl based notifications."
+    SEND_PUSHOVER="NO"
+    SEND_PUSHBULLET="NO"
+    SEND_TELEGRAM="NO"
+    SEND_SLACK="NO"
+    SEND_MSTEAM="NO"
+    SEND_ROCKETCHAT="NO"
+    SEND_ALERTA="NO"
+    SEND_PD="NO"
+    SEND_FLOCK="NO"
+    SEND_DISCORD="NO"
+    SEND_TWILIO="NO"
+    SEND_HIPCHAT="NO"
+    SEND_MESSAGEBIRD="NO"
+    SEND_KAVENEGAR="NO"
+    SEND_KAFKA="NO"
+    SEND_FLEEP="NO"
+    SEND_PROWL="NO"
+    SEND_HANGOUTS="NO"
+    SEND_MATRIX="NO"
+    SEND_CUSTOM="NO"
+    SEND_DYNATRACE="NO"
+    SEND_STACKPULSE="NO"
+    SEND_OPSGENIE="NO"
+  fi
+fi
+
+if [ "${SEND_SMS}" = "YES" ]; then
+  if [ -z "${sendsms}" ]; then
+    sendsms="$(command -v sendsms 2>/dev/null)"
+  fi
+  if [ -z "${sendsms}" ]; then
+    SEND_SMS="NO"
+  fi
+fi
+# if we need sendmail, check for the sendmail command
+if [ "${SEND_EMAIL}" = "YES" ] && [ -z "${sendmail}" ]; then
+  sendmail="$(command -v sendmail 2>/dev/null)"
+  if [ -z "${sendmail}" ]; then
+    debug "Cannot find sendmail command in the system path. Disabling email notifications."
+    SEND_EMAIL="NO"
+  fi
+fi
+
+# if we need logger, check for the logger command
+if [ "${SEND_SYSLOG}" = "YES" ] && [ -z "${logger}" ]; then
+  logger="$(command -v logger 2>/dev/null)"
+  if [ -z "${logger}" ]; then
+    debug "Cannot find logger command in the system path. Disabling syslog notifications."
+    SEND_SYSLOG="NO"
+  fi
+fi
+
+# if we need aws, check for the aws command
+if [ "${SEND_AWSSNS}" = "YES" ] && [ -z "${aws}" ]; then
+  aws="$(command -v aws 2>/dev/null)"
+  if [ -z "${aws}" ]; then
+    debug "Cannot find aws command in the system path.  Disabling Amazon SNS notifications."
+    SEND_AWSSNS="NO"
+  fi
+fi
+
+# -----------------------------------------------------------------------------
+# find the recipients' addresses per method
+
+# netdata may call us with multiple roles, and roles may have multiple but
+# overlapping recipients - so, here we find the unique recipients.
+for method_name in ${method_names}; do
+  send_var="SEND_${method_name^^}"
+  if [ "${!send_var}" = "NO" ]; then
+    continue
+  fi
+
+  declare -A arr_var=()
+
+  for x in ${roles//,/ }; do
+    # the roles 'silent' and 'disabled' mean:
+    # don't send a notification for this role
+    if [ "${x}" = "silent" ] || [ "${x}" = "disabled" ]; then
+      continue
+    fi
+
+    role_recipients="role_recipients_${method_name}[$x]"
+    default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}"
+
+    a="${!role_recipients}"
+    [ -z "${a}" ] && a="${!default_recipient_var}"
+    for r in ${a//,/ }; do
+      [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1"
+    done
+  done
+
+  # build the list of recipients
+  to_var="to_${method_name}"
+  declare to_${method_name}="${!arr_var[*]}"
+
+  [ -z "${!to_var}" ] && declare ${send_var}="NO"
+done
+
+# -----------------------------------------------------------------------------
+# handle fixup of the email recipient list.
+
+fix_to_email() {
+  to_email=
+  while [ -n "${1}" ]; do
+    [ -n "${to_email}" ] && to_email="${to_email}, "
+    to_email="${to_email}${1}"
+    shift 1
+  done
+}
+
+# ${to_email} without quotes here
+fix_to_email ${to_email}
+
+# -----------------------------------------------------------------------------
+# handle output if we're running in unit test mode
+if [ ${unittest} ]; then
+  for method_name in ${method_names}; do
+    to_var="to_${method_name}"
+    echo "results: ${method_name}: ${!to_var}"
+  done
+  exit 0
+fi
+
+# -----------------------------------------------------------------------------
+# check that we have at least a method enabled
+proceed=0
+for method in "${SEND_EMAIL}" \
+  "${SEND_PUSHOVER}" \
+  "${SEND_TELEGRAM}" \
+  "${SEND_SLACK}" \
+  "${SEND_ROCKETCHAT}" \
+  "${SEND_ALERTA}" \
+  "${SEND_FLOCK}" \
+  "${SEND_DISCORD}" \
+  "${SEND_TWILIO}" \
+  "${SEND_HIPCHAT}" \
+  "${SEND_MESSAGEBIRD}" \
+  "${SEND_KAVENEGAR}" \
+  "${SEND_PUSHBULLET}" \
+  "${SEND_KAFKA}" \
+  "${SEND_PD}" \
+  "${SEND_FLEEP}" \
+  "${SEND_PROWL}" \
+  "${SEND_MATRIX}" \
+  "${SEND_CUSTOM}" \
+  "${SEND_IRC}" \
+  "${SEND_HANGOUTS}" \
+  "${SEND_AWSSNS}" \
+  "${SEND_SYSLOG}" \
+  "${SEND_SMS}" \
+  "${SEND_MSTEAM}" \
+  "${SEND_DYNATRACE}" \
+  "${SEND_STACKPULSE}" \
+  "${SEND_OPSGENIE}" ; do
+
+  if [ "${method}" == "YES" ]; then
+    proceed=1
+    break
+  fi
+done
+if [ "$proceed" -eq 0 ]; then
+  fatal "All notification methods are disabled. Not sending notification for host '${host}', chart '${chart}' to '${roles}' for '${name}' = '${value}' for status '${status}'."
+fi
+
+# -----------------------------------------------------------------------------
+# get the date the alarm happened
+
+date=$(date --date=@${when} "${date_format}" 2>/dev/null)
+[ -z "${date}" ] && date=$(date "${date_format}" 2>/dev/null)
+[ -z "${date}" ] && date=$(date --date=@${when} 2>/dev/null)
+[ -z "${date}" ] && date=$(date 2>/dev/null)
+
+# ----------------------------------------------------------------------------
+# prepare some extra headers if we've been asked to thread e-mails
+if [ "${SEND_EMAIL}" == "YES" ] && [ "${EMAIL_THREADING}" != "NO" ]; then
+  email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\\r\\nReferences: <${chart}-${name}@${host}>"
+else
+  email_thread_headers=
+fi
+
+# -----------------------------------------------------------------------------
+# function to URL encode a string
+
+urlencode() {
+  local string="${1}" strlen encoded pos c o
+
+  strlen=${#string}
+  for ((pos = 0; pos < strlen; pos++)); do
+    c=${string:pos:1}
+    case "${c}" in
+    [-_.~a-zA-Z0-9])
+      o="${c}"
+      ;;
+
+    *)
+      printf -v o '%%%02x' "'${c}"
+      ;;
+    esac
+    encoded+="${o}"
+  done
+
+  REPLY="${encoded}"
+  echo "${REPLY}"
+}
+
+# -----------------------------------------------------------------------------
+# function to convert a duration in seconds, to a human readable duration
+# using DAYS, MINUTES, SECONDS
+
+duration4human() {
+  local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second" ret
+  d=$((s / 86400))
+  s=$((s - (d * 86400)))
+  h=$((s / 3600))
+  s=$((s - (h * 3600)))
+  m=$((s / 60))
+  s=$((s - (m * 60)))
+
+  if [ ${d} -gt 0 ]; then
+    [ ${m} -ge 30 ] && h=$((h + 1))
+    [ ${d} -gt 1 ] && ds="days"
+    [ ${h} -gt 1 ] && hs="hours"
+    if [ ${h} -gt 0 ]; then
+      ret="${d} ${ds} and ${h} ${hs}"
+    else
+      ret="${d} ${ds}"
+    fi
+  elif [ ${h} -gt 0 ]; then
+    [ ${s} -ge 30 ] && m=$((m + 1))
+    [ ${h} -gt 1 ] && hs="hours"
+    [ ${m} -gt 1 ] && ms="minutes"
+    if [ ${m} -gt 0 ]; then
+      ret="${h} ${hs} and ${m} ${ms}"
+    else
+      ret="${h} ${hs}"
+    fi
+  elif [ ${m} -gt 0 ]; then
+    [ ${m} -gt 1 ] && ms="minutes"
+    [ ${s} -gt 1 ] && ss="seconds"
+    if [ ${s} -gt 0 ]; then
+      ret="${m} ${ms} and ${s} ${ss}"
+    else
+      ret="${m} ${ms}"
+    fi
+  else
+    [ ${s} -gt 1 ] && ss="seconds"
+    ret="${s} ${ss}"
+  fi
+
+  REPLY="${ret}"
+  echo "${REPLY}"
+}
+
+# -----------------------------------------------------------------------------
+# email sender
+
+send_email() {
+  local ret opts=() sender_email="${EMAIL_SENDER}" sender_name=
+  if [ "${SEND_EMAIL}" = "YES" ]; then
+
+    if [ -n "${EMAIL_SENDER}" ]; then
+      if [[ ${EMAIL_SENDER} =~ ^\".*\"\ \<.*\>$ ]]; then
+        # the name includes double quotes
+        sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+        sender_name="$(echo "${EMAIL_SENDER}" | cut -d '"' -f 2)"
+      elif [[ ${EMAIL_SENDER} =~ ^\'.*\'\ \<.*\>$ ]]; then
+        # the name includes single quotes
+        sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+        sender_name="$(echo "${EMAIL_SENDER}" | cut -d "'" -f 2)"
+      elif [[ ${EMAIL_SENDER} =~ ^.*\ \<.*\>$ ]]; then
+        # the name does not have any quotes
+        sender_email="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 2 | cut -d '>' -f 1)"
+        sender_name="$(echo "${EMAIL_SENDER}" | cut -d '<' -f 1)"
+      fi
+    fi
+
+    [ -n "${sender_email}" ] && opts+=(-f "${sender_email}")
+    [ -n "${sender_name}" ] && sendmail --help 2>&1 | grep -q "\-F " && opts+=(-F "${sender_name}")
+
+    if [ "${debug}" = "1" ]; then
+      echo >&2 "--- BEGIN sendmail command ---"
+      printf >&2 "%q " "${sendmail}" -t "${opts[@]}"
+      echo >&2
+      echo >&2 "--- END sendmail command ---"
+    fi
+
+    "${sendmail}" -t "${opts[@]}"
+    ret=$?
+
+    if [ ${ret} -eq 0 ]; then
+      info "sent email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}'"
+      return 0
+    else
+      error "failed to send email notification for: ${host} ${chart}.${name} is ${status} to '${to_email}' with error code ${ret}."
+      return 1
+    fi
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# pushover sender
+
+send_pushover() {
+  local apptoken="${1}" usertokens="${2}" when="${3}" url="${4}" status="${5}" title="${6}" message="${7}" httpcode sent=0 user priority
+
+  if [ "${SEND_PUSHOVER}" = "YES" ] && [ -n "${apptoken}" ] && [ -n "${usertokens}" ] && [ -n "${title}" ] && [ -n "${message}" ]; then
+
+    # https://pushover.net/api
+    priority=-2
+    case "${status}" in
+    CLEAR) priority=-1 ;; # low priority: no sound or vibration
+    WARNING) priority=0 ;; # normal priority: respect quiet hours
+    CRITICAL) priority=1 ;; # high priority: bypass quiet hours
+    *) priority=-2 ;; # lowest priority: no notification at all
+    esac
+
+    for user in ${usertokens}; do
+      httpcode=$(docurl \
+        --form-string "token=${apptoken}" \
+        --form-string "user=${user}" \
+        --form-string "html=1" \
+        --form-string "title=${title}" \
+        --form-string "message=${message}" \
+        --form-string "timestamp=${when}" \
+        --form-string "url=${url}" \
+        --form-string "url_title=Open netdata dashboard to view the alarm" \
+        --form-string "priority=${priority}" \
+        https://api.pushover.net/1/messages.json)
+
+      if [ "${httpcode}" = "200" ]; then
+        info "sent pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send pushover notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# pushbullet sender
+
+send_pushbullet() {
+  local userapikey="${1}" source_device="${2}" recipients="${3}" url="${4}" title="${5}" message="${6}" httpcode sent=0 user
+  if [ "${SEND_PUSHBULLET}" = "YES" ] && [ -n "${userapikey}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
+    #https://docs.pushbullet.com/#create-push
+    for user in ${recipients}; do
+      httpcode=$(docurl \
+        --header 'Access-Token: '${userapikey}'' \
+        --header 'Content-Type: application/json' \
+        --data-binary @<(
+          cat <<EOF
+                              {"title": "${title}",
+                              "type": "link",
+                              "email": "${user}",
+                              "body": "$(echo -n ${message})",
+                              "url": "${url}",
+                              "source_device_iden": "${source_device}"}
+EOF
+        ) "https://api.pushbullet.com/v2/pushes" -X POST)
+
+      if [ "${httpcode}" = "200" ]; then
+        info "sent pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send pushbullet notification for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# kafka sender
+
+send_kafka() {
+  local httpcode sent=0
+  if [ "${SEND_KAFKA}" = "YES" ]; then
+    httpcode=$(docurl -X POST \
+      --data "{host_ip:\"${KAFKA_SENDER_IP}\",when:${when},name:\"${name}\",chart:\"${chart}\",family:\"${family}\",status:\"${status}\",old_status:\"${old_status}\",value:${value},old_value:${old_value},duration:${duration},non_clear_duration:${non_clear_duration},units:\"${units}\",info:\"${info}\"}" \
+      "${KAFKA_URL}")
+
+    if [ "${httpcode}" = "204" ]; then
+      info "sent kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}'"
+      sent=$((sent + 1))
+    else
+      error "failed to send kafka data for: ${host} ${chart}.${name} is ${status} and ip '${KAFKA_SENDER_IP}' with HTTP response status code ${httpcode}."
+    fi
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# pagerduty.com sender
+
+send_pd() {
+  local recipients="${1}" sent=0 severity current_time payload url response_code
+  unset t
+  case ${status} in
+  CLEAR) t='resolve' ; severity='info' ;;
+  WARNING) t='trigger' ; severity='warning' ;;
+  CRITICAL) t='trigger' ; severity='critical' ;;
+  esac
+
+  if [ ${SEND_PD} = "YES" ] && [ -n "${t}" ]; then
+    if [ "$(uname)" == "Linux" ]; then
+      current_time=$(date -d @${when} +'%Y-%m-%dT%H:%M:%S.000')
+    else
+      current_time=$(date -r ${when} +'%Y-%m-%dT%H:%M:%S.000')
+    fi
+    for PD_SERVICE_KEY in ${recipients}; do
+      d="${status} ${name} = ${value_string} - ${host}, ${family}"
+      if [ ${USE_PD_VERSION} = "2" ]; then
+        payload="$(
+          cat <<EOF
+              {
+                "payload" : {
+                  "summary": "${info:0:1024}",
+                  "source" : "${args_host}",
+                  "severity" : "${severity}",
+                  "timestamp" : "${current_time}",
+                  "group" : "${family}",
+                  "class" : "${chart}",
+                  "custom_details": {
+                    "value_w_units": "${value_string}",
+                    "when": "${when}",
+                    "duration" : "${duration}",
+                    "roles": "${roles}",
+                    "alarm_id" : "${alarm_id}",
+                    "name" : "${name}",
+                    "chart" : "${chart}",
+                    "family" : "${family}",
+                    "status" : "${status}",
+                    "old_status" : "${old_status}",
+                    "value" : "${value}",
+                    "old_value" : "${old_value}",
+                    "src" : "${src}",
+                    "non_clear_duration" : "${non_clear_duration}",
+                    "units" : "${units}",
+                    "info" : "${info}"
+                  }
+                },
+                "routing_key": "${PD_SERVICE_KEY}",
+                "event_action": "${t}",
+                "dedup_key": "${unique_id}"
+              }
+EOF
+        )"
+        url="https://events.pagerduty.com/v2/enqueue"
+        response_code="202"
+      else
+        payload="$(
+          cat <<EOF
+            {
+              "service_key": "${PD_SERVICE_KEY}",
+              "event_type": "${t}",
+              "incident_key" : "${alarm_id}",
+              "description": "${d}",
+              "details": {
+                "value_w_units": "${value_string}",
+                "when": "${when}",
+                "duration" : "${duration}",
+                "roles": "${roles}",
+                "alarm_id" : "${alarm_id}",
+                "name" : "${name}",
+                "chart" : "${chart}",
+                "family" : "${family}",
+                "status" : "${status}",
+                "old_status" : "${old_status}",
+                "value" : "${value}",
+                "old_value" : "${old_value}",
+                "src" : "${src}",
+                "non_clear_duration" : "${non_clear_duration}",
+                "units" : "${units}",
+                "info" : "${info}"
+              }
+            }
+EOF
+        )"
+        url="https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+        response_code="200"
+      fi
+      httpcode=$(docurl -X POST --data "${payload}" ${url})
+      if [ "${httpcode}" = "${response_code}" ]; then
+        info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# twilio sender
+
+send_twilio() {
+  local accountsid="${1}" accounttoken="${2}" twilionumber="${3}" recipients="${4}" title="${5}" message="${6}" httpcode sent=0 user
+  if [ "${SEND_TWILIO}" = "YES" ] && [ -n "${accountsid}" ] && [ -n "${accounttoken}" ] && [ -n "${twilionumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
+    #https://www.twilio.com/packages/labs/code/bash/twilio-sms
+    for user in ${recipients}; do
+      httpcode=$(docurl -X POST \
+        --data-urlencode "From=${twilionumber}" \
+        --data-urlencode "To=${user}" \
+        --data-urlencode "Body=${title} ${message}" \
+        -u "${accountsid}:${accounttoken}" \
+        "https://api.twilio.com/2010-04-01/Accounts/${accountsid}/Messages.json")
+
+      if [ "${httpcode}" = "201" ]; then
+        info "sent Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send Twilio SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# hipchat sender
+
+send_hipchat() {
+  local authtoken="${1}" recipients="${2}" message="${3}" httpcode sent=0 room color msg_format notify
+
+  # remove <small></small> from the message
+  message="${message//<small>/}"
+  message="${message//<\/small>/}"
+
+  if [ "${SEND_HIPCHAT}" = "YES" ] && [ -n "${HIPCHAT_SERVER}" ] && [ -n "${authtoken}" ] && [ -n "${recipients}" ] && [ -n "${message}" ]; then
+    # Valid values: html, text.
+    # Defaults to 'html'.
+    msg_format="html"
+
+    # Background color for message. Valid values: yellow, green, red, purple, gray, random. Defaults to 'yellow'.
+    case "${status}" in
+    WARNING) color="yellow" ;;
+    CRITICAL) color="red" ;;
+    CLEAR) color="green" ;;
+    *) color="gray" ;;
+    esac
+
+    # Whether this message should trigger a user notification (change the tab color, play a sound, notify mobile phones, etc).
+    # Each recipient's notification preferences are taken into account.
+    # Defaults to false.
+    notify="true"
+
+    for room in ${recipients}; do
+      httpcode=$(docurl -X POST \
+        -H "Content-type: application/json" \
+        -H "Authorization: Bearer ${authtoken}" \
+        -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \
+        "https://${HIPCHAT_SERVER}/v2/room/${room}/notification")
+
+      if [ "${httpcode}" = "204" ]; then
+        info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# messagebird sender
+
+send_messagebird() {
+  local accesskey="${1}" messagebirdnumber="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
+  if [ "${SEND_MESSAGEBIRD}" = "YES" ] && [ -n "${accesskey}" ] && [ -n "${messagebirdnumber}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
+    #https://developers.messagebird.com/docs/messaging
+    for user in ${recipients}; do
+      httpcode=$(docurl -X POST \
+        --data-urlencode "originator=${messagebirdnumber}" \
+        --data-urlencode "recipients=${user}" \
+        --data-urlencode "body=${title} ${message}" \
+        --data-urlencode "datacoding=auto" \
+        -H "Authorization: AccessKey ${accesskey}" \
+        "https://rest.messagebird.com/messages")
+
+      if [ "${httpcode}" = "201" ]; then
+        info "sent Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send Messagebird SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# kavenegar sender
+
+send_kavenegar() {
+  local API_KEY="${1}" kavenegarsender="${2}" recipients="${3}" title="${4}" message="${5}" httpcode sent=0 user
+  if [ "${SEND_KAVENEGAR}" = "YES" ] && [ -n "${API_KEY}" ] && [ -n "${kavenegarsender}" ] && [ -n "${recipients}" ] && [ -n "${message}" ] && [ -n "${title}" ]; then
+    # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
+    for user in ${recipients}; do
+      httpcode=$(docurl -X POST http://api.kavenegar.com/v1/${API_KEY}/sms/send.json \
+        --data-urlencode "sender=${kavenegarsender}" \
+        --data-urlencode "receptor=${user}" \
+        --data-urlencode "message=${title} ${message}")
+
+      if [ "${httpcode}" = "200" ]; then
+        info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# telegram sender
+
+send_telegram() {
+  local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification=""
+
+  if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi
+
+  case "${status}" in
+  WARNING) emoji="⚠️" ;;
+  CRITICAL) emoji="🔴" ;;
+  CLEAR) emoji="✅" ;;
+  *) emoji="⚪️" ;;
+  esac
+
+  if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
+    for chatid in ${chatids}; do
+      # https://core.telegram.org/bots/api#sendmessage
+      httpcode=$(docurl ${disableNotification} \
+        --data-urlencode "parse_mode=HTML" \
+        --data-urlencode "disable_web_page_preview=true" \
+        --data-urlencode "text=${emoji} ${message}" \
+        "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
+
+      if [ "${httpcode}" = "200" ]; then
+        info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
+        sent=$((sent + 1))
+      elif [ "${httpcode}" = "401" ]; then
+        error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
+      else
+        error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# Microsoft Team sender
+
+send_msteam() {
+
+  local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+  [ "${SEND_MSTEAM}" != "YES" ] && return 1
+
+  case "${status}" in
+  WARNING) icon="${MSTEAM_ICON_WARNING}" && color="${MSTEAM_COLOR_WARNING}" ;;
+  CRITICAL) icon="${MSTEAM_ICON_CRITICAL}" && color="${MSTEAM_COLOR_CRITICAL}" ;;
+  CLEAR) icon="${MSTEAM_ICON_CLEAR}" && color="${MSTEAM_COLOR_CLEAR}" ;;
+  *) icon="${MSTEAM_ICON_DEFAULT}" && color="${MSTEAM_COLOR_DEFAULT}" ;;
+  esac
+
+  for channel in ${channels}; do
+    ## More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
+    ## Online designer : https://acdesignerbeta.azurewebsites.net/
+    payload="$(
+      cat <<EOF
+        {
+            "@context": "http://schema.org/extensions",
+            "@type": "MessageCard",
+            "themeColor": "${color}",
+            "title": "$icon Alert ${status} from netdata for ${host}",
+            "text": "${host} ${status_message}, ${chart} (_${family}_), *${alarm}*",
+            "potentialAction": [
+            {
+                "@type": "OpenUri",
+                "name": "Netdata",
+                "targets": [
+                { "os": "default", "uri": "${goto_url}" }
+                ]
+            }
+            ]
+        }
+EOF
+    )"
+
+    # Replacing in the webhook CHANNEL string by the MS Teams channel name from conf file.
+    webhook="${webhook//CHANNEL/${channel}}"
+
+    httpcode=$(docurl -H "Content-Type: application/json" -d "${payload}" "${webhook}")
+
+    if [ "${httpcode}" = "200" ]; then
+      info "sent Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}'"
+      sent=$((sent + 1))
+    else
+      error "failed to send Microsoft team notification for: ${host} ${chart}.${name} is ${status} to '${webhook}', with HTTP response status code ${httpcode}."
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# slack sender
+
+send_slack() {
+  local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+  [ "${SEND_SLACK}" != "YES" ] && return 1
+
+  case "${status}" in
+  WARNING) color="warning" ;;
+  CRITICAL) color="danger" ;;
+  CLEAR) color="good" ;;
+  *) color="#777777" ;;
+  esac
+
+  for channel in ${channels}; do
+    # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user.
+    if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ]; then channel="#$channel"; fi
+
+    # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks.
+    if [ "${channel}" = "#" ]; then
+      ch=""
+      chstr="without specifying a channel"
+    else
+      ch="\"channel\": \"${channel}\","
+      chstr="to '${channel}'"
+    fi
+
+    payload="$(
+      cat <<EOF
+        {
+            $ch
+            "username": "netdata on ${host}",
+            "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
+            "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
+            "attachments": [
+                {
+                    "fallback": "${alarm} - ${chart} (${family}) - ${info}",
+                    "color": "${color}",
+                    "title": "${alarm}",
+                    "title_link": "${goto_url}",
+                    "text": "${info}",
+                    "fields": [
+                        {
+                            "title": "${chart}",
+                            "short": true
+                        },
+                        {
+                            "title": "${family}",
+                            "short": true
+                        }
+                    ],
+                    "thumb_url": "${image}",
+                    "footer": "by ${host}",
+                    "ts": ${when}
+                }
+            ]
+        }
+EOF
+    )"
+
+    httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
+    if [ "${httpcode}" = "200" ]; then
+      info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}"
+      sent=$((sent + 1))
+    else
+      error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP response status code ${httpcode}."
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# rocketchat sender
+
+send_rocketchat() {
+  local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+  [ "${SEND_ROCKETCHAT}" != "YES" ] && return 1
+
+  case "${status}" in
+  WARNING) color="warning" ;;
+  CRITICAL) color="danger" ;;
+  CLEAR) color="good" ;;
+  *) color="#777777" ;;
+  esac
+
+  for channel in ${channels}; do
+    payload="$(
+      cat <<EOF
+        {
+            "channel": "#${channel}",
+            "alias": "netdata on ${host}",
+            "avatar": "${images_base_url}/images/banner-icon-144x144.png",
+            "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
+            "attachments": [
+                {
+                    "color": "${color}",
+                    "title": "${alarm}",
+                    "title_link": "${goto_url}",
+                    "text": "${info}",
+                    "fields": [
+                        {
+                            "title": "${chart}",
+                            "short": true,
+                            "value": "chart"
+                        },
+                        {
+                            "title": "${family}",
+                            "short": true,
+                            "value": "family"
+                        }
+                    ],
+                    "thumb_url": "${image}",
+                    "ts": "${when}"
+                }
+            ]
+        }
+EOF
+    )"
+
+    httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
+    if [ "${httpcode}" = "200" ]; then
+      info "sent rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+      sent=$((sent + 1))
+    else
+      error "failed to send rocketchat notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# alerta sender
+
+send_alerta() {
+  local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
+
+  [ "${SEND_ALERTA}" != "YES" ] && return 1
+
+  case "${status}" in
+  CRITICAL) severity="critical" ;;
+  WARNING) severity="warning" ;;
+  CLEAR) severity="cleared" ;;
+  *) severity="indeterminate" ;;
+  esac
+
+  if [[ ${chart} == httpcheck* ]]; then
+    resource=$chart
+    event=$name
+  else
+    resource="${host}:${family}"
+    event="${chart}.${name}"
+  fi
+
+  for channel in ${channels}; do
+    payload="$(
+      cat <<EOF
+        {
+            "resource": "${resource}",
+            "event": "${event}",
+            "environment": "${channel}",
+            "severity": "${severity}",
+            "service": ["Netdata"],
+            "group": "Performance",
+            "value": "${value_string}",
+            "text": "${info}",
+            "tags": ["alarm_id:${alarm_id}"],
+            "attributes": {
+                "roles": "${roles}",
+                "name": "${name}",
+                "chart": "${chart}",
+                "family": "${family}",
+                "source": "${src}",
+                "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
+            },
+            "origin": "netdata/${host}",
+            "type": "netdataAlarm",
+            "rawData": "${BASH_ARGV[@]}"
+        }
+EOF
+    )"
+
+    if [ -n "${ALERTA_API_KEY}" ]; then
+      auth="Key ${ALERTA_API_KEY}"
+    fi
+
+    httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
+
+    if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "201" ]; then
+      info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+      sent=$((sent + 1))
+    elif [ "${httpcode}" = "202" ]; then
+      info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+    else
+      error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# flock sender
+
+send_flock() {
+  local webhook="${1}" channels="${2}" httpcode sent=0 channel color payload
+
+  [ "${SEND_FLOCK}" != "YES" ] && return 1
+
+  case "${status}" in
+  WARNING) color="warning" ;;
+  CRITICAL) color="danger" ;;
+  CLEAR) color="good" ;;
+  *) color="#777777" ;;
+  esac
+
+  for channel in ${channels}; do
+    httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
+            \"sendAs\": {
+                \"name\" : \"netdata on ${host}\",
+                \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
+            },
+            \"text\": \"${host} *${status_message}*\",
+            \"timestamp\": \"${when}\",
+            \"attachments\": [
+                {
+                    \"description\": \"${chart} (${family}) - ${info}\",
+                    \"color\": \"${color}\",
+                    \"title\": \"${alarm}\",
+                    \"url\": \"${goto_url}\",
+                    \"text\": \"${info}\",
+                    \"views\": {
+                        \"image\": {
+                            \"original\": { \"src\": \"${image}\", \"width\": 400, \"height\": 400 },
+                            \"thumbnail\": { \"src\": \"${image}\", \"width\": 50, \"height\": 50 },
+                            \"filename\": \"${image}\"
+                            }
+                    }
+                }
+            ]
+        }")
+    if [ "${httpcode}" = "200" ]; then
+      info "sent flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+      sent=$((sent + 1))
+    else
+      error "failed to send flock notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# discord sender
+
+send_discord() {
+  local webhook="${1}/slack" channels="${2}" httpcode sent=0 channel color payload username
+
+  [ "${SEND_DISCORD}" != "YES" ] && return 1
+
+  case "${status}" in
+  WARNING) color="warning" ;;
+  CRITICAL) color="danger" ;;
+  CLEAR) color="good" ;;
+  *) color="#777777" ;;
+  esac
+
+  for channel in ${channels}; do
+    username="netdata on ${host}"
+    [ ${#username} -gt 32 ] && username="${username:0:29}..."
+
+    payload="$(
+      cat <<EOF
+        {
+            "channel": "#${channel}",
+            "username": "${username}",
+            "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
+            "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
+            "attachments": [
+                {
+                    "color": "${color}",
+                    "title": "${alarm}",
+                    "title_link": "${goto_url}",
+                    "text": "${info}",
+                    "fields": [
+                        {
+                            "title": "${chart}",
+                            "value": "${family}"
+                        }
+                    ],
+                    "thumb_url": "${image}",
+                    "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
+                    "footer": "${host}",
+                    "ts": ${when}
+                }
+            ]
+        }
+EOF
+    )"
+
+    httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}")
+    if [ "${httpcode}" = "200" ]; then
+      info "sent discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+      sent=$((sent + 1))
+    else
+      error "failed to send discord notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP response status code ${httpcode}."
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# fleep sender
+
+send_fleep() {
+  local httpcode sent=0 webhooks="${1}" data message
+  if [ "${SEND_FLEEP}" = "YES" ]; then
+    message="${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}"
+
+    for hook in ${webhooks}; do
+      data="{ "
+      data="${data} 'message': '${message}', "
+      data="${data} 'user': '${FLEEP_SENDER}' "
+      data="${data} }"
+
+      httpcode=$(docurl -X POST --data "${data}" "https://fleep.io/hook/${hook}")
+
+      if [ "${httpcode}" = "200" ]; then
+        info "sent fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send fleep data for: ${host} ${chart}.${name} is ${status} and user '${FLEEP_SENDER}' with HTTP response status code ${httpcode}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# Prowl sender
+
+send_prowl() {
+  local httpcode sent=0 data message keys prio=0 alarm_url event
+  if [ "${SEND_PROWL}" = "YES" ]; then
+    message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")"
+    message="description=${message}"
+    keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")"
+    keys="apikey=${keys}"
+    app="application=Netdata"
+
+    case "${status}" in
+    CRITICAL)
+      prio=2
+      ;;
+    WARNING)
+      prio=1
+      ;;
+    esac
+    prio="priority=${prio}"
+
+    alarm_url="$(urlencode ${goto_url})"
+    alarm_url="url=${alarm_url}"
+    event="$(urlencode "${host} ${status_message}")"
+    event="event=${event}"
+
+    data="${keys}&${prio}&${alarm_url}&${app}&${event}&${message}"
+
+    httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add")
+
+    if [ "${httpcode}" = "200" ]; then
+      info "sent prowl data for: ${host} ${chart}.${name} is ${status}"
+      sent=1
+    else
+      error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}."
+    fi
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# irc sender
+
+send_irc() {
+  local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error
+
+  if [ "${SEND_IRC}" = "YES" ] && [ -n "${NICKNAME}" ] && [ -n "${REALNAME}" ] && [ -n "${CHANNELS}" ] && [ -n "${NETWORK}" ] && [ -n "${SERVERNAME}" ]; then
+    case "${status}" in
+    WARNING) color="warning" ;;
+    CRITICAL) color="danger" ;;
+    CLEAR) color="good" ;;
+    *) color="#777777" ;;
+    esac
+
+    SNDMESSAGE="${MESSAGE//$'\n'/", "}"
+    for CHANNEL in ${CHANNELS}; do
+      error=0
+      send_alarm=$(echo -e "USER ${NICKNAME} guest ${REALNAME} ${SERVERNAME}\\nNICK ${NICKNAME}\\nJOIN ${CHANNEL}\\nPRIVMSG ${CHANNEL} :${SNDMESSAGE}\\nQUIT\\n" \  | nc "${NETWORK}" 6667)
+      reply_codes=$(echo "${send_alarm}" | cut -d ' ' -f 2 | grep -o '[0-9]*')
+      for code in ${reply_codes}; do
+        if [ "${code}" -ge 400 ] && [ "${code}" -le 599 ]; then
+          error=1
+          break
+        fi
+      done
+
+      if [ "${error}" -eq 0 ]; then
+        info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}."
+      fi
+    done
+  fi
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# Amazon SNS sender
+
+send_awssns() {
+  local targets="${1}" message='' sent=0 region=''
+  local default_format="${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+  [ "${SEND_AWSSNS}" = "YES" ] || return 1
+
+  message=${AWSSNS_MESSAGE_FORMAT:-${default_format}}
+
+  for target in ${targets}; do
+    # Extract the region from the target ARN.  We need to explicitly specify the region so that it matches up correctly.
+    region="$(echo ${target} | cut -f 4 -d ':')"
+    if ${aws} sns publish --region "${region}" --subject "${host} ${status_message} - ${name//_/ } - ${chart}" --message "${message}" --target-arn ${target} &>/dev/null; then
+      info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
+      sent=$((sent + 1))
+    else
+      error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'"
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# Matrix sender
+
+send_matrix() {
+  local homeserver="${1}" webhook accesstoken rooms="${2}" httpcode sent=0 payload
+
+  [ "${SEND_MATRIX}" != "YES" ] && return 1
+  [ -z "${MATRIX_ACCESSTOKEN}" ] && return 1
+
+  accesstoken="${MATRIX_ACCESSTOKEN}"
+
+  case "${status}" in
+  WARNING) emoji="⚠️" ;;
+  CRITICAL) emoji="🔴" ;;
+  CLEAR) emoji="✅" ;;
+  *) emoji="⚪️" ;;
+  esac
+
+  for room in ${rooms}; do
+    webhook="$homeserver/_matrix/client/r0/rooms/$(urlencode $room)/send/m.room.message?access_token=$accesstoken"
+    payload="$(
+  cat <<EOF
+  {
+    "msgtype": "m.notice",
+    "format": "org.matrix.custom.html",
+      "formatted_body": "${emoji} ${host} ${status_message} - <b>${name//_/ }</b><br>${chart} (${family})<br><a href=\"${goto_url}\">${alarm}</a><br><i>${info}</i>",
+      "body": "${emoji} ${host} ${status_message} - ${name//_/ } ${chart} (${family}) ${goto_url} ${alarm} ${info}"
+  }
+EOF
+    )"
+
+    httpcode=$(docurl -X POST --data "${payload}" "${webhook}")
+    if [ "${httpcode}" == "200" ]; then
+      info "sent Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
+      sent=$((sent + 1))
+    else
+      error "failed to send Matrix notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP response status code ${httpcode}."
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# syslog sender
+
+send_syslog() {
+  local facility=${SYSLOG_FACILITY:-"local6"} level='info' targets="${1}"
+  local priority='' message='' server='' port='' prefix=''
+  local temp1='' temp2=''
+
+  [ "${SEND_SYSLOG}" = "YES" ] || return 1
+
+  if [ "${status}" = "CRITICAL" ]; then
+    level='crit'
+  elif [ "${status}" = "WARNING" ]; then
+    level='warning'
+  fi
+
+  for target in ${targets}; do
+    priority="${facility}.${level}"
+    message=''
+    server=''
+    port=''
+    prefix=''
+    temp1=''
+    temp2=''
+
+    prefix=$(echo ${target} | cut -d '/' -f 2)
+    temp1=$(echo ${target} | cut -d '/' -f 1)
+
+    if [ ${prefix} != ${temp1} ]; then
+      if (echo ${temp1} | grep -q '@'); then
+        temp2=$(echo ${temp1} | cut -d '@' -f 1)
+        server=$(echo ${temp1} | cut -d '@' -f 2)
+
+        if [ ${temp2} != ${server} ]; then
+          priority=${temp2}
+        fi
+
+        port=$(echo ${server} | rev | cut -d ':' -f 1 | rev)
+
+        if (echo ${server} | grep -E -q '\[.*\]'); then
+          if (echo ${port} | grep -q ']'); then
+            port=''
+          else
+            server=$(echo ${server} | rev | cut -d ':' -f 2- | rev)
+          fi
+        else
+          if [ ${port} = ${server} ]; then
+            port=''
+          else
+            server=$(echo ${server} | cut -d ':' -f 1)
+          fi
+        fi
+      else
+        priority=${temp1}
+      fi
+    fi
+
+    message="${prefix} ${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+    if [ ${server} ]; then
+      logger_options="${logger_options} -n ${server}"
+      if [ ${port} ]; then
+        logger_options="${logger_options} -P ${port}"
+      fi
+    fi
+
+    ${logger} -p ${priority} ${logger_options} "${message}"
+  done
+
+  return $?
+}
+
+# -----------------------------------------------------------------------------
+# SMS sender
+
+send_sms() {
+  local recipients="${1}" errcode errmessage sent=0
+
+    # Human readable SMS
+    local msg="${host} ${status_message}: ${chart} (${family}), ${alarm}"
+
+    # limit it to 160 characters
+    msg="${msg:0:160}"
+
+  if [ "${SEND_SMS}" = "YES" ] && [ -n "${sendsms}" ] && [ -n "${recipients}" ] && [ -n "${msg}" ]; then
+    # http://api.kavenegar.com/v1/{API-KEY}/sms/send.json
+    for phone in ${recipients}; do
+      errmessage=$($sendsms $phone "$msg" 2>&1)
+      errcode=$?
+      if [ ${errcode} -eq 0 ]; then
+        info "sent smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send smstools3 SMS for: ${host} ${chart}.${name} is ${status} to '${user}' with error code ${errcode}: ${errmessage}."
+      fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+  fi
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# hangouts sender
+
+send_hangouts() {
+  local rooms="${1}" httpcode sent=0 room color payload webhook thread
+
+  [ "${SEND_HANGOUTS}" != "YES" ] && return 1
+
+  case "${status}" in
+  WARNING) color="#ffa700" ;;
+  CRITICAL) color="#d62d20" ;;
+  CLEAR) color="#008744" ;;
+  *) color="#777777" ;;
+  esac
+
+  for room in ${rooms}; do
+    if [ -z "${HANGOUTS_WEBHOOK_URI[$room]}" ] ; then
+      info "Can't send Hangouts notification for: ${host} ${chart}.${name} to room ${room}. HANGOUTS_WEBHOOK_URI[$room] not defined"
+    else
+      if [ -n "${HANGOUTS_WEBHOOK_THREAD[$room]}" ]; then
+        thread="\"name\" : \"${HANGOUTS_WEBHOOK_THREAD[$room]}\""
+      fi
+      webhook="${HANGOUTS_WEBHOOK_URI[$room]}"
+      payload="$(
+        cat <<EOF
+        {
+          "cards": [
+            {
+              "header": {
+                "title": "Netdata on ${host}",
+                "imageUrl": "${images_base_url}/images/banner-icon-144x144.png",
+                "imageStyle": "IMAGE"
+              },
+              "sections": [
+                {
+                  "header": "<b>${host}</b>",
+                  "widgets": [
+                    {
+                      "keyValue": {
+                        "topLabel": "Status Message",
+                        "content": "<b>${status_message}</b>",
+                        "contentMultiline": "true",
+                        "iconUrl": "${image}",
+                        "onClick": {
+                          "openLink": {
+                            "url": "${goto_url}"
+                          }
+                        }
+                      }
+                    },
+                    {
+                      "keyValue": {
+                        "topLabel": "${chart} | ${family}",
+                        "content": "<font color=${color}>${alarm}</font>",
+                        "contentMultiline": "true"
+                      }
+                    }
+                  ]
+                },
+                {
+                  "widgets": [
+                    {
+                      "textParagraph": {
+                      "text": "<font color=\"#0057e7\">@ ${date}\n<b>${info}</b></font>"
+                      }
+                    }
+                  ]
+                },
+                {
+                  "widgets": [
+                    {
+                      "buttons": [
+                        {
+                          "textButton": {
+                            "text": "Go to ${host}",
+                            "onClick": {
+                              "openLink": {
+                                "url": "${goto_url}"
+                              }
+                            }
+                          }
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }
+          ],
+          "thread": {
+            $thread
+          }
+        }
+EOF
+        )"
+
+      httpcode=$(docurl -H "Content-Type: application/json" -X POST -d "${payload}" "${webhook}")
+
+      if [ "${httpcode}" = "200" ]; then
+        info "sent hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}'"
+        sent=$((sent + 1))
+      else
+        error "failed to send hangouts notification for: ${host} ${chart}.${name} is ${status} to '${room}', with HTTP response status code ${httpcode}."
+      fi
+    fi
+  done
+
+  [ ${sent} -gt 0 ] && return 0
+
+  return 1
+}
+
+# -----------------------------------------------------------------------------
+# Dynatrace sender
+
+send_dynatrace() {
+  [ "${SEND_DYNATRACE}" != "YES" ] && return 1
+
+  local dynatrace_url="${DYNATRACE_SERVER}/e/${DYNATRACE_SPACE}/api/v1/events"
+  local description="NetData Notification for: ${host} ${chart}.${name} is ${status}"
+  local payload=""
+
+  payload=$(cat <<EOF
+{
+  "title": "NetData Alarm from ${host}",
+  "source" : "${DYNATRACE_ANNOTATION_TYPE}",
+  "description" : "${description}",
+  "eventType": "${DYNATRACE_EVENT}",
+   "attachRules":{
+     "tagRule":[{
+        "meTypes":["HOST"],
+        "tags":["${DYNATRACE_TAG_VALUE}"]
+     }]
+  },
+  "customProperties":{
+    "description": "${description}"
+  }
+}
+EOF
+)
+
+  # echo ${payload}
+
+  httpcode=$(docurl -X POST  -H "Authorization: Api-token ${DYNATRACE_TOKEN}"  -H "Content-Type: application/json" -d "${payload}"    ${dynatrace_url})
+  ret=$?
+
+
+  if [ ${ret} -eq 0 ]; then
+     if [ "${httpcode}" = "200" ]; then
+        info "sent ${DYNATRACE_EVENT} to ${DYNATRACE_SERVER}"
+        return 0
+     else
+        warning "Dynatrace ${DYNATRACE_SERVER} responded ${httpcode} notification for: ${host} ${chart}.${name} is ${status} was not sent!"
+        return 1
+     fi
+  else
+     error  "failed to sent ${DYNATRACE_EVENT} notification for: ${host} ${chart}.${name} is ${status} to  ${DYNATRACE_SERVER} with error code ${ret}."
+     return 1
+  fi
+}
+
+
+# -----------------------------------------------------------------------------
+# Stackpulse sender
+
+send_stackpulse() {
+  local payload httpcode oldv currv
+  [ "${SEND_STACKPULSE}" != "YES" ] && return 1
+
+  # We are sending null when values are nan to avoid errors while JSON message is parsed
+  [ "${old_value}" != "nan" ] && oldv="${old_value}" ||  oldv="null"
+  [ "${value}" != "nan" ] && currv="${value}" || currv="null"
+
+  payload=$(cat <<EOF
+  {
+    "Node" : "${host}",
+    "Chart" : "${chart}",
+    "OldValue" : ${oldv},
+    "Value" : ${currv},
+    "Units" : "${units}",
+    "OldStatus" : "${old_status}",
+    "Status" : "${status}",
+    "Alarm" : "${name}",
+    "Date": ${when},
+    "Duration": ${duration},
+    "NonClearDuration": ${non_clear_duration},
+    "Description" : "${status_message}, ${info}",
+    "CalcExpression" : "${calc_expression}",
+    "CalcParamValues" : "${calc_param_values}",
+    "TotalWarnings" : "${total_warnings}",
+    "TotalCritical" : "${total_critical}",
+    "ID" : ${alarm_id}
+  }
+EOF
+)
+
+  httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" ${STACKPULSE_WEBHOOK})
+  if [ "${httpcode}" = "200" ]; then
+    info "sent stackpulse notification for: ${host} ${chart}.${name} is ${status}"
+  else
+    error "failed to send stackpulse notification for: ${host} ${chart}.${name} is ${status}, with HTTP response status code ${httpcode}."
+    return 1
+  fi
+
+  return 0
+}
+# -----------------------------------------------------------------------------
+# Opsgenie sender
+
+send_opsgenie() {
+  local payload httpcode oldv currv
+  [ "${SEND_OPSGENIE}" != "YES" ] && return 1
+
+  if [ -z "${OPSGENIE_API_KEY}" ] ; then
+    info "Can't send Opsgenie notification, because OPSGENIE_API_KEY is not defined"
+    return 1
+  fi
+
+  # We are sending null when values are nan to avoid errors while JSON message is parsed
+  [ "${old_value}" != "nan" ] && oldv="${old_value}" ||  oldv="null"
+  [ "${value}" != "nan" ] && currv="${value}" || currv="null"
+
+  payload=$(cat <<EOF
+  {
+    "host" : "${host}",
+    "unique_id" : "${unique_id}",
+    "alarmId" : ${alarm_id},
+    "eventId" : ${event_id},
+    "chart" : "${chart}",
+    "when": ${when},
+    "name" : "${name}",
+    "family" : "${family}",
+    "status" : "${status}",
+    "old_status" : "${old_status}",
+    "value" : ${currv},
+    "old_value" : ${oldv},
+    "duration": ${duration},
+    "non_clear_duration": ${non_clear_duration},
+    "units" : "${units}",
+    "info" : "${status_message}, ${info}",
+    "calc_expression" : "${calc_expression}",
+    "total_warnings" : "${total_warnings}",
+    "total_critical" : "${total_critical}",
+    "src" : "${src}"
+  }
+EOF
+)
+
+  httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${OPSGENIE_API_URL}/v1/json/integrations/webhooks/netdata?apiKey=${OPSGENIE_API_KEY}")
+  # https://docs.opsgenie.com/docs/alert-api#create-alert
+  if [ "${httpcode}" = "200" ]; then
+    info "sent opsgenie notification for: ${host} ${chart}.${name} is ${status}"
+  else
+    error "failed to send opsgenie notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}."
+    return 1
+  fi
+
+  return 0
+}
+
+# -----------------------------------------------------------------------------
+# prepare the content of the notification
+
+# the url to send the user on click
+urlencode "${args_host}" >/dev/null
+url_host="${REPLY}"
+urlencode "${chart}" >/dev/null
+url_chart="${REPLY}"
+urlencode "${family}" >/dev/null
+url_family="${REPLY}"
+urlencode "${name}" >/dev/null
+url_name="${REPLY}"
+
+redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}&alarm_when=${when}"
+GOTOCLOUD=0
+
+if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ]; then
+  if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
+    if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
+      NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
+    fi
+  fi
+  if [ -n "${NETDATA_REGISTRY_UNIQUE_ID}" ]; then
+    GOTOCLOUD=1
+  fi
+fi
+
+if [ ${GOTOCLOUD} -eq 0 ]; then
+  goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}"
+else
+  # Temporarily disable alarm redirection, as the cloud endpoint no longer exists. This functionality will be restored after discussion on #9487. For now, just lead to netdata.cloud
+  #goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}/alarms/redirect?agentID=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}"
+  goto_url="${NETDATA_REGISTRY_CLOUD_BASE_URL}"
+fi
+
+# the severity of the alarm
+severity="${status}"
+
+# the time the alarm was raised
+duration4human ${duration} >/dev/null
+duration_txt="${REPLY}"
+duration4human ${non_clear_duration} >/dev/null
+non_clear_duration_txt="${REPLY}"
+raised_for="(was ${old_status,,} for ${duration_txt})"
+
+# the key status message
+status_message="status unknown"
+
+# the color of the alarm
+color="grey"
+
+# the alarm value
+alarm="${name//_/ } = ${value_string}"
+
+# the image of the alarm
+image="${images_base_url}/images/banner-icon-144x144.png"
+
+# prepare the title based on status
+case "${status}" in
+CRITICAL)
+  image="${images_base_url}/images/alert-128-red.png"
+  status_message="is critical"
+  color="#ca414b"
+  ;;
+
+WARNING)
+  image="${images_base_url}/images/alert-128-orange.png"
+  status_message="needs attention"
+  color="#ffc107"
+  ;;
+
+CLEAR)
+  image="${images_base_url}/images/check-mark-2-128-green.png"
+  status_message="recovered"
+  color="#77ca6d"
+  ;;
+esac
+
+if [ "${status}" = "CLEAR" ]; then
+  severity="Recovered from ${old_status}"
+  if [ ${non_clear_duration} -gt ${duration} ]; then
+    raised_for="(alarm was raised for ${non_clear_duration_txt})"
+  fi
+
+  # don't show the value when the status is CLEAR
+  # for certain alarms, this value might not have any meaning
+  alarm="${name//_/ } ${raised_for}"
+
+elif { [ "${old_status}" = "WARNING" ] && [ "${status}" = "CRITICAL" ]; }; then
+  severity="Escalated to ${status}"
+  if [ ${non_clear_duration} -gt ${duration} ]; then
+    raised_for="(alarm is raised for ${non_clear_duration_txt})"
+  fi
+
+elif { [ "${old_status}" = "CRITICAL" ] && [ "${status}" = "WARNING" ]; }; then
+  severity="Demoted to ${status}"
+  if [ ${non_clear_duration} -gt ${duration} ]; then
+    raised_for="(alarm is raised for ${non_clear_duration_txt})"
+  fi
+
+else
+  raised_for=
+fi
+
+# prepare HTML versions of elements
+info_html=
+[ -n "${info}" ] && info_html=" <small><br/>${info}</small>"
+
+raised_for_html=
+[ -n "${raised_for}" ] && raised_for_html="<br/><small>${raised_for}</small>"
+
+# -----------------------------------------------------------------------------
+# send the slack notification
+
+# slack aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
+SENT_SLACK=$?
+
+# -----------------------------------------------------------------------------
+# send the hangouts notification
+
+# hangouts aggregates posts from the same room
+# so we use "${host} ${status}" as the room, to make them diff
+
+send_hangouts "${to_hangouts}"
+SENT_HANGOUTS=$?
+
+# -----------------------------------------------------------------------------
+# send the Microsoft notification
+
+# Microsoft team aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_msteam "${MSTEAM_WEBHOOK_URL}" "${to_msteam}"
+SENT_MSTEAM=$?
+
+# -----------------------------------------------------------------------------
+# send the rocketchat notification
+
+# rocketchat aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_rocketchat "${ROCKETCHAT_WEBHOOK_URL}" "${to_rocketchat}"
+SENT_ROCKETCHAT=$?
+
+# -----------------------------------------------------------------------------
+# send the alerta notification
+
+# alerta aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
+SENT_ALERTA=$?
+
+# -----------------------------------------------------------------------------
+# send the flock notification
+
+# flock aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_flock "${FLOCK_WEBHOOK_URL}" "${to_flock}"
+SENT_FLOCK=$?
+
+# -----------------------------------------------------------------------------
+# send the discord notification
+
+# discord aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_discord "${DISCORD_WEBHOOK_URL}" "${to_discord}"
+SENT_DISCORD=$?
+
+# -----------------------------------------------------------------------------
+# send the pushover notification
+
+send_pushover "${PUSHOVER_APP_TOKEN}" "${to_pushover}" "${when}" "${goto_url}" "${status}" "${host} ${status_message} - ${name//_/ } - ${chart}" "
+<font color=\"${color}\"><b>${alarm}</b></font>${info_html}<br/>&nbsp;
+<small><b>${chart}</b><br/>Chart<br/>&nbsp;</small>
+<small><b>${family}</b><br/>Family<br/>&nbsp;</small>
+<small><b>${severity}</b><br/>Severity<br/>&nbsp;</small>
+<small><b>${date}${raised_for_html}</b><br/>Time<br/>&nbsp;</small>
+<a href=\"${goto_url}\">View Netdata</a><br/>&nbsp;
+<small><small>The source of this alarm is line ${src}</small></small>
+"
+
+SENT_PUSHOVER=$?
+
+# -----------------------------------------------------------------------------
+# send the pushbullet notification
+
+send_pushbullet "${PUSHBULLET_ACCESS_TOKEN}" "${PUSHBULLET_SOURCE_DEVICE}" "${to_pushbullet}" "${goto_url}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}\\n
+Severity: ${severity}\\n
+Chart: ${chart}\\n
+Family: ${family}\\n
+${date}\\n
+The source of this alarm is line ${src}"
+
+SENT_PUSHBULLET=$?
+
+# -----------------------------------------------------------------------------
+# send the twilio SMS
+
+send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_TWILIO=$?
+
+# -----------------------------------------------------------------------------
+# send the messagebird SMS
+
+send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_MESSAGEBIRD=$?
+
+# -----------------------------------------------------------------------------
+# send the kavenegar SMS
+
+send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_KAVENEGAR=$?
+
+# -----------------------------------------------------------------------------
+# send the telegram.org message
+
+# https://core.telegram.org/bots/api#formatting-options
+send_telegram "${TELEGRAM_BOT_TOKEN}" "${to_telegram}" "${host} ${status_message} - <b>${name//_/ }</b>
+${chart} (${family})
+<a href=\"${goto_url}\">${alarm}</a>
+<i>${info}</i>"
+
+SENT_TELEGRAM=$?
+
+# -----------------------------------------------------------------------------
+# send the kafka message
+
+send_kafka
+SENT_KAFKA=$?
+
+# -----------------------------------------------------------------------------
+# send the pagerduty.com message
+
+send_pd "${to_pd}"
+SENT_PD=$?
+
+# -----------------------------------------------------------------------------
+# send the fleep message
+
+send_fleep "${to_fleep}"
+SENT_FLEEP=$?
+
+# -----------------------------------------------------------------------------
+# send the Prowl message
+
+send_prowl "${to_prowl}"
+SENT_PROWL=$?
+
+# -----------------------------------------------------------------------------
+# send the irc message
+
+send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm}
+Severity: ${severity}
+Chart: ${chart}
+Family: ${family}
+${info}"
+
+SENT_IRC=$?
+
+# -----------------------------------------------------------------------------
+# send the SMS message with smstools3
+
+send_sms "${to_sms}"
+
+SENT_SMS=$?
+
+# -----------------------------------------------------------------------------
+# send the custom message
+
+send_custom() {
+  # is it enabled?
+  [ "${SEND_CUSTOM}" != "YES" ] && return 1
+
+  # do we have any sender?
+  [ -z "${1}" ] && return 1
+
+  # call the custom_sender function
+  custom_sender "${@}"
+}
+
+send_custom "${to_custom}"
+SENT_CUSTOM=$?
+
+# -----------------------------------------------------------------------------
+# send hipchat message
+
+send_hipchat "${HIPCHAT_AUTH_TOKEN}" "${to_hipchat}" " \
+${host} ${status_message}<br/> \
+<b>${alarm}</b> ${info_html}<br/> \
+<b>${chart}</b> (family <b>${family}</b>)<br/> \
+<b>${date}${raised_for_html}</b><br/> \
+<a href=\\\"${goto_url}\\\">View netdata dashboard</a> \
+(source of alarm ${src}) \
+"
+
+SENT_HIPCHAT=$?
+
+# -----------------------------------------------------------------------------
+# send the Amazon SNS message
+
+send_awssns "${to_awssns}"
+
+SENT_AWSSNS=$?
+
+# -----------------------------------------------------------------------------
+#  send the Matrix message
+send_matrix "${MATRIX_HOMESERVER}" "${to_matrix}"
+
+SENT_MATRIX=$?
+
+
+# -----------------------------------------------------------------------------
+# send the syslog message
+
+send_syslog "${to_syslog}"
+
+SENT_SYSLOG=$?
+
+# -----------------------------------------------------------------------------
+# send the email
+
+IFS='' read -r -d '' email_plaintext_part <<EOF
+Content-Type: text/plain; encoding=${EMAIL_CHARSET}
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+${host} ${status_message}
+
+${alarm} ${info}
+${raised_for}
+
+Chart   : ${chart}
+Family  : ${family}
+Severity: ${severity}
+URL     : ${goto_url}
+Source  : ${src}
+Date    : ${date}
+Notification generated on ${host}
+
+Evaluated Expression :  ${calc_expression}
+Expression Variables :  ${calc_param_values}
+
+The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
+EOF
+
+if [[ "${EMAIL_PLAINTEXT_ONLY}" == "YES" ]]; then
+
+send_email <<EOF
+To: ${to_email}
+Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="multipart-boundary"
+${email_thread_headers}
+
+This is a MIME-encoded multipart message
+
+--multipart-boundary
+${email_plaintext_part}
+--multipart-boundary--
+EOF
+
+else
+
+IFS='' read -r -d '' email_html_part <<EOF
+Content-Type: text/html; encoding=${EMAIL_CHARSET}
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0; padding: 0;">
+<body style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 14px; width: 100% !important; min-height: 100%; line-height: 1.6; background: #f6f6f6; margin:0; padding: 0;">
+<table>
+    <tbody>
+    <tr>
+        <td style="vertical-align: top;" valign="top"></td>
+        <td width="700" style="vertical-align: top; display: block !important; max-width: 700px !important; clear: both !important; margin: 0 auto; padding: 0;" valign="top">
+            <div style="max-width: 700px; display: block; margin: 0 auto; padding: 20px;">
+                <table width="100%" cellpadding="0" cellspacing="0" style="background: #fff; border: 1px solid #e9e9e9;">
+                    <tbody>
+                    <tr>
+                        <td bgcolor="#eee" style="padding: 5px 20px 5px 20px; background-color: #eee;">
+                            <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 20px; color: #777; font-weight: bold;">netdata notification</div>
+                        </td>
+                    </tr>
+                    <tr>
+                        <td bgcolor="${color}" style="font-size: 16px; vertical-align: top; font-weight: 400; text-align: center; margin: 0; padding: 10px; color: #ffffff; background: ${color} !important; border: 1px solid ${color}; border-top-color: ${color};" align="center" valign="top">
+                            <h1 style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-weight: 400; margin: 0;">${host} ${status_message}</h1>
+                        </td>
+                    </tr>
+                    <tr>
+                        <td style="vertical-align: top;" valign="top">
+                            <div style="margin: 0; padding: 20px; max-width: 700px;">
+                                <table width="100%" cellpadding="0" cellspacing="0" style="max-width:700px">
+                                    <tbody>
+                                    <tr>
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding:0 0 20px;" align="left" valign="top">
+                                            <span>${chart}</span>
+                                            <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Chart</span>
+                                        </td>
+                                    </tr>
+                                    <tr style="margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+                                            <span><b>${alarm}</b>${info_html}</span>
+                                            <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Alarm</span>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+                                            <span>${family}</span>
+                                            <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Family</span>
+                                        </td>
+                                    </tr>
+                                    <tr style="margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+                                            <span>${severity}</span>
+                                            <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Severity</span>
+                                        </td>
+                                    </tr>
+                                    <tr style="margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"><span>${date}</span>
+                                            <span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
+                                        </td>
+                                    </tr>
+                                    <tr style="margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+                                            <span>${calc_expression}</span>
+                                            <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span>
+                                        </td>
+                                    </tr>
+                                     <tr style="margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+                                            <span>${calc_param_values}</span>
+                                            <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span>
+                                        </td>
+                                    </tr>
+                                     <tr style="margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
+                                            The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
+                                         </td>
+                                    </tr>
+
+                                    <tr style="margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
+                                            <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>
+                                        </td>
+                                    </tr>
+                                    <tr style="text-align: center; margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 11px; vertical-align: top; margin: 0; padding: 10px 0 0 0; color: #666666;" align="center" valign="bottom">The source of this alarm is line <code>${src}</code><br/>(alarms are configurable, edit this file to adapt the alarm to your needs)
+                                        </td>
+                                    </tr>
+                                    <tr style="text-align: center; margin: 0; padding: 0;">
+                                        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by
+                                            <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>.
+                                        </td>
+                                    </tr>
+                                    </tbody>
+                                </table>
+                            </div>
+                        </td>
+                    </tr>
+                    </tbody>
+                </table>
+            </div>
+        </td>
+    </tr>
+    </tbody>
+</table>
+</body>
+</html>
+EOF
+
+send_email <<EOF
+To: ${to_email}
+Subject: ${host} ${status_message} - ${name//_/ } - ${chart}
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="multipart-boundary"
+${email_thread_headers}
+
+This is a MIME-encoded multipart message
+
+--multipart-boundary
+${email_plaintext_part}
+--multipart-boundary
+${email_html_part}
+--multipart-boundary--
+EOF
+
+fi
+
+SENT_EMAIL=$?
+
+# -----------------------------------------------------------------------------
+# send the EVENT to Dynatrace
+send_dynatrace "${host}" "${chart}" "${name}"  "${status}"
+SENT_DYNATRACE=$?
+
+# -----------------------------------------------------------------------------
+# send the EVENT to Stackpulse
+send_stackpulse
+SENT_STACKPULSE=$?
+
+# -----------------------------------------------------------------------------
+# send messages to Opsgenie
+send_opsgenie
+SENT_OPSGENIE=$?
+
+# -----------------------------------------------------------------------------
+# let netdata know
+for state in "${SENT_EMAIL}" \
+  "${SENT_PUSHOVER}" \
+  "${SENT_TELEGRAM}" \
+  "${SENT_SLACK}" \
+  "${SENT_HANGOUTS}" \
+  "${SENT_ROCKETCHAT}" \
+  "${SENT_ALERTA}" \
+  "${SENT_FLOCK}" \
+  "${SENT_DISCORD}" \
+  "${SENT_TWILIO}" \
+  "${SENT_HIPCHAT}" \
+  "${SENT_MESSAGEBIRD}" \
+  "${SENT_KAVENEGAR}" \
+  "${SENT_PUSHBULLET}" \
+  "${SENT_KAFKA}" \
+  "${SENT_PD}" \
+  "${SENT_FLEEP}" \
+  "${SENT_PROWL}" \
+  "${SENT_CUSTOM}" \
+  "${SENT_IRC}" \
+  "${SENT_AWSSNS}" \
+  "${SENT_MATRIX}" \
+  "${SENT_SYSLOG}" \
+  "${SENT_SMS}" \
+  "${SENT_MSTEAM}" \
+  "${SENT_DYNATRACE}" \
+  "${SENT_STACKPULSE}" \
+  "${SENT_OPSGENIE}"; do
+  if [ "${state}" -eq 0 ]; then
+    # we sent something
+    exit 0
+  fi
+done
+# we did not send anything
+exit 1
diff --git a/health/notifications/alarm-test.sh b/health/notifications/alarm-test.sh
new file mode 100755
index 0000000..828aa75
--- /dev/null
+++ b/health/notifications/alarm-test.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# netdata
+# real-time performance and health monitoring, done right!
+# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Script to test alarm notifications for netdata
+
+dir="$(dirname "${0}")"
+"${dir}/alarm-notify.sh" test "${1}"
+exit $?
diff --git a/health/notifications/alerta/Makefile.inc b/health/notifications/alerta/Makefile.inc
new file mode 100644
index 0000000..10f26b0
--- /dev/null
+++ b/health/notifications/alerta/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    alerta/README.md \
+    alerta/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/alerta/README.md b/health/notifications/alerta/README.md
new file mode 100644
index 0000000..8f1679a
--- /dev/null
+++ b/health/notifications/alerta/README.md
@@ -0,0 +1,94 @@
+<!--
+title: "alerta.io"
+description: "Send alarm notifications to Alerta to see the latest health status updates from multiple nodes in a single interface."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/alerta/README.md
+-->
+
+# alerta.io
+
+The [Alerta](https://alerta.io) monitoring system is a tool used to
+consolidate and de-duplicate alerts from multiple sources for quick
+‘at-a-glance’ visualisation. With just one system you can monitor
+alerts from many other monitoring tools on a single screen.
+
+![](https://docs.alerta.io/en/latest/_images/alerta-screen-shot-3.png)
+
+Netadata alarms can be sent to Alerta so you can see in one place
+alerts coming from many Netdata hosts or also from a multi-host
+Netadata configuration. The big advantage over other notifications
+systems is that there is a main view of all active alarms with
+the most recent state, and it is also possible to view alarm history.
+
+## Deploying Alerta
+
+It is recommended to set up the server in a separated server, VM or
+container. If you have other Nginx or Apache server in your organization,
+it is recommended to proxy to this new server.
+
+The easiest way to install Alerta is to use the Docker image available
+on [Docker hub][1]. Alternatively, follow the ["getting started"][2]
+tutorial to deploy Alerta to an Ubuntu server. More advanced
+configurations are out os scope of this tutorial but information
+about different deployment scenarios can be found in the  [docs][3].
+
+[1]: https://hub.docker.com/r/alerta/alerta-web/
+
+[2]: http://alerta.readthedocs.io/en/latest/gettingstarted/tutorial-1-deploy-alerta.html
+
+[3]: http://docs.alerta.io/en/latest/deployment.html
+
+## Send alarms to Alerta
+
+Step 1. Create an API key (if authentication is enabled)
+
+You will need an API key to send messages from any source, if
+Alerta is configured to use authentication (recommended). To
+create an API key go to "Configuration -> API Keys" and create
+a new API key called "netdata" with `write:alerts` permission.
+
+Step 2. configure Netdata to send alarms to Alerta
+
+On your system run:
+
+```sh
+/etc/netdata/edit-config health_alarm_notify.conf
+```
+
+and modify the file as below:
+
+```
+# enable/disable sending alerta notifications
+SEND_ALERTA="YES"
+
+# here set your alerta server API url
+# this is the API url you defined when installed Alerta server, 
+# it is the same for all users. Do not include last slash.
+ALERTA_WEBHOOK_URL="http://yourserver/alerta/api"
+
+# Login with an administrative user to you Alerta server and create an API KEY
+# with write permissions.
+ALERTA_API_KEY="INSERT_YOUR_API_KEY_HERE"
+
+# you can define environments in /etc/alertad.conf option ALLOWED_ENVIRONMENTS
+# standard environments are Production and Development
+# if a role's recipients are not configured, a notification will be send to
+# this Environment (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_ALERTA="Production"
+```
+
+## Test alarms
+
+We can test alarms using the standard approach:
+
+```sh
+/opt/netdata/netdata-plugins/plugins.d/alarm-notify.sh test
+```
+
+Note: Netdata will send 3 alarms, and because last alarm is "CLEAR"
+you will not see them in main Alerta page, you need to select to see
+"closed" alarm in top-right lookup. A little change in `alarm-notify.sh`
+that let us test each state one by one will be useful.
+
+For more information see <https://docs.alerta.io>
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Falerta%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/awssns/Makefile.inc b/health/notifications/awssns/Makefile.inc
new file mode 100644
index 0000000..ee86f4b
--- /dev/null
+++ b/health/notifications/awssns/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    awssns/README.md \
+    awssns/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/awssns/README.md b/health/notifications/awssns/README.md
new file mode 100644
index 0000000..c682513
--- /dev/null
+++ b/health/notifications/awssns/README.md
@@ -0,0 +1,43 @@
+<!--
+title: "Amazon SNS"
+description: "hello"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/awssns/README.md
+-->
+
+# Amazon SNS
+
+As part of it's AWS suite, Amazon provides a notification broker service called 'Simple Notification Service' or SNS.  Amazon SNS works kind of similarly to Netdata's own notification system, allowing dispatch of a single notification to multiple subscribers of different types.  Among other things, SNS supports sending notifications to:
+
+-   Email addresses.
+-   Mobile Phones via SMS.
+-   HTTP or HTTPS web hooks.
+-   AWS Lambda functions.
+-   AWS SQS queues.
+-   Mobile applications via push notifications.
+
+To get this working, you will need:
+
+-   The Amazon Web Services CLI tools.  Most distributions provide these with the package name `awscli`.
+-   An actual home directory for the user you run Netdata as, instead of just using `/` as a home directory.  Setup of this is distribution specific.  `/var/lib/netdata` is the recommended directory (because the permissions will already be correct) if you are using a dedicated user (which is how most distributions work).
+-   An Amazon SNS topic to send notifications to with one or more subscribers.  The [Getting
+    Started](https://docs.aws.amazon.com/sns/latest/dg/sns-getting-started.html) section of the Amazon SNS documentation
+    covers the basics of how to set this up.  Make note of the Topic ARN when you create the topic.
+-   While not mandatory, it is highly recommended to create a dedicated IAM user on your account for Netdata to send notifications.  This user needs to have programmatic access, and should only allow access to SNS.  If you're really paranoid, you can create one for each system or group of systems.
+
+Once you have all the above, run the following command as the user Netdata runs under:
+
+```
+aws configure
+```
+
+THis will prompt you for the access key and secret key for accessing Amazon SNS (as well as the default region and output format, but you can leave those blank because we don't use them).
+
+Once that's done, you're ready to go and can specify the desired topic ARN as a recipient.
+
+Notes:
+
+-   Netdata's native email notification support is far better in almost all respects than it's support through Amazon SNS.  If you want email notifications, use the native support, not SNS.
+    -   If you need to change the notification format for SNS notifications, you can do so by specifying the format in `AWSSNS_MESSAGE_FORMAT` in the configuration.  This variable supports all the same variables you can use in custom notifications.
+    -   While Amazon SNS supports sending differently formatted messages for different delivery methods, Netdata does not currently support this functionality.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fawssns%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/custom/Makefile.inc b/health/notifications/custom/Makefile.inc
new file mode 100644
index 0000000..c64ebda
--- /dev/null
+++ b/health/notifications/custom/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    custom/README.md \
+    custom/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/custom/README.md b/health/notifications/custom/README.md
new file mode 100644
index 0000000..04376d5
--- /dev/null
+++ b/health/notifications/custom/README.md
@@ -0,0 +1,88 @@
+<!--
+title: "Custom"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/custom/README.md
+-->
+
+# Custom
+
+Netdata allows you to send custom notifications to any endpoint you choose.
+
+To configure custom notifications, you will need to customize `health_alarm_notify.conf`. You can look at the other senders in `/usr/libexec/netdata/plugins.d/alarm-notify.sh` for examples of how to modify the `custom_sender()` function in `health_alarm_notify.conf`. Ensure you follow the instructions of changing any configuration file to [persist your configuration](/docs/configuration-guide.md#persist-my-configuration).
+
+As with other notifications, you will also need to define the recipient list in `DEFAULT_RECIPIENT_CUSTOM` and/or the `role_recipients_custom` array.
+
+The following is a sample `custom_sender` function in `health_alarm_notify.conf`, to send an SMS via an imaginary HTTPS endpoint to the SMS gateway:
+
+```
+ custom_sender() {
+    # example human readable SMS
+    local msg="${host} ${status_message}: ${alarm} ${raised_for}"
+
+    # limit it to 160 characters and encode it for use in a URL
+    urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}"
+
+    # a space separated list of the recipients to send alarms to
+    to="${1}"
+
+    for phone in ${to}; do
+      httpcode=$(docurl -X POST \
+				    --data-urlencode "From=XXX" \
+				    --data-urlencode "To=${phone}" \
+				    --data-urlencode "Body=${msg}" \
+				    -u "${accountsid}:${accounttoken}" \
+        https://domain.website.com/)
+
+       if [ "${httpcode}" = "200" ]; then
+         info "sent custom notification ${msg} to ${phone}"
+         sent=$((sent + 1))
+       else
+         error "failed to send custom notification ${msg} to ${phone} with HTTP error code ${httpcode}."
+       fi
+    done
+}
+```
+
+Variables available to the custom_sender:
+
+-   `${to_custom}`          the list of recipients for the alarm
+-   `${host}`               the host generated this event
+-   `${url_host}`           same as `${host}` but URL encoded
+-   `${unique_id}`          the unique id of this event
+-   `${alarm_id}`           the unique id of the alarm that generated this event
+-   `${event_id}`           the incremental id of the event, for this alarm id
+-   `${when}`               the timestamp this event occurred
+-   `${name}`               the name of the alarm, as given in Netdata health.d entries
+-   `${url_name}`           same as `${name}` but URL encoded
+-   `${chart}`              the name of the chart (type.id)
+-   `${url_chart}`          same as `${chart}` but URL encoded
+-   `${family}`             the family of the chart
+-   `${url_family}`         same as `${family}` but URL encoded
+-   `${status}`             the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+-   `${old_status}`         the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+-   `${value}`              the current value of the alarm
+-   `${old_value}`          the previous value of the alarm
+-   `${src}`                the line number and file the alarm has been configured
+-   `${duration}`           the duration in seconds of the previous alarm state
+-   `${duration_txt}`       same as `${duration}` for humans
+-   `${non_clear_duration}` the total duration in seconds this is/was non-clear
+-   `${non_clear_duration_txt}` same as `${non_clear_duration}` for humans
+-   `${units}`              the units of the value
+-   `${info}`               a short description of the alarm
+-   `${value_string}`       friendly value (with units)
+-   `${old_value_string}`   friendly old value (with units)
+-   `${image}`              the URL of an image to represent the status of the alarm
+-   `${color}`              a color in #AABBCC format for the alarm
+-   `${goto_url}`           the URL the user can click to see the Netdata dashboard
+-   `${calc_expression}`    the expression evaluated to provide the value for the alarm
+-   `${calc_param_values}`  the value of the variables in the evaluated expression
+-   `${total_warnings}`     the total number of alarms in WARNING state on the host
+-   `${total_critical}`     the total number of alarms in CRITICAL state on the host
+
+The following are more human friendly:
+
+-   `${alarm}`              like "name = value units"
+-   `${status_message}`     like "needs attention", "recovered", "is critical"
+-   `${severity}`           like "Escalated to CRITICAL", "Recovered from WARNING"
+-   `${raised_for}`         like "(alarm was raised for 10 minutes)"
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fcustom%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/discord/Makefile.inc b/health/notifications/discord/Makefile.inc
new file mode 100644
index 0000000..78de723
--- /dev/null
+++ b/health/notifications/discord/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    discord/README.md \
+    discord/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/discord/README.md b/health/notifications/discord/README.md
new file mode 100644
index 0000000..1650d9c
--- /dev/null
+++ b/health/notifications/discord/README.md
@@ -0,0 +1,50 @@
+<!--
+title: "Discordapp.com"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/discord/README.md
+-->
+
+# Discordapp.com
+
+This is what you will get:
+
+![image](https://cloud.githubusercontent.com/assets/7321975/22215935/b49ede7e-e162-11e6-98d0-ae8541e6b92e.png)
+
+You need:
+
+1.  The **incoming webhook URL** as given by Discord. Create a webhook by following the official [Discord documentation](https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks). You can use the same on all your Netdata servers (or you can have multiple if you like - your decision).
+2.  One or more Discord channels to post the messages to.
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# sending discord notifications
+
+# note: multiple recipients can be given like this:
+#                  "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending discord notifications
+SEND_DISCORD="YES"
+
+# Create a webhook by following the official documentation -
+# https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks
+DISCORD_WEBHOOK_URL="https://discordapp.com/api/webhooks/XXXXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+# if a role's recipients are not configured, a notification will be send to
+# this discord channel (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_DISCORD="alarms"
+```
+
+You can define multiple channels like this: `alarms systems`.
+You can give different channels per **role** using these (at the same file):
+
+```
+role_recipients_discord[sysadmin]="systems"
+role_recipients_discord[dba]="databases systems"
+role_recipients_discord[webmaster]="marketing development"
+```
+
+The keywords `systems`, `databases`, `marketing`, `development` are discordapp.com channels (they should already exist within your discord server).
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fdiscord%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/dynatrace/Makefile.inc b/health/notifications/dynatrace/Makefile.inc
new file mode 100644
index 0000000..a2ae623
--- /dev/null
+++ b/health/notifications/dynatrace/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    dynatrace/README.md \
+    dynatrace/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/dynatrace/README.md b/health/notifications/dynatrace/README.md
new file mode 100644
index 0000000..cc82ee7
--- /dev/null
+++ b/health/notifications/dynatrace/README.md
@@ -0,0 +1,36 @@
+<!--
+title: "Dynatrace"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/dynatrace/README.md
+-->
+
+# Dynatrace
+
+Dynatrace allows you to receive notifications using their Events REST API.
+
+See [the Dynatrace documentation](https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/environment-api/events/post-event/) about POSTing an event in the Events API for more details.
+
+
+
+You need:
+
+1.  Dynatrace Server. You can use the same on all your Netdata servers but make sure the server is network visible from your Netdata hosts.
+The Dynatrace server should be with protocol prefixed (`http://` or `https://`). For example: `https://monitor.example.com`
+This is a required parameter.
+2.  API Token. Generate a secure access API token that enables access to your Dynatrace monitoring data via the REST-based API.
+Generate a Dynatrace API authentication token. On your Dynatrace server, go to **Settings** --> **Integration** --> **Dynatrace API** --> **Generate token**.
+See [Dynatrace API - Authentication](https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/basics/dynatrace-api-authentication/) for more details.
+This is a required parameter.
+3.  API Space. This is the URL part of the page you have access in order to generate the API Token. For example, the URL
+    for a generated API token might look like:
+    `https://monitor.illumineit.com/e/2a93fe0e-4cd5-469a-9d0d-1a064235cfce/#settings/integration/apikeys;gf=all` In that
+    case, my space is _2a93fe0e-4cd5-469a-9d0d-1a064235cfce_ This is a required parameter.
+4. Generate a Server Tag. On your Dynatrace Server, go to **Settings** --> **Tags** --> **Manually applied tags** and create the Tag.
+The Netdata alarm is sent as a Dynatrace Event to be correlated with all those hosts tagged with this Tag you have created.
+This is a required parameter.
+5. Specify the Dynatrace event. This can be one of `CUSTOM_INFO`, `CUSTOM_ANNOTATION`, `CUSTOM_CONFIGURATION`, and `CUSTOM_DEPLOYMENT`. 
+The default value is `CUSTOM_INFO`.
+This is a required parameter.
+6. Specify the annotation type. This is the source of the Dynatrace event. Put whatever it fits you, for example, 
+_Netdata Alarm_, which is also the default value.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fdynatrace%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]()
diff --git a/health/notifications/email/Makefile.inc b/health/notifications/email/Makefile.inc
new file mode 100644
index 0000000..95dc7cf
--- /dev/null
+++ b/health/notifications/email/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    email/README.md \
+    email/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/email/README.md b/health/notifications/email/README.md
new file mode 100644
index 0000000..827a9c0
--- /dev/null
+++ b/health/notifications/email/README.md
@@ -0,0 +1,63 @@
+<!--
+title: "Email"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/email/README.md
+-->
+
+# Email
+
+You need a working `sendmail` command for email alerts to work. Almost all MTAs provide a `sendmail` interface.
+
+Netdata sends all emails as user `netdata`, so make sure your `sendmail` works for local users.
+
+email notifications look like this:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/18407294/e9218c68-7714-11e6-8739-e4dd8a498252.png)
+
+## configuration
+
+To edit `health_alarm_notify.conf` on your system run `/etc/netdata/edit-config health_alarm_notify.conf`.
+
+You can configure recipients in [`/etc/netdata/health_alarm_notify.conf`](https://github.com/netdata/netdata/blob/99d44b7d0c4e006b11318a28ba4a7e7d3f9b3bae/conf.d/health_alarm_notify.conf#L101).
+
+You can also configure per role recipients [in the same file, a few lines below](https://github.com/netdata/netdata/blob/99d44b7d0c4e006b11318a28ba4a7e7d3f9b3bae/conf.d/health_alarm_notify.conf#L313).
+
+Changes to this file do not require a Netdata restart.
+
+You can test your configuration by issuing the commands:
+
+```sh
+# become user netdata
+sudo su -s /bin/bash netdata
+
+# send a test alarm
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test [ROLE]
+```
+
+Where `[ROLE]` is the role you want to test. The default (if you don't give a `[ROLE]`) is `sysadmin`.
+
+Note that in versions before 1.16, the plugins.d directory may be installed in a different location in certain OSs (e.g. under `/usr/lib/netdata`). 
+You can always find the location of the alarm-notify.sh script in `netdata.conf`.
+
+## Simple SMTP transport configuration
+
+If you want an alternative to `sendmail` in order to have a simple MTA configuration for sending emails and auth to an existing SMTP server, you can do the following:
+
+- Install `msmtp`.
+- Modify the `sendmail` path in `health_alarm_notify.conf` to point to the location of `mstmp`:
+```
+# The full path to the sendmail command.
+# If empty, the system $PATH will be searched for it.
+# If not found, email notifications will be disabled (silently).
+sendmail="/usr/bin/msmtp"
+```
+- Login as netdata :
+```sh
+(sudo) su -s /bin/bash netdata
+```
+- Configure `~/.msmtprc` as shown [in the documentation](https://marlam.de/msmtp/documentation/).
+- Finally set the appropriate permissions on the `.msmtprc` file :
+```sh
+chmod 600 ~/.msmtprc
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Femail%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/flock/Makefile.inc b/health/notifications/flock/Makefile.inc
new file mode 100644
index 0000000..5bde161
--- /dev/null
+++ b/health/notifications/flock/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    flock/README.md \
+    flock/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/flock/README.md b/health/notifications/flock/README.md
new file mode 100644
index 0000000..b24ecdb
--- /dev/null
+++ b/health/notifications/flock/README.md
@@ -0,0 +1,37 @@
+<!--
+title: "Flock"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/flock/README.md
+-->
+
+# Flock
+
+This is what you will get:
+
+![Flock](https://i.imgur.com/ok9bRzw.png)
+
+You need:
+
+The **incoming webhook URL** as given by flock.com. 
+You can use the same on all your Netdata servers (or you can have multiple if you like - your decision).
+
+Get them here: <https://admin.flock.com/webhooks>
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# sending flock notifications
+
+# enable/disable sending pushover notifications
+SEND_FLOCK="YES"
+
+# Login to flock.com and create an incoming webhook.
+# You need only one for all your Netdata servers.
+# Without it, Netdata cannot send flock notifications.
+FLOCK_WEBHOOK_URL="https://api.flock.com/hooks/sendMessage/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+# if a role recipient is not configured, no notification will be sent
+DEFAULT_RECIPIENT_FLOCK="alarms"
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fflock%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/hangouts/Makefile.inc b/health/notifications/hangouts/Makefile.inc
new file mode 100644
index 0000000..6ff1dff
--- /dev/null
+++ b/health/notifications/hangouts/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    hangouts/README.md \
+    hangouts/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/hangouts/README.md b/health/notifications/hangouts/README.md
new file mode 100644
index 0000000..886abfc
--- /dev/null
+++ b/health/notifications/hangouts/README.md
@@ -0,0 +1,55 @@
+<!--
+title: "Send notifications to Google Hangouts"
+description: "Send alerts to Send notifications to Google Hangouts any time an anomaly or performance issue strikes a node in your infrastructure."
+sidebar_label: "Google Hangouts"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/hangouts/README.md
+-->
+
+# Send notifications to Google Hangouts
+
+[Google Hangouts](https://hangouts.google.com/) is a cross-platform messaging app developed by Google. You can configure
+Netdata to send alarm notifications to a Hangouts room in order to stay aware of possible health or performance issues
+on your nodes. Here's an example of the notification in action:
+
+![Netdata on Hangouts](https://user-images.githubusercontent.com/1153921/66427166-47de6900-e9c8-11e9-8322-b4b03f084dc1.png)
+
+To receive notifications in Google Hangouts, you need the following in your Hangouts setup:
+
+1.   One or more rooms.
+2.   An **incoming webhook** for each room.
+
+Follow [Google's documentation](https://developers.google.com/hangouts/chat/how-tos/webhooks) to create an incoming
+webhook for each room you want to send Netdata notifications to.
+
+Set the webhook URIs and room names in `health_alarm_notify.conf`. To edit it on your system, run
+`/etc/netdata/edit-config health_alarm_notify.conf`):
+
+## Threads (optional)
+
+Instead to receive alarms on different threads, Netdata allows you to concentrate them inside an unique thread when you
+set the variable `HANGOUTS_WEBHOOK_THREAD[NAME]`.
+
+```
+#------------------------------------------------------------------------------
+# hangouts (google hangouts chat) global notification options
+# enable/disable sending hangouts notifications
+SEND_HANGOUTS="YES"
+# On Hangouts, in the room you choose, create an incoming webhook,
+# copy the link and paste it below and also identify the room name.
+# Without it, netdata cannot send hangouts notifications to that room.
+# HANGOUTS_WEBHOOK_URI[ROOM_NAME]="URLforroom1"
+HANGOUTS_WEBHOOK_URI[systems]="https://chat.googleapis.com/v1/spaces/AAAAXXXXXXX/..."
+HANGOUTS_WEBHOOK_URI[development]="https://chat.googleapis.com/v1/spaces/AAAAYYYYY/..."
+# On Hangouts, copy a thread link and change the values for space and thread
+# HANGOUTS_WEBHOOK_THREAD[systems]="spaces/AAAAXXXXXXX/threads/XXXXXXXXXXX"
+# if a DEFAULT_RECIPIENT_HANGOUTS are not configured,
+# notifications wouldn't be send to hangouts rooms.
+# DEFAULT_RECIPIENT_HANGOUTS="systems development|critical"
+DEFAULT_RECIPIENT_HANGOUTS="sysadmin devops alarms|critical"
+```
+
+You can define multiple rooms like this: `sysadmin devops alarms|critical`.
+
+The keywords `sysadmin`, `devops`, and `alarms` are Hangouts rooms.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fhangouts%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf
new file mode 100755
index 0000000..be669e1
--- /dev/null
+++ b/health/notifications/health_alarm_notify.conf
@@ -0,0 +1,1251 @@
+# Configuration for alarm notifications
+#
+# This configuration is used by: alarm-notify.sh
+# changes take effect immediately (the next alarm will use them).
+#
+# alarm-notify.sh can send:
+# - e-mails (using the sendmail command),
+# - push notifications to your mobile phone (pushover.net),
+# - messages to your slack team (slack.com),
+# - messages to your alerta server (alerta.io),
+# - messages to your flock team (flock.com),
+# - messages to your discord guild (discordapp.com),
+# - messages to your telegram chat / group chat (telegram.org)
+# - sms messages to your cell phone or any sms enabled device (twilio.com)
+# - sms messages to your cell phone or any sms enabled device (messagebird.com)
+# - sms messages to your cell phone or any sms enabled device (smstools3)
+# - notifications to users on pagerduty.com
+# - push notifications to iOS devices (via prowlapp.com)
+# - notifications to Amazon SNS topics (aws.amazon.com)
+# - messages to your irc channel on your selected network
+# - messages to a local or remote syslog daemon
+# - message to Microsoft Team (through webhook)
+# - message to Rocket.Chat (through webhook)
+# - message to Google Hangouts Chat (through webhook)
+#
+# The 'to' line given at netdata alarms defines a *role*, so that many
+# people can be notified for each role.
+#
+# This file is a BASH script itself.
+#
+#
+#------------------------------------------------------------------------------
+# proxy configuration
+#
+# If you need to send curl based notifications (pushover, pushbullet, slack, alerta,
+# flock, discord, telegram) via a proxy, set these to your proxy address:
+#export http_proxy="http://10.0.0.1:3128/"
+#export https_proxy="http://10.0.0.1:3128/"
+
+
+#------------------------------------------------------------------------------
+# notifications images
+#
+# Images in notifications need to be downloaded from an Internet facing site.
+# To allow notification providers fetch the icons/images, by default we set
+# the URL of the global public netdata registry.
+# If you have an Internet facing netdata (or you have copied the images/ folder
+# of netdata to your web server), set its URL here, to fetch the notification
+# images from it.
+#images_base_url="http://my.public.netdata.server:19999"
+
+
+#------------------------------------------------------------------------------
+# date handling
+#
+# You can configure netdata alerts to send dates in any format you want.
+# This uses standard `date` command format strings.  See `man date` for
+# more info on what you can put in here.  Note that this has to start with a '+', otherwise it won't work.
+#
+# For ISO 8601 dates, use '+%FT%T%z'
+# For RFC 5322 dates, use '+%a, %d %b %Y %H:%M:%S %z'
+# For RFC 3339 dates, use '+%F %T%:z'
+# For RFC 1123 dates, use '+%a, %d %b %Y %H:%M:%S %Z'
+# For RFC 1036 dates, use '+%A, %d-%b-%y %H:%M:%S %Z'
+# For a reasonably local date and time (in that order), use '+%x %X'
+# For the old default behavior (compatible with ANSI C's asctime() function), leave this empty.
+date_format=''
+
+
+#------------------------------------------------------------------------------
+# hostname handling
+#
+# By default, Netdata will use the simple hostname for the system (the
+# hostname with everything after the first `.` removed) when displaying
+# the hostname in alert notifications.  If you prefer, you can uncomment
+# the line below to have Netdata instead use the host's fully qualified
+# domain name.
+#
+# This does not report correct FQDN's for child systems for which this
+# system is a parent.
+#
+# Additionally, if the system host name is overridden in /etc/netdata.conf
+# with the `hostname` option, that name will be used unconditionally
+# instead of this.
+#use_fqdn='YES'
+
+
+#------------------------------------------------------------------------------
+# external commands
+
+# The full path to the sendmail command.
+# If empty, the system $PATH will be searched for it.
+# If not found, email notifications will be disabled (silently).
+sendmail=""
+
+# The full path of the curl command.
+# If empty, the system $PATH will be searched for it.
+# If not found, most notifications will be silently disabled.
+curl=""
+
+# The full path of the nc command.
+# If empty, the system $PATH will be searched for it.
+# If not found, irc notifications will be silently disabled.
+nc=""
+
+# The full path of the logger command.
+# If empty, the system $PATH will be searched for it.
+# If not found, syslog notifications will be silently disabled.
+logger=""
+
+# The full path of the aws command.
+# If empty, the system $PATH will be searched for it.
+# If not found, Amazon SNS notifications will be silently disabled.
+aws=""
+
+# The full path of the sendsms command (smstools3).
+# If empty, the system $PATH will be searched for it.
+# If not found, SMS notifications will be silently disabled.
+sendsms=""
+
+#------------------------------------------------------------------------------
+# extra options for external commands
+#
+# In some cases, you may need to change what options get passed to an
+# external command.  Such cases are covered here.
+
+# Extra options to pass to curl.  In most cases, you shouldn't need to add anything
+# to this.  If you're having issues with HTTPS connections, you might try adding
+# '--insecure' here, but be warned that it will make it much easier for
+# third-parties to block notification delivery, and may allow disclosure
+# of potentially sensitive information.
+#curl_options="--insecure"
+
+# Extra options to pass to logger.  You shouldn't have to specify anything
+# here in most cases.
+#logger_options=""
+
+#------------------------------------------------------------------------------
+# extra options
+
+# By default don't do anything if this is CLEAR, but it was not WARNING or CRITICAL.
+# You can send it always if your system makes deduplication for alarms.
+#clear_alarm_always='YES'
+
+#
+#------------------------------------------------------------------------------
+# NOTE ABOUT RECIPIENTS
+#
+# When you define recipients (all types):
+#
+#  - emails addresses
+#  - pushover user tokens
+#  - telegram chat ids
+#  - slack channels
+#  - alerta environment
+#  - flock rooms
+#  - discord channels
+#  - hipchat rooms
+#  - sms phone numbers
+#  - pagerduty.com (pd) services
+#  - irc channels
+#
+# You can append |critical to limit the notifications to be sent.
+#
+# In these examples, the first recipient receives all the alarms
+# while the second one receives only notifications for alarms that 
+# have at some point become critical. The second user may still receive
+# warning and clear notifications, but only for the event that previously 
+# caused a critical alarm.
+#
+#  email      : "user1@example.com user2@example.com|critical"
+#  pushover   : "2987343...9437837 8756278...2362736|critical"
+#  telegram   : "111827421 112746832|critical"
+#  slack      : "alarms disasters|critical"
+#  alerta     : "alarms disasters|critical"
+#  flock      : "alarms disasters|critical"
+#  discord    : "alarms disasters|critical"
+#  twilio     : "+15555555555 +17777777777|critical"
+#  messagebird: "+15555555555 +17777777777|critical"
+#  kavenegar  : "09155555555 09177777777|critical"
+#  pd         : "<pd_service_key_1> <pd_service_key_2>|critical"
+#  irc        : "<irc_channel_1> <irc_channel_2>|critical"
+#  hangouts   : "alarms disasters|critical"
+#
+# If a recipient is set to empty string, the default recipient of the given
+# notification method (email, pushover, telegram, slack, alerta, etc) will be used.
+# To disable a notification, use the recipient called: disabled
+# This works for all notification methods (including the default recipients).
+
+
+#------------------------------------------------------------------------------
+# email global notification options
+
+# multiple recipients can be given like this:
+#              "admin1@example.com admin2@example.com ..."
+
+# the email address sending email notifications
+# the default is the system user netdata runs as (usually: netdata)
+# The following formats are supported:
+# EMAIL_SENDER="user@domain"
+# EMAIL_SENDER="User Name <user@domain>"
+# EMAIL_SENDER="'User Name' <user@domain>"
+# EMAIL_SENDER="\"User Name\" <user@domain>"
+EMAIL_SENDER=""
+
+# enable/disable sending emails
+SEND_EMAIL="YES"
+
+# if a role recipient is not configured, an email will be send to:
+DEFAULT_RECIPIENT_EMAIL="root"
+# to receive only critical alarms, set it to "root|critical"
+
+# Optionally specify the encoding to list in the Content-Type header.
+# This doesn't change what encoding the e-mail is sent with, just what
+# the headers say it was encoded as.
+# This shouldn't need to be changed as it will almost always be
+# autodetected from the environment.
+#EMAIL_CHARSET="UTF-8"
+
+# You can also have netdata add headers to the message that will
+# cause most e-mail clients to treat all notifications for a given
+# chart+alarm+host combination as a single thread.  This can help
+# simplify tracking of alarms, as it provides an easy wway for scripts
+# to corelate messages and also will cause most clients to group all the
+# messages together.  This is enabled by default, uncomment the line
+# below if you want to disable it.
+#EMAIL_THREADING="NO"
+
+# By default, netdata sends HTML and Plain Text emails, some clients
+# do not parse HTML emails such as command line clients.
+# To make emails readable in these clients, you can configure netdata
+# to not send HTML but Plain Text only emails.
+#EMAIL_PLAINTEXT_ONLY="YES"
+
+#------------------------------------------------------------------------------
+# Dynatrace global notification options
+#------------------------------------------------------------------------------
+# enable/disable sending Dynatrace notifications
+SEND_DYNATRACE="YES"
+
+# The Dynatrace server with protocol prefix (http:// or https://), example https://monitor.illumineit.com
+# Required
+DYNATRACE_SERVER=""
+
+# Generate a Dynatrace API authentication token
+# Read https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/basics/dynatrace-api-authentication/ 
+# On Dynatrace server goto Settings --> Integration --> Dynatrace API --> Generate token
+# Required
+DYNATRACE_TOKEN=""
+
+# Beware: Space is taken from dynatrace URL from browser when you create the TOKEN
+# Required
+DYNATRACE_SPACE=""
+
+# Generate a Server Tag. On the Dynatrace Server go to Settings --> Tags --> Manually applied tags create the Tag
+# The NetData alarm will be sent as a Dynatrace Event to be correlated with all those hosts tagged with this Tag 
+# you created.
+# Required
+DYNATRACE_TAG_VALUE=""
+
+# Change this to what you want
+DYNATRACE_ANNOTATION_TYPE="NetData Alarm"
+
+# This can be CUSTOM_INFO, CUSTOM_ANNOTATION, CUSTOM_CONFIGURATION, CUSTOM_DEPLOYMENT
+# Applying default value
+# Required
+DYNATRACE_EVENT="CUSTOM_INFO"
+
+
+DEFAULT_RECIPIENT_DYNATRACE=""
+
+#------------------------------------------------------------------------------
+# Stackpulse global notification options
+SEND_STACKPULSE="YES"
+
+# Webhook
+STACKPULSE_WEBHOOK=""
+
+DEFAULT_RECIPIENT_STACKPULSE=""
+
+#------------------------------------------------------------------------------
+# opsgenie global notification options
+SEND_OPSGENIE="YES"
+
+# Api key
+OPSGENIE_API_KEY=""
+OPSGENIE_API_URL=""
+
+DEFAULT_RECIPIENT_OPSGENIE=""
+
+#------------------------------------------------------------------------------
+# hangouts (google hangouts chat) global notification options
+
+# enable/disable sending hangouts notifications
+SEND_HANGOUTS="YES"
+
+# On Hangouts, in the room you choose, create an incoming webhook,
+# copy the link and paste it below and also give it a room name.
+# Without it, netdata cannot send hangouts notifications to that room.
+# You will then use the same room name in your recipients list. For each URI, you need
+# HANGOUTS_WEBHOOK_URI[room_name]="WEBHOOK_URI"
+# e.g. to define systems and development rooms/recipients:
+# HANGOUTS_WEBHOOK_URI[systems]="URLforroom1"
+# HANGOUTS_WEBHOOK_URI[development]="URLforroom2"
+
+# if a DEFAULT_RECIPIENT_HANGOUTS is not configured,
+# notifications won't be send to hangouts rooms. For the example above,
+# a valid recipients list is the following
+# DEFAULT_RECIPIENT_HANGOUTS="systems development|critical"
+DEFAULT_RECIPIENT_HANGOUTS=""
+
+#------------------------------------------------------------------------------
+# pushover (pushover.net) global notification options
+
+# multiple recipients can be given like this:
+#                  "USERTOKEN1 USERTOKEN2 ..."
+
+# enable/disable sending pushover notifications
+SEND_PUSHOVER="YES"
+
+# Login to pushover.net to get your pushover app token.
+# You need only one for all your netdata servers (or you can have one for
+# each of your netdata - your call).
+# Without an app token, netdata cannot send pushover notifications.
+PUSHOVER_APP_TOKEN=""
+
+# if a role's recipients are not configured, a notification will be send to
+# this pushover user token (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_PUSHOVER=""
+
+
+#------------------------------------------------------------------------------
+# pushbullet (pushbullet.com) push notification options
+
+# multiple recipients can be given like this:
+#                  "user1@email.com user2@mail.com"
+
+# enable/disable sending pushbullet notifications
+SEND_PUSHBULLET="YES"
+
+# Signup and Login to pushbullet.com
+# To get your Access Token, go to https://www.pushbullet.com/#settings/account
+# Create a new access token and paste it below.
+# Then just set the recipients' emails.
+# Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does
+# not have a pushbullet account, the pushbullet service will send an email
+# to that address instead.
+
+# Without an access token, netdata cannot send pushbullet notifications.
+PUSHBULLET_ACCESS_TOKEN=""
+DEFAULT_RECIPIENT_PUSHBULLET=""
+
+# Device iden of the sending device. Optional.
+PUSHBULLET_SOURCE_DEVICE=""
+
+
+#------------------------------------------------------------------------------
+# Twilio (twilio.com) SMS options
+
+# multiple recipients can be given like this:
+#                  "+15555555555 +17777777777"
+
+# enable/disable sending twilio SMS
+SEND_TWILIO="YES"
+
+# Signup for free trial and select a SMS capable Twilio Number
+# To get your Account SID and Token, go to https://www.twilio.com/console
+# Place your sid, token and number below.
+# Then just set the recipients' phone numbers.
+# The trial account is only allowed to use the number specified when set up.
+
+# Without an account sid and token, netdata cannot send Twilio text messages.
+TWILIO_ACCOUNT_SID=""
+TWILIO_ACCOUNT_TOKEN=""
+TWILIO_NUMBER=""
+DEFAULT_RECIPIENT_TWILIO=""
+
+
+#------------------------------------------------------------------------------
+# Messagebird (messagebird.com) SMS options
+
+# multiple recipients can be given like this:
+#                  "+15555555555 +17777777777"
+
+# enable/disable sending messagebird SMS
+SEND_MESSAGEBIRD="YES"
+
+# to get an access key, create a free account at https://www.messagebird.com
+# verify and activate the account (no CC info needed)
+# login to your account and enter your phonenumber to get some free credits
+# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)'
+# click 'Add access key' and fill in data (you want a live key to send SMS)
+
+# Without an access key, netdata cannot send Messagebird text messages.
+MESSAGEBIRD_ACCESS_KEY=""
+MESSAGEBIRD_NUMBER=""
+DEFAULT_RECIPIENT_MESSAGEBIRD=""
+
+
+#------------------------------------------------------------------------------
+# Kavenegar (Kavenegar.com) SMS options
+
+# multiple recipients can be given like this:
+#                  "09155555555 09177777777"
+
+# enable/disable sending kavenegar SMS
+SEND_KAVENEGAR="YES"
+
+# to get an access key, after selecting and purchasing your desired service
+# at http://kavenegar.com/pricing.html
+# login to your account, go to your dashboard and my account are
+# https://panel.kavenegar.com/Client/setting/account from API Key
+# copy your api key. You can generate new API Key too.
+# You can find and select kevenegar sender number from this place.
+
+# Without an API key, netdata cannot send KAVENEGAR text messages.
+KAVENEGAR_API_KEY=""
+KAVENEGAR_SENDER=""
+DEFAULT_RECIPIENT_KAVENEGAR=""
+
+
+#------------------------------------------------------------------------------
+# telegram (telegram.org) global notification options
+
+# multiple recipients can be given like this:
+#                  "CHAT_ID_1 CHAT_ID_2 ..."
+
+# enable/disable sending telegram messages
+SEND_TELEGRAM="YES"
+
+# Contact the bot @BotFather to create a new bot and receive a bot token.
+# Without it, netdata cannot send telegram messages.
+TELEGRAM_BOT_TOKEN=""
+
+# To get your chat ID send the command /getid to telegram bot @myidbot
+# (https://t.me/myidbot). Each user also needs to open a conversation with the
+# bot that will be sending notifications.
+# If a role's recipients are not configured, a message will be sent to
+# this chat id (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_TELEGRAM=""
+
+
+#------------------------------------------------------------------------------
+# slack (slack.com) global notification options
+
+# multiple recipients can be given like this:
+#                  "RECIPIENT1 RECIPIENT2 ..."
+
+# enable/disable sending slack notifications
+SEND_SLACK="YES"
+
+# Login to your slack.com workspace and create an incoming webhook, using the "Incoming Webhooks" App: https://slack.com/apps/A0F7XDUAZ-incoming-webhooks
+# Do not use the instructions in https://api.slack.com/incoming-webhooks#enable_webhooks, as those webhooks work only for a single channel.
+# You need only one for all your netdata servers (or you can have one for each of your netdata).
+# Without the app and a webhook, netdata cannot send slack notifications.
+SLACK_WEBHOOK_URL=""
+
+# if a role's recipients are not configured, a notification will be send to:
+# - A slack channel (syntax: '#channel' or 'channel')
+# - A slack user (syntax: '@user')
+# - The channel or user defined in slack for the webhook (syntax: '#')
+# empty = do not send a notification for unconfigured roles
+DEFAULT_RECIPIENT_SLACK=""
+
+#------------------------------------------------------------------------------
+# Microsoft Team (office.com) global notification options
+# More details are available here regarding the payload syntax options : https://docs.microsoft.com/en-us/outlook/actionable-messages/message-card-reference
+# Online designer : https://acdesignerbeta.azurewebsites.net/
+# multiple recipients can be given like this:
+#                  "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending team notifications
+SEND_MSTEAM="YES"
+
+# if a role's recipients are not configured, a notification will be send to
+# this slack channel (empty = do not send a notification for unconfigured
+# roles):
+# For team the channel name is encoded in the URI after ....IncomingWebhook/___/.....
+# This value will be replaced in the webhook value to publish to several channels in a same Team.
+# In order to get it working properly, you have to replace the value between [] ....IncomingWebhook/[___]/..... by "CHANNEL" string.
+DEFAULT_RECIPIENT_MSTEAM=""
+# Based on the way MS Teams is working, put the differents channels here like : "CHANNEL1 CHANNEL2 ..."
+# AT LEAST ONE CHANNEL IS MANDATORY
+MSTEAM_WEBHOOK_URL=""
+
+# Define the default color scheme for alert to MS Team - icon and color
+# Icons - go to https://emojipedia.org/bomb/
+MSTEAM_ICON_DEFAULT="♡"
+MSTEAM_ICON_CLEAR="💚"
+MSTEAM_ICON_WARNING="⚠️"
+MSTEAM_ICON_CRITICAL="🔥"
+
+# Colors
+MSTEAM_COLOR_DEFAULT="0076D7"
+MSTEAM_COLOR_CLEAR="65A677"
+MSTEAM_COLOR_WARNING="FFA500"
+MSTEAM_COLOR_CRITICAL="D93F3C"
+
+
+#------------------------------------------------------------------------------
+# rocketchat (rocket.chat) global notification options
+
+# multiple recipients can be given like this:
+#                  "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending rocketchat notifications
+SEND_ROCKETCHAT="YES"
+
+# Login to rocket.chat and create an incoming webhook. You need only one for all
+# your netdata servers (or you can have one for each of your netdata).
+# Without it, netdata cannot send rocketchat notifications.
+ROCKETCHAT_WEBHOOK_URL=""
+
+# if a role's recipients are not configured, a notification will be send to
+# this rocketchat channel (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_ROCKETCHAT=""
+
+
+#------------------------------------------------------------------------------
+# alerta (alerta.io) global notification options
+
+# multiple recipients (Environments) can be given like this:
+#                  "Production Development ..."
+
+# enable/disable sending alerta notifications
+SEND_ALERTA="YES"
+
+# here set your alerta server API url
+# this is the API url you defined when installed Alerta server,
+# it is the same for all users. Do not include last slash.
+# ALERTA_WEBHOOK_URL="https://<server>/alerta/api"
+ALERTA_WEBHOOK_URL=""
+
+# Login with an administrative user to you Alerta server and create an API KEY
+# with write permissions.
+ALERTA_API_KEY=""
+
+# you can define environments in /etc/alertad.conf option ALLOWED_ENVIRONMENTS
+# standard environments are Production and Development
+# if a role's recipients are not configured, a notification will be send to
+# this Environment (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_ALERTA=""
+
+
+#------------------------------------------------------------------------------
+# flock (flock.com) global notification options
+
+# enable/disable sending flock notifications
+SEND_FLOCK="YES"
+
+# Login to flock.com and create an incoming webhook. You need only one for all
+# your netdata servers (or you can have one for each of your netdata).
+# Without it, netdata cannot send flock notifications.
+FLOCK_WEBHOOK_URL=""
+
+# if a role recipient is not configured, no notification will be sent
+DEFAULT_RECIPIENT_FLOCK=""
+
+
+#------------------------------------------------------------------------------
+# discord (discordapp.com) global notification options
+
+# multiple recipients can be given like this:
+#                  "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending discord notifications
+SEND_DISCORD="YES"
+
+# Create a webhook by following the official documentation -
+# https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks
+DISCORD_WEBHOOK_URL=""
+
+# if a role's recipients are not configured, a notification will be send to
+# this discord channel (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_DISCORD=""
+
+
+#------------------------------------------------------------------------------
+# hipchat global notification options
+
+# multiple recipients can be given like this:
+#                  "ROOM1 ROOM2 ..."
+
+# enable/disable sending hipchat notifications
+SEND_HIPCHAT="YES"
+
+# define hipchat server
+HIPCHAT_SERVER="api.hipchat.com"
+
+# api.hipchat.com authorization token
+# Without this, netdata cannot send hipchat notifications.
+HIPCHAT_AUTH_TOKEN=""
+
+# if a role's recipients are not configured, a notification will be send to
+# this hipchat room (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_HIPCHAT=""
+
+
+#------------------------------------------------------------------------------
+# kafka notification options
+
+# enable/disable sending kafka notifications
+SEND_KAFKA="YES"
+
+# The URL to POST kafka alarm data to. It should be the full URL.
+KAFKA_URL=""
+
+# The IP to be used in the kafka message as the sender.
+KAFKA_SENDER_IP=""
+
+
+#------------------------------------------------------------------------------
+# pagerduty.com notification options
+#
+# pagerduty.com notifications require a "Generic API" (Events v1)
+# pagerduty service.
+# https://support.pagerduty.com/docs/services-and-integrations
+
+# multiple recipients can be given like this:
+#              "<pd_service_key_1> <pd_service_key_2> ..."
+
+# enable/disable sending pagerduty notifications
+SEND_PD="YES"
+
+# if a role's recipients are not configured, a notification will be sent to
+# the "General API" pagerduty.com service that uses this service key.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_PD=""
+
+# Which PD API are we going to use? For version 2 or newer, it is necessary to do a request for Pagerduty
+# before to set the version(https://developer.pagerduty.com/docs/events-api-v2/overview/).
+USE_PD_VERSION="1"
+
+#------------------------------------------------------------------------------
+# fleep notification options
+#
+# To send fleep.io notifications, you will need a webhook for the
+# conversation you want to send to.
+
+# Fleep recipients are specified as the last part of the webhook URL.
+# So, for a webhook URL of: https://fleep.io/hook/IJONmBuuSlWlkb_ttqyXJg, the
+# recipient name would be: 'IJONmBuuSlWlkb_ttqyXJg'.
+
+# enable/disable sending fleep notifications
+SEND_FLEEP="YES"
+
+# if a role's recipients are not configured, a notification will not be sent.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_FLEEP=""
+
+# The user name to label the messages with.  If this is unset,
+# the hostname of the system the notification is for will be used.
+FLEEP_SENDER=""
+
+
+#------------------------------------------------------------------------------
+# irc notification options
+#
+# irc notifications require only the nc utility to be installed.
+
+# multiple recipients can be given like this:
+#              "<irc_channel_1> <irc_channel_2> ..."
+
+# enable/disable sending irc notifications
+SEND_IRC="YES"
+
+# if a role's recipients are not configured, a notification will not be sent.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_IRC=""
+
+# The irc network to which the recipients belong. It must be the full network.
+# e.g. "irc.freenode.net"
+IRC_NETWORK=""
+
+# The irc nickname which is required to send the notification. It must not be
+# an already registered name as the connection's MODE is defined as a 'guest'.
+IRC_NICKNAME=""
+
+# The irc realname which is required in order to make the connection and is an
+# extra identifier.
+IRC_REALNAME=""
+
+
+#------------------------------------------------------------------------------
+# syslog notifications
+#
+# syslog notifications only need you to have a working logger command, which
+# should be the case on pretty much any Linux system.
+
+# enable/disable sending syslog notifications
+# NOTE: make sure you have everything else configured the way you want
+# it _before_ turning this on.
+SEND_SYSLOG="NO"
+
+# A note on log levels and facilities:
+#
+# The traditional UNIX syslog mechanism has the concept of both log
+# levels and facilities.  A log level indicates the relaitve severity of
+# the message, while a facility specifies a generic source for the message
+# (for example, the `mail` facility is where sendmail and postfix log
+# their messages).  All major syslog daemons have the ability to filter
+# messages based on both log level and facility, and can often also make
+# routing decisions for messages based on both factors.
+#
+# On Linux, the eight log levels in decreasing order of severity are:
+# emerg, alert, crit, err, warning, notice, info, debug
+#
+# By default, warnings will be logged at the warning level, critical
+# alerts at the crit level, and clear notifications at the invo level.
+#
+# And the 19 facilities you can log to are:
+# auth, authpriv, cron, daemon, ftp, lpr, mail, news, syslog, user,
+# uucp, local0, local1, local2, local3, local4, local5, local6, and local7
+#
+# By default, netdata alerts will be logged to the local6 facility.
+#
+# Depending on your distribution, this means that either all your
+# netdata alerts will by default end up in the main system log (usually
+# /var/log/messages), or they won't be logged to a file at all.
+# Neither of these are likely to be what you actually want, but any
+# configuration to change that needs to happen in the syslog daemon
+# configuration, not here.
+
+# This controls which facility is used by default for logging.  Defaults
+# to local6.
+SYSLOG_FACILITY=''
+
+# If a role's recipients are not configured, use the following.
+# (empty = do not send a notification for unconfigured roles)
+#
+# The recipient format for syslog uses the following format:
+# [[facility.level][@host[:port]]/]prefix
+#
+# `prefix` gets appended to the front of all log messages generated for
+# that recipient.  The prefix is mandatory.
+# 'host' and 'port' can be used to specify a remote syslog server to
+# send messages to.  Leave these out if you want messages to be delivered
+# locally.  'host' can be either a hostname or an IP address.
+# IPv6 addresses must have square around them.
+# 'facility' and 'level' are used to override the default logging facility
+# set above and the log level.  If one is specified, both must be present.
+#
+# For example, to send messages with a 'netdata' prefix to a syslog
+# daemon listening on port 514 on 'loghost' using the daemon facility and
+# notice log level:
+# DEFAULT_RECIPIENT_SYSLOG='daemon.notice@loghost:514/netdata'
+#
+DEFAULT_RECIPIENT_SYSLOG="netdata"
+
+#------------------------------------------------------------------------------
+# iOS Push Notifications
+
+# enable/disable sending iOS push notifications
+SEND_PROWL="YES"
+
+# If a role's recipients are not configured, use the following,
+# (empty = do not send a notiication for unconfigured roles)
+#
+# Recipients for iOS push notifications are Prowl API keys.
+#
+# A recipient may also consist of multiple Prowl API keys separated by
+# commas, in which case notifications will be simultaneously sent for all
+# of those API keys.
+DEFAULT_RECIPIENT_PROWL=""
+
+#------------------------------------------------------------------------------
+# Amazon SNS notifications
+#
+# This method requires potentially complex manual configuration.  See the
+# netdata wiki for information on what is needed.
+
+# enable/disable sending Amazon SNS notifications
+SEND_AWSSNS="YES"
+
+# Specify a template for the Amazon SNS notifications.  This supports
+# the same set of variables that are usable in the `custom_sender()`
+# function in the custom notification configuration below.
+#
+AWSSNS_MESSAGE_FORMAT="${status} on ${host} at ${date}: ${chart} ${value_string}"
+
+# If a role's recipients are not configured, use the following.
+# (empty = do not send a notification for unconfigured roles)
+#
+# Recipients for AWS SNS notifications are specified as topic ARN's.
+#
+DEFAULT_RECIPIENT_AWSSNS=""
+
+#------------------------------------------------------------------------------
+# SMS Server Tools 3 (smstools3) global notification options
+
+# enable/disable sending SMS Server Tools 3 SMS notifications
+SEND_SMS="YES"
+
+# if a role's recipients are not configured, a notification will be sent to
+# this SMS channel (empty = do not send a notification for unconfigured
+# roles). Multiple recipients can be given like this: "PHONE1 PHONE2 ..."
+
+DEFAULT_RECIPIENT_SMS=""
+
+# Matrix notifications
+#
+
+# enable/disable Matrix notifications
+SEND_MATRIX="YES"
+
+# The url of the Matrix homeserver
+# e.g https://matrix.org:8448
+MATRIX_HOMESERVER=
+
+# An access token from a valid Matrix account. Tokens usually don't expire,
+# can be controlled from a Matrix client.
+# See https://matrix.org/docs/guides/client-server.html
+MATRIX_ACCESSTOKEN=
+
+# Specify the default rooms to receive the notification if no rooms are provided
+# in a role's recipients.
+# The format is !roomid:homeservername
+DEFAULT_RECIPIENT_MATRIX=""
+
+#------------------------------------------------------------------------------
+# custom notifications
+#
+
+# enable/disable sending custom notifications
+SEND_CUSTOM="YES"
+
+# if a role's recipients are not configured, use the following.
+# (empty = do not send a notification for unconfigured roles)
+DEFAULT_RECIPIENT_CUSTOM=""
+
+# The custom_sender() is a custom function to do whatever you need to do
+custom_sender() {
+    # variables you can use:
+    # ${host}               the host generated this event
+    # ${url_host}           same as ${host} but URL encoded
+    # ${unique_id}          the unique id of this event
+    # ${alarm_id}           the unique id of the alarm that generated this event
+    # ${event_id}           the incremental id of the event, for this alarm id
+    # ${when}               the timestamp this event occurred
+    # ${name}               the name of the alarm, as given in netdata health.d entries
+    # ${url_name}           same as ${name} but URL encoded
+    # ${chart}              the name of the chart (type.id)
+    # ${url_chart}          same as ${chart} but URL encoded
+    # ${family}             the family of the chart
+    # ${url_family}         same as ${family} but URL encoded
+    # ${status}             the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+    # ${old_status}         the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+    # ${value}              the current value of the alarm
+    # ${old_value}          the previous value of the alarm
+    # ${src}                the line number and file the alarm has been configured
+    # ${duration}           the duration in seconds of the previous alarm state
+    # ${duration_txt}       same as ${duration} for humans
+    # ${non_clear_duration} the total duration in seconds this is/was non-clear
+    # ${non_clear_duration_txt} same as ${non_clear_duration} for humans
+    # ${units}              the units of the value
+    # ${info}               a short description of the alarm
+    # ${value_string}       friendly value (with units)
+    # ${old_value_string}   friendly old value (with units)
+    # ${image}              the URL of an image to represent the status of the alarm
+    # ${color}              a color in #AABBCC format for the alarm
+    # ${goto_url}           the URL the user can click to see the netdata dashboard
+    # ${calc_expression}    the expression evaluated to provide the value for the alarm
+    # ${calc_param_values}  the value of the variables in the evaluated expression
+    # ${total_warnings}     the total number of alarms in WARNING state on the host
+    # ${total_critical}     the total number of alarms in CRITICAL state on the host
+
+    # these are more human friendly:
+    # ${alarm}              like "name = value units"
+    # ${status_message}     like "needs attention", "recovered", "is critical"
+    # ${severity}           like "Escalated to CRITICAL", "Recovered from WARNING"
+    # ${raised_for}         like "(alarm was raised for 10 minutes)"
+
+    # example human readable SMS
+    local msg="${host} ${status_message}: ${alarm} ${raised_for}"
+
+    # limit it to 160 characters and encode it for use in a URL
+    urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}"
+
+    # a space separated list of the recipients to send alarms to
+    to="${1}"
+
+    # Sample send SMS to an imaginary SMS gateway accessible via HTTPS
+    #for phone in ${to}; do
+    #  httpcode=$(docurl -X POST \
+	#			    --data-urlencode "From=XXX" \
+	#			    --data-urlencode "To=${phone}" \
+	#			    --data-urlencode "Body=${msg}" \
+	#			    -u "${accountsid}:${accounttoken}" \
+    #   https://domain.website.com/)
+    #
+    #   if [ "${httpcode}" = "200" ]; then
+    #     info "sent custom notification ${msg} to ${phone}"
+    #     sent=$((sent + 1))
+    #   else
+    #     error "failed to send custom notification ${msg} to ${phone} with HTTP error code ${httpcode}."
+    #   fi
+    #done
+
+    info "not sending custom notification to ${to}, for ${status} of '${host}.${chart}.${name}' - custom_sender() is not configured."
+}
+
+
+###############################################################################
+# RECIPIENTS PER ROLE
+
+# -----------------------------------------------------------------------------
+# generic system alarms
+# CPU, disks, network interfaces, entropy, etc
+
+role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_hangouts[sysadmin]="${DEFAULT_RECIPIENT_HANGOUTS}"
+
+role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[sysadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_alerta[sysadmin]="${DEFAULT_RECIPIENT_ALERTA}"
+
+role_recipients_flock[sysadmin]="${DEFAULT_RECIPIENT_FLOCK}"
+
+role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_kavenegar[sysadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"
+
+role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_fleep[sysadmin]="${DEFAULT_RECIPIENT_FLEEP}"
+
+role_recipients_irc[sysadmin]="${DEFAULT_RECIPIENT_IRC}"
+
+role_recipients_syslog[sysadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_prowl[sysadming]="${DEFAULT_RECIPIENT_PROWL}"
+
+role_recipients_awssns[sysadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
+
+role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+role_recipients_msteam[sysadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
+
+role_recipients_rocketchat[sysadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+
+role_recipients_dynatrace[sysadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
+
+role_recipients_opsgenie[sysadmin]="${DEFAULT_RECIPIENT_OPSGENIE}"
+
+role_recipients_matrix[sysadmin]="${DEFAULT_RECIPIENT_MATRIX}"
+
+role_recipients_stackpulse[sysadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
+
+# -----------------------------------------------------------------------------
+# DNS related alarms
+
+role_recipients_email[domainadmin]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_hangouts[domainadmin]="${DEFAULT_RECIPIENT_HANGOUTS}"
+
+role_recipients_pushover[domainadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[domainadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_alerta[domainadmin]="${DEFAULT_RECIPIENT_ALERTA}"
+
+role_recipients_flock[domainadmin]="${DEFAULT_RECIPIENT_FLOCK}"
+
+role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_kavenegar[domainadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"
+
+role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_fleep[domainadmin]="${DEFAULT_RECIPIENT_FLEEP}"
+
+role_recipients_irc[domainadmin]="${DEFAULT_RECIPIENT_IRC}"
+
+role_recipients_syslog[domainadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_prowl[domainadmin]="${DEFAULT_RECIPIENT_PROWL}"
+
+role_recipients_awssns[domainadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
+
+role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+role_recipients_msteam[domainadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
+
+role_recipients_rocketchat[domainadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+
+role_recipients_sms[domainadmin]="${DEFAULT_RECIPIENT_SMS}"
+
+role_recipients_dynatrace[domainadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
+
+role_recipients_opsgenie[domainadmin]="${DEFAULT_RECIPIENT_OPSGENIE}"
+
+role_recipients_matrix[domainadmin]="${DEFAULT_RECIPIENT_MATRIX}"
+
+role_recipients_stackpulse[domainadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
+
+# -----------------------------------------------------------------------------
+# database servers alarms
+# mysql, redis, memcached, postgres, etc
+
+role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_hangouts[dba]="${DEFAULT_RECIPIENT_HANGOUTS}"
+
+role_recipients_pushover[dba]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[dba]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_alerta[dba]="${DEFAULT_RECIPIENT_ALERTA}"
+
+role_recipients_flock[dba]="${DEFAULT_RECIPIENT_FLOCK}"
+
+role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_kavenegar[dba]="${DEFAULT_RECIPIENT_KAVENEGAR}"
+
+role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_fleep[dba]="${DEFAULT_RECIPIENT_FLEEP}"
+
+role_recipients_irc[dba]="${DEFAULT_RECIPIENT_IRC}"
+
+role_recipients_syslog[dba]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_prowl[dba]="${DEFAULT_RECIPIENT_PROWL}"
+
+role_recipients_awssns[dba]="${DEFAULT_RECIPIENT_AWSSNS}"
+
+role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+role_recipients_msteam[dba]="${DEFAULT_RECIPIENT_MSTEAM}"
+
+role_recipients_rocketchat[dba]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+
+role_recipients_sms[dba]="${DEFAULT_RECIPIENT_SMS}"
+
+role_recipients_dynatrace[dba]="${DEFAULT_RECIPIENT_DYNATRACE}"
+
+role_recipients_opsgenie[dba]="${DEFAULT_RECIPIENT_OPSGENIE}"
+
+role_recipients_matrix[dba]="${DEFAULT_RECIPIENT_MATRIX}"
+
+role_recipients_stackpulse[dba]="${DEFAULT_RECIPIENT_STACKPULSE}"
+
+# -----------------------------------------------------------------------------
+# web servers alarms
+# apache, nginx, lighttpd, etc
+
+role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_hangouts[webmaster]="${DEFAULT_RECIPIENT_HANGOUTS}"
+
+role_recipients_pushover[webmaster]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[webmaster]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_alerta[webmaster]="${DEFAULT_RECIPIENT_ALERTA}"
+
+role_recipients_flock[webmaster]="${DEFAULT_RECIPIENT_FLOCK}"
+
+role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_kavenegar[webmaster]="${DEFAULT_RECIPIENT_KAVENEGAR}"
+
+role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_fleep[webmaster]="${DEFAULT_RECIPIENT_FLEEP}"
+
+role_recipients_irc[webmaster]="${DEFAULT_RECIPIENT_IRC}"
+
+role_recipients_syslog[webmaster]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_prowl[webmaster]="${DEFAULT_RECIPIENT_PROWL}"
+
+role_recipients_awssns[webmaster]="${DEFAULT_RECIPIENT_AWSSNS}"
+
+role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+role_recipients_msteam[webmaster]="${DEFAULT_RECIPIENT_MSTEAM}"
+
+role_recipients_rocketchat[webmaster]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+
+role_recipients_sms[webmaster]="${DEFAULT_RECIPIENT_SMS}"
+
+role_recipients_dynatrace[webmaster]="${DEFAULT_RECIPIENT_DYNATRACE}"
+
+role_recipients_opsgenie[webmaster]="${DEFAULT_RECIPIENT_OPSGENIE}"
+
+role_recipients_matrix[webmaster]="${DEFAULT_RECIPIENT_MATRIX}"
+
+role_recipients_stackpulse[webmaster]="${DEFAULT_RECIPIENT_STACKPULSE}"
+
+# -----------------------------------------------------------------------------
+# proxy servers alarms
+# squid, etc
+
+role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_hangouts[proxyadmin]="${DEFAULT_RECIPIENT_HANGOUTS}"
+
+role_recipients_pushover[proxyadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[proxyadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_alerta[proxyadmin]="${DEFAULT_RECIPIENT_ALERTA}"
+
+role_recipients_flock[proxyadmin]="${DEFAULT_RECIPIENT_FLOCK}"
+
+role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_kavenegar[proxyadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"
+
+role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_fleep[proxyadmin]="${DEFAULT_RECIPIENT_FLEEP}"
+
+role_recipients_irc[proxyadmin]="${DEFAULT_RECIPIENT_IRC}"
+
+role_recipients_syslog[proxyadmin]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_prowl[proxyadmin]="${DEFAULT_RECIPIENT_PROWL}"
+
+role_recipients_awssns[porxyadmin]="${DEFAULT_RECIPIENT_AWSSNS}"
+
+role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+role_recipients_msteam[proxyadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
+
+role_recipients_rocketchat[proxyadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+
+role_recipients_sms[proxyadmin]="${DEFAULT_RECIPIENT_SMS}"
+
+role_recipients_dynatrace[proxyadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
+
+role_recipients_opsgenie[proxyadmin]="${DEFAULT_RECIPIENT_OPSGENIE}"
+
+role_recipients_matrix[proxyadmin]="${DEFAULT_RECIPIENT_MATRIX}"
+
+role_recipients_stackpulse[proxyadmin]="${DEFAULT_RECIPIENT_STACKPULSE}"
+
+# -----------------------------------------------------------------------------
+# peripheral devices
+# UPS, photovoltaics, etc
+
+role_recipients_email[sitemgr]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_hangouts[sitemgr]="${DEFAULT_RECIPIENT_HANGOUTS}"
+
+role_recipients_pushover[sitemgr]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[sitemgr]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[sitemgr]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[sitemgr]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_alerta[sitemgr]="${DEFAULT_RECIPIENT_ALERTA}"
+
+role_recipients_flock[sitemgr]="${DEFAULT_RECIPIENT_FLOCK}"
+
+role_recipients_discord[sitemgr]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[sitemgr]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[sitemgr]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[sitemgr]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_kavenegar[sitemgr]="${DEFAULT_RECIPIENT_KAVENEGAR}"
+
+role_recipients_pd[sitemgr]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_fleep[sitemgr]="${DEFAULT_RECIPIENT_FLEEP}"
+
+role_recipients_syslog[sitemgr]="${DEFAULT_RECIPIENT_SYSLOG}"
+
+role_recipients_prowl[sitemgr]="${DEFAULT_RECIPIENT_PROWL}"
+
+role_recipients_awssns[sitemgr]="${DEFAULT_RECIPIENT_AWSSNS}"
+
+role_recipients_custom[sitemgr]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+role_recipients_msteam[sitemgr]="${DEFAULT_RECIPIENT_MSTEAM}"
+
+role_recipients_rocketchat[sitemgr]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
+
+role_recipients_sms[sitemgr]="${DEFAULT_RECIPIENT_SMS}"
+
+role_recipients_dynatrace[sitemgr]="${DEFAULT_RECIPIENT_DYNATRACE}"
+
+role_recipients_opsgenie[sitemgr]="${DEFAULT_RECIPIENT_OPSGENIE}"
+
+role_recipients_matrix[sitemgr]="${DEFAULT_RECIPIENT_MATRIX}"
+
+role_recipients_stackpulse[sitemgr]="${DEFAULT_RECIPIENT_STACKPULSE}"
diff --git a/health/notifications/health_email_recipients.conf b/health/notifications/health_email_recipients.conf
new file mode 100644
index 0000000..f56c6c6
--- /dev/null
+++ b/health/notifications/health_email_recipients.conf
@@ -0,0 +1,2 @@
+# OBSOLETE FILE
+# REPLACED WITH health_alarm_notify.conf
diff --git a/health/notifications/irc/Makefile.inc b/health/notifications/irc/Makefile.inc
new file mode 100644
index 0000000..1a68f65
--- /dev/null
+++ b/health/notifications/irc/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    irc/README.md \
+    irc/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/irc/README.md b/health/notifications/irc/README.md
new file mode 100644
index 0000000..e7f22e1
--- /dev/null
+++ b/health/notifications/irc/README.md
@@ -0,0 +1,78 @@
+<!--
+title: "IRC"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/irc/README.md
+-->
+
+# IRC
+
+This is what you will get:
+
+IRCCloud web client:\
+![image](https://user-images.githubusercontent.com/31221999/36793487-3735673e-1ca6-11e8-8880-d1d8b6cd3bc0.png)
+
+Irssi terminal client:
+![image](https://user-images.githubusercontent.com/31221999/36793486-3713ada6-1ca6-11e8-8c12-70d956ad801e.png)
+
+You need:
+
+1.  The `nc` utility. If you do not set the path, Netdata will search for it in your system `$PATH`.
+
+Set the path for `nc` in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# external commands
+#
+# The full path of the nc command.
+# If empty, the system $PATH will be searched for it.
+# If not found, irc notifications will be silently disabled.
+nc="/usr/bin/nc"
+```
+
+2.  Αn `IRC_NETWORK` to which your preferred channels belong to.   
+3.  One or more channels ( `DEFAULT_RECIPIENT_IRC` ) to post the messages to.   
+4.  An `IRC_NICKNAME` and an `IRC_REALNAME` to identify in IRC.   
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# irc notification options
+#
+# irc notifications require only the nc utility to be installed. 
+
+# multiple recipients can be given like this:
+#              "<irc_channel_1> <irc_channel_2> ..."
+
+# enable/disable sending irc notifications
+SEND_IRC="YES"
+
+# if a role's recipients are not configured, a notification will not be sent.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_IRC="#system-alarms"
+
+# The irc network to which the recipients belong. It must be the full network.
+IRC_NETWORK="irc.freenode.net"
+
+# The irc nickname which is required to send the notification. It must not be 
+# an already registered name as the connection's MODE is defined as a 'guest'.
+IRC_NICKNAME="netdata-alarm-user"
+
+# The irc realname which is required in order to make the connection and is an
+# extra identifier.
+IRC_REALNAME="netdata-user"
+```
+
+You can define multiple channels like this: `#system-alarms #networking-alarms`.\
+You can also filter the notifications like this: `#system-alarms|critical`.\
+You can give different channels per **role** using these (at the same file):  
+
+```
+role_recipients_irc[sysadmin]="#user-alarms #networking-alarms #system-alarms"
+role_recipients_irc[dba]="#databases-alarms"
+role_recipients_irc[webmaster]="#networking-alarms"
+```
+
+The keywords `#user-alarms`, `#networking-alarms`, `#system-alarms`, `#databases-alarms` are irc channels which belong to the specified IRC network.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Firc%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/kavenegar/Makefile.inc b/health/notifications/kavenegar/Makefile.inc
new file mode 100644
index 0000000..b98e794
--- /dev/null
+++ b/health/notifications/kavenegar/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    kavenegar/README.md \
+    kavenegar/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/kavenegar/README.md b/health/notifications/kavenegar/README.md
new file mode 100644
index 0000000..b59799f
--- /dev/null
+++ b/health/notifications/kavenegar/README.md
@@ -0,0 +1,46 @@
+<!--
+title: "Kavenegar"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/kavenegar/README.md
+-->
+
+# Kavenegar
+
+[Kavenegar](https://kavenegar.com/) as service for software developers, based in Iran, provides send and receive SMS, calling voice by using its APIs.
+
+Will look like this on your Android device:
+
+![image](https://cloud.githubusercontent.com/assets/17090999/20034652/620b6100-a39b-11e6-96af-4f83b8e830e2.png)
+
+You will need:
+
+1.  Signup and Login to kavenegar.com
+2.  Get your APIKEY and Sender from `http://panel.kavenegar.com/client/setting/account`
+3.  Fill in KAVENEGAR_API_KEY="" KAVENEGAR_SENDER=""
+4.  Add the recipient phone numbers to DEFAULT_RECIPIENT_KAVENEGAR=""
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# Kavenegar (kavenegar.com) SMS options
+
+# multiple recipients can be given like this:
+#                 "09155555555 09177777777"
+
+# enable/disable sending kavenegar SMS
+SEND_KAVENEGAR="YES"
+
+# to get an access key, after selecting and purchasing your desired service
+# at http://kavenegar.com/pricing.html
+# login to your account, go to your dashboard and my account are
+# https://panel.kavenegar.com/Client/setting/account from API Key
+# copy your api key. You can generate new API Key too.
+# You can find and select kevenegar sender number from this place.
+
+# Without an API key, Netdata cannot send KAVENEGAR text messages.
+KAVENEGAR_API_KEY=""
+KAVENEGAR_SENDER=""
+DEFAULT_RECIPIENT_KAVENEGAR=""
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fkavenegar%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/matrix/Makefile.inc b/health/notifications/matrix/Makefile.inc
new file mode 100644
index 0000000..9937d80
--- /dev/null
+++ b/health/notifications/matrix/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+	matrix/README.md \
+	matrix/Makefile.inc \
+	$(NULL)
+
diff --git a/health/notifications/matrix/README.md b/health/notifications/matrix/README.md
new file mode 100644
index 0000000..ea22b4a
--- /dev/null
+++ b/health/notifications/matrix/README.md
@@ -0,0 +1,58 @@
+<!--
+title: "Send Netdata notifications to Matrix network rooms"
+description: "Stay aware of warning or critical anomalies by sending health alarms to Matrix network rooms with Netdata's health monitoring watchdog."
+sidebar_label: "Matrix"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/matrix/README.md
+-->
+
+# Matrix
+
+Send notifications to [Matrix](https://matrix.org/) network rooms.
+
+The requirements for this notification method are:
+
+1.  The url of the homeserver (`https://homeserver:port`).
+2.  Credentials for connecting to the homeserver, in the form of a valid access token for your account (or for a
+    dedicated notification account). These tokens usually don't expire.
+3.  The room ids that you want to sent the notification to.
+
+To obtain the access token, you can use the following `curl` command:
+
+```bash
+curl -XPOST -d '{"type":"m.login.password", "user":"example", "password":"wordpass"}' "https://homeserver:8448/_matrix/client/r0/login"
+```
+
+The room ids are unique identifiers and can be obtained from the room settings in a Matrix client (e.g. Riot). Their
+format is `!uniqueid:homeserver`.
+
+Multiple room ids can be defined by separating with a space character.
+
+Detailed information about the Matrix client API is available at the [official
+site](https://matrix.org/docs/guides/client-server.html).
+
+Your `health_alarm_notify.conf` should look like this:
+
+```conf
+###############################################################################
+# Matrix notifications
+#
+
+# enable/disable Matrix notifications
+SEND_MATRIX="YES"
+
+# The url of the Matrix homeserver
+# e.g https://matrix.org:8448
+MATRIX_HOMESERVER="https://matrix.org:8448"
+
+# A access token from a valid Matrix account. Tokens usually don't expire,
+# can be controlled from a Matrix client.
+# See https://matrix.org/docs/guides/client-server.html
+MATRIX_ACCESSTOKEN="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+# Specify the default rooms to receive the notification if no rooms are provided
+# in a role's recipients.
+# The format is !roomid:homeservername
+DEFAULT_RECIPIENT_MATRIX="!XXXXXXXXXXXX:matrix.org"
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fmatrix%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/messagebird/Makefile.inc b/health/notifications/messagebird/Makefile.inc
new file mode 100644
index 0000000..f8d2332
--- /dev/null
+++ b/health/notifications/messagebird/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    messagebird/README.md \
+    messagebird/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/messagebird/README.md b/health/notifications/messagebird/README.md
new file mode 100644
index 0000000..8e3d1a5
--- /dev/null
+++ b/health/notifications/messagebird/README.md
@@ -0,0 +1,45 @@
+<!--
+title: "Messagebird"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/messagebird/README.md
+-->
+
+# Messagebird
+
+The messagebird notifications will look like this on your Android device:
+
+![image](https://cloud.githubusercontent.com/assets/17090999/20034652/620b6100-a39b-11e6-96af-4f83b8e830e2.png)
+
+You will need:
+
+1.  Signup and Login to messagebird.com
+2.  Pick an SMS capable number after sign up to get some free credits
+3.  Go to <https://www.messagebird.com/app/settings/developers/access>
+4.  Create a new access key under 'API ACCESS (REST)' (you will want a live key)
+5.  Fill in MESSAGEBIRD_ACCESS_KEY="XXXXXXXX" MESSAGEBIRD_NUMBER="+XXXXXXXXXXX"
+6.  Add the recipient phone numbers to DEFAULT_RECIPIENT_MESSAGEBIRD="+XXXXXXXXXXX"
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# Messagebird (messagebird.com) SMS options
+
+# multiple recipients can be given like this:
+#                  "+15555555555 +17777777777"
+
+# enable/disable sending messagebird SMS
+SEND_MESSAGEBIRD="YES"
+
+# to get an access key, create a free account at https://www.messagebird.com
+# verify and activate the account (no CC info needed)
+# login to your account and enter your phonenumber to get some free credits
+# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)' 
+# click 'Add access key' and fill in data (you want a live key to send SMS)
+
+# Without an access key, Netdata cannot send Messagebird text messages.
+MESSAGEBIRD_ACCESS_KEY="XXXXXXXX"
+MESSAGEBIRD_NUMBER="XXXXXXX"
+DEFAULT_RECIPIENT_MESSAGEBIRD="XXXXXXX"
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fmessagebird%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/opsgenie/Makefile.inc b/health/notifications/opsgenie/Makefile.inc
new file mode 100644
index 0000000..c85bb7c
--- /dev/null
+++ b/health/notifications/opsgenie/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    opsgenie/README.md \
+    opsgenie/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/opsgenie/README.md b/health/notifications/opsgenie/README.md
new file mode 100644
index 0000000..10b9f52
--- /dev/null
+++ b/health/notifications/opsgenie/README.md
@@ -0,0 +1,62 @@
+<!--
+title: "Send notifications to Opsgenie"
+description: "Send alerts to your Opsgenie incident response account any time an anomaly or performance issue strikes a node in your infrastructure."
+sidebar_label: "Opsgenie"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/opsgenie/README.md
+-->
+
+# Send notifications to Opsgenie
+
+[Opsgenie](https://www.atlassian.com/software/opsgenie) is an alerting and incident response tool. It is designed to
+group and filter alarms, build custom routing rules for on-call teams, and correlate deployments and commits to
+incidents.
+
+The first step is to create a [Netdata integration](https://docs.opsgenie.com/docs/api-integration) in the
+[Opsgenie](https://www.atlassian.com/software/opsgenie) dashboard. After this, you need to edit
+`health_alarm_notify.conf` on your system, by running the following from your [config
+directory](/docs/configure/nodes.md):
+ 
+```bash
+./edit-config health_alarm_notify.conf
+```
+
+Change the variable `OPSGENIE_API_KEY` with the API key you got from Opsgenie. `OPSGENIE_API_URL` defaults to
+`https://api.opsgenie.com`, however there are region-specific API URLs such as `https://eu.api.opsgenie.com`, so set
+this if required.
+
+```conf
+SEND_OPSGENIE="YES"
+
+# Api key
+# Default Opsgenie API
+OPSGENIE_API_KEY="11111111-2222-3333-4444-555555555555"
+OPSGENIE_API_URL=""
+```
+
+Changes to `health_alarm_notify.conf` do not require a Netdata restart. You can test your Opsgenie notifications
+configuration by issuing the commands, replacing `ROLE` with your preferred role:
+
+```sh
+# become user netdata
+sudo su -s /bin/bash netdata
+
+# send a test alarm
+/usr/libexec/netdata/plugins.d/alarm-notify.sh test ROLE
+```
+
+If everything works, you'll see alarms in your Opsgenie platform:
+
+![Example alarm notifications in
+Opsgenie](https://user-images.githubusercontent.com/49162938/92184518-f725f900-ee40-11ea-9afa-e7c639c72206.png)
+
+If sending the test notifications fails, you can look in `/var/log/netdata/error.log` to find the relevant error
+message:
+
+```log
+2020-09-03 23:07:00: alarm-notify.sh: ERROR: failed to send opsgenie notification for: hades test.chart.test_alarm is CRITICAL, with HTTP error code 401.
+```
+
+You can find more details about the Opsgenie error codes in their [response
+docs](https://docs.opsgenie.com/docs/response).
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fopsgenie%2FREADME%2FDonations-netdata-has-received&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/pagerduty/Makefile.inc b/health/notifications/pagerduty/Makefile.inc
new file mode 100644
index 0000000..ee9b091
--- /dev/null
+++ b/health/notifications/pagerduty/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    pagerduty/README.md \
+    pagerduty/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/pagerduty/README.md b/health/notifications/pagerduty/README.md
new file mode 100644
index 0000000..b1f60d4
--- /dev/null
+++ b/health/notifications/pagerduty/README.md
@@ -0,0 +1,46 @@
+<!--
+title: "PagerDuty"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/pagerduty/README.md
+-->
+
+# PagerDuty
+
+[PagerDuty](https://www.pagerduty.com/company/) is the enterprise incident resolution service that integrates with ITOps and DevOps monitoring stacks to improve operational reliability and agility. From enriching and aggregating events to correlating them into incidents, PagerDuty streamlines the incident management process by reducing alert noise and resolution times.
+
+Here is an example of a PagerDuty dashboard with Netdata notifications:
+
+![PagerDuty dashboard with Netdata notifications](https://cloud.githubusercontent.com/assets/19278582/21233877/b466a08a-c2a5-11e6-8d66-ee6eed43818f.png)
+
+To have Netdata send notifications to PagerDuty, you'll first need to set up a PagerDuty `Generic API` service and install the PagerDuty agent on the host running Netdata.  See the following guide for details:
+
+<https://www.pagerduty.com/docs/guides/agent-install-guide/>
+
+During the setup of the `Generic API` PagerDuty service, you'll obtain a `pagerduty service key`.  Keep this **service key** handy.
+
+Once the PagerDuty agent is installed on your host and can send notifications from your host to your `Generic API` service on PagerDuty, add the **service key** to `DEFAULT_RECIPIENT_PD` in `health_alarm_notify.conf`:
+
+```
+#------------------------------------------------------------------------------
+# pagerduty.com notification options
+#
+# pagerduty.com notifications require the pagerduty agent to be installed and 
+# a "Generic API" pagerduty service.
+# https://www.pagerduty.com/docs/guides/agent-install-guide/
+
+# multiple recipients can be given like this:
+#              "<pd_service_key_1> <pd_service_key_2> ..."
+
+# enable/disable sending pagerduty notifications
+SEND_PD="YES"
+
+# if a role's recipients are not configured, a notification will be sent to
+# the "General API" pagerduty.com service that uses this service key.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_PD="<service key>"
+
+# Which PD API are we going to use? For version 2 or newer, it is necessary to do a request for Pagerduty
+# before to set the version(https://developer.pagerduty.com/docs/events-api-v2/overview/).
+USE_PD_VERSION="1"
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fpagerduty%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/prowl/Makefile.inc b/health/notifications/prowl/Makefile.inc
new file mode 100644
index 0000000..64a1deb
--- /dev/null
+++ b/health/notifications/prowl/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    prowl/README.md \
+    prowl/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/prowl/README.md b/health/notifications/prowl/README.md
new file mode 100644
index 0000000..7c60de2
--- /dev/null
+++ b/health/notifications/prowl/README.md
@@ -0,0 +1,29 @@
+<!--
+title: "Prowl"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/prowl/README.md
+-->
+
+# Prowl
+
+[Prowl](https://www.prowlapp.com/) is a push notification service for iOS devices.  Netdata
+supports delivering notifications to iOS devices through Prowl.
+
+Because of how Netdata integrates with Prowl, there is a hard limit of
+at most 1000 notifications per hour (starting from the first notification
+sent).  Any alerts beyond the first thousand in an hour will be dropped.
+
+Warning messages will be sent with the 'High' priority, critical messages
+will be sent with the 'Emergency' priority, and all other messages will
+be sent with the normal priority.  Opening the notification's associated
+URL will take you to the Netdata dashboard of the system that issued
+the alert, directly to the chart that it triggered on.
+
+## configuration
+
+To use this, you will need a Prowl API key, which can be requested through
+the Prowl website after registering.
+
+Once you have an API key, simply specify that as a recipient for Prowl
+notifications.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fprowl%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]()
diff --git a/health/notifications/pushbullet/Makefile.inc b/health/notifications/pushbullet/Makefile.inc
new file mode 100644
index 0000000..d3a9459
--- /dev/null
+++ b/health/notifications/pushbullet/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    pushbullet/README.md \
+    pushbullet/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/pushbullet/README.md b/health/notifications/pushbullet/README.md
new file mode 100644
index 0000000..7a098d6
--- /dev/null
+++ b/health/notifications/pushbullet/README.md
@@ -0,0 +1,48 @@
+<!--
+title: "PushBullet"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/pushbullet/README.md
+-->
+
+# PushBullet
+
+Will look like this on your browser:
+![image](https://cloud.githubusercontent.com/assets/4300670/19109636/278b1c0c-8aee-11e6-8a09-7fc94fdbfec8.png)
+
+And like this on your Android device:
+
+![image](https://cloud.githubusercontent.com/assets/4300670/19109635/278a1dde-8aee-11e6-9984-0bc87a13312d.png)
+
+You will need:
+
+1.  Signup and Login to pushbullet.com
+2.  Get your Access Token, go to <https://www.pushbullet.com/#settings/account> and create a new one
+3.  Fill in the PUSHBULLET_ACCESS_TOKEN with that value
+4.  Add the recipient emails to DEFAULT_RECIPIENT_PUSHBULLET
+    !!PLEASE NOTE THAT IF THE RECIPIENT DOES NOT HAVE A PUSHBULLET ACCOUNT, PUSHBULLET SERVICE WILL SEND AN EMAIL!!
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# pushbullet (pushbullet.com) push notification options
+
+# multiple recipients can be given like this:
+#                  "user1@email.com user2@mail.com"
+
+# enable/disable sending pushbullet notifications
+SEND_PUSHBULLET="YES"
+
+# Signup and Login to pushbullet.com
+# To get your Access Token, go to https://www.pushbullet.com/#settings/account
+# And create a new access token
+# Then just set the recipients emails
+# Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does
+# not have a pushbullet account, the pushbullet service will send an email
+# to that address instead
+
+# Without an access token, Netdata cannot send pushbullet notifications.
+PUSHBULLET_ACCESS_TOKEN="o.Sometokenhere"
+DEFAULT_RECIPIENT_PUSHBULLET="admin1@example.com admin3@somemail.com"
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fpushbullet%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/pushover/Makefile.inc b/health/notifications/pushover/Makefile.inc
new file mode 100644
index 0000000..9b703a1
--- /dev/null
+++ b/health/notifications/pushover/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    pushover/README.md \
+    pushover/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/pushover/README.md b/health/notifications/pushover/README.md
new file mode 100644
index 0000000..3ba97fb
--- /dev/null
+++ b/health/notifications/pushover/README.md
@@ -0,0 +1,23 @@
+<!--
+title: "PushOver"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/pushover/README.md
+-->
+
+# PushOver
+
+pushover.net allows you to receive push notifications on your mobile phone. The service seems free for up to 7.500 messages per month.
+
+Netdata will send warning messages with priority `0` and critical messages with priority `1`. pushover.net allows you to select do-not-disturb hours. The way this is configured, critical notifications will ring and vibrate your phone, even during the do-not-disturb-hours. All other notifications will be delivered silently.
+
+You need:
+
+1.  APP TOKEN. You can use the same on all your Netdata servers.
+2.  USER TOKEN for each user you are going to send notifications to. This is the actual recipient of the notification.
+
+The configuration is like above (slack messages).
+
+pushover.net notifications look like this:
+
+![image](https://cloud.githubusercontent.com/assets/2662304/18407319/839c10c4-7715-11e6-92c0-12f8215128d3.png)
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fpushover%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/rocketchat/Makefile.inc b/health/notifications/rocketchat/Makefile.inc
new file mode 100644
index 0000000..58f210b
--- /dev/null
+++ b/health/notifications/rocketchat/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    rocketchat/README.md \
+    rocketchat/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/rocketchat/README.md b/health/notifications/rocketchat/README.md
new file mode 100644
index 0000000..a54f582
--- /dev/null
+++ b/health/notifications/rocketchat/README.md
@@ -0,0 +1,52 @@
+<!--
+title: "Rocket.Chat"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/rocketchat/README.md
+-->
+
+# Rocket.Chat
+
+This is what you will get:
+![Netdata on RocketChat](https://i.imgur.com/Zu4t3j3.png)
+You need:
+
+1.  The **incoming webhook URL** as given by RocketChat. You can use the same on all your Netdata servers (or you can have multiple if you like - your decision).
+2.  One or more channels to post the messages to.
+
+Get them here: <https://rocket.chat/docs/administrator-guides/integrations/index.html#how-to-create-a-new-incoming-webhook>
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+#------------------------------------------------------------------------------
+# rocketchat (rocket.chat) global notification options
+
+# multiple recipients can be given like this:
+#                  "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending rocketchat notifications
+SEND_ROCKETCHAT="YES"
+
+# Login to rocket.chat and create an incoming webhook. You need only one for all
+# your Netdata servers (or you can have one for each of your Netdata).
+# Without it, Netdata cannot send rocketchat notifications.
+ROCKETCHAT_WEBHOOK_URL="<your_incoming_webhook_url>"
+
+# if a role's recipients are not configured, a notification will be send to
+# this rocketchat channel (empty = do not send a notification for unconfigured
+# roles).
+DEFAULT_RECIPIENT_ROCKETCHAT="monitoring_alarms"
+```
+
+You can define multiple channels like this: `alarms systems`.
+You can give different channels per **role** using these (at the same file):
+
+```
+role_recipients_rocketchat[sysadmin]="systems"
+role_recipients_rocketchat[dba]="databases systems"
+role_recipients_rocketchat[webmaster]="marketing development"
+```
+
+The keywords `systems`, `databases`, `marketing`, `development` are RocketChat channels (they should already exist).
+Both public and private channels can be used, even if they differ from the channel configured in your RocketChat incoming webhook.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Frocketchat%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/slack/Makefile.inc b/health/notifications/slack/Makefile.inc
new file mode 100644
index 0000000..043bfaf
--- /dev/null
+++ b/health/notifications/slack/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    slack/README.md \
+    slack/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/slack/README.md b/health/notifications/slack/README.md
new file mode 100644
index 0000000..e338e9a
--- /dev/null
+++ b/health/notifications/slack/README.md
@@ -0,0 +1,50 @@
+<!--
+title: "Slack"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/slack/README.md
+-->
+
+# Slack
+
+This is what you will get:
+![image](https://cloud.githubusercontent.com/assets/2662304/18407116/bbd0fee6-7710-11e6-81cf-58c0defaee2b.png)
+
+You need:
+
+1.  The **incoming webhook URL** as given by slack.com. You can use the same on all your Netdata servers (or you can have multiple if you like - your decision).
+2.  One or more channels to post the messages to.
+
+To get a webhook that works on multiple channels, you will need to login to your slack.com workspace and create an incoming webhook using the [Incoming Webhooks App](https://slack.com/apps/A0F7XDUAZ-incoming-webhooks).
+Do NOT use the instructions in <https://api.slack.com/incoming-webhooks#enable_webhooks>, as the particular webhooks work only for a single channel.
+
+Set the webhook and the recipients in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+SEND_SLACK="YES"
+
+SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXXXXXXX/XXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+
+# if a role's recipients are not configured, a notification will be send to:
+# - A slack channel (syntax: '#channel' or 'channel')  
+# - A slack user (syntax: '@user')
+# - The channel or user defined in slack for the webhook (syntax: '#')
+# empty = do not send a notification for unconfigured roles 
+DEFAULT_RECIPIENT_SLACK="alarms"
+```
+
+You can define multiple recipients like this: `# #alarms systems @myuser`. 
+This example will send the alarm to:
+
+-   The recipient defined in slack for the webhook (not known to Netdata)
+-   The channel 'alarms'
+-   The channel 'systems'
+-   The user @myuser
+
+You can give different recipients per **role** using these (at the same file):
+
+```
+role_recipients_slack[sysadmin]="systems"
+role_recipients_slack[dba]="databases systems"
+role_recipients_slack[webmaster]="marketing development"
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fslack%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/smstools3/Makefile.inc b/health/notifications/smstools3/Makefile.inc
new file mode 100644
index 0000000..4764b9e
--- /dev/null
+++ b/health/notifications/smstools3/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    smstools3/README.md \
+    smstools3/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/smstools3/README.md b/health/notifications/smstools3/README.md
new file mode 100644
index 0000000..6d90e70
--- /dev/null
+++ b/health/notifications/smstools3/README.md
@@ -0,0 +1,44 @@
+<!--
+title: "SMS Server Tools 3"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/smstools3/README.md
+-->
+
+# SMS Server Tools 3
+
+The [SMS Server Tools 3](http://smstools3.kekekasvi.com/) is a SMS Gateway software which can send and receive short messages through GSM modems and mobile phones.
+
+To have Netdata send notifications via SMS Server Tools 3, you'll first need to [install](http://smstools3.kekekasvi.com/index.php?p=compiling) and [configure](http://smstools3.kekekasvi.com/index.php?p=configure) smsd.
+
+Ensure that the user `netdata` can execute `sendsms`. Any user executing `sendsms` needs to:
+
+-   Have write permissions to `/tmp` and `/var/spool/sms/outgoing`
+-   Be a member of group `smsd`
+
+To ensure that the steps above are successful, just `su netdata` and execute `sendsms phone message`.
+
+You then just need to configure the recipient phone numbers in `health_alarm_notify.conf`:
+
+```sh
+#------------------------------------------------------------------------------
+# SMS Server Tools 3 (smstools3) global notification options
+
+# enable/disable sending SMS Server Tools 3 SMS notifications
+SEND_SMS="YES"
+
+# if a role's recipients are not configured, a notification will be sent to
+# this SMS channel (empty = do not send a notification for unconfigured
+# roles). Multiple recipients can be given like this: "PHONE1 PHONE2 ..."
+
+DEFAULT_RECIPIENT_SMS=""
+```
+
+Netdata uses the script `sendsms` that is installed by `smstools3` and just passes a phone number and a message to it. If `sendsms` is not in `$PATH`, you can pass its location in `health_alarm_notify.conf`:
+
+```sh
+# The full path of the sendsms command (smstools3).
+# If empty, the system $PATH will be searched for it.
+# If not found, SMS notifications will be silently disabled.
+sendsms=""
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fsmstools3%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/stackpulse/Makefile.inc b/health/notifications/stackpulse/Makefile.inc
new file mode 100644
index 0000000..eabcb4b
--- /dev/null
+++ b/health/notifications/stackpulse/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    stackpulse/README.md \
+    stackpulse/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/stackpulse/README.md b/health/notifications/stackpulse/README.md
new file mode 100644
index 0000000..13d2f72
--- /dev/null
+++ b/health/notifications/stackpulse/README.md
@@ -0,0 +1,80 @@
+<!--
+title: "Send notifications to StackPulse"
+description: "Send alerts to your StackPulse Netdata integration any time an anomaly or performance issue strikes a node in your infrastructure."
+sidebar_label: "StackPulse"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/stackpulse/README.md
+-->
+
+# Send notifications to StackPulse
+
+[StackPulse](https://stackpulse.com/) is a software-as-a-service platform for site reliability engineering.
+It helps SREs, DevOps Engineers and Software Developers reduce toil and alert fatigue while improving reliability of 
+software services by managing, analyzing and automating incident response activities.
+
+Sending Netdata alarm notifications to StackPulse allows you to create smart automated response workflows 
+(StackPulse playbooks) that will help you drive down your MTTD and MTTR by performing any of the following:
+
+-   Enriching the incident with data from multiple sources
+-   Performing triage actions and analyzing their results
+-   Orchestrating incident management and notification flows
+-   Performing automatic and semi-automatic remediation actions
+-   Analyzing incident data and remediation patterns to improve reliability of your services
+
+To send the notification you need:
+
+1.  Create a Netdata integration in the `StackPulse Administration Portal`, and copy the `Endpoint` URL.
+
+![Creating a Netdata integration in StackPulse](https://user-images.githubusercontent.com/49162938/93023348-d9455a80-f5dd-11ea-8e05-67d07dce93e4.png)
+
+2.  On your node, navigate to `/etc/netdata/` and run the following command:
+
+```sh
+$ ./edit-config health_alarm_notify.conf
+```
+
+3.  Set the `STACKPULSE_WEBHOOK` variable to `Endpoint` URL you copied earlier:
+
+```
+SEND_STACKPULSE="YES"
+STACKPULSE_WEBHOOK="https://hooks.stackpulse.io/v1/webhooks/YOUR_UNIQUE_ID"
+```
+
+4.  Now [restart Netdata](/docs/getting-started.md#start-stop-and-restart-netdata). When your node creates an alarm, you
+    can see the associated notification on your StackPulse Administration Portal 
+
+## React to alarms with playbooks
+
+StackPulse allow users to create `Playbooks` giving additional information about events that happen in specific 
+scenarios. For example, you could create a Playbook that responds to a "low disk space" alarm by compressing and 
+cleaning up storage partitions with dynamic data.
+
+![image](https://user-images.githubusercontent.com/49162938/93207961-4c201400-f74b-11ea-94d1-42a29d007b62.png)
+ 
+![The StackPulse Administration Portal with a Netdata
+alarm](https://user-images.githubusercontent.com/49162938/93208199-bfc22100-f74b-11ea-83c4-728be23dcf4d.png) 
+### Create Playbooks for Netdata alarms
+
+To create a Playbook, you need to access the StackPulse Administration Portal. After the initial setup, you need to
+access the **TRIGGER** tab to define the scenarios used to trigger the event. The following variables are available:
+
+-  `Hostname`: The host that generated the event.
+-  `Chart`: The name of the chart.
+-  `OldValue` : The previous value of the alarm.
+-  `Value`: The current value of the alarm.
+-  `Units` : The units of the value.
+-  `OldStatus` : The previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL.
+-  `State`: The current alarm  status, the acceptable values are the same of `OldStatus`.
+-  `Alarm` : The name of the alarm, as given in Netdata's health.d entries.
+-  `Date` : The timestamp this event occurred.
+-  `Duration` : The duration in seconds of the previous alarm state.
+-  `NonClearDuration` : The total duration in seconds this is/was non-clear.
+-  `Description` : A short description of the alarm copied from the alarm definition.
+-  `CalcExpression` : The expression that was evaluated to trigger the alarm.
+-  `CalcParamValues` : The values of the parameters in the expression, at the time of the evaluation.
+-  `TotalWarnings` : Total number of alarms in WARNING state.
+-  `TotalCritical` : Total number of alarms in CRITICAL state.
+-  `ID` : The unique id of the alarm that generated this event.
+
+For more details how to create a scenario, take a look at the [StackPulse documentation](https://docs.stackpulse.io).
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fopsgenie%2FREADME%2FDonations-netdata-has-received&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/syslog/Makefile.inc b/health/notifications/syslog/Makefile.inc
new file mode 100644
index 0000000..94a8acc
--- /dev/null
+++ b/health/notifications/syslog/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    syslog/README.md \
+    syslog/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/syslog/README.md b/health/notifications/syslog/README.md
new file mode 100644
index 0000000..456394d
--- /dev/null
+++ b/health/notifications/syslog/README.md
@@ -0,0 +1,34 @@
+<!--
+title: "Syslog"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/syslog/README.md
+-->
+
+# Syslog
+
+You need a working `logger` command for this to work.  This is the case on pretty much every Linux system in existence, and most BSD systems.
+
+Logged messages will look like this:
+
+```
+netdata WARNING on hostname at Tue Apr 3 09:00:00 EDT 2018: disk_space._ out of disk space time = 5h
+```
+
+## configuration
+
+System log targets are configured as recipients in [`/etc/netdata/health_alarm_notify.conf`](https://github.com/netdata/netdata/blob/36bedc044584dea791fd29455bdcd287c3306cb2/conf.d/health_alarm_notify.conf#L534) (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`).
+
+You can als configure per-role targets in the same file a bit further down.
+
+Targets are defined as follows:
+
+```
+[[facility.level][@host[:port]]/]prefix
+```
+
+`prefix` defines what the log messages are prefixed with.  By default, all lines are prefixed with 'netdata'.
+
+The `facility` and `level` are the standard syslog facility and level options, for more info on them see your local `logger` and `syslog` documentation.  By default, Netdata will log to the `local6` facility, with a log level dependent on the type of message (`crit` for CRITICAL, `warning` for WARNING, and `info` for everything else).
+
+You can configure sending directly to remote log servers by specifying a host (and optionally a port).  However, this has a somewhat high overhead, so it is much preferred to use your local syslog daemon to handle the forwarding of messages to remote systems (pretty much all of them allow at least simple forwarding, and most of the really popular ones support complex queueing and routing of messages to remote log servers).
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fsyslog%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/telegram/Makefile.inc b/health/notifications/telegram/Makefile.inc
new file mode 100644
index 0000000..ffca071
--- /dev/null
+++ b/health/notifications/telegram/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    telegram/README.md \
+    telegram/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/telegram/README.md b/health/notifications/telegram/README.md
new file mode 100644
index 0000000..c1c6f2a
--- /dev/null
+++ b/health/notifications/telegram/README.md
@@ -0,0 +1,45 @@
+<!--
+title: "Telegram"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/telegram/README.md
+-->
+
+# Telegram
+
+[Telegram](https://telegram.org/) is a messaging app with a focus on speed and security, it’s super-fast, simple and free. You can use Telegram on all your devices at the same time — your messages sync seamlessly across any number of your phones, tablets or computers.
+
+With Telegram, you can send messages, photos, videos and files of any type (doc, zip, mp3, etc), as well as create groups for up to 100,000 people or channels for broadcasting to unlimited audiences. You can write to your phone contacts and find people by their usernames. As a result, Telegram is like SMS and email combined — and can take care of all your personal or business messaging needs.
+
+Netdata will send warning messages without vibration.
+
+You need to:
+
+1.  Get a bot token. To get one, contact the [@BotFather](https://t.me/BotFather) bot and send the command `/newbot`. Follow the instructions.
+2.  Start a conversation with your bot or invite it into a group where you want it to send messages.
+3.  Find the chat ID for every chat you want to send messages to. Contact the [@myidbot](https://t.me/myidbot) bot and send the `/getid` command to get your personal chat ID or invite it into a group and use the `/getgroupid` command to get the group chat ID. Group IDs start with a hyphen, supergroup IDs start with `-100`.  
+    Alternatively, you can get the chat ID directly from the bot API. Send *your* bot a command in the chat you want to use, then check `https://api.telegram.org/bot{YourBotToken}/getUpdates`, eg.  `https://api.telegram.org/bot111122223:7OpFlFFRzRBbrUUmIjj5HF9Ox2pYJZy5/getUpdates`
+4.  Set the bot token and the chat ID of the recipient in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+```
+SEND_TELEGRAM="YES"
+TELEGRAM_BOT_TOKEN="111122223:7OpFlFFRzRBbrUUmIjj5HF9Ox2pYJZy5"
+DEFAULT_RECIPIENT_TELEGRAM="-100233335555"
+```
+
+You can define multiple recipients like this: `"-100311112222 212341234|critical"`.
+This example will send:
+
+-   All alerts to the group with ID -100311112222
+-   Critical alerts to the user with ID 212341234
+
+You can give different recipients per **role** using these (in the same file):
+
+```
+role_recipients_telegram[sysadmin]="212341234"
+role_recipients_telegram[dba]="-1004444333321"
+role_recipients_telegram[webmaster]="49999333322 -1009999222255"
+```
+
+Telegram messages look like this:
+
+![Netdata notifications via Telegram](https://user-images.githubusercontent.com/1153921/66612223-f07dfb80-eb75-11e9-976f-5734ffd93ecd.png)
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Ftelegram%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/twilio/Makefile.inc b/health/notifications/twilio/Makefile.inc
new file mode 100644
index 0000000..0f2d8d8
--- /dev/null
+++ b/health/notifications/twilio/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    twilio/README.md \
+    twilio/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/twilio/README.md b/health/notifications/twilio/README.md
new file mode 100644
index 0000000..b36d40b
--- /dev/null
+++ b/health/notifications/twilio/README.md
@@ -0,0 +1,47 @@
+<!--
+title: "Twilio"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/twilio/README.md
+-->
+
+# Twilio
+
+Will look like this on your Android device:
+
+![image](https://cloud.githubusercontent.com/assets/17090999/20034652/620b6100-a39b-11e6-96af-4f83b8e830e2.png)
+
+You will need:
+
+1.  Signup and Login to twilio.com
+2.  Pick an SMS capable number during sign up.
+3.  Get your SID, and Token from <https://www.twilio.com/console>
+4.  Fill in TWILIO_ACCOUNT_SID="XXXXXXXX" TWILIO_ACCOUNT_TOKEN="XXXXXXXXX" TWILIO_NUMBER="+XXXXXXXXXXX"
+5.  Add the recipient phone numbers to DEFAULT_RECIPIENT_TWILIO="+XXXXXXXXXXX"
+
+!!PLEASE NOTE THAT IF YOUR ACCOUNT IS A TRIAL ACCOUNT YOU WILL ONLY BE ABLE TO SEND NOTIFICATIONS TO THE NUMBER YOU SIGNED UP WITH
+
+Set them in `/etc/netdata/health_alarm_notify.conf` (to edit it on your system run `/etc/netdata/edit-config health_alarm_notify.conf`), like this:
+
+```
+###############################################################################
+# Twilio (twilio.com) SMS options
+
+# multiple recipients can be given like this:
+#                  "+15555555555 +17777777777"
+
+# enable/disable sending twilio SMS
+SEND_TWILIO="YES"
+
+# Signup for free trial and select a SMS capable Twilio Number
+# To get your Account SID and Token, go to https://www.twilio.com/console
+# Place your sid, token and number below.
+# Then just set the recipients' phone numbers.
+# The trial account is only allowed to use the number specified when set up.
+
+# Without an account sid and token, Netdata cannot send Twilio text messages.
+TWILIO_ACCOUNT_SID="xxxxxxxxx"
+TWILIO_ACCOUNT_TOKEN="xxxxxxxxxx"
+TWILIO_NUMBER="xxxxxxxxxxx"
+DEFAULT_RECIPIENT_TWILIO="+15555555555"
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Ftwilio%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/health/notifications/web/Makefile.inc b/health/notifications/web/Makefile.inc
new file mode 100644
index 0000000..b564d83
--- /dev/null
+++ b/health/notifications/web/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    web/README.md \
+    web/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/web/README.md b/health/notifications/web/README.md
new file mode 100644
index 0000000..9e49186
--- /dev/null
+++ b/health/notifications/web/README.md
@@ -0,0 +1,13 @@
+<!--
+title: "Dashboard"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/web/README.md
+-->
+
+# Dashboard
+
+The Netdata dashboard shows HTML notifications, when it is open.
+
+Such web notifications look like this:
+![image](https://cloud.githubusercontent.com/assets/2662304/18407279/82bac6a6-7714-11e6-847e-c2e84eeacbfb.png)
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fhealth%2Fnotifications%2Fweb%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)