summaryrefslogtreecommitdiffstats
path: root/health/notifications
diff options
context:
space:
mode:
Diffstat (limited to 'health/notifications')
-rwxr-xr-xhealth/notifications/alarm-notify.sh.in60
-rwxr-xr-xhealth/notifications/health_alarm_notify.conf5
2 files changed, 45 insertions, 20 deletions
diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in
index 0dfecade5..3edf3d083 100755
--- a/health/notifications/alarm-notify.sh.in
+++ b/health/notifications/alarm-notify.sh.in
@@ -199,7 +199,7 @@ fi
[ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
[ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
[ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
-[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud"
+[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://api.netdata.cloud"
# -----------------------------------------------------------------------------
# parse command line parameters
@@ -250,7 +250,7 @@ fi
# -----------------------------------------------------------------------------
# find a suitable hostname to use, if netdata did not supply a hostname
-if [ -z ${args_host} ]; then
+if [ -z "${args_host}" ]; then
this_host=$(hostname -s 2>/dev/null)
host="${this_host}"
args_host="${this_host}"
@@ -428,6 +428,10 @@ else
done
fi
+if [[ ! $curl_options =~ .*\--connect-timeout ]]; then
+ curl_options+=" --connect-timeout 5"
+fi
+
OPSGENIE_API_URL=${OPSGENIE_API_URL:-"https://api.opsgenie.com"}
# If we didn't autodetect the character set for e-mail and it wasn't
@@ -1335,21 +1339,37 @@ send_telegram() {
if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
for chatid in ${chatids}; do
- # https://core.telegram.org/bots/api#sendmessage
- httpcode=$(docurl ${disableNotification} \
- --data-urlencode "parse_mode=HTML" \
- --data-urlencode "disable_web_page_preview=true" \
- --data-urlencode "text=${emoji} ${message}" \
- "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
-
- if [ "${httpcode}" = "200" ]; then
- info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
- sent=$((sent + 1))
- elif [ "${httpcode}" = "401" ]; then
- error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
- else
- error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
- fi
+ notify_telegram=1
+ notify_retries=${TELEGRAM_RETRIES_ON_LIMIT:-0}
+
+ while [ ${notify_telegram} -eq 1 ]; do
+ # https://core.telegram.org/bots/api#sendmessage
+ httpcode=$(docurl ${disableNotification} \
+ --data-urlencode "parse_mode=HTML" \
+ --data-urlencode "disable_web_page_preview=true" \
+ --data-urlencode "text=${emoji} ${message}" \
+ "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
+
+ notify_telegram=0
+
+ if [ "${httpcode}" = "200" ]; then
+ info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
+ sent=$((sent + 1))
+ elif [ "${httpcode}" = "401" ]; then
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
+ elif [ "${httpcode}" = "429" ]; then
+ if [ "$notify_retries" -gt 0 ]; then
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded, retrying after 1s."
+ notify_retries=$((notify_retries - 1))
+ notify_telegram=1
+ sleep 1
+ else
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded."
+ fi
+ else
+ error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
+ fi
+ done
done
[ ${sent} -gt 0 ] && return 0
@@ -2398,7 +2418,7 @@ status_email_subject="${status}"
case "${status}" in
CRITICAL)
image="${images_base_url}/images/alert-128-red.png"
- alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_critical.png"
+ alarm_badge="https://app.netdata.cloud/static/email/img/label_critical.png"
status_message="is critical"
status_email_subject="Critical"
color="#ca414b"
@@ -2411,7 +2431,7 @@ CRITICAL)
WARNING)
image="${images_base_url}/images/alert-128-orange.png"
- alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_warning.png"
+ alarm_badge="https://app.netdata.cloud/static/email/img/label_warning.png"
status_message="needs attention"
status_email_subject="Warning"
color="#ffc107"
@@ -2424,7 +2444,7 @@ WARNING)
CLEAR)
image="${images_base_url}/images/check-mark-2-128-green.png"
- alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_recovered.png"
+ alarm_badge="https://app.netdata.cloud/static/email/img/label_recovered.png"
status_message="recovered"
status_email_subject="Clear"
color="#77ca6d"
diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf
index b69c6d538..52de86645 100755
--- a/health/notifications/health_alarm_notify.conf
+++ b/health/notifications/health_alarm_notify.conf
@@ -443,6 +443,11 @@ SEND_TELEGRAM="YES"
# Without it, netdata cannot send telegram messages.
TELEGRAM_BOT_TOKEN=""
+# If an API limit error is returned on sending a message, Netdata will retry this number of times before giving up.
+# Setting the number to 0 makes Netdata do no retries (which is the default).
+# See https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this
+TELEGRAM_RETRIES_ON_LIMIT="0"
+
# To get your chat ID send the command /getid to telegram bot @myidbot
# (https://t.me/myidbot). Each user also needs to open a conversation with the
# bot that will be sending notifications.