diff options
Diffstat (limited to 'health/notifications')
| -rwxr-xr-x | health/notifications/alarm-notify.sh.in | 60 | ||||
| -rwxr-xr-x | health/notifications/health_alarm_notify.conf | 5 |
2 files changed, 45 insertions, 20 deletions
diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in index 0dfecade5..3edf3d083 100755 --- a/health/notifications/alarm-notify.sh.in +++ b/health/notifications/alarm-notify.sh.in @@ -199,7 +199,7 @@ fi [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@" [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@" [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io" -[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud" +[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://api.netdata.cloud" # ----------------------------------------------------------------------------- # parse command line parameters @@ -250,7 +250,7 @@ fi # ----------------------------------------------------------------------------- # find a suitable hostname to use, if netdata did not supply a hostname -if [ -z ${args_host} ]; then +if [ -z "${args_host}" ]; then this_host=$(hostname -s 2>/dev/null) host="${this_host}" args_host="${this_host}" @@ -428,6 +428,10 @@ else done fi +if [[ ! $curl_options =~ .*\--connect-timeout ]]; then + curl_options+=" --connect-timeout 5" +fi + OPSGENIE_API_URL=${OPSGENIE_API_URL:-"https://api.opsgenie.com"} # If we didn't autodetect the character set for e-mail and it wasn't @@ -1335,21 +1339,37 @@ send_telegram() { if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then for chatid in ${chatids}; do - # https://core.telegram.org/bots/api#sendmessage - httpcode=$(docurl ${disableNotification} \ - --data-urlencode "parse_mode=HTML" \ - --data-urlencode "disable_web_page_preview=true" \ - --data-urlencode "text=${emoji} ${message}" \ - "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}") - - if [ "${httpcode}" = "200" ]; then - info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'" - sent=$((sent + 1)) - elif [ "${httpcode}" = "401" ]; then - error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token." - else - error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}." - fi + notify_telegram=1 + notify_retries=${TELEGRAM_RETRIES_ON_LIMIT:-0} + + while [ ${notify_telegram} -eq 1 ]; do + # https://core.telegram.org/bots/api#sendmessage + httpcode=$(docurl ${disableNotification} \ + --data-urlencode "parse_mode=HTML" \ + --data-urlencode "disable_web_page_preview=true" \ + --data-urlencode "text=${emoji} ${message}" \ + "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}") + + notify_telegram=0 + + if [ "${httpcode}" = "200" ]; then + info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'" + sent=$((sent + 1)) + elif [ "${httpcode}" = "401" ]; then + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token." + elif [ "${httpcode}" = "429" ]; then + if [ "$notify_retries" -gt 0 ]; then + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded, retrying after 1s." + notify_retries=$((notify_retries - 1)) + notify_telegram=1 + sleep 1 + else + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded." + fi + else + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}." + fi + done done [ ${sent} -gt 0 ] && return 0 @@ -2398,7 +2418,7 @@ status_email_subject="${status}" case "${status}" in CRITICAL) image="${images_base_url}/images/alert-128-red.png" - alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_critical.png" + alarm_badge="https://app.netdata.cloud/static/email/img/label_critical.png" status_message="is critical" status_email_subject="Critical" color="#ca414b" @@ -2411,7 +2431,7 @@ CRITICAL) WARNING) image="${images_base_url}/images/alert-128-orange.png" - alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_warning.png" + alarm_badge="https://app.netdata.cloud/static/email/img/label_warning.png" status_message="needs attention" status_email_subject="Warning" color="#ffc107" @@ -2424,7 +2444,7 @@ WARNING) CLEAR) image="${images_base_url}/images/check-mark-2-128-green.png" - alarm_badge="${NETDATA_REGISTRY_CLOUD_BASE_URL}/static/email/img/label_recovered.png" + alarm_badge="https://app.netdata.cloud/static/email/img/label_recovered.png" status_message="recovered" status_email_subject="Clear" color="#77ca6d" diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf index b69c6d538..52de86645 100755 --- a/health/notifications/health_alarm_notify.conf +++ b/health/notifications/health_alarm_notify.conf @@ -443,6 +443,11 @@ SEND_TELEGRAM="YES" # Without it, netdata cannot send telegram messages. TELEGRAM_BOT_TOKEN="" +# If an API limit error is returned on sending a message, Netdata will retry this number of times before giving up. +# Setting the number to 0 makes Netdata do no retries (which is the default). +# See https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this +TELEGRAM_RETRIES_ON_LIMIT="0" + # To get your chat ID send the command /getid to telegram bot @myidbot # (https://t.me/myidbot). Each user also needs to open a conversation with the # bot that will be sending notifications. |