diff options
Diffstat (limited to 'health/notifications/alarm-notify.sh.in')
| -rwxr-xr-x | health/notifications/alarm-notify.sh.in | 785 |
1 files changed, 341 insertions, 444 deletions
diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in index ea8223097..dd3cda917 100755 --- a/health/notifications/alarm-notify.sh.in +++ b/health/notifications/alarm-notify.sh.in @@ -27,6 +27,7 @@ # - messagebird.com notifications by @tech_no_logical #1453 # - hipchat notifications by @ktsaou #1561 # - fleep notifications by @Ferroin +# - prowlapp.com notifications by @Ferroin # - custom notifications by @ktsaou # - syslog messages by @Ferroin # - Microsoft Team notification by @tioumen @@ -54,7 +55,7 @@ then echo >&2 echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}" - "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" + "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0 if [ $? -ne 0 ] then echo >&2 "# FAILED" @@ -143,6 +144,31 @@ docurl() { } # ----------------------------------------------------------------------------- +# List of all the notification mechanisms we support. +# Used in a couple of places to write more compact code. + +method_names=" +email +pushover +pushbullet +telegram +slack +alerta +flock +discord +hipchat +twilio +messagebird +pd +fleep +syslog +custom +msteam +kavenegar +prowl +" + +# ----------------------------------------------------------------------------- # this is to be overwritten by the config file custom_sender() { @@ -167,32 +193,51 @@ custom_sender() { # ----------------------------------------------------------------------------- # parse command line parameters -roles="${1}" # the roles that should be notified for this event -host="${2}" # the host generated this event -unique_id="${3}" # the unique id of this event -alarm_id="${4}" # the unique id of the alarm that generated this event -event_id="${5}" # the incremental id of the event, for this alarm id -when="${6}" # the timestamp this event occurred -name="${7}" # the name of the alarm, as given in netdata health.d entries -chart="${8}" # the name of the chart (type.id) -family="${9}" # the family of the chart -status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL -old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL -value="${12}" # the current value of the alarm -old_value="${13}" # the previous value of the alarm -src="${14}" # the line number and file the alarm has been configured -duration="${15}" # the duration in seconds of the previous alarm state -non_clear_duration="${16}" # the total duration in seconds this is/was non-clear -units="${17}" # the units of the value -info="${18}" # a short description of the alarm -value_string="${19}" # friendly value (with units) -old_value_string="${20}" # friendly old value (with units) +if [ ${1} = "unittest" ] ; then + unittest=1 # enable unit testing mode + roles="${2}" # the role that should be used for unit testing + cfgfile="${3}" # the location of the config file to use for unit testing + status="${4}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + old_status="${5}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL +else + roles="${1}" # the roles that should be notified for this event + args_host="${2}" # the host generated this event + unique_id="${3}" # the unique id of this event + alarm_id="${4}" # the unique id of the alarm that generated this event + event_id="${5}" # the incremental id of the event, for this alarm id + when="${6}" # the timestamp this event occurred + name="${7}" # the name of the alarm, as given in netdata health.d entries + chart="${8}" # the name of the chart (type.id) + family="${9}" # the family of the chart + status="${10}" # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + old_status="${11}" # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + value="${12}" # the current value of the alarm + old_value="${13}" # the previous value of the alarm + src="${14}" # the line number and file the alarm has been configured + duration="${15}" # the duration in seconds of the previous alarm state + non_clear_duration="${16}" # the total duration in seconds this is/was non-clear + units="${17}" # the units of the value + info="${18}" # a short description of the alarm + value_string="${19}" # friendly value (with units) + old_value_string="${20}" # friendly old value (with units) + calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm + calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation + total_warnings="${23}" # Total number of alarms in WARNING state + total_critical="${24}" # Total number of alarms in CRITICAL state +fi + # ----------------------------------------------------------------------------- # find a suitable hostname to use, if netdata did not supply a hostname -this_host=$(hostname -s 2>/dev/null) -[ -z "${host}" ] && host="${this_host}" +if [ -z ${args_host} ] + then + this_host=$(hostname -s 2>/dev/null) + host="${this_host}" + args_host="${this_host}" +else + host="${args_host}" +fi # ----------------------------------------------------------------------------- # screen statuses we don't need to send a notification @@ -205,7 +250,7 @@ then fi # don't do anything if this is CLEAR, but it was not WARNING or CRITICAL -if [ "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ] +if [ "${clear_alarm_always}" != "YES" -a "${old_status}" != "WARNING" -a "${old_status}" != "CRITICAL" -a "${status}" = "CLEAR" ] then info "not sending notification for ${status} of '${host}.${chart}.${name}' (last status was ${old_status})" exit 1 @@ -223,118 +268,80 @@ images_base_url="https://registry.my-netdata.io" # curl options to use curl_options="" +# hostname handling +use_fqdn="NO" + # needed commands # if empty they will be searched in the system path curl= sendmail= # enable / disable features -SEND_SLACK="YES" -SEND_MSTEAM="YES" -SEND_ALERTA="YES" -SEND_FLOCK="YES" -SEND_DISCORD="YES" -SEND_PUSHOVER="YES" -SEND_TWILIO="YES" -SEND_HIPCHAT="YES" -SEND_MESSAGEBIRD="YES" -SEND_KAVENEGAR="YES" -SEND_TELEGRAM="YES" -SEND_EMAIL="YES" -SEND_PUSHBULLET="YES" -SEND_KAFKA="YES" -SEND_PD="YES" -SEND_FLEEP="YES" -SEND_IRC="YES" -SEND_AWSSNS="YES" -SEND_SYSLOG="NO" -SEND_CUSTOM="YES" +for method_name in ${method_names^^} ; do + declare SEND_${method_name}="YES" + declare DEFAULT_RECIPIENT_${method_name} +done + +for method_name in ${method_names} ; do + declare -A role_recipients_${method_name} +done # slack configs SLACK_WEBHOOK_URL= -DEFAULT_RECIPIENT_SLACK= -declare -A role_recipients_slack=() # Microsoft Team configs MSTEAM_WEBHOOK_URL= -DEFAULT_RECIPIENT_MSTEAM= -declare -A role_recipients_msteam=() # rocketchat configs ROCKETCHAT_WEBHOOK_URL= -DEFAULT_RECIPIENT_ROCKETCHAT= -declare -A role_recipients_rocketchat=() # alerta configs ALERTA_WEBHOOK_URL= ALERTA_API_KEY= -DEFAULT_RECIPIENT_ALERTA= -declare -A role_recipients_alerta=() # flock configs FLOCK_WEBHOOK_URL= -DEFAULT_RECIPIENT_FLOCK= -declare -A role_recipients_flock=() # discord configs DISCORD_WEBHOOK_URL= -DEFAULT_RECIPIENT_DISCORD= -declare -A role_recipients_discord=() # pushover configs PUSHOVER_APP_TOKEN= -DEFAULT_RECIPIENT_PUSHOVER= -declare -A role_recipients_pushover=() # pushbullet configs PUSHBULLET_ACCESS_TOKEN= PUSHBULLET_SOURCE_DEVICE= -DEFAULT_RECIPIENT_PUSHBULLET= -declare -A role_recipients_pushbullet=() # twilio configs TWILIO_ACCOUNT_SID= TWILIO_ACCOUNT_TOKEN= TWILIO_NUMBER= -DEFAULT_RECIPIENT_TWILIO= -declare -A role_recipients_twilio=() # hipchat configs HIPCHAT_SERVER= HIPCHAT_AUTH_TOKEN= -DEFAULT_RECIPIENT_HIPCHAT= -declare -A role_recipients_hipchat=() # messagebird configs MESSAGEBIRD_ACCESS_KEY= MESSAGEBIRD_NUMBER= -DEFAULT_RECIPIENT_MESSAGEBIRD= -declare -A role_recipients_messagebird=() # kavenegar configs -KAVENEGAR_API_KEY="" -KAVENEGAR_SENDER="" -DEFAULT_RECIPIENT_KAVENEGAR=() -declare -A role_recipients_kavenegar="" +KAVENEGAR_API_KEY= +KAVENEGAR_SENDER= # telegram configs TELEGRAM_BOT_TOKEN= -DEFAULT_RECIPIENT_TELEGRAM= -declare -A role_recipients_telegram=() # kafka configs +SEND_KAFKA="YES" KAFKA_URL= KAFKA_SENDER_IP= # pagerduty.com configs PD_SERVICE_KEY= -DEFAULT_RECIPIENT_PD= -declare -A role_recipients_pd=() # fleep.io configs FLEEP_SENDER="${host}" -DEFAULT_RECIPIENT_FLEEP= -declare -A role_recipients_fleep=() # Amazon SNS configs DEFAULT_RECIPIENT_AWSSNS= @@ -343,40 +350,38 @@ declare -A role_recipients_awssns=() # syslog configs SYSLOG_FACILITY= -declare -A role_recipients_syslog=() - -# custom configs -DEFAULT_RECIPIENT_CUSTOM= -declare -A role_recipients_custom=() # email configs EMAIL_SENDER= -DEFAULT_RECIPIENT_EMAIL="root" EMAIL_CHARSET=$(locale charmap 2>/dev/null) EMAIL_THREADING= -declare -A role_recipients_email=() +DEFAULT_RECIPIENT_EMAIL="root" # irc configs IRC_NICKNAME= IRC_REALNAME= -DEFAULT_RECIPIENT_IRC= IRC_NETWORK= -declare -A role_recipients_irc=() # load the stock and user configuration files # these will overwrite the variables above -for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf" -do - if [ -f "${CONFIG}" ] - then - debug "Loading config file '${CONFIG}'..." - source "${CONFIG}" - [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'." - else - warning "Cannot find file '${CONFIG}'." - fi -done +if [ ${unittest} ] ; + then + source "${cfgfile}" + [ $? -ne 0 ] && error "Failed to load requested config file." && exit 1 +else + for CONFIG in "${NETDATA_STOCK_CONFIG_DIR}/health_alarm_notify.conf" "${NETDATA_USER_CONFIG_DIR}/health_alarm_notify.conf" + do + if [ -f "${CONFIG}" ] + then + debug "Loading config file '${CONFIG}'..." + source "${CONFIG}" + [ $? -ne 0 ] && error "Failed to load config file '${CONFIG}'." + else + warning "Cannot find file '${CONFIG}'." + fi + done +fi # If we didn't autodetect the character set for e-mail and it wasn't # set by the user, we need to set it to a reasonable default. UTF-8 @@ -386,6 +391,14 @@ if [ -z ${EMAIL_CHARSET} ] EMAIL_CHARSET="UTF-8" fi +# If we've been asked to use FQDN's for the URL's in the alarm, do so, +# unless we're sending an alarm for a slave system which we can't get the +# FQDN of easily. +if [ "${use_fqdn}" = "YES" -a "${host}" = "$(hostname -s 2>/dev/null)" ] + then + host="$(hostname -f 2>/dev/null)" +fi + # ----------------------------------------------------------------------------- # filter a recipient based on alarm event severity @@ -444,285 +457,6 @@ filter_recipient_by_criticality() { } # ----------------------------------------------------------------------------- -# find the recipients' addresses per method - -declare -A arr_slack=() -declare -A arr_msteam=() -declare -A arr_rocketchat=() -declare -A arr_alerta=() -declare -A arr_flock=() -declare -A arr_discord=() -declare -A arr_pushover=() -declare -A arr_pushbullet=() -declare -A arr_twilio=() -declare -A arr_hipchat=() -declare -A arr_telegram=() -declare -A arr_pd=() -declare -A arr_email=() -declare -A arr_custom=() -declare -A arr_messagebird=() -declare -A arr_kavenegar=() -declare -A arr_fleep=() -declare -A arr_irc=() -declare -A arr_syslog=() -declare -A arr_awssns=() - -# netdata may call us with multiple roles, and roles may have multiple but -# overlapping recipients - so, here we find the unique recipients. -for x in ${roles//,/ } -do - # the roles 'silent' and 'disabled' mean: - # don't send a notification for this role - [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue - - # email - a="${role_recipients_email[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_EMAIL}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality email "${r}" && arr_email[${r/|*/}]="1" - done - - # pushover - a="${role_recipients_pushover[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHOVER}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushover "${r}" && arr_pushover[${r/|*/}]="1" - done - - # pushbullet - a="${role_recipients_pushbullet[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PUSHBULLET}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pushbullet "${r}" && arr_pushbullet[${r/|*/}]="1" - done - - # twilio - a="${role_recipients_twilio[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TWILIO}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality twilio "${r}" && arr_twilio[${r/|*/}]="1" - done - - # hipchat - a="${role_recipients_hipchat[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_HIPCHAT}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality hipchat "${r}" && arr_hipchat[${r/|*/}]="1" - done - - # messagebird - a="${role_recipients_messagebird[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MESSAGEBIRD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality messagebird "${r}" && arr_messagebird[${r/|*/}]="1" - done - - # kavenegar - a="${role_recipients_kavenegar[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_KAVENEGAR}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality kavenegar "${r}" && arr_kavenegar[${r/|*/}]="1" - done - - # telegram - a="${role_recipients_telegram[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_TELEGRAM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality telegram "${r}" && arr_telegram[${r/|*/}]="1" - done - - # slack - a="${role_recipients_slack[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SLACK}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1" - done - - # Microsoft Team - a="${role_recipients_msteam[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_MSTEAM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality msteam "${r}" && arr_msteam[${r/|*/}]="1" - done - - # rocketchat - a="${role_recipients_rocketchat[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ROCKETCHAT}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality rocketchat "${r}" && arr_rocketchat[${r/|*/}]="1" - done - - # alerta - a="${role_recipients_alerta[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ALERTA}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality alerta "${r}" && arr_alerta[${r/|*/}]="1" - done - - # flock - a="${role_recipients_flock[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLOCK}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality flock "${r}" && arr_flock[${r/|*/}]="1" - done - - # discord - a="${role_recipients_discord[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_DISCORD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality discord "${r}" && arr_discord[${r/|*/}]="1" - done - - # pagerduty.com - a="${role_recipients_pd[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_PD}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality pd "${r}" && arr_pd[${r/|*/}]="1" - done - - # fleep.io - a="${role_recipients_fleep[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLEEP}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality fleep "${r}" && arr_fleep[${r/|*/}]="1" - done - - # irc - a="${role_recipients_irc[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_IRC}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality irc "${r}" && arr_irc[${r/|*/}]="1" - done - - # amazon sns - a="${role_recipients_awssns[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_AWSSNS}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality awssns "${r}" && arr_awssns[${r/|*/}]="1" - done - - # syslog - a="${role_recipients_syslog[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_SYSLOG}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality syslog "${r}" && arr_syslog[${r/|*/}]="1" - done - - # custom - a="${role_recipients_custom[${x}]}" - [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_CUSTOM}" - for r in ${a//,/ } - do - [ "${r}" != "disabled" ] && filter_recipient_by_criticality custom "${r}" && arr_custom[${r/|*/}]="1" - done - -done - -# build the list of slack recipients (channels) -to_slack="${!arr_slack[*]}" -[ -z "${to_slack}" ] && SEND_SLACK="NO" - -# build the list of Microsoft team recipients (channels) -to_msteam="${!arr_msteam[*]}" -[ -z "${to_msteam}" ] && SEND_MSTEAM="NO" - -# build the list of rocketchat recipients (channels) -to_rocketchat="${!arr_rocketchat[*]}" -[ -z "${to_rocketchat}" ] && SEND_ROCKETCHAT="NO" - -# build the list of alerta recipients (channels) -to_alerta="${!arr_alerta[*]}" -[ -z "${to_alerta}" ] && SEND_ALERTA="NO" - -# build the list of flock recipients (channels) -to_flock="${!arr_flock[*]}" -[ -z "${to_flock}" ] && SEND_FLOCK="NO" - -# build the list of discord recipients (channels) -to_discord="${!arr_discord[*]}" -[ -z "${to_discord}" ] && SEND_DISCORD="NO" - -# build the list of pushover recipients (user tokens) -to_pushover="${!arr_pushover[*]}" -[ -z "${to_pushover}" ] && SEND_PUSHOVER="NO" - -# build the list of pushbulet recipients (user tokens) -to_pushbullet="${!arr_pushbullet[*]}" -[ -z "${to_pushbullet}" ] && SEND_PUSHBULLET="NO" - -# build the list of twilio recipients (phone numbers) -to_twilio="${!arr_twilio[*]}" -[ -z "${to_twilio}" ] && SEND_TWILIO="NO" - -# build the list of hipchat recipients (rooms) -to_hipchat="${!arr_hipchat[*]}" -[ -z "${to_hipchat}" ] && SEND_HIPCHAT="NO" - -# build the list of messagebird recipients (phone numbers) -to_messagebird="${!arr_messagebird[*]}" -[ -z "${to_messagebird}" ] && SEND_MESSAGEBIRD="NO" - -# build the list of kavenegar recipients (phone numbers) -to_kavenegar="${!arr_kavenegar[*]}" -[ -z "${to_kavenegar}" ] && SEND_KAVENEGAR="NO" - -# check array of telegram recipients (chat ids) -to_telegram="${!arr_telegram[*]}" -[ -z "${to_telegram}" ] && SEND_TELEGRAM="NO" - -# build the list of pagerduty recipients (service keys) -to_pd="${!arr_pd[*]}" -[ -z "${to_pd}" ] && SEND_PD="NO" - -# build the list of fleep recipients (conversation webhooks) -to_fleep="${!arr_fleep[*]}" -[ -z "${to_fleep}" ] && SEND_FLEEP="NO" - -# build the list of custom recipients -to_custom="${!arr_custom[*]}" -[ -z "${to_custom}" ] && SEND_CUSTOM="NO" - -# build the list of email recipients (email addresses) -to_email= -for x in "${!arr_email[@]}" -do - [ ! -z "${to_email}" ] && to_email="${to_email}, " - to_email="${to_email}${x}" -done -[ -z "${to_email}" ] && SEND_EMAIL="NO" - -# build the list of irc recipients (channels) -to_irc="${!arr_irc[*]}" -[ -z "${to_irc}" ] && SEND_IRC="NO" - -# build the list of awssns recipients (facilities, servers, and prefixes) -to_awssns="${!arr_awssns[*]}" -[ -z "${to_awssns}" ] && SEND_AWSSNS="NO" - -# build the list of syslog recipients (facilities, servers, and prefixes) -to_syslog="${!arr_syslog[*]}" -[ -z "${to_syslog}" ] && SEND_SYSLOG="NO" - -# ----------------------------------------------------------------------------- # verify the delivery methods supported # check slack @@ -770,25 +504,13 @@ to_syslog="${!arr_syslog[*]}" # check fleep [ -z "${FLEEP_SERVER}" -o -z "${FLEEP_SENDER}" ] && SEND_FLEEP="NO" -# check pagerduty.com -# if we need pd-send, check for the pd-send command -# https://www.pagerduty.com/docs/guides/agent-install-guide/ -if [ "${SEND_PD}" = "YES" ] - then - pd_send="$(which pd-send 2>/dev/null || command -v pd-send 2>/dev/null)" - if [ -z "${pd_send}" ] - then - error "Cannot find pd-send command in the system path. Disabling pagerduty.com notifications." - SEND_PD="NO" - fi -fi - # if we need curl, check for the curl command if [ \( \ "${SEND_PUSHOVER}" = "YES" \ -o "${SEND_SLACK}" = "YES" \ -o "${SEND_ROCKETCHAT}" = "YES" \ -o "${SEND_ALERTA}" = "YES" \ + -o "${SEND_PD}" = "YES" \ -o "${SEND_FLOCK}" = "YES" \ -o "${SEND_DISCORD}" = "YES" \ -o "${SEND_HIPCHAT}" = "YES" \ @@ -799,6 +521,7 @@ if [ \( \ -o "${SEND_PUSHBULLET}" = "YES" \ -o "${SEND_KAFKA}" = "YES" \ -o "${SEND_FLEEP}" = "YES" \ + -o "${SEND_PROWL}" = "YES" \ -o "${SEND_CUSTOM}" = "YES" \ -o "${SEND_MSTEAM}" = "YES" \ \) -a -z "${curl}" ] @@ -814,6 +537,7 @@ if [ \( \ SEND_MSTEAM="NO" SEND_ROCKETCHAT="NO" SEND_ALERTA="NO" + SEND_PD="NO" SEND_FLOCK="NO" SEND_DISCORD="NO" SEND_TWILIO="NO" @@ -822,6 +546,7 @@ if [ \( \ SEND_KAVENEGAR="NO" SEND_KAFKA="NO" SEND_FLEEP="NO" + SEND_PROWL="NO" SEND_CUSTOM="NO" fi fi @@ -859,6 +584,68 @@ if [ "${SEND_AWSSNS}" = "YES" -a -z "${aws}" ] fi fi +# ----------------------------------------------------------------------------- +# find the recipients' addresses per method + +# netdata may call us with multiple roles, and roles may have multiple but +# overlapping recipients - so, here we find the unique recipients. +for method_name in ${method_names} ; do + send_var="SEND_${method_name^^}" + if [ ${!send_var} = "NO" ] ; then + continue + fi + + declare -A arr_var=() + + for x in ${roles//,/ } ; do + # the roles 'silent' and 'disabled' mean: + # don't send a notification for this role + [ "${x}" = "silent" -o "${x}" = "disabled" ] && continue + + role_recipients="role_recipients_${method_name}[$x]" + default_recipient_var="DEFAULT_RECIPIENT_${method_name^^}" + + a="${!role_recipients}" + [ -z "${a}" ] && a="${!default_recipient_var}" + for r in ${a//,/ } ; do + [ "${r}" != "disabled" ] && filter_recipient_by_criticality ${method_name} "${r}" && arr_var[${r/|*/}]="1" + done + done + + # build the list of recipients + to_var="to_${method_name}" + declare to_${method_name}="${!arr_var[*]}" + + [ -z "${!to_var}" ] && declare ${send_var}="NO" +done + +# ----------------------------------------------------------------------------- +# handle fixup of the email recipient list. + +fix_to_email() { + to_email= + while [ ! -z "${1}" ] + do + [ ! -z "${to_email}" ] && to_email="${to_email}, " + to_email="${to_email}${1}" + shift 1 + done +} + +# ${to_email} without quotes here +fix_to_email ${to_email} + +# ----------------------------------------------------------------------------- +# handle output if we're running in unit test mode +if [ ${unittest} ] ; then + for method_name in ${method_names} ; do + to_var="to_${method_name}" + echo "results: ${method_name}: ${!to_var}" + done + exit 0 +fi + +# ----------------------------------------------------------------------------- # check that we have at least a method enabled if [ "${SEND_EMAIL}" != "YES" \ -a "${SEND_PUSHOVER}" != "YES" \ @@ -879,6 +666,7 @@ if [ "${SEND_EMAIL}" != "YES" \ -a "${SEND_CUSTOM}" != "YES" \ -a "${SEND_IRC}" != "YES" \ -a "${SEND_AWSSNS}" != "YES" \ + -a "${SEND_PROWL}" != "YES" \ -a "${SEND_SYSLOG}" != "YES" \ -a "${SEND_MSTEAM}" != "YES" \ ] @@ -1126,7 +914,7 @@ EOF # kafka sender send_kafka() { - local httpcode sent=0 + local httpcode sent=0 if [ "${SEND_KAFKA}" = "YES" ] then httpcode=$(docurl -X POST \ @@ -1163,38 +951,42 @@ send_pd() { then for PD_SERVICE_KEY in ${recipients} do - d="${status} ${name} = ${value_string} - ${host}, ${family}" - ${pd_send} -k ${PD_SERVICE_KEY} \ - -t ${t} \ - -d "${d}" \ - -i ${host}:${chart}:${name} \ - -f 'info'="${info}" \ - -f 'value_w_units'="${value_string}" \ - -f 'when'="${when}" \ - -f 'duration'="${duration}" \ - -f 'roles'="${roles}" \ - -f 'host'="${host}" \ - -f 'unique_id'="${unique_id}" \ - -f 'alarm_id'="${alarm_id}" \ - -f 'event_id'="${event_id}" \ - -f 'name'="${name}" \ - -f 'chart'="${chart}" \ - -f 'family'="${family}" \ - -f 'status'="${status}" \ - -f 'old_status'="${old_status}" \ - -f 'value'="${value}" \ - -f 'old_value'="${old_value}" \ - -f 'src'="${src}" \ - -f 'non_clear_duration'="${non_clear_duration}" \ - -f 'units'="${units}" - retval=$? - if [ ${retval} -eq 0 ] - then - info "sent pagerduty.com notification for host ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}....: ${d}" - sent=$((sent + 1)) - else - error "failed to send pagerduty.com notification for ${host} ${chart}.${name} using service key ${PD_SERVICE_KEY::-26}.... (error code ${retval}): ${d}" - fi + d="${status} ${name} = ${value_string} - ${host}, ${family}" + payload="$(cat << EOF + { + "service_key": "${PD_SERVICE_KEY}", + "event_type": "${t}", + "incident_key" : "${alarm_id}", + "description": "${d}", + "details": { + "value_w_units": "${value_string}", + "when": "${when}", + "duration" : "${duration}", + "roles": "${roles}", + "alarm_id" : "${alarm_id}", + "name" : "${name}", + "chart" : "${chart}", + "family" : "${family}", + "status" : "${status}", + "old_status" : "${old_status}", + "value" : "${value}", + "old_value" : "${old_value}", + "src" : "${src}", + "non_clear_duration" : "${non_clear_duration}", + "units" : "${units}", + "info" : "${info}" + } + } +EOF + )" + httpcode=$(docurl -X POST --data "${payload}" "https://events.pagerduty.com/generic/2010-04-15/create_event.json") + if [ "${httpcode}" = "200" ] + then + info "sent pagerduty notification for: ${host} ${chart}.${name} is ${status}'" + sent=$((sent + 1)) + else + error "failed to send pagerduty notification for: ${host} ${chart}.${name} is ${status}, with HTTP error code ${httpcode}." + fi done [ ${sent} -gt 0 ] && return 0 @@ -1249,7 +1041,7 @@ send_hipchat() { if [ "${SEND_HIPCHAT}" = "YES" -a ! -z "${HIPCHAT_SERVER}" -a ! -z "${authtoken}" -a ! -z "${recipients}" -a ! -z "${message}" ] then # A label to be shown in addition to the sender's name - # Valid length range: 0 - 64. + # Valid length range: 0 - 64. sender="netdata" # Valid values: html, text. @@ -1276,7 +1068,7 @@ send_hipchat() { -H "Authorization: Bearer ${authtoken}" \ -d "{\"color\": \"${color}\", \"from\": \"${host}\", \"message_format\": \"${msg_format}\", \"message\": \"${message}\", \"notify\": \"${notify}\"}" \ "https://${HIPCHAT_SERVER}/v2/room/${room}/notification") - + if [ "${httpcode}" = "204" ] then info "sent HipChat notification for: ${host} ${chart}.${name} is ${status} to '${room}'" @@ -1341,7 +1133,7 @@ send_kavenegar() { --data-urlencode "receptor=${user}" \ --data-urlencode "message=${title} ${message}") - if [ "${httpcode}" = "201" ] + if [ "${httpcode}" = "200" ] then info "sent Kavenegar SMS for: ${host} ${chart}.${name} is ${status} to '${user}'" sent=$((sent + 1)) @@ -1363,7 +1155,7 @@ send_telegram() { local bottoken="${1}" chatids="${2}" message="${3}" httpcode sent=0 chatid emoji disableNotification="" if [ "${status}" = "CLEAR" ]; then disableNotification="--data-urlencode disable_notification=true"; fi - + case "${status}" in WARNING) emoji="⚠️" ;; CRITICAL) emoji="🔴" ;; @@ -1476,9 +1268,21 @@ send_slack() { for channel in ${channels} do + # Default entry in the recipient is without a hash in front (backwards-compatible). Accept specification of channel or user. + if [ "${channel::1}" != "#" ] && [ "${channel::1}" != "@" ] ; then channel="#$channel"; fi + + # If channel is equal to "#" then do not send the channel attribute at all. Slack also defines channels and users in webhooks. + if [ "${channel}" = "#" ] ; then + ch="" + chstr="without specifying a channel" + else + ch="\"channel\": \"${channel}\"," + chstr="to '${channel}'" + fi + payload="$(cat <<EOF { - "channel": "#${channel}", + $ch "username": "netdata on ${host}", "icon_url": "${images_base_url}/images/banner-icon-144x144.png", "text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*", @@ -1500,7 +1304,7 @@ send_slack() { } ], "thumb_url": "${image}", - "footer": "by <${goto_url}|${this_host}>", + "footer": "by <${goto_url}|${host}>", "ts": ${when} } ] @@ -1511,10 +1315,10 @@ EOF httpcode=$(docurl -X POST --data-urlencode "payload=${payload}" "${webhook}") if [ "${httpcode}" = "200" ] then - info "sent slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}'" + info "sent slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}" sent=$((sent + 1)) else - error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}." + error "failed to send slack notification for: ${host} ${chart}.${name} is ${status} ${chstr}, with HTTP error code ${httpcode}." fi done @@ -1633,7 +1437,7 @@ send_alerta() { "source": "${src}", "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>" }, - "origin": "netdata/${this_host}", + "origin": "netdata/${host}", "type": "netdataAlarm", "rawData": "${BASH_ARGV[@]}" } @@ -1759,7 +1563,7 @@ send_discord() { ], "thumb_url": "${image}", "footer_icon": "${images_base_url}/images/banner-icon-144x144.png", - "footer": "${this_host}", + "footer": "${host}", "ts": ${when} } ] @@ -1813,11 +1617,55 @@ send_fleep() { } # ----------------------------------------------------------------------------- +# Prowl sender + +send_prowl() { + local httpcode sent=0 data message keys prio=0 alarm_url event + if [ "${SEND_PROWL}" = "YES" ] ; then + message="$(urlencode "${host} ${status_message}, \`${chart}\` (${family}), *${alarm}*\\n${info}")" + message="description=${message}" + keys="$(urlencode "$(echo "${1}" | tr ' ' ,)")" + keys="apikey=${keys}" + app="application=Netdata" + + case "${status}" in + CRITICAL) + prio=2 + ;; + WARNING) + prio=1 + ;; + esac + pri="priority=${pri}" + + alarm_url="$(urlencode ${goto_url})" + alarm_url="url=${alarm_url}" + event="$(urlencode "${host} ${status_message}")" + event="event=${event}" + + data="${keys}&${pri}&${alarm_url}&${app}&${event}&${message}" + + httpcode=$(docurl -X POST --data "${data}" "https://api.prowlapp.com/publicapi/add") + + if [ "${httpcode}" = "200" ] ; then + info "sent prowl data for: ${host} ${chart}.${name} is ${status}" + sent=1 + else + error "failed to send prowl data for: ${host} ${chart}.${name} is ${status} with with error code ${httpcode}." + fi + + [ ${sent} -gt 0 ] && return 0 + fi + + return 1 +} + +# ----------------------------------------------------------------------------- # irc sender send_irc() { local NICKNAME="${1}" REALNAME="${2}" CHANNELS="${3}" NETWORK="${4}" SERVERNAME="${5}" MESSAGE="${6}" sent=0 channel color send_alarm reply_codes error - + if [ "${SEND_IRC}" = "YES" -a ! -z "${NICKNAME}" -a ! -z "${REALNAME}" -a ! -z "${CHANNELS}" -a ! -z "${NETWORK}" -a ! -z "${SERVERNAME}" ] then case "${status}" in @@ -1842,13 +1690,13 @@ send_irc() { info "sent irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}'" sent=$((sent + 1)) else - error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}." + error "failed to send irc notification for: ${host} ${chart}.${name} is ${status} to '${CHANNEL}', with error code ${code}." fi done fi - + [ ${sent} -gt 0 ] && return 0 - + return 1 } @@ -1871,7 +1719,7 @@ send_awssns() { info "sent Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" sent=$((sent + 1)) else - error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" + error "failed to send Amazon SNS notification for: ${host} ${chart}.${name} is ${status} to '${target}'" fi done @@ -1957,11 +1805,30 @@ send_syslog() { # prepare the content of the notification # the url to send the user on click -urlencode "${host}" >/dev/null; url_host="${REPLY}" +urlencode "${args_host}" >/dev/null; url_host="${REPLY}" urlencode "${chart}" >/dev/null; url_chart="${REPLY}" urlencode "${family}" >/dev/null; url_family="${REPLY}" urlencode "${name}" >/dev/null; url_name="${REPLY}" -goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}" + +redirect_params="host=${url_host}&chart=${url_chart}&family=${url_family}&alarm=${url_name}&alarm_unique_id=${unique_id}&alarm_id=${alarm_id}&alarm_event_id=${event_id}" +GOTOCLOUD=0 + +if [ "${NETDATA_REGISTRY_URL}" == "https://registry.my-netdata.io" ] ; then + if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ] ; then + if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then + NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")" + fi + fi + if [ ! -z "${NETDATA_REGISTRY_UNIQUE_ID}" ] ; then + GOTOCLOUD=1 + fi +fi + +if [ ${GOTOCLOUD} -eq 0 ] ; then + goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?${redirect_params}" +else + goto_url="https://netdata.cloud/alarms/redirect?agentID=${NETDATA_REGISTRY_UNIQUE_ID}&${redirect_params}" +fi # the severity of the alarm severity="${status}" @@ -2127,7 +1994,7 @@ SENT_PUSHBULLET=$? # ----------------------------------------------------------------------------- # send the twilio SMS -send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} +send_twilio "${TWILIO_ACCOUNT_SID}" "${TWILIO_ACCOUNT_TOKEN}" "${TWILIO_NUMBER}" "${to_twilio}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2138,7 +2005,7 @@ SENT_TWILIO=$? # ----------------------------------------------------------------------------- # send the messagebird SMS -send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} +send_messagebird "${MESSAGEBIRD_ACCESS_KEY}" "${MESSAGEBIRD_NUMBER}" "${to_messagebird}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2150,7 +2017,7 @@ SENT_MESSAGEBIRD=$? # ----------------------------------------------------------------------------- # send the kavenegar SMS -send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} +send_kavenegar "${KAVENEGAR_API_KEY}" "${KAVENEGAR_SENDER}" "${to_kavenegar}" "${host} ${status_message} - ${name//_/ } - ${chart}" "${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2191,9 +2058,15 @@ send_fleep "${to_fleep}" SENT_FLEEP=$? # ----------------------------------------------------------------------------- +# send the Prowl message + +send_prowl "${to_prowl}" +SENT_PROWL=$? + +# ----------------------------------------------------------------------------- # send the irc message -send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm} +send_irc "${IRC_NICKNAME}" "${IRC_REALNAME}" "${to_irc}" "${IRC_NETWORK}" "${host}" "${host} ${status_message} - ${name//_/ } - ${chart} ----- ${alarm} Severity: ${severity} Chart: ${chart} Family: ${family} @@ -2278,7 +2151,12 @@ Severity: ${severity} URL : ${goto_url} Source : ${src} Date : ${date} -Notification generated on ${this_host} +Notification generated on ${host} + +Evaluated Expression : ${calc_expression} +Expression Variables : ${calc_param_values} + +The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised. --multipart-boundary Content-Type: text/html; encoding=${EMAIL_CHARSET} @@ -2341,6 +2219,24 @@ Content-Transfer-Encoding: 8bit </td> </tr> <tr style="margin: 0; padding: 0;"> + <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> + <span>${calc_expression}</span> + <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span> + </td> + </tr> + <tr style="margin: 0; padding: 0;"> + <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> + <span>${calc_param_values}</span> + <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span> + </td> + </tr> + <tr style="margin: 0; padding: 0;"> + <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top"> + The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised. + </td> + </tr> + + <tr style="margin: 0; padding: 0;"> <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;"> <a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a> </td> @@ -2351,7 +2247,7 @@ Content-Transfer-Encoding: 8bit </tr> <tr style="text-align: center; margin: 0; padding: 0;"> <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 12px; vertical-align: top; margin:0; padding: 20px 0 0 0; color: #666666; border-top: 1px solid #f0f0f0;" align="center" valign="bottom">Sent by - <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${this_host}</code>. + <a href="https://mynetdata.io/" target="_blank">netdata</a>, the real-time performance and health monitoring, on <code>${host}</code>. </td> </tr> </tbody> @@ -2393,6 +2289,7 @@ if [ ${SENT_EMAIL} -eq 0 \ -o ${SENT_KAFKA} -eq 0 \ -o ${SENT_PD} -eq 0 \ -o ${SENT_FLEEP} -eq 0 \ + -o ${SENT_PROWL} -eq 0 \ -o ${SENT_IRC} -eq 0 \ -o ${SENT_AWSSNS} -eq 0 \ -o ${SENT_CUSTOM} -eq 0 \ |