# Configuration for alarm notifications # # This configuration is used by: alarm-notify.sh # changes take effect immediately (the next alarm will use them). # # alarm-notify.sh can send: # - e-mails (using the sendmail command), # - push notifications to your mobile phone (pushover.net), # - messages to your slack team (slack.com), # - messages to your discord guild (discordapp.com), # - messages to your telegram chat / group chat (telegram.org) # - sms messages to your cell phone or any sms enabled device (twilio.com) # - sms messages to your cell phone or any sms enabled device (messagebird.com) # - notifications to users on pagerduty.com # # The 'to' line given at netdata alarms defines a *role*, so that many # people can be notified for each role. # # This file is a BASH script itself. # # #------------------------------------------------------------------------------ # proxy configuration # # If you need to send curl based notifications (pushover, pushbullet, slack, # discord, telegram) via a proxy, set these to your proxy address: #export http_proxy="http://10.0.0.1:3128/" #export https_proxy="http://10.0.0.1:3128/" #------------------------------------------------------------------------------ # notifications images # # Images in notifications need to be downloaded from an Internet facing site. # To allow notification providers fetch the icons/images, by default we set # the URL of the global public netdata registry. # If you have an Internet facing netdata (or you have copied the images/ folder # of netdata to your web server), set its URL here, to fetch the notification # images from it. #images_base_url="http://my.public.netdata.server:19999" #------------------------------------------------------------------------------ # external commands # The full path to the sendmail command. # If empty, the system $PATH will be searched for it. # If not found, email notifications will be disabled (silently). sendmail="" # The full path of the curl command. # If empty, the system $PATH will be searched for it. # If not found, most notifications will be silently disabled. curl="" #------------------------------------------------------------------------------ # NOTE ABOUT RECIPIENTS # # When you define recipients (all types): # # - emails addresses # - pushover user tokens # - telegram chat ids # - slack channels # - discord channels # - hipchat rooms # - sms phone numbers # - pagerduty.com (pd) services # # You can append |critical to limit the notifications to be sent. # # In these examples, the first recipient receives all the alarms # while the second one receives only the critical ones: # # email : "user1@example.com user2@example.com|critical" # pushover : "2987343...9437837 8756278...2362736|critical" # telegram : "111827421 112746832|critical" # slack : "alarms disasters|critical" # discord : "alarms disasters|critical" # twilio : "+15555555555 +17777777777|critical" # messagebird: "+15555555555 +17777777777|critical" # pd : " |critical" # # If a recipient is set to empty string, the default recipient of the given # notification method (email, pushover, telegram, slack, etc) will be used. # To disable a notification, use the recipient called: disabled # This works for all notification methods (including the default recipients). #------------------------------------------------------------------------------ # email global notification options # multiple recipients can be given like this: # "admin1@example.com admin2@example.com ..." # the email address sending email notifications # the default is the system user netdata runs as (usually: netdata) # The following formats are supported: # EMAIL_SENDER="user@domain" # EMAIL_SENDER="User Name " # EMAIL_SENDER="'User Name' " # EMAIL_SENDER="\"User Name\" " EMAIL_SENDER="" # enable/disable sending emails SEND_EMAIL="YES" # if a role recipient is not configured, an email will be send to: DEFAULT_RECIPIENT_EMAIL="root" # to receive only critical alarms, set it to "root|critical" # Optionally specify the encoding to list in the Content-Type header. # This doesn't change what encoding the e-mail is sent with, just what # the headers say it was encoded as. # This shouldn't need to be changed as it will almost always be # autodetected from the environment. #EMAIL_CHARSET="UTF-8" #------------------------------------------------------------------------------ # pushover (pushover.net) global notification options # multiple recipients can be given like this: # "USERTOKEN1 USERTOKEN2 ..." # enable/disable sending pushover notifications SEND_PUSHOVER="YES" # Login to pushover.net to get your pushover app token. # You need only one for all your netdata servers (or you can have one for # each of your netdata - your call). # Without an app token, netdata cannot send pushover notifications. PUSHOVER_APP_TOKEN="" # if a role's recipients are not configured, a notification will be send to # this pushover user token (empty = do not send a notification for unconfigured # roles): DEFAULT_RECIPIENT_PUSHOVER="" #------------------------------------------------------------------------------ # pushbullet (pushbullet.com) push notification options # multiple recipients can be given like this: # "user1@email.com user2@mail.com" # enable/disable sending pushbullet notifications SEND_PUSHBULLET="YES" # Signup and Login to pushbullet.com # To get your Access Token, go to https://www.pushbullet.com/#settings/account # Create a new access token and paste it below. # Then just set the recipients' emails. # Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does # not have a pushbullet account, the pushbullet service will send an email # to that address instead. # Without an access token, netdata cannot send pushbullet notifications. PUSHBULLET_ACCESS_TOKEN="" DEFAULT_RECIPIENT_PUSHBULLET="" #------------------------------------------------------------------------------ # Twilio (twilio.com) SMS options # multiple recipients can be given like this: # "+15555555555 +17777777777" # enable/disable sending twilio SMS SEND_TWILIO="YES" # Signup for free trial and select a SMS capable Twilio Number # To get your Account SID and Token, go to https://www.twilio.com/console # Place your sid, token and number below. # Then just set the recipients' phone numbers. # The trial account is only allowed to use the number specified when set up. # Without an account sid and token, netdata cannot send Twilio text messages. TWILIO_ACCOUNT_SID="" TWILIO_ACCOUNT_TOKEN="" TWILIO_NUMBER="" DEFAULT_RECIPIENT_TWILIO="" #------------------------------------------------------------------------------ # Messagebird (messagebird.com) SMS options # multiple recipients can be given like this: # "+15555555555 +17777777777" # enable/disable sending messagebird SMS SEND_MESSAGEBIRD="YES" # to get an access key, create a free account at https://www.messagebird.com # verify and activate the account (no CC info needed) # login to your account and enter your phonenumber to get some free credits # to get the API key, click on 'API' in the sidebar, then 'API Access (REST)' # click 'Add access key' and fill in data (you want a live key to send SMS) # Without an access key, netdata cannot send Messagebird text messages. MESSAGEBIRD_ACCESS_KEY="" MESSAGEBIRD_NUMBER="" DEFAULT_RECIPIENT_MESSAGEBIRD="" #------------------------------------------------------------------------------ # telegram (telegram.org) global notification options # To get your chat ID send the command /my_id to telegram bot @get_id. # Users also need to open a query with the bot (see below). # note: multiple recipients can be given like this: # "CHAT_ID_1 CHAT_ID_2 ..." # enable/disable sending telegram messages SEND_TELEGRAM="YES" # Contact the bot @BotFather to create a new bot and receive a bot token. # Without it, netdata cannot send telegram messages. TELEGRAM_BOT_TOKEN="" # If a role's recipients are not configured, a message will be send to # this chat id (empty = do not send a notification for unconfigured roles): DEFAULT_RECIPIENT_TELEGRAM="" #------------------------------------------------------------------------------ # slack (slack.com) global notification options # multiple recipients can be given like this: # "CHANNEL1 CHANNEL2 ..." # enable/disable sending slack notifications SEND_SLACK="YES" # Login to slack.com and create an incoming webhook. You need only one for all # your netdata servers (or you can have one for each of your netdata). # Without it, netdata cannot send slack notifications. # Get yours from: https://api.slack.com/incoming-webhooks SLACK_WEBHOOK_URL="" # if a role's recipients are not configured, a notification will be send to # this slack channel (empty = do not send a notification for unconfigured # roles): DEFAULT_RECIPIENT_SLACK="" #------------------------------------------------------------------------------ # discord (discordapp.com) global notification options # multiple recipients can be given like this: # "CHANNEL1 CHANNEL2 ..." # enable/disable sending discord notifications SEND_DISCORD="YES" # Create a webhook by following the official documentation - # https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks DISCORD_WEBHOOK_URL="" # if a role's recipients are not configured, a notification will be send to # this discord channel (empty = do not send a notification for unconfigured # roles): DEFAULT_RECIPIENT_DISCORD="" #------------------------------------------------------------------------------ # hipchat global notification options # multiple recipients can be given like this: # "ROOM1 ROOM2 ..." # enable/disable sending hipchat notifications SEND_HIPCHAT="YES" # define hipchat server HIPCHAT_SERVER="api.hipchat.com" # api.hipchat.com authorization token # Without this, netdata cannot send hipchat notifications. HIPCHAT_AUTH_TOKEN="" # if a role's recipients are not configured, a notification will be send to # this hipchat room (empty = do not send a notification for unconfigured # roles): DEFAULT_RECIPIENT_HIPCHAT="" #------------------------------------------------------------------------------ # kafka notification options # enable/disable sending kafka notifications SEND_KAFKA="YES" # The URL to POST kafka alarm data to. It should be the full URL. KAFKA_URL="" # The IP to be used in the kafka message as the sender. KAFKA_SENDER_IP="" #------------------------------------------------------------------------------ # pagerduty.com notification options # # pagerduty.com notifications require the pagerduty agent to be installed and # a "Generic API" pagerduty service. # https://www.pagerduty.com/docs/guides/agent-install-guide/ # multiple recipients can be given like this: # " ..." # enable/disable sending pagerduty notifications SEND_PD="YES" # if a role's recipients are not configured, a notification will be sent to # the "General API" pagerduty.com service that uses this service key. # (empty = do not send a notification for unconfigured roles): DEFAULT_RECIPIENT_PD="" #------------------------------------------------------------------------------ # custom notifications # # enable/disable sending custom notifications SEND_CUSTOM="YES" # if a role's recipients are not configured, use the following. # (empty = do not send a notification for unconfigured roles) DEFAULT_RECIPIENT_CUSTOM="" # The custom_sender() is a custom function to do whatever you need to do custom_sender() { # variables you can use: # ${host} the host generated this event # ${url_host} same as ${host} but URL encoded # ${unique_id} the unique id of this event # ${alarm_id} the unique id of the alarm that generated this event # ${event_id} the incremental id of the event, for this alarm id # ${when} the timestamp this event occurred # ${name} the name of the alarm, as given in netdata health.d entries # ${url_name} same as ${name} but URL encoded # ${chart} the name of the chart (type.id) # ${url_chart} same as ${chart} but URL encoded # ${family} the family of the chart # ${url_family} same as ${family} but URL encoded # ${status} the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL # ${old_status} the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL # ${value} the current value of the alarm # ${old_value} the previous value of the alarm # ${src} the line number and file the alarm has been configured # ${duration} the duration in seconds of the previous alarm state # ${duration_txt} same as ${duration} for humans # ${non_clear_duration} the total duration in seconds this is/was non-clear # ${non_clear_duration_txt} same as ${non_clear_duration} for humans # ${units} the units of the value # ${info} a short description of the alarm # ${value_string} friendly value (with units) # ${old_value_string} friendly old value (with units) # ${image} the URL of an image to represent the status of the alarm # ${color} a color in #AABBCC format for the alarm # ${goto_url} the URL the user can click to see the netdata dashboard # these are more human friendly: # ${alarm} like "name = value units" # ${status_message} like "needs attention", "recovered", "is critical" # ${severity} like "Escalated to CRITICAL", "Recovered from WARNING" # ${raised_for} like "(alarm was raised for 10 minutes)" # example human readable SMS local msg="${host} ${status_message}: ${alarm} ${raised_for}" # limit it to 160 characters and encode it for use in a URL urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}" # a space separated list of the recipients to send alarms to to="${1}" info "not sending custom notification to ${to}, for ${status} of '${host}.${chart}.${name}' - custom_sender() is not configured." } ############################################################################### # RECIPIENTS PER ROLE # ----------------------------------------------------------------------------- # generic system alarms # CPU, disks, network interfaces, entropy, etc role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}" role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}" role_recipients_pushbullet[sysadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}" role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}" role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}" # ----------------------------------------------------------------------------- # DNS related alarms role_recipients_email[domainadmin]="${DEFAULT_RECIPIENT_EMAIL}" role_recipients_pushover[domainadmin]="${DEFAULT_RECIPIENT_PUSHOVER}" role_recipients_pushbullet[domainadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}" role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}" role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}" # ----------------------------------------------------------------------------- # database servers alarms # mysql, redis, memcached, postgres, etc role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}" role_recipients_pushover[dba]="${DEFAULT_RECIPIENT_PUSHOVER}" role_recipients_pushbullet[dba]="${DEFAULT_RECIPIENT_PUSHBULLET}" role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}" role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}" role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}" # ----------------------------------------------------------------------------- # web servers alarms # apache, nginx, lighttpd, etc role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}" role_recipients_pushover[webmaster]="${DEFAULT_RECIPIENT_PUSHOVER}" role_recipients_pushbullet[webmaster]="${DEFAULT_RECIPIENT_PUSHBULLET}" role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}" role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}" role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}" # ----------------------------------------------------------------------------- # proxy servers alarms # squid, etc role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}" role_recipients_pushover[proxyadmin]="${DEFAULT_RECIPIENT_PUSHOVER}" role_recipients_pushbullet[proxyadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}" role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}" role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}" role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}" role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}" role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"