summaryrefslogtreecommitdiffstats
path: root/health/notifications
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2018-12-28 14:38:58 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2018-12-28 14:38:58 +0000
commitfa4ece01aed54c9a146af868be0d3db611ded229 (patch)
tree319cffc5f6c2abd7cce514383716153469fc6295 /health/notifications
parentNew upstream version 1.11.0+dfsg (diff)
downloadnetdata-fa4ece01aed54c9a146af868be0d3db611ded229.tar.xz
netdata-fa4ece01aed54c9a146af868be0d3db611ded229.zip
New upstream version 1.11.1+dfsgupstream/1.11.1+dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'health/notifications')
-rw-r--r--health/notifications/alarm-notify.sh73
-rwxr-xr-xhealth/notifications/alarm-notify.sh.in73
-rw-r--r--health/notifications/alerta/README.md238
-rwxr-xr-xhealth/notifications/health_alarm_notify.conf5
4 files changed, 146 insertions, 243 deletions
diff --git a/health/notifications/alarm-notify.sh b/health/notifications/alarm-notify.sh
index 33a59590e..3331dcd94 100644
--- a/health/notifications/alarm-notify.sh
+++ b/health/notifications/alarm-notify.sh
@@ -896,7 +896,7 @@ date=$(date --date=@${when} "${date_format}" 2>/dev/null)
# ----------------------------------------------------------------------------
# prepare some extra headers if we've been asked to thread e-mails
-if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" == "YES" ] ; then
+if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" != "NO" ] ; then
email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\nReferences: <${chart}-${name}@${host}>"
else
email_thread_headers=
@@ -1480,7 +1480,7 @@ send_slack() {
{
"channel": "#${channel}",
"username": "netdata on ${host}",
- "icon_url": "${images_base_url}/images/seo-performance-128.png",
+ "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
"text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
"attachments": [
{
@@ -1545,7 +1545,7 @@ send_rocketchat() {
{
"channel": "#${channel}",
"alias": "netdata on ${host}",
- "avatar": "${images_base_url}/images/seo-performance-128.png",
+ "avatar": "${images_base_url}/images/banner-icon-144x144.png",
"text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
"attachments": [
{
@@ -1592,39 +1592,68 @@ EOF
# alerta sender
send_alerta() {
- local webhook="${1}" channels="${2}" httpcode sent=0 channel severity content
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
[ "${SEND_ALERTA}" != "YES" ] && return 1
case "${status}" in
- WARNING) severity="warning" ;;
CRITICAL) severity="critical" ;;
+ WARNING) severity="warning" ;;
CLEAR) severity="cleared" ;;
- *) severity="unknown" ;;
+ *) severity="indeterminate" ;;
esac
- info=$( echo -n ${info})
+ if [[ "${chart}" == httpcheck* ]]
+ then
+ resource=$chart
+ event=$name
+ else
+ resource="${host}:${family}"
+ event="${chart}.${name}"
+ fi
- # the "event" property must be unique and repetible between states to let alerta do automatic correlation using severity value
for channel in ${channels}
do
- content="{"
- content="$content \"environment\": \"${channel}\","
- content="$content \"service\": [\"${host}\"],"
- content="$content \"resource\": \"${host}\","
- content="$content \"event\": \"${name}.${chart} (${family})\","
- content="$content \"severity\": \"${severity}\","
- content="$content \"value\": \"${alarm}\","
- content="$content \"text\": \"${info}\""
- content="$content }"
+ payload="$(cat <<EOF
+ {
+ "resource": "${resource}",
+ "event": "${event}",
+ "environment": "${channel}",
+ "severity": "${severity}",
+ "service": ["Netdata"],
+ "group": "Performance",
+ "value": "${value_string}",
+ "text": "${info}",
+ "tags": ["alarm_id:${alarm_id}"],
+ "attributes": {
+ "roles": "${roles}",
+ "name": "${name}",
+ "chart": "${chart}",
+ "family": "${family}",
+ "source": "${src}",
+ "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
+ },
+ "origin": "netdata/${this_host}",
+ "type": "netdataAlarm",
+ "rawData": "${BASH_ARGV[@]}"
+ }
+EOF
+ )"
+ if [[ -n "${ALERTA_API_KEY}" ]]
+ then
+ auth="Key ${ALERTA_API_KEY}"
+ fi
- httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: Key $ALERTA_API_KEY" -d "$content" )
+ httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]]
then
info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
sent=$((sent + 1))
+ elif [[ "${httpcode}" = "202" ]]
+ then
+ info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
else
error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
fi
@@ -1655,7 +1684,7 @@ send_flock() {
httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
\"sendAs\": {
\"name\" : \"netdata on ${host}\",
- \"profileImage\" : \"${images_base_url}/images/seo-performance-128.png\"
+ \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
},
\"text\": \"${host} *${status_message}*\",
\"timestamp\": \"${when}\",
@@ -1715,7 +1744,7 @@ send_discord() {
"channel": "#${channel}",
"username": "${username}",
"text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
- "icon_url": "${images_base_url}/images/seo-performance-128.png",
+ "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
"attachments": [
{
"color": "${color}",
@@ -1729,7 +1758,7 @@ send_discord() {
}
],
"thumb_url": "${image}",
- "footer_icon": "${images_base_url}/images/seo-performance-128.png",
+ "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
"footer": "${this_host}",
"ts": ${when}
}
@@ -1952,7 +1981,7 @@ color="grey"
alarm="${name//_/ } = ${value_string}"
# the image of the alarm
-image="${images_base_url}/images/seo-performance-128.png"
+image="${images_base_url}/images/banner-icon-144x144.png"
# prepare the title based on status
case "${status}" in
diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in
index 4aef3a521..ea8223097 100755
--- a/health/notifications/alarm-notify.sh.in
+++ b/health/notifications/alarm-notify.sh.in
@@ -896,7 +896,7 @@ date=$(date --date=@${when} "${date_format}" 2>/dev/null)
# ----------------------------------------------------------------------------
# prepare some extra headers if we've been asked to thread e-mails
-if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" == "YES" ] ; then
+if [ "${SEND_EMAIL}" == "YES" -a "${EMAIL_THREADING}" != "NO" ] ; then
email_thread_headers="In-Reply-To: <${chart}-${name}@${host}>\nReferences: <${chart}-${name}@${host}>"
else
email_thread_headers=
@@ -1480,7 +1480,7 @@ send_slack() {
{
"channel": "#${channel}",
"username": "netdata on ${host}",
- "icon_url": "${images_base_url}/images/seo-performance-128.png",
+ "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
"text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
"attachments": [
{
@@ -1545,7 +1545,7 @@ send_rocketchat() {
{
"channel": "#${channel}",
"alias": "netdata on ${host}",
- "avatar": "${images_base_url}/images/seo-performance-128.png",
+ "avatar": "${images_base_url}/images/banner-icon-144x144.png",
"text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
"attachments": [
{
@@ -1592,39 +1592,68 @@ EOF
# alerta sender
send_alerta() {
- local webhook="${1}" channels="${2}" httpcode sent=0 channel severity content
+ local webhook="${1}" channels="${2}" httpcode sent=0 channel severity resource event payload auth
[ "${SEND_ALERTA}" != "YES" ] && return 1
case "${status}" in
- WARNING) severity="warning" ;;
CRITICAL) severity="critical" ;;
+ WARNING) severity="warning" ;;
CLEAR) severity="cleared" ;;
- *) severity="unknown" ;;
+ *) severity="indeterminate" ;;
esac
- info=$( echo -n ${info})
+ if [[ "${chart}" == httpcheck* ]]
+ then
+ resource=$chart
+ event=$name
+ else
+ resource="${host}:${family}"
+ event="${chart}.${name}"
+ fi
- # the "event" property must be unique and repetible between states to let alerta do automatic correlation using severity value
for channel in ${channels}
do
- content="{"
- content="$content \"environment\": \"${channel}\","
- content="$content \"service\": [\"${host}\"],"
- content="$content \"resource\": \"${host}\","
- content="$content \"event\": \"${name}.${chart} (${family})\","
- content="$content \"severity\": \"${severity}\","
- content="$content \"value\": \"${alarm}\","
- content="$content \"text\": \"${info}\""
- content="$content }"
+ payload="$(cat <<EOF
+ {
+ "resource": "${resource}",
+ "event": "${event}",
+ "environment": "${channel}",
+ "severity": "${severity}",
+ "service": ["Netdata"],
+ "group": "Performance",
+ "value": "${value_string}",
+ "text": "${info}",
+ "tags": ["alarm_id:${alarm_id}"],
+ "attributes": {
+ "roles": "${roles}",
+ "name": "${name}",
+ "chart": "${chart}",
+ "family": "${family}",
+ "source": "${src}",
+ "moreInfo": "<a href=\"${goto_url}\">View Netdata</a>"
+ },
+ "origin": "netdata/${this_host}",
+ "type": "netdataAlarm",
+ "rawData": "${BASH_ARGV[@]}"
+ }
+EOF
+ )"
+ if [[ -n "${ALERTA_API_KEY}" ]]
+ then
+ auth="Key ${ALERTA_API_KEY}"
+ fi
- httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: Key $ALERTA_API_KEY" -d "$content" )
+ httpcode=$(docurl -X POST "${webhook}/alert" -H "Content-Type: application/json" -H "Authorization: $auth" --data "${payload}")
if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]]
then
info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
sent=$((sent + 1))
+ elif [[ "${httpcode}" = "202" ]]
+ then
+ info "suppressed alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
else
error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
fi
@@ -1655,7 +1684,7 @@ send_flock() {
httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -d "{
\"sendAs\": {
\"name\" : \"netdata on ${host}\",
- \"profileImage\" : \"${images_base_url}/images/seo-performance-128.png\"
+ \"profileImage\" : \"${images_base_url}/images/banner-icon-144x144.png\"
},
\"text\": \"${host} *${status_message}*\",
\"timestamp\": \"${when}\",
@@ -1715,7 +1744,7 @@ send_discord() {
"channel": "#${channel}",
"username": "${username}",
"text": "${host} ${status_message}, \`${chart}\` (_${family}_), *${alarm}*",
- "icon_url": "${images_base_url}/images/seo-performance-128.png",
+ "icon_url": "${images_base_url}/images/banner-icon-144x144.png",
"attachments": [
{
"color": "${color}",
@@ -1729,7 +1758,7 @@ send_discord() {
}
],
"thumb_url": "${image}",
- "footer_icon": "${images_base_url}/images/seo-performance-128.png",
+ "footer_icon": "${images_base_url}/images/banner-icon-144x144.png",
"footer": "${this_host}",
"ts": ${when}
}
@@ -1952,7 +1981,7 @@ color="grey"
alarm="${name//_/ } = ${value_string}"
# the image of the alarm
-image="${images_base_url}/images/seo-performance-128.png"
+image="${images_base_url}/images/banner-icon-144x144.png"
# prepare the title based on status
case "${status}" in
diff --git a/health/notifications/alerta/README.md b/health/notifications/alerta/README.md
index bbed23bac..cf43621ff 100644
--- a/health/notifications/alerta/README.md
+++ b/health/notifications/alerta/README.md
@@ -1,207 +1,50 @@
# alerta.io notifications
-The alerta monitoring system is a tool used to consolidate and de-duplicate alerts from multiple sources for quick ‘at-a-glance’ visualisation. With just one system you can monitor alerts from many other monitoring tools on a single screen.
+The [Alerta](https://alerta.io) monitoring system is a tool used to
+consolidate and de-duplicate alerts from multiple sources for quick
+‘at-a-glance’ visualisation. With just one system you can monitor
+alerts from many other monitoring tools on a single screen.
-![](http://docs.alerta.io/en/latest/_images/alerta-screen-shot-3.png)
+![](https://docs.alerta.io/en/latest/_images/alerta-screen-shot-3.png)
-When receiving alerts from multiple sources you can quickly become overwhelmed. With Alerta any alert with the same environment and resource is considered a duplicate if it has the same severity. If it has a different severity it is correlated so that you only see the most recent one. Awesome.
+Netadata alarms can be sent to Alerta so you can see in one place
+alerts coming from many Netdata hosts or also from a multi-host
+Netadata configuration. The big advantage over other notifications
+systems is that there is a main view of all active alarms with
+the most recent state, and it is also possible to view alarm history.
-main site http://www.alerta.io
+## Deploying Alerta
-We can send Netadata alarms to Alerta so yo can see in one place alerts coming from many Netdata hosts or also from a multihost Netadata configuration.\
-The big advantage over other notifications method is that you have in a main view all active alarms with only las state, but you can also search history.
+It is recommended to set up the server in a separated server, VM or
+container. If you have other Nginx or Apache server in your organization,
+it is recommended to proxy to this new server.
-## Setting up an Alerta server with Ubuntu 16.04
+The easiest way to install Alerta is to use the Docker image available
+on [Docker hub][1]. Alternatively, follow the ["getting started"][2]
+tutorial to deploy Alerta to an Ubuntu server. More advanced
+configurations are out os scope of this tutorial but information
+about different deployment scenaries can be found in the [docs][3].
-Here we will set a basic Alerta server to test it with Netdata alerts.\
-More advanced configurations are out os scope of this tutorial.
+[1]: https://hub.docker.com/r/alerta/alerta-web/
+[2]: http://alerta.readthedocs.io/en/latest/gettingstarted/tutorial-1-deploy-alerta.html
+[3]: http://docs.alerta.io/en/latest/deployment.html
-source: http://alerta.readthedocs.io/en/latest/gettingstarted/tutorial-1-deploy-alerta.html
+## Send alarms to Alerta
-I recommend to set up the server in a separated server, VM or container.\
-If you have other Nginx or Apache server in your organization, I recommend to proxy to this new server.
+Step 1. Create an API key (if authentication is enabled)
-Set us as root for easiest working
-```
-sudo su
-cd
-```
-
-Install Mongodb https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
-```
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
-echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list
-apt-get update
-apt-get install -y mongodb-org
-systemctl enable mongod
-systemctl start mongod
-systemctl status mongod
-```
-
-Install Nginx and Alerta uwsgi
-```
-apt-get install -y python-pip python-dev nginx
-pip install alerta-server uwsgi
-```
+You will need an API key to send messages from any source, if
+Alerta is configured to use authentication (recommended). To
+create an API key go to "Configuration -> API Keys" and create
+a new API key called "netdata" with `write:alerts` permission.
-Install web console
-```
-cd /var/www/html
-mkdir alerta
-cd alerta
-wget -q -O - https://github.com/alerta/angular-alerta-webui/tarball/master | tar zxf -
-mv alerta*/app/* .
-cd
-```
-## Services configuration
-
-Create a wsgi python file
-```
-nano /var/www/wsgi.py
-```
-fill with
-```
-from alerta import app
-```
-Create uWsgi configuration file
-```
-nano /etc/uwsgi.ini
-```
-fill with
-```
-[uwsgi]
-chdir = /var/www
-mount = /alerta/api=wsgi.py
-callable = app
-manage-script-name = true
-
-master = true
-processes = 5
-logger = syslog:alertad
-
-socket = /tmp/uwsgi.sock
-chmod-socket = 664
-uid = www-data
-gid = www-data
-vacuum = true
-
-die-on-term = true
-```
-Create a systemd configuration file
-```
-nano /etc/systemd/system/uwsgi.service
-```
-fill with
-```
-[Unit]
-Description=uWSGI service
-
-[Service]
-ExecStart=/usr/local/bin/uwsgi --ini /etc/uwsgi.ini
-
-[Install]
-WantedBy=multi-user.target
-```
-enable service
-```
-systemctl start uwsgi
-systemctl status uwsgi
-systemctl enable uwsgi
-```
-Configure nginx to serve Alerta as a uWsgi application on /alerta/api
-```
-nano /etc/nginx/sites-enabled/default
-```
-fill with
-```
-server {
- listen 80 default_server;
- listen [::]:80 default_server;
-
- location /alerta/api { try_files $uri @alerta/api; }
- location @alerta/api {
- include uwsgi_params;
- uwsgi_pass unix:/tmp/uwsgi.sock;
- proxy_set_header Host $host:$server_port;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- }
-
- location / {
- root /var/www/html;
- }
-}
-```
-restart nginx
-```
-service nginx restart
-```
-## Config web console
-```
-nano /var/www/html/config.js
-```
-fill with
-```
-'use strict';
-
-angular.module('config', [])
- .constant('config', {
- 'endpoint' : "/alerta/api",
- 'provider' : "basic",
- 'colors' : {},
- 'severity' : {},
- 'audio' : {}
- });
-```
-
-## Config Alerta server
-
-source: http://alerta.readthedocs.io/en/latest/configuration.html
-
-Create a random string to use as SECRET_KEY
-```
-cat /dev/urandom | tr -dc A-Za-z0-9_\!\@\#\$\%\^\&\*\(\)-+= | head -c 32 && echo
-```
-will output something like
-```
-0pv8Bw7VKfW6avDAz_TqzYPme_fYV%7g
-```
-Edit alertad.conf
-```
-nano /etc/alertad.conf
-```
-fill with (take care about all single quotes)
-```
-BASE_URL='/alerta/api'
-AUTH_REQUIRED=True
-SECRET_KEY='0pv8Bw7VKfW6avDAz_TqzYPme_fYV%7g'
-ADMIN_USERS=['<here put you email for future login>']
-```
-
-restart
-```
-systemctl restart uwsgi
-```
-
-* go to console to http://yourserver/alerta/
-* go to Login -> Create an account
-* use your email for login so and administrative account will be created
-
-## create an API KEY
-
-You need an API KEY to send messages from any source.\
-To create an API KEY go to Configuration -> Api Keys\
-Then create a API KEY with write permisions.
-
-## configure Netdata to send alarms to Alerta
+Step 2. configure Netdata to send alarms to Alerta
On your system run:
-```
-/etc/netdata/edit-config health_alarm_notify.conf
-```
+ $ /etc/netdata/edit-config health_alarm_notify.conf
-and set
+and modify the file as below:
```
# enable/disable sending alerta notifications
@@ -214,7 +57,7 @@ ALERTA_WEBHOOK_URL="http://yourserver/alerta/api"
# Login with an administrative user to you Alerta server and create an API KEY
# with write permissions.
-ALERTA_API_KEY="you last created API KEY"
+ALERTA_API_KEY="INSERT_YOUR_API_KEY_HERE"
# you can define environments in /etc/alertad.conf option ALLOWED_ENVIRONMENTS
# standard environments are Production and Development
@@ -225,12 +68,13 @@ DEFAULT_RECIPIENT_ALERTA="Production"
## Test alarms
-We can test alarms with standard
-```
-sudo su -s /bin/bash netdata
-/opt/netdata/netdata-plugins/plugins.d/alarm-notify.sh test
-exit
-```
-But the problem is that Netdata will send 3 alarms, and because last alarm is "CLEAR" you will not se them in main Alerta page, you need to select to see "closed" alarma in top-right lookup.
+We can test alarms using the standard approach:
+
+ $ /opt/netdata/netdata-plugins/plugins.d/alarm-notify.sh test
+
+Note: Netdata will send 3 alarms, and because last alarm is "CLEAR"
+you will not se them in main Alerta page, you need to select to see
+"closed" alarma in top-right lookup. A little change in `alarm-notify.sh`
+that let us test each state one by one will be useful.
-A little change in alarm-notify.sh that let us test each state one by one will be useful. \ No newline at end of file
+For more information see [https://docs.alerta.io](https://docs.alerta.io)
diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf
index 9e72aac4d..a997765a6 100755
--- a/health/notifications/health_alarm_notify.conf
+++ b/health/notifications/health_alarm_notify.conf
@@ -183,8 +183,9 @@ DEFAULT_RECIPIENT_EMAIL="root"
# chart+alarm+host combination as a single thread. This can help
# simplify tracking of alarms, as it provides an easy wway for scripts
# to corelate messages and also will cause most clients to group all the
-# messages together. THis is off by default.
-#EMAIL_THREADING="YES"
+# messages together. This is enabled by default, uncomment the line
+# below if you want to disable it.
+#EMAIL_THREADING="NO"
#------------------------------------------------------------------------------