diff options
Diffstat (limited to 'health')
42 files changed, 249 insertions, 148 deletions
diff --git a/health/Makefile.am b/health/Makefile.am index 349b86d61..d5eb88468 100644 --- a/health/Makefile.am +++ b/health/Makefile.am @@ -27,7 +27,6 @@ dist_healthconfig_DATA = \ health.d/adaptec_raid.conf \ health.d/anomalies.conf \ health.d/apcupsd.conf \ - health.d/backend.conf \ health.d/bcache.conf \ health.d/beanstalkd.conf \ health.d/bind_rndc.conf \ @@ -46,7 +45,6 @@ dist_healthconfig_DATA = \ health.d/fping.conf \ health.d/geth.conf \ health.d/ioping.conf \ - health.d/fronius.conf \ health.d/gearman.conf \ health.d/go.d.plugin.conf \ health.d/haproxy.conf \ @@ -66,6 +64,7 @@ dist_healthconfig_DATA = \ health.d/mysql.conf \ health.d/net.conf \ health.d/netfilter.conf \ + health.d/nut.conf \ health.d/pihole.conf \ health.d/portcheck.conf \ health.d/processes.conf \ @@ -77,7 +76,6 @@ dist_healthconfig_DATA = \ health.d/riakkv.conf \ health.d/scaleio.conf \ health.d/softnet.conf \ - health.d/stiebeleltron.conf \ health.d/synchronization.conf \ health.d/swap.conf \ health.d/systemdunits.conf \ diff --git a/health/QUICKSTART.md b/health/QUICKSTART.md index bc91caf7c..5cf6929dc 100644 --- a/health/QUICKSTART.md +++ b/health/QUICKSTART.md @@ -52,8 +52,8 @@ Navigate to the health configuration directory to see all the available files an cd /usr/lib/netdata/conf.d/health.d/ ls adaptec_raid.conf entropy.conf memory.conf squid.conf -am2320.conf fping.conf mongodb.conf stiebeleltron.conf -apache.conf fronius.conf mysql.conf swap.conf +am2320.conf fping.conf mongodb.conf +apache.conf mysql.conf swap.conf ... ``` @@ -140,4 +140,4 @@ Or, get guided insights into specific health configurations with our [health gui Finally, move on to Netdata's [notification system](/health/notifications/README.md) to learn more about how Netdata can let you know when the health of your systems or apps goes awry. -[](<>) + diff --git a/health/README.md b/health/README.md index 37f09e848..2b1caf548 100644 --- a/health/README.md +++ b/health/README.md @@ -35,4 +35,4 @@ content. **[Notifications](/health/notifications/README.md)**: Get notified about ongoing alarms from your Agents via your favorite platform(s), such as Slack, Discord, PagerDuty, email, and much more. -[](<>) + diff --git a/health/REFERENCE.md b/health/REFERENCE.md index 4feb782d6..3c1e53b2a 100644 --- a/health/REFERENCE.md +++ b/health/REFERENCE.md @@ -82,6 +82,7 @@ Netdata parses the following lines. Beneath the table is an in-depth explanation | [`repeat`](#alarm-line-repeat) | no | The interval for sending notifications when an alarm is in WARNING or CRITICAL mode. | | [`options`](#alarm-line-options) | no | Add an option to not clear alarms. | | [`host labels`](#alarm-line-host-labels) | no | List of labels present on a host. | +| [`info`](#alarm-line-info) | no | A brief description of the alarm. | The `alarm` or `template` line must be the first line of any entity. @@ -311,7 +312,7 @@ Everything is the same with [badges](/web/api/badges/README.md). In short: above too). - `OPTIONS` is a space separated list of `percentage`, `absolute`, `min2max`, `unaligned`, - `match-ids`, `match-names`. Check the badges documentation for more info. + `match-ids`, `match-names`. Check the [badges](/web/api/badges/README.md) documentation for more info. - `of DIMENSIONS` is optional and has to be the last parameter. Dimensions have to be separated by `,` or `|`. The space characters found in dimensions will be kept as-is (a few dimensions @@ -533,6 +534,14 @@ host labels: installed = 201* See our [simple patterns docs](/libnetdata/simple_pattern/README.md) for more examples. +#### Alarm line `info` + +The info field can contain a small piece of text describing the alarm or template. This will be rendered in notifications and UI elements whenever the specific alarm is in focus. An example for the `ram_available` alarm is: + +```yaml +info: percentage of estimated amount of RAM available for userspace processes, without causing swapping +``` + ## Expressions Netdata has an internal [infix expression parser](/libnetdata/eval). This parses expressions and creates an internal @@ -913,4 +922,4 @@ to temporary disable notifications (for instance when running backups triggers a notifications are runtime. The health checks can be controlled at runtime via the [health management api](/web/api/health/README.md). -[](<>) + diff --git a/health/health.c b/health/health.c index e94339fae..528238d74 100644 --- a/health/health.c +++ b/health/health.c @@ -223,6 +223,8 @@ void health_reload(void) { if (netdata_cloud_setting) aclk_single_update_disable(); #endif + sql_refresh_hashes(); + rrd_rdlock(); RRDHOST *host; @@ -653,6 +655,41 @@ static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) { return 0; } +// Create alarms for dimensions that have been added to charts +// since the previous iteration. +static void init_pending_foreach_alarms(RRDHOST *host) { + rrdhost_wrlock(host); + + if (host->alarms_with_foreach || host->alarms_template_with_foreach) { + if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS)) { + RRDSET *st; + + rrdset_foreach_read(st, host) { + rrdset_wrlock(st); + + if (rrdset_flag_check(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS)) { + RRDDIM *rd; + + rrddim_foreach_write(rd, st) { + if (rrddim_flag_check(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM)) { + rrdcalc_link_to_rrddim(rd, st, host); + rrddim_flag_clear(rd, RRDDIM_FLAG_PENDING_FOREACH_ALARM); + } + } + + rrdset_flag_clear(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS); + } + + rrdset_unlock(st); + } + + rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS); + } + } + + rrdhost_unlock(host); +} + /** * Health Main * @@ -737,6 +774,8 @@ void *health_main(void *ptr) { if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0)) sql_health_alarm_log_cleanup(host); + init_pending_foreach_alarms(host); + rrdhost_rdlock(host); // the first loop is to lookup values from the db @@ -745,6 +784,33 @@ void *health_main(void *ptr) { if (update_disabled_silenced(host, rc)) continue; + // create an alert removed event if the chart is obsolete and + // has stopped being collected for 60 seconds + if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED && + rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) && + now > (rc->rrdset->last_collected_time.tv_sec + 60))) { + if (!rrdcalc_isrepeating(rc)) { + time_t now = now_realtime_sec(); + ALARM_ENTRY *ae = health_create_alarm_entry( + host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id, + rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change, + rc->value, NAN, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info, 0, 0); + if (ae) { + health_alarm_log(host, ae); + rc->old_status = rc->status; + rc->status = RRDCALC_STATUS_REMOVED; + rc->last_status_change = now; + rc->last_updated = now; + rc->value = NAN; +#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL) + if (netdata_cloud_setting && likely(!aclk_alert_reloaded)) + sql_queue_removed_alerts_to_aclk(host); +#endif + } + } + continue; + } + if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) { if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_RUNNABLE)) rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUNNABLE; @@ -764,7 +830,7 @@ void *health_main(void *ptr) { int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rc->dimensions, 1, rc->after, rc->before, rc->group, 0, rc->options, &rc->db_after, - &rc->db_before, &value_is_null + &rc->db_before, &value_is_null, 0 ); if (unlikely(ret != 200)) { @@ -1004,7 +1070,7 @@ void *health_main(void *ptr) { RRDCALC *rc; for(rc = host->alarms; rc ; rc = rc->next) { int repeat_every = 0; - if(unlikely(rrdcalc_isrepeating(rc))) { + if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) { if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) { rc->rrdcalc_flags &= ~RRDCALC_FLAG_RUN_ONCE; repeat_every = rc->warn_repeat_every; diff --git a/health/health.d/backend.conf b/health/health.d/backend.conf deleted file mode 100644 index 91d469395..000000000 --- a/health/health.d/backend.conf +++ /dev/null @@ -1,42 +0,0 @@ -# Alert that backends subsystem will be disabled soon - alarm: backend_metrics_eol - on: netdata.backend_metrics - class: Errors - type: Netdata -component: Exporting engine - units: boolean - calc: $now - $last_collected_t - every: 1m - warn: $this > 0 - delay: down 5m multiplier 1.5 max 1h - info: the backends subsystem is deprecated and will be removed soon. Migrate your configuration to exporting.conf. - to: sysadmin - -# make sure we are sending data to backend - - alarm: backend_last_buffering - on: netdata.backend_metrics - class: Latency - type: Netdata -component: Exporting engine - calc: $now - $last_collected_t - units: seconds ago - every: 10s - warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) - crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) - delay: down 5m multiplier 1.5 max 1h - info: number of seconds since the last successful buffering of backend data - to: dba - - alarm: backend_metrics_sent - on: netdata.backend_metrics - class: Workload - type: Netdata -component: Exporting engine - units: % - calc: abs($sent) * 100 / abs($buffered) - every: 10s - warn: $this != 100 - delay: down 5m multiplier 1.5 max 1h - info: percentage of metrics sent to the backend server - to: dba diff --git a/health/health.d/cgroups.conf b/health/health.d/cgroups.conf index 45b34806c..aa416c795 100644 --- a/health/health.d/cgroups.conf +++ b/health/health.d/cgroups.conf @@ -32,3 +32,40 @@ component: Memory delay: down 15m multiplier 1.5 max 1h info: cgroup memory utilization to: sysadmin + +# ----------------------------------------------------------------------------- +# check for packet storms + +# 1. calculate the rate packets are received in 1m: 1m_received_packets_rate +# 2. do the same for the last 10s +# 3. raise an alarm if the later is 10x or 20x the first +# we assume the minimum packet storm should at least have +# 10000 packets/s, average of the last 10 seconds + + template: cgroup_1m_received_packets_rate + on: cgroup.net_packets + class: Workload + type: Cgroups +component: Network + hosts: * + lookup: average -1m unaligned of received + units: packets + every: 10s + info: average number of packets received by the network interface $family over the last minute + + template: cgroup_10s_received_packets_storm + on: cgroup.net_packets + class: Workload + type: Cgroups +component: Network + hosts: * + lookup: average -10s unaligned of received + calc: $this * 100 / (($1m_received_packets_rate < 1000)?(1000):($1m_received_packets_rate)) + every: 10s + units: % + warn: $this > (($status >= $WARNING)?(200):(5000)) + crit: $this > (($status == $CRITICAL)?(5000):(6000)) + options: no-clear-notification + info: ratio of average number of received packets for the network interface $family over the last 10 seconds, \ + compared to the rate over the last minute + to: sysadmin diff --git a/health/health.d/fronius.conf b/health/health.d/fronius.conf deleted file mode 100644 index 853bd7fbc..000000000 --- a/health/health.d/fronius.conf +++ /dev/null @@ -1,14 +0,0 @@ - template: fronius_last_collected_secs - families: * - on: fronius.power - class: Latency - type: Power Supply -component: Solar - calc: $now - $last_collected_t - every: 10s - units: seconds ago - warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) - crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) - delay: down 5m multiplier 1.5 max 1h - info: number of seconds since the last successful data collection - to: sitemgr diff --git a/health/health.d/ioping.conf b/health/health.d/ioping.conf index ee4befbea..8b498ad3c 100644 --- a/health/health.d/ioping.conf +++ b/health/health.d/ioping.conf @@ -4,12 +4,12 @@ class: Latency type: System component: Disk - lookup: average -10s unaligned of average - units: ms + lookup: average -10s unaligned of latency + units: microseconds every: 10s - green: 500 - red: 1000 - warn: $this > $green OR $max > $red + green: 5000 + red: 10000 + warn: $this > $green crit: $this > $red delay: down 30m multiplier 1.5 max 2h info: average I/O latency over the last 10 seconds diff --git a/health/health.d/net.conf b/health/health.d/net.conf index 028ca7b81..9d5b3b8d3 100644 --- a/health/health.d/net.conf +++ b/health/health.d/net.conf @@ -68,7 +68,7 @@ component: Network component: Network os: linux hosts: * - families: !net* * + families: * lookup: sum -10m unaligned absolute of inbound units: packets every: 1m @@ -81,7 +81,7 @@ component: Network component: Network os: linux hosts: * - families: !net* * + families: * lookup: sum -10m unaligned absolute of outbound units: packets every: 1m @@ -94,7 +94,7 @@ component: Network component: Network os: linux hosts: * - families: !net* !wl* * + families: !wl* * lookup: sum -10m unaligned absolute of received calc: (($inbound_packets_dropped != nan AND $this > 10000) ? ($inbound_packets_dropped * 100 / $this) : (0)) units: % @@ -111,7 +111,7 @@ component: Network component: Network os: linux hosts: * - families: !net* !wl* * + families: !wl* * lookup: sum -10m unaligned absolute of sent calc: (($outbound_packets_dropped != nan AND $this > 1000) ? ($outbound_packets_dropped * 100 / $this) : (0)) units: % diff --git a/health/health.d/nut.conf b/health/health.d/nut.conf new file mode 100644 index 000000000..6231dd97b --- /dev/null +++ b/health/health.d/nut.conf @@ -0,0 +1,47 @@ +# you can disable an alarm notification by setting the 'to' line to: silent + + template: nut_10min_ups_load + on: nut.load + class: Utilization + type: Power Supply +component: UPS + os: * + hosts: * + lookup: average -10m unaligned of load + units: % + every: 1m + warn: $this > (($status >= $WARNING) ? (70) : (80)) + crit: $this > (($status == $CRITICAL) ? (85) : (95)) + delay: down 10m multiplier 1.5 max 1h + info: average UPS load over the last 10 minutes + to: sitemgr + + template: nut_ups_charge + on: nut.charge + class: Errors + type: Power Supply +component: UPS + os: * + hosts: * + lookup: average -60s unaligned of battery_charge + units: % + every: 60s + warn: $this < 100 + crit: $this < (($status == $CRITICAL) ? (60) : (50)) + delay: down 10m multiplier 1.5 max 1h + info: average UPS charge over the last minute + to: sitemgr + + template: nut_last_collected_secs + on: nut.load + class: Latency + type: Power Supply +component: UPS device + calc: $now - $last_collected_t + every: 10s + units: seconds ago + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + delay: down 5m multiplier 1.5 max 1h + info: number of seconds since the last successful data collection + to: sitemgr diff --git a/health/health.d/stiebeleltron.conf b/health/health.d/stiebeleltron.conf deleted file mode 100644 index 493c8b73a..000000000 --- a/health/health.d/stiebeleltron.conf +++ /dev/null @@ -1,14 +0,0 @@ - template: stiebeleltron_last_collected_secs - families: * - on: stiebeleltron.heating.hc1 - class: Latency - type: Other -component: Sensors - calc: $now - $last_collected_t - every: 10s - units: seconds ago - warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) - crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) - delay: down 5m multiplier 1.5 max 1h - info: number of seconds since the last successful data collection - to: sitemgr diff --git a/health/health.d/tcp_resets.conf b/health/health.d/tcp_resets.conf index 190271e47..35cb6366c 100644 --- a/health/health.d/tcp_resets.conf +++ b/health/health.d/tcp_resets.conf @@ -26,7 +26,7 @@ component: Network lookup: average -10s unaligned absolute of OutRsts units: tcp resets/s every: 10s - warn: $this > ((($1m_ipv4_tcp_resets_sent < 5)?(5):($1m_ipv4_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (20))) + warn: $this > ((($1m_ipv4_tcp_resets_sent < 5)?(5):($1m_ipv4_tcp_resets_sent)) * (($status >= $WARNING) ? (1) : (10))) delay: up 20s down 60m multiplier 1.2 max 2h options: no-clear-notification info: average number of sent TCP RESETS over the last 10 seconds. \ diff --git a/health/health.d/timex.conf b/health/health.d/timex.conf index ea90c4000..23c18ba10 100644 --- a/health/health.d/timex.conf +++ b/health/health.d/timex.conf @@ -13,5 +13,5 @@ component: Clock every: 10s warn: $system.uptime.uptime > 17 * 60 AND $this == 0 delay: down 5m - info: the system time is not synchronized to a reliable server + info: when set to 0, the system kernel believes the system clock is not properly synchronized to a reliable server to: silent diff --git a/health/health.h b/health/health.h index 323279bac..f25ae6bc6 100644 --- a/health/health.h +++ b/health/health.h @@ -87,6 +87,7 @@ extern void *health_cmdapi_thread(void *ptr); extern void health_label_log_save(RRDHOST *host); extern char *health_edit_command_from_source(const char *source); +extern void sql_refresh_hashes(void); extern SIMPLE_PATTERN *health_pattern_from_foreach(char *s); diff --git a/health/health_config.c b/health/health_config.c index da71f13d8..e1f5f0e31 100644 --- a/health/health_config.c +++ b/health/health_config.c @@ -538,6 +538,7 @@ static inline void alert_config_free(struct alert_config *cfg) freez(cfg); } +int sql_store_hashes = 1; static int health_readfile(const char *filename, void *data) { RRDHOST *host = (RRDHOST *)data; @@ -662,17 +663,15 @@ static int health_readfile(const char *filename, void *data) { if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) { if(rc) { - if(ignore_this || !alert_hash_and_store_config(rc->config_hash_id, alert_cfg) || !rrdcalc_add_alarm_from_config(host, rc)) { + if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) { rrdcalc_free(rc); - alert_config_free(alert_cfg); } // health_add_alarms_loop(host, rc, ignore_this) ; } if(rt) { - if (ignore_this || !alert_hash_and_store_config(rt->config_hash_id, alert_cfg) || !rrdcalctemplate_add_template_from_config(host, rt)) { + if (!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) { rrdcalctemplate_free(rt); - alert_config_free(alert_cfg); } rt = NULL; } @@ -690,6 +689,8 @@ static int health_readfile(const char *filename, void *data) { rc->old_status = RRDCALC_STATUS_UNINITIALIZED; rc->warn_repeat_every = host->health_default_warn_repeat_every; rc->crit_repeat_every = host->health_default_crit_repeat_every; + if (alert_cfg) + alert_config_free(alert_cfg); alert_cfg = callocz(1, sizeof(struct alert_config)); if(rrdvar_fix_name(rc->name)) @@ -701,18 +702,16 @@ static int health_readfile(const char *filename, void *data) { else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) { if(rc) { // health_add_alarms_loop(host, rc, ignore_this) ; - if(ignore_this || !alert_hash_and_store_config(rc->config_hash_id, alert_cfg) || !rrdcalc_add_alarm_from_config(host, rc)) { + if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) { rrdcalc_free(rc); - alert_config_free(alert_cfg); } rc = NULL; } if(rt) { - if(ignore_this || !alert_hash_and_store_config(rt->config_hash_id, alert_cfg) || !rrdcalctemplate_add_template_from_config(host, rt)) { + if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) { rrdcalctemplate_free(rt); - alert_config_free(alert_cfg); } } @@ -725,6 +724,8 @@ static int health_readfile(const char *filename, void *data) { rt->delay_multiplier = 1.0; rt->warn_repeat_every = host->health_default_warn_repeat_every; rt->crit_repeat_every = host->health_default_crit_repeat_every; + if (alert_cfg) + alert_config_free(alert_cfg); alert_cfg = callocz(1, sizeof(struct alert_config)); if(rrdvar_fix_name(rt->name)) @@ -1225,13 +1226,13 @@ static int health_readfile(const char *filename, void *data) { if(rc) { //health_add_alarms_loop(host, rc, ignore_this) ; - if(ignore_this || !alert_hash_and_store_config(rc->config_hash_id, alert_cfg) || !rrdcalc_add_alarm_from_config(host, rc)) { + if(!alert_hash_and_store_config(rc->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalc_add_alarm_from_config(host, rc)) { rrdcalc_free(rc); } } if(rt) { - if(ignore_this || !alert_hash_and_store_config(rt->config_hash_id, alert_cfg) || !rrdcalctemplate_add_template_from_config(host, rt)) { + if(!alert_hash_and_store_config(rt->config_hash_id, alert_cfg, sql_store_hashes) || ignore_this || !rrdcalctemplate_add_template_from_config(host, rt)) { rrdcalctemplate_free(rt); } } @@ -1243,6 +1244,11 @@ static int health_readfile(const char *filename, void *data) { return 1; } +void sql_refresh_hashes(void) +{ + sql_store_hashes = 1; +} + void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath) { if(unlikely(!host->health_enabled)) { debug(D_HEALTH, "CONFIG health is not enabled for host '%s'", host->hostname); @@ -1258,4 +1264,5 @@ void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path } recursive_config_double_dir_load(user_path, stock_path, subpath, health_readfile, (void *) host, 0); + sql_store_hashes = 0; } diff --git a/health/notifications/README.md b/health/notifications/README.md index 53541981d..0bd6c7649 100644 --- a/health/notifications/README.md +++ b/health/notifications/README.md @@ -83,4 +83,4 @@ bash -x /usr/libexec/netdata/plugins.d/alarm-notify.sh test ``` Then look in the output for the alarm-notify.sh calls and run the one you want to trace with `bash -x`. -[](<>) + diff --git a/health/notifications/alerta/README.md b/health/notifications/alerta/README.md index 85a0cd488..9603aae01 100644 --- a/health/notifications/alerta/README.md +++ b/health/notifications/alerta/README.md @@ -78,4 +78,4 @@ We can test alarms using the standard approach: For more information see the [Alerta documentation](https://docs.alerta.io) -[](<>) + diff --git a/health/notifications/awssns/README.md b/health/notifications/awssns/README.md index c68251379..fc4a665e9 100644 --- a/health/notifications/awssns/README.md +++ b/health/notifications/awssns/README.md @@ -6,7 +6,8 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notificat # Amazon SNS -As part of it's AWS suite, Amazon provides a notification broker service called 'Simple Notification Service' or SNS. Amazon SNS works kind of similarly to Netdata's own notification system, allowing dispatch of a single notification to multiple subscribers of different types. Among other things, SNS supports sending notifications to: +As part of its AWS suite, Amazon provides a notification broker service called 'Simple Notification Service' (SNS). Amazon SNS works similarly to Netdata's own notification system, allowing to dispatch a single notification to multiple subscribers of different types. While Amazon SNS supports sending differently formatted messages for different delivery methods, Netdata does not currently support this functionality. +Among other things, SNS supports sending notifications to: - Email addresses. - Mobile Phones via SMS. @@ -15,29 +16,38 @@ As part of it's AWS suite, Amazon provides a notification broker service called - AWS SQS queues. - Mobile applications via push notifications. -To get this working, you will need: +For email notification support, we recommend using Netdata's email notifications, as it is has the following benefits: -- The Amazon Web Services CLI tools. Most distributions provide these with the package name `awscli`. -- An actual home directory for the user you run Netdata as, instead of just using `/` as a home directory. Setup of this is distribution specific. `/var/lib/netdata` is the recommended directory (because the permissions will already be correct) if you are using a dedicated user (which is how most distributions work). -- An Amazon SNS topic to send notifications to with one or more subscribers. The [Getting - Started](https://docs.aws.amazon.com/sns/latest/dg/sns-getting-started.html) section of the Amazon SNS documentation - covers the basics of how to set this up. Make note of the Topic ARN when you create the topic. -- While not mandatory, it is highly recommended to create a dedicated IAM user on your account for Netdata to send notifications. This user needs to have programmatic access, and should only allow access to SNS. If you're really paranoid, you can create one for each system or group of systems. +- In most cases, it requires less configuration. +- Netdata's emails are nicely pre-formatted and support features like threading, which requires a lot of manual effort in SNS. +- It is less resource intensive and more cost-efficient than SNS. -Once you have all the above, run the following command as the user Netdata runs under: +Read on to learn how to set up Amazon SNS in Netdata. -``` -aws configure -``` +## Prerequisites -THis will prompt you for the access key and secret key for accessing Amazon SNS (as well as the default region and output format, but you can leave those blank because we don't use them). +Before you can enable SNS, you need: -Once that's done, you're ready to go and can specify the desired topic ARN as a recipient. +- The [Amazon Web Services CLI tools](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) (`awscli`). +- An actual home directory for the user you run Netdata as, instead of just using `/` as a home directory. The setup depends on the distribution, but `/var/lib/netdata` is the recommended directory. If you are using Netdata as a dedicated user, the permissions will already be correct. +- An Amazon SNS topic to send notifications to with one or more subscribers. The [Getting Started](https://docs.aws.amazon.com/sns/latest/dg/sns-getting-started.html) section of the Amazon SNS documentation covers the basics of how to set this up. Make note of the **Topic ARN** when you create the topic. +- While not mandatory, it is highly recommended to create a dedicated IAM user on your account for Netdata to send notifications. This user needs to have programmatic access, and should only allow access to SNS. For an additional layer of security, you can create one for each system or group of systems. -Notes: +## Enabling Amazon SNS -- Netdata's native email notification support is far better in almost all respects than it's support through Amazon SNS. If you want email notifications, use the native support, not SNS. - - If you need to change the notification format for SNS notifications, you can do so by specifying the format in `AWSSNS_MESSAGE_FORMAT` in the configuration. This variable supports all the same variables you can use in custom notifications. - - While Amazon SNS supports sending differently formatted messages for different delivery methods, Netdata does not currently support this functionality. +To enable SNS: +1. Run the following command as the user Netdata runs under: + ``` + aws configure + ``` +2. Enter the access key and secret key for accessing Amazon SNS. The system also prompts you to enter the default region and output format, but you can leave those blank because Netdata doesn't use them. -[](<>) +3. Specify the desired topic ARN as a recipient, see [SNS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/US_SetupSNS.html#set-up-sns-topic-cli). +4. Optional: To change the notification format for SNS notifications, change the `AWSSNS_MESSAGE_FORMAT` variable in `health_alarm_notify.conf`. +This variable supports all the same variables you can use in custom notifications. + + The default format looks like this: + ```bash + AWSSNS_MESSAGE_FORMAT="${status} on ${host} at ${date}: ${chart} ${value_string}" + ``` + diff --git a/health/notifications/custom/README.md b/health/notifications/custom/README.md index bcb09ef53..edc42623d 100644 --- a/health/notifications/custom/README.md +++ b/health/notifications/custom/README.md @@ -89,4 +89,4 @@ The following are more human friendly: - `${severity}` like "Escalated to CRITICAL", "Recovered from WARNING" - `${raised_for}` like "(alarm was raised for 10 minutes)" -[](<>) + diff --git a/health/notifications/discord/README.md b/health/notifications/discord/README.md index 1650d9cec..568d03bc3 100644 --- a/health/notifications/discord/README.md +++ b/health/notifications/discord/README.md @@ -47,4 +47,4 @@ role_recipients_discord[webmaster]="marketing development" The keywords `systems`, `databases`, `marketing`, `development` are discordapp.com channels (they should already exist within your discord server). -[](<>) + diff --git a/health/notifications/dynatrace/README.md b/health/notifications/dynatrace/README.md index cc82ee78d..3f8ad85b6 100644 --- a/health/notifications/dynatrace/README.md +++ b/health/notifications/dynatrace/README.md @@ -32,5 +32,3 @@ The default value is `CUSTOM_INFO`. This is a required parameter. 6. Specify the annotation type. This is the source of the Dynatrace event. Put whatever it fits you, for example, _Netdata Alarm_, which is also the default value. - -[]() diff --git a/health/notifications/email/README.md b/health/notifications/email/README.md index 82786fca1..3dc84dd40 100644 --- a/health/notifications/email/README.md +++ b/health/notifications/email/README.md @@ -74,4 +74,4 @@ sendmail="/usr/bin/msmtp" chmod 600 ~/.msmtprc ``` -[](<>) + diff --git a/health/notifications/flock/README.md b/health/notifications/flock/README.md index b24ecdb25..b9e0025b3 100644 --- a/health/notifications/flock/README.md +++ b/health/notifications/flock/README.md @@ -34,4 +34,4 @@ FLOCK_WEBHOOK_URL="https://api.flock.com/hooks/sendMessage/XXXXXXXXXXXXXXXXXXXXX DEFAULT_RECIPIENT_FLOCK="alarms" ``` -[](<>) + diff --git a/health/notifications/hangouts/README.md b/health/notifications/hangouts/README.md index 886abfc43..7554b39cd 100644 --- a/health/notifications/hangouts/README.md +++ b/health/notifications/hangouts/README.md @@ -52,4 +52,4 @@ You can define multiple rooms like this: `sysadmin devops alarms|critical`. The keywords `sysadmin`, `devops`, and `alarms` are Hangouts rooms. -[](<>) + diff --git a/health/notifications/irc/README.md b/health/notifications/irc/README.md index e7f22e1fd..21c998d11 100644 --- a/health/notifications/irc/README.md +++ b/health/notifications/irc/README.md @@ -75,4 +75,4 @@ role_recipients_irc[webmaster]="#networking-alarms" The keywords `#user-alarms`, `#networking-alarms`, `#system-alarms`, `#databases-alarms` are irc channels which belong to the specified IRC network. -[](<>) + diff --git a/health/notifications/kavenegar/README.md b/health/notifications/kavenegar/README.md index b59799fc2..6123eb901 100644 --- a/health/notifications/kavenegar/README.md +++ b/health/notifications/kavenegar/README.md @@ -43,4 +43,4 @@ KAVENEGAR_SENDER="" DEFAULT_RECIPIENT_KAVENEGAR="" ``` -[](<>) + diff --git a/health/notifications/matrix/README.md b/health/notifications/matrix/README.md index ea22b4a8a..8eeecf55d 100644 --- a/health/notifications/matrix/README.md +++ b/health/notifications/matrix/README.md @@ -55,4 +55,4 @@ MATRIX_ACCESSTOKEN="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" DEFAULT_RECIPIENT_MATRIX="!XXXXXXXXXXXX:matrix.org" ``` -[](<>) + diff --git a/health/notifications/messagebird/README.md b/health/notifications/messagebird/README.md index 8e3d1a5b3..f70e86c68 100644 --- a/health/notifications/messagebird/README.md +++ b/health/notifications/messagebird/README.md @@ -42,4 +42,4 @@ MESSAGEBIRD_NUMBER="XXXXXXX" DEFAULT_RECIPIENT_MESSAGEBIRD="XXXXXXX" ``` -[](<>) + diff --git a/health/notifications/msteams/README.md b/health/notifications/msteams/README.md index 3ff5de68d..14dbe7511 100644 --- a/health/notifications/msteams/README.md +++ b/health/notifications/msteams/README.md @@ -42,4 +42,4 @@ role_recipients_msteams[dba]="YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY" role_recipients_msteams[webmaster]="ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" ``` -[](<>) + diff --git a/health/notifications/opsgenie/README.md b/health/notifications/opsgenie/README.md index 10b9f5242..640fcd42a 100644 --- a/health/notifications/opsgenie/README.md +++ b/health/notifications/opsgenie/README.md @@ -59,4 +59,4 @@ message: You can find more details about the Opsgenie error codes in their [response docs](https://docs.opsgenie.com/docs/response). -[](<>) + diff --git a/health/notifications/prowl/README.md b/health/notifications/prowl/README.md index 7c60de270..dc136820c 100644 --- a/health/notifications/prowl/README.md +++ b/health/notifications/prowl/README.md @@ -25,5 +25,3 @@ the Prowl website after registering. Once you have an API key, simply specify that as a recipient for Prowl notifications. - -[]() diff --git a/health/notifications/pushbullet/README.md b/health/notifications/pushbullet/README.md index 7c2f6774e..194050bc1 100644 --- a/health/notifications/pushbullet/README.md +++ b/health/notifications/pushbullet/README.md @@ -47,4 +47,4 @@ PUSHBULLET_ACCESS_TOKEN="o.Sometokenhere" DEFAULT_RECIPIENT_PUSHBULLET="admin1@example.com admin3@somemail.com #examplechanneltag #anotherchanneltag" ``` -[](<>) + diff --git a/health/notifications/pushover/README.md b/health/notifications/pushover/README.md index 3ba97fbc3..1e50f7140 100644 --- a/health/notifications/pushover/README.md +++ b/health/notifications/pushover/README.md @@ -20,4 +20,4 @@ pushover.net notifications look like this:  -[](<>) + diff --git a/health/notifications/rocketchat/README.md b/health/notifications/rocketchat/README.md index a54f5826c..96d6160b2 100644 --- a/health/notifications/rocketchat/README.md +++ b/health/notifications/rocketchat/README.md @@ -49,4 +49,4 @@ role_recipients_rocketchat[webmaster]="marketing development" The keywords `systems`, `databases`, `marketing`, `development` are RocketChat channels (they should already exist). Both public and private channels can be used, even if they differ from the channel configured in your RocketChat incoming webhook. -[](<>) + diff --git a/health/notifications/slack/README.md b/health/notifications/slack/README.md index e338e9af0..ad36ce34a 100644 --- a/health/notifications/slack/README.md +++ b/health/notifications/slack/README.md @@ -47,4 +47,4 @@ role_recipients_slack[dba]="databases systems" role_recipients_slack[webmaster]="marketing development" ``` -[](<>) + diff --git a/health/notifications/smstools3/README.md b/health/notifications/smstools3/README.md index 6d90e702a..6618dfa18 100644 --- a/health/notifications/smstools3/README.md +++ b/health/notifications/smstools3/README.md @@ -41,4 +41,4 @@ Netdata uses the script `sendsms` that is installed by `smstools3` and just pass sendsms="" ``` -[](<>) + diff --git a/health/notifications/stackpulse/README.md b/health/notifications/stackpulse/README.md index 4c44954ab..c478fd584 100644 --- a/health/notifications/stackpulse/README.md +++ b/health/notifications/stackpulse/README.md @@ -78,4 +78,4 @@ access the **TRIGGER** tab to define the scenarios used to trigger the event. Th For more details how to create a scenario, take a look at the [StackPulse documentation](https://docs.stackpulse.io). -[](<>) + diff --git a/health/notifications/syslog/README.md b/health/notifications/syslog/README.md index 360f6844d..8b7863a1a 100644 --- a/health/notifications/syslog/README.md +++ b/health/notifications/syslog/README.md @@ -31,4 +31,4 @@ The `facility` and `level` are the standard syslog facility and level options, f You can configure sending directly to remote log servers by specifying a host (and optionally a port). However, this has a somewhat high overhead, so it is much preferred to use your local syslog daemon to handle the forwarding of messages to remote systems (pretty much all of them allow at least simple forwarding, and most of the really popular ones support complex queueing and routing of messages to remote log servers). -[](<>) + diff --git a/health/notifications/telegram/README.md b/health/notifications/telegram/README.md index c1c6f2a4e..2a2ed5623 100644 --- a/health/notifications/telegram/README.md +++ b/health/notifications/telegram/README.md @@ -42,4 +42,4 @@ Telegram messages look like this:  -[](<>) + diff --git a/health/notifications/twilio/README.md b/health/notifications/twilio/README.md index b36d40b99..b563c66c1 100644 --- a/health/notifications/twilio/README.md +++ b/health/notifications/twilio/README.md @@ -44,4 +44,4 @@ TWILIO_NUMBER="xxxxxxxxxxx" DEFAULT_RECIPIENT_TWILIO="+15555555555" ``` -[](<>) + diff --git a/health/notifications/web/README.md b/health/notifications/web/README.md index 9e4918603..185843af5 100644 --- a/health/notifications/web/README.md +++ b/health/notifications/web/README.md @@ -10,4 +10,4 @@ The Netdata dashboard shows HTML notifications, when it is open. Such web notifications look like this:  -[](<>) + |