diff options
Diffstat (limited to 'conf.d/health.d')
-rw-r--r-- | conf.d/health.d/backend.conf | 2 | ||||
-rw-r--r-- | conf.d/health.d/btrfs.conf | 57 | ||||
-rw-r--r-- | conf.d/health.d/ceph.conf | 13 | ||||
-rw-r--r-- | conf.d/health.d/cpu.conf | 14 | ||||
-rw-r--r-- | conf.d/health.d/disks.conf | 51 | ||||
-rw-r--r-- | conf.d/health.d/fronius.conf | 11 | ||||
-rw-r--r-- | conf.d/health.d/httpcheck.conf | 99 | ||||
-rw-r--r-- | conf.d/health.d/isc_dhcpd.conf | 14 | ||||
-rw-r--r-- | conf.d/health.d/net.conf | 5 | ||||
-rw-r--r-- | conf.d/health.d/nginx_plus.conf | 14 | ||||
-rw-r--r-- | conf.d/health.d/portcheck.conf | 48 | ||||
-rw-r--r-- | conf.d/health.d/ram.conf | 42 | ||||
-rw-r--r-- | conf.d/health.d/softnet.conf | 14 | ||||
-rw-r--r-- | conf.d/health.d/stiebeleltron.conf | 11 | ||||
-rw-r--r-- | conf.d/health.d/swap.conf | 8 | ||||
-rw-r--r-- | conf.d/health.d/tcp_resets.conf | 6 | ||||
-rw-r--r-- | conf.d/health.d/udp_errors.conf | 4 | ||||
-rw-r--r-- | conf.d/health.d/web_log.conf | 1 |
18 files changed, 387 insertions, 27 deletions
diff --git a/conf.d/health.d/backend.conf b/conf.d/health.d/backend.conf index 9c193e7b9..7af100d8f 100644 --- a/conf.d/health.d/backend.conf +++ b/conf.d/health.d/backend.conf @@ -27,7 +27,7 @@ units: metrics calc: abs($lost) every: 10s - crit: $this != 0 + crit: ($this != 0) || ($status == $CRITICAL && abs($sent) == 0) delay: down 5m multiplier 1.5 max 1h info: number of metrics lost due to repeating failures to contact the backend server to: dba diff --git a/conf.d/health.d/btrfs.conf b/conf.d/health.d/btrfs.conf new file mode 100644 index 000000000..b27aa544f --- /dev/null +++ b/conf.d/health.d/btrfs.conf @@ -0,0 +1,57 @@ + +template: btrfs_allocated + on: btrfs.disk + os: * + hosts: * +families: * + calc: 100 - ($unallocated * 100 / ($unallocated + $data_used + $data_free + $meta_used + $meta_free + $sys_used + $sys_free)) + units: % + every: 10s + warn: $this > (($status >= $WARNING) ? (90) : (95)) + crit: $this > (($status == $CRITICAL) ? (95) : (98)) + delay: up 1m down 15m multiplier 1.5 max 1h + info: the percentage of allocated BTRFS physical disk space + to: sysadmin + +template: btrfs_data + on: btrfs.data + os: * + hosts: * +families: * + calc: $used * 100 / ($used + $free) + units: % + every: 10s + warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98 + crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98 + delay: up 1m down 15m multiplier 1.5 max 1h + info: the percentage of used BTRFS data space + to: sysadmin + +template: btrfs_metadata + on: btrfs.metadata + os: * + hosts: * +families: * + calc: ($used + $reserved) * 100 / ($used + $free + $reserved) + units: % + every: 10s + warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98 + crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98 + delay: up 1m down 15m multiplier 1.5 max 1h + info: the percentage of used BTRFS metadata space + to: sysadmin + +template: btrfs_system + on: btrfs.system + os: * + hosts: * +families: * + calc: $used * 100 / ($used + $free) + units: % + every: 10s + warn: $this > (($status >= $WARNING) ? (90) : (95)) && $btrfs_allocated > 98 + crit: $this > (($status == $CRITICAL) ? (95) : (98)) && $btrfs_allocated > 98 + delay: up 1m down 15m multiplier 1.5 max 1h + info: the percentage of used BTRFS system space + to: sysadmin + diff --git a/conf.d/health.d/ceph.conf b/conf.d/health.d/ceph.conf new file mode 100644 index 000000000..de16f7b6f --- /dev/null +++ b/conf.d/health.d/ceph.conf @@ -0,0 +1,13 @@ +# low ceph disk available + +template: cluster_space_usage + on: ceph.general_usage + calc: $avail * 100 / ($avail + $used) + units: % + every: 10s + warn: $this < 10 + crit: $this < 1 + delay: down 5m multiplier 1.2 max 1h + info: ceph disk usage is almost full + to: sysadmin + diff --git a/conf.d/health.d/cpu.conf b/conf.d/health.d/cpu.conf index db6285561..fa8189856 100644 --- a/conf.d/health.d/cpu.conf +++ b/conf.d/health.d/cpu.conf @@ -39,3 +39,17 @@ template: 20min_steal_cpu delay: down 1h multiplier 1.5 max 2h info: average CPU steal time for the last 20 minutes to: sysadmin + +## FreeBSD +template: 10min_cpu_usage + on: system.cpu + os: freebsd + hosts: * + lookup: average -10m unaligned of user,system,interrupt + units: % + every: 1m + warn: $this > (($status >= $WARNING) ? (75) : (85)) + crit: $this > (($status == $CRITICAL) ? (85) : (95)) + delay: down 15m multiplier 1.5 max 1h + info: average cpu utilization for the last 10 minutes (excluding nice) + to: sysadmin diff --git a/conf.d/health.d/disks.conf b/conf.d/health.d/disks.conf index 63053491e..26f85848a 100644 --- a/conf.d/health.d/disks.conf +++ b/conf.d/health.d/disks.conf @@ -11,7 +11,7 @@ template: disk_space_usage on: disk.space - os: linux + os: linux freebsd hosts: * families: * calc: $used * 100 / ($avail + $used) @@ -25,7 +25,7 @@ families: * template: disk_inode_usage on: disk.inodes - os: linux + os: linux freebsd hosts: * families: * calc: $used * 100 / ($avail + $used) @@ -51,7 +51,7 @@ families: * template: disk_fill_rate on: disk.space - os: linux + os: linux freebsd hosts: * families: * lookup: min -10m at -50m unaligned of avail @@ -67,7 +67,7 @@ families: * template: out_of_disk_space_time on: disk.space - os: linux + os: linux freebsd hosts: * families: * calc: ($disk_fill_rate > 0) ? ($avail / $disk_fill_rate) : (inf) @@ -81,6 +81,47 @@ families: * # ----------------------------------------------------------------------------- +# disk inode fill rate + +# calculate the rate the disk inodes are allocated +# use as base, the available inodes change +# during the last hour + +# this is just a calculation - it has no alarm +# we will use it in the next template to find +# the hours remaining + +template: disk_inode_rate + on: disk.inodes + os: linux freebsd + hosts: * +families: * + lookup: min -10m at -50m unaligned of avail + calc: ($this - $avail) / (($now - $after) / 3600) + every: 1m + units: inodes/hour + info: average rate at which disk inodes are allocated (positive), or freed (negative), for the last hour + +# calculate the hours remaining +# if the disk inodes are allocated +# in this rate + +template: out_of_disk_inodes_time + on: disk.inodes + os: linux freebsd + hosts: * +families: * + calc: ($disk_inode_rate > 0) ? ($avail / $disk_inode_rate) : (inf) + units: hours + every: 10s + warn: $this > 0 and $this < (($status >= $WARNING) ? (48) : (8)) + crit: $this > 0 and $this < (($status == $CRITICAL) ? (24) : (2)) + delay: down 15m multiplier 1.2 max 1h + info: estimated time the disk will run out of inodes, if the system continues to allocate inodes with the rate of the last hour + to: sysadmin + + +# ----------------------------------------------------------------------------- # disk congestion # raise an alarm if the disk is congested @@ -89,7 +130,7 @@ families: * template: 10min_disk_utilization on: disk.util - os: linux + os: linux freebsd hosts: * families: * lookup: average -10m unaligned diff --git a/conf.d/health.d/fronius.conf b/conf.d/health.d/fronius.conf new file mode 100644 index 000000000..cdf6c8fcb --- /dev/null +++ b/conf.d/health.d/fronius.conf @@ -0,0 +1,11 @@ +template: fronius_last_collected_secs +families: * + on: fronius.power + calc: $now - $last_collected_t + every: 10s + units: seconds ago + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + delay: down 5m multiplier 1.5 max 1h + info: number of seconds since the last successful data collection + to: sitemgr diff --git a/conf.d/health.d/httpcheck.conf b/conf.d/health.d/httpcheck.conf new file mode 100644 index 000000000..0ddf35eab --- /dev/null +++ b/conf.d/health.d/httpcheck.conf @@ -0,0 +1,99 @@ +template: httpcheck_last_collected_secs +families: * + on: httpcheck.status + calc: $now - $last_collected_t + every: 10s + units: seconds ago + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + delay: down 5m multiplier 1.5 max 1h + info: number of seconds since the last successful data collection + to: sysadmin + +# This is a fast-reacting no-notification alarm ideal for custom dashboards or badges +template: web_service_up +families: * + on: httpcheck.status + lookup: average -1m unaligned percentage of success + calc: ($this < 75) ? (0) : ($this) + every: 5s + units: up/down + info: at least 75% verified responses during last 60 seconds, ideal for badges + to: silent + +template: web_service_bad_content +families: * + on: httpcheck.status + lookup: average -5m unaligned percentage of bad_content + every: 10s + units: % + warn: $this >= 10 AND $this < 40 + crit: $this >= 40 + delay: down 5m multiplier 1.5 max 1h + info: average of unexpected http response content during the last 5 minutes + options: no-clear-notification + to: webmaster + +template: web_service_bad_status +families: * + on: httpcheck.status + lookup: average -5m unaligned percentage of bad_status + every: 10s + units: % + warn: $this >= 10 AND $this < 40 + crit: $this >= 40 + delay: down 5m multiplier 1.5 max 1h + info: average of unexpected http status during the last 5 minutes + options: no-clear-notification + to: webmaster + +template: web_service_timeouts +families: * + on: httpcheck.status + lookup: average -5m unaligned percentage of timeout + every: 10s + units: % + info: average of timeouts during the last 5 minutes + +template: no_web_service_connections +families: * + on: httpcheck.status + lookup: average -5m unaligned percentage of no_connection + every: 10s + units: % + info: average of failed requests during the last 5 minutes + +# combined timeout & no connection alarm +template: web_service_unreachable +families: * + on: httpcheck.status + calc: ($no_web_service_connections >= $web_service_timeouts) ? ($no_web_service_connections) : ($web_service_timeouts) + units: % + every: 10s + warn: ($no_web_service_connections >= 10 OR $web_service_timeouts >= 10) AND ($no_web_service_connections < 40 OR $web_service_timeouts < 40) + crit: $no_web_service_connections >= 40 OR $web_service_timeouts >= 40 + delay: down 5m multiplier 1.5 max 1h + info: average of failed requests either due to timeouts or no connection during the last 5 minutes + options: no-clear-notification + to: webmaster + +template: 1h_web_service_response_time +families: * + on: httpcheck.responsetime + lookup: average -1h unaligned of time + every: 30s + units: ms + info: average response time over the last hour + +template: web_service_slow +families: * + on: httpcheck.responsetime + lookup: average -3m unaligned of time + units: ms + every: 10s + warn: ($this > ($1h_web_service_response_time * 2) ) + crit: ($this > ($1h_web_service_response_time * 3) ) + info: average response time over the last 3 minutes, compared to the average over the last hour + delay: down 5m multiplier 1.5 max 1h + options: no-clear-notification + to: webmaster diff --git a/conf.d/health.d/isc_dhcpd.conf b/conf.d/health.d/isc_dhcpd.conf index 4345619aa..8054656ff 100644 --- a/conf.d/health.d/isc_dhcpd.conf +++ b/conf.d/health.d/isc_dhcpd.conf @@ -1,10 +1,10 @@ - alarm: isc_dhcpd_parse_time - on: isc_dhcpd.parse_time - units: ms + template: isc_dhcpd_leases_size + on: isc_dhcpd.leases_total + units: KB every: 60 - calc: $ptime - warn: $this > 100 - crit: $this > 250 + calc: $leases_size + warn: $this > 3072 + crit: $this > 6144 delay: up 2m down 5m - info: Parsing too slow! It can slow down your server. Check dhcpd.leases file size. + info: dhcpd.leases file too big! Module can slow down your server. to: sysadmin diff --git a/conf.d/health.d/net.conf b/conf.d/health.d/net.conf index 00a198612..22a88927d 100644 --- a/conf.d/health.d/net.conf +++ b/conf.d/health.d/net.conf @@ -98,7 +98,7 @@ families: * template: 1m_received_packets_rate on: net.packets - os: linux + os: linux freebsd hosts: * families: * lookup: average -1m of received @@ -108,7 +108,7 @@ families: * template: 10s_received_packets_storm on: net.packets - os: linux + os: linux freebsd hosts: * families: * lookup: average -10s of received @@ -120,4 +120,3 @@ families: * options: no-clear-notification info: the % of the rate of received packets in the last 10 seconds, compared to the rate of the last minute (clear notification for this alarm will not be sent) to: sysadmin - diff --git a/conf.d/health.d/nginx_plus.conf b/conf.d/health.d/nginx_plus.conf new file mode 100644 index 000000000..5a171a76d --- /dev/null +++ b/conf.d/health.d/nginx_plus.conf @@ -0,0 +1,14 @@ + +# make sure nginx_plus is running + +template: nginx_plus_last_collected_secs + on: nginx_plus.requests_total + calc: $now - $last_collected_t + units: seconds ago + every: 10s + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + delay: down 5m multiplier 1.5 max 1h + info: number of seconds since the last successful data collection + to: webmaster + diff --git a/conf.d/health.d/portcheck.conf b/conf.d/health.d/portcheck.conf new file mode 100644 index 000000000..f42b63d30 --- /dev/null +++ b/conf.d/health.d/portcheck.conf @@ -0,0 +1,48 @@ +template: portcheck_last_collected_secs +families: * + on: portcheck.status + calc: $now - $last_collected_t + every: 10s + units: seconds ago + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + delay: down 5m multiplier 1.5 max 1h + info: number of seconds since the last successful data collection + to: sysadmin + +# This is a fast-reacting no-notification alarm ideal for custom dashboards or badges +template: service_reachable +families: * + on: portcheck.status + lookup: average -1m unaligned percentage of success + calc: ($this < 75) ? (0) : ($this) + every: 5s + units: up/down + info: at least 75% successful connections during last 60 seconds, ideal for badges + to: silent + +template: connection_timeouts +families: * + on: portcheck.status + lookup: average -5m unaligned percentage of timeout + every: 10s + units: % + warn: $this >= 10 AND $this < 40 + crit: $this >= 40 + delay: down 5m multiplier 1.5 max 1h + info: average of timeouts during the last 5 minutes + options: no-clear-notification + to: sysadmin + +template: connection_fails +families: * + on: portcheck.status + lookup: average -5m unaligned percentage of no_connection + every: 10s + units: % + warn: $this >= 10 AND $this < 40 + crit: $this >= 40 + delay: down 5m multiplier 1.5 max 1h + info: average of failed connections during the last 5 minutes + options: no-clear-notification + to: sysadmin diff --git a/conf.d/health.d/ram.conf b/conf.d/health.d/ram.conf index 8d0e8838d..b6dc5f945 100644 --- a/conf.d/health.d/ram.conf +++ b/conf.d/health.d/ram.conf @@ -20,5 +20,45 @@ warn: $this > (($status >= $WARNING) ? (80) : (90)) crit: $this > (($status == $CRITICAL) ? (90) : (98)) delay: down 15m multiplier 1.5 max 1h - info: system RAM usage + info: system RAM used to: sysadmin + + alarm: ram_available + on: mem.available + os: linux + hosts: * + calc: ($avail + $used_ram_to_ignore) * 100 / ($system.ram.used + $system.ram.cached + $system.ram.free + $system.ram.buffers) + units: % + every: 10s + warn: $this < (($status >= $WARNING) ? ( 5) : (10)) + crit: $this < (($status == $CRITICAL) ? (10) : ( 5)) + delay: down 15m multiplier 1.5 max 1h + info: estimated amount of RAM available for userspace processes, without causing swapping + to: sysadmin + +## FreeBSD +alarm: ram_in_use + on: system.ram + os: freebsd +hosts: * + calc: (($active + $wired) - $used_ram_to_ignore) * 100 / (($active + $wired) - $used_ram_to_ignore + $cached + $free) +units: % +every: 10s + warn: $this > (($status >= $WARNING) ? (80) : (90)) + crit: $this > (($status == $CRITICAL) ? (90) : (98)) +delay: down 15m multiplier 1.5 max 1h + info: system RAM usage + to: sysadmin + + alarm: ram_available + on: system.ram + os: freebsd + hosts: * + calc: ($free + $inactive + $used_ram_to_ignore) * 100 / ($free + $active + $inactive + $wired + $cache + $buffers) + units: % + every: 10s + warn: $this < (($status >= $WARNING) ? ( 5) : (10)) + crit: $this < (($status == $CRITICAL) ? (10) : ( 5)) + delay: down 15m multiplier 1.5 max 1h + info: estimated amount of RAM available for userspace processes, without causing swapping + to: sysadmin diff --git a/conf.d/health.d/softnet.conf b/conf.d/health.d/softnet.conf index 64e1c6784..77c804bfd 100644 --- a/conf.d/health.d/softnet.conf +++ b/conf.d/health.d/softnet.conf @@ -24,5 +24,17 @@ every: 1m warn: $this > (($status >= $WARNING) ? (0) : (10)) delay: down 1h multiplier 1.5 max 2h - info: number of times, during the last 10min, ksoftirq ran out of sysctl net.core.netdev_budget or time slice, with work remaining (this can be a cause for dropped packets) + info: number of times, during the last 10min, ksoftirq ran out of sysctl net.core.netdev_budget or net.core.netdev_budget_usecs, with work remaining (this can be a cause for dropped packets) to: silent + + alarm: 10min_netisr_backlog_exceeded + on: system.softnet_stat + os: freebsd + hosts: * + lookup: sum -10m unaligned absolute of qdrops + units: packets + every: 1m + warn: $this > 0 + delay: down 1h multiplier 1.5 max 2h + info: number of drops in the last 10min, because sysctl net.route.netisr_maxqlen was exceeded (this can be a cause for dropped packets) + to: sysadmin diff --git a/conf.d/health.d/stiebeleltron.conf b/conf.d/health.d/stiebeleltron.conf new file mode 100644 index 000000000..e0361eb20 --- /dev/null +++ b/conf.d/health.d/stiebeleltron.conf @@ -0,0 +1,11 @@ +template: stiebeleltron_last_collected_secs +families: * + on: stiebeleltron.heating.hc1 + calc: $now - $last_collected_t + every: 10s + units: seconds ago + warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) + crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) + delay: down 5m multiplier 1.5 max 1h + info: number of seconds since the last successful data collection + to: sitemgr diff --git a/conf.d/health.d/swap.conf b/conf.d/health.d/swap.conf index 830a9af95..f920b0807 100644 --- a/conf.d/health.d/swap.conf +++ b/conf.d/health.d/swap.conf @@ -3,7 +3,7 @@ alarm: 30min_ram_swapped_out on: system.swapio - os: linux + os: linux freebsd hosts: * lookup: sum -30m unaligned absolute of out # we have to convert KB to MB by dividing $this (i.e. the result of the lookup) with 1024 @@ -25,19 +25,19 @@ every: 10s warn: $this > (($status >= $WARNING) ? (15) : (20)) crit: $this > (($status == $CRITICAL) ? (40) : (50)) - delay: up 0 down 15m multiplier 1.5 max 1h + delay: up 30s down 15m multiplier 1.5 max 1h info: the swap memory used, as a percentage of the system RAM to: sysadmin alarm: used_swap on: system.swap - os: linux + os: linux freebsd hosts: * calc: $used * 100 / ( $used + $free ) units: % every: 10s warn: $this > (($status >= $WARNING) ? (80) : (90)) crit: $this > (($status == $CRITICAL) ? (90) : (98)) - delay: up 0 down 15m multiplier 1.5 max 1h + delay: up 30s down 15m multiplier 1.5 max 1h info: the percentage of swap memory used to: sysadmin diff --git a/conf.d/health.d/tcp_resets.conf b/conf.d/health.d/tcp_resets.conf index e6cfd39ab..91dad3c6a 100644 --- a/conf.d/health.d/tcp_resets.conf +++ b/conf.d/health.d/tcp_resets.conf @@ -5,7 +5,7 @@ alarm: ipv4_tcphandshake_last_collected_secs on: ipv4.tcphandshake - os: linux + os: linux freebsd hosts: * calc: $now - $last_collected_t units: seconds ago @@ -46,7 +46,7 @@ alarm: 1m_ipv4_tcp_resets_received on: ipv4.tcphandshake - os: linux + os: linux freebsd hosts: * lookup: average -1m at -10s unaligned absolute of AttemptFails units: tcp resets/s @@ -55,7 +55,7 @@ alarm: 10s_ipv4_tcp_resets_received on: ipv4.tcphandshake - os: linux + os: linux freebsd hosts: * lookup: average -10s unaligned absolute of AttemptFails units: tcp resets/s diff --git a/conf.d/health.d/udp_errors.conf b/conf.d/health.d/udp_errors.conf index 33338b83e..382b39658 100644 --- a/conf.d/health.d/udp_errors.conf +++ b/conf.d/health.d/udp_errors.conf @@ -5,7 +5,7 @@ alarm: ipv4_udperrors_last_collected_secs on: ipv4.udperrors - os: linux + os: linux freebsd hosts: * calc: $now - $last_collected_t units: seconds ago @@ -21,7 +21,7 @@ alarm: 1m_ipv4_udp_receive_buffer_errors on: ipv4.udperrors - os: linux + os: linux freebsd hosts: * lookup: sum -1m unaligned absolute of RcvbufErrors units: errors diff --git a/conf.d/health.d/web_log.conf b/conf.d/health.d/web_log.conf index d18088172..d8be88b47 100644 --- a/conf.d/health.d/web_log.conf +++ b/conf.d/health.d/web_log.conf @@ -116,6 +116,7 @@ families: * crit: ($1m_requests > 120) ? ($this > $red && $this > ($10m_response_time * 4) ) : ( 0 ) delay: down 15m multiplier 1.5 max 1h info: the average time to respond to HTTP requests, over the last 1 minute + options: no-clear-notification to: webmaster # ----------------------------------------------------------------------------- |