summaryrefslogtreecommitdiffstats
path: root/web/api/health
diff options
context:
space:
mode:
Diffstat (limited to 'web/api/health')
-rw-r--r--web/api/health/Makefile.am8
-rw-r--r--web/api/health/README.md225
-rw-r--r--web/api/health/health_cmdapi.c206
-rw-r--r--web/api/health/health_cmdapi.h31
4 files changed, 470 insertions, 0 deletions
diff --git a/web/api/health/Makefile.am b/web/api/health/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/web/api/health/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/web/api/health/README.md b/web/api/health/README.md
new file mode 100644
index 0000000..9ec8f31
--- /dev/null
+++ b/web/api/health/README.md
@@ -0,0 +1,225 @@
+<!--
+title: "Health API Calls"
+date: 2020-04-27
+custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/health/README.md
+-->
+
+# Health API Calls
+
+## Health Read API
+
+### Enabled Alarms
+
+Netdata enables alarms on demand, i.e. when the chart they should be linked to starts collecting data. So, although many
+more alarms are configured, only the useful ones are enabled.
+
+To get the list of all enabled alarms, open your browser and navigate to `http://NODE:19999/api/v1/alarms?all`,
+replacing `NODE` with the IP address or hostname for your Agent dashboard.
+
+### Raised Alarms
+
+This API call will return the alarms currently in WARNING or CRITICAL state.
+
+`http://NODE:19999/api/v1/alarms`
+
+### Event Log
+
+The size of the alarm log is configured in `netdata.conf`. There are 2 settings: the rotation of the alarm log file and the in memory size of the alarm log.
+
+```
+[health]
+ in memory max health log entries = 1000
+ rotate log every lines = 2000
+```
+
+The API call retrieves all entries of the alarm log:
+
+`http://NODE:19999/api/v1/alarm_log`
+
+### Alarm Log Incremental Updates
+
+`http://NODE:19999/api/v1/alarm_log?after=UNIQUEID`
+
+The above returns all the events in the alarm log that occurred after UNIQUEID (you poll it once without `after=`, remember the last UNIQUEID of the returned set, which you give back to get incrementally the next events).
+
+### Alarm badges
+
+The following will return an SVG badge of the alarm named `NAME`, attached to the chart named `CHART`.
+
+`http://NODE:19999/api/v1/badge.svg?alarm=NAME&chart=CHART`
+
+## Health Management API
+
+Netdata v1.12 and beyond provides a command API to control health checks and notifications at runtime. The feature is especially useful for maintenance periods, during which you receive meaningless alarms.
+From Netdata v1.16.0 and beyond, the configuration controlled via the API commands is [persisted across Netdata restarts](#persistence).
+
+Specifically, the API allows you to:
+
+- Disable health checks completely. Alarm conditions will not be evaluated at all and no entries will be added to the alarm log.
+- Silence alarm notifications. Alarm conditions will be evaluated, the alarms will appear in the log and the Netdata UI will show the alarms as active, but no notifications will be sent.
+- Disable or Silence specific alarms that match selectors on alarm/template name, chart, context, host and family.
+
+The API is available by default, but it is protected by an `api authorization token` that is stored in the file you will see in the following entry of `http://NODE:19999/netdata.conf`:
+
+```
+[registry]
+ # netdata management api key file = /var/lib/netdata/netdata.api.key
+```
+
+You can access the API via GET requests, by adding the bearer token to an `Authorization` http header, like this:
+
+```
+curl "http://NODE:19999/api/v1/manage/health?cmd=RESET" -H "X-Auth-Token: Mytoken"
+```
+
+By default access to the health management API is only allowed from `localhost`. Accessing the API from anything else will return a 403 error with the message `You are not allowed to access this resource.`. You can change permissions by editing the `allow management from` variable in `netdata.conf` within the [web] section. See [web server access lists](/web/server/README.md#access-lists) for more information.
+
+The command `RESET` just returns Netdata to the default operation, with all health checks and notifications enabled.
+If you've configured and entered your token correctly, you should see the plain text response `All health checks and notifications are enabled`.
+
+### Disable or silence all alarms
+
+If all you need is temporarily disable all health checks, then you issue the following before your maintenance period starts:
+
+```sh
+curl "http://NODE:19999/api/v1/manage/health?cmd=DISABLE ALL" -H "X-Auth-Token: Mytoken"
+```
+
+The effect of disabling health checks is that the alarm criteria are not evaluated at all and nothing is written in the alarm log.
+If you want the health checks to be running but to not receive any notifications during your maintenance period, you can instead use this:
+
+```sh
+curl "http://NODE:19999/api/v1/manage/health?cmd=SILENCE ALL" -H "X-Auth-Token: Mytoken"
+```
+
+Alarms may then still be raised and logged in Netdata, so you'll be able to see them via the UI.
+
+Regardless of the option you choose, at the end of your maintenance period you revert to the normal state via the RESET command.
+
+```sh
+ curl "http://NODE:19999/api/v1/manage/health?cmd=RESET" -H "X-Auth-Token: Mytoken"
+```
+
+### Disable or silence specific alarms
+
+If you do not wish to disable/silence all alarms, then the `DISABLE ALL` and `SILENCE ALL` commands can't be used.
+Instead, the following commands expect that one or more alarm selectors will be added, so that only alarms that match the selectors are disabled or silenced.
+
+- `DISABLE` : Set the mode to disable health checks.
+- `SILENCE` : Set the mode to silence notifications.
+
+You will normally put one of these commands in the same request with your first alarm selector, but it's possible to issue them separately as well.
+You will get a warning in the response, if a selector was added without a SILENCE/DISABLE command, or vice versa.
+
+Each request can specify a single alarm `selector`, with one or more `selection criteria`.
+A single alarm will match a `selector` if all selection criteria match the alarm.
+You can add as many selectors as you like.
+In essence, the rule is: IF (alarm matches all the criteria in selector1 OR all the criteria in selector2 OR ...) THEN apply the DISABLE or SILENCE command.
+
+To clear all selectors and reset the mode to default, use the `RESET` command.
+
+The following example silences notifications for all the alarms with context=load:
+
+```
+curl "http://NODE:19999/api/v1/manage/health?cmd=SILENCE&context=load" -H "X-Auth-Token: Mytoken"
+```
+
+#### Selection criteria
+
+The `selection criteria` are key/value pairs, in the format `key : value`, where value is a Netdata [simple pattern](/libnetdata/simple_pattern/README.md). This means that you can create very powerful selectors (you will rarely need more than one or two).
+
+The accepted keys for the `selection criteria` are the following:
+
+- `alarm` : The expression provided will match both `alarm` and `template` names.
+- `chart` : Chart ids/names, as shown on the dashboard. These will match the `on` entry of a configured `alarm`.
+- `context` : Chart context, as shown on the dashboard. These will match the `on` entry of a configured `template`.
+- `hosts` : The hostnames that will need to match.
+- `families` : The alarm families.
+
+You can add any of the selection criteria you need on the request, to ensure that only the alarms you are interested in are matched and disabled/silenced. e.g. there is no reason to add `hosts: *`, if you want the criteria to be applied to alarms for all hosts.
+
+Example 1: Disable all health checks for context = `random`
+
+```
+http://NODE:19999/api/v1/manage/health?cmd=DISABLE&context=random
+```
+
+Example 2: Silence all alarms and templates with name starting with `out_of` on host `myhost`
+
+```
+http://NODE:19999/api/v1/manage/health?cmd=SILENCE&alarm=out_of*&hosts=myhost
+```
+
+Example 2.2: Add one more selector, to also silence alarms for cpu1 and cpu2
+
+```
+http://NODE:19999/api/v1/manage/health?families=cpu1 cpu2
+```
+
+### List silencers
+
+The command `LIST` was added in Netdata v1.16.0 and returns a JSON with the current status of the silencers.
+
+```
+ curl "http://NODE:19999/api/v1/manage/health?cmd=LIST" -H "X-Auth-Token: Mytoken"
+```
+
+As an example, the following response shows that we have two silencers configured, one for an alarm called `samplealarm` and one for alarms with context `random` on host `myhost`
+
+```
+json
+{
+ "all": false,
+ "type": "SILENCE",
+ "silencers": [
+ {
+ "alarm": "samplealarm"
+ },
+ {
+ "context": "random",
+ "hosts": "myhost"
+ }
+ ]
+}
+```
+
+The response below shows that we have disabled all health checks.
+
+```
+json
+{
+ "all": true,
+ "type": "DISABLE",
+ "silencers": []
+}
+```
+
+### Responses
+
+- "Auth Error" : Token authentication failed
+- "All alarm notifications are silenced" : Successful response to cmd=SILENCE ALL
+- "All health checks are disabled" : Successful response to cmd=DISABLE ALL
+- "All health checks and notifications are enabled" : Successful response to cmd=RESET
+- "Health checks disabled for alarms matching the selectors" : Added to the response for a cmd=DISABLE
+- "Alarm notifications silenced for alarms matching the selectors" : Added to the response for a cmd=SILENCE
+- "Alarm selector added" : Added to the response when a new selector is added
+- "Invalid key. Ignoring it." : Wrong name of a parameter. Added to the response and ignored.
+- "WARNING: Added alarm selector to silence/disable alarms without a SILENCE or DISABLE command." : Added to the response if a selector is added without a selector-specific command.
+- "WARNING: SILENCE or DISABLE command is ineffective without defining any alarm selectors." : Added to the response if a selector-specific command is issued without a selector.
+
+### Persistence
+
+From Netdata v1.16.0 and beyond, the silencers configuration is persisted to disk and loaded when Netdata starts.
+The JSON string returned by the [LIST command](#list-silencers) is automatically saved to the `silencers file`, every time a command alters the silencers configuration.
+The file's location is configurable in `netdata.conf`. The default is shown below:
+
+```
+[health]
+ # silencers file = /var/lib/netdata/health.silencers.json
+```
+
+### Further reading
+
+The test script under [tests/health_mgmtapi](/tests/health_mgmtapi/README.md) contains a series of tests that you can either run or read through to understand the various calls and responses better.
+
+
diff --git a/web/api/health/health_cmdapi.c b/web/api/health/health_cmdapi.c
new file mode 100644
index 0000000..bad3e96
--- /dev/null
+++ b/web/api/health/health_cmdapi.c
@@ -0,0 +1,206 @@
+//
+// Created by Christopher on 11/12/18.
+//
+
+#include "health_cmdapi.h"
+
+/**
+ * Free Silencers
+ *
+ * Clean the silencer structure
+ *
+ * @param t is the structure that will be cleaned.
+ */
+void free_silencers(SILENCER *t) {
+ if (!t) return;
+ if (t->next) free_silencers(t->next);
+ debug(D_HEALTH, "HEALTH command API: Freeing silencer %s:%s:%s:%s:%s", t->alarms,
+ t->charts, t->contexts, t->hosts, t->families);
+ simple_pattern_free(t->alarms_pattern);
+ simple_pattern_free(t->charts_pattern);
+ simple_pattern_free(t->contexts_pattern);
+ simple_pattern_free(t->hosts_pattern);
+ simple_pattern_free(t->families_pattern);
+ freez(t->alarms);
+ freez(t->charts);
+ freez(t->contexts);
+ freez(t->hosts);
+ freez(t->families);
+ freez(t);
+ return;
+}
+
+/**
+ * Silencers to JSON Entry
+ *
+ * Fill the buffer with the other values given.
+ *
+ * @param wb a pointer to the output buffer
+ * @param var the json variable
+ * @param val the json value
+ * @param hasprev has it a previous value?
+ *
+ * @return
+ */
+int health_silencers2json_entry(BUFFER *wb, char* var, char* val, int hasprev) {
+ if (val) {
+ buffer_sprintf(wb, "%s\n\t\t\t\"%s\": \"%s\"", (hasprev)?",":"", var, val);
+ return 1;
+ } else {
+ return hasprev;
+ }
+}
+
+/**
+ * Silencer to JSON
+ *
+ * Write the silencer values using JSON format inside a buffer.
+ *
+ * @param wb is the buffer to write the silencers.
+ */
+void health_silencers2json(BUFFER *wb) {
+ buffer_sprintf(wb, "{\n\t\"all\": %s,"
+ "\n\t\"type\": \"%s\","
+ "\n\t\"silencers\": [",
+ (silencers->all_alarms)?"true":"false",
+ (silencers->stype == STYPE_NONE)?"None":((silencers->stype == STYPE_DISABLE_ALARMS)?"DISABLE":"SILENCE"));
+
+ SILENCER *silencer;
+ int i = 0, j = 0;
+ for(silencer = silencers->silencers; silencer ; silencer = silencer->next) {
+ if(likely(i)) buffer_strcat(wb, ",");
+ buffer_strcat(wb, "\n\t\t{");
+ j=health_silencers2json_entry(wb, HEALTH_ALARM_KEY, silencer->alarms, j);
+ j=health_silencers2json_entry(wb, HEALTH_CHART_KEY, silencer->charts, j);
+ j=health_silencers2json_entry(wb, HEALTH_CONTEXT_KEY, silencer->contexts, j);
+ j=health_silencers2json_entry(wb, HEALTH_HOST_KEY, silencer->hosts, j);
+ health_silencers2json_entry(wb, HEALTH_FAMILIES_KEY, silencer->families, j);
+ j=0;
+ buffer_strcat(wb, "\n\t\t}");
+ i++;
+ }
+ if(likely(i)) buffer_strcat(wb, "\n\t");
+ buffer_strcat(wb, "]\n}\n");
+}
+
+/**
+ * Silencer to FILE
+ *
+ * Write the silencer buffer to a file.
+ * @param wb
+ */
+void health_silencers2file(BUFFER *wb) {
+ if (wb->len == 0) return;
+
+ FILE *fd = fopen(silencers_filename, "wb");
+ if(fd) {
+ size_t written = (size_t)fprintf(fd, "%s", wb->buffer) ;
+ if (written == wb->len ) {
+ info("Silencer changes written to %s", silencers_filename);
+ }
+ fclose(fd);
+ return;
+ }
+ error("Silencer changes could not be written to %s. Error %s", silencers_filename, strerror(errno));
+}
+
+/**
+ * Request V1 MGMT Health
+ *
+ * Function called by api to management the health.
+ *
+ * @param host main structure with client information!
+ * @param w is the structure with all information of the client request.
+ * @param url is the url that netdata is working
+ *
+ * @return It returns 200 on success and another code otherwise.
+ */
+int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, char *url) {
+ int ret;
+ (void) host;
+
+ BUFFER *wb = w->response.data;
+ buffer_flush(wb);
+ wb->contenttype = CT_TEXT_PLAIN;
+
+ buffer_flush(w->response.data);
+
+ //Local instance of the silencer
+ SILENCER *silencer = NULL;
+ int config_changed = 1;
+
+ if (!w->auth_bearer_token) {
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_AUTHERROR);
+ ret = HTTP_RESP_FORBIDDEN;
+ } else {
+ debug(D_HEALTH, "HEALTH command API: Comparing secret '%s' to '%s'", w->auth_bearer_token, api_secret);
+ if (strcmp(w->auth_bearer_token, api_secret)) {
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_AUTHERROR);
+ ret = HTTP_RESP_FORBIDDEN;
+ } else {
+ while (url) {
+ char *value = mystrsep(&url, "&");
+ if (!value || !*value) continue;
+
+ char *key = mystrsep(&value, "=");
+ if (!key || !*key) continue;
+ if (!value || !*value) continue;
+
+ debug(D_WEB_CLIENT, "%llu: API v1 health query param '%s' with value '%s'", w->id, key, value);
+
+ // name and value are now the parameters
+ if (!strcmp(key, "cmd")) {
+ if (!strcmp(value, HEALTH_CMDAPI_CMD_SILENCEALL)) {
+ silencers->all_alarms = 1;
+ silencers->stype = STYPE_SILENCE_NOTIFICATIONS;
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_SILENCEALL);
+ } else if (!strcmp(value, HEALTH_CMDAPI_CMD_DISABLEALL)) {
+ silencers->all_alarms = 1;
+ silencers->stype = STYPE_DISABLE_ALARMS;
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_DISABLEALL);
+ } else if (!strcmp(value, HEALTH_CMDAPI_CMD_SILENCE)) {
+ silencers->stype = STYPE_SILENCE_NOTIFICATIONS;
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_SILENCE);
+ } else if (!strcmp(value, HEALTH_CMDAPI_CMD_DISABLE)) {
+ silencers->stype = STYPE_DISABLE_ALARMS;
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_DISABLE);
+ } else if (!strcmp(value, HEALTH_CMDAPI_CMD_RESET)) {
+ silencers->all_alarms = 0;
+ silencers->stype = STYPE_NONE;
+ free_silencers(silencers->silencers);
+ silencers->silencers = NULL;
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_RESET);
+ } else if (!strcmp(value, HEALTH_CMDAPI_CMD_LIST)) {
+ w->response.data->contenttype = CT_APPLICATION_JSON;
+ health_silencers2json(wb);
+ config_changed=0;
+ }
+ } else {
+ silencer = health_silencers_addparam(silencer, key, value);
+ }
+ }
+
+ if (likely(silencer)) {
+ health_silencers_add(silencer);
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_ADDED);
+ if (silencers->stype == STYPE_NONE) {
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_STYPEWARNING);
+ }
+ }
+ if (unlikely(silencers->stype != STYPE_NONE && !silencers->all_alarms && !silencers->silencers)) {
+ buffer_strcat(wb, HEALTH_CMDAPI_MSG_NOSELECTORWARNING);
+ }
+ ret = HTTP_RESP_OK;
+ }
+ }
+ w->response.data = wb;
+ buffer_no_cacheable(w->response.data);
+ if (ret == HTTP_RESP_OK && config_changed) {
+ BUFFER *jsonb = buffer_create(200);
+ health_silencers2json(jsonb);
+ health_silencers2file(jsonb);
+ buffer_free(jsonb);
+ }
+
+ return ret;
+}
diff --git a/web/api/health/health_cmdapi.h b/web/api/health/health_cmdapi.h
new file mode 100644
index 0000000..d5309c7
--- /dev/null
+++ b/web/api/health/health_cmdapi.h
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_WEB_HEALTH_SVG_H
+#define NETDATA_WEB_HEALTH_SVG_H 1
+
+#include "libnetdata/libnetdata.h"
+#include "web/server/web_client.h"
+#include "health/health.h"
+
+#define HEALTH_CMDAPI_CMD_SILENCEALL "SILENCE ALL"
+#define HEALTH_CMDAPI_CMD_DISABLEALL "DISABLE ALL"
+#define HEALTH_CMDAPI_CMD_SILENCE "SILENCE"
+#define HEALTH_CMDAPI_CMD_DISABLE "DISABLE"
+#define HEALTH_CMDAPI_CMD_RESET "RESET"
+#define HEALTH_CMDAPI_CMD_LIST "LIST"
+
+#define HEALTH_CMDAPI_MSG_AUTHERROR "Auth Error\n"
+#define HEALTH_CMDAPI_MSG_SILENCEALL "All alarm notifications are silenced\n"
+#define HEALTH_CMDAPI_MSG_DISABLEALL "All health checks are disabled\n"
+#define HEALTH_CMDAPI_MSG_RESET "All health checks and notifications are enabled\n"
+#define HEALTH_CMDAPI_MSG_DISABLE "Health checks disabled for alarms matching the selectors\n"
+#define HEALTH_CMDAPI_MSG_SILENCE "Alarm notifications silenced for alarms matching the selectors\n"
+#define HEALTH_CMDAPI_MSG_ADDED "Alarm selector added\n"
+#define HEALTH_CMDAPI_MSG_STYPEWARNING "WARNING: Added alarm selector to silence/disable alarms without a SILENCE or DISABLE command.\n"
+#define HEALTH_CMDAPI_MSG_NOSELECTORWARNING "WARNING: SILENCE or DISABLE command is ineffective without defining any alarm selectors.\n"
+
+int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, char *url);
+
+#include "web/api/web_api_v1.h"
+
+#endif /* NETDATA_WEB_HEALTH_SVG_H */
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-04 16:19:34 +0000