Adding upstream version 1.29.3.upstream/1.29.3upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

10 files changed, 441 insertions, 0 deletions

diff --git a/web/api/exporters/Makefile.am b/web/api/exporters/Makefile.am
new file mode 100644
index 0000000..06fda51
--- /dev/null
+++ b/web/api/exporters/Makefile.am
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+SUBDIRS = \
+    shell \
+    prometheus \
+    $(NULL)
+
+dist_noinst_DATA = \
+    README.md \
+    $(NULL)
diff --git a/web/api/exporters/README.md b/web/api/exporters/README.md
new file mode 100644
index 0000000..4019647
--- /dev/null
+++ b/web/api/exporters/README.md
@@ -0,0 +1,10 @@
+<!--
+title: "Exporters"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/exporters/README.md
+-->
+
+# Exporters
+
+TBD
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fweb%2Fapi%2Fexporters%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/web/api/exporters/allmetrics.c b/web/api/exporters/allmetrics.c
new file mode 100644
index 0000000..d10de3d
--- /dev/null
+++ b/web/api/exporters/allmetrics.c
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "allmetrics.h"
+
+struct prometheus_output_options {
+    char *name;
+    PROMETHEUS_OUTPUT_OPTIONS flag;
+} prometheus_output_flags_root[] = {
+    { "help",       PROMETHEUS_OUTPUT_HELP       },
+    { "types",      PROMETHEUS_OUTPUT_TYPES      },
+    { "names",      PROMETHEUS_OUTPUT_NAMES      },
+    { "timestamps", PROMETHEUS_OUTPUT_TIMESTAMPS },
+    { "variables",  PROMETHEUS_OUTPUT_VARIABLES  },
+    { "oldunits",   PROMETHEUS_OUTPUT_OLDUNITS   },
+    { "hideunits",  PROMETHEUS_OUTPUT_HIDEUNITS  },
+    // terminator
+    { NULL, PROMETHEUS_OUTPUT_NONE },
+};
+
+inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) {
+    int format = ALLMETRICS_SHELL;
+    const char *prometheus_server = w->client_ip;
+
+    uint32_t prometheus_exporting_options;
+    if (prometheus_exporter_instance)
+        prometheus_exporting_options = prometheus_exporter_instance->config.options;
+    else
+        prometheus_exporting_options = global_backend_options;
+
+    PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options =
+        PROMETHEUS_OUTPUT_TIMESTAMPS |
+        ((prometheus_exporting_options & BACKEND_OPTION_SEND_NAMES) ? PROMETHEUS_OUTPUT_NAMES : 0);
+
+    const char *prometheus_prefix;
+    if (prometheus_exporter_instance)
+        prometheus_prefix = prometheus_exporter_instance->config.prefix;
+    else
+        prometheus_prefix = global_backend_prefix;
+
+    while(url) {
+        char *value = mystrsep(&url, "&");
+        if (!value || !*value) continue;
+
+        char *name = mystrsep(&value, "=");
+        if(!name || !*name) continue;
+        if(!value || !*value) continue;
+
+        if(!strcmp(name, "format")) {
+            if(!strcmp(value, ALLMETRICS_FORMAT_SHELL))
+                format = ALLMETRICS_SHELL;
+            else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS))
+                format = ALLMETRICS_PROMETHEUS;
+            else if(!strcmp(value, ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS))
+                format = ALLMETRICS_PROMETHEUS_ALL_HOSTS;
+            else if(!strcmp(value, ALLMETRICS_FORMAT_JSON))
+                format = ALLMETRICS_JSON;
+            else
+                format = 0;
+        }
+        else if(!strcmp(name, "server")) {
+            prometheus_server = value;
+        }
+        else if(!strcmp(name, "prefix")) {
+            prometheus_prefix = value;
+        }
+        else if(!strcmp(name, "data") || !strcmp(name, "source") || !strcmp(name, "data source") || !strcmp(name, "data-source") || !strcmp(name, "data_source") || !strcmp(name, "datasource")) {
+            prometheus_exporting_options = backend_parse_data_source(value, prometheus_exporting_options);
+        }
+        else {
+            int i;
+            for(i = 0; prometheus_output_flags_root[i].name ; i++) {
+                if(!strcmp(name, prometheus_output_flags_root[i].name)) {
+                    if(!strcmp(value, "yes") || !strcmp(value, "1") || !strcmp(value, "true"))
+                        prometheus_output_options |= prometheus_output_flags_root[i].flag;
+                    else
+                        prometheus_output_options &= ~prometheus_output_flags_root[i].flag;
+
+                    break;
+                }
+            }
+        }
+    }
+
+    buffer_flush(w->response.data);
+    buffer_no_cacheable(w->response.data);
+
+    switch(format) {
+        case ALLMETRICS_JSON:
+            w->response.data->contenttype = CT_APPLICATION_JSON;
+            rrd_stats_api_v1_charts_allmetrics_json(host, w->response.data);
+            return HTTP_RESP_OK;
+
+        case ALLMETRICS_SHELL:
+            w->response.data->contenttype = CT_TEXT_PLAIN;
+            rrd_stats_api_v1_charts_allmetrics_shell(host, w->response.data);
+            return HTTP_RESP_OK;
+
+        case ALLMETRICS_PROMETHEUS:
+            w->response.data->contenttype = CT_PROMETHEUS;
+            rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
+                    host
+                    , w->response.data
+                    , prometheus_server
+                    , prometheus_prefix
+                    , prometheus_exporting_options
+                    , prometheus_output_options
+            );
+            return HTTP_RESP_OK;
+
+        case ALLMETRICS_PROMETHEUS_ALL_HOSTS:
+            w->response.data->contenttype = CT_PROMETHEUS;
+            rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
+                    host
+                    , w->response.data
+                    , prometheus_server
+                    , prometheus_prefix
+                    , prometheus_exporting_options
+                    , prometheus_output_options
+            );
+            return HTTP_RESP_OK;
+
+        default:
+            w->response.data->contenttype = CT_TEXT_PLAIN;
+            buffer_strcat(w->response.data, "Which format? '" ALLMETRICS_FORMAT_SHELL "', '" ALLMETRICS_FORMAT_PROMETHEUS "', '" ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS "' and '" ALLMETRICS_FORMAT_JSON "' are currently supported.");
+            return HTTP_RESP_BAD_REQUEST;
+    }
+}
diff --git a/web/api/exporters/allmetrics.h b/web/api/exporters/allmetrics.h
new file mode 100644
index 0000000..f076ff0
--- /dev/null
+++ b/web/api/exporters/allmetrics.h
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_ALLMETRICS_H
+#define NETDATA_API_ALLMETRICS_H
+
+#include "web/api/formatters/rrd2json.h"
+#include "shell/allmetrics_shell.h"
+#include "web/server/web_client.h"
+
+extern int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url);
+
+#endif //NETDATA_API_ALLMETRICS_H
diff --git a/web/api/exporters/prometheus/Makefile.am b/web/api/exporters/prometheus/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/web/api/exporters/prometheus/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+    README.md \
+    $(NULL)
diff --git a/web/api/exporters/prometheus/README.md b/web/api/exporters/prometheus/README.md
new file mode 100644
index 0000000..d26c6e4
--- /dev/null
+++ b/web/api/exporters/prometheus/README.md
@@ -0,0 +1,10 @@
+<!--
+title: "prometheus exporter"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/exporters/prometheus/README.md
+-->
+
+# prometheus exporter
+
+The prometheus exporter for Netdata is located at the [backends section for prometheus](/backends/prometheus/README.md).
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fweb%2Fapi%2Fexporters%2Fprometheus%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/web/api/exporters/shell/Makefile.am b/web/api/exporters/shell/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/web/api/exporters/shell/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+    README.md \
+    $(NULL)
diff --git a/web/api/exporters/shell/README.md b/web/api/exporters/shell/README.md
new file mode 100644
index 0000000..b919045
--- /dev/null
+++ b/web/api/exporters/shell/README.md
@@ -0,0 +1,71 @@
+<!--
+title: "shell exporter"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/exporters/shell/README.md
+-->
+
+# shell exporter
+
+Shell scripts can now query Netdata:
+
+```sh
+eval "$(curl -s 'http://localhost:19999/api/v1/allmetrics')"
+```
+
+after this command, all the Netdata metrics are exposed to shell. Check:
+
+```sh
+# source the metrics
+eval "$(curl -s 'http://localhost:19999/api/v1/allmetrics')"
+
+# let's see if there are variables exposed by Netdata for system.cpu
+set | grep "^NETDATA_SYSTEM_CPU"
+
+NETDATA_SYSTEM_CPU_GUEST=0
+NETDATA_SYSTEM_CPU_GUEST_NICE=0
+NETDATA_SYSTEM_CPU_IDLE=95
+NETDATA_SYSTEM_CPU_IOWAIT=0
+NETDATA_SYSTEM_CPU_IRQ=0
+NETDATA_SYSTEM_CPU_NICE=0
+NETDATA_SYSTEM_CPU_SOFTIRQ=0
+NETDATA_SYSTEM_CPU_STEAL=0
+NETDATA_SYSTEM_CPU_SYSTEM=1
+NETDATA_SYSTEM_CPU_USER=4
+NETDATA_SYSTEM_CPU_VISIBLETOTAL=5
+
+# let's see the total cpu utilization of the system
+echo ${NETDATA_SYSTEM_CPU_VISIBLETOTAL}
+5
+
+# what about alarms?
+set | grep "^NETDATA_ALARM_SYSTEM_SWAP_"
+NETDATA_ALARM_SYSTEM_SWAP_RAM_IN_SWAP_STATUS=CRITICAL
+NETDATA_ALARM_SYSTEM_SWAP_RAM_IN_SWAP_VALUE=53
+NETDATA_ALARM_SYSTEM_SWAP_USED_SWAP_STATUS=CLEAR
+NETDATA_ALARM_SYSTEM_SWAP_USED_SWAP_VALUE=51
+
+# let's get the current status of the alarm 'ram in swap'
+echo ${NETDATA_ALARM_SYSTEM_SWAP_RAM_IN_SWAP_STATUS}
+CRITICAL
+
+# is it fast?
+time curl -s 'http://localhost:19999/api/v1/allmetrics' >/dev/null
+
+real  0m0,070s
+user  0m0,000s
+sys   0m0,007s
+
+# it is...
+# 0.07 seconds for curl to be loaded, connect to Netdata and fetch the response back...
+```
+
+The `_VISIBLETOTAL` variable sums up all the dimensions of each chart.
+
+The format of the variables is:
+
+```sh
+NETDATA_${chart_id^^}_${dimension_id^^}="${value}"
+```
+
+The value is rounded to the closest integer, since shell script cannot process decimal numbers.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fweb%2Fapi%2Fexporters%2Fshell%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/web/api/exporters/shell/allmetrics_shell.c b/web/api/exporters/shell/allmetrics_shell.c
new file mode 100644
index 0000000..daa0049
--- /dev/null
+++ b/web/api/exporters/shell/allmetrics_shell.c
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "allmetrics_shell.h"
+
+// ----------------------------------------------------------------------------
+// BASH
+// /api/v1/allmetrics?format=bash
+
+static inline size_t shell_name_copy(char *d, const char *s, size_t usable) {
+    size_t n;
+
+    for(n = 0; *s && n < usable ; d++, s++, n++) {
+        register char c = *s;
+
+        if(unlikely(!isalnum(c))) *d = '_';
+        else *d = (char)toupper(c);
+    }
+    *d = '\0';
+
+    return n;
+}
+
+#define SHELL_ELEMENT_MAX 100
+
+void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) {
+    rrdhost_rdlock(host);
+
+    // for each chart
+    RRDSET *st;
+    rrdset_foreach_read(st, host) {
+        calculated_number total = 0.0;
+        char chart[SHELL_ELEMENT_MAX + 1];
+        shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX);
+
+        buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name);
+        if(rrdset_is_available_for_viewers(st)) {
+            rrdset_rdlock(st);
+
+            // for each dimension
+            RRDDIM *rd;
+            rrddim_foreach_read(rd, st) {
+                if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
+                    char dimension[SHELL_ELEMENT_MAX + 1];
+                    shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX);
+
+                    calculated_number n = rd->last_stored_value;
+
+                    if(isnan(n) || isinf(n))
+                        buffer_sprintf(wb, "NETDATA_%s_%s=\"\"      # %s\n", chart, dimension, st->units);
+                    else {
+                        if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
+                        n = calculated_number_round(n);
+                        if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
+                        buffer_sprintf(wb, "NETDATA_%s_%s=\"" CALCULATED_NUMBER_FORMAT_ZERO "\"      # %s\n", chart, dimension, n, st->units);
+                    }
+                }
+            }
+
+            total = calculated_number_round(total);
+            buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" CALCULATED_NUMBER_FORMAT_ZERO "\"      # %s\n", chart, total, st->units);
+            rrdset_unlock(st);
+        }
+    }
+
+    buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n");
+
+    RRDCALC *rc;
+    for(rc = host->alarms; rc ;rc = rc->next) {
+        if(!rc->rrdset) continue;
+
+        char chart[SHELL_ELEMENT_MAX + 1];
+        shell_name_copy(chart, rc->rrdset->name?rc->rrdset->name:rc->rrdset->id, SHELL_ELEMENT_MAX);
+
+        char alarm[SHELL_ELEMENT_MAX + 1];
+        shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
+
+        calculated_number n = rc->value;
+
+        if(isnan(n) || isinf(n))
+            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\"      # %s\n", chart, alarm, rc->units);
+        else {
+            n = calculated_number_round(n);
+            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" CALCULATED_NUMBER_FORMAT_ZERO "\"      # %s\n", chart, alarm, n, rc->units);
+        }
+
+        buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
+    }
+
+    rrdhost_unlock(host);
+}
+
+// ----------------------------------------------------------------------------
+
+void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb) {
+    rrdhost_rdlock(host);
+
+    buffer_strcat(wb, "{");
+
+    size_t chart_counter = 0;
+    size_t dimension_counter = 0;
+
+    // for each chart
+    RRDSET *st;
+    rrdset_foreach_read(st, host) {
+        if(rrdset_is_available_for_viewers(st)) {
+            rrdset_rdlock(st);
+
+            buffer_sprintf(wb, "%s\n"
+                               "\t\"%s\": {\n"
+                               "\t\t\"name\":\"%s\",\n"
+                               "\t\t\"family\":\"%s\",\n"
+                               "\t\t\"context\":\"%s\",\n"
+                               "\t\t\"units\":\"%s\",\n"
+                               "\t\t\"last_updated\": %ld,\n"
+                               "\t\t\"dimensions\": {"
+                           , chart_counter?",":""
+                           , st->id
+                           , st->name
+                           , st->family
+                           , st->context
+                           , st->units
+                           , rrdset_last_entry_t_nolock(st)
+            );
+
+            chart_counter++;
+            dimension_counter = 0;
+
+            // for each dimension
+            RRDDIM *rd;
+            rrddim_foreach_read(rd, st) {
+                if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
+
+                    buffer_sprintf(wb, "%s\n"
+                                       "\t\t\t\"%s\": {\n"
+                                       "\t\t\t\t\"name\": \"%s\",\n"
+                                       "\t\t\t\t\"value\": "
+                                   , dimension_counter?",":""
+                                   , rd->id
+                                   , rd->name
+                    );
+
+                    if(isnan(rd->last_stored_value))
+                        buffer_strcat(wb, "null");
+                    else
+                        buffer_sprintf(wb, CALCULATED_NUMBER_FORMAT, rd->last_stored_value);
+
+                    buffer_strcat(wb, "\n\t\t\t}");
+
+                    dimension_counter++;
+                }
+            }
+
+            buffer_strcat(wb, "\n\t\t}\n\t}");
+            rrdset_unlock(st);
+        }
+    }
+
+    buffer_strcat(wb, "\n}");
+    rrdhost_unlock(host);
+}
+
diff --git a/web/api/exporters/shell/allmetrics_shell.h b/web/api/exporters/shell/allmetrics_shell.h
new file mode 100644
index 0000000..1d7611a
--- /dev/null
+++ b/web/api/exporters/shell/allmetrics_shell.h
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_ALLMETRICS_SHELL_H
+#define NETDATA_API_ALLMETRICS_SHELL_H
+
+#include "../allmetrics.h"
+
+#define ALLMETRICS_FORMAT_SHELL                 "shell"
+#define ALLMETRICS_FORMAT_PROMETHEUS            "prometheus"
+#define ALLMETRICS_FORMAT_PROMETHEUS_ALL_HOSTS  "prometheus_all_hosts"
+#define ALLMETRICS_FORMAT_JSON                  "json"
+
+#define ALLMETRICS_SHELL                        1
+#define ALLMETRICS_PROMETHEUS                   2
+#define ALLMETRICS_JSON                         3
+#define ALLMETRICS_PROMETHEUS_ALL_HOSTS         4
+
+extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb);
+extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb);
+
+#endif //NETDATA_API_ALLMETRICS_SHELL_H