Merging upstream version 1.42.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

12 files changed, 237 insertions, 179 deletions

diff --git a/web/api/badges/README.md b/web/api/badges/README.md
index e40e706eb..e83186bb2 100644
--- a/web/api/badges/README.md
+++ b/web/api/badges/README.md
@@ -157,7 +157,7 @@ Here is what you can put for `options` (these are standard Netdata API options):
 
     These fine tune various options of the API. Here is what you can use for badges (the API has more option, but only these are useful for badges):
 
-    -   `percentage`, instead of returning the value, calculate the percentage of the sum of the selected dimensions, versus the sum of all the dimensions of the chart. This also sets the units to `%`.
+    -   `percentage`, instead of returning a value, calculate the percentage of the sum of the values of the selected dimensions (selected sum / total sum * 100). This also sets the units to `%`.
 
     -   `absolute` or `abs`, turn all values positive and then sum them.
 
diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index 36150b93e..9b81ba4fb 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -1057,8 +1057,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
             buffer_sprintf(w->response.header, "Refresh: %d\r\n", refresh);
             w->response.data->date = now_realtime_sec();
             w->response.data->expires = w->response.data->date + refresh;
+            buffer_cacheable(w->response.data);
         }
-        else buffer_no_cacheable(w->response.data);
+        else
+            buffer_no_cacheable(w->response.data);
 
         if(!value_color) {
             switch(rc->status) {
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 6a66cbcca..52025c9fc 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -874,7 +874,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
         sq[0] = '"';
     }
 
-    buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
+    buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
 
     buffer_json_member_add_uint64(wb, "api", 1);
     buffer_json_member_add_string(wb, "id", qt->id);
@@ -1289,7 +1289,7 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
         sq[0] = '\'';
     }
 
-    buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
+    buffer_json_initialize(wb, kq, sq, 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
     buffer_json_member_add_uint64(wb, "api", 2);
 
     if(options & RRDR_OPTION_DEBUG) {
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index ca3a41aae..f0c0c39ba 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -38,8 +38,6 @@ typedef enum {
 
 #define HOSTNAME_MAX 1024
 
-#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
-
 #define DATASOURCE_FORMAT_JSON "json"
 #define DATASOURCE_FORMAT_JSON2 "json2"
 #define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h
index dd6483b2c..f110c5861 100644
--- a/web/api/queries/incremental_sum/incremental_sum.h
+++ b/web/api/queries/incremental_sum/incremental_sum.h
@@ -12,19 +12,20 @@ struct tg_incremental_sum {
     size_t count;
 };
 
-static inline void tg_incremental_sum_create(RRDR *r, const char *options __maybe_unused) {
-    r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_incremental_sum));
-}
-
 // resets when switches dimensions
 // so, clear everything to restart
 static inline void tg_incremental_sum_reset(RRDR *r) {
     struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data;
-    g->first = 0;
-    g->last = 0;
+    g->first = NAN;
+    g->last = NAN;
     g->count = 0;
 }
 
+static inline void tg_incremental_sum_create(RRDR *r, const char *options __maybe_unused) {
+    r->time_grouping.data = onewayalloc_mallocz(r->internal.owa, sizeof(struct tg_incremental_sum));
+    tg_incremental_sum_reset(r);
+}
+
 static inline void tg_incremental_sum_free(RRDR *r) {
     onewayalloc_freez(r->internal.owa, r->time_grouping.data);
     r->time_grouping.data = NULL;
@@ -34,7 +35,11 @@ static inline void tg_incremental_sum_add(RRDR *r, NETDATA_DOUBLE value) {
     struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data;
 
     if(unlikely(!g->count)) {
-        g->first = value;
+        if(isnan(g->first))
+            g->first = value;
+        else
+            g->last = value;
+
         g->count++;
     }
     else {
@@ -48,19 +53,16 @@ static inline NETDATA_DOUBLE tg_incremental_sum_flush(RRDR *r, RRDR_VALUE_FLAGS
 
     NETDATA_DOUBLE value;
 
-    if(unlikely(!g->count)) {
+    if(unlikely(!g->count || isnan(g->first) || isnan(g->last))) {
         value = 0.0;
         *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
     }
-    else if(unlikely(g->count == 1)) {
-        value = 0.0;
-    }
     else {
         value = g->last - g->first;
     }
 
-    g->first = 0.0;
-    g->last = 0.0;
+    g->first = g->last;
+    g->last = NAN;
     g->count = 0;
 
     return value;
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index 74a925bc3..6f3cbd5fd 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -17,6 +17,7 @@
 #include "percentile/percentile.h"
 #include "trimmed_mean/trimmed_mean.h"
 
+#define QUERY_PLAN_MIN_POINTS 10
 #define POINTS_TO_EXPAND_QUERY 5
 
 // ----------------------------------------------------------------------------
@@ -996,6 +997,10 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
     if(unlikely(after_wanted == before_wanted || points_wanted <= 0))
         return query_metric_first_working_tier(qm);
 
+    if(points_wanted < QUERY_PLAN_MIN_POINTS)
+        // when selecting tiers, aim for a resolution of at least QUERY_PLAN_MIN_POINTS points
+        points_wanted = (before_wanted - after_wanted) > QUERY_PLAN_MIN_POINTS ? QUERY_PLAN_MIN_POINTS : before_wanted - after_wanted;
+
     time_t min_first_time_s = 0;
     time_t max_last_time_s = 0;
 
@@ -2070,88 +2075,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
 }
 #endif // NETDATA_INTERNAL_CHECKS
 
-// Returns 1 if an absolute period was requested or 0 if it was a relative period
-bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr) {
-    time_t now = now_realtime_sec() - 1;
-
-    if(now_ptr)
-        *now_ptr = now;
-
-    int absolute_period_requested = -1;
-    long long after_requested, before_requested;
-
-    before_requested = *before;
-    after_requested = *after;
-
-    // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
-    if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) {
-        // if the user asked for a positive relative time,
-        // flip it to a negative
-        if(before_requested > 0)
-            before_requested = -before_requested;
-
-        before_requested = now + before_requested;
-        absolute_period_requested = 0;
-    }
-
-    // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
-    if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
-        if(after_requested > 0)
-            after_requested = -after_requested;
-
-        // if the user didn't give an after, use the number of points
-        // to give a sane default
-        if(after_requested == 0)
-            after_requested = -600;
-
-        // since the query engine now returns inclusive timestamps
-        // it is awkward to return 6 points when after=-5 is given
-        // so for relative queries we add 1 second, to give
-        // more predictable results to users.
-        after_requested = before_requested + after_requested + 1;
-        absolute_period_requested = 0;
-    }
-
-    if(absolute_period_requested == -1)
-        absolute_period_requested = 1;
-
-    // check if the parameters are flipped
-    if(after_requested > before_requested) {
-        long long t = before_requested;
-        before_requested = after_requested;
-        after_requested = t;
-    }
-
-    // if the query requests future data
-    // shift the query back to be in the present time
-    // (this may also happen because of the rules above)
-    if(before_requested > now) {
-        long long delta = before_requested - now;
-        before_requested -= delta;
-        after_requested  -= delta;
-    }
-
-    time_t absolute_minimum_time = now - (10 * 365 * 86400);
-    time_t absolute_maximum_time = now + (1 * 365 * 86400);
-
-    if (after_requested < absolute_minimum_time && !unittest_running)
-        after_requested = absolute_minimum_time;
-
-    if (after_requested > absolute_maximum_time && !unittest_running)
-        after_requested = absolute_maximum_time;
-
-    if (before_requested < absolute_minimum_time && !unittest_running)
-        before_requested = absolute_minimum_time;
-
-    if (before_requested > absolute_maximum_time && !unittest_running)
-        before_requested = absolute_maximum_time;
-
-    *before = before_requested;
-    *after = after_requested;
-
-    return (absolute_period_requested != 1);
-}
-
 // #define DEBUG_QUERY_LOGIC 1
 
 #ifdef DEBUG_QUERY_LOGIC
@@ -2278,7 +2201,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
     }
 
     // convert our before_wanted and after_wanted to absolute
-    rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL);
+    rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL, unittest_running);
     query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
 
     if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
@@ -2983,11 +2906,11 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
         }
 
         // initialize partial trimming
-        r->partial_data_trimming.max_update_every = update_every_max;
+        r->partial_data_trimming.max_update_every = update_every_max * 2;
         r->partial_data_trimming.expected_after =
                 (!query_target_aggregatable(qt) &&
-                 qt->window.before >= qt->window.now - update_every_max) ?
-                qt->window.before - update_every_max :
+                 qt->window.before >= qt->window.now - r->partial_data_trimming.max_update_every) ?
+                qt->window.before - r->partial_data_trimming.max_update_every :
                 qt->window.before;
         r->partial_data_trimming.trimmed_after = qt->window.before;
 
@@ -3139,6 +3062,8 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
     if(unlikely(i < 0))
         return;
 
+    // internal_error(true, "Found trimmable index %zd (from 0 to %zu)", i, r->n - 1);
+
     size_t last_row_gbc = 0;
     for (; i < (ssize_t)r->n; i++) {
         size_t row_gbc = 0;
@@ -3149,8 +3074,11 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {
             row_gbc += r->gbc[ i * r->d + d ];
         }
 
-        if (unlikely(r->t[i] >= trimmable_after && row_gbc < last_row_gbc)) {
+        // internal_error(true, "GBC of index %zd is %zu", i, row_gbc);
+
+        if (unlikely(r->t[i] >= trimmable_after && (row_gbc < last_row_gbc || !row_gbc))) {
             // discard the rest of the points
+            // internal_error(true, "Discarding points %zd to %zu", i, r->n - 1);
             r->partial_data_trimming.trimmed_after = r->t[i];
             r->rows = i;
             break;
diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h
index c4a1f83f2..e02e00675 100644
--- a/web/api/queries/rrdr.h
+++ b/web/api/queries/rrdr.h
@@ -206,8 +206,6 @@ RRDR *rrd2rrdr_legacy(
 RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
 bool query_target_calculate_window(struct query_target *qt);
 
-bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr);
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c
index 8ffd8951a..2782aef60 100644
--- a/web/api/queries/weights.c
+++ b/web/api/queries/weights.c
@@ -169,7 +169,7 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
                                                 size_t examined_dimensions, usec_t duration,
                                                 WEIGHTS_STATS *stats) {
 
-    buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
 
     results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
                            points, method, group, options, shifts, examined_dimensions, duration, stats);
@@ -221,7 +221,7 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w
                                                   size_t examined_dimensions, usec_t duration,
                                                   WEIGHTS_STATS *stats) {
 
-    buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
 
     results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
                            points, method, group, options, shifts, examined_dimensions, duration, stats);
@@ -739,7 +739,7 @@ static size_t registered_results_to_json_multinode_no_group_by(
         size_t examined_dimensions, struct query_weights_data *qwd,
         WEIGHTS_STATS *stats,
         struct query_versions *versions) {
-    buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
     buffer_json_member_add_uint64(wb, "api", 2);
 
     results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
@@ -958,7 +958,7 @@ static size_t registered_results_to_json_multinode_group_by(
         size_t examined_dimensions, struct query_weights_data *qwd,
         WEIGHTS_STATS *stats,
         struct query_versions *versions) {
-    buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
     buffer_json_member_add_uint64(wb, "api", 2);
 
     results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before,
@@ -1806,8 +1806,10 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
             }
     };
 
-    if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL))
+    if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL, false))
         buffer_no_cacheable(wb);
+    else
+        buffer_cacheable(wb);
 
     if (qwr->before <= qwr->after) {
         resp = HTTP_RESP_BAD_REQUEST;
@@ -1821,7 +1823,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
         if(qwr->baseline_before <= API_RELATIVE_TIME_MAX)
             qwr->baseline_before += qwr->after;
 
-        rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL);
+        rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL, false);
 
         if (qwr->baseline_before <= qwr->baseline_after) {
             resp = HTTP_RESP_BAD_REQUEST;
diff --git a/web/api/web_api.c b/web/api/web_api.c
index 4372bb8cb..7a4704bd5 100644
--- a/web/api/web_api.c
+++ b/web/api/web_api.c
@@ -6,7 +6,7 @@ bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at
 DICTIONARY *netdata_authorized_bearers = NULL;
 
 static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL endpoint_acl) {
-    if(endpoint_acl == WEB_CLIENT_ACL_NOCHECK)
+    if(endpoint_acl == WEB_CLIENT_ACL_NONE || (endpoint_acl & WEB_CLIENT_ACL_NOCHECK))
         // the endpoint is totally public
         return true;
 
@@ -23,27 +23,58 @@ static bool web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL
         // endpoint does not require a bearer
         return true;
 
-    if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)) || api_check_bearer_token(w))
+    if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)))
         // the request is coming from ACLK or WEBRTC (authorized already),
-        // or we have a valid bearer on the request
         return true;
 
+    // at this point we need a bearer to serve the request
+    // either because:
+    //
+    // 1. WEB_CLIENT_ACL_BEARER_REQUIRED, or
+    // 2. netdata_is_protected_by_bearer == true
+    //
+
+    BEARER_STATUS t = api_check_bearer_token(w);
+    if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED)
+        // we have a valid bearer on the request
+        return true;
+
+    netdata_log_info("BEARER: bearer is required for request: code %d", t);
+
     return false;
 }
 
 int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_path_endpoint, struct web_api_command *api_commands) {
+    buffer_no_cacheable(w->response.data);
+
     if(unlikely(!url_path_endpoint || !*url_path_endpoint)) {
         buffer_flush(w->response.data);
         buffer_sprintf(w->response.data, "Which API command?");
         return HTTP_RESP_BAD_REQUEST;
     }
 
-    uint32_t hash = simple_hash(url_path_endpoint);
+    char *api_command = strchr(url_path_endpoint, '/');
+    if (likely(api_command == NULL)) // only config command supports subpaths for now
+        api_command = url_path_endpoint;
+    else {
+        size_t api_command_len = api_command - url_path_endpoint;
+        api_command = callocz(1, api_command_len + 1);
+        memcpy(api_command, url_path_endpoint, api_command_len);
+    }
+
+    uint32_t hash = simple_hash(api_command);
 
     for(int i = 0; api_commands[i].command ; i++) {
-        if(unlikely(hash == api_commands[i].hash && !strcmp(url_path_endpoint, api_commands[i].command))) {
+        if(unlikely(hash == api_commands[i].hash && !strcmp(api_command, api_commands[i].command))) {
+            if(unlikely(!api_commands[i].allow_subpaths && api_command != url_path_endpoint)) {
+                buffer_flush(w->response.data);
+                buffer_sprintf(w->response.data, "API command '%s' does not support subpaths.", api_command);
+                freez(api_command);
+                return HTTP_RESP_BAD_REQUEST;
+            }
+
             if(unlikely(!web_client_check_acl_and_bearer(w, api_commands[i].acl)))
-                return web_client_bearer_required(w);
+                return web_client_permission_denied(w);
 
             char *query_string = (char *)buffer_tostring(w->url_query_string_decoded);
 
@@ -54,6 +85,9 @@ int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_pat
         }
     }
 
+    if (api_command != url_path_endpoint)
+        freez(api_command);
+
     buffer_flush(w->response.data);
     buffer_strcat(w->response.data, "Unsupported API command: ");
     buffer_strcat_htmlescape(w->response.data, url_path_endpoint);
diff --git a/web/api/web_api.h b/web/api/web_api.h
index 840ac8dcb..f7ae45ad0 100644
--- a/web/api/web_api.h
+++ b/web/api/web_api.h
@@ -11,15 +11,26 @@
 
 extern bool netdata_is_protected_by_bearer;
 extern DICTIONARY *netdata_authorized_bearers;
-bool api_check_bearer_token(struct web_client *w);
-bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
-void bearer_tokens_init(void);
+typedef enum __attribute__((packed)) {
+    BEARER_STATUS_NO_BEARER_IN_HEADERS,
+    BEARER_STATUS_BEARER_DOES_NOT_FIT,
+    BEARER_STATUS_NOT_PARSABLE,
+    BEARER_STATUS_EXTRACTED_FROM_HEADER,
+    BEARER_STATUS_NO_BEARERS_DICTIONARY,
+    BEARER_STATUS_NOT_FOUND_IN_DICTIONARY,
+    BEARER_STATUS_EXPIRED,
+    BEARER_STATUS_AVAILABLE_AND_VALIDATED,
+} BEARER_STATUS;
+
+BEARER_STATUS api_check_bearer_token(struct web_client *w);
+BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
 
 struct web_api_command {
     const char *command;
     uint32_t hash;
     WEB_CLIENT_ACL acl;
     int (*callback)(RRDHOST *host, struct web_client *w, char *url);
+    unsigned int allow_subpaths;
 };
 
 struct web_client;
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index 1e4f9c41c..60962413d 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -883,6 +883,11 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
     else if(format == DATASOURCE_JSONP)
         buffer_strcat(w->response.data, ");");
 
+    if(qt->internal.relative)
+        buffer_no_cacheable(w->response.data);
+    else
+        buffer_cacheable(w->response.data);
+
 cleanup:
     query_target_release(qt);
     onewayalloc_destroy(owa);
@@ -936,7 +941,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
     char *cookie = strstr(w->response.data->buffer, NETDATA_REGISTRY_COOKIE_NAME "=");
     if(cookie)
         strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME)], UUID_STR_LEN - 1);
-    else if(!extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)))
+    else if(extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)) != BEARER_STATUS_EXTRACTED_FROM_HEADER)
         person_guid[0] = '\0';
 
     char action = '\0';
@@ -1194,7 +1199,7 @@ static void host_collectors(RRDHOST *host, BUFFER *wb) {
 
 extern int aclk_connected;
 inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb) {
-    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
 
     buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));
     buffer_json_member_add_string(wb, "uid", host->machine_guid);
@@ -1314,7 +1319,7 @@ int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char
     buffer_flush(wb);
     wb->content_type = CT_APPLICATION_JSON;
 
-    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     ml_host_get_detection_info(host, wb);
     buffer_json_finalize(wb);
 
@@ -1419,7 +1424,7 @@ int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, cha
     wb->content_type = CT_APPLICATION_JSON;
     buffer_no_cacheable(wb);
 
-    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     host_functions2json(host, wb);
     buffer_json_finalize(wb);
 
@@ -1520,43 +1525,43 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
 #endif
 
 static struct web_api_command api_commands_v1[] = {
-        { "info",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info                       },
-        { "data",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data                       },
-        { "chart",           0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_chart                      },
-        { "charts",          0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_charts                     },
-        { "context",         0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_context                    },
-        { "contexts",        0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts                   },
+        { "info",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info,     0                     },
+        { "data",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data,     0                     },
+        { "chart",           0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_chart,    0                     },
+        { "charts",          0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_charts,   0                     },
+        { "context",         0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_context,  0                     },
+        { "contexts",        0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts, 0                     },
 
         // registry checks the ACL by itself, so we allow everything
-        { "registry",        0, WEB_CLIENT_ACL_NOCHECK,               web_client_api_request_v1_registry                   },
+        { "registry",        0, WEB_CLIENT_ACL_NOCHECK,               web_client_api_request_v1_registry, 0                     },
 
         // badges can be fetched with both dashboard and badge permissions
-        { "badge.svg",       0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC | WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge },
+        { "badge.svg",       0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC | WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge, 0 },
 
-        { "alarms",          0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms                     },
-        { "alarms_values",   0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms_values             },
-        { "alarm_log",       0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_log                 },
-        { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_variables           },
-        { "alarm_count",     0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_count               },
-        { "allmetrics",      0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_allmetrics                },
+        { "alarms",          0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms,          0              },
+        { "alarms_values",   0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms_values,   0              },
+        { "alarm_log",       0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_log,       0              },
+        { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_variables, 0              },
+        { "alarm_count",     0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_count,     0              },
+        { "allmetrics",      0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_allmetrics,      0              },
 
 #if defined(ENABLE_ML)
-        { "ml_info",         0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info            },
+        { "ml_info",         0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info, 0                      },
         // { "ml_models",       0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models          },
 #endif
 
-        {"manage/health",        0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER, web_client_api_request_v1_mgmt_health },
-        { "aclk",                0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state            },
-        { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations   },
-        { "weights",             0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights               },
+        {"manage/health",        0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_mgmt_health, 0 },
+        { "aclk",                0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state, 0            },
+        { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations, 0   },
+        { "weights",             0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights, 0               },
 
-        {"function",             0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function },
-        {"functions",            0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions },
+        {"function",             0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function, 0 },
+        {"functions",            0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions, 0 },
 
-        { "dbengine_stats",      0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats },
+        { "dbengine_stats",      0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats, 0 },
 
         // terminator
-        { NULL,                  0, WEB_CLIENT_ACL_NONE,      NULL },
+        { NULL,                  0, WEB_CLIENT_ACL_NONE,      NULL, 0 },
 };
 
 inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url_path_endpoint) {
diff --git a/web/api/web_api_v2.c b/web/api/web_api_v2.c
index 97647f5d6..850282121 100644
--- a/web/api/web_api_v2.c
+++ b/web/api/web_api_v2.c
@@ -54,13 +54,13 @@ static time_t bearer_get_token(uuid_t *uuid) {
 
 #define HTTP_REQUEST_AUTHORIZATION_BEARER "\r\nAuthorization: Bearer "
 
-bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
+BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
     const char *req = buffer_tostring(w->response.data);
     size_t req_len = buffer_strlen(w->response.data);
     const char *bearer = strcasestr(req, HTTP_REQUEST_AUTHORIZATION_BEARER);
 
     if(!bearer)
-        return false;
+        return BEARER_STATUS_NO_BEARER_IN_HEADERS;
 
     const char *token_start = bearer + sizeof(HTTP_REQUEST_AUTHORIZATION_BEARER) - 1;
 
@@ -69,26 +69,33 @@ bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t d
 
     const char *token_end = token_start + UUID_STR_LEN - 1 + 2;
     if (token_end > req + req_len)
-        return false;
+        return BEARER_STATUS_BEARER_DOES_NOT_FIT;
 
     strncpyz(dst, token_start, dst_len - 1);
     uuid_t uuid;
     if (uuid_parse(dst, uuid) != 0)
-        return false;
+        return BEARER_STATUS_NOT_PARSABLE;
 
-    return true;
+    return BEARER_STATUS_EXTRACTED_FROM_HEADER;
 }
 
-bool api_check_bearer_token(struct web_client *w) {
+BEARER_STATUS api_check_bearer_token(struct web_client *w) {
     if(!netdata_authorized_bearers)
-        return false;
+        return BEARER_STATUS_NO_BEARERS_DICTIONARY;
 
     char token[UUID_STR_LEN];
-    if(!extract_bearer_token_from_request(w, token, sizeof(token)))
-        return false;
+    BEARER_STATUS t = extract_bearer_token_from_request(w, token, sizeof(token));
+    if(t != BEARER_STATUS_EXTRACTED_FROM_HEADER)
+        return t;
 
     struct bearer_token *z = dictionary_get(netdata_authorized_bearers, token);
-    return z && z->expires_s > now_monotonic_sec();
+    if(!z)
+        return BEARER_STATUS_NOT_FOUND_IN_DICTIONARY;
+
+    if(z->expires_s < now_monotonic_sec())
+        return BEARER_STATUS_EXPIRED;
+
+    return BEARER_STATUS_AVAILABLE_AND_VALIDATED;
 }
 
 static bool verify_agent_uuids(const char *machine_guid, const char *node_id, const char *claim_id) {
@@ -153,7 +160,7 @@ int api_v2_bearer_protection(RRDHOST *host __maybe_unused, struct web_client *w
 
     BUFFER *wb = w->response.data;
     buffer_flush(wb);
-    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
     buffer_json_finalize(wb);
 
@@ -192,7 +199,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may
 
     BUFFER *wb = w->response.data;
     buffer_flush(wb);
-    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
     buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer);
     buffer_json_member_add_uuid(wb, "token", &uuid);
@@ -629,6 +636,11 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
     else if(format == DATASOURCE_JSONP)
         buffer_strcat(w->response.data, ");");
 
+    if(qt->internal.relative)
+        buffer_no_cacheable(w->response.data);
+    else
+        buffer_cacheable(w->response.data);
+
 cleanup:
     query_target_release(qt);
     onewayalloc_destroy(owa);
@@ -639,31 +651,97 @@ static int web_client_api_request_v2_webrtc(RRDHOST *host __maybe_unused, struct
     return webrtc_new_connection(w->post_payload, w->response.data);
 }
 
+#define CONFIG_API_V2_URL "/api/v2/config"
+static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct web_client *w, char *query) {
+
+    char *url = strdupz(buffer_tostring(w->url_as_received));
+    char *url_full = url;
+
+    if (strncmp(url, CONFIG_API_V2_URL, strlen(CONFIG_API_V2_URL)) != 0) {
+        buffer_sprintf(w->response.data, "Invalid URL");
+        return HTTP_RESP_BAD_REQUEST;
+    }
+    url += strlen(CONFIG_API_V2_URL);
+
+    char *save_ptr = NULL;
+    char *plugin = strtok_r(url, "/", &save_ptr);
+    char *module = strtok_r(NULL, "/", &save_ptr);
+    char *job_id = strtok_r(NULL, "/", &save_ptr);
+    char *extra = strtok_r(NULL, "/", &save_ptr);
+
+    buffer_flush(w->response.data);
+    if (extra != NULL) {
+        buffer_sprintf(w->response.data, "Invalid URL");
+        freez(url_full);
+        return HTTP_RESP_BAD_REQUEST;
+    }
+
+    int http_method;
+    switch (w->mode)
+    {
+        case WEB_CLIENT_MODE_GET:
+            http_method = HTTP_METHOD_GET;
+            break;
+        case WEB_CLIENT_MODE_POST:
+            http_method = HTTP_METHOD_POST;
+            break;
+        case WEB_CLIENT_MODE_PUT:
+            http_method = HTTP_METHOD_PUT;
+            break;
+        case WEB_CLIENT_MODE_DELETE:
+            http_method = HTTP_METHOD_DELETE;
+            break;
+        default:
+            buffer_sprintf(w->response.data, "Invalid HTTP method");
+            freez(url_full);
+            return HTTP_RESP_BAD_REQUEST;
+    }
+
+    struct uni_http_response resp = dyn_conf_process_http_request(http_method, plugin, module, job_id, w->post_payload, w->post_payload_size);
+    if (resp.content[resp.content_length - 1] != '\0') {
+        char *con = mallocz(resp.content_length + 1);
+        memcpy(con, resp.content, resp.content_length);
+        con[resp.content_length] = '\0';
+        if (resp.content_free)
+            resp.content_free(resp.content);
+        resp.content = con;
+        resp.content_free = freez_dyncfg;
+    }
+    buffer_strcat(w->response.data, resp.content);
+    if (resp.content_free)
+        resp.content_free(resp.content);
+    w->response.data->content_type = resp.content_type;
+    freez(url_full);
+    return resp.status;
+}
+
 static struct web_api_command api_commands_v2[] = {
-        {"info",                0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_info},
+        {"info",                0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_info, 0},
+
+        {"data",                0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_data, 0},
+        {"weights",             0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_weights, 0},
 
-        {"data",                0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_data},
-        {"weights",             0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_weights},
+        {"contexts",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_contexts, 0},
+        {"nodes",               0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_nodes, 0},
+        {"node_instances",      0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_node_instances, 0},
+        {"versions",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_versions, 0},
+        {"functions",           0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS,     web_client_api_request_v2_functions, 0},
+        {"q",                   0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_q, 0},
+        {"alerts",              0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_alerts, 0},
 
-        {"contexts",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_contexts},
-        {"nodes",               0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_nodes},
-        {"node_instances",      0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_node_instances},
-        {"versions",            0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_versions},
-        {"functions",           0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS,     web_client_api_request_v2_functions},
-        {"q",                   0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_q},
-        {"alerts",              0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_alerts},
+        {"alert_transitions",   0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_alert_transitions, 0},
+        {"alert_config",        0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_alert_config, 0},
 
-        {"alert_transitions",   0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_alert_transitions},
-        {"alert_config",        0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC,                                       web_client_api_request_v2_alert_config},
+        {"claim",               0, WEB_CLIENT_ACL_NOCHECK,                                                     web_client_api_request_v2_claim, 0},
 
-        {"claim",               0, WEB_CLIENT_ACL_NOCHECK,                                                     web_client_api_request_v2_claim},
+        {"rtc_offer",           0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS,                                  web_client_api_request_v2_webrtc, 0},
+        {"bearer_protection",   0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0},
+        {"bearer_get_token",    0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0},
 
-        {"rtc_offer",           0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS,                                  web_client_api_request_v2_webrtc},
-        {"bearer_protection",   0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection},
-        {"bearer_get_token",    0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token},
+        {"config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1},
 
         // terminator
-        {NULL,                  0, WEB_CLIENT_ACL_NONE,                 NULL},
+        {NULL,                  0, WEB_CLIENT_ACL_NONE,                 NULL, 0},
 };
 
 inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint) {