Merging upstream version 1.36.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 79 insertions, 25 deletions

diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 1de6be4e3..7aa478d95 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "web/api/web_api_v1.h"
+#include "database/storage_engine.h"
 
 static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
 {
@@ -18,7 +19,18 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc
         }
     }
 
-    onewayalloc_freez(owa, temp_rd->state);
+    for(int tier = 0; tier < storage_tiers ;tier++) {
+        if(!temp_rd->tiers[tier]) continue;
+
+        if(archive_mode) {
+            STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
+            if (eng)
+                eng->api.free(temp_rd->tiers[tier]->db_metric_handle);
+        }
+
+        onewayalloc_freez(owa, temp_rd->tiers[tier]);
+    }
+
     onewayalloc_freez(owa, temp_rd);
 }
 
@@ -50,10 +62,22 @@ void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_
     RRDDIM *temp_rd = context_param_list->rd;
     RRDDIM *new_rd_list = NULL, *t;
     int is_archived = (context_param_list->flags & CONTEXT_FLAGS_ARCHIVE);
+
+    RRDSET *st = temp_rd->rrdset;
+    RRDSET *last_st = st;
+    time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
+    time_t last_last_entry_t = last_entry_t;
     while (temp_rd) {
         t = temp_rd->next;
-        RRDSET *st = temp_rd->rrdset;
-        time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
+
+        st = temp_rd->rrdset;
+        if (st == last_st) {
+            last_entry_t = last_last_entry_t;
+        }else {
+            last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
+            last_last_entry_t = last_entry_t;
+            last_st = st;
+        }
 
         if (last_entry_t >= after_requested) {
             temp_rd->next = new_rd_list;
@@ -86,10 +110,15 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis
     (*param_list)->last_entry_t  = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st));
 
     rrddim_foreach_read(rd1, st) {
-        RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize);
+        RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM));
         rd->id = onewayalloc_strdupz(owa, rd1->id);
         rd->name = onewayalloc_strdupz(owa, rd1->name);
-        rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state));
+        for(int tier = 0; tier < storage_tiers ;tier++) {
+            if(rd1->tiers[tier])
+                rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier]));
+            else
+                rd->tiers[tier] = NULL;
+        }
         rd->next = (*param_list)->rd;
         (*param_list)->rd = rd;
     }
@@ -152,24 +181,32 @@ void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)  {
 int rrdset2value_api_v1(
           RRDSET *st
         , BUFFER *wb
-        , calculated_number *n
+        , NETDATA_DOUBLE *n
         , const char *dimensions
         , long points
         , long long after
         , long long before
         , int group_method
+        , const char *group_options
         , long group_time
         , uint32_t options
         , time_t *db_after
         , time_t *db_before
+        , size_t *db_points_read
+        , size_t *db_points_per_tier
+        , size_t *result_points_generated
         , int *value_is_null
+        , NETDATA_DOUBLE *anomaly_rate
         , int timeout
+        , int tier
 ) {
     int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
 
     ONEWAYALLOC *owa = onewayalloc_create(0);
 
-    RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout);
+    RRDR *r = rrd2rrdr(owa, st, points, after, before,
+                       group_method, group_time, options, dimensions, NULL,
+                       group_options, timeout, tier);
 
     if(!r) {
         if(value_is_null) *value_is_null = 1;
@@ -177,9 +214,18 @@ int rrdset2value_api_v1(
         goto cleanup;
     }
 
-    if(rrdr_rows(r) == 0) {
-        rrdr_free(owa, r);
+    if(db_points_read)
+        *db_points_read += r->internal.db_points_read;
+
+    if(db_points_per_tier) {
+        for(int t = 0; t < storage_tiers ;t++)
+            db_points_per_tier[t] += r->internal.tier_points_read[t];
+    }
+
+    if(result_points_generated)
+        *result_points_generated += r->internal.result_points_generated;
 
+    if(rrdr_rows(r) == 0) {
         if(db_after)  *db_after  = 0;
         if(db_before) *db_before = 0;
         if(value_is_null) *value_is_null = 1;
@@ -199,7 +245,7 @@ int rrdset2value_api_v1(
     if(db_before) *db_before = r->before;
 
     long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
-    *n = rrdr2value(r, i, options, value_is_null, NULL);
+    *n = rrdr2value(r, i, options, value_is_null, anomaly_rate, NULL);
     ret = HTTP_RESP_OK;
 
 cleanup:
@@ -218,9 +264,11 @@ int rrdset2anything_api_v1(
         , long long after
         , long long before
         , int group_method
+        , const char *group_options
         , long group_time
         , uint32_t options
         , time_t *latest_timestamp
+        , int tier
 )
 {
     BUFFER *wb = query_params->wb;
@@ -238,7 +286,8 @@ int rrdset2anything_api_v1(
         options,
         dimensions ? buffer_tostring(dimensions) : NULL,
         query_params->context_param_list,
-        query_params->timeout);
+        group_options,
+        query_params->timeout, tier);
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
         return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -249,7 +298,7 @@ int rrdset2anything_api_v1(
         return HTTP_RESP_BACKEND_FETCH_FAILED;
     }
 
-    if (st && st->state && st->state->is_ar_chart)
+    if (st->state && st->state->is_ar_chart)
         ml_process_rrdr(r, query_params->max_anomaly_rates);
 
     RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;
@@ -266,7 +315,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -279,7 +328,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_SSV_COMMA:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -292,7 +341,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JS_ARRAY:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
             rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
@@ -305,7 +354,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -318,7 +367,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_MARKDOWN:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -331,7 +380,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_CSV_JSON_ARRAY:
         wb->contenttype = CT_APPLICATION_JSON;
         if(options & RRDR_OPTION_JSON_WRAP) {
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
             buffer_strcat(wb, "[\n");
             rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
             buffer_strcat(wb, "\n]");
@@ -348,7 +397,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_TSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
@@ -361,7 +410,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_HTML:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
             rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
@@ -379,7 +428,7 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 1, query_params->context_param_list);
 
@@ -391,7 +440,7 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 1, query_params->context_param_list);
 
@@ -402,7 +451,7 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JSONP:
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 0, query_params->context_param_list);
 
@@ -415,12 +464,17 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
 
         rrdr2json(r, wb, options, 0, query_params->context_param_list);
 
-        if(options & RRDR_OPTION_JSON_WRAP)
+        if(options & RRDR_OPTION_JSON_WRAP) {
+            if(options & RRDR_OPTION_RETURN_JWAR) {
+                rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0);
+                rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0, query_params->context_param_list);
+            }
             rrdr_json_wrapper_end(r, wb, format, options, 0);
+        }
         break;
     }