From 81581f9719bc56f01d5aa08952671d65fda9867a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 8 May 2023 18:27:08 +0200 Subject: Merging upstream version 1.39.0. Signed-off-by: Daniel Baumann --- web/api/formatters/README.md | 4 + web/api/formatters/charts2json.c | 52 -- web/api/formatters/charts2json.h | 1 - web/api/formatters/csv/README.md | 4 + web/api/formatters/csv/csv.c | 61 +- web/api/formatters/json/README.md | 4 + web/api/formatters/json/json.c | 170 ++-- web/api/formatters/json/json.h | 1 + web/api/formatters/json_wrapper.c | 1806 +++++++++++++++++++++++++++++------- web/api/formatters/json_wrapper.h | 14 +- web/api/formatters/rrd2json.c | 231 +++-- web/api/formatters/rrd2json.h | 53 +- web/api/formatters/rrdset2json.c | 8 +- web/api/formatters/ssv/README.md | 4 + web/api/formatters/ssv/ssv.c | 10 +- web/api/formatters/value/README.md | 4 + web/api/formatters/value/value.c | 86 +- web/api/formatters/value/value.h | 4 +- 18 files changed, 1827 insertions(+), 690 deletions(-) (limited to 'web/api/formatters') diff --git a/web/api/formatters/README.md b/web/api/formatters/README.md index 4c281f064..ddc70d90f 100644 --- a/web/api/formatters/README.md +++ b/web/api/formatters/README.md @@ -1,6 +1,10 @@ # Query formatting diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c index 61a9ecf2f..4b6b095c2 100644 --- a/web/api/formatters/charts2json.c +++ b/web/api/formatters/charts2json.c @@ -137,55 +137,3 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived buffer_sprintf(wb, "\n\t]\n}\n"); } - -// generate collectors list for the api/v1/info call - -struct collector { - const char *plugin; - const char *module; -}; - -struct array_printer { - int c; - BUFFER *wb; -}; - -static int print_collector_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { - struct array_printer *ap = (struct array_printer *)data; - BUFFER *wb = ap->wb; - struct collector *col=(struct collector *) entry; - if(ap->c) buffer_strcat(wb, ","); - buffer_strcat(wb, "\n\t\t{\n\t\t\t\"plugin\": \""); - buffer_strcat(wb, col->plugin); - buffer_strcat(wb, "\",\n\t\t\t\"module\": \""); - buffer_strcat(wb, col->module); - buffer_strcat(wb, "\"\n\t\t}"); - (ap->c)++; - return 0; -} - -void chartcollectors2json(RRDHOST *host, BUFFER *wb) { - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); - RRDSET *st; - char name[500]; - - time_t now = now_realtime_sec(); - rrdset_foreach_read(st, host) { - if (rrdset_is_available_for_viewers(st)) { - struct collector col = { - .plugin = rrdset_plugin_name(st), - .module = rrdset_module_name(st) - }; - sprintf(name, "%s:%s", col.plugin, col.module); - dictionary_set(dict, name, &col, sizeof(struct collector)); - st->last_accessed_time_s = now; - } - } - rrdset_foreach_done(st); - struct array_printer ap = { - .c = 0, - .wb = wb - }; - dictionary_walkthrough_read(dict, print_collector_callback, &ap); - dictionary_destroy(dict); -} diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h index d4b04af58..96720d4b4 100644 --- a/web/api/formatters/charts2json.h +++ b/web/api/formatters/charts2json.h @@ -6,7 +6,6 @@ #include "rrd2json.h" void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived); -void chartcollectors2json(RRDHOST *host, BUFFER *wb); const char* get_release_channel(); #endif //NETDATA_API_FORMATTER_CHARTS2JSON_H diff --git a/web/api/formatters/csv/README.md b/web/api/formatters/csv/README.md index fc5ffec1b..4585710b4 100644 --- a/web/api/formatters/csv/README.md +++ b/web/api/formatters/csv/README.md @@ -1,6 +1,10 @@ # CSV formatter diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c index 18009f146..8f4950ddd 100644 --- a/web/api/formatters/csv/csv.c +++ b/web/api/formatters/csv/csv.c @@ -5,15 +5,13 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) { //info("RRD2CSV(): %s: BEGIN", r->st->id); - QUERY_TARGET *qt = r->internal.qt; long c, i; - const long used = qt->query.used; + const long used = (long)r->d; // print the csv header for(c = 0, i = 0; c < used ; c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; if(!i) { buffer_strcat(wb, startline); @@ -23,7 +21,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const } buffer_strcat(wb, separator); if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - buffer_strcat(wb, string2str(qt->query.array[c].dimension.name)); + buffer_strcat(wb, string2str(r->dn[c])); if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); i++; } @@ -32,9 +30,8 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const if(format == DATASOURCE_CSV_MARKDOWN) { // print the --- line after header for(c = 0, i = 0; c < used ;c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; if(!i) { buffer_strcat(wb, startline); @@ -64,7 +61,6 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const } // for each line in the array - NETDATA_DOUBLE total = 1; for(i = start; i != end ;i += step) { NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; @@ -76,7 +72,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { // print the timestamp of the line - buffer_rrd_value(wb, (NETDATA_DOUBLE)now); + buffer_print_netdata_double(wb, (NETDATA_DOUBLE) now); // in ms if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); } @@ -87,29 +83,10 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); } - int set_min_max = 0; - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - total = 0; - for(c = 0; c < used ;c++) { - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - - NETDATA_DOUBLE n = cn[c]; - - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; - - total += n; - } - // prevent a division by zero - if(total == 0) total = 1; - set_min_max = 1; - } - // for each dimension for(c = 0; c < used ;c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; buffer_strcat(wb, separator); @@ -121,24 +98,8 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const else buffer_strcat(wb, "null"); } - else { - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; - - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - n = n * 100 / total; - - if(unlikely(set_min_max)) { - r->min = r->max = n; - set_min_max = 0; - } - - if(n < r->min) r->min = n; - if(n > r->max) r->max = n; - } - - buffer_rrd_value(wb, n); - } + else + buffer_print_netdata_double(wb, n); } buffer_strcat(wb, endline); diff --git a/web/api/formatters/json/README.md b/web/api/formatters/json/README.md index 75f729ada..bc70aec02 100644 --- a/web/api/formatters/json/README.md +++ b/web/api/formatters/json/README.md @@ -1,6 +1,10 @@ # JSON formatter diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c index 3cad3e914..d5b8c7570 100644 --- a/web/api/formatters/json/json.c +++ b/web/api/formatters/json/json.c @@ -42,7 +42,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { strcpy(post_value, "}"); strcpy(post_line, "]}"); snprintfz(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq); - strcpy(finish, "\n]\n}"); + strcpy(finish, "\n ]\n }"); snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); @@ -69,9 +69,9 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { dates_with_new = 0; } if( options & RRDR_OPTION_OBJECTSROWS ) - strcpy(pre_date, " { "); + strcpy(pre_date, " {"); else - strcpy(pre_date, " [ "); + strcpy(pre_date, " ["); strcpy(pre_label, ",\""); strcpy(post_label, "\""); strcpy(pre_value, ","); @@ -79,10 +79,10 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { strcpy(post_line, "}"); else strcpy(post_line, "]"); - snprintfz(data_begin, 100, "],\n %sdata%s:\n [\n", kq, kq); - strcpy(finish, "\n]\n}"); + snprintfz(data_begin, 100, "],\n %sdata%s:[\n", kq, kq); + strcpy(finish, "\n ]\n }"); - buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq); + buffer_sprintf(wb, "{\n %slabels%s:[", kq, kq); buffer_sprintf(wb, "%stime%s", sq, sq); if( options & RRDR_OPTION_OBJECTSROWS ) @@ -104,18 +104,16 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { // ------------------------------------------------------------------------- // print the JSON header - QUERY_TARGET *qt = r->internal.qt; long c, i; - const long used = qt->query.used; + const long used = (long)r->d; // print the header lines for(c = 0, i = 0; c < used ; c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; buffer_fast_strcat(wb, pre_label, pre_label_len); - buffer_strcat(wb, string2str(qt->query.array[c].dimension.name)); + buffer_strcat(wb, string2str(r->dn[c])); buffer_fast_strcat(wb, post_label, post_label_len); i++; } @@ -151,7 +149,6 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { ); // for each line in the array - NETDATA_DOUBLE total = 1; for(i = start; i != end ;i += step) { NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; @@ -203,7 +200,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); - buffer_rrd_value(wb, (NETDATA_DOUBLE)r->t[i]); + buffer_print_netdata_double(wb, (NETDATA_DOUBLE) r->t[i]); // in ms if(unlikely(options & RRDR_OPTION_MILLISECONDS)) @@ -212,33 +209,10 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { buffer_fast_strcat(wb, post_date, post_date_len); } - int set_min_max = 0; - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - total = 0; - for(c = 0; c < used ;c++) { - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - - NETDATA_DOUBLE n; - if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) - n = ar[c]; - else - n = cn[c]; - - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; - - total += n; - } - // prevent a division by zero - if(total == 0) total = 1; - set_min_max = 1; - } - // for each dimension for(c = 0; c < used ;c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; NETDATA_DOUBLE n; if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) @@ -249,39 +223,119 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { buffer_fast_strcat(wb, pre_value, pre_value_len); if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) - buffer_sprintf(wb, "%s%s%s: ", kq, string2str(qt->query.array[c].dimension.name), kq); + buffer_sprintf(wb, "%s%s%s: ", kq, string2str(r->dn[c]), kq); - if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) { + if(co[c] & RRDR_VALUE_EMPTY && !(options & (RRDR_OPTION_INTERNAL_AR))) { if(unlikely(options & RRDR_OPTION_NULL2ZERO)) buffer_fast_strcat(wb, "0", 1); else buffer_fast_strcat(wb, "null", 4); } - else { - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; + else + buffer_print_netdata_double(wb, n); - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - n = n * 100 / total; + buffer_fast_strcat(wb, post_value, post_value_len); + } - if(unlikely(set_min_max)) { - r->min = r->max = n; - set_min_max = 0; - } + buffer_fast_strcat(wb, post_line, post_line_len); + } + + buffer_strcat(wb, finish); + //info("RRD2JSON(): %s: END", r->st->id); +} + + +void rrdr2json_v2(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + RRDR_OPTIONS options = qt->window.options; + + bool expose_gbc = query_target_aggregatable(qt); + + buffer_json_member_add_object(wb, "result"); - if(n < r->min) r->min = n; - if(n > r->max) r->max = n; + buffer_json_member_add_array(wb, "labels"); + buffer_json_add_array_item_string(wb, "time"); + long d, i; + const long used = (long)r->d; + for(d = 0, i = 0; d < used ; d++) { + if(!rrdr_dimension_should_be_exposed(r->od[d], options)) + continue; + + buffer_json_add_array_item_string(wb, string2str(r->di[d])); + i++; + } + buffer_json_array_close(wb); // labels + + buffer_json_member_add_object(wb, "point"); + buffer_json_member_add_uint64(wb, "value", 0); + buffer_json_member_add_uint64(wb, "arp", 1); + buffer_json_member_add_uint64(wb, "pa", 2); + if(expose_gbc) + buffer_json_member_add_uint64(wb, "count", 3); + buffer_json_object_close(wb); + + buffer_json_member_add_array(wb, "data"); + if(i) { + long start = 0, end = rrdr_rows(r), step = 1; + if (!(options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + for (i = start; i != end; i += step) { + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; + uint32_t *gbc = &r->gbc [ i * r->d ]; + time_t now = r->t[i]; + + buffer_json_add_array_item_array(wb); // row + + if (options & RRDR_OPTION_MILLISECONDS) + buffer_json_add_array_item_time_ms(wb, now); // the time + else + buffer_json_add_array_item_time_t(wb, now); // the time + + for (d = 0; d < used; d++) { + if (!rrdr_dimension_should_be_exposed(r->od[d], options)) + continue; + + RRDR_VALUE_FLAGS o = co[d]; + + buffer_json_add_array_item_array(wb); // point + + // add the value + NETDATA_DOUBLE n = cn[d]; + + if(o & RRDR_VALUE_EMPTY) { + if (unlikely(options & RRDR_OPTION_NULL2ZERO)) + buffer_json_add_array_item_double(wb, 0); + else + buffer_json_add_array_item_double(wb, NAN); } + else + buffer_json_add_array_item_double(wb, n); + + // add the anomaly + buffer_json_add_array_item_double(wb, ar[d]); - buffer_rrd_value(wb, n); + // add the point annotations + buffer_json_add_array_item_uint64(wb, o); + + // add the count + if(expose_gbc) + buffer_json_add_array_item_uint64(wb, gbc[d]); + + buffer_json_array_close(wb); // point } - buffer_fast_strcat(wb, post_value, post_value_len); + buffer_json_array_close(wb); // row } - - buffer_fast_strcat(wb, post_line, post_line_len); } - buffer_strcat(wb, finish); - //info("RRD2JSON(): %s: END", r->st->id); + buffer_json_array_close(wb); // data + + buffer_json_object_close(wb); // annotations } diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h index fb59e5c9a..d1ab4f901 100644 --- a/web/api/formatters/json/json.h +++ b/web/api/formatters/json/json.h @@ -6,5 +6,6 @@ #include "../rrd2json.h" void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable); +void rrdr2json_v2(RRDR *r, BUFFER *wb); #endif //NETDATA_API_FORMATTER_JSON_H diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index aa663495a..6bcbb8d5a 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -2,441 +2,868 @@ #include "json_wrapper.h" -struct value_output { - int c; - BUFFER *wb; -}; +static void jsonwrap_query_metric_plan(BUFFER *wb, QUERY_METRIC *qm) { + buffer_json_member_add_array(wb, "plans"); + for (size_t p = 0; p < qm->plan.used; p++) { + QUERY_PLAN_ENTRY *qp = &qm->plan.array[p]; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tr", qp->tier); + buffer_json_member_add_time_t(wb, "af", qp->after); + buffer_json_member_add_time_t(wb, "bf", qp->before); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "tiers"); + for (size_t tier = 0; tier < storage_tiers; tier++) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tr", tier); + buffer_json_member_add_time_t(wb, "fe", qm->tiers[tier].db_first_time_s); + buffer_json_member_add_time_t(wb, "le", qm->tiers[tier].db_last_time_s); + buffer_json_member_add_int64(wb, "wg", qm->tiers[tier].weight); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); +} -static int value_list_output_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { - struct value_output *ap = (struct value_output *)data; - BUFFER *wb = ap->wb; - char *output = (char *) entry; - if(ap->c) buffer_strcat(wb, ","); - buffer_strcat(wb, output); - (ap->c)++; - return 0; +void jsonwrap_query_plan(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + + buffer_json_member_add_object(wb, "query_plan"); + for(size_t m = 0; m < qt->query.used; m++) { + QUERY_METRIC *qm = query_metric(qt, m); + buffer_json_member_add_object(wb, query_metric_id(qt, qm)); + jsonwrap_query_metric_plan(wb, qm); + buffer_json_object_close(wb); + } + buffer_json_object_close(wb); } -static int fill_formatted_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { - (void)ls; - DICTIONARY *dict = (DICTIONARY *)data; - char n[RRD_ID_LENGTH_MAX * 2 + 2]; - char output[RRD_ID_LENGTH_MAX * 2 + 8]; - char v[RRD_ID_LENGTH_MAX * 2 + 1]; +static inline size_t rrdr_dimension_names(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + const size_t dimensions = r->d; + size_t c, i; - sanitize_json_string(v, (char *)value, RRD_ID_LENGTH_MAX * 2); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", name, v); - snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, v); - dictionary_set(dict, n, output, len + 1); + buffer_json_member_add_array(wb, key); + for(c = 0, i = 0; c < dimensions ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - return 1; + buffer_json_add_array_item_string(wb, string2str(r->dn[c])); + i++; + } + buffer_json_array_close(wb); + + return i; } -void rrdr_show_plan(RRDR *r, BUFFER *wb, const char *kq, const char *sq __maybe_unused) { +static inline size_t rrdr_dimension_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + const size_t dimensions = r->d; + size_t c, i; + + buffer_json_member_add_array(wb, key); + for(c = 0, i = 0; c < dimensions ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_string(wb, string2str(r->di[c])); + i++; + } + buffer_json_array_close(wb); + + return i; +} + +static inline long jsonwrap_v1_chart_ids(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; - buffer_sprintf(wb, "\n\t%squery_plan%s: {", kq, kq); + buffer_json_member_add_array(wb, key); + for (c = 0, i = 0; c < query_used; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - for(size_t m = 0; m < qt->query.used; m++) { - QUERY_METRIC *qm = &qt->query.array[m]; + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria)); + i++; + } + buffer_json_array_close(wb); - if(m) - buffer_strcat(wb, ","); + return i; +} - buffer_sprintf(wb, "\n\t\t%s%s%s: {", kq, string2str(qm->dimension.id), kq); +struct summary_total_counts { + size_t selected; + size_t excluded; + size_t queried; + size_t failed; +}; - buffer_sprintf(wb, "\n\t\t\t%splans%s: [", kq, kq); - for(size_t p = 0; p < qm->plan.used ;p++) { - QUERY_PLAN_ENTRY *qp = &qm->plan.array[p]; - if(p) - buffer_strcat(wb, ","); +static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, QUERY_METRICS_COUNTS *metrics) { + if(unlikely(!totals || !metrics)) + return; - buffer_strcat(wb, "\n\t\t\t\t{"); - buffer_sprintf(wb, "\n\t\t\t\t\t%stier%s: %zu,", kq, kq, qp->tier); - buffer_sprintf(wb, "\n\t\t\t\t\t%safter%s: %ld,", kq, kq, qp->after); - buffer_sprintf(wb, "\n\t\t\t\t\t%sbefore%s: %ld", kq, kq, qp->before); - buffer_strcat(wb, "\n\t\t\t\t}"); - } - buffer_strcat(wb, "\n\t\t\t],"); - - buffer_sprintf(wb, "\n\t\t\t%stiers%s: [", kq, kq); - for(size_t tier = 0; tier < storage_tiers ;tier++) { - if(tier) - buffer_strcat(wb, ","); - - buffer_strcat(wb, "\n\t\t\t\t{"); - buffer_sprintf(wb, "\n\t\t\t\t\t%stier%s: %zu,", kq, kq, tier); - buffer_sprintf(wb, "\n\t\t\t\t\t%sdb_first_time%s: %ld,", kq, kq, qm->tiers[tier].db_first_time_s); - buffer_sprintf(wb, "\n\t\t\t\t\t%sdb_last_time%s: %ld,", kq, kq, qm->tiers[tier].db_last_time_s); - buffer_sprintf(wb, "\n\t\t\t\t\t%sweight%s: %ld", kq, kq, qm->tiers[tier].weight); - buffer_strcat(wb, "\n\t\t\t\t}"); - } - buffer_strcat(wb, "\n\t\t\t]"); + if(metrics->selected) { + totals->selected++; + + if(metrics->queried) + totals->queried++; - buffer_strcat(wb, "\n\t\t}"); + else if(metrics->failed) + totals->failed++; } + else + totals->excluded++; +} + +static inline void query_target_total_counts(BUFFER *wb, const char *key, struct summary_total_counts *totals) { + if(!totals->selected && !totals->queried && !totals->failed && !totals->excluded) + return; - buffer_strcat(wb, "\n\t},"); + buffer_json_member_add_object(wb, key); + + if(totals->selected) + buffer_json_member_add_uint64(wb, "sl", totals->selected); + + if(totals->excluded) + buffer_json_member_add_uint64(wb, "ex", totals->excluded); + + if(totals->queried) + buffer_json_member_add_uint64(wb, "qr", totals->queried); + + if(totals->failed) + buffer_json_member_add_uint64(wb, "fl", totals->failed); + + buffer_json_object_close(wb); } -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - RRDR_GROUPING group_method) -{ - QUERY_TARGET *qt = r->internal.qt; +static inline void query_target_metric_counts(BUFFER *wb, QUERY_METRICS_COUNTS *metrics) { + if(!metrics->selected && !metrics->queried && !metrics->failed && !metrics->excluded) + return; - long rows = rrdr_rows(r); - long c, i; - const long query_used = qt->query.used; + buffer_json_member_add_object(wb, "ds"); - //info("JSONWRAPPER(): %s: BEGIN", r->st->id); - char kq[2] = "", // key quote - sq[2] = ""; // string quote + if(metrics->selected) + buffer_json_member_add_uint64(wb, "sl", metrics->selected); - if( options & RRDR_OPTION_GOOGLE_JSON ) { - kq[0] = '\0'; - sq[0] = '\''; + if(metrics->excluded) + buffer_json_member_add_uint64(wb, "ex", metrics->excluded); + + if(metrics->queried) + buffer_json_member_add_uint64(wb, "qr", metrics->queried); + + if(metrics->failed) + buffer_json_member_add_uint64(wb, "fl", metrics->failed); + + buffer_json_object_close(wb); +} + +static inline void query_target_instance_counts(BUFFER *wb, QUERY_INSTANCES_COUNTS *instances) { + if(!instances->selected && !instances->queried && !instances->failed && !instances->excluded) + return; + + buffer_json_member_add_object(wb, "is"); + + if(instances->selected) + buffer_json_member_add_uint64(wb, "sl", instances->selected); + + if(instances->excluded) + buffer_json_member_add_uint64(wb, "ex", instances->excluded); + + if(instances->queried) + buffer_json_member_add_uint64(wb, "qr", instances->queried); + + if(instances->failed) + buffer_json_member_add_uint64(wb, "fl", instances->failed); + + buffer_json_object_close(wb); +} + +static inline void query_target_alerts_counts(BUFFER *wb, QUERY_ALERTS_COUNTS *alerts, const char *name, bool array) { + if(!alerts->clear && !alerts->other && !alerts->critical && !alerts->warning) + return; + + if(array) + buffer_json_add_array_item_object(wb); + else + buffer_json_member_add_object(wb, "al"); + + if(name) + buffer_json_member_add_string(wb, "nm", name); + + if(alerts->clear) + buffer_json_member_add_uint64(wb, "cl", alerts->clear); + + if(alerts->warning) + buffer_json_member_add_uint64(wb, "wr", alerts->warning); + + if(alerts->critical) + buffer_json_member_add_uint64(wb, "cr", alerts->critical); + + if(alerts->other) + buffer_json_member_add_uint64(wb, "ot", alerts->other); + + buffer_json_object_close(wb); +} + +static inline void query_target_points_statistics(BUFFER *wb, QUERY_TARGET *qt, STORAGE_POINT *sp) { + if(!sp->count) + return; + + buffer_json_member_add_object(wb, "sts"); + + buffer_json_member_add_double(wb, "min", sp->min); + buffer_json_member_add_double(wb, "max", sp->max); + + if(query_target_aggregatable(qt)) { + buffer_json_member_add_uint64(wb, "cnt", sp->count); + + if(sp->sum != 0.0) { + buffer_json_member_add_double(wb, "sum", sp->sum); + buffer_json_member_add_double(wb, "vol", sp->sum * (NETDATA_DOUBLE) query_view_update_every(qt)); + } + + if(sp->anomaly_count != 0) + buffer_json_member_add_uint64(wb, "arc", sp->anomaly_count); } else { - kq[0] = '"'; - sq[0] = '"'; + NETDATA_DOUBLE avg = (sp->count) ? sp->sum / (NETDATA_DOUBLE)sp->count : 0.0; + if(avg != 0.0) + buffer_json_member_add_double(wb, "avg", avg); + + NETDATA_DOUBLE arp = storage_point_anomaly_rate(*sp); + if(arp != 0.0) + buffer_json_member_add_double(wb, "arp", arp); + + NETDATA_DOUBLE con = (qt->query_points.sum > 0.0) ? sp->sum * 100.0 / qt->query_points.sum : 0.0; + if(con != 0.0) + buffer_json_member_add_double(wb, "con", con); } + buffer_json_object_close(wb); +} - buffer_sprintf(wb, "{\n" - " %sapi%s: 1,\n" - " %sid%s: %s%s%s,\n" - " %sname%s: %s%s%s,\n" - " %sview_update_every%s: %lld,\n" - " %supdate_every%s: %lld,\n" - " %sfirst_entry%s: %lld,\n" - " %slast_entry%s: %lld,\n" - " %sbefore%s: %lld,\n" - " %safter%s: %lld,\n" - " %sgroup%s: %s%s%s,\n" - " %soptions%s: %s" - , kq, kq - , kq, kq, sq, qt->id, sq - , kq, kq, sq, qt->id, sq - , kq, kq, (long long)r->update_every - , kq, kq, (long long)qt->db.minimum_latest_update_every_s - , kq, kq, (long long)qt->db.first_time_s - , kq, kq, (long long)qt->db.last_time_s - , kq, kq, (long long)r->before - , kq, kq, (long long)r->after - , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq - , kq, kq, sq); - - web_client_api_request_v1_data_options_to_buffer(wb, r->internal.query_options); - - buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq); - - for(c = 0, i = 0; c < query_used ; c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; - - if(i) buffer_strcat(wb, ", "); - buffer_strcat(wb, sq); - buffer_strcat(wb, string2str(qt->query.array[c].dimension.name)); - buffer_strcat(wb, sq); - i++; +static void query_target_summary_nodes_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { + buffer_json_member_add_array(wb, key); + for (size_t c = 0; c < qt->nodes.used; c++) { + QUERY_NODE *qn = query_node(qt, c); + RRDHOST *host = qn->rrdhost; + buffer_json_add_array_item_object(wb); + buffer_json_node_add_v2(wb, host, qn->slot, qn->duration_ut); + query_target_instance_counts(wb, &qn->instances); + query_target_metric_counts(wb, &qn->metrics); + query_target_alerts_counts(wb, &qn->alerts, NULL, false); + query_target_points_statistics(wb, qt, &qn->query_points); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &qn->metrics); } - if(!i) { -#ifdef NETDATA_INTERNAL_CHECKS - error("QUERY: '%s', RRDR is empty, %zu dimensions, options is 0x%08x", qt->id, r->d, options); -#endif - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); + buffer_json_array_close(wb); +} + +static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { + buffer_json_member_add_array(wb, key); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + + struct { + STORAGE_POINT query_points; + QUERY_INSTANCES_COUNTS instances; + QUERY_METRICS_COUNTS metrics; + QUERY_ALERTS_COUNTS alerts; + } *z; + + for (long c = 0; c < (long) qt->contexts.used; c++) { + QUERY_CONTEXT *qc = query_context(qt, c); + + z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), NULL, sizeof(*z)); + + z->instances.selected += qc->instances.selected; + z->instances.excluded += qc->instances.selected; + z->instances.queried += qc->instances.queried; + z->instances.failed += qc->instances.failed; + + z->metrics.selected += qc->metrics.selected; + z->metrics.excluded += qc->metrics.excluded; + z->metrics.queried += qc->metrics.queried; + z->metrics.failed += qc->metrics.failed; + + z->alerts.clear += qc->alerts.clear; + z->alerts.warning += qc->alerts.warning; + z->alerts.critical += qc->alerts.critical; + + storage_point_merge_to(z->query_points, qc->query_points); } - buffer_sprintf(wb, "],\n" - " %sdimension_ids%s: [" - , kq, kq); + size_t unique_contexts = dictionary_entries(dict); + dfe_start_read(dict, z) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", z_dfe.name); + query_target_instance_counts(wb, &z->instances); + query_target_metric_counts(wb, &z->metrics); + query_target_alerts_counts(wb, &z->alerts, NULL, false); + query_target_points_statistics(wb, qt, &z->query_points); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &z->metrics); + } + dfe_done(z); + buffer_json_array_close(wb); + dictionary_destroy(dict); - for(c = 0, i = 0; c < query_used ; c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + return unique_contexts; +} - if(i) buffer_strcat(wb, ", "); - buffer_strcat(wb, sq); - buffer_strcat(wb, string2str(qt->query.array[c].dimension.id)); - buffer_strcat(wb, sq); - i++; +static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, const char *key) { + char name[RRD_ID_LENGTH_MAX * 2 + 2]; + + buffer_json_member_add_array(wb, key); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); + + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdinstance_acquired_id(qi->ria), + rrdinstance_acquired_name(qi->ria)); + + bool *set = dictionary_set(dict, name, NULL, sizeof(*set)); + if (!*set) { + *set = true; + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_id(qi->ria)); + buffer_json_add_array_item_string(wb, rrdinstance_acquired_name(qi->ria)); + buffer_json_array_close(wb); + } } - if(!i) { - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); + dictionary_destroy(dict); + buffer_json_array_close(wb); +} + +static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key, struct summary_total_counts *totals) { + buffer_json_member_add_array(wb, key); + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); +// QUERY_HOST *qh = query_host(qt, qi->query_host_id); + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", rrdinstance_acquired_id(qi->ria)); + + if(!rrdinstance_acquired_id_and_name_are_same(qi->ria)) + buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(qi->ria)); + + buffer_json_member_add_uint64(wb, "ni", qi->query_host_id); +// buffer_json_member_add_string(wb, "id", string2str(qi->id_fqdn)); +// buffer_json_member_add_string(wb, "nm", string2str(qi->name_fqdn)); +// buffer_json_member_add_string(wb, "lc", rrdinstance_acquired_name(qi->ria)); +// buffer_json_member_add_string(wb, "mg", qh->host->machine_guid); +// if(qh->node_id[0]) +// buffer_json_member_add_string(wb, "nd", qh->node_id); + query_target_metric_counts(wb, &qi->metrics); + query_target_alerts_counts(wb, &qi->alerts, NULL, false); + query_target_points_statistics(wb, qt, &qi->query_points); + buffer_json_object_close(wb); + + aggregate_into_summary_totals(totals, &qi->metrics); } - buffer_strcat(wb, "],\n"); + buffer_json_array_close(wb); +} - if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) { - buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq); +struct dimensions_sorted_walkthrough_data { + BUFFER *wb; + struct summary_total_counts *totals; + QUERY_TARGET *qt; +}; - char name[RRD_ID_LENGTH_MAX * 2 + 2]; - char output[RRD_ID_LENGTH_MAX * 2 + 8]; +struct dimensions_sorted_entry { + const char *id; + const char *name; + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; + uint32_t priority; +}; - struct value_output co = {.c = 0, .wb = wb}; +static int dimensions_sorted_walktrhough_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + struct dimensions_sorted_walkthrough_data *sdwd = data; + BUFFER *wb = sdwd->wb; + struct summary_total_counts *totals = sdwd->totals; + QUERY_TARGET *qt = sdwd->qt; + struct dimensions_sorted_entry *z = value; - DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - for (c = 0; c < (long)qt->metrics.used ;c++) { - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", - rrdmetric_acquired_id(qt->metrics.array[c]), - rrdmetric_acquired_name(qt->metrics.array[c])); + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", z->id); + if (z->id != z->name && z->name) + buffer_json_member_add_string(wb, "nm", z->name); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", - rrdmetric_acquired_id(qt->metrics.array[c]), - rrdmetric_acquired_name(qt->metrics.array[c])); + query_target_metric_counts(wb, &z->metrics); + query_target_points_statistics(wb, qt, &z->query_points); + buffer_json_member_add_uint64(wb, "pri", z->priority); + buffer_json_object_close(wb); - dictionary_set(dict, name, output, len + 1); - } - dictionary_walkthrough_read(dict, value_list_output_callback, &co); - dictionary_destroy(dict); + aggregate_into_summary_totals(totals, &z->metrics); - co.c = 0; - buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq); - dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - for (c = 0; c < (long)qt->instances.used ; c++) { - RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; + return 1; +} - snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", - rrdinstance_acquired_id(ria), - rrdinstance_acquired_name(ria)); +int dimensions_sorted_compar(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2) { + struct dimensions_sorted_entry *z1 = dictionary_acquired_item_value(*item1); + struct dimensions_sorted_entry *z2 = dictionary_acquired_item_value(*item2); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", - rrdinstance_acquired_id(ria), - rrdinstance_acquired_name(ria)); + if(z1->priority == z2->priority) + return strcmp(dictionary_acquired_item_name(*item1), dictionary_acquired_item_name(*item2)); + else if(z1->priority < z2->priority) + return -1; + else + return 1; +} - dictionary_set(dict, name, output, len + 1); +static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *totals) { + char buf[RRD_ID_LENGTH_MAX * 2 + 2]; + + buffer_json_member_add_array(wb, key); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + struct dimensions_sorted_entry *z; + size_t q = 0; + for (long c = 0; c < (long) qt->dimensions.used; c++) { + QUERY_DIMENSION * qd = query_dimension(qt, c); + RRDMETRIC_ACQUIRED *rma = qd->rma; + + QUERY_METRIC *qm = NULL; + for( ; q < qt->query.used ;q++) { + QUERY_METRIC *tqm = query_metric(qt, q); + QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id); + if(tqd->rma != rma) break; + qm = tqm; } - dictionary_walkthrough_read(dict, value_list_output_callback, &co); - dictionary_destroy(dict); - co.c = 0; - buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq); - dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - for (c = 0; c < (long)qt->instances.used ; c++) { - RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; - rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), fill_formatted_callback, dict); + const char *key, *id, *name; + + if(v2) { + key = rrdmetric_acquired_name(rma); + id = key; + name = key; + } + else { + snprintfz(buf, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdmetric_acquired_id(rma), + rrdmetric_acquired_name(rma)); + key = buf; + id = rrdmetric_acquired_id(rma); + name = rrdmetric_acquired_name(rma); } - dictionary_walkthrough_read(dict, value_list_output_callback, &co); - dictionary_destroy(dict); - buffer_strcat(wb, "],\n"); - } - // functions - { - DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); - RRDINSTANCE_ACQUIRED *ria = NULL; - for (c = 0; c < query_used ; c++) { - QUERY_METRIC *qm = &qt->query.array[c]; - if(qm->link.ria == ria) - continue; + z = dictionary_set(dict, key, NULL, sizeof(*z)); + if(!z->id) { + z->id = id; + z->name = name; + z->priority = qd->priority; + } + else { + if(qd->priority < z->priority) + z->priority = qd->priority; + } - ria = qm->link.ria; - chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs); + if(qm) { + z->metrics.selected += (qm->status & RRDR_DIMENSION_SELECTED) ? 1 : 0; + z->metrics.failed += (qm->status & RRDR_DIMENSION_FAILED) ? 1 : 0; + + if(qm->status & RRDR_DIMENSION_QUERIED) { + z->metrics.queried++; + storage_point_merge_to(z->query_points, qm->query_points); + } } + else + z->metrics.excluded++; + } - buffer_sprintf(wb, " %sfunctions%s: [", kq, kq); - void *t; (void)t; - dfe_start_read(funcs, t) { - const char *comma = ""; - if(t_dfe.counter) comma = ", "; - buffer_sprintf(wb, "%s%s%s%s", comma, sq, t_dfe.name, sq); + if(v2) { + struct dimensions_sorted_walkthrough_data t = { + .wb = wb, + .totals = totals, + .qt = qt, + }; + dictionary_sorted_walkthrough_rw(dict, DICTIONARY_LOCK_READ, dimensions_sorted_walktrhough_cb, + &t, dimensions_sorted_compar); + } + else { + // v1 + dfe_start_read(dict, z) { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, z->id); + buffer_json_add_array_item_string(wb, z->name); + buffer_json_array_close(wb); } - dfe_done(t); - dictionary_destroy(funcs); - buffer_strcat(wb, "],\n"); + dfe_done(z); } + dictionary_destroy(dict); + buffer_json_array_close(wb); +} - // context query - if (!qt->request.st) { - buffer_sprintf( - wb, - " %schart_ids%s: [", - kq, kq); +struct rrdlabels_formatting_v2 { + DICTIONARY *keys; + QUERY_INSTANCE *qi; + bool v2; +}; - for (c = 0, i = 0; c < query_used; c++) { - QUERY_METRIC *qm = &qt->query.array[c]; +struct rrdlabels_keys_dict_entry { + const char *name; + DICTIONARY *values; + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; +}; - if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) - continue; +struct rrdlabels_key_value_dict_entry { + const char *key; + const char *value; + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; +}; - if (i) - buffer_strcat(wb, ", "); - buffer_strcat(wb, sq); - buffer_strcat(wb, string2str(qm->chart.id)); - buffer_strcat(wb, sq); - i++; - } - if (!i) { - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); - } - buffer_strcat(wb, "],\n"); - if (qt->instances.chart_label_key_pattern) { - buffer_sprintf(wb, " %schart_labels%s: { ", kq, kq); - - SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern; - char *label_key = NULL; - int keys = 0; - while (pattern && (label_key = simple_pattern_iterate(&pattern))) { - if (keys) - buffer_strcat(wb, ", "); - buffer_sprintf(wb, "%s%s%s : [", kq, label_key, kq); - keys++; - - for (c = 0, i = 0; c < query_used; c++) { - QUERY_METRIC *qm = &qt->query.array[c]; - - if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) - continue; +static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) { + struct rrdlabels_formatting_v2 *t = data; - if (i) - buffer_strcat(wb, ", "); - rrdlabels_get_value_to_buffer_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key, sq, "null"); - i++; + struct rrdlabels_keys_dict_entry *d = dictionary_set(t->keys, name, NULL, sizeof(*d)); + if(!d->values) { + d->name = name; + d->values = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + } + + char n[RRD_ID_LENGTH_MAX * 2 + 2]; + snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value); + + struct rrdlabels_key_value_dict_entry *z = dictionary_set(d->values, n, NULL, sizeof(*z)); + if(!z->key) { + z->key = name; + z->value = value; + } + + if(t->v2) { + QUERY_INSTANCE *qi = t->qi; + + z->metrics.selected += qi->metrics.selected; + z->metrics.excluded += qi->metrics.excluded; + z->metrics.queried += qi->metrics.queried; + z->metrics.failed += qi->metrics.failed; + + d->metrics.selected += qi->metrics.selected; + d->metrics.excluded += qi->metrics.excluded; + d->metrics.queried += qi->metrics.queried; + d->metrics.failed += qi->metrics.failed; + + storage_point_merge_to(z->query_points, qi->query_points); + storage_point_merge_to(d->query_points, qi->query_points); + } + + return 1; +} + +static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const char *key, bool v2, struct summary_total_counts *key_totals, struct summary_total_counts *value_totals) { + buffer_json_member_add_array(wb, key); + struct rrdlabels_formatting_v2 t = { + .keys = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE), + .v2 = v2, + }; + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); + RRDINSTANCE_ACQUIRED *ria = qi->ria; + t.qi = qi; + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), rrdlabels_formatting_v2, &t); + } + struct rrdlabels_keys_dict_entry *d; + dfe_start_read(t.keys, d) { + if(v2) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", d_dfe.name); + query_target_metric_counts(wb, &d->metrics); + query_target_points_statistics(wb, qt, &d->query_points); + aggregate_into_summary_totals(key_totals, &d->metrics); + buffer_json_member_add_array(wb, "vl"); } - if (!i) { - rows = 0; - buffer_strcat(wb, sq); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, sq); + struct rrdlabels_key_value_dict_entry *z; + dfe_start_read(d->values, z){ + if (v2) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", z->value); + query_target_metric_counts(wb, &z->metrics); + query_target_points_statistics(wb, qt, &z->query_points); + buffer_json_object_close(wb); + aggregate_into_summary_totals(value_totals, &z->metrics); + } else { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_string(wb, z->key); + buffer_json_add_array_item_string(wb, z->value); + buffer_json_array_close(wb); + } + } + dfe_done(z); + dictionary_destroy(d->values); + if(v2) { + buffer_json_array_close(wb); + buffer_json_object_close(wb); } - buffer_strcat(wb, "]"); } - buffer_strcat(wb, "},\n"); + dfe_done(d); + dictionary_destroy(t.keys); + buffer_json_array_close(wb); +} + +static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) { + buffer_json_member_add_array(wb, key); + QUERY_ALERTS_COUNTS *z; + + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + for (long c = 0; c < (long) qt->instances.used; c++) { + QUERY_INSTANCE *qi = query_instance(qt, c); + RRDSET *st = rrdinstance_acquired_rrdset(qi->ria); + if (st) { + netdata_rwlock_rdlock(&st->alerts.rwlock); + if (st->alerts.base) { + for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) { + z = dictionary_set(dict, string2str(rc->name), NULL, sizeof(*z)); + + switch(rc->status) { + case RRDCALC_STATUS_CLEAR: + z->clear++; + break; + + case RRDCALC_STATUS_WARNING: + z->warning++; + break; + + case RRDCALC_STATUS_CRITICAL: + z->critical++; + break; + + default: + case RRDCALC_STATUS_UNINITIALIZED: + case RRDCALC_STATUS_UNDEFINED: + case RRDCALC_STATUS_REMOVED: + z->other++; + break; + } + } + } + netdata_rwlock_unlock(&st->alerts.rwlock); + } + } + dfe_start_read(dict, z) + query_target_alerts_counts(wb, z, z_dfe.name, true); + dfe_done(z); + dictionary_destroy(dict); + buffer_json_array_close(wb); // alerts +} + +static inline void query_target_functions(BUFFER *wb, const char *key, RRDR *r) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + + DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + RRDINSTANCE_ACQUIRED *ria = NULL; + for (long c = 0; c < query_used ; c++) { + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + if(qi->ria == ria) + continue; + + ria = qi->ria; + chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs); + } + + buffer_json_member_add_array(wb, key); + void *t; (void)t; + dfe_start_read(funcs, t) + buffer_json_add_array_item_string(wb, t_dfe.name); + dfe_done(t); + dictionary_destroy(funcs); + buffer_json_array_close(wb); +} + +static inline long query_target_chart_labels_filter_v1(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i = 0; + + buffer_json_member_add_object(wb, key); + + SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern; + char *label_key = NULL; + while (pattern && (label_key = simple_pattern_iterate(&pattern))) { + buffer_json_member_add_array(wb, label_key); + + for (c = 0, i = 0; c < query_used; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + rrdlabels_value_to_buffer_array_item_or_null(rrdinstance_acquired_labels(qi->ria), wb, label_key); + i++; } + buffer_json_array_close(wb); } - buffer_sprintf(wb, " %slatest_values%s: [" - , kq, kq); + buffer_json_object_close(wb); + + return i; +} + +static inline long query_target_metrics_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + const long query_used = qt->query.used; + long c, i; + + buffer_json_member_add_array(wb, key); for(c = 0, i = 0; c < query_used ;c++) { - QUERY_METRIC *qm = &qt->query.array[c]; + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + QUERY_METRIC *qm = query_metric(qt, c); + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); + buffer_json_add_array_item_double(wb, rrdmetric_acquired_last_stored_value(qd->rma)); + i++; + } - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + buffer_json_array_close(wb); + + return i; +} + +static inline size_t rrdr_dimension_view_latest_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + buffer_json_member_add_array(wb, key); + + size_t c, i; + for(c = 0, i = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - if(i) buffer_strcat(wb, ", "); i++; - NETDATA_DOUBLE value = rrdmetric_acquired_last_stored_value(qm->link.rma); - if (NAN == value) - buffer_strcat(wb, "null"); + NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; + NETDATA_DOUBLE n = cn[c]; + + if(co[c] & RRDR_VALUE_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_json_add_array_item_double(wb, 0.0); + else + buffer_json_add_array_item_double(wb, NAN); + } else - buffer_rrd_value(wb, value); - } - if(!i) { - rows = 0; - buffer_strcat(wb, "null"); + buffer_json_add_array_item_double(wb, n); } - buffer_sprintf(wb, "],\n" - " %sview_latest_values%s: [" - , kq, kq); + buffer_json_array_close(wb); - i = 0; - if(rows) { - NETDATA_DOUBLE total = 1; + return i; +} - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - total = 0; - for(c = 0; c < query_used ;c++) { - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; +static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool dview) { + STORAGE_POINT *sp = (dview) ? r->dview : r->dqp; + NETDATA_DOUBLE anomaly_rate_multiplier = (dview) ? RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER : 1.0; - NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; - NETDATA_DOUBLE n = cn[c]; + if(unlikely(!sp)) + return; - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; + if(key) + buffer_json_member_add_object(wb, key); - total += n; - } - // prevent a division by zero - if(total == 0) total = 1; - } + buffer_json_member_add_array(wb, "min"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - for(c = 0, i = 0; c < query_used ;c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + buffer_json_add_array_item_double(wb, sp[c].min); + } + buffer_json_array_close(wb); - if(i) buffer_strcat(wb, ", "); - i++; + buffer_json_member_add_array(wb, "max"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; - RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; - NETDATA_DOUBLE n = cn[c]; + buffer_json_add_array_item_double(wb, sp[c].max); + } + buffer_json_array_close(wb); - if(co[c] & RRDR_VALUE_EMPTY) { - if(options & RRDR_OPTION_NULL2ZERO) - buffer_strcat(wb, "0"); - else - buffer_strcat(wb, "null"); - } - else { - if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; + if(options & RRDR_OPTION_RETURN_RAW) { + buffer_json_member_add_array(wb, "sum"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) - n = n * 100 / total; + buffer_json_add_array_item_double(wb, sp[c].sum); + } + buffer_json_array_close(wb); - buffer_rrd_value(wb, n); - } + buffer_json_member_add_array(wb, "cnt"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, sp[c].count); } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "arc"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier / 100.0 * sp[c].count); + } + buffer_json_array_close(wb); } - if(!i) { - rows = 0; - buffer_strcat(wb, "null"); - } + else { + NETDATA_DOUBLE sum = 0.0; + for(size_t c = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - buffer_sprintf(wb, "],\n" - " %sdimensions%s: %ld,\n" - " %spoints%s: %ld,\n" - " %sformat%s: %s" - , kq, kq, i - , kq, kq, rows - , kq, kq, sq - ); + sum += ABS(sp[c].sum); + } - rrdr_buffer_print_format(wb, format); + buffer_json_member_add_array(wb, "avg"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - buffer_sprintf(wb, "%s,\n" - " %sdb_points_per_tier%s: [ " - , sq - , kq, kq - ); + buffer_json_add_array_item_double(wb, storage_point_average_value(sp[c])); + } + buffer_json_array_close(wb); - for(size_t tier = 0; tier < storage_tiers ; tier++) - buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]); + buffer_json_member_add_array(wb, "arp"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - buffer_strcat(wb, " ],"); + buffer_json_add_array_item_double(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier); + } + buffer_json_array_close(wb); - if(options & RRDR_OPTION_SHOW_PLAN) - rrdr_show_plan(r, wb, kq, sq); + buffer_json_member_add_array(wb, "con"); + for(size_t c = 0; c < r->d ; c++) { + if (!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; - buffer_sprintf(wb, "\n %sresult%s: ", kq, kq); + NETDATA_DOUBLE con = (sum > 0.0) ? ABS(sp[c].sum) * 100.0 / sum : 0.0; + buffer_json_add_array_item_double(wb, con); + } + buffer_json_array_close(wb); + } - if(string_value) buffer_strcat(wb, sq); - //info("JSONWRAPPER(): %s: END", r->st->id); + if(key) + buffer_json_object_close(wb); } -void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) { - (void)r; - (void)format; +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + DATASOURCE_FORMAT format = qt->request.format; + RRDR_OPTIONS options = qt->window.options; + + long rows = rrdr_rows(r); char kq[2] = "", // key quote - sq[2] = ""; // string quote + sq[2] = ""; // string quote if( options & RRDR_OPTION_GOOGLE_JSON ) { kq[0] = '\0'; @@ -447,31 +874,702 @@ void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint3 sq[0] = '"'; } - if(string_value) buffer_strcat(wb, sq); + buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY); + + buffer_json_member_add_uint64(wb, "api", 1); + buffer_json_member_add_string(wb, "id", qt->id); + buffer_json_member_add_string(wb, "name", qt->id); + buffer_json_member_add_time_t(wb, "view_update_every", r->view.update_every); + buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s); + buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s); + buffer_json_member_add_time_t(wb, "after", r->view.after); + buffer_json_member_add_time_t(wb, "before", r->view.before); + buffer_json_member_add_string(wb, "group", time_grouping_tostring(qt->request.time_group_method)); + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options); + + if(!rrdr_dimension_names(wb, "dimension_names", r, options)) + rows = 0; + + if(!rrdr_dimension_ids(wb, "dimension_ids", r, options)) + rows = 0; + + if (options & RRDR_OPTION_ALL_DIMENSIONS) { + query_target_summary_instances_v1(wb, qt, "full_chart_list"); + query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false, NULL); + query_target_summary_labels_v12(wb, qt, "full_chart_labels", false, NULL, NULL); + } + + query_target_functions(wb, "functions", r); + + if (!qt->request.st && !jsonwrap_v1_chart_ids(wb, "chart_ids", r, options)) + rows = 0; + + if (qt->instances.chart_label_key_pattern && !query_target_chart_labels_filter_v1(wb, "chart_labels", r, options)) + rows = 0; + + if(!query_target_metrics_latest_values(wb, "latest_values", r, options)) + rows = 0; + + size_t dimensions = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options); + if(!dimensions) + rows = 0; + + buffer_json_member_add_uint64(wb, "dimensions", dimensions); + buffer_json_member_add_uint64(wb, "points", rows); + buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format)); + + buffer_json_member_add_array(wb, "db_points_per_tier"); + for(size_t tier = 0; tier < storage_tiers ; tier++) + buffer_json_add_array_item_uint64(wb, qt->db.tiers[tier].points); + buffer_json_array_close(wb); + + if(options & RRDR_OPTION_DEBUG) + jsonwrap_query_plan(r, wb); +} + +static void rrdset_rrdcalc_entries_v2(BUFFER *wb, RRDINSTANCE_ACQUIRED *ria) { + RRDSET *st = rrdinstance_acquired_rrdset(ria); + if(st) { + netdata_rwlock_rdlock(&st->alerts.rwlock); + if(st->alerts.base) { + buffer_json_member_add_object(wb, "alerts"); + for(RRDCALC *rc = st->alerts.base; rc ;rc = rc->next) { + if(rc->status < RRDCALC_STATUS_CLEAR) + continue; + + buffer_json_member_add_object(wb, string2str(rc->name)); + buffer_json_member_add_string(wb, "st", rrdcalc_status2string(rc->status)); + buffer_json_member_add_double(wb, "vl", rc->value); + buffer_json_member_add_string(wb, "un", string2str(rc->units)); + buffer_json_object_close(wb); + } + buffer_json_object_close(wb); + } + netdata_rwlock_unlock(&st->alerts.rwlock); + } +} + +static void query_target_combined_units_v2(BUFFER *wb, QUERY_TARGET *qt, size_t contexts, bool ignore_percentage) { + if(!ignore_percentage && query_target_has_percentage_units(qt)) { + buffer_json_member_add_string(wb, "units", "%"); + } + else if(contexts == 1) { + buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca)); + } + else if(contexts > 1) { + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + for(size_t c = 0; c < qt->contexts.used ;c++) + dictionary_set(dict, rrdcontext_acquired_units(qt->contexts.array[c].rca), NULL, 0); + + if(dictionary_entries(dict) == 1) + buffer_json_member_add_string(wb, "units", rrdcontext_acquired_units(qt->contexts.array[0].rca)); + else { + buffer_json_member_add_array(wb, "units"); + const char *s; + dfe_start_read(dict, s) + buffer_json_add_array_item_string(wb, s_dfe.name); + dfe_done(s); + buffer_json_array_close(wb); + } + dictionary_destroy(dict); + } +} - buffer_sprintf(wb, ",\n %sanomaly_rates%s: ", kq, kq); +static void query_target_combined_chart_type(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) { + if(contexts >= 1) + buffer_json_member_add_string(wb, "chart_type", rrdset_type_name(rrdcontext_acquired_chart_type(qt->contexts.array[0].rca))); } -void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) { - (void)format; +static void rrdr_grouped_by_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options __maybe_unused) { + QUERY_TARGET *qt = r->internal.qt; - char kq[2] = "", // key quote - sq[2] = ""; // string quote + buffer_json_member_add_array(wb, key); - if( options & RRDR_OPTION_GOOGLE_JSON ) { + // find the deeper group-by + ssize_t g = 0; + for(g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) + break; + } + + if(g > 0) + g--; + + RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by; + + if(group_by & RRDR_GROUP_BY_SELECTED) + buffer_json_add_array_item_string(wb, "selected"); + + else if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + buffer_json_add_array_item_string(wb, "percentage-of-instance"); + + else { + + if(group_by & RRDR_GROUP_BY_DIMENSION) + buffer_json_add_array_item_string(wb, "dimension"); + + if(group_by & RRDR_GROUP_BY_INSTANCE) + buffer_json_add_array_item_string(wb, "instance"); + + if(group_by & RRDR_GROUP_BY_LABEL) { + BUFFER *b = buffer_create(0, NULL); + for (size_t l = 0; l < qt->group_by[g].used; l++) { + buffer_flush(b); + buffer_fast_strcat(b, "label:", 6); + buffer_strcat(b, qt->group_by[g].label_keys[l]); + buffer_json_add_array_item_string(wb, buffer_tostring(b)); + } + buffer_free(b); + } + + if(group_by & RRDR_GROUP_BY_NODE) + buffer_json_add_array_item_string(wb, "node"); + + if(group_by & RRDR_GROUP_BY_CONTEXT) + buffer_json_add_array_item_string(wb, "context"); + + if(group_by & RRDR_GROUP_BY_UNITS) + buffer_json_add_array_item_string(wb, "units"); + } + + buffer_json_array_close(wb); // group_by_order +} + +static void rrdr_dimension_units_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool ignore_percentage) { + if(!r->du) + return; + + bool percentage = !ignore_percentage && query_target_has_percentage_units(r->internal.qt); + + buffer_json_member_add_array(wb, key); + for(size_t c = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + if(percentage) + buffer_json_add_array_item_string(wb, "%"); + else + buffer_json_add_array_item_string(wb, string2str(r->du[c])); + } + buffer_json_array_close(wb); +} + +static void rrdr_dimension_priority_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + if(!r->dp) + return; + + buffer_json_member_add_array(wb, key); + for(size_t c = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, r->dp[c]); + } + buffer_json_array_close(wb); +} + +static void rrdr_dimension_aggregated_array_v2(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + if(!r->dgbc) + return; + + buffer_json_member_add_array(wb, key); + for(size_t c = 0; c < r->d ;c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; + + buffer_json_add_array_item_uint64(wb, r->dgbc[c]); + } + buffer_json_array_close(wb); +} + +static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) { + if(contexts == 1) { + buffer_json_member_add_string(wb, "title", rrdcontext_acquired_title(qt->contexts.array[0].rca)); + } + else if(contexts > 1) { + BUFFER *t = buffer_create(0, NULL); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + + buffer_strcat(t, "Chart for contexts: "); + + size_t added = 0; + for(size_t c = 0; c < qt->contexts.used ;c++) { + bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), NULL, sizeof(*set)); + if(!*set) { + *set = true; + if(added) + buffer_fast_strcat(t, ", ", 2); + + buffer_strcat(t, rrdcontext_acquired_id(qt->contexts.array[c].rca)); + added++; + } + } + buffer_json_member_add_string(wb, "title", buffer_tostring(t)); + dictionary_destroy(dict); + buffer_free(t); + } +} + +static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS options) { + QUERY_TARGET *qt = r->internal.qt; + buffer_json_member_add_object(wb, "nodes"); + + time_t now_s = now_realtime_sec(); + RRDHOST *last_host = NULL; + RRDCONTEXT_ACQUIRED *last_rca = NULL; + RRDINSTANCE_ACQUIRED *last_ria = NULL; + + size_t h = 0, c = 0, i = 0, m = 0, q = 0; + for(; h < qt->nodes.used ; h++) { + QUERY_NODE *qn = query_node(qt, h); + RRDHOST *host = qn->rrdhost; + + for( ;c < qt->contexts.used ;c++) { + QUERY_CONTEXT *qc = query_context(qt, c); + RRDCONTEXT_ACQUIRED *rca = qc->rca; + if(!rrdcontext_acquired_belongs_to_host(rca, host)) break; + + for( ;i < qt->instances.used ;i++) { + QUERY_INSTANCE *qi = query_instance(qt, i); + RRDINSTANCE_ACQUIRED *ria = qi->ria; + if(!rrdinstance_acquired_belongs_to_context(ria, rca)) break; + + for( ; m < qt->dimensions.used ; m++) { + QUERY_DIMENSION *qd = query_dimension(qt, m); + RRDMETRIC_ACQUIRED *rma = qd->rma; + if(!rrdmetric_acquired_belongs_to_instance(rma, ria)) break; + + QUERY_METRIC *qm = NULL; + bool queried = false; + for( ; q < qt->query.used ;q++) { + QUERY_METRIC *tqm = query_metric(qt, q); + QUERY_DIMENSION *tqd = query_dimension(qt, tqm->link.query_dimension_id); + if(tqd->rma != rma) break; + + queried = tqm->status & RRDR_DIMENSION_QUERIED; + qm = tqm; + } + + if(!queried & !(options & RRDR_OPTION_ALL_DIMENSIONS)) + continue; + + if(host != last_host) { + if(last_host) { + if(last_rca) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + buffer_json_object_close(wb); // instances + buffer_json_object_close(wb); // context + last_rca = NULL; + } + buffer_json_object_close(wb); // contexts + buffer_json_object_close(wb); // host + last_host = NULL; + } + + buffer_json_member_add_object(wb, host->machine_guid); + if(qn->node_id[0]) + buffer_json_member_add_string(wb, "nd", qn->node_id); + buffer_json_member_add_uint64(wb, "ni", qn->slot); + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host)); + buffer_json_member_add_object(wb, "contexts"); + + last_host = host; + } + + if(rca != last_rca) { + if(last_rca) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + buffer_json_object_close(wb); // instances + buffer_json_object_close(wb); // context + last_rca = NULL; + } + + buffer_json_member_add_object(wb, rrdcontext_acquired_id(rca)); + buffer_json_member_add_object(wb, "instances"); + + last_rca = rca; + } + + if(ria != last_ria) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + + buffer_json_member_add_object(wb, rrdinstance_acquired_id(ria)); + buffer_json_member_add_string(wb, "nm", rrdinstance_acquired_name(ria)); + buffer_json_member_add_time_t(wb, "ue", rrdinstance_acquired_update_every(ria)); + DICTIONARY *labels = rrdinstance_acquired_labels(ria); + if(labels) { + buffer_json_member_add_object(wb, "labels"); + rrdlabels_to_buffer_json_members(labels, wb); + buffer_json_object_close(wb); + } + rrdset_rrdcalc_entries_v2(wb, ria); + buffer_json_member_add_object(wb, "dimensions"); + + last_ria = ria; + } + + buffer_json_member_add_object(wb, rrdmetric_acquired_id(rma)); + { + buffer_json_member_add_string(wb, "nm", rrdmetric_acquired_name(rma)); + buffer_json_member_add_uint64(wb, "qr", queried ? 1 : 0); + time_t first_entry_s = rrdmetric_acquired_first_entry(rma); + time_t last_entry_s = rrdmetric_acquired_last_entry(rma); + buffer_json_member_add_time_t(wb, "fe", first_entry_s); + buffer_json_member_add_time_t(wb, "le", last_entry_s ? last_entry_s : now_s); + + if(qm) { + if(qm->status & RRDR_DIMENSION_GROUPED) { + // buffer_json_member_add_string(wb, "grouped_as_id", string2str(qm->grouped_as.id)); + buffer_json_member_add_string(wb, "as", string2str(qm->grouped_as.name)); + } + + query_target_points_statistics(wb, qt, &qm->query_points); + + if(options & RRDR_OPTION_DEBUG) + jsonwrap_query_metric_plan(wb, qm); + } + } + buffer_json_object_close(wb); // metric + } + } + } + } + + if(last_host) { + if(last_rca) { + if(last_ria) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // instance + last_ria = NULL; + } + buffer_json_object_close(wb); // instances + buffer_json_object_close(wb); // context + last_rca = NULL; + } + buffer_json_object_close(wb); // contexts + buffer_json_object_close(wb); // host + last_host = NULL; + } + buffer_json_object_close(wb); // hosts +} + +void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) { + buffer_json_member_add_object(wb, "versions"); + buffer_json_member_add_uint64(wb, "nodes_hard_hash", dictionary_version(rrdhost_root_index)); + buffer_json_member_add_uint64(wb, "contexts_hard_hash", versions->contexts_hard_hash); + buffer_json_member_add_uint64(wb, "contexts_soft_hash", versions->contexts_soft_hash); + buffer_json_member_add_uint64(wb, "alerts_hard_hash", versions->alerts_hard_hash); + buffer_json_member_add_uint64(wb, "alerts_soft_hash", versions->alerts_soft_hash); + buffer_json_object_close(wb); +} + +void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + RRDR_OPTIONS options = qt->window.options; + + char kq[2] = "\"", // key quote + sq[2] = "\""; // string quote + + if(unlikely(options & RRDR_OPTION_GOOGLE_JSON)) { kq[0] = '\0'; sq[0] = '\''; } - else { - kq[0] = '"'; - sq[0] = '"'; + + buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_member_add_uint64(wb, "api", 2); + + if(options & RRDR_OPTION_DEBUG) { + buffer_json_member_add_string(wb, "id", qt->id); + buffer_json_member_add_object(wb, "request"); + { + buffer_json_member_add_string(wb, "format", rrdr_format_to_string(qt->request.format)); + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", qt->request.options); + + buffer_json_member_add_object(wb, "scope"); + buffer_json_member_add_string(wb, "scope_nodes", qt->request.scope_nodes); + buffer_json_member_add_string(wb, "scope_contexts", qt->request.scope_contexts); + buffer_json_object_close(wb); // scope + + buffer_json_member_add_object(wb, "selectors"); + if (qt->request.host) + buffer_json_member_add_string(wb, "nodes", rrdhost_hostname(qt->request.host)); + else + buffer_json_member_add_string(wb, "nodes", qt->request.nodes); + buffer_json_member_add_string(wb, "contexts", qt->request.contexts); + buffer_json_member_add_string(wb, "instances", qt->request.instances); + buffer_json_member_add_string(wb, "dimensions", qt->request.dimensions); + buffer_json_member_add_string(wb, "labels", qt->request.labels); + buffer_json_member_add_string(wb, "alerts", qt->request.alerts); + buffer_json_object_close(wb); // selectors + + buffer_json_member_add_object(wb, "window"); + buffer_json_member_add_time_t(wb, "after", qt->request.after); + buffer_json_member_add_time_t(wb, "before", qt->request.before); + buffer_json_member_add_uint64(wb, "points", qt->request.points); + if (qt->request.options & RRDR_OPTION_SELECTED_TIER) + buffer_json_member_add_uint64(wb, "tier", qt->request.tier); + else + buffer_json_member_add_string(wb, "tier", NULL); + buffer_json_object_close(wb); // window + + buffer_json_member_add_object(wb, "aggregations"); + { + buffer_json_member_add_object(wb, "time"); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method)); + buffer_json_member_add_string(wb, "time_group_options", qt->request.time_group_options); + if (qt->request.resampling_time > 0) + buffer_json_member_add_time_t(wb, "time_resampling", qt->request.resampling_time); + else + buffer_json_member_add_string(wb, "time_resampling", NULL); + buffer_json_object_close(wb); // time + + buffer_json_member_add_array(wb, "metrics"); + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) + break; + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_array(wb, "group_by"); + buffer_json_group_by_to_array(wb, qt->request.group_by[g].group_by); + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "group_by_label"); + for (size_t l = 0; l < qt->group_by[g].used; l++) + buffer_json_add_array_item_string(wb, qt->group_by[g].label_keys[l]); + buffer_json_array_close(wb); + + buffer_json_member_add_string( + wb, "aggregation",group_by_aggregate_function_to_string(qt->request.group_by[g].aggregation)); + } + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); // group_by + } + buffer_json_object_close(wb); // aggregations + + buffer_json_member_add_uint64(wb, "timeout", qt->request.timeout_ms); + } + buffer_json_object_close(wb); // request + } + + version_hashes_api_v2(wb, &qt->versions); + + buffer_json_member_add_object(wb, "summary"); + struct summary_total_counts + nodes_totals = { 0 }, + contexts_totals = { 0 }, + instances_totals = { 0 }, + metrics_totals = { 0 }, + label_key_totals = { 0 }, + label_key_value_totals = { 0 }; + { + query_target_summary_nodes_v2(wb, qt, "nodes", &nodes_totals); + r->internal.contexts = query_target_summary_contexts_v2(wb, qt, "contexts", &contexts_totals); + query_target_summary_instances_v2(wb, qt, "instances", &instances_totals); + query_target_summary_dimensions_v12(wb, qt, "dimensions", true, &metrics_totals); + query_target_summary_labels_v12(wb, qt, "labels", true, &label_key_totals, &label_key_value_totals); + query_target_summary_alerts_v2(wb, qt, "alerts"); + } + if(query_target_aggregatable(qt)) { + buffer_json_member_add_object(wb, "globals"); + query_target_points_statistics(wb, qt, &qt->query_points); + buffer_json_object_close(wb); // globals + } + buffer_json_object_close(wb); // summary + + buffer_json_member_add_object(wb, "totals"); + query_target_total_counts(wb, "nodes", &nodes_totals); + query_target_total_counts(wb, "contexts", &contexts_totals); + query_target_total_counts(wb, "instances", &instances_totals); + query_target_total_counts(wb, "dimensions", &metrics_totals); + query_target_total_counts(wb, "label_keys", &label_key_totals); + query_target_total_counts(wb, "label_key_values", &label_key_value_totals); + buffer_json_object_close(wb); // totals + + if(options & RRDR_OPTION_SHOW_DETAILS) { + buffer_json_member_add_object(wb, "detailed"); + query_target_detailed_objects_tree(wb, r, options); + buffer_json_object_close(wb); // detailed + } + + query_target_functions(wb, "functions", r); +} + +//static void annotations_range_for_value_flags(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options, RRDR_VALUE_FLAGS flags, const char *type) { +// const size_t dims = r->d, rows = r->rows; +// size_t next_d_idx = 0; +// for(size_t d = 0; d < dims ; d++) { +// if(!rrdr_dimension_should_be_exposed(r->od[d], options)) +// continue; +// +// size_t d_idx = next_d_idx++; +// +// size_t t = 0; +// while(t < rows) { +// +// // find the beginning +// time_t started = 0; +// for(; t < rows ;t++) { +// RRDR_VALUE_FLAGS o = r->o[t * r->d + d]; +// if(o & flags) { +// started = r->t[t]; +// break; +// } +// } +// +// if(started) { +// time_t ended = 0; +// for(; t < rows ;t++) { +// RRDR_VALUE_FLAGS o = r->o[t * r->d + d]; +// if(!(o & flags)) { +// ended = r->t[t]; +// break; +// } +// } +// +// if(!ended) +// ended = r->t[rows - 1]; +// +// buffer_json_add_array_item_object(wb); +// buffer_json_member_add_string(wb, "t", type); +// // buffer_json_member_add_string(wb, "d", string2str(r->dn[d])); +// buffer_json_member_add_uint64(wb, "d", d_idx); +// if(started == ended) { +// if(options & RRDR_OPTION_MILLISECONDS) +// buffer_json_member_add_time_t2ms(wb, "x", started); +// else +// buffer_json_member_add_time_t(wb, "x", started); +// } +// else { +// buffer_json_member_add_array(wb, "x"); +// if(options & RRDR_OPTION_MILLISECONDS) { +// buffer_json_add_array_item_time_t2ms(wb, started); +// buffer_json_add_array_item_time_t2ms(wb, ended); +// } +// else { +// buffer_json_add_array_item_time_t(wb, started); +// buffer_json_add_array_item_time_t(wb, ended); +// } +// buffer_json_array_close(wb); +// } +// buffer_json_object_close(wb); +// } +// } +// } +//} +// +//void rrdr_json_wrapper_annotations(RRDR *r, BUFFER *wb, DATASOURCE_FORMAT format __maybe_unused, RRDR_OPTIONS options) { +// buffer_json_member_add_array(wb, "annotations"); +// +// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_EMPTY, "G"); // Gap +// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_RESET, "O"); // Overflow +// annotations_range_for_value_flags(r, wb, format, options, RRDR_VALUE_PARTIAL, "P"); // Partial +// +// buffer_json_array_close(wb); // annotations +//} + +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) { + buffer_json_member_add_double(wb, "min", r->view.min); + buffer_json_member_add_double(wb, "max", r->view.max); + + buffer_json_query_timings(wb, "timings", &r->internal.qt->timings); + buffer_json_finalize(wb); +} + +void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) { + QUERY_TARGET *qt = r->internal.qt; + DATASOURCE_FORMAT format = qt->request.format; + RRDR_OPTIONS options = qt->window.options; + + buffer_json_member_add_object(wb, "db"); + { + buffer_json_member_add_uint64(wb, "tiers", storage_tiers); + buffer_json_member_add_time_t(wb, "update_every", qt->db.minimum_latest_update_every_s); + buffer_json_member_add_time_t(wb, "first_entry", qt->db.first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", qt->db.last_time_s); + + query_target_combined_units_v2(wb, qt, r->internal.contexts, true); + buffer_json_member_add_object(wb, "dimensions"); + { + rrdr_dimension_ids(wb, "ids", r, options); + rrdr_dimension_units_array_v2(wb, "units", r, options, true); + rrdr_dimension_query_points_statistics(wb, "sts", r, options, false); + } + buffer_json_object_close(wb); // dimensions + + buffer_json_member_add_array(wb, "per_tier"); + for(size_t tier = 0; tier < storage_tiers ; tier++) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tier", tier); + buffer_json_member_add_uint64(wb, "queries", qt->db.tiers[tier].queries); + buffer_json_member_add_uint64(wb, "points", qt->db.tiers[tier].points); + buffer_json_member_add_time_t(wb, "update_every", qt->db.tiers[tier].update_every); + buffer_json_member_add_time_t(wb, "first_entry", qt->db.tiers[tier].retention.first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", qt->db.tiers[tier].retention.last_time_s); + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); } + buffer_json_object_close(wb); - if(string_value) buffer_strcat(wb, sq); + buffer_json_member_add_object(wb, "view"); + { + query_target_title(wb, qt, r->internal.contexts); + buffer_json_member_add_time_t(wb, "update_every", r->view.update_every); + buffer_json_member_add_time_t(wb, "after", r->view.after); + buffer_json_member_add_time_t(wb, "before", r->view.before); + + if(options & RRDR_OPTION_DEBUG) { + buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format)); + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method)); + } + + if(options & RRDR_OPTION_DEBUG) { + buffer_json_member_add_object(wb, "partial_data_trimming"); + buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every); + buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after); + buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after); + buffer_json_object_close(wb); + } + + if(options & RRDR_OPTION_RETURN_RAW) + buffer_json_member_add_uint64(wb, "points", rrdr_rows(r)); + + query_target_combined_units_v2(wb, qt, r->internal.contexts, false); + query_target_combined_chart_type(wb, qt, r->internal.contexts); + buffer_json_member_add_object(wb, "dimensions"); + { + rrdr_grouped_by_array_v2(wb, "grouped_by", r, options); + rrdr_dimension_ids(wb, "ids", r, options); + rrdr_dimension_names(wb, "names", r, options); + rrdr_dimension_units_array_v2(wb, "units", r, options, false); + rrdr_dimension_priority_array_v2(wb, "priorities", r, options); + rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options); + rrdr_dimension_query_points_statistics(wb, "sts", r, options, true); + rrdr_json_group_by_labels(wb, "labels", r, options); + } + buffer_json_object_close(wb); // dimensions + buffer_json_member_add_double(wb, "min", r->view.min); + buffer_json_member_add_double(wb, "max", r->view.max); + } + buffer_json_object_close(wb); // view - buffer_sprintf(wb, ",\n %smin%s: ", kq, kq); - buffer_rrd_value(wb, r->min); - buffer_sprintf(wb, ",\n %smax%s: ", kq, kq); - buffer_rrd_value(wb, r->max); - buffer_strcat(wb, "\n}\n"); + buffer_json_agents_array_v2(wb, &r->internal.qt->timings, 0); + buffer_json_cloud_timings(wb, "timings", &r->internal.qt->timings); + buffer_json_finalize(wb); } diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h index 91c1475c5..a702f3a5c 100644 --- a/web/api/formatters/json_wrapper.h +++ b/web/api/formatters/json_wrapper.h @@ -6,10 +6,16 @@ #include "rrd2json.h" #include "web/api/queries/query.h" +typedef void (*wrapper_begin_t)(RRDR *r, BUFFER *wb); +typedef void (*wrapper_end_t)(RRDR *r, BUFFER *wb); -void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - RRDR_GROUPING group_method); -void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); -void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb); +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb); + +void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb); +void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb); + +struct query_versions; +void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions); #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 64cde5b2b..139fa6ec8 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -7,63 +7,55 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) { rrdset2json(st, wb, NULL, NULL, 0); } -void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) { +const char *rrdr_format_to_string(DATASOURCE_FORMAT format) { switch(format) { case DATASOURCE_JSON: - buffer_strcat(wb, DATASOURCE_FORMAT_JSON); - break; + return DATASOURCE_FORMAT_JSON; + + case DATASOURCE_JSON2: + return DATASOURCE_FORMAT_JSON2; case DATASOURCE_DATATABLE_JSON: - buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON); - break; + return DATASOURCE_FORMAT_DATATABLE_JSON; case DATASOURCE_DATATABLE_JSONP: - buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP); - break; + return DATASOURCE_FORMAT_DATATABLE_JSONP; case DATASOURCE_JSONP: - buffer_strcat(wb, DATASOURCE_FORMAT_JSONP); - break; + return DATASOURCE_FORMAT_JSONP; case DATASOURCE_SSV: - buffer_strcat(wb, DATASOURCE_FORMAT_SSV); - break; + return DATASOURCE_FORMAT_SSV; case DATASOURCE_CSV: - buffer_strcat(wb, DATASOURCE_FORMAT_CSV); - break; + return DATASOURCE_FORMAT_CSV; case DATASOURCE_TSV: - buffer_strcat(wb, DATASOURCE_FORMAT_TSV); - break; + return DATASOURCE_FORMAT_TSV; case DATASOURCE_HTML: - buffer_strcat(wb, DATASOURCE_FORMAT_HTML); - break; + return DATASOURCE_FORMAT_HTML; case DATASOURCE_JS_ARRAY: - buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY); - break; + return DATASOURCE_FORMAT_JS_ARRAY; case DATASOURCE_SSV_COMMA: - buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA); - break; + return DATASOURCE_FORMAT_SSV_COMMA; default: - buffer_strcat(wb, "unknown"); - break; + return "unknown"; } } int rrdset2value_api_v1( - RRDSET *st + RRDSET *st , BUFFER *wb , NETDATA_DOUBLE *n , const char *dimensions , size_t points , time_t after , time_t before - , RRDR_GROUPING group_method + , RRDR_TIME_GROUPING group_method , const char *group_options , time_t resampling_time , uint32_t options @@ -105,15 +97,15 @@ int rrdset2value_api_v1( } if(db_points_read) - *db_points_read += r->internal.db_points_read; + *db_points_read += r->stats.db_points_read; if(db_points_per_tier) { for(size_t t = 0; t < storage_tiers ;t++) - db_points_per_tier[t] += r->internal.tier_points_read[t]; + db_points_per_tier[t] += r->internal.qt->db.tiers[t].points; } if(result_points_generated) - *result_points_generated += r->internal.result_points_generated; + *result_points_generated += r->stats.result_points_generated; if(rrdr_rows(r) == 0) { if(db_after) *db_after = 0; @@ -125,14 +117,14 @@ int rrdset2value_api_v1( } if(wb) { - if (r->result_options & RRDR_RESULT_OPTION_RELATIVE) + if (r->view.flags & RRDR_RESULT_FLAG_RELATIVE) buffer_no_cacheable(wb); - else if (r->result_options & RRDR_RESULT_OPTION_ABSOLUTE) + else if (r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE) buffer_cacheable(wb); } - if(db_after) *db_after = r->after; - if(db_before) *db_before = r->before; + if(db_after) *db_after = r->view.after; + if(db_before) *db_before = r->view.before; long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; *n = rrdr2value(r, i, options, value_is_null, anomaly_rate); @@ -144,108 +136,141 @@ cleanup: return ret; } +static inline void buffer_json_member_add_key_only(BUFFER *wb, const char *key) { + buffer_print_json_comma_newline_spacing(wb); + buffer_print_json_key(wb, key); + buffer_fast_strcat(wb, ":", 1); + wb->json.stack[wb->json.depth].count++; +} + +static inline void buffer_json_member_add_string_open(BUFFER *wb, const char *key) { + buffer_json_member_add_key_only(wb, key); + buffer_strcat(wb, wb->json.value_quote); +} + +static inline void buffer_json_member_add_string_close(BUFFER *wb) { + buffer_strcat(wb, wb->json.value_quote); +} + int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) { + wrapper_begin_t wrapper_begin = rrdr_json_wrapper_begin; + wrapper_end_t wrapper_end = rrdr_json_wrapper_end; + + if(qt->request.version == 2) { + wrapper_begin = rrdr_json_wrapper_begin2; + wrapper_end = rrdr_json_wrapper_end2; + } RRDR *r = rrd2rrdr(owa, qt); + if(!r) { buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); return HTTP_RESP_INTERNAL_SERVER_ERROR; } - if (r->result_options & RRDR_RESULT_OPTION_CANCEL) { + if (r->view.flags & RRDR_RESULT_FLAG_CANCEL) { rrdr_free(owa, r); return HTTP_RESP_BACKEND_FETCH_FAILED; } - if(r->result_options & RRDR_RESULT_OPTION_RELATIVE) + if(r->view.flags & RRDR_RESULT_FLAG_RELATIVE) buffer_no_cacheable(wb); - else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE) + else if(r->view.flags & RRDR_RESULT_FLAG_ABSOLUTE) buffer_cacheable(wb); if(latest_timestamp && rrdr_rows(r) > 0) - *latest_timestamp = r->before; + *latest_timestamp = r->view.before; DATASOURCE_FORMAT format = qt->request.format; - RRDR_OPTIONS options = qt->request.options; - RRDR_GROUPING group_method = qt->request.group_method; + RRDR_OPTIONS options = qt->window.options; switch(format) { case DATASOURCE_SSV: if(options & RRDR_OPTION_JSON_WRAP) { - wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); rrdr2ssv(r, wb, options, "", " ", ""); - rrdr_json_wrapper_end(r, wb, format, options, 1); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_TEXT_PLAIN; + wb->content_type = CT_TEXT_PLAIN; rrdr2ssv(r, wb, options, "", " ", ""); } break; case DATASOURCE_SSV_COMMA: if(options & RRDR_OPTION_JSON_WRAP) { - wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); rrdr2ssv(r, wb, options, "", ",", ""); - rrdr_json_wrapper_end(r, wb, format, options, 1); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_TEXT_PLAIN; + wb->content_type = CT_TEXT_PLAIN; rrdr2ssv(r, wb, options, "", ",", ""); } break; case DATASOURCE_JS_ARRAY: if(options & RRDR_OPTION_JSON_WRAP) { - wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); - rrdr2ssv(r, wb, options, "[", ",", "]"); - rrdr_json_wrapper_end(r, wb, format, options, 0); + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_array(wb, "result"); + rrdr2ssv(r, wb, options, "", ",", ""); + buffer_json_array_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_APPLICATION_JSON; + wb->content_type = CT_APPLICATION_JSON; rrdr2ssv(r, wb, options, "[", ",", "]"); } break; case DATASOURCE_CSV: if(options & RRDR_OPTION_JSON_WRAP) { - wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); rrdr2csv(r, wb, format, options, "", ",", "\\n", ""); - rrdr_json_wrapper_end(r, wb, format, options, 1); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_TEXT_PLAIN; + wb->content_type = CT_TEXT_PLAIN; rrdr2csv(r, wb, format, options, "", ",", "\r\n", ""); } break; case DATASOURCE_CSV_MARKDOWN: if(options & RRDR_OPTION_JSON_WRAP) { - wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); rrdr2csv(r, wb, format, options, "", "|", "\\n", ""); - rrdr_json_wrapper_end(r, wb, format, options, 1); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_TEXT_PLAIN; + wb->content_type = CT_TEXT_PLAIN; rrdr2csv(r, wb, format, options, "", "|", "\r\n", ""); } break; case DATASOURCE_CSV_JSON_ARRAY: - wb->contenttype = CT_APPLICATION_JSON; + wb->content_type = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) { - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); - buffer_strcat(wb, "[\n"); + wrapper_begin(r, wb); + buffer_json_member_add_array(wb, "result"); rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); - buffer_strcat(wb, "\n]"); - rrdr_json_wrapper_end(r, wb, format, options, 0); + buffer_json_array_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_APPLICATION_JSON; + wb->content_type = CT_APPLICATION_JSON; buffer_strcat(wb, "[\n"); rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); buffer_strcat(wb, "\n]"); @@ -254,28 +279,32 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l case DATASOURCE_TSV: if(options & RRDR_OPTION_JSON_WRAP) { - wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); rrdr2csv(r, wb, format, options, "", "\t", "\\n", ""); - rrdr_json_wrapper_end(r, wb, format, options, 1); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_TEXT_PLAIN; + wb->content_type = CT_TEXT_PLAIN; rrdr2csv(r, wb, format, options, "", "\t", "\r\n", ""); } break; case DATASOURCE_HTML: if(options & RRDR_OPTION_JSON_WRAP) { - wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + buffer_json_member_add_string_open(wb, "result"); buffer_strcat(wb, "\\n
\\n\\n"); rrdr2csv(r, wb, format, options, "\\n", ""); buffer_strcat(wb, "
", "", "
\\n
\\n\\n"); - rrdr_json_wrapper_end(r, wb, format, options, 1); + buffer_json_member_add_string_close(wb); + wrapper_end(r, wb); } else { - wb->contenttype = CT_TEXT_HTML; + wb->content_type = CT_TEXT_HTML; buffer_strcat(wb, "\n
\n\n"); rrdr2csv(r, wb, format, options, "\n", ""); buffer_strcat(wb, "
", "", "
\n
\n\n"); @@ -283,57 +312,75 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l break; case DATASOURCE_DATATABLE_JSONP: - wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; + wb->content_type = CT_APPLICATION_X_JAVASCRIPT; - if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } rrdr2json(r, wb, options, 1); if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_end(r, wb, format, options, 0); + wrapper_end(r, wb); + break; case DATASOURCE_DATATABLE_JSON: - wb->contenttype = CT_APPLICATION_JSON; + wb->content_type = CT_APPLICATION_JSON; - if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } rrdr2json(r, wb, options, 1); if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_end(r, wb, format, options, 0); + wrapper_end(r, wb); + break; case DATASOURCE_JSONP: - wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; - if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); + wb->content_type = CT_APPLICATION_X_JAVASCRIPT; + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } rrdr2json(r, wb, options, 0); if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_end(r, wb, format, options, 0); + wrapper_end(r, wb); + break; case DATASOURCE_JSON: default: - wb->contenttype = CT_APPLICATION_JSON; + wb->content_type = CT_APPLICATION_JSON; - if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); + if(options & RRDR_OPTION_JSON_WRAP) { + wrapper_begin(r, wb); + buffer_json_member_add_key_only(wb, "result"); + } rrdr2json(r, wb, options, 0); if(options & RRDR_OPTION_JSON_WRAP) { - if(options & RRDR_OPTION_RETURN_JWAR) { - rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0); - rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0); + if (options & RRDR_OPTION_RETURN_JWAR) { + buffer_json_member_add_key_only(wb, "anomaly_rates"); + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, false); } - rrdr_json_wrapper_end(r, wb, format, options, 0); + wrapper_end(r, wb); } break; + + case DATASOURCE_JSON2: + wb->content_type = CT_APPLICATION_JSON; + wrapper_begin(r, wb); + rrdr2json_v2(r, wb); + wrapper_end(r, wb); + break; } rrdr_free(owa, r); diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index 88b9f773f..def26c754 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -3,6 +3,23 @@ #ifndef NETDATA_RRD2JSON_H #define NETDATA_RRD2JSON_H 1 +// type of JSON generations +typedef enum { + DATASOURCE_JSON = 0, + DATASOURCE_DATATABLE_JSON = 1, + DATASOURCE_DATATABLE_JSONP = 2, + DATASOURCE_SSV = 3, + DATASOURCE_CSV = 4, + DATASOURCE_JSONP = 5, + DATASOURCE_TSV = 6, + DATASOURCE_HTML = 7, + DATASOURCE_JS_ARRAY = 8, + DATASOURCE_SSV_COMMA = 9, + DATASOURCE_CSV_JSON_ARRAY = 10, + DATASOURCE_CSV_MARKDOWN = 11, + DATASOURCE_JSON2 = 12, +} DATASOURCE_FORMAT; + #include "web/api/web_api_v1.h" #include "web/api/exporters/allmetrics.h" @@ -23,23 +40,8 @@ #define API_RELATIVE_TIME_MAX (3 * 365 * 86400) -// type of JSON generations -typedef enum { - DATASOURCE_JSON = 0, - DATASOURCE_DATATABLE_JSON = 1, - DATASOURCE_DATATABLE_JSONP = 2, - DATASOURCE_SSV = 3, - DATASOURCE_CSV = 4, - DATASOURCE_JSONP = 5, - DATASOURCE_TSV = 6, - DATASOURCE_HTML = 7, - DATASOURCE_JS_ARRAY = 8, - DATASOURCE_SSV_COMMA = 9, - DATASOURCE_CSV_JSON_ARRAY = 10, - DATASOURCE_CSV_MARKDOWN = 11, -} DATASOURCE_FORMAT; - #define DATASOURCE_FORMAT_JSON "json" +#define DATASOURCE_FORMAT_JSON2 "json2" #define DATASOURCE_FORMAT_DATATABLE_JSON "datatable" #define DATASOURCE_FORMAT_DATATABLE_JSONP "datasource" #define DATASOURCE_FORMAT_JSONP "jsonp" @@ -53,19 +55,21 @@ typedef enum { #define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown" void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); -void rrdr_buffer_print_format(BUFFER *wb, uint32_t format); +const char *rrdr_format_to_string(DATASOURCE_FORMAT format); int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp); +void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options); + int rrdset2value_api_v1( - RRDSET *st + RRDSET *st , BUFFER *wb , NETDATA_DOUBLE *n , const char *dimensions , size_t points , time_t after , time_t before - , RRDR_GROUPING group_method + , RRDR_TIME_GROUPING group_method , const char *group_options , time_t resampling_time , uint32_t options @@ -82,4 +86,15 @@ int rrdset2value_api_v1( , STORAGE_PRIORITY priority ); +static inline bool rrdr_dimension_should_be_exposed(RRDR_DIMENSION_FLAGS rrdr_dim_flags, RRDR_OPTIONS options) { + if(unlikely(options & RRDR_OPTION_RETURN_RAW)) + return true; + + if(unlikely(rrdr_dim_flags & RRDR_DIMENSION_HIDDEN)) return false; + if(unlikely(!(rrdr_dim_flags & RRDR_DIMENSION_QUERIED))) return false; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(rrdr_dim_flags & RRDR_DIMENSION_NONZERO))) return false; + + return true; +} + #endif /* NETDATA_RRD2JSON_H */ diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c index 449d4ddf5..156f4486b 100644 --- a/web/api/formatters/rrdset2json.c +++ b/web/api/formatters/rrdset2json.c @@ -96,9 +96,9 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_strcat(wb, ",\n\t\t\t\t\""); else buffer_strcat(wb, "\t\t\t\t\""); - buffer_strcat_jsonescape(wb, rrddim_id(rd)); + buffer_json_strcat(wb, rrddim_id(rd)); buffer_strcat(wb, "\": { \"name\": \""); - buffer_strcat_jsonescape(wb, rrddim_name(rd)); + buffer_json_strcat(wb, rrddim_name(rd)); buffer_strcat(wb, "\" }"); dimensions++; @@ -112,9 +112,9 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor health_api_v1_chart_custom_variables2json(st, wb); buffer_strcat(wb, ",\n\t\t\t\"green\": "); - buffer_rrd_value(wb, st->green); + buffer_print_netdata_double(wb, st->green); buffer_strcat(wb, ",\n\t\t\t\"red\": "); - buffer_rrd_value(wb, st->red); + buffer_print_netdata_double(wb, st->red); if (likely(!skip_volatile)) { buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n"); diff --git a/web/api/formatters/ssv/README.md b/web/api/formatters/ssv/README.md index 4ca2a64ca..434d56721 100644 --- a/web/api/formatters/ssv/README.md +++ b/web/api/formatters/ssv/README.md @@ -1,6 +1,10 @@ # SSV formatter diff --git a/web/api/formatters/ssv/ssv.c b/web/api/formatters/ssv/ssv.c index d561980d9..65de0464b 100644 --- a/web/api/formatters/ssv/ssv.c +++ b/web/api/formatters/ssv/ssv.c @@ -20,12 +20,12 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL); if(likely(i != start)) { - if(r->min > v) r->min = v; - if(r->max < v) r->max = v; + if(r->view.min > v) r->view.min = v; + if(r->view.max < v) r->view.max = v; } else { - r->min = v; - r->max = v; + r->view.min = v; + r->view.max = v; } if(likely(i != start)) @@ -38,7 +38,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con buffer_strcat(wb, "null"); } else - buffer_rrd_value(wb, v); + buffer_print_netdata_double(wb, v); } buffer_strcat(wb, suffix); //info("RRD2SSV(): %s: END", r->st->id); diff --git a/web/api/formatters/value/README.md b/web/api/formatters/value/README.md index 5b75ded7c..5631d8207 100644 --- a/web/api/formatters/value/README.md +++ b/web/api/formatters/value/README.md @@ -1,6 +1,10 @@ # Value formatter diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c index fd9188057..1d07f62f6 100644 --- a/web/api/formatters/value/value.c +++ b/web/api/formatters/value/value.c @@ -4,9 +4,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) { - QUERY_TARGET *qt = r->internal.qt; - long c; - const long used = qt->query.used; + size_t c; NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; @@ -15,49 +13,15 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all NETDATA_DOUBLE sum = 0, min = 0, max = 0, v; int all_null = 1, init = 1; - NETDATA_DOUBLE total = 1; NETDATA_DOUBLE total_anomaly_rate = 0; - int set_min_max = 0; - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - total = 0; - for (c = 0; c < used; c++) { - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - NETDATA_DOUBLE n = cn[c]; - - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; - - total += n; - } - // prevent a division by zero - if(total == 0) total = 1; - set_min_max = 1; - } - // for each dimension - for (c = 0; c < used; c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if(unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + for (c = 0; c < r->d ; c++) { + if(!rrdr_dimension_should_be_exposed(r->od[c], options)) + continue; NETDATA_DOUBLE n = cn[c]; - if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) - n = -n; - - if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { - n = n * 100 / total; - - if(unlikely(set_min_max)) { - r->min = r->max = n; - set_min_max = 0; - } - - if(n < r->min) r->min = n; - if(n > r->max) r->max = n; - } - if(unlikely(init)) { if(n > 0) { min = 0; @@ -107,10 +71,11 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all QUERY_VALUE rrdmetric2value(RRDHOST *host, struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, time_t after, time_t before, - RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority ) { QUERY_TARGET_REQUEST qtr = { + .version = 1, .host = host, .rca = rca, .ria = ria, @@ -119,16 +84,17 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host, .before = before, .points = 1, .options = options, - .group_method = group_method, - .group_options = group_options, + .time_group_method = time_group_method, + .time_group_options = time_group_options, .tier = tier, - .timeout = timeout, + .timeout_ms = timeout, .query_source = query_source, .priority = priority, }; ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); - RRDR *r = rrd2rrdr(owa, query_target_create(&qtr)); + QUERY_TARGET *qt = query_target_create(&qtr); + RRDR *r = rrd2rrdr(owa, qt); QUERY_VALUE qv; @@ -136,18 +102,37 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host, qv = (QUERY_VALUE) { .value = NAN, .anomaly_rate = NAN, + .sp = { + .count = 0, + .min = NAN, + .max = NAN, + .sum = NAN, + .anomaly_count = 0, + }, + .duration_ut = (r) ? r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut : 0, }; } else { qv = (QUERY_VALUE) { - .after = r->after, - .before = r->before, - .points_read = r->internal.db_points_read, - .result_points = r->internal.result_points_generated, + .after = r->view.after, + .before = r->view.before, + .points_read = r->stats.db_points_read, + .result_points = r->stats.result_points_generated, + .sp = { + .count = 0, + }, + .duration_ut = r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut, }; + for(size_t d = 0; d < r->internal.qt->query.used ;d++) { + if(!rrdr_dimension_should_be_exposed(r->internal.qt->query.array[d].status, options)) + continue; + + storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points); + } + for(size_t t = 0; t < storage_tiers ;t++) - qv.storage_points_per_tier[t] = r->internal.tier_points_read[t]; + qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points; long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; int all_values_are_null = 0; @@ -159,6 +144,7 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host, } rrdr_free(owa, r); + query_target_release(qt); onewayalloc_destroy(owa); return qv; diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h index 3f7f51ccb..072ca14f8 100644 --- a/web/api/formatters/value/value.h +++ b/web/api/formatters/value/value.h @@ -13,6 +13,8 @@ typedef struct storage_value { size_t points_read; size_t storage_points_per_tier[RRD_STORAGE_TIERS]; size_t result_points; + STORAGE_POINT sp; + usec_t duration_ut; } QUERY_VALUE; struct rrdmetric_acquired; @@ -22,7 +24,7 @@ struct rrdcontext_acquired; QUERY_VALUE rrdmetric2value(RRDHOST *host, struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, time_t after, time_t before, - RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier, time_t timeout, QUERY_SOURCE query_source, STORAGE_PRIORITY priority ); -- cgit v1.2.3