From 3c315f0fff93aa072472abc10815963ac0035268 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 12 Aug 2022 09:26:11 +0200 Subject: Adding upstream version 1.36.0. Signed-off-by: Daniel Baumann --- web/api/formatters/csv/csv.c | 10 ++-- web/api/formatters/json/json.c | 21 +++++--- web/api/formatters/json_wrapper.c | 102 ++++++++++++++++++++++++------------- web/api/formatters/json_wrapper.h | 5 +- web/api/formatters/rrd2json.c | 104 +++++++++++++++++++++++++++++--------- web/api/formatters/rrd2json.h | 14 +++-- web/api/formatters/rrdset2json.c | 24 +++------ web/api/formatters/ssv/ssv.c | 2 +- web/api/formatters/value/value.c | 25 +++++---- web/api/formatters/value/value.h | 2 +- 10 files changed, 206 insertions(+), 103 deletions(-) (limited to 'web/api/formatters') diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c index da0a6b583..6d87ca374 100644 --- a/web/api/formatters/csv/csv.c +++ b/web/api/formatters/csv/csv.c @@ -63,9 +63,9 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const } // for each line in the array - calculated_number total = 1; + NETDATA_DOUBLE total = 1; for(i = start; i != end ;i += step) { - calculated_number *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; buffer_strcat(wb, betweenlines); @@ -75,7 +75,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { // print the timestamp of the line - buffer_rrd_value(wb, (calculated_number)now); + buffer_rrd_value(wb, (NETDATA_DOUBLE)now); // in ms if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); } @@ -90,7 +90,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { - calculated_number n = cn[c]; + NETDATA_DOUBLE n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) n = -n; @@ -109,7 +109,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const buffer_strcat(wb, separator); - calculated_number n = cn[c]; + NETDATA_DOUBLE n = cn[c]; if(co[c] & RRDR_VALUE_EMPTY) { if(options & RRDR_OPTION_NULL2ZERO) diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c index af1156d27..6f07b9aa4 100644 --- a/web/api/formatters/json/json.c +++ b/web/api/formatters/json/json.c @@ -158,10 +158,11 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct ); // for each line in the array - calculated_number total = 1; + NETDATA_DOUBLE total = 1; for(i = start; i != end ;i += step) { - calculated_number *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; time_t now = r->t[i]; @@ -209,7 +210,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); - buffer_rrd_value(wb, (calculated_number)r->t[i]); + buffer_rrd_value(wb, (NETDATA_DOUBLE)r->t[i]); // in ms if(unlikely(options & RRDR_OPTION_MILLISECONDS)) @@ -222,7 +223,11 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { - calculated_number n = cn[c]; + NETDATA_DOUBLE n; + if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) + n = ar[c]; + else + n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) n = -n; @@ -239,14 +244,18 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; - calculated_number n = cn[c]; + NETDATA_DOUBLE n; + if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) + n = ar[c]; + else + n = cn[c]; buffer_fast_strcat(wb, pre_value, pre_value_len); if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq); - if(co[c] & RRDR_VALUE_EMPTY) { + if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) { if(unlikely(options & RRDR_OPTION_NULL2ZERO)) buffer_fast_strcat(wb, "0", 1); else diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index 7097a5b77..04cace2fb 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -19,8 +19,23 @@ static int value_list_output(const char *name, void *entry, void *data) { return 0; } +static int fill_formatted_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { + (void)ls; + DICTIONARY *dict = (DICTIONARY *)data; + char n[RRD_ID_LENGTH_MAX * 2 + 2]; + char output[RRD_ID_LENGTH_MAX * 2 + 8]; + char v[RRD_ID_LENGTH_MAX * 2 + 1]; + + sanitize_json_string(v, (char *)value, RRD_ID_LENGTH_MAX * 2); + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", name, v); + snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, v); + dictionary_set(dict, n, output, len + 1); + + return 1; +} + void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - QUERY_PARAMS *rrdset_query_data) + RRDR_GROUPING group_method, QUERY_PARAMS *rrdset_query_data) { struct context_param *context_param_list = rrdset_query_data->context_param_list; char *chart_label_key = rrdset_query_data->chart_label_key; @@ -61,7 +76,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS " %slast_entry%s: %u,\n" " %sbefore%s: %u,\n" " %safter%s: %u,\n" - " %sdimension_names%s: [" + " %sgroup%s: %s%s%s,\n" + " %soptions%s: %s" , kq, kq , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq @@ -71,7 +87,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t_nolock(r->st)) , kq, kq, (uint32_t)r->before , kq, kq, (uint32_t)r->after - , kq, kq); + , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq + , kq, kq, sq); + + web_client_api_request_v1_data_options_to_string(wb, r->internal.query_options); + + buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq); + if (should_lock) rrdset_unlock(r->st); @@ -122,7 +144,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS char name[RRD_ID_LENGTH_MAX * 2 + 2]; char output[RRD_ID_LENGTH_MAX * 2 + 8]; - char value[RRD_ID_LENGTH_MAX * 2 + 1]; struct value_output co = {.c = 0, .wb = wb}; @@ -153,19 +174,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { st = rd->rrdset; - if (likely(st->state)) { - struct label_index *labels = &st->state->labels; - if (labels->head) { - netdata_rwlock_rdlock(&labels->labels_rwlock); - for (struct label *label = labels->head; label; label = label->next) { - sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value); - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value); - dictionary_set(dict, name, output, len + 1); - } - netdata_rwlock_unlock(&labels->labels_rwlock); - } - } + if (st->state && st->state->chart_labels) + rrdlabels_walkthrough_read(st->state->chart_labels, fill_formatted_callback, dict); } dictionary_walkthrough_read(dict, value_list_output, &co); dictionary_destroy(dict); @@ -207,8 +217,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS char *label_key = NULL; int keys = 0; while (pattern && (label_key = simple_pattern_iterate(&pattern))) { - uint32_t key_hash = simple_hash(label_key); - struct label *current_label; if (keys) buffer_strcat(wb, ", "); @@ -223,13 +231,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if (i) buffer_strcat(wb, ", "); - current_label = rrdset_lookup_label_key(rd->rrdset, label_key, key_hash); - if (current_label) { - buffer_strcat(wb, sq); - buffer_strcat(wb, current_label->value); - buffer_strcat(wb, sq); - } else - buffer_strcat(wb, "null"); + rrdlabels_get_value_to_buffer_or_null(rd->rrdset->state->chart_labels, wb, label_key, sq, "null"); i++; } if (!i) { @@ -255,7 +257,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if(i) buffer_strcat(wb, ", "); i++; - calculated_number value = rd->last_stored_value; + NETDATA_DOUBLE value = rd->last_stored_value; if (NAN == value) buffer_strcat(wb, "null"); else @@ -280,13 +282,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS i = 0; if(rows) { - calculated_number total = 1; + NETDATA_DOUBLE total = 1; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { - calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; - calculated_number n = cn[c]; + NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + NETDATA_DOUBLE n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) n = -n; @@ -304,9 +306,9 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if(i) buffer_strcat(wb, ", "); i++; - calculated_number *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; + NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ (rrdr_rows(r) - 1) * r->d ]; - calculated_number n = cn[c]; + NETDATA_DOUBLE n = cn[c]; if(co[c] & RRDR_VALUE_EMPTY) { if(options & RRDR_OPTION_NULL2ZERO) @@ -341,12 +343,21 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS rrdr_buffer_print_format(wb, format); + buffer_sprintf(wb, "%s,\n" + " %sdb_points_per_tier%s: [ " + , sq + , kq, kq + ); + + for(int tier = 0; tier < storage_tiers ; tier++) + buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]); + + buffer_strcat(wb, " ]"); + if((options & RRDR_OPTION_CUSTOM_VARS) && (options & RRDR_OPTION_JSON_WRAP)) { - buffer_sprintf(wb, "%s,\n %schart_variables%s: ", sq, kq, kq); + buffer_sprintf(wb, ",\n %schart_variables%s: ", kq, kq); health_api_v1_chart_custom_variables2json(r->st, wb); } - else - buffer_sprintf(wb, "%s", sq); buffer_sprintf(wb, ",\n %sresult%s: ", kq, kq); @@ -354,6 +365,27 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS //info("JSONWRAPPER(): %s: END", r->st->id); } +void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) { + (void)r; + (void)format; + + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + if(string_value) buffer_strcat(wb, sq); + + buffer_sprintf(wb, ",\n %sanomaly_rates%s: ", kq, kq); +} + void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) { (void)format; diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h index 65dbd5b65..bfadc883e 100644 --- a/web/api/formatters/json_wrapper.h +++ b/web/api/formatters/json_wrapper.h @@ -4,9 +4,12 @@ #define NETDATA_API_FORMATTER_JSON_WRAPPER_H #include "rrd2json.h" +#include "web/api/queries/query.h" + extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - QUERY_PARAMS *query_params); + RRDR_GROUPING group_method, QUERY_PARAMS *query_params); +extern void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 1de6be4e3..7aa478d95 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "web/api/web_api_v1.h" +#include "database/storage_engine.h" static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode) { @@ -18,7 +19,18 @@ static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int arc } } - onewayalloc_freez(owa, temp_rd->state); + for(int tier = 0; tier < storage_tiers ;tier++) { + if(!temp_rd->tiers[tier]) continue; + + if(archive_mode) { + STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode); + if (eng) + eng->api.free(temp_rd->tiers[tier]->db_metric_handle); + } + + onewayalloc_freez(owa, temp_rd->tiers[tier]); + } + onewayalloc_freez(owa, temp_rd); } @@ -50,10 +62,22 @@ void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_ RRDDIM *temp_rd = context_param_list->rd; RRDDIM *new_rd_list = NULL, *t; int is_archived = (context_param_list->flags & CONTEXT_FLAGS_ARCHIVE); + + RRDSET *st = temp_rd->rrdset; + RRDSET *last_st = st; + time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st); + time_t last_last_entry_t = last_entry_t; while (temp_rd) { t = temp_rd->next; - RRDSET *st = temp_rd->rrdset; - time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st); + + st = temp_rd->rrdset; + if (st == last_st) { + last_entry_t = last_last_entry_t; + }else { + last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st); + last_last_entry_t = last_entry_t; + last_st = st; + } if (last_entry_t >= after_requested) { temp_rd->next = new_rd_list; @@ -86,10 +110,15 @@ void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_lis (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); rrddim_foreach_read(rd1, st) { - RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize); + RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM)); rd->id = onewayalloc_strdupz(owa, rd1->id); rd->name = onewayalloc_strdupz(owa, rd1->name); - rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state)); + for(int tier = 0; tier < storage_tiers ;tier++) { + if(rd1->tiers[tier]) + rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier])); + else + rd->tiers[tier] = NULL; + } rd->next = (*param_list)->rd; (*param_list)->rd = rd; } @@ -152,24 +181,32 @@ void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) { int rrdset2value_api_v1( RRDSET *st , BUFFER *wb - , calculated_number *n + , NETDATA_DOUBLE *n , const char *dimensions , long points , long long after , long long before , int group_method + , const char *group_options , long group_time , uint32_t options , time_t *db_after , time_t *db_before + , size_t *db_points_read + , size_t *db_points_per_tier + , size_t *result_points_generated , int *value_is_null + , NETDATA_DOUBLE *anomaly_rate , int timeout + , int tier ) { int ret = HTTP_RESP_INTERNAL_SERVER_ERROR; ONEWAYALLOC *owa = onewayalloc_create(0); - RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout); + RRDR *r = rrd2rrdr(owa, st, points, after, before, + group_method, group_time, options, dimensions, NULL, + group_options, timeout, tier); if(!r) { if(value_is_null) *value_is_null = 1; @@ -177,9 +214,18 @@ int rrdset2value_api_v1( goto cleanup; } - if(rrdr_rows(r) == 0) { - rrdr_free(owa, r); + if(db_points_read) + *db_points_read += r->internal.db_points_read; + + if(db_points_per_tier) { + for(int t = 0; t < storage_tiers ;t++) + db_points_per_tier[t] += r->internal.tier_points_read[t]; + } + + if(result_points_generated) + *result_points_generated += r->internal.result_points_generated; + if(rrdr_rows(r) == 0) { if(db_after) *db_after = 0; if(db_before) *db_before = 0; if(value_is_null) *value_is_null = 1; @@ -199,7 +245,7 @@ int rrdset2value_api_v1( if(db_before) *db_before = r->before; long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0; - *n = rrdr2value(r, i, options, value_is_null, NULL); + *n = rrdr2value(r, i, options, value_is_null, anomaly_rate, NULL); ret = HTTP_RESP_OK; cleanup: @@ -218,9 +264,11 @@ int rrdset2anything_api_v1( , long long after , long long before , int group_method + , const char *group_options , long group_time , uint32_t options , time_t *latest_timestamp + , int tier ) { BUFFER *wb = query_params->wb; @@ -238,7 +286,8 @@ int rrdset2anything_api_v1( options, dimensions ? buffer_tostring(dimensions) : NULL, query_params->context_param_list, - query_params->timeout); + group_options, + query_params->timeout, tier); if(!r) { buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); return HTTP_RESP_INTERNAL_SERVER_ERROR; @@ -249,7 +298,7 @@ int rrdset2anything_api_v1( return HTTP_RESP_BACKEND_FETCH_FAILED; } - if (st && st->state && st->state->is_ar_chart) + if (st->state && st->state->is_ar_chart) ml_process_rrdr(r, query_params->max_anomaly_rates); RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL; @@ -266,7 +315,7 @@ int rrdset2anything_api_v1( case DATASOURCE_SSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); rrdr2ssv(r, wb, options, "", " ", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -279,7 +328,7 @@ int rrdset2anything_api_v1( case DATASOURCE_SSV_COMMA: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); rrdr2ssv(r, wb, options, "", ",", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -292,7 +341,7 @@ int rrdset2anything_api_v1( case DATASOURCE_JS_ARRAY: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 0); } @@ -305,7 +354,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -318,7 +367,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV_MARKDOWN: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -331,7 +380,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV_JSON_ARRAY: wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) { - rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); buffer_strcat(wb, "[\n"); rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd); buffer_strcat(wb, "\n]"); @@ -348,7 +397,7 @@ int rrdset2anything_api_v1( case DATASOURCE_TSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -361,7 +410,7 @@ int rrdset2anything_api_v1( case DATASOURCE_HTML: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); buffer_strcat(wb, "\\n
\\n\\n"); rrdr2csv(r, wb, format, options, "\\n", "", temp_rd); buffer_strcat(wb, "
", "", "
\\n
\\n\\n"); @@ -379,7 +428,7 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); rrdr2json(r, wb, options, 1, query_params->context_param_list); @@ -391,7 +440,7 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); rrdr2json(r, wb, options, 1, query_params->context_param_list); @@ -402,7 +451,7 @@ int rrdset2anything_api_v1( case DATASOURCE_JSONP: wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); rrdr2json(r, wb, options, 0, query_params->context_param_list); @@ -415,12 +464,17 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); rrdr2json(r, wb, options, 0, query_params->context_param_list); - if(options & RRDR_OPTION_JSON_WRAP) + if(options & RRDR_OPTION_JSON_WRAP) { + if(options & RRDR_OPTION_RETURN_JWAR) { + rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0); + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0, query_params->context_param_list); + } rrdr_json_wrapper_end(r, wb, format, options, 0); + } break; } diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index 60bed5b90..6be53ff8a 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -67,33 +67,41 @@ extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format); extern int rrdset2anything_api_v1( ONEWAYALLOC *owa , RRDSET *st - , - QUERY_PARAMS *query_params, BUFFER *dimensions + , QUERY_PARAMS *query_params + , BUFFER *dimensions , uint32_t format , long points , long long after , long long before , int group_method + , const char *group_options , long group_time , uint32_t options , time_t *latest_timestamp + , int tier ); extern int rrdset2value_api_v1( RRDSET *st , BUFFER *wb - , calculated_number *n + , NETDATA_DOUBLE *n , const char *dimensions , long points , long long after , long long before , int group_method + , const char *group_options , long group_time , uint32_t options , time_t *db_after , time_t *db_before + , size_t *db_points_read + , size_t *db_points_per_tier + , size_t *result_points_generated , int *value_is_null + , NETDATA_DOUBLE *anomaly_rate , int timeout + , int tier ); extern void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st); diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c index c83b22e63..de8d87bae 100644 --- a/web/api/formatters/rrdset2json.c +++ b/web/api/formatters/rrdset2json.c @@ -4,32 +4,22 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation) { + if(unlikely(!st->state || !st->state->chart_labels)) + return; + char tabs[11]; - struct label_index *labels = &st->state->labels; if (indentation > 10) indentation = 10; tabs[0] = '\0'; while (indentation) { - strcat(tabs, "\t"); + strcat(tabs, "\t\t"); indentation--; } - int count = 0; - netdata_rwlock_rdlock(&labels->labels_rwlock); - for (struct label *label = labels->head; label; label = label->next) { - if(count > 0) buffer_strcat(wb, ",\n"); - buffer_strcat(wb, tabs); - - char value[CONFIG_MAX_VALUE * 2 + 1]; - sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2); - buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value); - - count++; - } + rrdlabels_to_buffer(st->state->chart_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL); buffer_strcat(wb, "\n"); - netdata_rwlock_unlock(&labels->labels_rwlock); } // generate JSON for the /api/v1/chart API call @@ -95,14 +85,14 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\"dimensions\": {\n", st->update_every); - unsigned long memory = st->memsize; + unsigned long memory = sizeof(RRDSET) + st->memsize; size_t dimensions = 0; RRDDIM *rd; rrddim_foreach_read(rd, st) { if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; - memory += rd->memsize; + memory += sizeof(RRDDIM) + rd->memsize; if (dimensions) buffer_strcat(wb, ",\n\t\t\t\t\""); diff --git a/web/api/formatters/ssv/ssv.c b/web/api/formatters/ssv/ssv.c index 8d3ddbfdf..850182da1 100644 --- a/web/api/formatters/ssv/ssv.c +++ b/web/api/formatters/ssv/ssv.c @@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con // for each line in the array for(i = start; i != end ;i += step) { int all_values_are_null = 0; - calculated_number v = rrdr2value(r, i, options, &all_values_are_null, temp_rd); + NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd); if(likely(i != start)) { if(r->min > v) r->min = v; diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c index 9ac91f509..30e00c068 100644 --- a/web/api/formatters/value/value.c +++ b/web/api/formatters/value/value.c @@ -3,25 +3,25 @@ #include "value.h" -inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, RRDDIM *temp_rd) { - if (r->st_needs_lock) - rrdset_check_rdlock(r->st); - +inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd) { long c; RRDDIM *d; - calculated_number *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; - calculated_number sum = 0, min = 0, max = 0, v; + NETDATA_DOUBLE sum = 0, min = 0, max = 0, v; int all_null = 1, init = 1; - calculated_number total = 1; + NETDATA_DOUBLE total = 1; + NETDATA_DOUBLE total_anomaly_rate = 0; + int set_min_max = 0; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) { - calculated_number n = cn[c]; + NETDATA_DOUBLE n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) n = -n; @@ -38,7 +38,7 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int * if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; - calculated_number n = cn[c]; + NETDATA_DOUBLE n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) n = -n; @@ -74,6 +74,13 @@ inline calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int * if(n < min) min = n; if(n > max) max = n; + + total_anomaly_rate += ar[c]; + } + + if(anomaly_rate) { + if(!r->d) *anomaly_rate = 0; + else *anomaly_rate = total_anomaly_rate / r->d; } if(unlikely(all_null)) { diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h index 2d6bd1242..fc1c7bf08 100644 --- a/web/api/formatters/value/value.h +++ b/web/api/formatters/value/value.h @@ -5,6 +5,6 @@ #include "../rrd2json.h" -extern calculated_number rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, RRDDIM *temp_rd); +extern NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd); #endif //NETDATA_API_FORMATTER_VALUE_H -- cgit v1.2.3