From 03bf87dcb06f7021bfb2df2fa8691593c6148aff Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 30 Nov 2022 19:47:00 +0100 Subject: Adding upstream version 1.37.0. Signed-off-by: Daniel Baumann --- web/api/formatters/charts2json.c | 37 +++-- web/api/formatters/charts2json.h | 6 +- web/api/formatters/csv/csv.c | 17 ++- web/api/formatters/csv/csv.h | 2 +- web/api/formatters/json/json.c | 30 ++--- web/api/formatters/json/json.h | 2 +- web/api/formatters/json_wrapper.c | 192 +++++++++++++------------- web/api/formatters/json_wrapper.h | 8 +- web/api/formatters/rrd2json.c | 276 +++++++++----------------------------- web/api/formatters/rrd2json.h | 79 ++++------- web/api/formatters/rrdset2json.c | 52 +++---- web/api/formatters/rrdset2json.h | 2 +- web/api/formatters/ssv/ssv.c | 4 +- web/api/formatters/ssv/ssv.h | 2 +- web/api/formatters/value/value.c | 70 +++++++++- web/api/formatters/value/value.h | 23 +++- 16 files changed, 357 insertions(+), 445 deletions(-) (limited to 'web/api/formatters') diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c index 4325b6530..1fc20b493 100644 --- a/web/api/formatters/charts2json.c +++ b/web/api/formatters/charts2json.c @@ -57,11 +57,11 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived ",\n\t\"memory_mode\": \"%s\"" ",\n\t\"custom_info\": \"%s\"" ",\n\t\"charts\": {" - , host->hostname - , host->program_version + , rrdhost_hostname(host) + , rrdhost_program_version(host) , get_release_channel() - , host->os - , host->timezone + , rrdhost_os(host) + , rrdhost_timezone(host) , host->rrd_update_every , host->rrd_history_entries , rrd_memory_mode_name(host->rrd_memory_mode) @@ -69,12 +69,11 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived ); c = 0; - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) { if(c) buffer_strcat(wb, ","); buffer_strcat(wb, "\n\t\t\""); - buffer_strcat(wb, st->id); + buffer_strcat(wb, rrdset_id(st)); buffer_strcat(wb, "\": "); rrdset2json(st, wb, &dimensions, &memory, skip_volatile); @@ -82,13 +81,14 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived st->last_accessed_time = now; } } + rrdset_foreach_done(st); RRDCALC *rc; - for(rc = host->alarms; rc ; rc = rc->next) { + foreach_rrdcalc_in_rrdhost_read(host, rc) { if(rc->rrdset) alarms++; } - rrdhost_unlock(host); + foreach_rrdcalc_in_rrdhost_done(rc); buffer_sprintf(wb , "\n\t}" @@ -117,7 +117,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived "\n\t\t\t\"hostname\": \"%s\"" "\n\t\t}" , (found > 0) ? "," : "" - , h->hostname + , rrdhost_hostname(h) ); found++; @@ -131,7 +131,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived , "\n\t\t{" "\n\t\t\t\"hostname\": \"%s\"" "\n\t\t}" - , host->hostname + , rrdhost_hostname(host) ); } @@ -141,8 +141,8 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived // generate collectors list for the api/v1/info call struct collector { - char *plugin; - char *module; + const char *plugin; + const char *module; }; struct array_printer { @@ -150,9 +150,7 @@ struct array_printer { BUFFER *wb; }; -static int print_collector_callback(const char *name, void *entry, void *data) { - (void)name; - +static int print_collector_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { struct array_printer *ap = (struct array_printer *)data; BUFFER *wb = ap->wb; struct collector *col=(struct collector *) entry; @@ -167,24 +165,23 @@ static int print_collector_callback(const char *name, void *entry, void *data) { } void chartcollectors2json(RRDHOST *host, BUFFER *wb) { - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED); RRDSET *st; char name[500]; time_t now = now_realtime_sec(); - rrdhost_rdlock(host); rrdset_foreach_read(st, host) { if (rrdset_is_available_for_viewers(st)) { struct collector col = { - .plugin = st->plugin_name ? st->plugin_name : "", - .module = st->module_name ? st->module_name : "" + .plugin = rrdset_plugin_name(st), + .module = rrdset_module_name(st) }; sprintf(name, "%s:%s", col.plugin, col.module); dictionary_set(dict, name, &col, sizeof(struct collector)); st->last_accessed_time = now; } } - rrdhost_unlock(host); + rrdset_foreach_done(st); struct array_printer ap = { .c = 0, .wb = wb diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h index 2d8cce310..d4b04af58 100644 --- a/web/api/formatters/charts2json.h +++ b/web/api/formatters/charts2json.h @@ -5,8 +5,8 @@ #include "rrd2json.h" -extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived); -extern void chartcollectors2json(RRDHOST *host, BUFFER *wb); -extern const char* get_release_channel(); +void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived); +void chartcollectors2json(RRDHOST *host, BUFFER *wb); +const char* get_release_channel(); #endif //NETDATA_API_FORMATTER_CHARTS2JSON_H diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c index 6d87ca374..603a17169 100644 --- a/web/api/formatters/csv/csv.c +++ b/web/api/formatters/csv/csv.c @@ -3,15 +3,14 @@ #include "libnetdata/libnetdata.h" #include "csv.h" -void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd) { - rrdset_check_rdlock(r->st); - +void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) { //info("RRD2CSV(): %s: BEGIN", r->st->id); + QUERY_TARGET *qt = r->internal.qt; long c, i; - RRDDIM *d; + const long used = qt->query.used; // print the csv header - for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + for(c = 0, i = 0; c < used ; c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; @@ -23,7 +22,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const } buffer_strcat(wb, separator); if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); - buffer_strcat(wb, d->name); + buffer_strcat(wb, string2str(qt->query.array[c].dimension.name)); if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); i++; } @@ -31,7 +30,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const if(format == DATASOURCE_CSV_MARKDOWN) { // print the --- line after header - for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + for(c = 0, i = 0; c < used ;c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; @@ -89,7 +88,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const int set_min_max = 0; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + for(c = 0; c < used ;c++) { NETDATA_DOUBLE n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) @@ -103,7 +102,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const } // for each dimension - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) { + for(c = 0; c < used ;c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; diff --git a/web/api/formatters/csv/csv.h b/web/api/formatters/csv/csv.h index cf6020de4..666d4c660 100644 --- a/web/api/formatters/csv/csv.h +++ b/web/api/formatters/csv/csv.h @@ -5,7 +5,7 @@ #include "web/api/queries/rrdr.h" -extern void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd); +void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines); #include "../rrd2json.h" diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c index 6f07b9aa4..608150cba 100644 --- a/web/api/formatters/json/json.c +++ b/web/api/formatters/json/json.c @@ -5,15 +5,7 @@ #define JSON_DATES_JS 1 #define JSON_DATES_TIMESTAMP 2 -void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list) -{ - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; - - int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); - - if (should_lock) - rrdset_check_rdlock(r->st); - +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) { //info("RRD2JSON(): %s: BEGIN", r->st->id); int row_annotations = 0, dates, dates_with_new = 0; char kq[2] = "", // key quote @@ -112,21 +104,21 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct // ------------------------------------------------------------------------- // print the JSON header + QUERY_TARGET *qt = r->internal.qt; long c, i; - RRDDIM *rd; + const long used = qt->query.used; // print the header lines - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0, i = 0; c < used ; c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; buffer_fast_strcat(wb, pre_label, pre_label_len); - buffer_strcat(wb, rd->name); -// buffer_strcat(wb, "."); -// buffer_strcat(wb, rd->rrdset->name); + buffer_strcat(wb, string2str(qt->query.array[c].dimension.name)); buffer_fast_strcat(wb, post_label, post_label_len); i++; } + if(!i) { buffer_fast_strcat(wb, pre_label, pre_label_len); buffer_fast_strcat(wb, "no data", 7); @@ -134,7 +126,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct } size_t total_number_of_dimensions = i; - // print the begin of row data + // print the beginning of row data buffer_strcat(wb, data_begin); // if all dimensions are hidden, print a null @@ -187,7 +179,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct if(unlikely(row_annotations)) { // google supports one annotation per row int annotation_found = 0; - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd ;c++, rd = rd->next) { + for(c = 0; c < used ; c++) { if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue; if(unlikely(co[c] & RRDR_VALUE_RESET)) { @@ -222,7 +214,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct int set_min_max = 0; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0; c < used ;c++) { NETDATA_DOUBLE n; if(unlikely(options & RRDR_OPTION_INTERNAL_AR)) n = ar[c]; @@ -240,7 +232,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct } // for each dimension - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0; c < used ;c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; @@ -253,7 +245,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct buffer_fast_strcat(wb, pre_value, pre_value_len); if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) - buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq); + buffer_sprintf(wb, "%s%s%s: ", kq, string2str(qt->query.array[c].dimension.name), kq); if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) { if(unlikely(options & RRDR_OPTION_NULL2ZERO)) diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h index 5c4e11371..fb59e5c9a 100644 --- a/web/api/formatters/json/json.h +++ b/web/api/formatters/json/json.h @@ -5,6 +5,6 @@ #include "../rrd2json.h" -extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list); +void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable); #endif //NETDATA_API_FORMATTER_JSON_H diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index 04cace2fb..8b9b7522c 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -7,9 +7,7 @@ struct value_output { BUFFER *wb; }; -static int value_list_output(const char *name, void *entry, void *data) { - (void)name; - +static int value_list_output_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) { struct value_output *ap = (struct value_output *)data; BUFFER *wb = ap->wb; char *output = (char *) entry; @@ -35,25 +33,17 @@ static int fill_formatted_callback(const char *name, const char *value, RRDLABEL } void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - RRDR_GROUPING group_method, QUERY_PARAMS *rrdset_query_data) + RRDR_GROUPING group_method) { - struct context_param *context_param_list = rrdset_query_data->context_param_list; - char *chart_label_key = rrdset_query_data->chart_label_key; - - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; - int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); - uint8_t context_mode = (!context_param_list || (context_param_list->flags & CONTEXT_FLAGS_CONTEXT)); - - if (should_lock) - rrdset_check_rdlock(r->st); + QUERY_TARGET *qt = r->internal.qt; long rows = rrdr_rows(r); long c, i; - RRDDIM *rd; + const long query_used = qt->query.used; //info("JSONWRAPPER(): %s: BEGIN", r->st->id); char kq[2] = "", // key quote - sq[2] = ""; // string quote + sq[2] = ""; // string quote if( options & RRDR_OPTION_GOOGLE_JSON ) { kq[0] = '\0'; @@ -64,52 +54,47 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS sq[0] = '"'; } - if (should_lock) - rrdset_rdlock(r->st); buffer_sprintf(wb, "{\n" " %sapi%s: 1,\n" " %sid%s: %s%s%s,\n" " %sname%s: %s%s%s,\n" - " %sview_update_every%s: %d,\n" - " %supdate_every%s: %d,\n" - " %sfirst_entry%s: %u,\n" - " %slast_entry%s: %u,\n" - " %sbefore%s: %u,\n" - " %safter%s: %u,\n" + " %sview_update_every%s: %lld,\n" + " %supdate_every%s: %lld,\n" + " %sfirst_entry%s: %lld,\n" + " %slast_entry%s: %lld,\n" + " %sbefore%s: %lld,\n" + " %safter%s: %lld,\n" " %sgroup%s: %s%s%s,\n" " %soptions%s: %s" , kq, kq - , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq - , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq - , kq, kq, r->update_every - , kq, kq, r->st->update_every - , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t_nolock(r->st)) - , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t_nolock(r->st)) - , kq, kq, (uint32_t)r->before - , kq, kq, (uint32_t)r->after + , kq, kq, sq, qt->id, sq + , kq, kq, sq, qt->id, sq + , kq, kq, (long long)r->update_every + , kq, kq, (long long)qt->db.minimum_latest_update_every + , kq, kq, (long long)qt->db.first_time_t + , kq, kq, (long long)qt->db.last_time_t + , kq, kq, (long long)r->before + , kq, kq, (long long)r->after , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq , kq, kq, sq); - web_client_api_request_v1_data_options_to_string(wb, r->internal.query_options); + web_client_api_request_v1_data_options_to_buffer(wb, r->internal.query_options); buffer_sprintf(wb, "%s,\n %sdimension_names%s: [", sq, kq, kq); - if (should_lock) - rrdset_unlock(r->st); - - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0, i = 0; c < query_used ; c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); - buffer_strcat(wb, rd->name); + buffer_strcat(wb, string2str(qt->query.array[c].dimension.name)); buffer_strcat(wb, sq); i++; } if(!i) { #ifdef NETDATA_INTERNAL_CHECKS - error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options); + error("QUERY: '%s', RRDR is empty, %zu dimensions, options is 0x%08x", qt->id, r->d, options); #endif rows = 0; buffer_strcat(wb, sq); @@ -121,13 +106,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS " %sdimension_ids%s: [" , kq, kq); - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0, i = 0; c < query_used ; c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); - buffer_strcat(wb, rd->id); + buffer_strcat(wb, string2str(qt->query.array[c].dimension.id)); buffer_strcat(wb, sq); i++; } @@ -139,7 +124,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS } buffer_strcat(wb, "],\n"); - if (rrdset_query_data->show_dimensions) { + if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) { buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq); char name[RRD_ID_LENGTH_MAX * 2 + 2]; @@ -147,58 +132,97 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS struct value_output co = {.c = 0, .wb = wb}; - DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); - for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name); - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name); - dictionary_set(dict, name, output, len+1); + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + for (c = 0; c < (long)qt->metrics.used ;c++) { + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdmetric_acquired_id(qt->metrics.array[c]), + rrdmetric_acquired_name(qt->metrics.array[c])); + + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", + rrdmetric_acquired_id(qt->metrics.array[c]), + rrdmetric_acquired_name(qt->metrics.array[c])); + + dictionary_set(dict, name, output, len + 1); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); co.c = 0; buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); - for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name); - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + for (c = 0; c < (long)qt->instances.used ; c++) { + RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; + + snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s", + rrdinstance_acquired_id(ria), + rrdinstance_acquired_name(ria)); + + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", + rrdinstance_acquired_id(ria), + rrdinstance_acquired_name(ria)); + dictionary_set(dict, name, output, len + 1); } - - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); - RRDSET *st; co.c = 0; buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); - for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { - st = rd->rrdset; - if (st->state && st->state->chart_labels) - rrdlabels_walkthrough_read(st->state->chart_labels, fill_formatted_callback, dict); + dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + for (c = 0; c < (long)qt->instances.used ; c++) { + RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c]; + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), fill_formatted_callback, dict); } - dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_walkthrough_read(dict, value_list_output_callback, &co); dictionary_destroy(dict); buffer_strcat(wb, "],\n"); } - // Composite charts - if (context_mode && temp_rd) { + // functions + { + DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + RRDINSTANCE_ACQUIRED *ria = NULL; + for (c = 0; c < query_used ; c++) { + QUERY_METRIC *qm = &qt->query.array[c]; + if(qm->link.ria == ria) + continue; + + ria = qm->link.ria; + chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs); + } + + buffer_sprintf(wb, " %sfunctions%s: [", kq, kq); + void *t; (void)t; + dfe_start_read(funcs, t) { + const char *comma = ""; + if(t_dfe.counter) comma = ", "; + buffer_sprintf(wb, "%s%s%s%s", comma, sq, t_dfe.name, sq); + } + dfe_done(t); + dictionary_destroy(funcs); + buffer_strcat(wb, "],\n"); + } + + // context query + if (!qt->request.st) { buffer_sprintf( wb, " %schart_ids%s: [", kq, kq); - for (c = 0, i = 0, rd = temp_rd ; rd && c < r->d; c++, rd = rd->next) { + for (c = 0, i = 0; c < query_used; c++) { + QUERY_METRIC *qm = &qt->query.array[c]; + if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if (i) buffer_strcat(wb, ", "); buffer_strcat(wb, sq); - buffer_strcat(wb, rd->rrdset->id); + buffer_strcat(wb, string2str(qm->chart.id)); buffer_strcat(wb, sq); i++; } @@ -209,29 +233,29 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS buffer_strcat(wb, sq); } buffer_strcat(wb, "],\n"); - if (chart_label_key) { + if (qt->instances.chart_label_key_pattern) { buffer_sprintf(wb, " %schart_labels%s: { ", kq, kq); - SIMPLE_PATTERN *pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT); - SIMPLE_PATTERN *original_pattern = pattern; + SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern; char *label_key = NULL; int keys = 0; while (pattern && (label_key = simple_pattern_iterate(&pattern))) { - if (keys) buffer_strcat(wb, ", "); buffer_sprintf(wb, "%s%s%s : [", kq, label_key, kq); keys++; - for (c = 0, i = 0, rd = temp_rd; rd && c < r->d; c++, rd = rd->next) { + for (c = 0, i = 0; c < query_used; c++) { + QUERY_METRIC *qm = &qt->query.array[c]; + if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; + if (i) buffer_strcat(wb, ", "); - - rrdlabels_get_value_to_buffer_or_null(rd->rrdset->state->chart_labels, wb, label_key, sq, "null"); + rrdlabels_get_value_to_buffer_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key, sq, "null"); i++; } if (!i) { @@ -243,33 +267,26 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS buffer_strcat(wb, "]"); } buffer_strcat(wb, "},\n"); - simple_pattern_free(original_pattern); } } buffer_sprintf(wb, " %slatest_values%s: [" , kq, kq); - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0, i = 0; c < query_used ;c++) { + QUERY_METRIC *qm = &qt->query.array[c]; + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; if(i) buffer_strcat(wb, ", "); i++; - NETDATA_DOUBLE value = rd->last_stored_value; + NETDATA_DOUBLE value = rrdmetric_acquired_last_stored_value(qm->link.rma); if (NAN == value) buffer_strcat(wb, "null"); else buffer_rrd_value(wb, value); - /* - storage_number n = rd->values[rrdset_last_slot(r->st)]; - - if(!does_storage_number_exist(n)) - buffer_strcat(wb, "null"); - else - buffer_rrd_value(wb, unpack_storage_number(n)); - */ } if(!i) { rows = 0; @@ -286,7 +303,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; - for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0; c < query_used ;c++) { NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ]; NETDATA_DOUBLE n = cn[c]; @@ -299,7 +316,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS if(total == 0) total = 1; } - for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + for(c = 0, i = 0; c < query_used ;c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; @@ -349,16 +366,11 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS , kq, kq ); - for(int tier = 0; tier < storage_tiers ; tier++) + for(size_t tier = 0; tier < storage_tiers ; tier++) buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]); buffer_strcat(wb, " ]"); - if((options & RRDR_OPTION_CUSTOM_VARS) && (options & RRDR_OPTION_JSON_WRAP)) { - buffer_sprintf(wb, ",\n %schart_variables%s: ", kq, kq); - health_api_v1_chart_custom_variables2json(r->st, wb); - } - buffer_sprintf(wb, ",\n %sresult%s: ", kq, kq); if(string_value) buffer_strcat(wb, sq); diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h index bfadc883e..91c1475c5 100644 --- a/web/api/formatters/json_wrapper.h +++ b/web/api/formatters/json_wrapper.h @@ -7,9 +7,9 @@ #include "web/api/queries/query.h" -extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - RRDR_GROUPING group_method, QUERY_PARAMS *query_params); -extern void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); -extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, + RRDR_GROUPING group_method); +void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 7aa478d95..8bf547192 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -3,129 +3,6 @@ #include "web/api/web_api_v1.h" #include "database/storage_engine.h" -static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode) -{ - if (unlikely(!temp_rd)) - return; - - onewayalloc_freez(owa, (char *)temp_rd->id); - - if (unlikely(archive_mode)) { - temp_rd->rrdset->counter--; - if (!temp_rd->rrdset->counter) { - onewayalloc_freez(owa, (char *)temp_rd->rrdset->name); - onewayalloc_freez(owa, temp_rd->rrdset->context); - onewayalloc_freez(owa, temp_rd->rrdset); - } - } - - for(int tier = 0; tier < storage_tiers ;tier++) { - if(!temp_rd->tiers[tier]) continue; - - if(archive_mode) { - STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode); - if (eng) - eng->api.free(temp_rd->tiers[tier]->db_metric_handle); - } - - onewayalloc_freez(owa, temp_rd->tiers[tier]); - } - - onewayalloc_freez(owa, temp_rd); -} - -static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode) -{ - if (unlikely(!temp_rd)) - return; - - RRDDIM *t; - while (temp_rd) { - t = temp_rd->next; - free_single_rrdrim(owa, temp_rd, archive_mode); - temp_rd = t; - } -} - -void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list) -{ - if (unlikely(!param_list || !*param_list)) - return; - - free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE); - onewayalloc_freez(owa, (*param_list)); - *param_list = NULL; -} - -void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested) -{ - RRDDIM *temp_rd = context_param_list->rd; - RRDDIM *new_rd_list = NULL, *t; - int is_archived = (context_param_list->flags & CONTEXT_FLAGS_ARCHIVE); - - RRDSET *st = temp_rd->rrdset; - RRDSET *last_st = st; - time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st); - time_t last_last_entry_t = last_entry_t; - while (temp_rd) { - t = temp_rd->next; - - st = temp_rd->rrdset; - if (st == last_st) { - last_entry_t = last_last_entry_t; - }else { - last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st); - last_last_entry_t = last_entry_t; - last_st = st; - } - - if (last_entry_t >= after_requested) { - temp_rd->next = new_rd_list; - new_rd_list = temp_rd; - } else - free_single_rrdrim(owa, temp_rd, is_archived); - temp_rd = t; - } - context_param_list->rd = new_rd_list; -}; - -void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st) -{ - if (unlikely(!param_list || !st)) - return; - - if (unlikely(!(*param_list))) { - *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param)); - (*param_list)->first_entry_t = LONG_MAX; - (*param_list)->last_entry_t = 0; - (*param_list)->flags = CONTEXT_FLAGS_CONTEXT; - (*param_list)->rd = NULL; - } - - RRDDIM *rd1; - st->last_accessed_time = now_realtime_sec(); - rrdset_rdlock(st); - - (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t_nolock(st)); - (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); - - rrddim_foreach_read(rd1, st) { - RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM)); - rd->id = onewayalloc_strdupz(owa, rd1->id); - rd->name = onewayalloc_strdupz(owa, rd1->name); - for(int tier = 0; tier < storage_tiers ;tier++) { - if(rd1->tiers[tier]) - rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier])); - else - rd->tiers[tier] = NULL; - } - rd->next = (*param_list)->rd; - (*param_list)->rd = rd; - } - - rrdset_unlock(st); -} - void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) { rrdset2json(st, wb, NULL, NULL, 0); } @@ -183,12 +60,12 @@ int rrdset2value_api_v1( , BUFFER *wb , NETDATA_DOUBLE *n , const char *dimensions - , long points - , long long after - , long long before - , int group_method + , size_t points + , time_t after + , time_t before + , RRDR_GROUPING group_method , const char *group_options - , long group_time + , time_t resampling_time , uint32_t options , time_t *db_after , time_t *db_before @@ -197,16 +74,27 @@ int rrdset2value_api_v1( , size_t *result_points_generated , int *value_is_null , NETDATA_DOUBLE *anomaly_rate - , int timeout - , int tier + , time_t timeout + , size_t tier + , QUERY_SOURCE query_source ) { int ret = HTTP_RESP_INTERNAL_SERVER_ERROR; ONEWAYALLOC *owa = onewayalloc_create(0); - - RRDR *r = rrd2rrdr(owa, st, points, after, before, - group_method, group_time, options, dimensions, NULL, - group_options, timeout, tier); + RRDR *r = rrd2rrdr_legacy( + owa, + st, + points, + after, + before, + group_method, + resampling_time, + options, + dimensions, + group_options, + timeout, + tier, + query_source); if(!r) { if(value_is_null) *value_is_null = 1; @@ -218,7 +106,7 @@ int rrdset2value_api_v1( *db_points_read += r->internal.db_points_read; if(db_points_per_tier) { - for(int t = 0; t < storage_tiers ;t++) + for(size_t t = 0; t < storage_tiers ;t++) db_points_per_tier[t] += r->internal.tier_points_read[t]; } @@ -244,50 +132,19 @@ int rrdset2value_api_v1( if(db_after) *db_after = r->after; if(db_before) *db_before = r->before; - long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0; - *n = rrdr2value(r, i, options, value_is_null, anomaly_rate, NULL); + long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; + *n = rrdr2value(r, i, options, value_is_null, anomaly_rate); ret = HTTP_RESP_OK; cleanup: - if(r) rrdr_free(owa, r); + rrdr_free(owa, r); onewayalloc_destroy(owa); return ret; } -int rrdset2anything_api_v1( - ONEWAYALLOC *owa - , RRDSET *st - , QUERY_PARAMS *query_params - , BUFFER *dimensions - , uint32_t format - , long points - , long long after - , long long before - , int group_method - , const char *group_options - , long group_time - , uint32_t options - , time_t *latest_timestamp - , int tier -) -{ - BUFFER *wb = query_params->wb; - if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) - st->last_accessed_time = now_realtime_sec(); - - RRDR *r = rrd2rrdr( - owa, - st, - points, - after, - before, - group_method, - group_time, - options, - dimensions ? buffer_tostring(dimensions) : NULL, - query_params->context_param_list, - group_options, - query_params->timeout, tier); +int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) { + + RRDR *r = rrd2rrdr(owa, qt); if(!r) { buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); return HTTP_RESP_INTERNAL_SERVER_ERROR; @@ -298,11 +155,6 @@ int rrdset2anything_api_v1( return HTTP_RESP_BACKEND_FETCH_FAILED; } - if (st->state && st->state->is_ar_chart) - ml_process_rrdr(r, query_params->max_anomaly_rates); - - RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL; - if(r->result_options & RRDR_RESULT_OPTION_RELATIVE) buffer_no_cacheable(wb); else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE) @@ -311,85 +163,89 @@ int rrdset2anything_api_v1( if(latest_timestamp && rrdr_rows(r) > 0) *latest_timestamp = r->before; + DATASOURCE_FORMAT format = qt->request.format; + RRDR_OPTIONS options = qt->request.options; + RRDR_GROUPING group_method = qt->request.group_method; + switch(format) { case DATASOURCE_SSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); - rrdr2ssv(r, wb, options, "", " ", "", temp_rd); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + rrdr2ssv(r, wb, options, "", " ", ""); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_PLAIN; - rrdr2ssv(r, wb, options, "", " ", "", temp_rd); + rrdr2ssv(r, wb, options, "", " ", ""); } break; case DATASOURCE_SSV_COMMA: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); - rrdr2ssv(r, wb, options, "", ",", "", temp_rd); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + rrdr2ssv(r, wb, options, "", ",", ""); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_PLAIN; - rrdr2ssv(r, wb, options, "", ",", "", temp_rd); + rrdr2ssv(r, wb, options, "", ",", ""); } break; case DATASOURCE_JS_ARRAY: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); - rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); + rrdr2ssv(r, wb, options, "[", ",", "]"); rrdr_json_wrapper_end(r, wb, format, options, 0); } else { wb->contenttype = CT_APPLICATION_JSON; - rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd); + rrdr2ssv(r, wb, options, "[", ",", "]"); } break; case DATASOURCE_CSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); - rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + rrdr2csv(r, wb, format, options, "", ",", "\\n", ""); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_PLAIN; - rrdr2csv(r, wb, format, options, "", ",", "\r\n", "", temp_rd); + rrdr2csv(r, wb, format, options, "", ",", "\r\n", ""); } break; case DATASOURCE_CSV_MARKDOWN: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); - rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + rrdr2csv(r, wb, format, options, "", "|", "\\n", ""); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_PLAIN; - rrdr2csv(r, wb, format, options, "", "|", "\r\n", "", temp_rd); + rrdr2csv(r, wb, format, options, "", "|", "\r\n", ""); } break; case DATASOURCE_CSV_JSON_ARRAY: wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) { - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); buffer_strcat(wb, "[\n"); - rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd); + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); buffer_strcat(wb, "\n]"); rrdr_json_wrapper_end(r, wb, format, options, 0); } else { wb->contenttype = CT_APPLICATION_JSON; buffer_strcat(wb, "[\n"); - rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd); + rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); buffer_strcat(wb, "\n]"); } break; @@ -397,29 +253,29 @@ int rrdset2anything_api_v1( case DATASOURCE_TSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); - rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); + rrdr2csv(r, wb, format, options, "", "\t", "\\n", ""); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_PLAIN; - rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "", temp_rd); + rrdr2csv(r, wb, format, options, "", "\t", "\r\n", ""); } break; case DATASOURCE_HTML: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method); buffer_strcat(wb, "\\n
\\n\\n"); - rrdr2csv(r, wb, format, options, "\\n", "", temp_rd); + rrdr2csv(r, wb, format, options, "\\n", ""); buffer_strcat(wb, "
", "", "
", "", "
\\n
\\n\\n"); rrdr_json_wrapper_end(r, wb, format, options, 1); } else { wb->contenttype = CT_TEXT_HTML; buffer_strcat(wb, "\n
\n\n"); - rrdr2csv(r, wb, format, options, "\n", "", temp_rd); + rrdr2csv(r, wb, format, options, "\n", ""); buffer_strcat(wb, "
", "", "
", "", "
\n
\n\n"); } break; @@ -428,9 +284,9 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); - rrdr2json(r, wb, options, 1, query_params->context_param_list); + rrdr2json(r, wb, options, 1); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -440,9 +296,9 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); - rrdr2json(r, wb, options, 1, query_params->context_param_list); + rrdr2json(r, wb, options, 1); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -451,9 +307,9 @@ int rrdset2anything_api_v1( case DATASOURCE_JSONP: wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); - rrdr2json(r, wb, options, 0, query_params->context_param_list); + rrdr2json(r, wb, options, 0); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -464,14 +320,14 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params); + rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method); - rrdr2json(r, wb, options, 0, query_params->context_param_list); + rrdr2json(r, wb, options, 0); if(options & RRDR_OPTION_JSON_WRAP) { if(options & RRDR_OPTION_RETURN_JWAR) { rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0); - rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0, query_params->context_param_list); + rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0); } rrdr_json_wrapper_end(r, wb, format, options, 0); } diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index 6be53ff8a..048281d7e 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -5,16 +5,6 @@ #include "web/api/web_api_v1.h" -typedef struct query_params { - struct context_param *context_param_list; - BUFFER *wb; - char *chart_label_key; - int max_anomaly_rates; - int timeout; - int show_dimensions; -} QUERY_PARAMS; - - #include "web/api/exporters/allmetrics.h" #include "web/api/queries/rrdr.h" @@ -34,19 +24,20 @@ typedef struct query_params { #define API_RELATIVE_TIME_MAX (3 * 365 * 86400) // type of JSON generations -#define DATASOURCE_INVALID (-1) -#define DATASOURCE_JSON 0 -#define DATASOURCE_DATATABLE_JSON 1 -#define DATASOURCE_DATATABLE_JSONP 2 -#define DATASOURCE_SSV 3 -#define DATASOURCE_CSV 4 -#define DATASOURCE_JSONP 5 -#define DATASOURCE_TSV 6 -#define DATASOURCE_HTML 7 -#define DATASOURCE_JS_ARRAY 8 -#define DATASOURCE_SSV_COMMA 9 -#define DATASOURCE_CSV_JSON_ARRAY 10 -#define DATASOURCE_CSV_MARKDOWN 11 +typedef enum { + DATASOURCE_JSON = 0, + DATASOURCE_DATATABLE_JSON = 1, + DATASOURCE_DATATABLE_JSONP = 2, + DATASOURCE_SSV = 3, + DATASOURCE_CSV = 4, + DATASOURCE_JSONP = 5, + DATASOURCE_TSV = 6, + DATASOURCE_HTML = 7, + DATASOURCE_JS_ARRAY = 8, + DATASOURCE_SSV_COMMA = 9, + DATASOURCE_CSV_JSON_ARRAY = 10, + DATASOURCE_CSV_MARKDOWN = 11, +} DATASOURCE_FORMAT; #define DATASOURCE_FORMAT_JSON "json" #define DATASOURCE_FORMAT_DATATABLE_JSON "datatable" @@ -61,37 +52,22 @@ typedef struct query_params { #define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray" #define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown" -extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); -extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format); +void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); +void rrdr_buffer_print_format(BUFFER *wb, uint32_t format); -extern int rrdset2anything_api_v1( - ONEWAYALLOC *owa - , RRDSET *st - , QUERY_PARAMS *query_params - , BUFFER *dimensions - , uint32_t format - , long points - , long long after - , long long before - , int group_method - , const char *group_options - , long group_time - , uint32_t options - , time_t *latest_timestamp - , int tier -); +int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp); -extern int rrdset2value_api_v1( +int rrdset2value_api_v1( RRDSET *st , BUFFER *wb , NETDATA_DOUBLE *n , const char *dimensions - , long points - , long long after - , long long before - , int group_method + , size_t points + , time_t after + , time_t before + , RRDR_GROUPING group_method , const char *group_options - , long group_time + , time_t resampling_time , uint32_t options , time_t *db_after , time_t *db_before @@ -100,12 +76,9 @@ extern int rrdset2value_api_v1( , size_t *result_points_generated , int *value_is_null , NETDATA_DOUBLE *anomaly_rate - , int timeout - , int tier + , time_t timeout + , size_t tier + , QUERY_SOURCE query_source ); -extern void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st); -extern void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested); -extern void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list); - #endif /* NETDATA_RRD2JSON_H */ diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c index de8d87bae..1e8106335 100644 --- a/web/api/formatters/rrdset2json.c +++ b/web/api/formatters/rrdset2json.c @@ -4,7 +4,7 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation) { - if(unlikely(!st->state || !st->state->chart_labels)) + if(unlikely(!st->rrdlabels)) return; char tabs[11]; @@ -18,17 +18,15 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation) indentation--; } - rrdlabels_to_buffer(st->state->chart_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL); + rrdlabels_to_buffer(st->rrdlabels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL); buffer_strcat(wb, "\n"); } // generate JSON for the /api/v1/chart API call void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile) { - rrdset_rdlock(st); - - time_t first_entry_t = rrdset_first_entry_t_nolock(st); - time_t last_entry_t = rrdset_last_entry_t_nolock(st); + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); buffer_sprintf( wb, @@ -45,18 +43,18 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\"units\": \"%s\",\n" "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n" "\t\t\t\"chart_type\": \"%s\",\n", - st->id, - st->name, - st->type, - st->family, - st->context, - st->title, - st->name, + rrdset_id(st), + rrdset_name(st), + rrdset_parts_type(st), + rrdset_family(st), + rrdset_context(st), + rrdset_title(st), + rrdset_name(st), st->priority, - st->plugin_name ? st->plugin_name : "", - st->module_name ? st->module_name : "", - st->units, - st->name, + rrdset_plugin_name(st), + rrdset_module_name(st), + rrdset_units(st), + rrdset_name(st), rrdset_type_name(st->chart_type)); if (likely(!skip_volatile)) @@ -90,7 +88,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor size_t dimensions = 0; RRDDIM *rd; rrddim_foreach_read(rd, st) { - if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; + if(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue; memory += sizeof(RRDDIM) + rd->memsize; @@ -98,13 +96,14 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_strcat(wb, ",\n\t\t\t\t\""); else buffer_strcat(wb, "\t\t\t\t\""); - buffer_strcat_jsonescape(wb, rd->id); + buffer_strcat_jsonescape(wb, rrddim_id(rd)); buffer_strcat(wb, "\": { \"name\": \""); - buffer_strcat_jsonescape(wb, rd->name); + buffer_strcat_jsonescape(wb, rrddim_name(rd)); buffer_strcat(wb, "\" }"); dimensions++; } + rrddim_foreach_done(rd); if(dimensions_count) *dimensions_count += dimensions; if(memory_used) *memory_used += memory; @@ -121,7 +120,8 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n"); size_t alarms = 0; RRDCALC *rc; - for (rc = st->alarms; rc; rc = rc->rrdset_next) { + netdata_rwlock_rdlock(&st->alerts.rwlock); + DOUBLE_LINKED_LIST_FOREACH_FORWARD(st->alerts.base, rc, prev, next) { buffer_sprintf( wb, "%s" @@ -131,23 +131,25 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\t\t\"units\": \"%s\",\n" "\t\t\t\t\t\"update_every\": %d\n" "\t\t\t\t}", - (alarms) ? ",\n" : "", rc->name, rc->id, rrdcalc_status2string(rc->status), rc->units, + (alarms) ? ",\n" : "", rrdcalc_name(rc), rc->id, rrdcalc_status2string(rc->status), rrdcalc_units(rc), rc->update_every); alarms++; } + netdata_rwlock_unlock(&st->alerts.rwlock); buffer_sprintf(wb, "\n\t\t\t}" ); } buffer_strcat(wb, ",\n\t\t\t\"chart_labels\": {\n"); chart_labels2json(st, wb, 2); - buffer_strcat(wb, "\t\t\t}\n"); + buffer_strcat(wb, "\t\t\t}"); + buffer_strcat(wb, ",\n\t\t\t\"functions\": {\n"); + chart_functions2json(st, wb, 4, "\"", "\""); + buffer_strcat(wb, "\t\t\t}"); buffer_sprintf(wb, "\n\t\t}" ); - - rrdset_unlock(st); } diff --git a/web/api/formatters/rrdset2json.h b/web/api/formatters/rrdset2json.h index 697c84634..b2908e225 100644 --- a/web/api/formatters/rrdset2json.h +++ b/web/api/formatters/rrdset2json.h @@ -5,6 +5,6 @@ #include "rrd2json.h" -extern void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile); +void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile); #endif //NETDATA_API_FORMATTER_RRDSET2JSON_H diff --git a/web/api/formatters/ssv/ssv.c b/web/api/formatters/ssv/ssv.c index 850182da1..d561980d9 100644 --- a/web/api/formatters/ssv/ssv.c +++ b/web/api/formatters/ssv/ssv.c @@ -2,7 +2,7 @@ #include "ssv.h" -void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd) { +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) { //info("RRD2SSV(): %s: BEGIN", r->st->id); long i; @@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con // for each line in the array for(i = start; i != end ;i += step) { int all_values_are_null = 0; - NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd); + NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL); if(likely(i != start)) { if(r->min > v) r->min = v; diff --git a/web/api/formatters/ssv/ssv.h b/web/api/formatters/ssv/ssv.h index 66716b9c9..f7d4a9548 100644 --- a/web/api/formatters/ssv/ssv.h +++ b/web/api/formatters/ssv/ssv.h @@ -5,6 +5,6 @@ #include "../rrd2json.h" -extern void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd); +void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix); #endif //NETDATA_API_FORMATTER_SSV_H diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c index 30e00c068..46a71303e 100644 --- a/web/api/formatters/value/value.c +++ b/web/api/formatters/value/value.c @@ -3,9 +3,10 @@ #include "value.h" -inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd) { +inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) { + QUERY_TARGET *qt = r->internal.qt; long c; - RRDDIM *d; + const long used = qt->query.used; NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; @@ -20,7 +21,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all int set_min_max = 0; if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { total = 0; - for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) { + for (c = 0; c < used; c++) { NETDATA_DOUBLE n = cn[c]; if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) @@ -34,7 +35,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all } // for each dimension - for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) { + for (c = 0; c < used; c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; @@ -80,7 +81,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all if(anomaly_rate) { if(!r->d) *anomaly_rate = 0; - else *anomaly_rate = total_anomaly_rate / r->d; + else *anomaly_rate = total_anomaly_rate / (NETDATA_DOUBLE)r->d; } if(unlikely(all_null)) { @@ -100,3 +101,62 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all return v; } + +QUERY_VALUE rrdmetric2value(RRDHOST *host, + struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, + time_t after, time_t before, + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + size_t tier, time_t timeout, QUERY_SOURCE query_source +) { + QUERY_TARGET_REQUEST qtr = { + .host = host, + .rca = rca, + .ria = ria, + .rma = rma, + .after = after, + .before = before, + .points = 1, + .options = options, + .group_method = group_method, + .group_options = group_options, + .tier = tier, + .timeout = timeout, + .query_source = query_source, + }; + + ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); + RRDR *r = rrd2rrdr(owa, query_target_create(&qtr)); + + QUERY_VALUE qv; + + if(!r || rrdr_rows(r) == 0) { + qv = (QUERY_VALUE) { + .value = NAN, + .anomaly_rate = NAN, + }; + } + else { + qv = (QUERY_VALUE) { + .after = r->after, + .before = r->before, + .points_read = r->internal.db_points_read, + .result_points = r->internal.result_points_generated, + }; + + for(size_t t = 0; t < storage_tiers ;t++) + qv.storage_points_per_tier[t] = r->internal.tier_points_read[t]; + + long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0; + int all_values_are_null = 0; + qv.value = rrdr2value(r, i, options, &all_values_are_null, &qv.anomaly_rate); + if(all_values_are_null) { + qv.value = NAN; + qv.anomaly_rate = NAN; + } + } + + rrdr_free(owa, r); + onewayalloc_destroy(owa); + + return qv; +} diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h index fc1c7bf08..76b1869f3 100644 --- a/web/api/formatters/value/value.h +++ b/web/api/formatters/value/value.h @@ -5,6 +5,27 @@ #include "../rrd2json.h" -extern NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd); +typedef struct storage_value { + NETDATA_DOUBLE value; + NETDATA_DOUBLE anomaly_rate; + time_t after; + time_t before; + size_t points_read; + size_t storage_points_per_tier[RRD_STORAGE_TIERS]; + size_t result_points; +} QUERY_VALUE; + +struct rrdmetric_acquired; +struct rrdinstance_acquired; +struct rrdcontext_acquired; + +QUERY_VALUE rrdmetric2value(RRDHOST *host, + struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma, + time_t after, time_t before, + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + size_t tier, time_t timeout, QUERY_SOURCE query_source +); + +NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate); #endif //NETDATA_API_FORMATTER_VALUE_H -- cgit v1.2.3