diff options
Diffstat (limited to 'web/api')
38 files changed, 604 insertions, 328 deletions
diff --git a/web/api/badges/README.md b/web/api/badges/README.md index 601eae479..84409471a 100644 --- a/web/api/badges/README.md +++ b/web/api/badges/README.md @@ -324,9 +324,11 @@ On modern hardware, Netdata can generate about **2.000 badges per second per cor Of course these timing are for badges that use recent data. If you need badges that do calculations over long durations (a day, or more), timing will differ. Netdata logs its timings at its `access.log`, so take a look there before adding a heavy badge on a busy web site. Of course, you can cache such badges or have a cron job get them from Netdata and save them at your web server at regular intervals. -#### Embedding badges in github +#### Embedding badges in GitHub -You have 2 options a) SVG images with markdown and b) SVG images with HTML (directly in .md files). +You have 2 options: +- SVG images with markdown +- SVG images with HTML (directly in .md files) For example, this is the cpu badge shown above: @@ -350,9 +352,9 @@ Both produce this: <img src="https://registry.my-netdata.io/api/v1/badge.svg?chart=users.cpu&dimensions=root&value_color=grey:null%7Cgreen%3C10%7Cyellow%3C20%7Corange%3C50%7Cblue%3C100%7Cred&label=root%20user%20cpu%20now&units=%25"></img> </a> -#### auto-refreshing badges in github +#### Auto-refreshing badges in GitHub -Unfortunately it cannot be done. Github fetches all the images using a proxy and rewrites all the URLs to be served by the proxy. +Unfortunately it cannot be done. GitHub fetches all the images using a proxy and rewrites all the URLs to be served by the proxy. You can refresh them from your browser console though. Press F12 to open the web browser console (switch to the console too), paste the following and press enter. They will refresh: diff --git a/web/api/exporters/allmetrics.c b/web/api/exporters/allmetrics.c index 1eba815c3..88065400d 100644 --- a/web/api/exporters/allmetrics.c +++ b/web/api/exporters/allmetrics.c @@ -19,6 +19,7 @@ struct prometheus_output_options { inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) { int format = ALLMETRICS_SHELL; + const char *filter = NULL; const char *prometheus_server = w->client_ip; uint32_t prometheus_exporting_options; @@ -57,6 +58,9 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client else format = 0; } + else if(!strcmp(name, "filter")) { + filter = value; + } else if(!strcmp(name, "server")) { prometheus_server = value; } @@ -87,18 +91,19 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client switch(format) { case ALLMETRICS_JSON: w->response.data->contenttype = CT_APPLICATION_JSON; - rrd_stats_api_v1_charts_allmetrics_json(host, w->response.data); + rrd_stats_api_v1_charts_allmetrics_json(host, filter, w->response.data); return HTTP_RESP_OK; case ALLMETRICS_SHELL: w->response.data->contenttype = CT_TEXT_PLAIN; - rrd_stats_api_v1_charts_allmetrics_shell(host, w->response.data); + rrd_stats_api_v1_charts_allmetrics_shell(host, filter, w->response.data); return HTTP_RESP_OK; case ALLMETRICS_PROMETHEUS: w->response.data->contenttype = CT_PROMETHEUS; rrd_stats_api_v1_charts_allmetrics_prometheus_single_host( host + , filter , w->response.data , prometheus_server , prometheus_prefix @@ -111,6 +116,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client w->response.data->contenttype = CT_PROMETHEUS; rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts( host + , filter , w->response.data , prometheus_server , prometheus_prefix diff --git a/web/api/exporters/shell/allmetrics_shell.c b/web/api/exporters/shell/allmetrics_shell.c index 0cbaf3069..b9b6c904b 100644 --- a/web/api/exporters/shell/allmetrics_shell.c +++ b/web/api/exporters/shell/allmetrics_shell.c @@ -22,13 +22,17 @@ static inline size_t shell_name_copy(char *d, const char *s, size_t usable) { #define SHELL_ELEMENT_MAX 100 -void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) { +void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb) { analytics_log_shell(); + SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); rrdhost_rdlock(host); // for each chart RRDSET *st; rrdset_foreach_read(st, host) { + if (filter && !simple_pattern_matches(filter, st->name)) + continue; + calculated_number total = 0.0; char chart[SHELL_ELEMENT_MAX + 1]; shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX); @@ -88,12 +92,14 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) { } rrdhost_unlock(host); + simple_pattern_free(filter); } // ---------------------------------------------------------------------------- -void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb) { +void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb) { analytics_log_json(); + SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); rrdhost_rdlock(host); buffer_strcat(wb, "{"); @@ -104,6 +110,9 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb) { // for each chart RRDSET *st; rrdset_foreach_read(st, host) { + if (filter && !(simple_pattern_matches(filter, st->id) || simple_pattern_matches(filter, st->name))) + continue; + if(rrdset_is_available_for_viewers(st)) { rrdset_rdlock(st); @@ -160,5 +169,6 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb) { buffer_strcat(wb, "\n}"); rrdhost_unlock(host); + simple_pattern_free(filter); } diff --git a/web/api/exporters/shell/allmetrics_shell.h b/web/api/exporters/shell/allmetrics_shell.h index 1d7611a2d..1ee9aa717 100644 --- a/web/api/exporters/shell/allmetrics_shell.h +++ b/web/api/exporters/shell/allmetrics_shell.h @@ -15,7 +15,7 @@ #define ALLMETRICS_JSON 3 #define ALLMETRICS_PROMETHEUS_ALL_HOSTS 4 -extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb); -extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb); +extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb); +extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb); #endif //NETDATA_API_ALLMETRICS_SHELL_H diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c index 856ffb5eb..4325b6530 100644 --- a/web/api/formatters/charts2json.c +++ b/web/api/formatters/charts2json.c @@ -8,30 +8,30 @@ const char* get_release_channel() { static int use_stable = -1; if (use_stable == -1) { - char filename[FILENAME_MAX + 1]; + char filename[FILENAME_MAX + 1]; snprintfz(filename, FILENAME_MAX, "%s/.environment", netdata_configured_user_config_dir); procfile *ff = procfile_open(filename, "=", PROCFILE_FLAG_DEFAULT); - if(!ff) { - use_stable=1; - } else { + if (ff) { procfile_set_quotes(ff, "'\""); ff = procfile_readall(ff); - if(!ff) { - use_stable=1; - } else { + if (ff) { unsigned int i; - for(i = 0; i < procfile_lines(ff); i++) { - if (!procfile_linewords(ff, i)) continue; - - if (!strcmp(procfile_lineword(ff, i, 0), "RELEASE_CHANNEL") && !strcmp(procfile_lineword(ff, i, 1), "stable")) { - use_stable = 1; + for (i = 0; i < procfile_lines(ff); i++) { + if (!procfile_linewords(ff, i)) + continue; + if (!strcmp(procfile_lineword(ff, i, 0), "RELEASE_CHANNEL")) { + if (!strcmp(procfile_lineword(ff, i, 1), "stable")) + use_stable = 1; + else if (!strcmp(procfile_lineword(ff, i, 1), "nightly")) + use_stable = 0; break; } } procfile_close(ff); - if (use_stable == -1) use_stable = 0; } } + if (use_stable == -1) + use_stable = strchr(program_version, '-') ? 0 : 1; } return (use_stable)?"stable":"nightly"; } @@ -150,7 +150,9 @@ struct array_printer { BUFFER *wb; }; -int print_collector(void *entry, void *data) { +static int print_collector_callback(const char *name, void *entry, void *data) { + (void)name; + struct array_printer *ap = (struct array_printer *)data; BUFFER *wb = ap->wb; struct collector *col=(struct collector *) entry; @@ -187,6 +189,6 @@ void chartcollectors2json(RRDHOST *host, BUFFER *wb) { .c = 0, .wb = wb }; - dictionary_get_all(dict, print_collector, &ap); + dictionary_walkthrough_read(dict, print_collector_callback, &ap); dictionary_destroy(dict); } diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c index bf311e22c..af1156d27 100644 --- a/web/api/formatters/json/json.c +++ b/web/api/formatters/json/json.c @@ -16,7 +16,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct //info("RRD2JSON(): %s: BEGIN", r->st->id); int row_annotations = 0, dates, dates_with_new = 0; - char kq[2] = "", // key quote + char kq[2] = "", // key quote sq[2] = "", // string quote pre_label[101] = "", // before each label post_label[101] = "", // after each label @@ -28,7 +28,8 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct normal_annotation[201] = "", // default row annotation overflow_annotation[201] = "", // overflow row annotation data_begin[101] = "", // between labels and values - finish[101] = ""; // at the end of everything + finish[101] = "", // at the end of everything + object_rows_time[101] = ""; if(datatable) { dates = JSON_DATES_JS; @@ -49,7 +50,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct strcpy(post_value, "}"); strcpy(post_line, "]}"); snprintfz(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq); - strcpy(finish, "\n ]\n}"); + strcpy(finish, "\n]\n}"); snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); @@ -76,23 +77,38 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct dates_with_new = 0; } if( options & RRDR_OPTION_OBJECTSROWS ) - strcpy(pre_date, " { "); + strcpy(pre_date, " { "); else - strcpy(pre_date, " [ "); - strcpy(pre_label, ", \""); + strcpy(pre_date, " [ "); + strcpy(pre_label, ",\""); strcpy(post_label, "\""); - strcpy(pre_value, ", "); + strcpy(pre_value, ","); if( options & RRDR_OPTION_OBJECTSROWS ) strcpy(post_line, "}"); else strcpy(post_line, "]"); snprintfz(data_begin, 100, "],\n %sdata%s:\n [\n", kq, kq); - strcpy(finish, "\n ]\n}"); + strcpy(finish, "\n]\n}"); buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq); buffer_sprintf(wb, "%stime%s", sq, sq); + + if( options & RRDR_OPTION_OBJECTSROWS ) + snprintfz(object_rows_time, 100, "%stime%s: ", kq, kq); + } + size_t pre_value_len = strlen(pre_value); + size_t post_value_len = strlen(post_value); + size_t pre_label_len = strlen(pre_label); + size_t post_label_len = strlen(post_label); + size_t pre_date_len = strlen(pre_date); + size_t post_date_len = strlen(post_date); + size_t post_line_len = strlen(post_line); + size_t normal_annotation_len = strlen(normal_annotation); + size_t overflow_annotation_len = strlen(overflow_annotation); + size_t object_rows_time_len = strlen(object_rows_time); + // ------------------------------------------------------------------------- // print the JSON header @@ -104,18 +120,19 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue; - buffer_strcat(wb, pre_label); + buffer_fast_strcat(wb, pre_label, pre_label_len); buffer_strcat(wb, rd->name); // buffer_strcat(wb, "."); // buffer_strcat(wb, rd->rrdset->name); - buffer_strcat(wb, post_label); + buffer_fast_strcat(wb, post_label, post_label_len); i++; } if(!i) { - buffer_strcat(wb, pre_label); - buffer_strcat(wb, "no data"); - buffer_strcat(wb, post_label); + buffer_fast_strcat(wb, pre_label, pre_label_len); + buffer_fast_strcat(wb, "no data", 7); + buffer_fast_strcat(wb, post_label, post_label_len); } + size_t total_number_of_dimensions = i; // print the begin of row data buffer_strcat(wb, data_begin); @@ -133,6 +150,13 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct step = -1; } + // pre-allocate a large enough buffer for us + // this does not need to be accurate - it is just a hint to avoid multiple realloc(). + buffer_need_bytes(wb, + ( 20 * rrdr_rows(r)) // timestamp + json overhead + + ( (pre_value_len + post_value_len + 4) * total_number_of_dimensions * rrdr_rows(r) ) // number + ); + // for each line in the array calculated_number total = 1; for(i = start; i != end ;i += step) { @@ -146,48 +170,52 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); if(!tm) { error("localtime_r() failed."); continue; } - if(likely(i != start)) buffer_strcat(wb, ",\n"); - buffer_strcat(wb, pre_date); + if(likely(i != start)) buffer_fast_strcat(wb, ",\n", 2); + buffer_fast_strcat(wb, pre_date, pre_date_len); if( options & RRDR_OPTION_OBJECTSROWS ) - buffer_sprintf(wb, "%stime%s: ", kq, kq); + buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); - if(dates_with_new) - buffer_strcat(wb, "new "); + if(unlikely(dates_with_new)) + buffer_fast_strcat(wb, "new ", 4); buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); - buffer_strcat(wb, post_date); + buffer_fast_strcat(wb, post_date, post_date_len); - if(row_annotations) { + if(unlikely(row_annotations)) { // google supports one annotation per row int annotation_found = 0; for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd ;c++, rd = rd->next) { if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue; - if(co[c] & RRDR_VALUE_RESET) { - buffer_strcat(wb, overflow_annotation); + if(unlikely(co[c] & RRDR_VALUE_RESET)) { + buffer_fast_strcat(wb, overflow_annotation, overflow_annotation_len); annotation_found = 1; break; } } - if(!annotation_found) - buffer_strcat(wb, normal_annotation); + if(likely(!annotation_found)) + buffer_fast_strcat(wb, normal_annotation, normal_annotation_len); } } else { // print the timestamp of the line - if(likely(i != start)) buffer_strcat(wb, ",\n"); - buffer_strcat(wb, pre_date); + if(likely(i != start)) + buffer_fast_strcat(wb, ",\n", 2); - if( options & RRDR_OPTION_OBJECTSROWS ) - buffer_sprintf(wb, "%stime%s: ", kq, kq); + buffer_fast_strcat(wb, pre_date, pre_date_len); + + if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) + buffer_fast_strcat(wb, object_rows_time, object_rows_time_len); buffer_rrd_value(wb, (calculated_number)r->t[i]); + // in ms - if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + if(unlikely(options & RRDR_OPTION_MILLISECONDS)) + buffer_fast_strcat(wb, "000", 3); - buffer_strcat(wb, post_date); + buffer_fast_strcat(wb, post_date, post_date_len); } int set_min_max = 0; @@ -213,16 +241,16 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct calculated_number n = cn[c]; - buffer_strcat(wb, pre_value); + buffer_fast_strcat(wb, pre_value, pre_value_len); - if( options & RRDR_OPTION_OBJECTSROWS ) + if(unlikely( options & RRDR_OPTION_OBJECTSROWS )) buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq); if(co[c] & RRDR_VALUE_EMPTY) { - if(options & RRDR_OPTION_NULL2ZERO) - buffer_strcat(wb, "0"); + if(unlikely(options & RRDR_OPTION_NULL2ZERO)) + buffer_fast_strcat(wb, "0", 1); else - buffer_strcat(wb, "null"); + buffer_fast_strcat(wb, "null", 4); } else { if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) @@ -243,10 +271,10 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct buffer_rrd_value(wb, n); } - buffer_strcat(wb, post_value); + buffer_fast_strcat(wb, post_value, post_value_len); } - buffer_strcat(wb, post_line); + buffer_fast_strcat(wb, post_line, post_line_len); } buffer_strcat(wb, finish); diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c index 264377e20..7097a5b77 100644 --- a/web/api/formatters/json_wrapper.c +++ b/web/api/formatters/json_wrapper.c @@ -2,9 +2,28 @@ #include "json_wrapper.h" +struct value_output { + int c; + BUFFER *wb; +}; + +static int value_list_output(const char *name, void *entry, void *data) { + (void)name; + + struct value_output *ap = (struct value_output *)data; + BUFFER *wb = ap->wb; + char *output = (char *) entry; + if(ap->c) buffer_strcat(wb, ","); + buffer_strcat(wb, output); + (ap->c)++; + return 0; +} + void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, - struct context_param *context_param_list, char *chart_label_key) + QUERY_PARAMS *rrdset_query_data) { + struct context_param *context_param_list = rrdset_query_data->context_param_list; + char *chart_label_key = rrdset_query_data->chart_label_key; RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); @@ -98,6 +117,61 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS } buffer_strcat(wb, "],\n"); + if (rrdset_query_data->show_dimensions) { + buffer_sprintf(wb, " %sfull_dimension_list%s: [", kq, kq); + + char name[RRD_ID_LENGTH_MAX * 2 + 2]; + char output[RRD_ID_LENGTH_MAX * 2 + 8]; + char value[RRD_ID_LENGTH_MAX * 2 + 1]; + + struct value_output co = {.c = 0, .wb = wb}; + + DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name); + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name); + dictionary_set(dict, name, output, len+1); + } + dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_destroy(dict); + + co.c = 0; + buffer_sprintf(wb, "],\n %sfull_chart_list%s: [", kq, kq); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name); + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name); + dictionary_set(dict, name, output, len + 1); + } + + dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_destroy(dict); + + RRDSET *st; + co.c = 0; + buffer_sprintf(wb, "],\n %sfull_chart_labels%s: [", kq, kq); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) { + st = rd->rrdset; + if (likely(st->state)) { + struct label_index *labels = &st->state->labels; + if (labels->head) { + netdata_rwlock_rdlock(&labels->labels_rwlock); + for (struct label *label = labels->head; label; label = label->next) { + sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2); + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value); + snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value); + dictionary_set(dict, name, output, len + 1); + } + netdata_rwlock_unlock(&labels->labels_rwlock); + } + } + } + dictionary_walkthrough_read(dict, value_list_output, &co); + dictionary_destroy(dict); + buffer_strcat(wb, "],\n"); + } + // Composite charts if (context_mode && temp_rd) { buffer_sprintf( diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h index 14662db74..65dbd5b65 100644 --- a/web/api/formatters/json_wrapper.h +++ b/web/api/formatters/json_wrapper.h @@ -5,7 +5,8 @@ #include "rrd2json.h" -extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, struct context_param *context_param_list, char *chart_key); +extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value, + QUERY_PARAMS *query_params); extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value); #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 1a8b07c7c..1de6be4e3 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -2,27 +2,27 @@ #include "web/api/web_api_v1.h" -static inline void free_single_rrdrim(RRDDIM *temp_rd, int archive_mode) +static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode) { if (unlikely(!temp_rd)) return; - freez((char *)temp_rd->id); - freez((char *)temp_rd->name); + onewayalloc_freez(owa, (char *)temp_rd->id); if (unlikely(archive_mode)) { temp_rd->rrdset->counter--; if (!temp_rd->rrdset->counter) { - freez((char *)temp_rd->rrdset->name); - freez(temp_rd->rrdset->context); - freez(temp_rd->rrdset); + onewayalloc_freez(owa, (char *)temp_rd->rrdset->name); + onewayalloc_freez(owa, temp_rd->rrdset->context); + onewayalloc_freez(owa, temp_rd->rrdset); } } - freez(temp_rd->state); - freez(temp_rd); + + onewayalloc_freez(owa, temp_rd->state); + onewayalloc_freez(owa, temp_rd); } -static inline void free_rrddim_list(RRDDIM *temp_rd, int archive_mode) +static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode) { if (unlikely(!temp_rd)) return; @@ -30,22 +30,22 @@ static inline void free_rrddim_list(RRDDIM *temp_rd, int archive_mode) RRDDIM *t; while (temp_rd) { t = temp_rd->next; - free_single_rrdrim(temp_rd, archive_mode); + free_single_rrdrim(owa, temp_rd, archive_mode); temp_rd = t; } } -void free_context_param_list(struct context_param **param_list) +void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list) { if (unlikely(!param_list || !*param_list)) return; - free_rrddim_list(((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE); - freez((*param_list)); + free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE); + onewayalloc_freez(owa, (*param_list)); *param_list = NULL; } -void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested) +void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested) { RRDDIM *temp_rd = context_param_list->rd; RRDDIM *new_rd_list = NULL, *t; @@ -59,19 +59,19 @@ void rebuild_context_param_list(struct context_param *context_param_list, time_t temp_rd->next = new_rd_list; new_rd_list = temp_rd; } else - free_single_rrdrim(temp_rd, is_archived); + free_single_rrdrim(owa, temp_rd, is_archived); temp_rd = t; } context_param_list->rd = new_rd_list; }; -void build_context_param_list(struct context_param **param_list, RRDSET *st) +void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st) { if (unlikely(!param_list || !st)) return; if (unlikely(!(*param_list))) { - *param_list = mallocz(sizeof(struct context_param)); + *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param)); (*param_list)->first_entry_t = LONG_MAX; (*param_list)->last_entry_t = 0; (*param_list)->flags = CONTEXT_FLAGS_CONTEXT; @@ -86,14 +86,10 @@ void build_context_param_list(struct context_param **param_list, RRDSET *st) (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); rrddim_foreach_read(rd1, st) { - RRDDIM *rd = mallocz(rd1->memsize); - memcpy(rd, rd1, rd1->memsize); - rd->id = strdupz(rd1->id); - rd->name = strdupz(rd1->name); - rd->state = mallocz(sizeof(*rd->state)); - memcpy(rd->state, rd1->state, sizeof(*rd->state)); - memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops)); - memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops)); + RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize); + rd->id = onewayalloc_strdupz(owa, rd1->id); + rd->name = onewayalloc_strdupz(owa, rd1->name); + rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state)); rd->next = (*param_list)->rd; (*param_list)->rd = rd; } @@ -169,22 +165,27 @@ int rrdset2value_api_v1( , int *value_is_null , int timeout ) { + int ret = HTTP_RESP_INTERNAL_SERVER_ERROR; + + ONEWAYALLOC *owa = onewayalloc_create(0); - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout); + RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout); if(!r) { if(value_is_null) *value_is_null = 1; - return HTTP_RESP_INTERNAL_SERVER_ERROR; + ret = HTTP_RESP_INTERNAL_SERVER_ERROR; + goto cleanup; } if(rrdr_rows(r) == 0) { - rrdr_free(r); + rrdr_free(owa, r); if(db_after) *db_after = 0; if(db_before) *db_before = 0; if(value_is_null) *value_is_null = 1; - return HTTP_RESP_BAD_REQUEST; + ret = HTTP_RESP_BAD_REQUEST; + goto cleanup; } if(wb) { @@ -199,14 +200,18 @@ int rrdset2value_api_v1( long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0; *n = rrdr2value(r, i, options, value_is_null, NULL); + ret = HTTP_RESP_OK; - rrdr_free(r); - return HTTP_RESP_OK; +cleanup: + if(r) rrdr_free(owa, r); + onewayalloc_destroy(owa); + return ret; } int rrdset2anything_api_v1( - RRDSET *st - , BUFFER *wb + ONEWAYALLOC *owa + , RRDSET *st + , QUERY_PARAMS *query_params , BUFFER *dimensions , uint32_t format , long points @@ -216,30 +221,38 @@ int rrdset2anything_api_v1( , long group_time , uint32_t options , time_t *latest_timestamp - , struct context_param *context_param_list - , char *chart_label_key - , int max_anomaly_rates - , int timeout ) { - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) + BUFFER *wb = query_params->wb; + if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) st->last_accessed_time = now_realtime_sec(); - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout); + RRDR *r = rrd2rrdr( + owa, + st, + points, + after, + before, + group_method, + group_time, + options, + dimensions ? buffer_tostring(dimensions) : NULL, + query_params->context_param_list, + query_params->timeout); if(!r) { buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); return HTTP_RESP_INTERNAL_SERVER_ERROR; } if (r->result_options & RRDR_RESULT_OPTION_CANCEL) { - rrdr_free(r); + rrdr_free(owa, r); return HTTP_RESP_BACKEND_FETCH_FAILED; } if (st && st->state && st->state->is_ar_chart) - ml_process_rrdr(r, max_anomaly_rates); + ml_process_rrdr(r, query_params->max_anomaly_rates); - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; + RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL; if(r->result_options & RRDR_RESULT_OPTION_RELATIVE) buffer_no_cacheable(wb); @@ -253,7 +266,7 @@ int rrdset2anything_api_v1( case DATASOURCE_SSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); rrdr2ssv(r, wb, options, "", " ", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -266,7 +279,7 @@ int rrdset2anything_api_v1( case DATASOURCE_SSV_COMMA: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); rrdr2ssv(r, wb, options, "", ",", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -279,7 +292,7 @@ int rrdset2anything_api_v1( case DATASOURCE_JS_ARRAY: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 0); } @@ -292,7 +305,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -305,7 +318,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV_MARKDOWN: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -318,7 +331,7 @@ int rrdset2anything_api_v1( case DATASOURCE_CSV_JSON_ARRAY: wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) { - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); buffer_strcat(wb, "[\n"); rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd); buffer_strcat(wb, "\n]"); @@ -335,7 +348,7 @@ int rrdset2anything_api_v1( case DATASOURCE_TSV: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd); rrdr_json_wrapper_end(r, wb, format, options, 1); } @@ -348,7 +361,7 @@ int rrdset2anything_api_v1( case DATASOURCE_HTML: if(options & RRDR_OPTION_JSON_WRAP) { wb->contenttype = CT_APPLICATION_JSON; - rrdr_json_wrapper_begin(r, wb, format, options, 1, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 1, query_params); buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n"); rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd); buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n"); @@ -366,9 +379,9 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); - rrdr2json(r, wb, options, 1, context_param_list); + rrdr2json(r, wb, options, 1, query_params->context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -378,9 +391,9 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); - rrdr2json(r, wb, options, 1, context_param_list); + rrdr2json(r, wb, options, 1, query_params->context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -389,9 +402,9 @@ int rrdset2anything_api_v1( case DATASOURCE_JSONP: wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); - rrdr2json(r, wb, options, 0, context_param_list); + rrdr2json(r, wb, options, 0, query_params->context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); @@ -402,15 +415,15 @@ int rrdset2anything_api_v1( wb->contenttype = CT_APPLICATION_JSON; if(options & RRDR_OPTION_JSON_WRAP) - rrdr_json_wrapper_begin(r, wb, format, options, 0, context_param_list, chart_label_key); + rrdr_json_wrapper_begin(r, wb, format, options, 0, query_params); - rrdr2json(r, wb, options, 0, context_param_list); + rrdr2json(r, wb, options, 0, query_params->context_param_list); if(options & RRDR_OPTION_JSON_WRAP) rrdr_json_wrapper_end(r, wb, format, options, 0); break; } - rrdr_free(r); + rrdr_free(owa, r); return HTTP_RESP_OK; } diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index af809c54f..60bed5b90 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -4,6 +4,17 @@ #define NETDATA_RRD2JSON_H 1 #include "web/api/web_api_v1.h" + +typedef struct query_params { + struct context_param *context_param_list; + BUFFER *wb; + char *chart_label_key; + int max_anomaly_rates; + int timeout; + int show_dimensions; +} QUERY_PARAMS; + + #include "web/api/exporters/allmetrics.h" #include "web/api/queries/rrdr.h" @@ -54,9 +65,10 @@ extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format); extern int rrdset2anything_api_v1( - RRDSET *st - , BUFFER *wb - , BUFFER *dimensions + ONEWAYALLOC *owa + , RRDSET *st + , + QUERY_PARAMS *query_params, BUFFER *dimensions , uint32_t format , long points , long long after @@ -65,10 +77,6 @@ extern int rrdset2anything_api_v1( , long group_time , uint32_t options , time_t *latest_timestamp - , struct context_param *context_param_list - , char *chart_label_key - , int max_anomaly_rates - , int timeout ); extern int rrdset2value_api_v1( @@ -88,8 +96,8 @@ extern int rrdset2value_api_v1( , int timeout ); -extern void build_context_param_list(struct context_param **param_list, RRDSET *st); -extern void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested); -extern void free_context_param_list(struct context_param **param_list); +extern void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st); +extern void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested); +extern void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list); #endif /* NETDATA_RRD2JSON_H */ diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c index ce237ab23..c83b22e63 100644 --- a/web/api/formatters/rrdset2json.c +++ b/web/api/formatters/rrdset2json.c @@ -52,7 +52,6 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor "\t\t\t\"priority\": %ld,\n" "\t\t\t\"plugin\": \"%s\",\n" "\t\t\t\"module\": \"%s\",\n" - "\t\t\t\"enabled\": %s,\n" "\t\t\t\"units\": \"%s\",\n" "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n" "\t\t\t\"chart_type\": \"%s\",\n", @@ -66,7 +65,6 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor st->priority, st->plugin_name ? st->plugin_name : "", st->module_name ? st->module_name : "", - rrdset_flag_check(st, RRDSET_FLAG_ENABLED) ? "true" : "false", st->units, st->name, rrdset_type_name(st->chart_type)); diff --git a/web/api/health/health_cmdapi.c b/web/api/health/health_cmdapi.c index c5fb71195..bad3e960a 100644 --- a/web/api/health/health_cmdapi.c +++ b/web/api/health/health_cmdapi.c @@ -199,6 +199,7 @@ int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, c BUFFER *jsonb = buffer_create(200); health_silencers2json(jsonb); health_silencers2file(jsonb); + buffer_free(jsonb); } return ret; diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json index 4952e6116..97427d323 100644 --- a/web/api/netdata-swagger.json +++ b/web/api/netdata-swagger.json @@ -712,6 +712,16 @@ } }, { + "name": "filter", + "in": "query", + "description": "Allows to filter charts out using simple patterns.", + "required": false, + "schema": { + "type": "string", + "format": "any text" + } + }, + { "name": "variables", "in": "query", "description": "When enabled, netdata will expose various system configuration metrics.", @@ -898,7 +908,7 @@ "/alarms_values": { "get": { "summary": "Get a list of active or raised alarms on the server", - "description": "The alarms_values endpoint returns the list of all raised or enabled alarms on the netdata server. Called without any parameters, the raised alarms in state WARNING or CRITICAL are returned. By passing \"?all\", all the enabled alarms are returned. This option output differs from `/alarms` in the number of variables delivered. This endpoint gives to user `id`, `value` and alarm `status`.", + "description": "The alarms_values endpoint returns the list of all raised or enabled alarms on the netdata server. Called without any parameters, the raised alarms in state WARNING or CRITICAL are returned. By passing \"?all\", all the enabled alarms are returned. This option output differs from `/alarms` in the number of variables delivered. This endpoint gives to user `id`, `value`, `last_updated` time and alarm `status`.", "parameters": [ { "name": "all", @@ -2099,6 +2109,10 @@ "value": { "type": "integer" }, + "last_updated": { + "type": "integer", + "format": "int32" + }, "status": { "type": "string", "enum": [ diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml index 1e20ad0f5..96920375e 100644 --- a/web/api/netdata-swagger.yaml +++ b/web/api/netdata-swagger.yaml @@ -593,6 +593,13 @@ paths: - prometheus_all_hosts - json default: shell + - name: filter + in: query + description: Allows to filter charts out using simple patterns. + required: false + schema: + type: string + format: any text - name: variables in: query description: When enabled, netdata will expose various system @@ -738,7 +745,7 @@ paths: state WARNING or CRITICAL are returned. By passing "?all", all the enabled alarms are returned. This option output differs from `/alarms` in the number of variables delivered. This endpoint gives - to user `id`, `value` and alarm `status`. + to user `id`, `value`, `last_updated` time, and alarm `status`. parameters: - name: all in: query @@ -1643,6 +1650,9 @@ components: type: integer value: type: integer + last_updated: + type: integer + format: int32 status: type: string enum: diff --git a/web/api/queries/average/average.c b/web/api/queries/average/average.c index 2c64358e6..2ed33da50 100644 --- a/web/api/queries/average/average.c +++ b/web/api/queries/average/average.c @@ -10,9 +10,8 @@ struct grouping_average { size_t count; }; -void *grouping_create_average(RRDR *r) { - (void)r; - return callocz(1, sizeof(struct grouping_average)); +void grouping_create_average(RRDR *r) { + r->internal.grouping_data = callocz(1, sizeof(struct grouping_average)); } // resets when switches dimensions @@ -29,11 +28,9 @@ void grouping_free_average(RRDR *r) { } void grouping_add_average(RRDR *r, calculated_number value) { - if(!isnan(value)) { - struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; - g->sum += value; - g->count++; - } + struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; + g->sum += value; + g->count++; } calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { diff --git a/web/api/queries/average/average.h b/web/api/queries/average/average.h index 9fb7de21a..23ecfac6f 100644 --- a/web/api/queries/average/average.h +++ b/web/api/queries/average/average.h @@ -6,7 +6,7 @@ #include "../query.h" #include "../rrdr.h" -extern void *grouping_create_average(RRDR *r); +extern void grouping_create_average(RRDR *r); extern void grouping_reset_average(RRDR *r); extern void grouping_free_average(RRDR *r); extern void grouping_add_average(RRDR *r, calculated_number value); diff --git a/web/api/queries/des/des.c b/web/api/queries/des/des.c index c6236f31a..8e4ca4bd4 100644 --- a/web/api/queries/des/des.c +++ b/web/api/queries/des/des.c @@ -69,14 +69,14 @@ static inline void set_beta(RRDR *r, struct grouping_des *g) { //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta); } -void *grouping_create_des(RRDR *r) { - struct grouping_des *g = (struct grouping_des *)malloc(sizeof(struct grouping_des)); +void grouping_create_des(RRDR *r) { + struct grouping_des *g = (struct grouping_des *)mallocz(sizeof(struct grouping_des)); set_alpha(r, g); set_beta(r, g); g->level = 0.0; g->trend = 0.0; g->count = 0; - return g; + r->internal.grouping_data = g; } // resets when switches dimensions @@ -99,28 +99,26 @@ void grouping_free_des(RRDR *r) { void grouping_add_des(RRDR *r, calculated_number value) { struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; - if(calculated_number_isnumber(value)) { - if(likely(g->count > 0)) { - // we have at least a number so far + if(likely(g->count > 0)) { + // we have at least a number so far - if(unlikely(g->count == 1)) { - // the second value we got - g->trend = value - g->trend; - g->level = value; - } - - // for the values, except the first - calculated_number last_level = g->level; - g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); - g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); - } - else { - // the first value we got - g->level = g->trend = value; + if(unlikely(g->count == 1)) { + // the second value we got + g->trend = value - g->trend; + g->level = value; } - g->count++; + // for the values, except the first + calculated_number last_level = g->level; + g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); + g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); } + else { + // the first value we got + g->level = g->trend = value; + } + + g->count++; //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend); } diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h index 360513e9c..bd361b865 100644 --- a/web/api/queries/des/des.h +++ b/web/api/queries/des/des.h @@ -8,7 +8,7 @@ extern void grouping_init_des(void); -extern void *grouping_create_des(RRDR *r); +extern void grouping_create_des(RRDR *r); extern void grouping_reset_des(RRDR *r); extern void grouping_free_des(RRDR *r); extern void grouping_add_des(RRDR *r, calculated_number value); diff --git a/web/api/queries/incremental_sum/incremental_sum.c b/web/api/queries/incremental_sum/incremental_sum.c index 131d85d78..304d9aa74 100644 --- a/web/api/queries/incremental_sum/incremental_sum.c +++ b/web/api/queries/incremental_sum/incremental_sum.c @@ -11,9 +11,8 @@ struct grouping_incremental_sum { size_t count; }; -void *grouping_create_incremental_sum(RRDR *r) { - (void)r; - return callocz(1, sizeof(struct grouping_incremental_sum)); +void grouping_create_incremental_sum(RRDR *r) { + r->internal.grouping_data = callocz(1, sizeof(struct grouping_incremental_sum)); } // resets when switches dimensions @@ -31,17 +30,15 @@ void grouping_free_incremental_sum(RRDR *r) { } void grouping_add_incremental_sum(RRDR *r, calculated_number value) { - if(!isnan(value)) { - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; + struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; - if(unlikely(!g->count)) { - g->first = value; - g->count++; - } - else { - g->last = value; - g->count++; - } + if(unlikely(!g->count)) { + g->first = value; + g->count++; + } + else { + g->last = value; + g->count++; } } diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h index 990a2ac4a..5b55ad3c8 100644 --- a/web/api/queries/incremental_sum/incremental_sum.h +++ b/web/api/queries/incremental_sum/incremental_sum.h @@ -6,7 +6,7 @@ #include "../query.h" #include "../rrdr.h" -extern void *grouping_create_incremental_sum(RRDR *r); +extern void grouping_create_incremental_sum(RRDR *r); extern void grouping_reset_incremental_sum(RRDR *r); extern void grouping_free_incremental_sum(RRDR *r); extern void grouping_add_incremental_sum(RRDR *r, calculated_number value); diff --git a/web/api/queries/max/max.c b/web/api/queries/max/max.c index a4be36add..b6e723314 100644 --- a/web/api/queries/max/max.c +++ b/web/api/queries/max/max.c @@ -10,9 +10,8 @@ struct grouping_max { size_t count; }; -void *grouping_create_max(RRDR *r) { - (void)r; - return callocz(1, sizeof(struct grouping_max)); +void grouping_create_max(RRDR *r) { + r->internal.grouping_data = callocz(1, sizeof(struct grouping_max)); } // resets when switches dimensions @@ -29,13 +28,11 @@ void grouping_free_max(RRDR *r) { } void grouping_add_max(RRDR *r, calculated_number value) { - if(!isnan(value)) { - struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; + struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; - if(!g->count || calculated_number_fabs(value) > calculated_number_fabs(g->max)) { - g->max = value; - g->count++; - } + if(!g->count || calculated_number_fabs(value) > calculated_number_fabs(g->max)) { + g->max = value; + g->count++; } } diff --git a/web/api/queries/max/max.h b/web/api/queries/max/max.h index d839fe3f9..7b606ce34 100644 --- a/web/api/queries/max/max.h +++ b/web/api/queries/max/max.h @@ -6,7 +6,7 @@ #include "../query.h" #include "../rrdr.h" -extern void *grouping_create_max(RRDR *r); +extern void grouping_create_max(RRDR *r); extern void grouping_reset_max(RRDR *r); extern void grouping_free_max(RRDR *r); extern void grouping_add_max(RRDR *r, calculated_number value); diff --git a/web/api/queries/median/median.c b/web/api/queries/median/median.c index 31916c546..bffcee12f 100644 --- a/web/api/queries/median/median.c +++ b/web/api/queries/median/median.c @@ -13,14 +13,14 @@ struct grouping_median { LONG_DOUBLE series[]; }; -void *grouping_create_median(RRDR *r) { +void grouping_create_median(RRDR *r) { long entries = r->group; if(entries < 0) entries = 0; struct grouping_median *g = (struct grouping_median *)callocz(1, sizeof(struct grouping_median) + entries * sizeof(LONG_DOUBLE)); g->series_size = (size_t)entries; - return g; + r->internal.grouping_data = g; } // resets when switches dimensions @@ -41,10 +41,8 @@ void grouping_add_median(RRDR *r, calculated_number value) { if(unlikely(g->next_pos >= g->series_size)) { error("INTERNAL ERROR: median buffer overflow on chart '%s' - next_pos = %zu, series_size = %zu, r->group = %ld.", r->st->name, g->next_pos, g->series_size, r->group); } - else { - if(calculated_number_isnumber(value)) - g->series[g->next_pos++] = (LONG_DOUBLE)value; - } + else + g->series[g->next_pos++] = (LONG_DOUBLE)value; } calculated_number grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h index dd2c1ffc5..28d52b31e 100644 --- a/web/api/queries/median/median.h +++ b/web/api/queries/median/median.h @@ -6,7 +6,7 @@ #include "../query.h" #include "../rrdr.h" -extern void *grouping_create_median(RRDR *r); +extern void grouping_create_median(RRDR *r); extern void grouping_reset_median(RRDR *r); extern void grouping_free_median(RRDR *r); extern void grouping_add_median(RRDR *r, calculated_number value); diff --git a/web/api/queries/min/min.c b/web/api/queries/min/min.c index 9bd7460e0..497bae04d 100644 --- a/web/api/queries/min/min.c +++ b/web/api/queries/min/min.c @@ -10,9 +10,8 @@ struct grouping_min { size_t count; }; -void *grouping_create_min(RRDR *r) { - (void)r; - return callocz(1, sizeof(struct grouping_min)); +void grouping_create_min(RRDR *r) { + r->internal.grouping_data = callocz(1, sizeof(struct grouping_min)); } // resets when switches dimensions @@ -29,13 +28,11 @@ void grouping_free_min(RRDR *r) { } void grouping_add_min(RRDR *r, calculated_number value) { - if(!isnan(value)) { - struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data; + struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data; - if(!g->count || calculated_number_fabs(value) < calculated_number_fabs(g->min)) { - g->min = value; - g->count++; - } + if(!g->count || calculated_number_fabs(value) < calculated_number_fabs(g->min)) { + g->min = value; + g->count++; } } diff --git a/web/api/queries/min/min.h b/web/api/queries/min/min.h index 74703605c..9207c74f7 100644 --- a/web/api/queries/min/min.h +++ b/web/api/queries/min/min.h @@ -6,7 +6,7 @@ #include "../query.h" #include "../rrdr.h" -extern void *grouping_create_min(RRDR *r); +extern void grouping_create_min(RRDR *r); extern void grouping_reset_min(RRDR *r); extern void grouping_free_min(RRDR *r); extern void grouping_add_min(RRDR *r, calculated_number value); diff --git a/web/api/queries/query.c b/web/api/queries/query.c index c55a97060..5c6c70411 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -3,6 +3,7 @@ #include "query.h" #include "web/api/formatters/rrd2json.h" #include "rrdr.h" +#include "database/ram/rrddim_mem.h" #include "average/average.h" #include "incremental_sum/incremental_sum.h" @@ -27,7 +28,7 @@ static struct { // Allocate all required structures for a query. // This is called once for each netdata query. - void *(*create)(struct rrdresult *r); + void (*create)(struct rrdresult *r); // Cleanup collected values, but don't destroy the structures. // This is called when the query engine switches dimensions, @@ -464,7 +465,9 @@ static inline void do_dimension_variablestep( } } // add this value to grouping - r->internal.grouping_add(r, value); + if(likely(!isnan(value))) + r->internal.grouping_add(r, value); + values_in_group++; db_points_read++; } @@ -537,25 +540,18 @@ static inline void do_dimension_fixedstep( , time_t before_wanted , uint32_t options ){ -#ifdef NETDATA_INTERNAL_CHECKS - RRDSET *st = r->st; -#endif - - time_t - now = after_wanted, + time_t now = after_wanted, dt = r->update_every / r->group, /* usually is st->update_every */ max_date = 0, min_date = 0; - long - group_size = r->group, + long group_size = r->group, points_added = 0, values_in_group = 0, values_in_group_non_zero = 0, rrdr_line = -1; - RRDR_VALUE_FLAGS - group_value_flags = RRDR_VALUE_NOTHING; + RRDR_VALUE_FLAGS group_value_flags = RRDR_VALUE_NOTHING; struct rrddim_query_handle handle; @@ -563,6 +559,13 @@ static inline void do_dimension_fixedstep( size_t db_points_read = 0; time_t db_now = now; time_t first_time_t = rrddim_first_entry_t(rd); + + // cache the function pointers we need in the loop + storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time) = rd->state->query_ops.next_metric; + void (*grouping_add)(struct rrdresult *r, calculated_number value) = r->internal.grouping_add; + calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) = r->internal.grouping_flush; + RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode; + for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) { // make sure we return data in the proper time range if(unlikely(now > before_wanted)) { @@ -571,44 +574,74 @@ static inline void do_dimension_fixedstep( #endif break; } + if(unlikely(now < after_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS r->internal.log = "skipped, because attempted to access the db before 'wanted after'"; #endif continue; } + // read the value from the database //storage_number n = rd->values[slot]; + #ifdef NETDATA_INTERNAL_CHECKS - if ((rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) && - (rrdset_time2slot(st, now) != (long unsigned)handle.slotted.slot)) { - error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)handle.slotted.slot, rrdset_time2slot(st, now)); + struct mem_query_handle* mem_handle = (struct mem_query_handle*)handle.handle; + if ((rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) && + (rrdset_time2slot(r->st, now) != (long unsigned)(mem_handle->slot))) { + error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)mem_handle->slot, rrdset_time2slot(r->st, now)); } #endif + db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it + storage_number n; - if (rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t) + calculated_number value; + + if (unlikely(rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t)) { n = SN_EMPTY_SLOT; - else - n = rd->state->query_ops.next_metric(&handle, &db_now); + value = NAN; + } + else { + // load the metric value + n = next_metric(&handle, &db_now); + db_points_read++; + + // and unpack it + if(likely(does_storage_number_exist(n))) { + if (options & RRDR_OPTION_ANOMALY_BIT) + value = (n & SN_ANOMALY_BIT) ? 0.0 : 100.0; + else + value = unpack_storage_number(n); + } + else + value = NAN; + } + if(unlikely(db_now > before_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; #endif break; } - for ( ; now <= db_now ; now += dt) { - calculated_number value = NAN; - if(likely(now >= db_now && does_storage_number_exist(n))) { + + // this loop exists only to fill nulls + // so, if there is a value already, we use it for the first iteration + // but the following iterations will just fill nulls to the destination + for ( ; now <= db_now ; now += dt, value = NAN, n = SN_EMPTY_SLOT) { + if(likely(does_storage_number_exist(n))) { + #if defined(NETDATA_INTERNAL_CHECKS) && defined(ENABLE_DBENGINE) - if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != handle.rrdeng.now)) { - error("INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", rd->id, (long)handle.rrdeng.now, (long)now); + if(now >= db_now) { + struct rrdeng_query_handle *rrd_handle = (struct rrdeng_query_handle *)handle.handle; + if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != rrd_handle->now)) + error( + "INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", + rd->id, + (long)rrd_handle->now, + (long)now); } #endif - if (options & RRDR_OPTION_ANOMALY_BIT) - value = (n & SN_ANOMALY_BIT) ? 0.0 : 100.0; - else - value = unpack_storage_number(n); if(likely(value != 0.0)) values_in_group_non_zero++; @@ -616,21 +649,21 @@ static inline void do_dimension_fixedstep( if(unlikely(did_storage_number_reset(n))) group_value_flags |= RRDR_VALUE_RESET; + grouping_add(r, value); } // add this value for grouping - r->internal.grouping_add(r, value); values_in_group++; - db_points_read++; if(unlikely(values_in_group == group_size)) { rrdr_line = rrdr_line_init(r, now, rrdr_line); + size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr; if(unlikely(!min_date)) min_date = now; max_date = now; // find the place to store our values - RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr]; + RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index]; // update the dimension options if(likely(values_in_group_non_zero)) @@ -639,21 +672,21 @@ static inline void do_dimension_fixedstep( // store the specific point options *rrdr_value_options_ptr = group_value_flags; - // store the value - calculated_number value = r->internal.grouping_flush(r, rrdr_value_options_ptr); - r->v[rrdr_line * r->d + dim_id_in_rrdr] = value; + // store the group value + calculated_number group_value = grouping_flush(r, rrdr_value_options_ptr); + r->v[rrdr_o_v_index] = group_value; if(likely(points_added || dim_id_in_rrdr)) { // find the min/max across all dimensions - if(unlikely(value < min)) min = value; - if(unlikely(value > max)) max = value; + if(unlikely(group_value < min)) min = group_value; + if(unlikely(group_value > max)) max = group_value; } else { // runs only when dim_id_in_rrdr == 0 && points_added == 0 // so, on the first point added for the query. - min = max = value; + min = max = group_value; } points_added++; @@ -831,7 +864,8 @@ static int rrdr_convert_before_after_to_absolute( } static RRDR *rrd2rrdr_fixedstep( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , long points_requested , long long after_requested , long long before_requested @@ -855,7 +889,7 @@ static RRDR *rrd2rrdr_fixedstep( RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; if(duration <= 0 || available_points <= 0) - return rrdr_create(st, 1, context_param_list); + return rrdr_create(owa, st, 1, context_param_list); // check the number of wanted points in the result if(unlikely(points_requested < 0)) points_requested = -points_requested; @@ -1013,7 +1047,7 @@ static RRDR *rrd2rrdr_fixedstep( // initialize our result set // this also locks the chart for us - RRDR *r = rrdr_create(st, points_wanted, context_param_list); + RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); if(unlikely(!r)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); @@ -1077,7 +1111,7 @@ static RRDR *rrd2rrdr_fixedstep( } // allocate any memory required by the grouping method - r->internal.grouping_data = r->internal.grouping_create(r); + r->internal.grouping_create(r); // ------------------------------------------------------------------------- @@ -1216,7 +1250,8 @@ static RRDR *rrd2rrdr_fixedstep( #ifdef ENABLE_DBENGINE static RRDR *rrd2rrdr_variablestep( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , long points_requested , long long after_requested , long long before_requested @@ -1242,7 +1277,7 @@ static RRDR *rrd2rrdr_variablestep( if(duration <= 0 || available_points <= 0) { freez(region_info_array); - return rrdr_create(st, 1, context_param_list); + return rrdr_create(owa, st, 1, context_param_list); } // check the number of wanted points in the result @@ -1401,7 +1436,7 @@ static RRDR *rrd2rrdr_variablestep( // initialize our result set // this also locks the chart for us - RRDR *r = rrdr_create(st, points_wanted, context_param_list); + RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); if(unlikely(!r)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); @@ -1468,7 +1503,7 @@ static RRDR *rrd2rrdr_variablestep( } // allocate any memory required by the grouping method - r->internal.grouping_data = r->internal.grouping_create(r); + r->internal.grouping_create(r); // ------------------------------------------------------------------------- @@ -1608,7 +1643,8 @@ static RRDR *rrd2rrdr_variablestep( #endif //#ifdef ENABLE_DBENGINE RRDR *rrd2rrdr( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , long points_requested , long long after_requested , long long before_requested @@ -1644,7 +1680,7 @@ RRDR *rrd2rrdr( first_entry_t = after_requested; if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { - rebuild_context_param_list(context_param_list, after_requested); + rebuild_context_param_list(owa, context_param_list, after_requested); st = context_param_list->rd ? context_param_list->rd->rrdset : NULL; if (unlikely(!st)) return NULL; @@ -1669,7 +1705,7 @@ RRDR *rrd2rrdr( } freez(region_info_array); } - return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, + return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); } else { @@ -1680,13 +1716,13 @@ RRDR *rrd2rrdr( rrd_update_every, first_entry_t, last_entry_t, options); } - return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method, + return rrd2rrdr_variablestep(owa, st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout); } } #endif - return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, + return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); } diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c index b64868222..4d05778c1 100644 --- a/web/api/queries/rrdr.c +++ b/web/api/queries/rrdr.c @@ -78,12 +78,12 @@ inline static void rrdr_unlock_rrdset(RRDR *r) { } if(likely(r->has_st_lock)) { - rrdset_unlock(r->st); r->has_st_lock = 0; + rrdset_unlock(r->st); } } -inline void rrdr_free(RRDR *r) +inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) { if(unlikely(!r)) { error("NULL value given!"); @@ -91,21 +91,21 @@ inline void rrdr_free(RRDR *r) } rrdr_unlock_rrdset(r); - freez(r->t); - freez(r->v); - freez(r->o); - freez(r->od); - freez(r); + onewayalloc_freez(owa, r->t); + onewayalloc_freez(owa, r->v); + onewayalloc_freez(owa, r->o); + onewayalloc_freez(owa, r->od); + onewayalloc_freez(owa, r); } -RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list) +RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) { if (unlikely(!st)) { error("NULL value given!"); return NULL; } - RRDR *r = callocz(1, sizeof(RRDR)); + RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR)); r->st = st; if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { @@ -126,10 +126,10 @@ RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param r->n = n; - r->t = callocz((size_t)n, sizeof(time_t)); - r->v = mallocz(n * r->d * sizeof(calculated_number)); - r->o = mallocz(n * r->d * sizeof(RRDR_VALUE_FLAGS)); - r->od = mallocz(r->d * sizeof(RRDR_DIMENSION_FLAGS)); + r->t = onewayalloc_callocz(owa, (size_t)n, sizeof(time_t)); + r->v = onewayalloc_mallocz(owa, n * r->d * sizeof(calculated_number)); + r->o = onewayalloc_mallocz(owa, n * r->d * sizeof(RRDR_VALUE_FLAGS)); + r->od = onewayalloc_mallocz(owa, r->d * sizeof(RRDR_DIMENSION_FLAGS)); // set the hidden flag on hidden dimensions int c; diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index bd94e56e2..87ba6c86b 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -83,7 +83,7 @@ typedef struct rrdresult { long resampling_group; calculated_number resampling_divisor; - void *(*grouping_create)(struct rrdresult *r); + void (*grouping_create)(struct rrdresult *r); void (*grouping_reset)(struct rrdresult *r); void (*grouping_free)(struct rrdresult *r); void (*grouping_add)(struct rrdresult *r, calculated_number value); @@ -102,13 +102,14 @@ typedef struct rrdresult { #define rrdr_rows(r) ((r)->rows) #include "database/rrd.h" -extern void rrdr_free(RRDR *r); -extern RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list); +extern void rrdr_free(ONEWAYALLOC *owa, RRDR *r); +extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list); #include "../web_api_v1.h" #include "web/api/queries/query.h" extern RRDR *rrd2rrdr( + ONEWAYALLOC *owa, RRDSET *st, long points_requested, long long after_requested, long long before_requested, RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions, struct context_param *context_param_list, int timeout); diff --git a/web/api/queries/ses/ses.c b/web/api/queries/ses/ses.c index 772505f93..ae4a0fa0d 100644 --- a/web/api/queries/ses/ses.c +++ b/web/api/queries/ses/ses.c @@ -48,11 +48,11 @@ static inline void set_alpha(RRDR *r, struct grouping_ses *g) { g->alpha_other = 1.0 - g->alpha; } -void *grouping_create_ses(RRDR *r) { +void grouping_create_ses(RRDR *r) { struct grouping_ses *g = (struct grouping_ses *)callocz(1, sizeof(struct grouping_ses)); set_alpha(r, g); g->level = 0.0; - return g; + r->internal.grouping_data = g; } // resets when switches dimensions @@ -71,13 +71,11 @@ void grouping_free_ses(RRDR *r) { void grouping_add_ses(RRDR *r, calculated_number value) { struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data; - if(calculated_number_isnumber(value)) { - if(unlikely(!g->count)) - g->level = value; + if(unlikely(!g->count)) + g->level = value; - g->level = g->alpha * value + g->alpha_other * g->level; - g->count++; - } + g->level = g->alpha * value + g->alpha_other * g->level; + g->count++; } calculated_number grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { diff --git a/web/api/queries/ses/ses.h b/web/api/queries/ses/ses.h index 603fdb57c..c05f208f3 100644 --- a/web/api/queries/ses/ses.h +++ b/web/api/queries/ses/ses.h @@ -8,7 +8,7 @@ extern void grouping_init_ses(void); -extern void *grouping_create_ses(RRDR *r); +extern void grouping_create_ses(RRDR *r); extern void grouping_reset_ses(RRDR *r); extern void grouping_free_ses(RRDR *r); extern void grouping_add_ses(RRDR *r, calculated_number value); diff --git a/web/api/queries/stddev/stddev.c b/web/api/queries/stddev/stddev.c index 16258445c..ffe7a47c0 100644 --- a/web/api/queries/stddev/stddev.c +++ b/web/api/queries/stddev/stddev.c @@ -14,9 +14,8 @@ struct grouping_stddev { calculated_number m_oldM, m_newM, m_oldS, m_newS; }; -void *grouping_create_stddev(RRDR *r) { - UNUSED (r); - return callocz(1, sizeof(struct grouping_stddev)); +void grouping_create_stddev(RRDR *r) { + r->internal.grouping_data = callocz(1, sizeof(struct grouping_stddev)); } // resets when switches dimensions @@ -34,22 +33,20 @@ void grouping_free_stddev(RRDR *r) { void grouping_add_stddev(RRDR *r, calculated_number value) { struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - if(calculated_number_isnumber(value)) { - g->count++; + g->count++; - // See Knuth TAOCP vol 2, 3rd edition, page 232 - if (g->count == 1) { - g->m_oldM = g->m_newM = value; - g->m_oldS = 0.0; - } - else { - g->m_newM = g->m_oldM + (value - g->m_oldM) / g->count; - g->m_newS = g->m_oldS + (value - g->m_oldM) * (value - g->m_newM); + // See Knuth TAOCP vol 2, 3rd edition, page 232 + if (g->count == 1) { + g->m_oldM = g->m_newM = value; + g->m_oldS = 0.0; + } + else { + g->m_newM = g->m_oldM + (value - g->m_oldM) / g->count; + g->m_newS = g->m_oldS + (value - g->m_oldM) * (value - g->m_newM); - // set up for next iteration - g->m_oldM = g->m_newM; - g->m_oldS = g->m_newS; - } + // set up for next iteration + g->m_oldM = g->m_newM; + g->m_oldS = g->m_newS; } } diff --git a/web/api/queries/stddev/stddev.h b/web/api/queries/stddev/stddev.h index 7a4697572..ab58fbe50 100644 --- a/web/api/queries/stddev/stddev.h +++ b/web/api/queries/stddev/stddev.h @@ -6,7 +6,7 @@ #include "../query.h" #include "../rrdr.h" -extern void *grouping_create_stddev(RRDR *r); +extern void grouping_create_stddev(RRDR *r); extern void grouping_reset_stddev(RRDR *r); extern void grouping_free_stddev(RRDR *r); extern void grouping_add_stddev(RRDR *r, calculated_number value); diff --git a/web/api/queries/sum/sum.c b/web/api/queries/sum/sum.c index 0da9937a3..6bb012bb0 100644 --- a/web/api/queries/sum/sum.c +++ b/web/api/queries/sum/sum.c @@ -10,9 +10,8 @@ struct grouping_sum { size_t count; }; -void *grouping_create_sum(RRDR *r) { - (void)r; - return callocz(1, sizeof(struct grouping_sum)); +void grouping_create_sum(RRDR *r) { + r->internal.grouping_data = callocz(1, sizeof(struct grouping_sum)); } // resets when switches dimensions @@ -29,11 +28,9 @@ void grouping_free_sum(RRDR *r) { } void grouping_add_sum(RRDR *r, calculated_number value) { - if(!isnan(value)) { - struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data; - g->sum += value; - g->count++; - } + struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data; + g->sum += value; + g->count++; } calculated_number grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { diff --git a/web/api/queries/sum/sum.h b/web/api/queries/sum/sum.h index 9dc8d209c..05cb6185e 100644 --- a/web/api/queries/sum/sum.h +++ b/web/api/queries/sum/sum.h @@ -6,7 +6,7 @@ #include "../query.h" #include "../rrdr.h" -extern void *grouping_create_sum(RRDR *r); +extern void grouping_create_sum(RRDR *r); extern void grouping_reset_sum(RRDR *r); extern void grouping_free_sum(RRDR *r); extern void grouping_add_sum(RRDR *r, calculated_number value); diff --git a/web/api/tests/valid_urls.c b/web/api/tests/valid_urls.c index 30dc29828..91cd19b09 100644 --- a/web/api/tests/valid_urls.c +++ b/web/api/tests/valid_urls.c @@ -8,6 +8,18 @@ #include <cmocka.h> #include <stdbool.h> +void free_temporary_host(RRDHOST *host) +{ + (void) host; +} + +void *__wrap_free_temporary_host(RRDHOST *host) +{ + (void) host; + return NULL; +} + + RRDHOST *sql_create_host_by_uuid(char *hostname) { (void) hostname; diff --git a/web/api/tests/web_api.c b/web/api/tests/web_api.c index df4efdabd..fd9a86ef6 100644 --- a/web/api/tests/web_api.c +++ b/web/api/tests/web_api.c @@ -8,6 +8,17 @@ #include <cmocka.h> #include <stdbool.h> +void free_temporary_host(RRDHOST *host) +{ + (void) host; +} + +void *__wrap_free_temporary_host(RRDHOST *host) +{ + (void) host; + return NULL; +} + RRDHOST *sql_create_host_by_uuid(char *hostname) { (void) hostname; diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index 8cf89d38d..cb73f7c02 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -422,6 +422,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c char *chart_labels_filter = NULL; int group = RRDR_GROUPING_AVERAGE; + int show_dimensions = 0; uint32_t format = DATASOURCE_JSON; uint32_t options = 0x00000000; @@ -447,6 +448,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c buffer_strcat(dimensions, "|"); buffer_strcat(dimensions, value); } + else if(!strcmp(name, "show_dimensions")) show_dimensions = 1; else if(!strcmp(name, "after")) after_str = value; else if(!strcmp(name, "before")) before_str = value; else if(!strcmp(name, "points")) points_str = value; @@ -512,6 +514,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c fix_google_param(outFileName); RRDSET *st = NULL; + ONEWAYALLOC *owa = onewayalloc_create(0); if((!chart || !*chart) && (!context)) { buffer_sprintf(w->response.data, "No chart id is given at the request."); @@ -519,8 +522,10 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c } struct context_param *context_param_list = NULL; + if (context && !chart) { RRDSET *st1; + uint32_t context_hash = simple_hash(context); rrdhost_rdlock(host); @@ -532,14 +537,14 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c (!chart_label_key || rrdset_contains_label_keylist(st1, chart_label_key)) && (!chart_labels_filter || rrdset_matches_label_keys(st1, chart_labels_filter, words, hash_key_list, &word_count, MAX_CHART_LABELS_FILTER))) - build_context_param_list(&context_param_list, st1); + build_context_param_list(owa, &context_param_list, st1); } rrdhost_unlock(host); if (likely(context_param_list && context_param_list->rd)) // Just set the first one st = context_param_list->rd->rrdset; else { if (!chart_label_key && !chart_labels_filter) - sql_build_context_param_list(&context_param_list, host, context, NULL); + sql_build_context_param_list(owa, &context_param_list, host, context, NULL); } } else { @@ -549,14 +554,14 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c if (likely(st)) st->last_accessed_time = now_realtime_sec(); else - sql_build_context_param_list(&context_param_list, host, NULL, chart); + sql_build_context_param_list(owa, &context_param_list, host, NULL, chart); } if (!st) { if (likely(context_param_list && context_param_list->rd && context_param_list->rd->rrdset)) st = context_param_list->rd->rrdset; else { - free_context_param_list(&context_param_list); + free_context_param_list(owa, &context_param_list); context_param_list = NULL; } } @@ -589,6 +594,18 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c long group_time = (group_time_str && *group_time_str)?str2l(group_time_str):0; int max_anomaly_rates = (max_anomaly_rates_str && *max_anomaly_rates_str) ? str2i(max_anomaly_rates_str) : 0; + if (timeout) { + struct timeval now; + now_realtime_timeval(&now); + int inqueue = (int)dt_usec(&w->tv_in, &now) / 1000; + timeout -= inqueue; + if (timeout <= 0) { + buffer_flush(w->response.data); + buffer_strcat(w->response.data, "Query timeout exceeded"); + return HTTP_RESP_BACKEND_FETCH_FAILED; + } + } + debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', format '%u', options '0x%08x'" , w->id , chart @@ -630,12 +647,18 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c buffer_strcat(w->response.data, "("); } - ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, - points, after, before, group, group_time, - options, &last_timestamp_in_data, context_param_list, - chart_label_key, max_anomaly_rates, timeout); + QUERY_PARAMS query_params = { + .context_param_list = context_param_list, + .timeout = timeout, + .max_anomaly_rates = max_anomaly_rates, + .show_dimensions = show_dimensions, + .chart_label_key = chart_label_key, + .wb = w->response.data}; + + ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format, + points, after, before, group, group_time, options, &last_timestamp_in_data); - free_context_param_list(&context_param_list); + free_context_param_list(owa, &context_param_list); if(format == DATASOURCE_DATATABLE_JSONP) { if(google_timestamp < last_timestamp_in_data) @@ -652,7 +675,8 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c else if(format == DATASOURCE_JSONP) buffer_strcat(w->response.data, ");"); - cleanup: +cleanup: + onewayalloc_destroy(owa); buffer_free(dimensions); return ret; } @@ -899,23 +923,27 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) { netdata_mutex_lock(&host->receiver_lock); buffer_sprintf( - wb, "\t\t{ \"guid\": \"%s\", \"reachable\": %s, \"hops\": %d, \"claim_id\": ", host->machine_guid, - (host->receiver || host == localhost) ? "true" : "false", host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1); + wb, "\t\t{ \"guid\": \"%s\", \"hostname\": \"%s\", \"reachable\": %s, \"hops\": %d" + , host->machine_guid + , host->hostname + , (host->receiver || host == localhost) ? "true" : "false" + , host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1 + ); netdata_mutex_unlock(&host->receiver_lock); rrdhost_aclk_state_lock(host); if (host->aclk_state.claimed_id) - buffer_sprintf(wb, "\"%s\", ", host->aclk_state.claimed_id); + buffer_sprintf(wb, ", \"claim_id\": \"%s\"", host->aclk_state.claimed_id); else - buffer_strcat(wb, "null, "); + buffer_strcat(wb, ", \"claim_id\": null"); rrdhost_aclk_state_unlock(host); if (host->node_id) { char node_id_str[GUID_LEN + 1]; uuid_unparse_lower(*host->node_id, node_id_str); - buffer_sprintf(wb, "\"node_id\": \"%s\" }", node_id_str); + buffer_sprintf(wb, ", \"node_id\": \"%s\" }", node_id_str); } else - buffer_strcat(wb, "\"node_id\": null }"); + buffer_strcat(wb, ", \"node_id\": null }"); count++; } @@ -1295,6 +1323,50 @@ static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client return HTTP_RESP_OK; } +int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_client *w, char *url) { + if (!netdata_ready) + return HTTP_RESP_BACKEND_FETCH_FAILED; + + long long baseline_after = 0, baseline_before = 0, highlight_after = 0, highlight_before = 0, max_points = 0; + + while (url) { + char *value = mystrsep(&url, "&"); + if (!value || !*value) + continue; + + char *name = mystrsep(&value, "="); + if (!name || !*name) + continue; + if (!value || !*value) + continue; + + if (!strcmp(name, "baseline_after")) + baseline_after = (long long) strtoul(value, NULL, 0); + else if (!strcmp(name, "baseline_before")) + baseline_before = (long long) strtoul(value, NULL, 0); + else if (!strcmp(name, "highlight_after")) + highlight_after = (long long) strtoul(value, NULL, 0); + else if (!strcmp(name, "highlight_before")) + highlight_before = (long long) strtoul(value, NULL, 0); + else if (!strcmp(name, "max_points")) + max_points = (long long) strtoul(value, NULL, 0); + + } + + BUFFER *wb = w->response.data; + buffer_flush(wb); + wb->contenttype = CT_APPLICATION_JSON; + buffer_no_cacheable(wb); + + if (!highlight_after || !highlight_before) + buffer_strcat(wb, "{\"error\": \"Missing or invalid required highlight after and before parameters.\" }"); + else { + metric_correlations(host, wb, baseline_after, baseline_before, highlight_after, highlight_before, max_points); + } + + return HTTP_RESP_OK; +} + static struct api_command { const char *command; uint32_t hash; @@ -1326,8 +1398,9 @@ static struct api_command { { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info }, #endif - { "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health }, - { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state }, + { "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health }, + { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state }, + { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_metric_correlations }, // terminator { NULL, 0, WEB_CLIENT_ACL_NONE, NULL }, }; |