diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2022-08-12 07:26:17 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2022-08-12 07:26:17 +0000 |
| commit | 7877a98bd9c00db5e81dd2f8c734cba2bab20be7 (patch) | |
| tree | d18b767250f7c7ced9b8abe2ece784ac1fe24d3e /web/api/queries | |
| parent | Releasing debian version 1.35.1-2. (diff) | |
| download | netdata-7877a98bd9c00db5e81dd2f8c734cba2bab20be7.tar.xz netdata-7877a98bd9c00db5e81dd2f8c734cba2bab20be7.zip | |
Merging upstream version 1.36.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'web/api/queries')
38 files changed, 3718 insertions, 1233 deletions
diff --git a/web/api/queries/Makefile.am b/web/api/queries/Makefile.am index 34bfdb85b..7c4c43520 100644 --- a/web/api/queries/Makefile.am +++ b/web/api/queries/Makefile.am @@ -5,14 +5,17 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in SUBDIRS = \ average \ + countif \ des \ incremental_sum \ max \ min \ sum \ median \ + percentile \ ses \ stddev \ + trimmed_mean \ $(NULL) dist_noinst_DATA = \ diff --git a/web/api/queries/average/average.c b/web/api/queries/average/average.c index 2ed33da50..0719d57fa 100644 --- a/web/api/queries/average/average.c +++ b/web/api/queries/average/average.c @@ -6,12 +6,12 @@ // average struct grouping_average { - calculated_number sum; + NETDATA_DOUBLE sum; size_t count; }; -void grouping_create_average(RRDR *r) { - r->internal.grouping_data = callocz(1, sizeof(struct grouping_average)); +void grouping_create_average(RRDR *r, const char *options __maybe_unused) { + r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_average)); } // resets when switches dimensions @@ -23,20 +23,20 @@ void grouping_reset_average(RRDR *r) { } void grouping_free_average(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_average(RRDR *r, calculated_number value) { +void grouping_add_average(RRDR *r, NETDATA_DOUBLE value) { struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; g->sum += value; g->count++; } -calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(unlikely(!g->count)) { value = 0.0; diff --git a/web/api/queries/average/average.h b/web/api/queries/average/average.h index 23ecfac6f..55c51722c 100644 --- a/web/api/queries/average/average.h +++ b/web/api/queries/average/average.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_average(RRDR *r); +extern void grouping_create_average(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_average(RRDR *r); extern void grouping_free_average(RRDR *r); -extern void grouping_add_average(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_average(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_AVERAGE_H diff --git a/web/api/queries/countif/Makefile.am b/web/api/queries/countif/Makefile.am new file mode 100644 index 000000000..161784b8f --- /dev/null +++ b/web/api/queries/countif/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/queries/countif/README.md b/web/api/queries/countif/README.md new file mode 100644 index 000000000..200a4c9ed --- /dev/null +++ b/web/api/queries/countif/README.md @@ -0,0 +1,36 @@ +<!-- +title: "CountIf" +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/countif/README.md +--> + +# CountIf + +> This query is available as `countif`. + +CountIf returns the percentage of points in the database that satisfy the condition supplied. + +The following conditions are available: + +- `!` or `!=` or `<>`, different than +- `=` or `:`, equal to +- `>`, greater than +- `<`, less than +- `>=`, greater or equal to +- `<=`, less or equal to + +The target number and the desired condition can be set using the `group_options` query parameter, as a string, like in these examples: + +- `!0`, to match any number except zero. +- `>=-3` to match any number bigger or equal to -3. + +. When an invalid condition is given, the web server can deliver a not accurate response. + +## how to use + +This query cannot be used in alarms. + +`countif` changes the units of charts. The result of the calculation is always from zero to 1, expressing the percentage of database points that matched the condition. + +In APIs and badges can be used like this: `&group=countif&group_options=>10` in the URL. + + diff --git a/web/api/queries/countif/countif.c b/web/api/queries/countif/countif.c new file mode 100644 index 000000000..369b20be9 --- /dev/null +++ b/web/api/queries/countif/countif.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "countif.h" + +// ---------------------------------------------------------------------------- +// countif + +struct grouping_countif { + size_t (*comparison)(NETDATA_DOUBLE, NETDATA_DOUBLE); + NETDATA_DOUBLE target; + size_t count; + size_t matched; +}; + +static size_t countif_equal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { + return (v == target); +} + +static size_t countif_notequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { + return (v != target); +} + +static size_t countif_less(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { + return (v < target); +} + +static size_t countif_lessequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { + return (v <= target); +} + +static size_t countif_greater(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { + return (v > target); +} + +static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { + return (v >= target); +} + +void grouping_create_countif(RRDR *r, const char *options __maybe_unused) { + struct grouping_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_countif)); + r->internal.grouping_data = g; + + if(options && *options) { + // skip any leading spaces + while(isspace(*options)) options++; + + // find the comparison function + switch(*options) { + case '!': + options++; + if(*options != '=' && *options != ':') + options--; + g->comparison = countif_notequal; + break; + + case '>': + options++; + if(*options == '=' || *options == ':') { + g->comparison = countif_greaterequal; + } + else { + options--; + g->comparison = countif_greater; + } + break; + + case '<': + options++; + if(*options == '>') { + g->comparison = countif_notequal; + } + else if(*options == '=' || *options == ':') { + g->comparison = countif_lessequal; + } + else { + options--; + g->comparison = countif_less; + } + break; + + default: + case '=': + case ':': + g->comparison = countif_equal; + break; + } + if(*options) options++; + + // skip everything up to the first digit + while(isspace(*options)) options++; + + g->target = str2ndd(options, NULL); + } + else { + g->target = 0.0; + g->comparison = countif_equal; + } +} + +// resets when switches dimensions +// so, clear everything to restart +void grouping_reset_countif(RRDR *r) { + struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; + g->matched = 0; + g->count = 0; +} + +void grouping_free_countif(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); + r->internal.grouping_data = NULL; +} + +void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value) { + struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; + g->matched += g->comparison(value, g->target); + g->count++; +} + +NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count; + } + + g->matched = 0; + g->count = 0; + + return value; +} diff --git a/web/api/queries/countif/countif.h b/web/api/queries/countif/countif.h new file mode 100644 index 000000000..0c7d2d7d1 --- /dev/null +++ b/web/api/queries/countif/countif.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_QUERY_COUNTIF_H +#define NETDATA_API_QUERY_COUNTIF_H + +#include "../query.h" +#include "../rrdr.h" + +extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused); +extern void grouping_reset_countif(RRDR *r); +extern void grouping_free_countif(RRDR *r); +extern void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + +#endif //NETDATA_API_QUERY_COUNTIF_H diff --git a/web/api/queries/des/des.c b/web/api/queries/des/des.c index 8e4ca4bd4..a6c4e4051 100644 --- a/web/api/queries/des/des.c +++ b/web/api/queries/des/des.c @@ -8,13 +8,13 @@ // single exponential smoothing struct grouping_des { - calculated_number alpha; - calculated_number alpha_other; - calculated_number beta; - calculated_number beta_other; + NETDATA_DOUBLE alpha; + NETDATA_DOUBLE alpha_other; + NETDATA_DOUBLE beta; + NETDATA_DOUBLE beta_other; - calculated_number level; - calculated_number trend; + NETDATA_DOUBLE level; + NETDATA_DOUBLE trend; size_t count; }; @@ -31,22 +31,22 @@ void grouping_init_des(void) { } } -static inline calculated_number window(RRDR *r, struct grouping_des *g) { +static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) { (void)g; - calculated_number points; + NETDATA_DOUBLE points; if(r->group == 1) { // provide a running DES - points = r->internal.points_wanted; + points = (NETDATA_DOUBLE)r->internal.points_wanted; } else { // provide a SES with flush points - points = r->group; + points = (NETDATA_DOUBLE)r->group; } // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average // A commonly used value for alpha is 2 / (N + 1) - return (points > max_window_size) ? max_window_size : points; + return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points; } static inline void set_alpha(RRDR *r, struct grouping_des *g) { @@ -69,8 +69,8 @@ static inline void set_beta(RRDR *r, struct grouping_des *g) { //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta); } -void grouping_create_des(RRDR *r) { - struct grouping_des *g = (struct grouping_des *)mallocz(sizeof(struct grouping_des)); +void grouping_create_des(RRDR *r, const char *options __maybe_unused) { + struct grouping_des *g = (struct grouping_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct grouping_des)); set_alpha(r, g); set_beta(r, g); g->level = 0.0; @@ -92,11 +92,11 @@ void grouping_reset_des(RRDR *r) { } void grouping_free_des(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_des(RRDR *r, calculated_number value) { +void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) { struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; if(likely(g->count > 0)) { @@ -109,7 +109,7 @@ void grouping_add_des(RRDR *r, calculated_number value) { } // for the values, except the first - calculated_number last_level = g->level; + NETDATA_DOUBLE last_level = g->level; g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); } @@ -123,10 +123,10 @@ void grouping_add_des(RRDR *r, calculated_number value) { //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend); } -calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; - if(unlikely(!g->count || !calculated_number_isnumber(g->level))) { + if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; return 0.0; } diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h index bd361b865..8906d14eb 100644 --- a/web/api/queries/des/des.h +++ b/web/api/queries/des/des.h @@ -8,10 +8,10 @@ extern void grouping_init_des(void); -extern void grouping_create_des(RRDR *r); +extern void grouping_create_des(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_des(RRDR *r); extern void grouping_free_des(RRDR *r); -extern void grouping_add_des(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_des(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_DES_H diff --git a/web/api/queries/incremental_sum/incremental_sum.c b/web/api/queries/incremental_sum/incremental_sum.c index 304d9aa74..afca530c3 100644 --- a/web/api/queries/incremental_sum/incremental_sum.c +++ b/web/api/queries/incremental_sum/incremental_sum.c @@ -6,13 +6,13 @@ // incremental sum struct grouping_incremental_sum { - calculated_number first; - calculated_number last; + NETDATA_DOUBLE first; + NETDATA_DOUBLE last; size_t count; }; -void grouping_create_incremental_sum(RRDR *r) { - r->internal.grouping_data = callocz(1, sizeof(struct grouping_incremental_sum)); +void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused) { + r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_incremental_sum)); } // resets when switches dimensions @@ -25,11 +25,11 @@ void grouping_reset_incremental_sum(RRDR *r) { } void grouping_free_incremental_sum(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_incremental_sum(RRDR *r, calculated_number value) { +void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) { struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; if(unlikely(!g->count)) { @@ -42,10 +42,10 @@ void grouping_add_incremental_sum(RRDR *r, calculated_number value) { } } -calculated_number grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(unlikely(!g->count)) { value = 0.0; diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h index 5b55ad3c8..6d908cef6 100644 --- a/web/api/queries/incremental_sum/incremental_sum.h +++ b/web/api/queries/incremental_sum/incremental_sum.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_incremental_sum(RRDR *r); +extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_incremental_sum(RRDR *r); extern void grouping_free_incremental_sum(RRDR *r); -extern void grouping_add_incremental_sum(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H diff --git a/web/api/queries/max/max.c b/web/api/queries/max/max.c index b6e723314..73cf9fa66 100644 --- a/web/api/queries/max/max.c +++ b/web/api/queries/max/max.c @@ -6,12 +6,12 @@ // max struct grouping_max { - calculated_number max; + NETDATA_DOUBLE max; size_t count; }; -void grouping_create_max(RRDR *r) { - r->internal.grouping_data = callocz(1, sizeof(struct grouping_max)); +void grouping_create_max(RRDR *r, const char *options __maybe_unused) { + r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_max)); } // resets when switches dimensions @@ -23,23 +23,23 @@ void grouping_reset_max(RRDR *r) { } void grouping_free_max(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_max(RRDR *r, calculated_number value) { +void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) { struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; - if(!g->count || calculated_number_fabs(value) > calculated_number_fabs(g->max)) { + if(!g->count || fabsndd(value) > fabsndd(g->max)) { g->max = value; g->count++; } } -calculated_number grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(unlikely(!g->count)) { value = 0.0; diff --git a/web/api/queries/max/max.h b/web/api/queries/max/max.h index 7b606ce34..28913686b 100644 --- a/web/api/queries/max/max.h +++ b/web/api/queries/max/max.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_max(RRDR *r); +extern void grouping_create_max(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_max(RRDR *r); extern void grouping_free_max(RRDR *r); -extern void grouping_add_max(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_max(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_MAX_H diff --git a/web/api/queries/median/README.md b/web/api/queries/median/README.md index bb7d4c66b..5600284c2 100644 --- a/web/api/queries/median/README.md +++ b/web/api/queries/median/README.md @@ -13,6 +13,20 @@ The median is the value separating the higher half from the lower half of a data `median` is not an accurate average. However, it eliminates all spikes, by sorting all the values in a period, and selecting the value in the middle of the sorted array. +Netdata also supports `trimmed-median`, which trims a percentage of the smaller and bigger values prior to finding the +median. The following `trimmed-median` functions are defined: + +- `trimmed-median1` +- `trimmed-median2` +- `trimmed-median3` +- `trimmed-median5` +- `trimmed-median10` +- `trimmed-median15` +- `trimmed-median20` +- `trimmed-median25` + +The function `trimmed-median` is an alias for `trimmed-median5`. + ## how to use Use it in alarms like this: @@ -27,7 +41,8 @@ lookup: median -1m unaligned of my_dimension `median` does not change the units. For example, if the chart units is `requests/sec`, the result will be again expressed in the same units. -It can also be used in APIs and badges as `&group=median` in the URL. +It can also be used in APIs and badges as `&group=median` in the URL. Additionally, a percentage may be given with +`&group_options=` to trim all small and big values before finding the median. ## Examples diff --git a/web/api/queries/median/median.c b/web/api/queries/median/median.c index bffcee12f..40fd4ec3a 100644 --- a/web/api/queries/median/median.c +++ b/web/api/queries/median/median.c @@ -2,27 +2,65 @@ #include "median.h" - // ---------------------------------------------------------------------------- // median struct grouping_median { size_t series_size; size_t next_pos; + NETDATA_DOUBLE percent; - LONG_DOUBLE series[]; + NETDATA_DOUBLE *series; }; -void grouping_create_median(RRDR *r) { +void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { long entries = r->group; - if(entries < 0) entries = 0; + if(entries < 10) entries = 10; - struct grouping_median *g = (struct grouping_median *)callocz(1, sizeof(struct grouping_median) + entries * sizeof(LONG_DOUBLE)); + struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); g->series_size = (size_t)entries; + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 50.0) g->percent = 50.0; + } + + g->percent = g->percent / 100.0; r->internal.grouping_data = g; } +void grouping_create_median(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 0.0); +} +void grouping_create_trimmed_median1(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 1.0); +} +void grouping_create_trimmed_median2(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 2.0); +} +void grouping_create_trimmed_median3(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 3.0); +} +void grouping_create_trimmed_median5(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 5.0); +} +void grouping_create_trimmed_median10(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 10.0); +} +void grouping_create_trimmed_median15(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 15.0); +} +void grouping_create_trimmed_median20(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 20.0); +} +void grouping_create_trimmed_median25(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 25.0); +} + // resets when switches dimensions // so, clear everything to restart void grouping_reset_median(RRDR *r) { @@ -31,47 +69,72 @@ void grouping_reset_median(RRDR *r) { } void grouping_free_median(RRDR *r) { - freez(r->internal.grouping_data); + struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_median(RRDR *r, calculated_number value) { +void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) { struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; if(unlikely(g->next_pos >= g->series_size)) { - error("INTERNAL ERROR: median buffer overflow on chart '%s' - next_pos = %zu, series_size = %zu, r->group = %ld.", r->st->name, g->next_pos, g->series_size, r->group); + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; } - else - g->series[g->next_pos++] = (LONG_DOUBLE)value; + + g->series[g->next_pos++] = value; } -calculated_number grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; - calculated_number value; + size_t available_slots = g->next_pos; + NETDATA_DOUBLE value; - if(unlikely(!g->next_pos)) { + if(unlikely(!available_slots)) { value = 0.0; *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; } + else if(available_slots == 1) { + value = g->series[0]; + } else { - if(g->next_pos > 1) { - sort_series(g->series, g->next_pos); - value = (calculated_number)median_on_sorted_series(g->series, g->next_pos); - } - else - value = (calculated_number)g->series[0]; + sort_series(g->series, available_slots); + + size_t start_slot = 0; + size_t end_slot = available_slots - 1; + + if(g->percent > 0.0) { + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + NETDATA_DOUBLE delta = (max - min) * g->percent; + + NETDATA_DOUBLE wanted_min = min + delta; + NETDATA_DOUBLE wanted_max = max - delta; + + for (start_slot = 0; start_slot < available_slots; start_slot++) + if (g->series[start_slot] >= wanted_min) break; - if(!calculated_number_isnumber(value)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--) + if (g->series[end_slot] <= wanted_max) break; } - //log_series_to_stderr(g->series, g->next_pos, value, "median"); + if(start_slot == end_slot) + value = g->series[start_slot]; + else + value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1); } + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "median"); + g->next_pos = 0; return value; } - diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h index 28d52b31e..dd1b3de61 100644 --- a/web/api/queries/median/median.h +++ b/web/api/queries/median/median.h @@ -6,10 +6,18 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_median(RRDR *r); +extern void grouping_create_median(RRDR *r, const char *options); +extern void grouping_create_trimmed_median1(RRDR *r, const char *options); +extern void grouping_create_trimmed_median2(RRDR *r, const char *options); +extern void grouping_create_trimmed_median3(RRDR *r, const char *options); +extern void grouping_create_trimmed_median5(RRDR *r, const char *options); +extern void grouping_create_trimmed_median10(RRDR *r, const char *options); +extern void grouping_create_trimmed_median15(RRDR *r, const char *options); +extern void grouping_create_trimmed_median20(RRDR *r, const char *options); +extern void grouping_create_trimmed_median25(RRDR *r, const char *options); extern void grouping_reset_median(RRDR *r); extern void grouping_free_median(RRDR *r); -extern void grouping_add_median(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_median(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_MEDIAN_H diff --git a/web/api/queries/min/min.c b/web/api/queries/min/min.c index 497bae04d..1752e9e0c 100644 --- a/web/api/queries/min/min.c +++ b/web/api/queries/min/min.c @@ -6,12 +6,12 @@ // min struct grouping_min { - calculated_number min; + NETDATA_DOUBLE min; size_t count; }; -void grouping_create_min(RRDR *r) { - r->internal.grouping_data = callocz(1, sizeof(struct grouping_min)); +void grouping_create_min(RRDR *r, const char *options __maybe_unused) { + r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_min)); } // resets when switches dimensions @@ -23,23 +23,23 @@ void grouping_reset_min(RRDR *r) { } void grouping_free_min(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_min(RRDR *r, calculated_number value) { +void grouping_add_min(RRDR *r, NETDATA_DOUBLE value) { struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data; - if(!g->count || calculated_number_fabs(value) < calculated_number_fabs(g->min)) { + if(!g->count || fabsndd(value) < fabsndd(g->min)) { g->min = value; g->count++; } } -calculated_number grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(unlikely(!g->count)) { value = 0.0; diff --git a/web/api/queries/min/min.h b/web/api/queries/min/min.h index 9207c74f7..b8627f667 100644 --- a/web/api/queries/min/min.h +++ b/web/api/queries/min/min.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_min(RRDR *r); +extern void grouping_create_min(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_min(RRDR *r); extern void grouping_free_min(RRDR *r); -extern void grouping_add_min(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_min(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_MIN_H diff --git a/web/api/queries/percentile/Makefile.am b/web/api/queries/percentile/Makefile.am new file mode 100644 index 000000000..161784b8f --- /dev/null +++ b/web/api/queries/percentile/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/queries/percentile/README.md b/web/api/queries/percentile/README.md new file mode 100644 index 000000000..70afc7420 --- /dev/null +++ b/web/api/queries/percentile/README.md @@ -0,0 +1,58 @@ +<!-- +title: "Percentile" +description: "Use percentile in API queries and health entities to find the 'percentile' value from a sample, eliminating any unwanted spikes in the returned metrics." +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/percentile/README.md +--> + +# Percentile + +The percentile is the average value of a series using only the smaller N percentile of the values. +(a population or a probability distribution). + +Netdata applies linear interpolation on the last point, if the percentile requested does not give a round number of +points. + +The following percentile aliases are defined: + +- `percentile25` +- `percentile50` +- `percentile75` +- `percentile80` +- `percentile90` +- `percentile95` +- `percentile97` +- `percentile98` +- `percentile99` + +The default `percentile` is an alias for `percentile95`. +Any percentile may be requested using the `group_options` query parameter. + +## how to use + +Use it in alarms like this: + +``` + alarm: my_alarm + on: my_chart +lookup: percentile95 -1m unaligned of my_dimension + warn: $this > 1000 +``` + +`percentile` does not change the units. For example, if the chart units is `requests/sec`, the result +will be again expressed in the same units. + +It can also be used in APIs and badges as `&group=percentile` in the URL and the additional parameter `group_options` +may be used to request any percentile (e.g. `&group=percentile&group_options=96`). + +## Examples + +Examining last 1 minute `successful` web server responses: + +-  +-  +-  +-  + +## References + +- <https://en.wikipedia.org/wiki/Percentile>. diff --git a/web/api/queries/percentile/percentile.c b/web/api/queries/percentile/percentile.c new file mode 100644 index 000000000..88f8600dd --- /dev/null +++ b/web/api/queries/percentile/percentile.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "percentile.h" + +// ---------------------------------------------------------------------------- +// median + +struct grouping_percentile { + size_t series_size; + size_t next_pos; + NETDATA_DOUBLE percent; + + NETDATA_DOUBLE *series; +}; + +static void grouping_create_percentile_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { + long entries = r->group; + if(entries < 10) entries = 10; + + struct grouping_percentile *g = (struct grouping_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_percentile)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); + g->series_size = (size_t)entries; + + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 100.0) g->percent = 100.0; + } + + g->percent = g->percent / 100.0; + r->internal.grouping_data = g; +} + +void grouping_create_percentile25(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 25.0); +} +void grouping_create_percentile50(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 50.0); +} +void grouping_create_percentile75(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 75.0); +} +void grouping_create_percentile80(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 80.0); +} +void grouping_create_percentile90(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 90.0); +} +void grouping_create_percentile95(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 95.0); +} +void grouping_create_percentile97(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 97.0); +} +void grouping_create_percentile98(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 98.0); +} +void grouping_create_percentile99(RRDR *r, const char *options) { + grouping_create_percentile_internal(r, options, 99.0); +} + +// resets when switches dimensions +// so, clear everything to restart +void grouping_reset_percentile(RRDR *r) { + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; + g->next_pos = 0; +} + +void grouping_free_percentile(RRDR *r) { + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); + r->internal.grouping_data = NULL; +} + +void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value) { + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; + + if(unlikely(g->next_pos >= g->series_size)) { + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; + } + + g->series[g->next_pos++] = value; +} + +NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; + + NETDATA_DOUBLE value; + size_t available_slots = g->next_pos; + + if(unlikely(!available_slots)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(available_slots == 1) { + value = g->series[0]; + } + else { + sort_series(g->series, available_slots); + + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + + if (min != max) { + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent); + if(!slots_to_use) slots_to_use = 1; + + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use; + + NETDATA_DOUBLE percent_interpolation_slot = 0.0; + NETDATA_DOUBLE percent_last_slot = 0.0; + if(percent_delta > 0.0) { + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use; + + percent_interpolation_slot = percent_delta / percent_1slot; + percent_last_slot = 1 - percent_interpolation_slot; + } + + int start_slot, stop_slot, step, last_slot, interpolation_slot; + if(min >= 0.0 && max >= 0.0) { + start_slot = 0; + stop_slot = start_slot + (int)slots_to_use; + last_slot = stop_slot - 1; + interpolation_slot = stop_slot; + step = 1; + } + else { + start_slot = (int)available_slots - 1; + stop_slot = start_slot - (int)slots_to_use; + last_slot = stop_slot + 1; + interpolation_slot = stop_slot; + step = -1; + } + + value = 0.0; + for(int slot = start_slot; slot != stop_slot ; slot += step) + value += g->series[slot]; + + size_t counted = slots_to_use; + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) { + value += g->series[interpolation_slot] * percent_interpolation_slot; + value += g->series[last_slot] * percent_last_slot; + counted++; + } + + value = value / (NETDATA_DOUBLE)counted; + } + else + value = min; + } + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "percentile"); + + g->next_pos = 0; + + return value; +} diff --git a/web/api/queries/percentile/percentile.h b/web/api/queries/percentile/percentile.h new file mode 100644 index 000000000..709717ebd --- /dev/null +++ b/web/api/queries/percentile/percentile.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_QUERIES_PERCENTILE_H +#define NETDATA_API_QUERIES_PERCENTILE_H + +#include "../query.h" +#include "../rrdr.h" + +extern void grouping_create_percentile25(RRDR *r, const char *options); +extern void grouping_create_percentile50(RRDR *r, const char *options); +extern void grouping_create_percentile75(RRDR *r, const char *options); +extern void grouping_create_percentile80(RRDR *r, const char *options); +extern void grouping_create_percentile90(RRDR *r, const char *options); +extern void grouping_create_percentile95(RRDR *r, const char *options); +extern void grouping_create_percentile97(RRDR *r, const char *options); +extern void grouping_create_percentile98(RRDR *r, const char *options); +extern void grouping_create_percentile99(RRDR *r, const char *options ); +extern void grouping_reset_percentile(RRDR *r); +extern void grouping_free_percentile(RRDR *r); +extern void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + +#endif //NETDATA_API_QUERIES_PERCENTILE_H diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 5c6c70411..d776f6d11 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -3,9 +3,9 @@ #include "query.h" #include "web/api/formatters/rrd2json.h" #include "rrdr.h" -#include "database/ram/rrddim_mem.h" #include "average/average.h" +#include "countif/countif.h" #include "incremental_sum/incremental_sum.h" #include "max/max.h" #include "median/median.h" @@ -14,6 +14,8 @@ #include "stddev/stddev.h" #include "ses/ses.h" #include "des/des.h" +#include "percentile/percentile.h" +#include "trimmed_mean/trimmed_mean.h" // ---------------------------------------------------------------------------- @@ -28,7 +30,7 @@ static struct { // Allocate all required structures for a query. // This is called once for each netdata query. - void (*create)(struct rrdresult *r); + void (*create)(struct rrdresult *r, const char *options); // Cleanup collected values, but don't destroy the structures. // This is called when the query engine switches dimensions, @@ -40,7 +42,7 @@ static struct { // Add a single value into the calculation. // The module may decide to cache it, or use it in the fly. - void (*add)(struct rrdresult *r, calculated_number value); + void (*add)(struct rrdresult *r, NETDATA_DOUBLE value); // Generate a single result for the values added so far. // More values and points may be requested later. @@ -48,7 +50,9 @@ static struct { // when flushing it (so for a few modules it may be better to // continue after a flush as if nothing changed, for others a // cleanup of the internal structures may be required). - calculated_number (*flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + NETDATA_DOUBLE (*flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + + TIER_QUERY_FETCH tier_query_fetch; } api_v1_data_groups[] = { {.name = "average", .hash = 0, @@ -58,7 +62,8 @@ static struct { .reset = grouping_reset_average, .free = grouping_free_average, .add = grouping_add_average, - .flush = grouping_flush_average + .flush = grouping_flush_average, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "mean", // alias on 'average' .hash = 0, @@ -68,7 +73,107 @@ static struct { .reset = grouping_reset_average, .free = grouping_free_average, .add = grouping_add_average, - .flush = grouping_flush_average + .flush = grouping_flush_average, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean1", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN1, + .init = NULL, + .create= grouping_create_trimmed_mean1, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean2", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN2, + .init = NULL, + .create= grouping_create_trimmed_mean2, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean3", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN3, + .init = NULL, + .create= grouping_create_trimmed_mean3, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean5", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN5, + .init = NULL, + .create= grouping_create_trimmed_mean5, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean10", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN10, + .init = NULL, + .create= grouping_create_trimmed_mean10, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean15", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN15, + .init = NULL, + .create= grouping_create_trimmed_mean15, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean20", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN20, + .init = NULL, + .create= grouping_create_trimmed_mean20, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean25", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN25, + .init = NULL, + .create= grouping_create_trimmed_mean25, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-mean", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEAN5, + .init = NULL, + .create= grouping_create_trimmed_mean5, + .reset = grouping_reset_trimmed_mean, + .free = grouping_free_trimmed_mean, + .add = grouping_add_trimmed_mean, + .flush = grouping_flush_trimmed_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "incremental_sum", .hash = 0, @@ -78,7 +183,8 @@ static struct { .reset = grouping_reset_incremental_sum, .free = grouping_free_incremental_sum, .add = grouping_add_incremental_sum, - .flush = grouping_flush_incremental_sum + .flush = grouping_flush_incremental_sum, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "incremental-sum", .hash = 0, @@ -88,7 +194,8 @@ static struct { .reset = grouping_reset_incremental_sum, .free = grouping_free_incremental_sum, .add = grouping_add_incremental_sum, - .flush = grouping_flush_incremental_sum + .flush = grouping_flush_incremental_sum, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "median", .hash = 0, @@ -98,7 +205,217 @@ static struct { .reset = grouping_reset_median, .free = grouping_free_median, .add = grouping_add_median, - .flush = grouping_flush_median + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median1", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN1, + .init = NULL, + .create= grouping_create_trimmed_median1, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median2", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN2, + .init = NULL, + .create= grouping_create_trimmed_median2, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median3", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN3, + .init = NULL, + .create= grouping_create_trimmed_median3, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median5", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN5, + .init = NULL, + .create= grouping_create_trimmed_median5, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median10", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN10, + .init = NULL, + .create= grouping_create_trimmed_median10, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median15", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN15, + .init = NULL, + .create= grouping_create_trimmed_median15, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median20", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN20, + .init = NULL, + .create= grouping_create_trimmed_median20, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median25", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN25, + .init = NULL, + .create= grouping_create_trimmed_median25, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "trimmed-median", + .hash = 0, + .value = RRDR_GROUPING_TRIMMED_MEDIAN5, + .init = NULL, + .create= grouping_create_trimmed_median5, + .reset = grouping_reset_median, + .free = grouping_free_median, + .add = grouping_add_median, + .flush = grouping_flush_median, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile25", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE25, + .init = NULL, + .create= grouping_create_percentile25, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile50", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE50, + .init = NULL, + .create= grouping_create_percentile50, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile75", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE75, + .init = NULL, + .create= grouping_create_percentile75, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile80", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE80, + .init = NULL, + .create= grouping_create_percentile80, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile90", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE90, + .init = NULL, + .create= grouping_create_percentile90, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile95", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE95, + .init = NULL, + .create= grouping_create_percentile95, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile97", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE97, + .init = NULL, + .create= grouping_create_percentile97, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile98", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE98, + .init = NULL, + .create= grouping_create_percentile98, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile99", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE99, + .init = NULL, + .create= grouping_create_percentile99, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "percentile", + .hash = 0, + .value = RRDR_GROUPING_PERCENTILE95, + .init = NULL, + .create= grouping_create_percentile95, + .reset = grouping_reset_percentile, + .free = grouping_free_percentile, + .add = grouping_add_percentile, + .flush = grouping_flush_percentile, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "min", .hash = 0, @@ -108,7 +425,8 @@ static struct { .reset = grouping_reset_min, .free = grouping_free_min, .add = grouping_add_min, - .flush = grouping_flush_min + .flush = grouping_flush_min, + .tier_query_fetch = TIER_QUERY_FETCH_MIN }, {.name = "max", .hash = 0, @@ -118,7 +436,8 @@ static struct { .reset = grouping_reset_max, .free = grouping_free_max, .add = grouping_add_max, - .flush = grouping_flush_max + .flush = grouping_flush_max, + .tier_query_fetch = TIER_QUERY_FETCH_MAX }, {.name = "sum", .hash = 0, @@ -128,7 +447,8 @@ static struct { .reset = grouping_reset_sum, .free = grouping_free_sum, .add = grouping_add_sum, - .flush = grouping_flush_sum + .flush = grouping_flush_sum, + .tier_query_fetch = TIER_QUERY_FETCH_SUM }, // standard deviation @@ -140,7 +460,8 @@ static struct { .reset = grouping_reset_stddev, .free = grouping_free_stddev, .add = grouping_add_stddev, - .flush = grouping_flush_stddev + .flush = grouping_flush_stddev, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "cv", // coefficient variation is calculated by stddev .hash = 0, @@ -150,7 +471,8 @@ static struct { .reset = grouping_reset_stddev, // not an error, stddev calculates this too .free = grouping_free_stddev, // not an error, stddev calculates this too .add = grouping_add_stddev, // not an error, stddev calculates this too - .flush = grouping_flush_coefficient_of_variation + .flush = grouping_flush_coefficient_of_variation, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "rsd", // alias of 'cv' .hash = 0, @@ -160,7 +482,8 @@ static struct { .reset = grouping_reset_stddev, // not an error, stddev calculates this too .free = grouping_free_stddev, // not an error, stddev calculates this too .add = grouping_add_stddev, // not an error, stddev calculates this too - .flush = grouping_flush_coefficient_of_variation + .flush = grouping_flush_coefficient_of_variation, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, /* @@ -172,7 +495,8 @@ static struct { .reset = grouping_reset_stddev, .free = grouping_free_stddev, .add = grouping_add_stddev, - .flush = grouping_flush_mean + .flush = grouping_flush_mean, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, */ @@ -185,7 +509,8 @@ static struct { .reset = grouping_reset_stddev, .free = grouping_free_stddev, .add = grouping_add_stddev, - .flush = grouping_flush_variance + .flush = grouping_flush_variance, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, */ @@ -193,44 +518,60 @@ static struct { {.name = "ses", .hash = 0, .value = RRDR_GROUPING_SES, - .init = grouping_init_ses, + .init = grouping_init_ses, .create= grouping_create_ses, .reset = grouping_reset_ses, .free = grouping_free_ses, .add = grouping_add_ses, - .flush = grouping_flush_ses + .flush = grouping_flush_ses, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "ema", // alias for 'ses' .hash = 0, .value = RRDR_GROUPING_SES, - .init = NULL, + .init = NULL, .create= grouping_create_ses, .reset = grouping_reset_ses, .free = grouping_free_ses, .add = grouping_add_ses, - .flush = grouping_flush_ses + .flush = grouping_flush_ses, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "ewma", // alias for ses .hash = 0, .value = RRDR_GROUPING_SES, - .init = NULL, + .init = NULL, .create= grouping_create_ses, .reset = grouping_reset_ses, .free = grouping_free_ses, .add = grouping_add_ses, - .flush = grouping_flush_ses + .flush = grouping_flush_ses, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, // double exponential smoothing {.name = "des", .hash = 0, .value = RRDR_GROUPING_DES, - .init = grouping_init_des, + .init = grouping_init_des, .create= grouping_create_des, .reset = grouping_reset_des, .free = grouping_free_des, .add = grouping_add_des, - .flush = grouping_flush_des + .flush = grouping_flush_des, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + + {.name = "countif", + .hash = 0, + .value = RRDR_GROUPING_COUNTIF, + .init = NULL, + .create= grouping_create_countif, + .reset = grouping_reset_countif, + .free = grouping_free_countif, + .add = grouping_add_countif, + .flush = grouping_flush_countif, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, // terminator @@ -242,7 +583,8 @@ static struct { .reset = grouping_reset_average, .free = grouping_free_average, .add = grouping_add_average, - .flush = grouping_flush_average + .flush = grouping_flush_average, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE } }; @@ -280,6 +622,41 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI return def; } +const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group) { + int i; + + for(i = 0; api_v1_data_groups[i].name ; i++) + if(unlikely(group == api_v1_data_groups[i].value)) + return api_v1_data_groups[i].name; + + return "unknown"; +} + +static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) { + int i, found = 0; + for(i = 0; !found && api_v1_data_groups[i].name ;i++) { + if(api_v1_data_groups[i].value == group_method) { + r->internal.grouping_create = api_v1_data_groups[i].create; + r->internal.grouping_reset = api_v1_data_groups[i].reset; + r->internal.grouping_free = api_v1_data_groups[i].free; + r->internal.grouping_add = api_v1_data_groups[i].add; + r->internal.grouping_flush = api_v1_data_groups[i].flush; + r->internal.tier_query_fetch = api_v1_data_groups[i].tier_query_fetch; + found = 1; + } + } + if(!found) { + errno = 0; + internal_error(true, "QUERY: grouping method %u not found. Using 'average'", (unsigned int)group_method); + r->internal.grouping_create = grouping_create_average; + r->internal.grouping_reset = grouping_reset_average; + r->internal.grouping_free = grouping_free_average; + r->internal.grouping_add = grouping_add_average; + r->internal.grouping_flush = grouping_flush_average; + r->internal.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE; + } +} + // ---------------------------------------------------------------------------- static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims, @@ -335,7 +712,7 @@ static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, // check if all dimensions are hidden if(unlikely(!dims_not_hidden_not_zero && dims_selected)) { - // there are a few selected dimensions + // there are a few selected dimensions, // but they are all zero // enable the selected ones // to avoid returning an empty chart @@ -352,22 +729,20 @@ static inline RRDR_VALUE_FLAGS *UNUSED_FUNCTION(rrdr_line_options)(RRDR *r, long return &r->o[ rrdr_line * r->d ]; } -static inline calculated_number *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rrdr_line) { +static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rrdr_line) { return &r->v[ rrdr_line * r->d ]; } static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) { rrdr_line++; - #ifdef NETDATA_INTERNAL_CHECKS - - if(unlikely(rrdr_line >= r->n)) - error("INTERNAL ERROR: requested to step above RRDR size for chart '%s'", r->st->name); + internal_error(rrdr_line >= r->n, + "QUERY: requested to step above RRDR size for chart '%s'", + r->st->name); - if(unlikely(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t)) - error("INTERNAL ERROR: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'", (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name); - - #endif + internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t, + "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'", + (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name); // save the time r->t[rrdr_line] = t; @@ -381,337 +756,737 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) { // ---------------------------------------------------------------------------- -// fill RRDR for a single dimension +// tier management -static inline void do_dimension_variablestep( - RRDR *r - , long points_wanted - , RRDDIM *rd - , long dim_id_in_rrdr - , time_t after_wanted - , time_t before_wanted - , uint32_t options -){ -// RRDSET *st = r->st; +static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, time_t before_wanted, long points_wanted) { + if(unlikely(storage_tiers < 2)) + return 0; + + if(unlikely(after_wanted == before_wanted || points_wanted <= 0 || !rd || !rd->rrdset)) { - time_t - now = after_wanted, - dt = r->update_every, - max_date = 0, - min_date = 0; + if(!rd) + internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation"); + else + internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation", + (rd->rrdset)?rd->rrdset->name:"unknown", rd->name); - long -// group_size = r->group, - points_added = 0, - values_in_group = 0, - values_in_group_non_zero = 0, - rrdr_line = -1; + return 0; + } - RRDR_VALUE_FLAGS - group_value_flags = RRDR_VALUE_NOTHING; + //BUFFER *wb = buffer_create(1000); + //buffer_sprintf(wb, "Best tier for chart '%s', dim '%s', from %ld to %ld (dur %ld, every %d), points %ld", + // rd->rrdset->name, rd->name, after_wanted, before_wanted, before_wanted - after_wanted, rd->update_every, points_wanted); - struct rrddim_query_handle handle; + long weight[storage_tiers]; + + for(int tier = 0; tier < storage_tiers ; tier++) { + if(unlikely(!rd->tiers[tier])) { + internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized", + tier, rd->rrdset->name, rd->name); + // buffer_free(wb); + return 0; + } + + time_t first_t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle); + time_t last_t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle); + + time_t common_after = MAX(first_t, after_wanted); + time_t common_before = MIN(last_t, before_wanted); + + long time_coverage = (common_before - common_after) * 1000 / (before_wanted - after_wanted); + if(time_coverage < 0) time_coverage = 0; + + int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every; + if(unlikely(update_every == 0)) { + internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d", + tier, rd->rrdset->name, rd->name, rd->tiers[tier]->tier_grouping, rd->update_every); + // buffer_free(wb); + return 0; + } - calculated_number min = r->min, max = r->max; - size_t db_points_read = 0; - time_t db_now = now; - storage_number n_curr, n_prev = SN_EMPTY_SLOT; - calculated_number value; + long points_available = (before_wanted - after_wanted) / update_every; + long points_delta = points_available - points_wanted; + long points_coverage = (points_delta < 0) ? points_available * 1000 / points_wanted: 1000; - for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) { - // make sure we return data in the proper time range - if (unlikely(now > before_wanted)) { + if(points_available <= 0) + weight[tier] = -LONG_MAX; + else + weight[tier] = points_coverage; + + // buffer_sprintf(wb, ": tier %d, first %ld, last %ld (dur %ld, tg %d, every %d), points %ld, tcoverage %ld, pcoverage %ld, weight %ld", + // tier, first_t, last_t, last_t - first_t, rd->tiers[tier]->tier_grouping, update_every, + // points_available, time_coverage, points_coverage, weight[tier]); + } + + int best_tier = 0; + for(int tier = 1; tier < storage_tiers ; tier++) { + if(weight[tier] >= weight[best_tier]) + best_tier = tier; + } + + if(weight[best_tier] == -LONG_MAX) + best_tier = 0; + + //buffer_sprintf(wb, ": final best tier %d", best_tier); + //internal_error(true, "%s", buffer_tostring(wb)); + //buffer_free(wb); + + return best_tier; +} + +static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) { + int ret = st->update_every; + + if(unlikely(!st->dimensions)) + return ret; + + rrdset_rdlock(st); + int best_tier; + + if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers) + best_tier = tier; + else + best_tier = rrddim_find_best_tier_for_timeframe(st->dimensions, after_wanted, before_wanted, points_wanted); + + if(!st->dimensions->tiers[best_tier]) { + internal_error( + true, + "QUERY: tier %d on chart '%s', is not initialized", best_tier, st->name); + } + else { + ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every; + if(unlikely(!ret)) { + internal_error( + true, + "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d", + st->name, st->dimensions->tiers[best_tier]->tier_grouping, st->update_every); + + ret = st->update_every; + } + } + + rrdset_unlock(st); + + return ret; +} + +// ---------------------------------------------------------------------------- +// query ops + +typedef struct query_point { + time_t end_time; + time_t start_time; + NETDATA_DOUBLE value; + NETDATA_DOUBLE anomaly; + SN_FLAGS flags; #ifdef NETDATA_INTERNAL_CHECKS - r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; + size_t id; #endif - break; - } - if (unlikely(now < after_wanted)) { +} QUERY_POINT; + +QUERY_POINT QUERY_POINT_EMPTY = { + .end_time = 0, + .start_time = 0, + .value = NAN, + .anomaly = 0, + .flags = SN_FLAG_NONE, #ifdef NETDATA_INTERNAL_CHECKS - r->internal.log = "skipped, because attempted to access the db before 'wanted after'"; + .id = 0, #endif - continue; - } +}; - while (now >= db_now && (!rd->state->query_ops.is_finished(&handle) || - does_storage_number_exist(n_prev))) { - value = NAN; - if (does_storage_number_exist(n_prev)) { - // use the previously read database value - n_curr = n_prev; - } else { - // read the value from the database - n_curr = rd->state->query_ops.next_metric(&handle, &db_now); - } - n_prev = SN_EMPTY_SLOT; - // db_now has a different value than above - if (likely(now >= db_now)) { - if (likely(does_storage_number_exist(n_curr))) { - if (options & RRDR_OPTION_ANOMALY_BIT) - value = (n_curr & SN_ANOMALY_BIT) ? 0.0 : 100.0; - else - value = unpack_storage_number(n_curr); - - if (likely(value != 0.0)) - values_in_group_non_zero++; - - if (unlikely(did_storage_number_reset(n_curr))) - group_value_flags |= RRDR_VALUE_RESET; - } - } else { - // We must postpone processing the value and fill the result with gaps instead - if (likely(does_storage_number_exist(n_curr))) { - n_prev = n_curr; - } - } - // add this value to grouping - if(likely(!isnan(value))) - r->internal.grouping_add(r, value); +#ifdef NETDATA_INTERNAL_CHECKS +#define query_point_set_id(point, point_id) (point).id = point_id +#else +#define query_point_set_id(point, point_id) debug_dummy() +#endif - values_in_group++; - db_points_read++; - } +typedef struct query_plan_entry { + size_t tier; + time_t after; + time_t before; +} QUERY_PLAN_ENTRY; - if (0 == values_in_group) { - // add NAN to grouping - r->internal.grouping_add(r, NAN); - } +typedef struct query_plan { + size_t entries; + QUERY_PLAN_ENTRY data[RRD_STORAGE_TIERS*2]; +} QUERY_PLAN; + +typedef struct query_engine_ops { + // configuration + RRDR *r; + RRDDIM *rd; + time_t view_update_every; + time_t query_granularity; + TIER_QUERY_FETCH tier_query_fetch; + + // query planer + QUERY_PLAN plan; + size_t current_plan; + time_t current_plan_expire_time; + + // storage queries + size_t tier; + struct rrddim_tier *tier_ptr; + struct rrddim_query_handle handle; + STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle); + int (*is_finished)(struct rrddim_query_handle *handle); + void (*finalize)(struct rrddim_query_handle *handle); - rrdr_line = rrdr_line_init(r, now, rrdr_line); + // aggregating points over time + void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value); + NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + size_t group_points_non_zero; + size_t group_points_added; + NETDATA_DOUBLE group_anomaly_rate; + RRDR_VALUE_FLAGS group_value_flags; - if(unlikely(!min_date)) min_date = now; - max_date = now; + // statistics + size_t db_total_points_read; + size_t db_points_read_per_tier[RRD_STORAGE_TIERS]; +} QUERY_ENGINE_OPS; - // find the place to store our values - RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr]; - // update the dimension options - if(likely(values_in_group_non_zero)) - r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO; +// ---------------------------------------------------------------------------- +// query planer - // store the specific point options - *rrdr_value_options_ptr = group_value_flags; +#define query_plan_should_switch_plan(ops, now) ((now) >= (ops).current_plan_expire_time) - // store the value - value = r->internal.grouping_flush(r, rrdr_value_options_ptr); - r->v[rrdr_line * r->d + dim_id_in_rrdr] = value; +static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, time_t overwrite_after) { + if(unlikely(plan_id >= ops->plan.entries)) + plan_id = ops->plan.entries - 1; - if(likely(points_added || dim_id_in_rrdr)) { - // find the min/max across all dimensions + time_t after = ops->plan.data[plan_id].after; + time_t before = ops->plan.data[plan_id].before; - if(unlikely(value < min)) min = value; - if(unlikely(value > max)) max = value; + if(overwrite_after > after && overwrite_after < before) + after = overwrite_after; - } - else { - // runs only when dim_id_in_rrdr == 0 && points_added == 0 - // so, on the first point added for the query. - min = max = value; + ops->tier = ops->plan.data[plan_id].tier; + ops->tier_ptr = ops->rd->tiers[ops->tier]; + ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch); + ops->next_metric = ops->tier_ptr->query_ops.next_metric; + ops->is_finished = ops->tier_ptr->query_ops.is_finished; + ops->finalize = ops->tier_ptr->query_ops.finalize; + ops->current_plan = plan_id; + ops->current_plan_expire_time = ops->plan.data[plan_id].before; +} + +static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) { + internal_error(now < ops->current_plan_expire_time && now < ops->plan.data[ops->current_plan].before, + "QUERY: switching query plan too early!"); + + time_t next_plan_before_time; + do { + ops->current_plan++; + + if (ops->current_plan >= ops->plan.entries) { + ops->current_plan = ops->plan.entries - 1; + return; } - points_added++; - values_in_group = 0; - group_value_flags = RRDR_VALUE_NOTHING; - values_in_group_non_zero = 0; + next_plan_before_time = ops->plan.data[ops->current_plan].before; + } while(now >= next_plan_before_time || last_point_end_time >= next_plan_before_time); + + if(ops->finalize) { + ops->finalize(&ops->handle); + ops->finalize = NULL; } - rd->state->query_ops.finalize(&handle); - r->internal.db_points_read += db_points_read; - r->internal.result_points_generated += points_added; + query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time)); - r->min = min; - r->max = max; - r->before = max_date; - r->after = min_date - (r->group - 1) * dt; - rrdr_done(r, rrdr_line); + // internal_error(true, "QUERY: switched plan to %zu (all is %zu), previous expiration was %ld, this starts at %ld, now is %ld, last_point_end_time %ld", ops->current_plan, ops->plan.entries, ops->plan.data[ops->current_plan-1].before, ops->plan.data[ops->current_plan].after, now, last_point_end_time); +} - #ifdef NETDATA_INTERNAL_CHECKS - if(unlikely(r->rows != points_added)) - error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows); - #endif +static int compare_query_plan_entries_on_start_time(const void *a, const void *b) { + QUERY_PLAN_ENTRY *p1 = (QUERY_PLAN_ENTRY *)a; + QUERY_PLAN_ENTRY *p2 = (QUERY_PLAN_ENTRY *)b; + return (p1->after < p2->after)?-1:1; } -static inline void do_dimension_fixedstep( - RRDR *r - , long points_wanted - , RRDDIM *rd - , long dim_id_in_rrdr - , time_t after_wanted - , time_t before_wanted - , uint32_t options -){ - time_t now = after_wanted, - dt = r->update_every / r->group, /* usually is st->update_every */ - max_date = 0, - min_date = 0; +static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, long points_wanted) { + RRDDIM *rd = ops->rd; - long group_size = r->group, - points_added = 0, - values_in_group = 0, - values_in_group_non_zero = 0, - rrdr_line = -1; + //BUFFER *wb = buffer_create(1000); + //buffer_sprintf(wb, "QUERY PLAN for chart '%s' dimension '%s', from %ld to %ld:", rd->rrdset->name, rd->name, after_wanted, before_wanted); - RRDR_VALUE_FLAGS group_value_flags = RRDR_VALUE_NOTHING; + // put our selected tier as the first plan + size_t selected_tier; - struct rrddim_query_handle handle; + if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER && ops->r->internal.query_tier >= 0 && ops->r->internal.query_tier < storage_tiers) { + selected_tier = ops->r->internal.query_tier; + } + else { - calculated_number min = r->min, max = r->max; - size_t db_points_read = 0; - time_t db_now = now; - time_t first_time_t = rrddim_first_entry_t(rd); + selected_tier = rrddim_find_best_tier_for_timeframe(rd, after_wanted, before_wanted, points_wanted); - // cache the function pointers we need in the loop - storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time) = rd->state->query_ops.next_metric; - void (*grouping_add)(struct rrdresult *r, calculated_number value) = r->internal.grouping_add; - calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) = r->internal.grouping_flush; - RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode; + if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER) + ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER; + } - for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) { - // make sure we return data in the proper time range - if(unlikely(now > before_wanted)) { -#ifdef NETDATA_INTERNAL_CHECKS - r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; -#endif - break; + ops->plan.entries = 1; + ops->plan.data[0].tier = selected_tier; + ops->plan.data[0].after = rd->tiers[selected_tier]->query_ops.oldest_time(rd->tiers[selected_tier]->db_metric_handle); + ops->plan.data[0].before = rd->tiers[selected_tier]->query_ops.latest_time(rd->tiers[selected_tier]->db_metric_handle); + + if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) { + // the selected tier + time_t selected_tier_first_time_t = ops->plan.data[0].after; + time_t selected_tier_last_time_t = ops->plan.data[0].before; + + //buffer_sprintf(wb, ": SELECTED tier %zu, from %ld to %ld", selected_tier, ops->plan.data[0].after, ops->plan.data[0].before); + + // check if our selected tier can start the query + if (selected_tier_first_time_t > after_wanted) { + // we need some help from other tiers + for (int tr = (int)selected_tier + 1; tr < storage_tiers; tr++) { + // find the first time of this tier + time_t first_time_t = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle); + + //buffer_sprintf(wb, ": EVAL AFTER tier %d, %ld", tier, first_time_t); + + // can it help? + if (first_time_t < selected_tier_first_time_t) { + // it can help us add detail at the beginning of the query + QUERY_PLAN_ENTRY t = { + .tier = tr, + .after = (first_time_t < after_wanted) ? after_wanted : first_time_t, + .before = selected_tier_first_time_t}; + ops->plan.data[ops->plan.entries++] = t; + + // prepare for the tier + selected_tier_first_time_t = t.after; + + if (t.after <= after_wanted) + break; + } + } } - if(unlikely(now < after_wanted)) { -#ifdef NETDATA_INTERNAL_CHECKS - r->internal.log = "skipped, because attempted to access the db before 'wanted after'"; -#endif - continue; + // check if our selected tier can finish the query + if (selected_tier_last_time_t < before_wanted) { + // we need some help from other tiers + for (int tr = (int)selected_tier - 1; tr >= 0; tr--) { + // find the last time of this tier + time_t last_time_t = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle); + + //buffer_sprintf(wb, ": EVAL BEFORE tier %d, %ld", tier, last_time_t); + + // can it help? + if (last_time_t > selected_tier_last_time_t) { + // it can help us add detail at the end of the query + QUERY_PLAN_ENTRY t = { + .tier = tr, + .after = selected_tier_last_time_t, + .before = (last_time_t > before_wanted) ? before_wanted : last_time_t}; + ops->plan.data[ops->plan.entries++] = t; + + // prepare for the tier + selected_tier_last_time_t = t.before; + + if (t.before >= before_wanted) + break; + } + } } + } - // read the value from the database - //storage_number n = rd->values[slot]; + // sort the query plan + if(ops->plan.entries > 1) + qsort(&ops->plan.data, ops->plan.entries, sizeof(QUERY_PLAN_ENTRY), compare_query_plan_entries_on_start_time); -#ifdef NETDATA_INTERNAL_CHECKS - struct mem_query_handle* mem_handle = (struct mem_query_handle*)handle.handle; - if ((rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) && - (rrdset_time2slot(r->st, now) != (long unsigned)(mem_handle->slot))) { - error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)mem_handle->slot, rrdset_time2slot(r->st, now)); - } -#endif + // make sure it has the whole timeframe we need + ops->plan.data[0].after = after_wanted; + ops->plan.data[ops->plan.entries - 1].before = before_wanted; - db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it + //buffer_sprintf(wb, ": FINAL STEPS %zu", ops->plan.entries); - storage_number n; - calculated_number value; + //for(size_t i = 0; i < ops->plan.entries ;i++) + // buffer_sprintf(wb, ": STEP %zu = use tier %zu from %ld to %ld", i+1, ops->plan.data[i].tier, ops->plan.data[i].after, ops->plan.data[i].before); - if (unlikely(rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t)) { - n = SN_EMPTY_SLOT; - value = NAN; - } - else { - // load the metric value - n = next_metric(&handle, &db_now); - db_points_read++; - - // and unpack it - if(likely(does_storage_number_exist(n))) { - if (options & RRDR_OPTION_ANOMALY_BIT) - value = (n & SN_ANOMALY_BIT) ? 0.0 : 100.0; - else - value = unpack_storage_number(n); - } - else - value = NAN; - } + //internal_error(true, "%s", buffer_tostring(wb)); - if(unlikely(db_now > before_wanted)) { -#ifdef NETDATA_INTERNAL_CHECKS - r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; -#endif - break; - } + query_planer_activate_plan(ops, 0, 0); +} - // this loop exists only to fill nulls - // so, if there is a value already, we use it for the first iteration - // but the following iterations will just fill nulls to the destination - for ( ; now <= db_now ; now += dt, value = NAN, n = SN_EMPTY_SLOT) { - if(likely(does_storage_number_exist(n))) { - -#if defined(NETDATA_INTERNAL_CHECKS) && defined(ENABLE_DBENGINE) - if(now >= db_now) { - struct rrdeng_query_handle *rrd_handle = (struct rrdeng_query_handle *)handle.handle; - if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != rrd_handle->now)) - error( - "INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", - rd->id, - (long)rrd_handle->now, - (long)now); - } -#endif - if(likely(value != 0.0)) - values_in_group_non_zero++; +// ---------------------------------------------------------------------------- +// dimension level query engine + +#define query_interpolate_point(this_point, last_point, now) do { \ + if(likely( \ + /* the point to interpolate is more than 1s wide */ \ + (this_point).end_time - (this_point).start_time > 1 \ + \ + /* the two points are exactly next to each other */ \ + && (last_point).end_time == (this_point).start_time \ + \ + /* both points are valid numbers */ \ + && netdata_double_isnumber((this_point).value) \ + && netdata_double_isnumber((last_point).value) \ + \ + )) { \ + (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).end_time - (now)) / (NETDATA_DOUBLE)((this_point).end_time - (this_point).start_time)); \ + (this_point).end_time = now; \ + } \ +} while(0) + +#define query_add_point_to_group(r, point, ops) do { \ + if(likely(netdata_double_isnumber((point).value))) { \ + if(likely(fpclassify((point).value) != FP_ZERO)) \ + (ops).group_points_non_zero++; \ + \ + if(unlikely((point).flags & SN_FLAG_RESET)) \ + (ops).group_value_flags |= RRDR_VALUE_RESET; \ + \ + (ops).grouping_add(r, (point).value); \ + } \ + \ + (ops).group_points_added++; \ + (ops).group_anomaly_rate += (point).anomaly; \ +} while(0) + +static inline void rrd2rrdr_do_dimension( + RRDR *r + , long points_wanted + , RRDDIM *rd + , long dim_id_in_rrdr + , time_t after_wanted + , time_t before_wanted +){ + time_t max_date = 0, + min_date = 0; + + size_t points_added = 0; + + QUERY_ENGINE_OPS ops = { + .r = r, + .rd = rd, + .grouping_add = r->internal.grouping_add, + .grouping_flush = r->internal.grouping_flush, + .tier_query_fetch = r->internal.tier_query_fetch, + .view_update_every = r->update_every, + .query_granularity = r->update_every / r->group, + .group_value_flags = RRDR_VALUE_NOTHING + }; + + long rrdr_line = -1; + bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false; + + query_plan(&ops, after_wanted, before_wanted, points_wanted); + + NETDATA_DOUBLE min = r->min, max = r->max; + + QUERY_POINT last2_point = QUERY_POINT_EMPTY; + QUERY_POINT last1_point = QUERY_POINT_EMPTY; + QUERY_POINT new_point = QUERY_POINT_EMPTY; + + time_t now_start_time = after_wanted - ops.query_granularity; + time_t now_end_time = after_wanted + ops.view_update_every - ops.query_granularity; - if(unlikely(did_storage_number_reset(n))) - group_value_flags |= RRDR_VALUE_RESET; + // The main loop, based on the query granularity we need + for( ; (long)points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) { - grouping_add(r, value); + if(query_plan_should_switch_plan(ops, now_end_time)) + query_planer_next_plan(&ops, now_end_time, new_point.end_time); + + // read all the points of the db, prior to the time we need (now_end_time) + + + size_t count_same_end_time = 0; + while(count_same_end_time < 100) { + if(likely(count_same_end_time == 0)) { + last2_point = last1_point; + last1_point = new_point; } - // add this value for grouping - values_in_group++; + if(unlikely(ops.is_finished(&ops.handle))) { + if(count_same_end_time != 0) { + last2_point = last1_point; + last1_point = new_point; + } + new_point = QUERY_POINT_EMPTY; + new_point.start_time = last1_point.end_time; + new_point.end_time = now_end_time; + break; + } + + // fetch the new point + { + STORAGE_POINT sp = ops.next_metric(&ops.handle); + + ops.db_points_read_per_tier[ops.tier]++; + ops.db_total_points_read++; + + new_point.start_time = sp.start_time; + new_point.end_time = sp.end_time; + new_point.anomaly = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0; + query_point_set_id(new_point, ops.db_total_points_read); + + // set the right value to the point we got + if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) { - if(unlikely(values_in_group == group_size)) { - rrdr_line = rrdr_line_init(r, now, rrdr_line); - size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr; + if(unlikely(use_anomaly_bit_as_value)) + new_point.value = new_point.anomaly; - if(unlikely(!min_date)) min_date = now; - max_date = now; + else { + switch (ops.tier_query_fetch) { + default: + case TIER_QUERY_FETCH_AVERAGE: + new_point.value = sp.sum / sp.count; + break; - // find the place to store our values - RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index]; + case TIER_QUERY_FETCH_MIN: + new_point.value = sp.min; + break; - // update the dimension options - if(likely(values_in_group_non_zero)) - r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO; + case TIER_QUERY_FETCH_MAX: + new_point.value = sp.max; + break; - // store the specific point options - *rrdr_value_options_ptr = group_value_flags; + case TIER_QUERY_FETCH_SUM: + new_point.value = sp.sum; + break; + }; + } + } + else { + new_point.value = NAN; + new_point.flags = SN_FLAG_NONE; + } + } + + // check if the db is giving us zero duration points + if(unlikely(new_point.start_time == new_point.end_time)) { + internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal", + rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time); - // store the group value - calculated_number group_value = grouping_flush(r, rrdr_value_options_ptr); - r->v[rrdr_o_v_index] = group_value; + new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every); + } + + // check if the db is advancing the query + if(unlikely(new_point.end_time <= last1_point.end_time)) { + internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld", + rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time, + last1_point.id, last1_point.end_time, now_start_time, now_end_time); + + count_same_end_time++; + continue; + } + count_same_end_time = 0; - if(likely(points_added || dim_id_in_rrdr)) { - // find the min/max across all dimensions + // decide how to use this point + if(likely(new_point.end_time < now_end_time)) { // likely to favor tier0 + // this db point ends before our now_end_time - if(unlikely(group_value < min)) min = group_value; - if(unlikely(group_value > max)) max = group_value; + if(likely(new_point.end_time >= now_start_time)) { // likely to favor tier0 + // this db point ends after our now_start time + query_add_point_to_group(r, new_point, ops); } else { - // runs only when dim_id_in_rrdr == 0 && points_added == 0 - // so, on the first point added for the query. - min = max = group_value; + // we don't need this db point + // it is totally outside our current time-frame + + // this is desirable for the first point of the query + // because it allows us to interpolate the next point + // at exactly the time we will want + + // we only log if this is not point 1 + internal_error(new_point.end_time < after_wanted && new_point.id > 1, + "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)", + rd->rrdset->name, rd->name, + new_point.id, new_point.start_time, new_point.end_time, + now_start_time, now_end_time, + after_wanted, before_wanted); } - points_added++; - values_in_group = 0; - group_value_flags = RRDR_VALUE_NOTHING; - values_in_group_non_zero = 0; + } + else { + // the point ends in the future + // so, we will interpolate it below, at the inner loop + break; } } - now = db_now; + + if(unlikely(count_same_end_time)) { + internal_error(true, + "QUERY: the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times", + last1_point.end_time, count_same_end_time); + + if(unlikely(new_point.end_time <= last1_point.end_time)) + new_point.end_time = now_end_time; + } + + // the inner loop + // we have 3 points in memory: last2, last1, new + // we select the one to use based on their timestamps + + size_t iterations = 0; + for ( ; now_end_time <= new_point.end_time && (long)points_added < points_wanted ; + now_end_time += ops.view_update_every, iterations++) { + + // now_start_time is wrong in this loop + // but, we don't need it + + QUERY_POINT current_point; + + if(likely(now_end_time > new_point.start_time)) { + // it is time for our NEW point to be used + current_point = new_point; + query_interpolate_point(current_point, last1_point, now_end_time); + + internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time, + "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld," + " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.", + rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, + current_point.id, current_point.start_time, current_point.end_time, now_end_time); + } + else if(likely(now_end_time <= last1_point.end_time)) { + // our LAST point is still valid + current_point = last1_point; + query_interpolate_point(current_point, last2_point, now_end_time); + + internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time, + "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld," + " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.", + rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, + current_point.id, current_point.start_time, current_point.end_time, now_end_time); + } + else { + // a GAP, we don't have a value this time + current_point = QUERY_POINT_EMPTY; + } + + query_add_point_to_group(r, current_point, ops); + + rrdr_line = rrdr_line_init(r, now_end_time, rrdr_line); + size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr; + + if(unlikely(!min_date)) min_date = now_end_time; + max_date = now_end_time; + + // find the place to store our values + RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index]; + + // update the dimension options + if(likely(ops.group_points_non_zero)) + r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO; + + // store the specific point options + *rrdr_value_options_ptr = ops.group_value_flags; + + // store the group value + NETDATA_DOUBLE group_value = ops.grouping_flush(r, rrdr_value_options_ptr); + r->v[rrdr_o_v_index] = group_value; + + // we only store uint8_t anomaly rates, + // so let's get double precision by storing + // anomaly rates in the range 0 - 200 + r->ar[rrdr_o_v_index] = ops.group_anomaly_rate / (NETDATA_DOUBLE)ops.group_points_added; + + if(likely(points_added || dim_id_in_rrdr)) { + // find the min/max across all dimensions + + if(unlikely(group_value < min)) min = group_value; + if(unlikely(group_value > max)) max = group_value; + + } + else { + // runs only when dim_id_in_rrdr == 0 && points_added == 0 + // so, on the first point added for the query. + min = max = group_value; + } + + points_added++; + ops.group_points_added = 0; + ops.group_value_flags = RRDR_VALUE_NOTHING; + ops.group_points_non_zero = 0; + ops.group_anomaly_rate = 0; + } + // the loop above increased "now" by query_granularity, + // but the main loop will increase it too, + // so, let's undo the last iteration of this loop + if(iterations) + now_end_time -= ops.view_update_every; } - rd->state->query_ops.finalize(&handle); + ops.finalize(&ops.handle); - r->internal.db_points_read += db_points_read; r->internal.result_points_generated += points_added; + r->internal.db_points_read += ops.db_total_points_read; + for(int tr = 0; tr < storage_tiers ; tr++) + r->internal.tier_points_read[tr] += ops.db_points_read_per_tier[tr]; r->min = min; r->max = max; r->before = max_date; - r->after = min_date - (r->group - 1) * dt; + r->after = min_date - ops.view_update_every + ops.query_granularity; rrdr_done(r, rrdr_line); -#ifdef NETDATA_INTERNAL_CHECKS - if(unlikely(r->rows != points_added)) - error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows); -#endif + internal_error((long)points_added != points_wanted, + "QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).", + r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read); +} + +// ---------------------------------------------------------------------------- +// fill the gap of a tier + +extern void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut); + +void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) { + if(unlikely(tier < 0 || tier >= storage_tiers)) return; + if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return; + + struct rrddim_tier *t = rd->tiers[tier]; + if(unlikely(!t)) return; + + time_t latest_time_t = t->query_ops.latest_time(t->db_metric_handle); + time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every; + time_t time_diff = now - latest_time_t; + + // if the user wants only NEW backfilling, and we don't have any data + if(storage_tiers_backfill[tier] == RRD_BACKFILL_NEW && latest_time_t <= 0) return; + + // there is really nothing we can do + if(now <= latest_time_t || time_diff < granularity) return; + + struct rrddim_query_handle handle; + + size_t all_points_read = 0; + + // for each lower tier + for(int tr = tier - 1; tr >= 0 ;tr--){ + time_t smaller_tier_first_time = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle); + time_t smaller_tier_last_time = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle); + if(smaller_tier_last_time <= latest_time_t) continue; // it is as bad as we are + + long after_wanted = (latest_time_t < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_t; + long before_wanted = smaller_tier_last_time; + + struct rrddim_tier *tmp = rd->tiers[tr]; + tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE); + + size_t points = 0; + + while(!tmp->query_ops.is_finished(&handle)) { + + STORAGE_POINT sp = tmp->query_ops.next_metric(&handle); + + if(sp.end_time > latest_time_t) { + latest_time_t = sp.end_time; + store_metric_at_tier(rd, t, sp, sp.end_time * USEC_PER_SEC); + points++; + } + } + + all_points_read += points; + tmp->query_ops.finalize(&handle); + + //internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d", + // rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr); + } + + rrdr_query_completed(all_points_read, all_points_read); } // ---------------------------------------------------------------------------- @@ -719,8 +1494,9 @@ static inline void do_dimension_fixedstep( #ifdef NETDATA_INTERNAL_CHECKS static void rrd2rrdr_log_request_response_metadata(RRDR *r + , RRDR_OPTIONS options __maybe_unused , RRDR_GROUPING group_method - , int aligned + , bool aligned , long group , long resampling_time , long resampling_group @@ -736,9 +1512,9 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r ) { netdata_rwlock_rdlock(&r->st->rrdset_rwlock); info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), " - "after (got: %zu, want: %zu, req: %zu, db: %zu), " - "before (got: %zu, want: %zu, req: %zu, db: %zu), " - "duration (got: %zu, want: %zu, req: %zu, db: %zu), " + "after (got: %zu, want: %zu, req: %ld, db: %zu), " + "before (got: %zu, want: %zu, req: %ld, db: %zu), " + "duration (got: %zu, want: %zu, req: %ld, db: %zu), " //"slot (after: %zu, before: %zu, delta: %zu), " "points (got: %ld, want: %ld, req: %ld, db: %ld), " "%s" @@ -755,19 +1531,19 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r // after , (size_t)r->after , (size_t)after_wanted - , (size_t)after_requested + , after_requested , (size_t)rrdset_first_entry_t_nolock(r->st) // before , (size_t)r->before , (size_t)before_wanted - , (size_t)before_requested + , before_requested , (size_t)rrdset_last_entry_t_nolock(r->st) // duration , (size_t)(r->before - r->after + r->st->update_every) , (size_t)(before_wanted - after_wanted + r->st->update_every) - , (size_t)(before_requested - after_requested) + , before_requested - after_requested , (size_t)((rrdset_last_entry_t_nolock(r->st) - rrdset_first_entry_t_nolock(r->st)) + r->st->update_every) // slot @@ -791,79 +1567,88 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r #endif // NETDATA_INTERNAL_CHECKS // Returns 1 if an absolute period was requested or 0 if it was a relative period -static int rrdr_convert_before_after_to_absolute( - long long *after_requestedp - , long long *before_requestedp - , int update_every - , time_t first_entry_t - , time_t last_entry_t - , RRDR_OPTIONS options -) { +int rrdr_relative_window_to_absolute(long long *after, long long *before) { + time_t now = now_realtime_sec() - 1; + int absolute_period_requested = -1; long long after_requested, before_requested; - before_requested = *before_requestedp; - after_requested = *after_requestedp; - - if(before_requested == 0 && after_requested == 0) { - // dump the all the data - before_requested = last_entry_t; - after_requested = first_entry_t; - absolute_period_requested = 0; - } + before_requested = *before; + after_requested = *after; // allow relative for before (smaller than API_RELATIVE_TIME_MAX) if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) { - if(ABS(before_requested) % update_every) { - // make sure it is multiple of st->update_every - if(before_requested < 0) before_requested = before_requested - update_every - - before_requested % update_every; - else before_requested = before_requested + update_every - before_requested % update_every; - } - if(before_requested > 0) before_requested = first_entry_t + before_requested; - else before_requested = last_entry_t + before_requested; //last_entry_t is not really now_t - //TODO: fix before_requested to be relative to now_t + // if the user asked for a positive relative time, + // flip it to a negative + if(before_requested > 0) + before_requested = -before_requested; + + before_requested = now + before_requested; absolute_period_requested = 0; } // allow relative for after (smaller than API_RELATIVE_TIME_MAX) if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) { - if(after_requested == 0) after_requested = -update_every; - if(ABS(after_requested) % update_every) { - // make sure it is multiple of st->update_every - if(after_requested < 0) after_requested = after_requested - update_every - after_requested % update_every; - else after_requested = after_requested + update_every - after_requested % update_every; - } - after_requested = before_requested + after_requested; + if(after_requested > 0) + after_requested = -after_requested; + + // if the user didn't give an after, use the number of points + // to give a sane default + if(after_requested == 0) + after_requested = -600; + + // since the query engine now returns inclusive timestamps + // it is awkward to return 6 points when after=-5 is given + // so for relative queries we add 1 second, to give + // more predictable results to users. + after_requested = before_requested + after_requested + 1; absolute_period_requested = 0; } if(absolute_period_requested == -1) absolute_period_requested = 1; - // make sure they are within our timeframe - if(before_requested > last_entry_t) before_requested = last_entry_t; - if(before_requested < first_entry_t && !(options & RRDR_OPTION_ALLOW_PAST)) - before_requested = first_entry_t; - - if(after_requested > last_entry_t) after_requested = last_entry_t; - if(after_requested < first_entry_t && !(options & RRDR_OPTION_ALLOW_PAST)) - after_requested = first_entry_t; - - // check if they are reversed + // check if the parameters are flipped if(after_requested > before_requested) { - time_t tmp = before_requested; + long long t = before_requested; before_requested = after_requested; - after_requested = tmp; + after_requested = t; + } + + // if the query requests future data + // shift the query back to be in the present time + // (this may also happen because of the rules above) + if(before_requested > now) { + long long delta = before_requested - now; + before_requested -= delta; + after_requested -= delta; } - *before_requestedp = before_requested; - *after_requestedp = after_requested; + *before = before_requested; + *after = after_requested; return absolute_period_requested; } -static RRDR *rrd2rrdr_fixedstep( +// #define DEBUG_QUERY_LOGIC 1 + +#ifdef DEBUG_QUERY_LOGIC +#define query_debug_log_init() BUFFER *debug_log = buffer_create(1000) +#define query_debug_log(args...) buffer_sprintf(debug_log, ##args) +#define query_debug_log_fin() { \ + info("QUERY: chart '%s', after:%lld, before:%lld, duration:%lld, points:%ld, res:%ld - wanted => after:%lld, before:%lld, points:%ld, group:%ld, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", st->name, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \ + buffer_free(debug_log); \ + debug_log = NULL; \ + } +#define query_debug_log_free() do { buffer_free(debug_log); } while(0) +#else +#define query_debug_log_init() debug_dummy() +#define query_debug_log(args...) debug_dummy() +#define query_debug_log_fin() debug_dummy() +#define query_debug_log_free() debug_dummy() +#endif + +RRDR *rrd2rrdr( ONEWAYALLOC *owa , RRDSET *st , long points_requested @@ -873,564 +1658,272 @@ static RRDR *rrd2rrdr_fixedstep( , long resampling_time_requested , RRDR_OPTIONS options , const char *dimensions - , int update_every - , time_t first_entry_t - , time_t last_entry_t - , int absolute_period_requested , struct context_param *context_param_list + , const char *group_options , int timeout + , int tier ) { - int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); - - // the duration of the chart - time_t duration = before_requested - after_requested; - long available_points = duration / update_every; - - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; - - if(duration <= 0 || available_points <= 0) - return rrdr_create(owa, st, 1, context_param_list); - - // check the number of wanted points in the result - if(unlikely(points_requested < 0)) points_requested = -points_requested; - if(unlikely(points_requested > available_points)) points_requested = available_points; - if(unlikely(points_requested == 0)) points_requested = available_points; - - // calculate the desired grouping of source data points - long group = available_points / points_requested; - if(unlikely(group <= 0)) group = 1; - if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer - - // resampling_time_requested enforces a certain grouping multiple - calculated_number resampling_divisor = 1.0; - long resampling_group = 1; - if(unlikely(resampling_time_requested > update_every)) { - if (unlikely(resampling_time_requested > duration)) { - // group_time is above the available duration - - #ifdef NETDATA_INTERNAL_CHECKS - info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, resampling_time_requested, duration); - #endif - - after_requested = before_requested - resampling_time_requested; - duration = before_requested - after_requested; - available_points = duration / update_every; - group = available_points / points_requested; - } - - // if the duration is not aligned to resampling time - // extend the duration to the past, to avoid a gap at the chart - // only when the missing duration is above 1/10th of a point - if(duration % resampling_time_requested) { - time_t delta = duration % resampling_time_requested; - if(delta > resampling_time_requested / 10) { - after_requested -= resampling_time_requested - delta; - duration = before_requested - after_requested; - available_points = duration / update_every; - group = available_points / points_requested; - } - } - - // the points we should group to satisfy gtime - resampling_group = resampling_time_requested / update_every; - if(unlikely(resampling_time_requested % update_every)) { - #ifdef NETDATA_INTERNAL_CHECKS - info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every); - #endif - - resampling_group++; - } - - // adapt group according to resampling_group - if(unlikely(group < resampling_group)) group = resampling_group; // do not allow grouping below the desired one - if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group - - //resampling_divisor = group / resampling_group; - resampling_divisor = (calculated_number)(group * update_every) / (calculated_number)resampling_time_requested; - } - - // now that we have group, - // align the requested timeframe to fit it. - - if(aligned) { - // alignment has been requested, so align the values - before_requested -= before_requested % (group * update_every); - after_requested -= after_requested % (group * update_every); - } - - // we align the request on requested_before - time_t before_wanted = before_requested; - if(likely(before_wanted > last_entry_t)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, before_wanted is after db max", st->name); - #endif - - before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * update_every )); - } - //size_t before_slot = rrdset_time2slot(st, before_wanted); - - // we need to estimate the number of points, for having - // an integer number of values per point - long points_wanted = (before_wanted - after_requested) / (update_every * group); - - time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; - if(unlikely(after_wanted < first_entry_t)) { - // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning - points_wanted = (before_wanted - first_entry_t) / group; - - // recalculate after wanted with the new number of points - after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; - - if(unlikely(after_wanted < first_entry_t)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name); - #endif - - after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every ); - } - } - //size_t after_slot = rrdset_time2slot(st, after_wanted); - - // check if they are reversed - if(unlikely(after_wanted > before_wanted)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, reversed wanted after/before", st->name); - #endif - time_t tmp = before_wanted; - before_wanted = after_wanted; - after_wanted = tmp; + // RULES + // points_requested = 0 + // the user wants all the natural points the database has + // + // after_requested = 0 + // the user wants to start the query from the oldest point in our database + // + // before_requested = 0 + // the user wants the query to end to the latest point in our database + // + // when natural points are wanted, the query has to be aligned to the update_every + // of the database + + long points_wanted = points_requested; + long long after_wanted = after_requested; + long long before_wanted = before_requested; + int update_every = st->update_every; + + bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED); + bool automatic_natural_points = (points_wanted == 0); + bool relative_period_requested = false; + bool natural_points = (options & RRDR_OPTION_NATURAL_POINTS) || automatic_natural_points; + bool before_is_aligned_to_db_end = false; + + query_debug_log_init(); + + // make sure points_wanted is positive + if(points_wanted < 0) { + points_wanted = -points_wanted; + query_debug_log(":-points_wanted %ld", points_wanted); } - // recalculate points_wanted using the final time-frame - points_wanted = (before_wanted - after_wanted) / update_every / group + 1; - if(unlikely(points_wanted < 0)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted); - #endif - points_wanted = 0; + if(ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) { + relative_period_requested = true; + natural_points = true; + options |= RRDR_OPTION_NATURAL_POINTS; + query_debug_log(":relative+natural"); } -#ifdef NETDATA_INTERNAL_CHECKS - duration = before_wanted - after_wanted; - - if(after_wanted < first_entry_t) - error("INTERNAL CHECK: after_wanted %u is too small, minimum %u", (uint32_t)after_wanted, (uint32_t)first_entry_t); - - if(after_wanted > last_entry_t) - error("INTERNAL CHECK: after_wanted %u is too big, maximum %u", (uint32_t)after_wanted, (uint32_t)last_entry_t); - - if(before_wanted < first_entry_t) - error("INTERNAL CHECK: before_wanted %u is too small, minimum %u", (uint32_t)before_wanted, (uint32_t)first_entry_t); - - if(before_wanted > last_entry_t) - error("INTERNAL CHECK: before_wanted %u is too big, maximum %u", (uint32_t)before_wanted, (uint32_t)last_entry_t); - -/* - if(before_slot >= (size_t)st->entries) - error("INTERNAL CHECK: before_slot is invalid %zu, expected 0 to %ld", before_slot, st->entries - 1); + // if the user wants virtual points, make sure we do it + if(options & RRDR_OPTION_VIRTUAL_POINTS) + natural_points = false; - if(after_slot >= (size_t)st->entries) - error("INTERNAL CHECK: after_slot is invalid %zu, expected 0 to %ld", after_slot, st->entries - 1); -*/ + // set the right flag about natural and virtual points + if(natural_points) { + options |= RRDR_OPTION_NATURAL_POINTS; - if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1) - error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1); - - if(group < resampling_group) - error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group); - - if(group > resampling_group && group % resampling_group) - error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, resampling_group); -#endif - - // ------------------------------------------------------------------------- - // initialize our result set - // this also locks the chart for us - - RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); - if(unlikely(!r)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); - #endif - return NULL; + if(options & RRDR_OPTION_VIRTUAL_POINTS) + options &= ~RRDR_OPTION_VIRTUAL_POINTS; } + else { + options |= RRDR_OPTION_VIRTUAL_POINTS; - if(unlikely(!r->d || !points_wanted)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); - #endif - return r; + if(options & RRDR_OPTION_NATURAL_POINTS) + options &= ~RRDR_OPTION_NATURAL_POINTS; } - if(unlikely(absolute_period_requested == 1)) - r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; - else - r->result_options |= RRDR_RESULT_OPTION_RELATIVE; - - // find how many dimensions we have - long dimensions_count = r->d; - - // ------------------------------------------------------------------------- - // initialize RRDR - - r->group = group; - r->update_every = (int)group * update_every; - r->before = before_wanted; - r->after = after_wanted; - r->internal.points_wanted = points_wanted; - r->internal.resampling_group = resampling_group; - r->internal.resampling_divisor = resampling_divisor; + if(after_wanted == 0 || before_wanted == 0) { + // for non-context queries we have to find the duration of the database + // for context queries we will assume 600 seconds duration + if(!context_param_list) { + relative_period_requested = true; - // ------------------------------------------------------------------------- - // assign the processor functions + rrdset_rdlock(st); + time_t first_entry_t = rrdset_first_entry_t_nolock(st); + time_t last_entry_t = rrdset_last_entry_t_nolock(st); + rrdset_unlock(st); - { - int i, found = 0; - for(i = 0; !found && api_v1_data_groups[i].name ;i++) { - if(api_v1_data_groups[i].value == group_method) { - r->internal.grouping_create= api_v1_data_groups[i].create; - r->internal.grouping_reset = api_v1_data_groups[i].reset; - r->internal.grouping_free = api_v1_data_groups[i].free; - r->internal.grouping_add = api_v1_data_groups[i].add; - r->internal.grouping_flush = api_v1_data_groups[i].flush; - found = 1; + if(first_entry_t == 0 || last_entry_t == 0) { + internal_error(true, "QUERY: chart without data detected on '%s'", st->name); + query_debug_log_free(); + return NULL; } - } - if(!found) { - errno = 0; - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name); - #endif - r->internal.grouping_create= grouping_create_average; - r->internal.grouping_reset = grouping_reset_average; - r->internal.grouping_free = grouping_free_average; - r->internal.grouping_add = grouping_add_average; - r->internal.grouping_flush = grouping_flush_average; - } - } - - // allocate any memory required by the grouping method - r->internal.grouping_create(r); - - - // ------------------------------------------------------------------------- - // disable the not-wanted dimensions - - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) - rrdset_check_rdlock(st); - - if(dimensions) - rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list); - - - // ------------------------------------------------------------------------- - // do the work for each dimension - - time_t max_after = 0, min_before = 0; - long max_rows = 0; - - RRDDIM *rd; - long c, dimensions_used = 0, dimensions_nonzero = 0; - struct timeval query_start_time; - struct timeval query_current_time; - if (timeout) - now_realtime_timeval(&query_start_time); - for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { - - // if we need a percentage, we need to calculate all dimensions - if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) { - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED; - continue; - } - r->od[c] |= RRDR_DIMENSION_SELECTED; - - // reset the grouping for the new dimension - r->internal.grouping_reset(r); - - do_dimension_fixedstep( - r - , points_wanted - , rd - , c - , after_wanted - , before_wanted - , options - ); - if (timeout) - now_realtime_timeval(&query_current_time); - if(r->od[c] & RRDR_DIMENSION_NONZERO) - dimensions_nonzero++; + query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t); - // verify all dimensions are aligned - if(unlikely(!dimensions_used)) { - min_before = r->before; - max_after = r->after; - max_rows = r->rows; - } - else { - if(r->after != max_after) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_after, rd->name, (size_t)r->after); - #endif - r->after = (r->after > max_after) ? r->after : max_after; + if (after_wanted == 0) { + after_wanted = first_entry_t; + query_debug_log(":zero after_wanted %lld", after_wanted); } - if(r->before != min_before) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)min_before, rd->name, (size_t)r->before); - #endif - r->before = (r->before < min_before) ? r->before : min_before; + if (before_wanted == 0) { + before_wanted = last_entry_t; + before_is_aligned_to_db_end = true; + query_debug_log(":zero before_wanted %lld", before_wanted); } - if(r->rows != max_rows) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_rows, rd->name, (size_t)r->rows); - #endif - r->rows = (r->rows > max_rows) ? r->rows : max_rows; + if(points_wanted == 0) { + points_wanted = (last_entry_t - first_entry_t) / update_every; + query_debug_log(":zero points_wanted %ld", points_wanted); } } - dimensions_used++; - if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) { - log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)", - dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout); - r->result_options |= RRDR_RESULT_OPTION_CANCEL; - break; + // if they are still zero, assume 600 + + if(after_wanted == 0) { + after_wanted = -600; + query_debug_log(":zero600 after_wanted %lld", after_wanted); } } - #ifdef NETDATA_INTERNAL_CHECKS - if (dimensions_used) { - if(r->internal.log) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log); - - if(r->rows != points_wanted) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'"); - - if(aligned && (r->before % group) != 0) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required"); + if(points_wanted == 0) { + points_wanted = 600; + query_debug_log(":zero600 points_wanted %ld", points_wanted); + } - // 'after' should not be aligned, since we start inside the first group - //if(aligned && (r->after % group) != 0) - // rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); + // convert our before_wanted and after_wanted to absolute + rrdr_relative_window_to_absolute(&after_wanted, &before_wanted); + query_debug_log(":relative2absolute after %lld, before %lld", after_wanted, before_wanted); - if(r->before != before_requested) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'"); + if(natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) { + update_every = rrdset_find_natural_update_every_for_timeframe(st, after_wanted, before_wanted, points_wanted, options, tier); + if(update_every <= 0) update_every = st->update_every; + query_debug_log(":natural update every %d", update_every); + } - if(r->before != before_wanted) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'"); + // this is the update_every of the query + // it may be different to the update_every of the database + time_t query_granularity = (natural_points)?update_every:1; + if(query_granularity <= 0) query_granularity = 1; + query_debug_log(":query_granularity %ld", query_granularity); - // reported 'after' varies, depending on group - if(r->after != after_wanted) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'"); + // align before_wanted and after_wanted to query_granularity + if (before_wanted % query_granularity) { + before_wanted -= before_wanted % query_granularity; + query_debug_log(":granularity align before_wanted %lld", before_wanted); } - #endif - // free all resources used by the grouping method - r->internal.grouping_free(r); + if (after_wanted % query_granularity) { + after_wanted -= after_wanted % query_granularity; + query_debug_log(":granularity align after_wanted %lld", after_wanted); + } - // when all the dimensions are zero, we should return all of them - if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) { - // all the dimensions are zero - // mark them as NONZERO to send them all - for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - r->od[c] |= RRDR_DIMENSION_NONZERO; - } + // automatic_natural_points is set when the user wants all the points available in the database + if(automatic_natural_points) { + points_wanted = (before_wanted - after_wanted + 1) / query_granularity; + if(unlikely(points_wanted <= 0)) points_wanted = 1; + query_debug_log(":auto natural points_wanted %ld", points_wanted); } - rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated); - return r; -} + time_t duration = before_wanted - after_wanted; -#ifdef ENABLE_DBENGINE -static RRDR *rrd2rrdr_variablestep( - ONEWAYALLOC *owa - , RRDSET *st - , long points_requested - , long long after_requested - , long long before_requested - , RRDR_GROUPING group_method - , long resampling_time_requested - , RRDR_OPTIONS options - , const char *dimensions - , int update_every - , time_t first_entry_t - , time_t last_entry_t - , int absolute_period_requested - , struct rrdeng_region_info *region_info_array - , struct context_param *context_param_list - , int timeout -) { - int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); + // if the resampling time is too big, extend the duration to the past + if (unlikely(resampling_time_requested > duration)) { + after_wanted = before_wanted - resampling_time_requested; + duration = before_wanted - after_wanted; + query_debug_log(":resampling after_wanted %lld", after_wanted); + } - // the duration of the chart - time_t duration = before_requested - after_requested; - long available_points = duration / update_every; + // if the duration is not aligned to resampling time + // extend the duration to the past, to avoid a gap at the chart + // only when the missing duration is above 1/10th of a point + if(resampling_time_requested > query_granularity && duration % resampling_time_requested) { + time_t delta = duration % resampling_time_requested; + if(delta > resampling_time_requested / 10) { + after_wanted -= resampling_time_requested - delta; + duration = before_wanted - after_wanted; + query_debug_log(":resampling2 after_wanted %lld", after_wanted); + } + } - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; + // the available points of the query + long points_available = (duration + 1) / query_granularity; + if(unlikely(points_available <= 0)) points_available = 1; + query_debug_log(":points_available %ld", points_available); - if(duration <= 0 || available_points <= 0) { - freez(region_info_array); - return rrdr_create(owa, st, 1, context_param_list); + if(points_wanted > points_available) { + points_wanted = points_available; + query_debug_log(":max points_wanted %ld", points_wanted); } - // check the number of wanted points in the result - if(unlikely(points_requested < 0)) points_requested = -points_requested; - if(unlikely(points_requested > available_points)) points_requested = available_points; - if(unlikely(points_requested == 0)) points_requested = available_points; - // calculate the desired grouping of source data points - long group = available_points / points_requested; - if(unlikely(group <= 0)) group = 1; - if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer - - // resampling_time_requested enforces a certain grouping multiple - calculated_number resampling_divisor = 1.0; - long resampling_group = 1; - if(unlikely(resampling_time_requested > update_every)) { - if (unlikely(resampling_time_requested > duration)) { - // group_time is above the available duration - - #ifdef NETDATA_INTERNAL_CHECKS - info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, resampling_time_requested, duration); - #endif - - after_requested = before_requested - resampling_time_requested; - duration = before_requested - after_requested; - available_points = duration / update_every; - group = available_points / points_requested; - } - - // if the duration is not aligned to resampling time - // extend the duration to the past, to avoid a gap at the chart - // only when the missing duration is above 1/10th of a point - if(duration % resampling_time_requested) { - time_t delta = duration % resampling_time_requested; - if(delta > resampling_time_requested / 10) { - after_requested -= resampling_time_requested - delta; - duration = before_requested - after_requested; - available_points = duration / update_every; - group = available_points / points_requested; - } - } + long group = points_available / points_wanted; + if(group <= 0) group = 1; - // the points we should group to satisfy gtime - resampling_group = resampling_time_requested / update_every; - if(unlikely(resampling_time_requested % update_every)) { - #ifdef NETDATA_INTERNAL_CHECKS - info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every); - #endif - - resampling_group++; - } + // round "group" to the closest integer + if(points_available % points_wanted > points_wanted / 2) + group++; - // adapt group according to resampling_group - if(unlikely(group < resampling_group)) group = resampling_group; // do not allow grouping below the desired one - if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group + query_debug_log(":group %ld", group); - //resampling_divisor = group / resampling_group; - resampling_divisor = (calculated_number)(group * update_every) / (calculated_number)resampling_time_requested; - } + if(points_wanted * group * query_granularity < duration) { + // the grouping we are going to do, is not enough + // to cover the entire duration requested, so + // we have to change the number of points, to make sure we will + // respect the timeframe as closely as possibly - // now that we have group, - // align the requested timeframe to fit it. + // let's see how many points are the optimal + points_wanted = points_available / group; - if(aligned) { - // alignment has been requested, so align the values - before_requested -= before_requested % (group * update_every); - after_requested -= after_requested % (group * update_every); - } + if(points_wanted * group < points_available) + points_wanted++; - // we align the request on requested_before - time_t before_wanted = before_requested; - if(likely(before_wanted > last_entry_t)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, before_wanted is after db max", st->name); - #endif + if(unlikely(points_wanted <= 0)) + points_wanted = 1; - before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * update_every )); + query_debug_log(":optimal points %ld", points_wanted); } - //size_t before_slot = rrdset_time2slot(st, before_wanted); - - // we need to estimate the number of points, for having - // an integer number of values per point - long points_wanted = (before_wanted - after_requested) / (update_every * group); - - time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; - if(unlikely(after_wanted < first_entry_t)) { - // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning - points_wanted = (before_wanted - first_entry_t) / group; - // recalculate after wanted with the new number of points - after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; + // resampling_time_requested enforces a certain grouping multiple + NETDATA_DOUBLE resampling_divisor = 1.0; + long resampling_group = 1; + if(unlikely(resampling_time_requested > query_granularity)) { + // the points we should group to satisfy gtime + resampling_group = resampling_time_requested / query_granularity; + if(unlikely(resampling_time_requested % query_granularity)) + resampling_group++; - if(unlikely(after_wanted < first_entry_t)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name); - #endif + query_debug_log(":resampling group %ld", resampling_group); - after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every ); + // adapt group according to resampling_group + if(unlikely(group < resampling_group)) { + group = resampling_group; // do not allow grouping below the desired one + query_debug_log(":group less res %ld", group); + } + if(unlikely(group % resampling_group)) { + group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group + query_debug_log(":group mod res %ld", group); } - } - //size_t after_slot = rrdset_time2slot(st, after_wanted); - - // check if they are reversed - if(unlikely(after_wanted > before_wanted)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, reversed wanted after/before", st->name); - #endif - time_t tmp = before_wanted; - before_wanted = after_wanted; - after_wanted = tmp; - } - // recalculate points_wanted using the final time-frame - points_wanted = (before_wanted - after_wanted) / update_every / group + 1; - if(unlikely(points_wanted < 0)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted); - #endif - points_wanted = 0; + // resampling_divisor = group / resampling_group; + resampling_divisor = (NETDATA_DOUBLE)(group * query_granularity) / (NETDATA_DOUBLE)resampling_time_requested; + query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor); } -#ifdef NETDATA_INTERNAL_CHECKS - duration = before_wanted - after_wanted; - - if(after_wanted < first_entry_t) - error("INTERNAL CHECK: after_wanted %u is too small, minimum %u", (uint32_t)after_wanted, (uint32_t)first_entry_t); - - if(after_wanted > last_entry_t) - error("INTERNAL CHECK: after_wanted %u is too big, maximum %u", (uint32_t)after_wanted, (uint32_t)last_entry_t); + // now that we have group, align the requested timeframe to fit it. + if(aligned && before_wanted % (group * query_granularity)) { + if(before_is_aligned_to_db_end) + before_wanted -= before_wanted % (group * query_granularity); + else + before_wanted += (group * query_granularity) - before_wanted % (group * query_granularity); + query_debug_log(":align before_wanted %lld", before_wanted); + } - if(before_wanted < first_entry_t) - error("INTERNAL CHECK: before_wanted %u is too small, minimum %u", (uint32_t)before_wanted, (uint32_t)first_entry_t); + after_wanted = before_wanted - (points_wanted * group * query_granularity) + query_granularity; + query_debug_log(":final after_wanted %lld", after_wanted); - if(before_wanted > last_entry_t) - error("INTERNAL CHECK: before_wanted %u is too big, maximum %u", (uint32_t)before_wanted, (uint32_t)last_entry_t); + duration = before_wanted - after_wanted; + query_debug_log(":final duration %ld", duration + 1); -/* - if(before_slot >= (size_t)st->entries) - error("INTERNAL CHECK: before_slot is invalid %zu, expected 0 to %ld", before_slot, st->entries - 1); + // check the context query based on the starting time of the query + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { + rebuild_context_param_list(owa, context_param_list, after_wanted); + st = context_param_list->rd ? context_param_list->rd->rrdset : NULL; - if(after_slot >= (size_t)st->entries) - error("INTERNAL CHECK: after_slot is invalid %zu, expected 0 to %ld", after_slot, st->entries - 1); -*/ + if(unlikely(!st)) + return NULL; + } - if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1) - error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1); + internal_error(points_wanted != duration / (query_granularity * group) + 1, + "QUERY: points_wanted %ld is not points %ld", + points_wanted, duration / (query_granularity * group) + 1); - if(group < resampling_group) - error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group); + internal_error(group < resampling_group, + "QUERY: group %ld is less than the desired group points %ld", + group, resampling_group); - if(group > resampling_group && group % resampling_group) - error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, resampling_group); -#endif + internal_error(group > resampling_group && group % resampling_group, + "QUERY: group %ld is not a multiple of the desired group points %ld", + group, resampling_group); // ------------------------------------------------------------------------- // initialize our result set @@ -1438,26 +1931,21 @@ static RRDR *rrd2rrdr_variablestep( RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); if(unlikely(!r)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); - #endif - freez(region_info_array); + internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", + st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); return NULL; } if(unlikely(!r->d || !points_wanted)) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); - #endif - freez(region_info_array); + internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", + st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); return r; } - r->result_options |= RRDR_RESULT_OPTION_VARIABLE_STEP; - if(unlikely(absolute_period_requested == 1)) - r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; - else + if(relative_period_requested) r->result_options |= RRDR_RESULT_OPTION_RELATIVE; + else + r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; // find how many dimensions we have long dimensions_count = r->d; @@ -1466,48 +1954,26 @@ static RRDR *rrd2rrdr_variablestep( // initialize RRDR r->group = group; - r->update_every = (int)group * update_every; + r->update_every = (int)(group * query_granularity); r->before = before_wanted; r->after = after_wanted; r->internal.points_wanted = points_wanted; r->internal.resampling_group = resampling_group; r->internal.resampling_divisor = resampling_divisor; - + r->internal.query_options = options; + r->internal.query_tier = tier; // ------------------------------------------------------------------------- // assign the processor functions - - { - int i, found = 0; - for(i = 0; !found && api_v1_data_groups[i].name ;i++) { - if(api_v1_data_groups[i].value == group_method) { - r->internal.grouping_create= api_v1_data_groups[i].create; - r->internal.grouping_reset = api_v1_data_groups[i].reset; - r->internal.grouping_free = api_v1_data_groups[i].free; - r->internal.grouping_add = api_v1_data_groups[i].add; - r->internal.grouping_flush = api_v1_data_groups[i].flush; - found = 1; - } - } - if(!found) { - errno = 0; - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name); - #endif - r->internal.grouping_create= grouping_create_average; - r->internal.grouping_reset = grouping_reset_average; - r->internal.grouping_free = grouping_free_average; - r->internal.grouping_add = grouping_add_average; - r->internal.grouping_flush = grouping_flush_average; - } - } + rrdr_set_grouping_function(r, group_method); // allocate any memory required by the grouping method - r->internal.grouping_create(r); + r->internal.grouping_create(r, group_options); // ------------------------------------------------------------------------- // disable the not-wanted dimensions + if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) rrdset_check_rdlock(st); @@ -1515,19 +1981,22 @@ static RRDR *rrd2rrdr_variablestep( rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list); + query_debug_log_fin(); + // ------------------------------------------------------------------------- // do the work for each dimension time_t max_after = 0, min_before = 0; long max_rows = 0; + RRDDIM *first_rd = context_param_list ? context_param_list->rd : st->dimensions; RRDDIM *rd; long c, dimensions_used = 0, dimensions_nonzero = 0; struct timeval query_start_time; struct timeval query_current_time; - if (timeout) - now_realtime_timeval(&query_start_time); - for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { + if (timeout) now_realtime_timeval(&query_start_time); + + for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { // if we need a percentage, we need to calculate all dimensions if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) { @@ -1539,15 +2008,7 @@ static RRDR *rrd2rrdr_variablestep( // reset the grouping for the new dimension r->internal.grouping_reset(r); - do_dimension_variablestep( - r - , points_wanted - , rd - , c - , after_wanted - , before_wanted - , options - ); + rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted); if (timeout) now_realtime_timeval(&query_current_time); @@ -1562,66 +2023,81 @@ static RRDR *rrd2rrdr_variablestep( } else { if(r->after != max_after) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_after, rd->name, (size_t)r->after); - #endif + internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", + st->name, (size_t)max_after, rd->name, (size_t)r->after); + r->after = (r->after > max_after) ? r->after : max_after; } if(r->before != min_before) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)min_before, rd->name, (size_t)r->before); - #endif + internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", + st->name, (size_t)min_before, rd->name, (size_t)r->before); + r->before = (r->before < min_before) ? r->before : min_before; } if(r->rows != max_rows) { - #ifdef NETDATA_INTERNAL_CHECKS - error("INTERNAL ERROR: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_rows, rd->name, (size_t)r->rows); - #endif + internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", + st->name, (size_t)max_rows, rd->name, (size_t)r->rows); + r->rows = (r->rows > max_rows) ? r->rows : max_rows; } } dimensions_used++; - if (timeout && (dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) { + if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) { log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)", - dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout); + (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout); r->result_options |= RRDR_RESULT_OPTION_CANCEL; break; } } - #ifdef NETDATA_INTERNAL_CHECKS - +#ifdef NETDATA_INTERNAL_CHECKS if (dimensions_used) { if(r->internal.log) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log); + rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, + after_wanted, after_requested, before_wanted, before_requested, + points_requested, points_wanted, /*after_slot, before_slot,*/ + r->internal.log); if(r->rows != points_wanted) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'"); + rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, + after_wanted, after_requested, before_wanted, before_requested, + points_requested, points_wanted, /*after_slot, before_slot,*/ + "got 'points' is not wanted 'points'"); - if(aligned && (r->before % group) != 0) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required"); + if(aligned && (r->before % (group * query_granularity)) != 0) + rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, + after_wanted, after_requested, before_wanted,before_wanted, + points_requested, points_wanted, /*after_slot, before_slot,*/ + "'before' is not aligned but alignment is required"); // 'after' should not be aligned, since we start inside the first group //if(aligned && (r->after % group) != 0) - // rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); + // rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); - if(r->before != before_requested) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'"); + if(r->before != before_wanted) + rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, + after_wanted, after_requested, before_wanted, before_requested, + points_requested, points_wanted, /*after_slot, before_slot,*/ + "chart is not aligned to requested 'before'"); if(r->before != before_wanted) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'"); + rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, + after_wanted, after_requested, before_wanted, before_requested, + points_requested, points_wanted, /*after_slot, before_slot,*/ + "got 'before' is not wanted 'before'"); // reported 'after' varies, depending on group if(r->after != after_wanted) - rrd2rrdr_log_request_response_metadata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'"); + rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, + after_wanted, after_requested, before_wanted, before_requested, + points_requested, points_wanted, /*after_slot, before_slot,*/ + "got 'after' is not wanted 'after'"); + } - #endif +#endif // free all resources used by the grouping method r->internal.grouping_free(r); @@ -1630,99 +2106,12 @@ static RRDR *rrd2rrdr_variablestep( if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) { // all the dimensions are zero // mark them as NONZERO to send them all - for(rd = temp_rd?temp_rd:st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { + for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; r->od[c] |= RRDR_DIMENSION_NONZERO; } } rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated); - freez(region_info_array); return r; } -#endif //#ifdef ENABLE_DBENGINE - -RRDR *rrd2rrdr( - ONEWAYALLOC *owa - , RRDSET *st - , long points_requested - , long long after_requested - , long long before_requested - , RRDR_GROUPING group_method - , long resampling_time_requested - , RRDR_OPTIONS options - , const char *dimensions - , struct context_param *context_param_list - , int timeout -) -{ - int rrd_update_every; - int absolute_period_requested; - - time_t first_entry_t; - time_t last_entry_t; - if (context_param_list) { - first_entry_t = context_param_list->first_entry_t; - last_entry_t = context_param_list->last_entry_t; - } else { - rrdset_rdlock(st); - first_entry_t = rrdset_first_entry_t_nolock(st); - last_entry_t = rrdset_last_entry_t_nolock(st); - rrdset_unlock(st); - } - - rrd_update_every = st->update_every; - absolute_period_requested = rrdr_convert_before_after_to_absolute(&after_requested, &before_requested, - rrd_update_every, first_entry_t, - last_entry_t, options); - if (options & RRDR_OPTION_ALLOW_PAST) - if (first_entry_t > after_requested) - first_entry_t = after_requested; - - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { - rebuild_context_param_list(owa, context_param_list, after_requested); - st = context_param_list->rd ? context_param_list->rd->rrdset : NULL; - if (unlikely(!st)) - return NULL; - } - -#ifdef ENABLE_DBENGINE - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - struct rrdeng_region_info *region_info_array; - unsigned regions, max_interval; - - /* This call takes the chart read-lock */ - regions = rrdeng_variable_step_boundaries(st, after_requested, before_requested, - ®ion_info_array, &max_interval, context_param_list); - if (1 == regions) { - if (region_info_array) { - if (rrd_update_every != region_info_array[0].update_every) { - rrd_update_every = region_info_array[0].update_every; - /* recalculate query alignment */ - absolute_period_requested = - rrdr_convert_before_after_to_absolute(&after_requested, &before_requested, rrd_update_every, - first_entry_t, last_entry_t, options); - } - freez(region_info_array); - } - return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method, - resampling_time_requested, options, dimensions, rrd_update_every, - first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); - } else { - if (rrd_update_every != (uint16_t)max_interval) { - rrd_update_every = (uint16_t) max_interval; - /* recalculate query alignment */ - absolute_period_requested = rrdr_convert_before_after_to_absolute(&after_requested, &before_requested, - rrd_update_every, first_entry_t, - last_entry_t, options); - } - return rrd2rrdr_variablestep(owa, st, points_requested, after_requested, before_requested, group_method, - resampling_time_requested, options, dimensions, rrd_update_every, - first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout); - } - } -#endif - return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method, - resampling_time_requested, options, dimensions, - rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); -} diff --git a/web/api/queries/query.h b/web/api/queries/query.h index 6b8a51c58..df876d9ac 100644 --- a/web/api/queries/query.h +++ b/web/api/queries/query.h @@ -3,6 +3,10 @@ #ifndef NETDATA_API_DATA_QUERY_H #define NETDATA_API_DATA_QUERY_H +#ifdef __cplusplus +extern "C" { +#endif + typedef enum rrdr_grouping { RRDR_GROUPING_UNDEFINED = 0, RRDR_GROUPING_AVERAGE, @@ -10,15 +14,46 @@ typedef enum rrdr_grouping { RRDR_GROUPING_MAX, RRDR_GROUPING_SUM, RRDR_GROUPING_INCREMENTAL_SUM, + RRDR_GROUPING_TRIMMED_MEAN1, + RRDR_GROUPING_TRIMMED_MEAN2, + RRDR_GROUPING_TRIMMED_MEAN3, + RRDR_GROUPING_TRIMMED_MEAN5, + RRDR_GROUPING_TRIMMED_MEAN10, + RRDR_GROUPING_TRIMMED_MEAN15, + RRDR_GROUPING_TRIMMED_MEAN20, + RRDR_GROUPING_TRIMMED_MEAN25, RRDR_GROUPING_MEDIAN, + RRDR_GROUPING_TRIMMED_MEDIAN1, + RRDR_GROUPING_TRIMMED_MEDIAN2, + RRDR_GROUPING_TRIMMED_MEDIAN3, + RRDR_GROUPING_TRIMMED_MEDIAN5, + RRDR_GROUPING_TRIMMED_MEDIAN10, + RRDR_GROUPING_TRIMMED_MEDIAN15, + RRDR_GROUPING_TRIMMED_MEDIAN20, + RRDR_GROUPING_TRIMMED_MEDIAN25, + RRDR_GROUPING_PERCENTILE25, + RRDR_GROUPING_PERCENTILE50, + RRDR_GROUPING_PERCENTILE75, + RRDR_GROUPING_PERCENTILE80, + RRDR_GROUPING_PERCENTILE90, + RRDR_GROUPING_PERCENTILE95, + RRDR_GROUPING_PERCENTILE97, + RRDR_GROUPING_PERCENTILE98, + RRDR_GROUPING_PERCENTILE99, RRDR_GROUPING_STDDEV, RRDR_GROUPING_CV, RRDR_GROUPING_SES, RRDR_GROUPING_DES, + RRDR_GROUPING_COUNTIF, } RRDR_GROUPING; extern const char *group_method2string(RRDR_GROUPING group); extern void web_client_api_v1_init_grouping(void); extern RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def); +extern const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group); + +#ifdef __cplusplus +} +#endif #endif //NETDATA_API_DATA_QUERY_H diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c index 4d05778c1..ecf4ca2ac 100644 --- a/web/api/queries/rrdr.c +++ b/web/api/queries/rrdr.c @@ -30,7 +30,7 @@ static void rrdr_dump(RRDR *r) // for each line in the array for(i = 0; i < r->rows ;i++) { - calculated_number *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; RRDR_DIMENSION_FLAGS *co = &r->o[ i * r->d ]; // print the id and the timestamp of the line @@ -44,7 +44,7 @@ static void rrdr_dump(RRDR *r) if(co[c] & RRDR_EMPTY) fprintf(stderr, "null "); else - fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s " + fprintf(stderr, NETDATA_DOUBLE_FORMAT " %s%s%s%s " , cn[c] , (co[c] & RRDR_EMPTY)?"E":" " , (co[c] & RRDR_RESET)?"R":" " @@ -58,78 +58,65 @@ static void rrdr_dump(RRDR *r) } */ +inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) { + if(unlikely(!r)) return; - - -inline static void rrdr_lock_rrdset(RRDR *r) { - if(unlikely(!r)) { - error("NULL value given!"); - return; - } - - rrdset_rdlock(r->st); - r->has_st_lock = 1; -} - -inline static void rrdr_unlock_rrdset(RRDR *r) { - if(unlikely(!r)) { - error("NULL value given!"); - return; - } - - if(likely(r->has_st_lock)) { - r->has_st_lock = 0; + if(likely(r->st_locked_by_rrdr_create)) rrdset_unlock(r->st); - } -} - -inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) -{ - if(unlikely(!r)) { - error("NULL value given!"); - return; - } - rrdr_unlock_rrdset(r); onewayalloc_freez(owa, r->t); onewayalloc_freez(owa, r->v); onewayalloc_freez(owa, r->o); onewayalloc_freez(owa, r->od); + onewayalloc_freez(owa, r->ar); onewayalloc_freez(owa, r); } -RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) -{ - if (unlikely(!st)) { - error("NULL value given!"); - return NULL; - } - +RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) { RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR)); - r->st = st; + r->internal.owa = owa; + + r->d = dimensions; + r->n = points; + + r->t = onewayalloc_callocz(owa, points, sizeof(time_t)); + r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE)); + r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS)); + r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE)); + r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS)); + r->group = 1; + r->update_every = 1; + + return r; +} + +RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) { + if (unlikely(!st)) return NULL; + + bool st_locked_by_rrdr_create = false; if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { - rrdr_lock_rrdset(r); - r->st_needs_lock = 1; + rrdset_rdlock(st); + st_locked_by_rrdr_create = true; } + // count the number of dimensions + int dimensions = 0; RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; RRDDIM *rd; if (temp_rd) { RRDDIM *t = temp_rd; while (t) { - r->d++; + dimensions++; t = t->next; } } else - rrddim_foreach_read(rd, st) r->d++; - - r->n = n; + rrddim_foreach_read(rd, st) dimensions++; - r->t = onewayalloc_callocz(owa, (size_t)n, sizeof(time_t)); - r->v = onewayalloc_mallocz(owa, n * r->d * sizeof(calculated_number)); - r->o = onewayalloc_mallocz(owa, n * r->d * sizeof(RRDR_VALUE_FLAGS)); - r->od = onewayalloc_mallocz(owa, r->d * sizeof(RRDR_DIMENSION_FLAGS)); + // create the rrdr + RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n); + r->st = st; + r->st_locked_by_rrdr_create = st_locked_by_rrdr_create; // set the hidden flag on hidden dimensions int c; @@ -140,8 +127,5 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_pa r->od[c] = RRDR_DIMENSION_DEFAULT; } - r->group = 1; - r->update_every = 1; - return r; } diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index 87ba6c86b..1c80e103f 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -4,27 +4,46 @@ #define NETDATA_QUERIES_RRDR_H #include "libnetdata/libnetdata.h" +#include "web/api/queries/query.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum tier_query_fetch { + TIER_QUERY_FETCH_SUM, + TIER_QUERY_FETCH_MIN, + TIER_QUERY_FETCH_MAX, + TIER_QUERY_FETCH_AVERAGE +} TIER_QUERY_FETCH; typedef enum rrdr_options { - RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values - RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest) - RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing - RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum - RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates - RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates - RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros - RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array - RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs - RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result - RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes - RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total - RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes - RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign - RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs - RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names - RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response - RRDR_OPTION_ALLOW_PAST = 0x00020000, // The after parameter can extend in the past before the first entry - RRDR_OPTION_ANOMALY_BIT = 0x00040000, // Return the anomaly bit stored in each collected_number + RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values + RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest) + RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing + RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum + RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates + RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates + RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros + RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array + RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs + RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result + RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes + RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total + RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes + RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign + RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs + RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names + RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response + RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database + RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points + RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number + RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes + RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap + RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query + + // internal ones - not to be exposed to the API + RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate } RRDR_OPTIONS; typedef enum rrdr_value_flag { @@ -62,40 +81,46 @@ typedef struct rrdresult { RRDR_DIMENSION_FLAGS *od; // the options for the dimensions time_t *t; // array of n timestamps - calculated_number *v; // array n x d values + NETDATA_DOUBLE *v; // array n x d values RRDR_VALUE_FLAGS *o; // array n x d options for each value returned + NETDATA_DOUBLE *ar; // array n x d of anomaly rates (0 - 100) long group; // how many collected values were grouped for each row int update_every; // what is the suggested update frequency in seconds - calculated_number min; - calculated_number max; + NETDATA_DOUBLE min; + NETDATA_DOUBLE max; time_t before; time_t after; - int has_st_lock; // if st is read locked by us - uint8_t st_needs_lock; // if ST should be locked + bool st_locked_by_rrdr_create; // if st is read locked by us // internal rrd2rrdr() members below this point struct { + int query_tier; // the selected tier + RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query) + long points_wanted; long resampling_group; - calculated_number resampling_divisor; + NETDATA_DOUBLE resampling_divisor; - void (*grouping_create)(struct rrdresult *r); + void (*grouping_create)(struct rrdresult *r, const char *options); void (*grouping_reset)(struct rrdresult *r); void (*grouping_free)(struct rrdresult *r); - void (*grouping_add)(struct rrdresult *r, calculated_number value); - calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value); + NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); void *grouping_data; + TIER_QUERY_FETCH tier_query_fetch; #ifdef NETDATA_INTERNAL_CHECKS const char *log; #endif size_t db_points_read; size_t result_points_generated; + size_t tier_points_read[RRD_STORAGE_TIERS]; + ONEWAYALLOC *owa; } internal; } RRDR; @@ -104,16 +129,21 @@ typedef struct rrdresult { #include "database/rrd.h" extern void rrdr_free(ONEWAYALLOC *owa, RRDR *r); extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list); +extern RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points); #include "../web_api_v1.h" #include "web/api/queries/query.h" extern RRDR *rrd2rrdr( ONEWAYALLOC *owa, - RRDSET *st, long points_requested, long long after_requested, long long before_requested, + RRDSET *st, long points_wanted, long long after_wanted, long long before_wanted, RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions, - struct context_param *context_param_list, int timeout); + struct context_param *context_param_list, const char *group_options, int timeout, int tier); + +extern int rrdr_relative_window_to_absolute(long long *after, long long *before); -#include "query.h" +#ifdef __cplusplus +} +#endif #endif //NETDATA_QUERIES_RRDR_H diff --git a/web/api/queries/ses/ses.c b/web/api/queries/ses/ses.c index ae4a0fa0d..5e94002c3 100644 --- a/web/api/queries/ses/ses.c +++ b/web/api/queries/ses/ses.c @@ -7,9 +7,9 @@ // single exponential smoothing struct grouping_ses { - calculated_number alpha; - calculated_number alpha_other; - calculated_number level; + NETDATA_DOUBLE alpha; + NETDATA_DOUBLE alpha_other; + NETDATA_DOUBLE level; size_t count; }; @@ -25,20 +25,20 @@ void grouping_init_ses(void) { } } -static inline calculated_number window(RRDR *r, struct grouping_ses *g) { +static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_ses *g) { (void)g; - calculated_number points; + NETDATA_DOUBLE points; if(r->group == 1) { // provide a running DES - points = r->internal.points_wanted; + points = (NETDATA_DOUBLE)r->internal.points_wanted; } else { // provide a SES with flush points - points = r->group; + points = (NETDATA_DOUBLE)r->group; } - return (points > max_window_size) ? max_window_size : points; + return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points; } static inline void set_alpha(RRDR *r, struct grouping_ses *g) { @@ -48,8 +48,8 @@ static inline void set_alpha(RRDR *r, struct grouping_ses *g) { g->alpha_other = 1.0 - g->alpha; } -void grouping_create_ses(RRDR *r) { - struct grouping_ses *g = (struct grouping_ses *)callocz(1, sizeof(struct grouping_ses)); +void grouping_create_ses(RRDR *r, const char *options __maybe_unused) { + struct grouping_ses *g = (struct grouping_ses *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_ses)); set_alpha(r, g); g->level = 0.0; r->internal.grouping_data = g; @@ -64,11 +64,11 @@ void grouping_reset_ses(RRDR *r) { } void grouping_free_ses(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_ses(RRDR *r, calculated_number value) { +void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value) { struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data; if(unlikely(!g->count)) @@ -78,10 +78,10 @@ void grouping_add_ses(RRDR *r, calculated_number value) { g->count++; } -calculated_number grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data; - if(unlikely(!g->count || !calculated_number_isnumber(g->level))) { + if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; return 0.0; } diff --git a/web/api/queries/ses/ses.h b/web/api/queries/ses/ses.h index c05f208f3..094b8de3f 100644 --- a/web/api/queries/ses/ses.h +++ b/web/api/queries/ses/ses.h @@ -8,10 +8,10 @@ extern void grouping_init_ses(void); -extern void grouping_create_ses(RRDR *r); +extern void grouping_create_ses(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_ses(RRDR *r); extern void grouping_free_ses(RRDR *r); -extern void grouping_add_ses(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_SES_H diff --git a/web/api/queries/stddev/stddev.c b/web/api/queries/stddev/stddev.c index ffe7a47c0..92a67b42d 100644 --- a/web/api/queries/stddev/stddev.c +++ b/web/api/queries/stddev/stddev.c @@ -11,11 +11,11 @@ struct grouping_stddev { long count; - calculated_number m_oldM, m_newM, m_oldS, m_newS; + NETDATA_DOUBLE m_oldM, m_newM, m_oldS, m_newS; }; -void grouping_create_stddev(RRDR *r) { - r->internal.grouping_data = callocz(1, sizeof(struct grouping_stddev)); +void grouping_create_stddev(RRDR *r, const char *options __maybe_unused) { + r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_stddev)); } // resets when switches dimensions @@ -26,11 +26,11 @@ void grouping_reset_stddev(RRDR *r) { } void grouping_free_stddev(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_stddev(RRDR *r, calculated_number value) { +void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value) { struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; g->count++; @@ -50,26 +50,26 @@ void grouping_add_stddev(RRDR *r, calculated_number value) { } } -static inline calculated_number mean(struct grouping_stddev *g) { +static inline NETDATA_DOUBLE mean(struct grouping_stddev *g) { return (g->count > 0) ? g->m_newM : 0.0; } -static inline calculated_number variance(struct grouping_stddev *g) { - return ( (g->count > 1) ? g->m_newS/(g->count - 1) : 0.0 ); +static inline NETDATA_DOUBLE variance(struct grouping_stddev *g) { + return ( (g->count > 1) ? g->m_newS/(NETDATA_DOUBLE)(g->count - 1) : 0.0 ); } -static inline calculated_number stddev(struct grouping_stddev *g) { - return sqrtl(variance(g)); +static inline NETDATA_DOUBLE stddev(struct grouping_stddev *g) { + return sqrtndd(variance(g)); } -calculated_number grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(likely(g->count > 1)) { value = stddev(g); - if(!calculated_number_isnumber(value)) { + if(!netdata_double_isnumber(value)) { value = 0.0; *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; } @@ -88,16 +88,16 @@ calculated_number grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_op } // https://en.wikipedia.org/wiki/Coefficient_of_variation -calculated_number grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(likely(g->count > 1)) { - calculated_number m = mean(g); + NETDATA_DOUBLE m = mean(g); value = 100.0 * stddev(g) / ((m < 0)? -m : m); - if(unlikely(!calculated_number_isnumber(value))) { + if(unlikely(!netdata_double_isnumber(value))) { value = 0.0; *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; } @@ -121,10 +121,10 @@ calculated_number grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FL /* * Mean = average * -calculated_number grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(unlikely(!g->count)) { value = 0.0; @@ -148,10 +148,10 @@ calculated_number grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_opti /* * It is not advised to use this version of variance directly * -calculated_number grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(unlikely(!g->count)) { value = 0.0; diff --git a/web/api/queries/stddev/stddev.h b/web/api/queries/stddev/stddev.h index ab58fbe50..c5c91f88d 100644 --- a/web/api/queries/stddev/stddev.h +++ b/web/api/queries/stddev/stddev.h @@ -6,13 +6,13 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_stddev(RRDR *r); +extern void grouping_create_stddev(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_stddev(RRDR *r); extern void grouping_free_stddev(RRDR *r); -extern void grouping_add_stddev(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -extern calculated_number grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -// extern calculated_number grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -// extern calculated_number grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +// extern NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +// extern NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_STDDEV_H diff --git a/web/api/queries/sum/sum.c b/web/api/queries/sum/sum.c index 6bb012bb0..eec6e2ad0 100644 --- a/web/api/queries/sum/sum.c +++ b/web/api/queries/sum/sum.c @@ -6,12 +6,12 @@ // sum struct grouping_sum { - calculated_number sum; + NETDATA_DOUBLE sum; size_t count; }; -void grouping_create_sum(RRDR *r) { - r->internal.grouping_data = callocz(1, sizeof(struct grouping_sum)); +void grouping_create_sum(RRDR *r, const char *options __maybe_unused) { + r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_sum)); } // resets when switches dimensions @@ -23,20 +23,20 @@ void grouping_reset_sum(RRDR *r) { } void grouping_free_sum(RRDR *r) { - freez(r->internal.grouping_data); + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_sum(RRDR *r, calculated_number value) { +void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value) { struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data; g->sum += value; g->count++; } -calculated_number grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data; - calculated_number value; + NETDATA_DOUBLE value; if(unlikely(!g->count)) { value = 0.0; diff --git a/web/api/queries/sum/sum.h b/web/api/queries/sum/sum.h index 05cb6185e..4e7e396e9 100644 --- a/web/api/queries/sum/sum.h +++ b/web/api/queries/sum/sum.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_sum(RRDR *r); +extern void grouping_create_sum(RRDR *r, const char *options __maybe_unused); extern void grouping_reset_sum(RRDR *r); extern void grouping_free_sum(RRDR *r); -extern void grouping_add_sum(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_SUM_H diff --git a/web/api/queries/trimmed_mean/Makefile.am b/web/api/queries/trimmed_mean/Makefile.am new file mode 100644 index 000000000..161784b8f --- /dev/null +++ b/web/api/queries/trimmed_mean/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/web/api/queries/trimmed_mean/README.md b/web/api/queries/trimmed_mean/README.md new file mode 100644 index 000000000..71cdb85db --- /dev/null +++ b/web/api/queries/trimmed_mean/README.md @@ -0,0 +1,56 @@ +<!-- +title: "Trimmed Mean" +description: "Use trimmed-mean in API queries and health entities to find the average value from a sample, eliminating any unwanted spikes in the returned metrics." +custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/trimmed_mean/README.md +--> + +# Trimmed Mean + +The trimmed mean is the average value of a series excluding the smallest and biggest points. + +Netdata applies linear interpolation on the last point, if the percentage requested to be excluded does not give a +round number of points. + +The following percentile aliases are defined: + +- `trimmed-mean1` +- `trimmed-mean2` +- `trimmed-mean3` +- `trimmed-mean5` +- `trimmed-mean10` +- `trimmed-mean15` +- `trimmed-mean20` +- `trimmed-mean25` + +The default `trimmed-mean` is an alias for `trimmed-mean5`. +Any percentage may be requested using the `group_options` query parameter. + +## how to use + +Use it in alarms like this: + +``` + alarm: my_alarm + on: my_chart +lookup: trimmed-mean5 -1m unaligned of my_dimension + warn: $this > 1000 +``` + +`trimmed-mean` does not change the units. For example, if the chart units is `requests/sec`, the result +will be again expressed in the same units. + +It can also be used in APIs and badges as `&group=trimmed-mean` in the URL and the additional parameter `group_options` +may be used to request any percentage (e.g. `&group=trimmed-mean&group_options=29`). + +## Examples + +Examining last 1 minute `successful` web server responses: + +-  +-  +-  +-  + +## References + +- <https://en.wikipedia.org/wiki/Truncated_mean>. diff --git a/web/api/queries/trimmed_mean/trimmed_mean.c b/web/api/queries/trimmed_mean/trimmed_mean.c new file mode 100644 index 000000000..2277208a7 --- /dev/null +++ b/web/api/queries/trimmed_mean/trimmed_mean.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "trimmed_mean.h" + +// ---------------------------------------------------------------------------- +// median + +struct grouping_trimmed_mean { + size_t series_size; + size_t next_pos; + NETDATA_DOUBLE percent; + + NETDATA_DOUBLE *series; +}; + +static void grouping_create_trimmed_mean_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { + long entries = r->group; + if(entries < 10) entries = 10; + + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_trimmed_mean)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); + g->series_size = (size_t)entries; + + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 50.0) g->percent = 50.0; + } + + g->percent = 1.0 - ((g->percent / 100.0) * 2.0); + r->internal.grouping_data = g; +} + +void grouping_create_trimmed_mean1(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 1.0); +} +void grouping_create_trimmed_mean2(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 2.0); +} +void grouping_create_trimmed_mean3(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 3.0); +} +void grouping_create_trimmed_mean5(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 5.0); +} +void grouping_create_trimmed_mean10(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 10.0); +} +void grouping_create_trimmed_mean15(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 15.0); +} +void grouping_create_trimmed_mean20(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 20.0); +} +void grouping_create_trimmed_mean25(RRDR *r, const char *options) { + grouping_create_trimmed_mean_internal(r, options, 25.0); +} + +// resets when switches dimensions +// so, clear everything to restart +void grouping_reset_trimmed_mean(RRDR *r) { + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; + g->next_pos = 0; +} + +void grouping_free_trimmed_mean(RRDR *r) { + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); + r->internal.grouping_data = NULL; +} + +void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value) { + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; + + if(unlikely(g->next_pos >= g->series_size)) { + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; + } + + g->series[g->next_pos++] = value; +} + +NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; + + NETDATA_DOUBLE value; + size_t available_slots = g->next_pos; + + if(unlikely(!available_slots)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(available_slots == 1) { + value = g->series[0]; + } + else { + sort_series(g->series, available_slots); + + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + + if (min != max) { + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent); + if(!slots_to_use) slots_to_use = 1; + + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use; + + NETDATA_DOUBLE percent_interpolation_slot = 0.0; + NETDATA_DOUBLE percent_last_slot = 0.0; + if(percent_delta > 0.0) { + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use; + + percent_interpolation_slot = percent_delta / percent_1slot; + percent_last_slot = 1 - percent_interpolation_slot; + } + + int start_slot, stop_slot, step, last_slot, interpolation_slot; + if(min >= 0.0 && max >= 0.0) { + start_slot = (int)((available_slots - slots_to_use) / 2); + stop_slot = start_slot + (int)slots_to_use; + last_slot = stop_slot - 1; + interpolation_slot = stop_slot; + step = 1; + } + else { + start_slot = (int)available_slots - 1 - (int)((available_slots - slots_to_use) / 2); + stop_slot = start_slot - (int)slots_to_use; + last_slot = stop_slot + 1; + interpolation_slot = stop_slot; + step = -1; + } + + value = 0.0; + for(int slot = start_slot; slot != stop_slot ; slot += step) + value += g->series[slot]; + + size_t counted = slots_to_use; + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) { + value += g->series[interpolation_slot] * percent_interpolation_slot; + value += g->series[last_slot] * percent_last_slot; + counted++; + } + + value = value / (NETDATA_DOUBLE)counted; + } + else + value = min; + } + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "trimmed_mean"); + + g->next_pos = 0; + + return value; +} diff --git a/web/api/queries/trimmed_mean/trimmed_mean.h b/web/api/queries/trimmed_mean/trimmed_mean.h new file mode 100644 index 000000000..1a4f63e9c --- /dev/null +++ b/web/api/queries/trimmed_mean/trimmed_mean.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_QUERIES_TRIMMED_MEAN_H +#define NETDATA_API_QUERIES_TRIMMED_MEAN_H + +#include "../query.h" +#include "../rrdr.h" + +extern void grouping_create_trimmed_mean1(RRDR *r, const char *options); +extern void grouping_create_trimmed_mean2(RRDR *r, const char *options); +extern void grouping_create_trimmed_mean3(RRDR *r, const char *options); +extern void grouping_create_trimmed_mean5(RRDR *r, const char *options); +extern void grouping_create_trimmed_mean10(RRDR *r, const char *options); +extern void grouping_create_trimmed_mean15(RRDR *r, const char *options); +extern void grouping_create_trimmed_mean20(RRDR *r, const char *options); +extern void grouping_create_trimmed_mean25(RRDR *r, const char *options); +extern void grouping_reset_trimmed_mean(RRDR *r); +extern void grouping_free_trimmed_mean(RRDR *r); +extern void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + +#endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c new file mode 100644 index 000000000..97a00f91c --- /dev/null +++ b/web/api/queries/weights.c @@ -0,0 +1,1220 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "daemon/common.h" +#include "database/KolmogorovSmirnovDist.h" + +#define MAX_POINTS 10000 +int enable_metric_correlations = CONFIG_BOOLEAN_YES; +int metric_correlations_version = 1; +WEIGHTS_METHOD default_metric_correlations_method = WEIGHTS_METHOD_MC_KS2; + +typedef struct weights_stats { + NETDATA_DOUBLE max_base_high_ratio; + size_t db_points; + size_t result_points; + size_t db_queries; + size_t db_points_per_tier[RRD_STORAGE_TIERS]; + size_t binary_searches; +} WEIGHTS_STATS; + +// ---------------------------------------------------------------------------- +// parse and render metric correlations methods + +static struct { + const char *name; + WEIGHTS_METHOD value; +} weights_methods[] = { + { "ks2" , WEIGHTS_METHOD_MC_KS2} + , { "volume" , WEIGHTS_METHOD_MC_VOLUME} + , { "anomaly-rate" , WEIGHTS_METHOD_ANOMALY_RATE} + , { NULL , 0 } +}; + +WEIGHTS_METHOD weights_string_to_method(const char *method) { + for(int i = 0; weights_methods[i].name ;i++) + if(strcmp(method, weights_methods[i].name) == 0) + return weights_methods[i].value; + + return default_metric_correlations_method; +} + +const char *weights_method_to_string(WEIGHTS_METHOD method) { + for(int i = 0; weights_methods[i].name ;i++) + if(weights_methods[i].value == method) + return weights_methods[i].name; + + return "unknown"; +} + +// ---------------------------------------------------------------------------- +// The results per dimension are aggregated into a dictionary + +typedef enum { + RESULT_IS_BASE_HIGH_RATIO = (1 << 0), + RESULT_IS_PERCENTAGE_OF_TIME = (1 << 1), +} RESULT_FLAGS; + +struct register_result { + RESULT_FLAGS flags; + RRDSET *st; + const char *chart_id; + const char *context; + const char *dim_name; + NETDATA_DOUBLE value; + + struct register_result *next; // used to link contexts together +}; + +static void register_result_insert_callback(const char *name, void *value, void *data) { + (void)name; + (void)data; + + struct register_result *t = (struct register_result *)value; + + if(t->chart_id) t->chart_id = strdupz(t->chart_id); + if(t->context) t->context = strdupz(t->context); + if(t->dim_name) t->dim_name = strdupz(t->dim_name); +} + +static void register_result_delete_callback(const char *name, void *value, void *data) { + (void)name; + (void)data; + struct register_result *t = (struct register_result *)value; + + freez((void *)t->chart_id); + freez((void *)t->context); + freez((void *)t->dim_name); +} + +static DICTIONARY *register_result_init() { + DICTIONARY *results = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dictionary_register_insert_callback(results, register_result_insert_callback, results); + dictionary_register_delete_callback(results, register_result_delete_callback, results); + return results; +} + +static void register_result_destroy(DICTIONARY *results) { + dictionary_destroy(results); +} + +static void register_result(DICTIONARY *results, + RRDSET *st, + RRDDIM *d, + NETDATA_DOUBLE value, + RESULT_FLAGS flags, + WEIGHTS_STATS *stats, + bool register_zero) { + + if(!netdata_double_isnumber(value)) return; + + // make it positive + NETDATA_DOUBLE v = fabsndd(value); + + // no need to store zero scored values + if(unlikely(fpclassify(v) == FP_ZERO && !register_zero)) + return; + + // keep track of the max of the baseline / highlight ratio + if(flags & RESULT_IS_BASE_HIGH_RATIO && v > stats->max_base_high_ratio) + stats->max_base_high_ratio = v; + + struct register_result t = { + .flags = flags, + .st = st, + .chart_id = st->id, + .context = st->context, + .dim_name = d->name, + .value = v + }; + + char buf[5000 + 1]; + snprintfz(buf, 5000, "%s:%s", st->id, d->name); + dictionary_set(results, buf, &t, sizeof(struct register_result)); +} + +// ---------------------------------------------------------------------------- +// Generation of JSON output for the results + +static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *wb, + long long after, long long before, + long long baseline_after, long long baseline_before, + long points, WEIGHTS_METHOD method, + RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions __maybe_unused, usec_t duration, + WEIGHTS_STATS *stats) { + + buffer_sprintf(wb, "{\n" + "\t\"after\": %lld,\n" + "\t\"before\": %lld,\n" + "\t\"duration\": %lld,\n" + "\t\"points\": %ld,\n", + after, + before, + before - after, + points + ); + + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) + buffer_sprintf(wb, "" + "\t\"baseline_after\": %lld,\n" + "\t\"baseline_before\": %lld,\n" + "\t\"baseline_duration\": %lld,\n" + "\t\"baseline_points\": %ld,\n", + baseline_after, + baseline_before, + baseline_before - baseline_after, + points << shifts + ); + + buffer_sprintf(wb, "" + "\t\"statistics\": {\n" + "\t\t\"query_time_ms\": %f,\n" + "\t\t\"db_queries\": %zu,\n" + "\t\t\"query_result_points\": %zu,\n" + "\t\t\"binary_searches\": %zu,\n" + "\t\t\"db_points_read\": %zu,\n" + "\t\t\"db_points_per_tier\": [ ", + (double)duration / (double)USEC_PER_MS, + stats->db_queries, + stats->result_points, + stats->binary_searches, + stats->db_points + ); + + for(int tier = 0; tier < storage_tiers ;tier++) + buffer_sprintf(wb, "%s%zu", tier?", ":"", stats->db_points_per_tier[tier]); + + buffer_sprintf(wb, " ]\n" + "\t},\n" + "\t\"group\": \"%s\",\n" + "\t\"method\": \"%s\",\n" + "\t\"options\": \"", + web_client_api_request_v1_data_group_to_string(group), + weights_method_to_string(method) + ); + + web_client_api_request_v1_data_options_to_string(wb, options); +} + +static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb, + long long after, long long before, + long long baseline_after, long long baseline_before, + long points, WEIGHTS_METHOD method, + RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions, usec_t duration, + WEIGHTS_STATS *stats) { + + results_header_to_json(results, wb, after, before, baseline_after, baseline_before, + points, method, group, options, shifts, examined_dimensions, duration, stats); + + buffer_strcat(wb, "\",\n\t\"correlated_charts\": {\n"); + + size_t charts = 0, chart_dims = 0, total_dimensions = 0; + struct register_result *t; + RRDSET *last_st = NULL; // never access this - we use it only for comparison + dfe_start_read(results, t) { + if(!last_st || t->st != last_st) { + last_st = t->st; + + if(charts) buffer_strcat(wb, "\n\t\t\t}\n\t\t},\n"); + buffer_strcat(wb, "\t\t\""); + buffer_strcat(wb, t->chart_id); + buffer_strcat(wb, "\": {\n"); + buffer_strcat(wb, "\t\t\t\"context\": \""); + buffer_strcat(wb, t->context); + buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n"); + charts++; + chart_dims = 0; + } + if (chart_dims) buffer_sprintf(wb, ",\n"); + buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, t->dim_name, t->value); + chart_dims++; + total_dimensions++; + } + dfe_done(t); + + // close dimensions and chart + if (total_dimensions) + buffer_strcat(wb, "\n\t\t\t}\n\t\t}\n"); + + // close correlated_charts + buffer_sprintf(wb, "\t},\n" + "\t\"correlated_dimensions\": %zu,\n" + "\t\"total_dimensions_count\": %zu\n" + "}\n", + total_dimensions, + examined_dimensions + ); + + return total_dimensions; +} + +static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *wb, + long long after, long long before, + long long baseline_after, long long baseline_before, + long points, WEIGHTS_METHOD method, + RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions, usec_t duration, + WEIGHTS_STATS *stats) { + + results_header_to_json(results, wb, after, before, baseline_after, baseline_before, + points, method, group, options, shifts, examined_dimensions, duration, stats); + + DICTIONARY *context_results = dictionary_create( + DICTIONARY_FLAG_SINGLE_THREADED + |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE + |DICTIONARY_FLAG_NAME_LINK_DONT_CLONE + |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE + ); + + struct register_result *t; + dfe_start_read(results, t) { + struct register_result *tc = dictionary_set(context_results, t->context, t, sizeof(*t)); + if(tc == t) + t->next = NULL; + else { + t->next = tc->next; + tc->next = t; + } + } + dfe_done(t); + + buffer_strcat(wb, "\",\n\t\"contexts\": {\n"); + + size_t contexts = 0, total_dimensions = 0, charts = 0, context_dims = 0, chart_dims = 0; + NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0; + RRDSET *last_st = NULL; // never access this - we use it only for comparison + dfe_start_read(context_results, t) { + + if(contexts) + buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims); + + contexts++; + context_dims = 0; + contexts_total_weight = 0.0; + + buffer_strcat(wb, "\t\t\""); + buffer_strcat(wb, t->context); + buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n"); + + charts = 0; + chart_dims = 0; + struct register_result *tt; + for(tt = t; tt ; tt = tt->next) { + if(!last_st || tt->st != last_st) { + last_st = tt->st; + + if(charts) + buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t},\n", charts_total_weight / chart_dims); + + buffer_strcat(wb, "\t\t\t\t\""); + buffer_strcat(wb, tt->chart_id); + buffer_strcat(wb, "\": {\n"); + buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n"); + charts++; + chart_dims = 0; + charts_total_weight = 0.0; + } + + if (chart_dims) buffer_sprintf(wb, ",\n"); + buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, tt->dim_name, tt->value); + charts_total_weight += tt->value; + contexts_total_weight += tt->value; + chart_dims++; + context_dims++; + total_dimensions++; + } + } + dfe_done(t); + + dictionary_destroy(context_results); + + // close dimensions and chart + if (total_dimensions) + buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t}\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims); + + // close correlated_charts + buffer_sprintf(wb, "\t},\n" + "\t\"weighted_dimensions\": %zu,\n" + "\t\"total_dimensions_count\": %zu\n" + "}\n", + total_dimensions, + examined_dimensions + ); + + return total_dimensions; +} + +// ---------------------------------------------------------------------------- +// KS2 algorithm functions + +typedef long int DIFFS_NUMBERS; +#define DOUBLE_TO_INT_MULTIPLIER 100000 + +static inline int binary_search_bigger_than(const DIFFS_NUMBERS arr[], int left, int size, DIFFS_NUMBERS K) { + // binary search to find the index the smallest index + // of the first value in the array that is greater than K + + int right = size; + while(left < right) { + int middle = (int)(((unsigned int)(left + right)) >> 1); + + if(arr[middle] > K) + right = middle; + + else + left = middle + 1; + } + + return left; +} + +int compare_diffs(const void *left, const void *right) { + DIFFS_NUMBERS lt = *(DIFFS_NUMBERS *)left; + DIFFS_NUMBERS rt = *(DIFFS_NUMBERS *)right; + + // https://stackoverflow.com/a/3886497/1114110 + return (lt > rt) - (lt < rt); +} + +static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, NETDATA_DOUBLE *arr, size_t size) { + NETDATA_DOUBLE *last = &arr[size - 1]; + size_t added = 0; + + while(last > arr) { + NETDATA_DOUBLE second = *last--; + NETDATA_DOUBLE first = *last; + *diffs++ = (DIFFS_NUMBERS)((first - second) * (NETDATA_DOUBLE)DOUBLE_TO_INT_MULTIPLIER); + added++; + } + + return added; +} + +static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMBERS highlight_diffs[], int high_size, uint32_t base_shifts) { + + qsort(baseline_diffs, base_size, sizeof(DIFFS_NUMBERS), compare_diffs); + qsort(highlight_diffs, high_size, sizeof(DIFFS_NUMBERS), compare_diffs); + + // Now we should be calculating this: + // + // For each number in the diffs arrays, we should find the index of the + // number bigger than them in both arrays and calculate the % of this index + // vs the total array size. Once we have the 2 percentages, we should find + // the min and max across the delta of all of them. + // + // It should look like this: + // + // base_pcent = binary_search_bigger_than(...) / base_size; + // high_pcent = binary_search_bigger_than(...) / high_size; + // delta = base_pcent - high_pcent; + // if(delta < min) min = delta; + // if(delta > max) max = delta; + // + // This would require a lot of multiplications and divisions. + // + // To speed it up, we do the binary search to find the index of each number + // but then we divide the base index by the power of two number (shifts) it + // is bigger than high index. So the 2 indexes are now comparable. + // We also keep track of the original indexes with min and max, to properly + // calculate their percentages once the loops finish. + + + // initialize min and max using the first number of baseline_diffs + DIFFS_NUMBERS K = baseline_diffs[0]; + int base_idx = binary_search_bigger_than(baseline_diffs, 1, base_size, K); + int high_idx = binary_search_bigger_than(highlight_diffs, 0, high_size, K); + int delta = base_idx - (high_idx << base_shifts); + int min = delta, max = delta; + int base_min_idx = base_idx; + int base_max_idx = base_idx; + int high_min_idx = high_idx; + int high_max_idx = high_idx; + + // do the baseline_diffs starting from 1 (we did position 0 above) + for(int i = 1; i < base_size; i++) { + K = baseline_diffs[i]; + base_idx = binary_search_bigger_than(baseline_diffs, i + 1, base_size, K); // starting from i, since data1 is sorted + high_idx = binary_search_bigger_than(highlight_diffs, 0, high_size, K); + + delta = base_idx - (high_idx << base_shifts); + if(delta < min) { + min = delta; + base_min_idx = base_idx; + high_min_idx = high_idx; + } + else if(delta > max) { + max = delta; + base_max_idx = base_idx; + high_max_idx = high_idx; + } + } + + // do the highlight_diffs starting from 0 + for(int i = 0; i < high_size; i++) { + K = highlight_diffs[i]; + base_idx = binary_search_bigger_than(baseline_diffs, 0, base_size, K); + high_idx = binary_search_bigger_than(highlight_diffs, i + 1, high_size, K); // starting from i, since data2 is sorted + + delta = base_idx - (high_idx << base_shifts); + if(delta < min) { + min = delta; + base_min_idx = base_idx; + high_min_idx = high_idx; + } + else if(delta > max) { + max = delta; + base_max_idx = base_idx; + high_max_idx = high_idx; + } + } + + // now we have the min, max and their indexes + // properly calculate min and max as dmin and dmax + double dbase_size = (double)base_size; + double dhigh_size = (double)high_size; + double dmin = ((double)base_min_idx / dbase_size) - ((double)high_min_idx / dhigh_size); + double dmax = ((double)base_max_idx / dbase_size) - ((double)high_max_idx / dhigh_size); + + dmin = -dmin; + if(islessequal(dmin, 0.0)) dmin = 0.0; + else if(isgreaterequal(dmin, 1.0)) dmin = 1.0; + + double d; + if(isgreaterequal(dmin, dmax)) d = dmin; + else d = dmax; + + double en = round(dbase_size * dhigh_size / (dbase_size + dhigh_size)); + + // under these conditions, KSfbar() crashes + if(unlikely(isnan(en) || isinf(en) || en == 0.0 || isnan(d) || isinf(d))) + return NAN; + + return KSfbar((int)en, d); +} + +static double kstwo( + NETDATA_DOUBLE baseline[], int baseline_points, + NETDATA_DOUBLE highlight[], int highlight_points, uint32_t base_shifts) { + // -1 in size, since the calculate_pairs_diffs() returns one less point + DIFFS_NUMBERS baseline_diffs[baseline_points - 1]; + DIFFS_NUMBERS highlight_diffs[highlight_points - 1]; + + int base_size = (int)calculate_pairs_diff(baseline_diffs, baseline, baseline_points); + int high_size = (int)calculate_pairs_diff(highlight_diffs, highlight, highlight_points); + + if(unlikely(!base_size || !high_size)) + return NAN; + + if(unlikely(base_size != baseline_points - 1 || high_size != highlight_points - 1)) { + error("Metric correlations: internal error - calculate_pairs_diff() returns the wrong number of entries"); + return NAN; + } + + return ks_2samp(baseline_diffs, base_size, highlight_diffs, high_size, base_shifts); +} + + +static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, + long long baseline_after, long long baseline_before, + long long after, long long before, + long long points, RRDR_OPTIONS options, + RRDR_GROUPING group, const char *group_options, int tier, + uint32_t shifts, int timeout, + WEIGHTS_STATS *stats, bool register_zero) { + options |= RRDR_OPTION_NATURAL_POINTS; + + long group_time = 0; + struct context_param *context_param_list = NULL; + + int examined_dimensions = 0; + + RRDR *high_rrdr = NULL; + RRDR *base_rrdr = NULL; + + // get first the highlight to find the number of points available + stats->db_queries++; + usec_t started_usec = now_realtime_usec(); + ONEWAYALLOC *owa = onewayalloc_create(0); + high_rrdr = rrd2rrdr(owa, st, points, + after, before, group, + group_time, options, NULL, context_param_list, group_options, + timeout, tier); + if(!high_rrdr) { + info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name); + goto cleanup; + } + + for(int i = 0; i < storage_tiers ;i++) + stats->db_points_per_tier[i] += high_rrdr->internal.tier_points_read[i]; + + stats->db_points += high_rrdr->internal.db_points_read; + stats->result_points += high_rrdr->internal.result_points_generated; + if(!high_rrdr->d) { + info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name); + goto cleanup; + } + if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) { + info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", st->name); + goto cleanup; + } + int high_points = rrdr_rows(high_rrdr); + + usec_t now_usec = now_realtime_usec(); + if(now_usec - started_usec > timeout * USEC_PER_MS) + goto cleanup; + + // get the baseline, requesting the same number of points as the highlight + stats->db_queries++; + base_rrdr = rrd2rrdr(owa, st,high_points << shifts, + baseline_after, baseline_before, group, + group_time, options, NULL, context_param_list, group_options, + (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), tier); + if(!base_rrdr) { + info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name); + goto cleanup; + } + + for(int i = 0; i < storage_tiers ;i++) + stats->db_points_per_tier[i] += base_rrdr->internal.tier_points_read[i]; + + stats->db_points += base_rrdr->internal.db_points_read; + stats->result_points += base_rrdr->internal.result_points_generated; + if(!base_rrdr->d) { + info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name); + goto cleanup; + } + if (base_rrdr->d != high_rrdr->d) { + info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", st->name); + goto cleanup; + } + if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) { + info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", st->name); + goto cleanup; + } + int base_points = rrdr_rows(base_rrdr); + + now_usec = now_realtime_usec(); + if(now_usec - started_usec > timeout * USEC_PER_MS) + goto cleanup; + + // we need at least 2 points to do the job + if(base_points < 2 || high_points < 2) + goto cleanup; + + // for each dimension + RRDDIM *d; + int i; + for(i = 0, d = base_rrdr->st->dimensions ; d && i < base_rrdr->d; i++, d = d->next) { + + // skip the not evaluated ones + if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN)) + continue; + + examined_dimensions++; + + // skip the dimensions that are just zero for both the baseline and the highlight + if(unlikely(!(base_rrdr->od[i] & RRDR_DIMENSION_NONZERO) && !(high_rrdr->od[i] & RRDR_DIMENSION_NONZERO))) + continue; + + // copy the baseline points of the dimension to a contiguous array + // there is no need to check for empty values, since empty are already zero + NETDATA_DOUBLE baseline[base_points]; + for(int c = 0; c < base_points; c++) + baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ]; + + // copy the highlight points of the dimension to a contiguous array + // there is no need to check for empty values, since empty values are already zero + // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43 + NETDATA_DOUBLE highlight[high_points]; + for(int c = 0; c < high_points; c++) + highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ]; + + stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1); + + double prob = kstwo(baseline, base_points, highlight, high_points, shifts); + if(!isnan(prob) && !isinf(prob)) { + + // these conditions should never happen, but still let's check + if(unlikely(prob < 0.0)) { + error("Metric correlations: kstwo() returned a negative number: %f", prob); + prob = -prob; + } + if(unlikely(prob > 1.0)) { + error("Metric correlations: kstwo() returned a number above 1.0: %f", prob); + prob = 1.0; + } + + // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated + // so we flip the result of kstwo() + register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero); + } + } + +cleanup: + rrdr_free(owa, high_rrdr); + rrdr_free(owa, base_rrdr); + onewayalloc_destroy(owa); + return examined_dimensions; +} + +// ---------------------------------------------------------------------------- +// VOLUME algorithm functions + +static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, + long long baseline_after, long long baseline_before, + long long after, long long before, + RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options, + int tier, int timeout, + WEIGHTS_STATS *stats, bool register_zero) { + + options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS; + long group_time = 0; + + int examined_dimensions = 0; + int ret, value_is_null; + usec_t started_usec = now_realtime_usec(); + + RRDDIM *d; + for(d = st->dimensions; d ; d = d->next) { + usec_t now_usec = now_realtime_usec(); + if(now_usec - started_usec > timeout * USEC_PER_MS) + return examined_dimensions; + + // we count how many metrics we evaluated + examined_dimensions++; + + // there is no point to pass a timeout to these queries + // since the query engine checks for a timeout between + // dimensions, and we query a single dimension at a time. + + stats->db_queries++; + NETDATA_DOUBLE baseline_average = NAN; + NETDATA_DOUBLE base_anomaly_rate = 0; + value_is_null = 1; + ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1, + baseline_after, baseline_before, + group, group_options, group_time, options, + NULL, NULL, + &stats->db_points, stats->db_points_per_tier, + &stats->result_points, + &value_is_null, &base_anomaly_rate, 0, tier); + + if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) { + // this means no data for the baseline window, but we may have data for the highlighted one - assume zero + baseline_average = 0.0; + } + + stats->db_queries++; + NETDATA_DOUBLE highlight_average = NAN; + NETDATA_DOUBLE high_anomaly_rate = 0; + value_is_null = 1; + ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1, + after, before, + group, group_options, group_time, options, + NULL, NULL, + &stats->db_points, stats->db_points_per_tier, + &stats->result_points, + &value_is_null, &high_anomaly_rate, 0, tier); + + if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) { + // this means no data for the highlighted duration - so skip it + continue; + } + + if(baseline_average == highlight_average) { + // they are the same - let's move on + continue; + } + + stats->db_queries++; + NETDATA_DOUBLE highlight_countif = NAN; + value_is_null = 1; + + char highlighted_countif_options[50 + 1]; + snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average); + + ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1, + after, before, + RRDR_GROUPING_COUNTIF,highlighted_countif_options, + group_time, options, + NULL, NULL, + &stats->db_points, stats->db_points_per_tier, + &stats->result_points, + &value_is_null, NULL, 0, tier); + + if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) { + info("MC: highlighted countif query failed, but highlighted average worked - strange..."); + continue; + } + + // this represents the percentage of time + // the highlighted window was above/below the baseline window + // (above or below depending on their averages) + highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0 + + RESULT_FLAGS flags; + NETDATA_DOUBLE pcent = NAN; + if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) { + flags = RESULT_IS_BASE_HIGH_RATIO; + pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif; + } + else { + flags = RESULT_IS_PERCENTAGE_OF_TIME; + pcent = highlight_countif; + } + + register_result(results, st, d, pcent, flags, stats, register_zero); + } + + return examined_dimensions; +} + +// ---------------------------------------------------------------------------- +// ANOMALY RATE algorithm functions + +static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results, + long long after, long long before, + RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options, + int tier, int timeout, + WEIGHTS_STATS *stats, bool register_zero) { + + options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ANOMALY_BIT | RRDR_OPTION_NATURAL_POINTS; + long group_time = 0; + + int examined_dimensions = 0; + int ret, value_is_null; + usec_t started_usec = now_realtime_usec(); + + RRDDIM *d; + for(d = st->dimensions; d ; d = d->next) { + usec_t now_usec = now_realtime_usec(); + if(now_usec - started_usec > timeout * USEC_PER_MS) + return examined_dimensions; + + // we count how many metrics we evaluated + examined_dimensions++; + + // there is no point to pass a timeout to these queries + // since the query engine checks for a timeout between + // dimensions, and we query a single dimension at a time. + + stats->db_queries++; + NETDATA_DOUBLE average = NAN; + NETDATA_DOUBLE anomaly_rate = 0; + value_is_null = 1; + ret = rrdset2value_api_v1(st, NULL, &average, d->id, 1, + after, before, + group, group_options, group_time, options, + NULL, NULL, + &stats->db_points, stats->db_points_per_tier, + &stats->result_points, + &value_is_null, &anomaly_rate, 0, tier); + + if(ret == HTTP_RESP_OK || !value_is_null || netdata_double_isnumber(average)) + register_result(results, st, d, average, 0, stats, register_zero); + } + + return examined_dimensions; +} + +// ---------------------------------------------------------------------------- + +int compare_netdata_doubles(const void *left, const void *right) { + NETDATA_DOUBLE lt = *(NETDATA_DOUBLE *)left; + NETDATA_DOUBLE rt = *(NETDATA_DOUBLE *)right; + + // https://stackoverflow.com/a/3886497/1114110 + return (lt > rt) - (lt < rt); +} + +static inline int binary_search_bigger_than_netdata_double(const NETDATA_DOUBLE arr[], int left, int size, NETDATA_DOUBLE K) { + // binary search to find the index the smallest index + // of the first value in the array that is greater than K + + int right = size; + while(left < right) { + int middle = (int)(((unsigned int)(left + right)) >> 1); + + if(arr[middle] > K) + right = middle; + + else + left = middle + 1; + } + + return left; +} + +// ---------------------------------------------------------------------------- +// spread the results evenly according to their value + +static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) { + struct register_result *t; + + // count the dimensions + size_t dimensions = dictionary_stats_entries(results); + if(!dimensions) return 0; + + if(stats->max_base_high_ratio == 0.0) + stats->max_base_high_ratio = 1.0; + + // create an array of the right size and copy all the values in it + NETDATA_DOUBLE slots[dimensions]; + dimensions = 0; + dfe_start_read(results, t) { + if(t->flags & (RESULT_IS_PERCENTAGE_OF_TIME)) + t->value = t->value * stats->max_base_high_ratio; + + slots[dimensions++] = t->value; + } + dfe_done(t); + + // sort the array with the values of all dimensions + qsort(slots, dimensions, sizeof(NETDATA_DOUBLE), compare_netdata_doubles); + + // skip the duplicates in the sorted array + NETDATA_DOUBLE last_value = NAN; + size_t unique_values = 0; + for(size_t i = 0; i < dimensions ;i++) { + if(likely(slots[i] != last_value)) + slots[unique_values++] = last_value = slots[i]; + } + + // this cannot happen, but coverity thinks otherwise... + if(!unique_values) + unique_values = dimensions; + + // calculate the weight of each slot, using the number of unique values + NETDATA_DOUBLE slot_weight = 1.0 / (NETDATA_DOUBLE)unique_values; + + dfe_start_read(results, t) { + int slot = binary_search_bigger_than_netdata_double(slots, 0, (int)unique_values, t->value); + NETDATA_DOUBLE v = slot * slot_weight; + if(unlikely(v > 1.0)) v = 1.0; + v = 1.0 - v; + t->value = v; + } + dfe_done(t); + + return dimensions; +} + +// ---------------------------------------------------------------------------- +// The main function + +int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, + RRDR_GROUPING group, const char *group_options, + long long baseline_after, long long baseline_before, + long long after, long long before, + long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout) { + WEIGHTS_STATS stats = {}; + + DICTIONARY *results = register_result_init(); + DICTIONARY *charts = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);; + char *error = NULL; + int resp = HTTP_RESP_OK; + + // if the user didn't give a timeout + // assume 60 seconds + if(!timeout) + timeout = 60 * MSEC_PER_SEC; + + // if the timeout is less than 1 second + // make it at least 1 second + if(timeout < (long)(1 * MSEC_PER_SEC)) + timeout = 1 * MSEC_PER_SEC; + + usec_t timeout_usec = timeout * USEC_PER_MS; + usec_t started_usec = now_realtime_usec(); + + if(!rrdr_relative_window_to_absolute(&after, &before)) + buffer_no_cacheable(wb); + + if (before <= after) { + resp = HTTP_RESP_BAD_REQUEST; + error = "Invalid selected time-range."; + goto cleanup; + } + + uint32_t shifts = 0; + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) { + if(!points) points = 500; + + if(baseline_before <= API_RELATIVE_TIME_MAX) + baseline_before += after; + + rrdr_relative_window_to_absolute(&baseline_after, &baseline_before); + + if (baseline_before <= baseline_after) { + resp = HTTP_RESP_BAD_REQUEST; + error = "Invalid baseline time-range."; + goto cleanup; + } + + // baseline should be a power of two multiple of highlight + long long base_delta = baseline_before - baseline_after; + long long high_delta = before - after; + uint32_t multiplier = (uint32_t)round((double)base_delta / (double)high_delta); + + // check if the multiplier is a power of two + // https://stackoverflow.com/a/600306/1114110 + if((multiplier & (multiplier - 1)) != 0) { + // it is not power of two + // let's find the closest power of two + // https://stackoverflow.com/a/466242/1114110 + multiplier--; + multiplier |= multiplier >> 1; + multiplier |= multiplier >> 2; + multiplier |= multiplier >> 4; + multiplier |= multiplier >> 8; + multiplier |= multiplier >> 16; + multiplier++; + } + + // convert the multiplier to the number of shifts + // we need to do, to divide baseline numbers to match + // the highlight ones + while(multiplier > 1) { + shifts++; + multiplier = multiplier >> 1; + } + + // if the baseline size will not comply to MAX_POINTS + // lower the window of the baseline + while(shifts && (points << shifts) > MAX_POINTS) + shifts--; + + // if the baseline size still does not comply to MAX_POINTS + // lower the resolution of the highlight and the baseline + while((points << shifts) > MAX_POINTS) + points = points >> 1; + + if(points < 15) { + resp = HTTP_RESP_BAD_REQUEST; + error = "Too few points available, at least 15 are needed."; + goto cleanup; + } + + // adjust the baseline to be multiplier times bigger than the highlight + baseline_after = baseline_before - (high_delta << shifts); + } + + // dont lock here and wait for results + // get the charts and run mc after + RRDSET *st; + rrdhost_rdlock(host); + rrdset_foreach_read(st, host) { + if (rrdset_is_available_for_viewers(st)) { + if(!contexts || simple_pattern_matches(contexts, st->context)) + dictionary_set(charts, st->name, NULL, 0); + } + } + rrdhost_unlock(host); + + size_t examined_dimensions = 0; + void *ptr; + + bool register_zero = true; + if(options & RRDR_OPTION_NONZERO) { + register_zero = false; + options &= ~RRDR_OPTION_NONZERO; + } + + // for every chart in the dictionary + dfe_start_read(charts, ptr) { + usec_t now_usec = now_realtime_usec(); + if(now_usec - started_usec > timeout_usec) { + error = "timed out"; + resp = HTTP_RESP_GATEWAY_TIMEOUT; + goto cleanup; + } + + st = rrdset_find_byname(host, ptr_name); + if(!st) continue; + + rrdset_rdlock(st); + + switch(method) { + case WEIGHTS_METHOD_ANOMALY_RATE: + options |= RRDR_OPTION_ANOMALY_BIT; + points = 1; + examined_dimensions += rrdset_weights_anomaly_rate(st, results, + after, before, + options, group, group_options, tier, + (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), + &stats, register_zero); + break; + + case WEIGHTS_METHOD_MC_VOLUME: + points = 1; + examined_dimensions += rrdset_metric_correlations_volume(st, results, + baseline_after, baseline_before, + after, before, + options, group, group_options, tier, + (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), + &stats, register_zero); + break; + + default: + case WEIGHTS_METHOD_MC_KS2: + examined_dimensions += rrdset_metric_correlations_ks2(st, results, + baseline_after, baseline_before, + after, before, + points, options, group, group_options, tier, shifts, + (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), + &stats, register_zero); + break; + } + + rrdset_unlock(st); + } + dfe_done(ptr); + + if(!register_zero) + options |= RRDR_OPTION_NONZERO; + + if(!(options & RRDR_OPTION_RETURN_RAW)) + spread_results_evenly(results, &stats); + + usec_t ended_usec = now_realtime_usec(); + + // generate the json output we need + buffer_flush(wb); + + size_t added_dimensions = 0; + switch(format) { + case WEIGHTS_FORMAT_CHARTS: + added_dimensions = registered_results_to_json_charts(results, wb, + after, before, + baseline_after, baseline_before, + points, method, group, options, shifts, + examined_dimensions, + ended_usec - started_usec, &stats); + break; + + default: + case WEIGHTS_FORMAT_CONTEXTS: + added_dimensions = registered_results_to_json_contexts(results, wb, + after, before, + baseline_after, baseline_before, + points, method, group, options, shifts, + examined_dimensions, + ended_usec - started_usec, &stats); + break; + } + + if(!added_dimensions) { + error = "no results produced."; + resp = HTTP_RESP_NOT_FOUND; + } + +cleanup: + if(charts) dictionary_destroy(charts); + if(results) register_result_destroy(results); + + if(error) { + buffer_flush(wb); + buffer_sprintf(wb, "{\"error\": \"%s\" }", error); + } + + return resp; +} + +// ---------------------------------------------------------------------------- +// unittest + +/* + +Unit tests against the output of this: + +https://github.com/scipy/scipy/blob/4cf21e753cf937d1c6c2d2a0e372fbc1dbbeea81/scipy/stats/_stats_py.py#L7275-L7449 + +import matplotlib.pyplot as plt +import pandas as pd +import numpy as np +import scipy as sp +from scipy import stats + +data1 = np.array([ 1111, -2222, 33, 100, 100, 15555, -1, 19999, 888, 755, -1, -730 ]) +data2 = np.array([365, -123, 0]) +data1 = np.sort(data1) +data2 = np.sort(data2) +n1 = data1.shape[0] +n2 = data2.shape[0] +data_all = np.concatenate([data1, data2]) +cdf1 = np.searchsorted(data1, data_all, side='right') / n1 +cdf2 = np.searchsorted(data2, data_all, side='right') / n2 +print(data_all) +print("\ndata1", data1, cdf1) +print("\ndata2", data2, cdf2) +cddiffs = cdf1 - cdf2 +print("\ncddiffs", cddiffs) +minS = np.clip(-np.min(cddiffs), 0, 1) +maxS = np.max(cddiffs) +print("\nmin", minS) +print("max", maxS) +m, n = sorted([float(n1), float(n2)], reverse=True) +en = m * n / (m + n) +d = max(minS, maxS) +prob = stats.distributions.kstwo.sf(d, np.round(en)) +print("\nprob", prob) + +*/ + +static int double_expect(double v, const char *str, const char *descr) { + char buf[100 + 1]; + snprintfz(buf, 100, "%0.6f", v); + int ret = strcmp(buf, str) ? 1 : 0; + + fprintf(stderr, "%s %s, expected %s, got %s\n", ret?"FAILED":"OK", descr, str, buf); + return ret; +} + +static int mc_unittest1(void) { + int bs = 3, hs = 3; + DIFFS_NUMBERS base[3] = { 1, 2, 3 }; + DIFFS_NUMBERS high[3] = { 3, 4, 6 }; + + double prob = ks_2samp(base, bs, high, hs, 0); + return double_expect(prob, "0.222222", "3x3"); +} + +static int mc_unittest2(void) { + int bs = 6, hs = 3; + DIFFS_NUMBERS base[6] = { 1, 2, 3, 10, 10, 15 }; + DIFFS_NUMBERS high[3] = { 3, 4, 6 }; + + double prob = ks_2samp(base, bs, high, hs, 1); + return double_expect(prob, "0.500000", "6x3"); +} + +static int mc_unittest3(void) { + int bs = 12, hs = 3; + DIFFS_NUMBERS base[12] = { 1, 2, 3, 10, 10, 15, 111, 19999, 8, 55, -1, -73 }; + DIFFS_NUMBERS high[3] = { 3, 4, 6 }; + + double prob = ks_2samp(base, bs, high, hs, 2); + return double_expect(prob, "0.347222", "12x3"); +} + +static int mc_unittest4(void) { + int bs = 12, hs = 3; + DIFFS_NUMBERS base[12] = { 1111, -2222, 33, 100, 100, 15555, -1, 19999, 888, 755, -1, -730 }; + DIFFS_NUMBERS high[3] = { 365, -123, 0 }; + + double prob = ks_2samp(base, bs, high, hs, 2); + return double_expect(prob, "0.777778", "12x3"); +} + +int mc_unittest(void) { + int errors = 0; + + errors += mc_unittest1(); + errors += mc_unittest2(); + errors += mc_unittest3(); + errors += mc_unittest4(); + + return errors; +} + diff --git a/web/api/queries/weights.h b/web/api/queries/weights.h new file mode 100644 index 000000000..f88a134f2 --- /dev/null +++ b/web/api/queries/weights.h @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_WEIGHTS_H +#define NETDATA_API_WEIGHTS_H 1 + +#include "query.h" + +typedef enum { + WEIGHTS_METHOD_MC_KS2 = 1, + WEIGHTS_METHOD_MC_VOLUME = 2, + WEIGHTS_METHOD_ANOMALY_RATE = 3, +} WEIGHTS_METHOD; + +typedef enum { + WEIGHTS_FORMAT_CHARTS = 1, + WEIGHTS_FORMAT_CONTEXTS = 2, +} WEIGHTS_FORMAT; + +extern int enable_metric_correlations; +extern int metric_correlations_version; +extern WEIGHTS_METHOD default_metric_correlations_method; + +extern int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, + RRDR_GROUPING group, const char *group_options, + long long baseline_after, long long baseline_before, + long long after, long long before, + long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout); + +extern WEIGHTS_METHOD weights_string_to_method(const char *method); +extern const char *weights_method_to_string(WEIGHTS_METHOD method); +extern int mc_unittest(void); + +#endif //NETDATA_API_WEIGHTS_H |