diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2023-05-08 16:27:04 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2023-05-08 16:27:04 +0000 |
| commit | a836a244a3d2bdd4da1ee2641e3e957850668cea (patch) | |
| tree | cb87c75b3677fab7144f868435243f864048a1e6 /web/api/queries | |
| parent | Adding upstream version 1.38.1. (diff) | |
| download | netdata-a836a244a3d2bdd4da1ee2641e3e957850668cea.tar.xz netdata-a836a244a3d2bdd4da1ee2641e3e957850668cea.zip | |
Adding upstream version 1.39.0.upstream/1.39.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'web/api/queries')
43 files changed, 4735 insertions, 2263 deletions
diff --git a/web/api/queries/README.md b/web/api/queries/README.md index 2a17ac78..dacd2900 100644 --- a/web/api/queries/README.md +++ b/web/api/queries/README.md @@ -1,11 +1,10 @@ -<!-- -title: "Database Queries" -custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/README.md ---> +# Database queries/lookup -# Database Queries +This document explains in detail the options available to retrieve data from the Netdata timeseries database in order to configure alerts, create badges or +create custom charts. -Netdata database can be queried with `/api/v1/data` and `/api/v1/badge.svg` REST API methods. +The Netdata database can be queried with the `/api/v1/data` and `/api/v1/badge.svg` REST API methods. The database is also queried from the `lookup` line +in an [alert configuration](https://github.com/netdata/netdata/blob/master/health/REFERENCE.md). Every data query accepts the following parameters: @@ -104,18 +103,24 @@ For each value it calls the **grouping method** given with the `&group=` query p The following grouping methods are supported. These are given all the values in the time-frame and they group the values every `group points`. --  finds the minimum value --  finds the maximum value --  finds the average value --  adds all the values and returns the sum --  sorts the values and returns the value in the middle of the list --  finds the standard deviation of the values --  finds the relative standard deviation (coefficient of variation) of the values --  finds the exponential weighted moving average of the values --  applies Holt-Winters double exponential smoothing --  finds the difference of the last vs the first value - -The examples shown above, are live information from the `successful` web requests of the global Netdata registry. +-  finds the minimum value +-  finds the maximum value +-  finds the average value +-  adds all the values and returns the sum +-  sorts the values and returns the value in the middle of the list +-  finds the standard deviation of the values +-  finds the relative standard deviation (coefficient of variation) of the values +-  finds the exponential weighted moving average of the values +-  applies Holt-Winters double exponential smoothing +-  finds the difference of the last vs the first value + +The examples shown above show live information from the `received` traffic on the `eth0` interface of the global Netdata registry. +Inspect any of the badges to see the parameters provided. You can directly issue the request to the registry server's API yourself, e.g. by +passing the following to get the value shown on the badge for the sum of the values within the period: + +``` +https://registry.my-netdata.io/api/v1/data?chart=net.eth0&options=unaligned&dimensions=received&group=sum&units=kilobits&after=-60&label=sum&points=1 +``` ## Further processing diff --git a/web/api/queries/average/README.md b/web/api/queries/average/README.md index b8d4ba7e..c9aa402c 100644 --- a/web/api/queries/average/README.md +++ b/web/api/queries/average/README.md @@ -1,6 +1,10 @@ <!-- title: "Average or Mean" +sidebar_label: "Average or Mean" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/average/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Average or Mean diff --git a/web/api/queries/average/average.c b/web/api/queries/average/average.c index 0719d57f..f54dcb24 100644 --- a/web/api/queries/average/average.c +++ b/web/api/queries/average/average.c @@ -2,58 +2,3 @@ #include "average.h" -// ---------------------------------------------------------------------------- -// average - -struct grouping_average { - NETDATA_DOUBLE sum; - size_t count; -}; - -void grouping_create_average(RRDR *r, const char *options __maybe_unused) { - r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_average)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_average(RRDR *r) { - struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; - g->sum = 0; - g->count = 0; -} - -void grouping_free_average(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_average(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; - g->sum += value; - g->count++; -} - -NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_average *g = (struct grouping_average *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - if(unlikely(r->internal.resampling_group != 1)) { - if (unlikely(r->result_options & RRDR_RESULT_OPTION_VARIABLE_STEP)) - value = g->sum / g->count / r->internal.resampling_divisor; - else - value = g->sum / r->internal.resampling_divisor; - } else - value = g->sum / g->count; - } - - g->sum = 0.0; - g->count = 0; - - return value; -} diff --git a/web/api/queries/average/average.h b/web/api/queries/average/average.h index b3196688..2d77cc57 100644 --- a/web/api/queries/average/average.h +++ b/web/api/queries/average/average.h @@ -6,10 +6,57 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_average(RRDR *r, const char *options __maybe_unused); -void grouping_reset_average(RRDR *r); -void grouping_free_average(RRDR *r); -void grouping_add_average(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +// ---------------------------------------------------------------------------- +// average + +struct tg_average { + NETDATA_DOUBLE sum; + size_t count; +}; + +static inline void tg_average_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_average)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_average_reset(RRDR *r) { + struct tg_average *g = (struct tg_average *)r->time_grouping.data; + g->sum = 0; + g->count = 0; +} + +static inline void tg_average_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_average_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_average *g = (struct tg_average *)r->time_grouping.data; + g->sum += value; + g->count++; +} + +static inline NETDATA_DOUBLE tg_average_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_average *g = (struct tg_average *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + if(unlikely(r->time_grouping.resampling_group != 1)) + value = g->sum / r->time_grouping.resampling_divisor; + else + value = g->sum / g->count; + } + + g->sum = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_AVERAGE_H diff --git a/web/api/queries/countif/README.md b/web/api/queries/countif/README.md index 200a4c9e..37b3f642 100644 --- a/web/api/queries/countif/README.md +++ b/web/api/queries/countif/README.md @@ -1,6 +1,10 @@ <!-- title: "CountIf" +sidebar_label: "CountIf" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/countif/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # CountIf diff --git a/web/api/queries/countif/countif.c b/web/api/queries/countif/countif.c index 369b20be..8a3a1f50 100644 --- a/web/api/queries/countif/countif.c +++ b/web/api/queries/countif/countif.c @@ -5,132 +5,3 @@ // ---------------------------------------------------------------------------- // countif -struct grouping_countif { - size_t (*comparison)(NETDATA_DOUBLE, NETDATA_DOUBLE); - NETDATA_DOUBLE target; - size_t count; - size_t matched; -}; - -static size_t countif_equal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v == target); -} - -static size_t countif_notequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v != target); -} - -static size_t countif_less(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v < target); -} - -static size_t countif_lessequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v <= target); -} - -static size_t countif_greater(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v > target); -} - -static size_t countif_greaterequal(NETDATA_DOUBLE v, NETDATA_DOUBLE target) { - return (v >= target); -} - -void grouping_create_countif(RRDR *r, const char *options __maybe_unused) { - struct grouping_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_countif)); - r->internal.grouping_data = g; - - if(options && *options) { - // skip any leading spaces - while(isspace(*options)) options++; - - // find the comparison function - switch(*options) { - case '!': - options++; - if(*options != '=' && *options != ':') - options--; - g->comparison = countif_notequal; - break; - - case '>': - options++; - if(*options == '=' || *options == ':') { - g->comparison = countif_greaterequal; - } - else { - options--; - g->comparison = countif_greater; - } - break; - - case '<': - options++; - if(*options == '>') { - g->comparison = countif_notequal; - } - else if(*options == '=' || *options == ':') { - g->comparison = countif_lessequal; - } - else { - options--; - g->comparison = countif_less; - } - break; - - default: - case '=': - case ':': - g->comparison = countif_equal; - break; - } - if(*options) options++; - - // skip everything up to the first digit - while(isspace(*options)) options++; - - g->target = str2ndd(options, NULL); - } - else { - g->target = 0.0; - g->comparison = countif_equal; - } -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_countif(RRDR *r) { - struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; - g->matched = 0; - g->count = 0; -} - -void grouping_free_countif(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; - g->matched += g->comparison(value, g->target); - g->count++; -} - -NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_countif *g = (struct grouping_countif *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count; - } - - g->matched = 0; - g->count = 0; - - return value; -} diff --git a/web/api/queries/countif/countif.h b/web/api/queries/countif/countif.h index dfe80565..896b9d87 100644 --- a/web/api/queries/countif/countif.h +++ b/web/api/queries/countif/countif.h @@ -6,10 +6,143 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_countif(RRDR *r, const char *options __maybe_unused); -void grouping_reset_countif(RRDR *r); -void grouping_free_countif(RRDR *r); -void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +enum tg_countif_cmp { + TG_COUNTIF_EQUAL, + TG_COUNTIF_NOTEQUAL, + TG_COUNTIF_LESS, + TG_COUNTIF_LESSEQUAL, + TG_COUNTIF_GREATER, + TG_COUNTIF_GREATEREQUAL, +}; + +struct tg_countif { + enum tg_countif_cmp comparison; + NETDATA_DOUBLE target; + size_t count; + size_t matched; +}; + +static inline void tg_countif_create(RRDR *r, const char *options __maybe_unused) { + struct tg_countif *g = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_countif)); + r->time_grouping.data = g; + + if(options && *options) { + // skip any leading spaces + while(isspace(*options)) options++; + + // find the comparison function + switch(*options) { + case '!': + options++; + if(*options != '=' && *options != ':') + options--; + g->comparison = TG_COUNTIF_NOTEQUAL; + break; + + case '>': + options++; + if(*options == '=' || *options == ':') { + g->comparison = TG_COUNTIF_GREATEREQUAL; + } + else { + options--; + g->comparison = TG_COUNTIF_GREATER; + } + break; + + case '<': + options++; + if(*options == '>') { + g->comparison = TG_COUNTIF_NOTEQUAL; + } + else if(*options == '=' || *options == ':') { + g->comparison = TG_COUNTIF_LESSEQUAL; + } + else { + options--; + g->comparison = TG_COUNTIF_LESS; + } + break; + + default: + case '=': + case ':': + g->comparison = TG_COUNTIF_EQUAL; + break; + } + if(*options) options++; + + // skip everything up to the first digit + while(isspace(*options)) options++; + + g->target = str2ndd(options, NULL); + } + else { + g->target = 0.0; + g->comparison = TG_COUNTIF_EQUAL; + } +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_countif_reset(RRDR *r) { + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data; + g->matched = 0; + g->count = 0; +} + +static inline void tg_countif_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_countif_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data; + switch(g->comparison) { + case TG_COUNTIF_GREATER: + if(value > g->target) g->matched++; + break; + + case TG_COUNTIF_GREATEREQUAL: + if(value >= g->target) g->matched++; + break; + + case TG_COUNTIF_LESS: + if(value < g->target) g->matched++; + break; + + case TG_COUNTIF_LESSEQUAL: + if(value <= g->target) g->matched++; + break; + + case TG_COUNTIF_EQUAL: + if(value == g->target) g->matched++; + break; + + case TG_COUNTIF_NOTEQUAL: + if(value != g->target) g->matched++; + break; + } + g->count++; +} + +static inline NETDATA_DOUBLE tg_countif_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_countif *g = (struct tg_countif *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = (NETDATA_DOUBLE)g->matched * 100 / (NETDATA_DOUBLE)g->count; + } + + g->matched = 0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_COUNTIF_H diff --git a/web/api/queries/des/README.md b/web/api/queries/des/README.md index 33c5f1a0..b12751a4 100644 --- a/web/api/queries/des/README.md +++ b/web/api/queries/des/README.md @@ -1,6 +1,10 @@ <!-- title: "double exponential smoothing" +sidebar_label: "double exponential smoothing" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/des/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # double exponential smoothing diff --git a/web/api/queries/des/des.c b/web/api/queries/des/des.c index a6c4e405..d0e234e2 100644 --- a/web/api/queries/des/des.c +++ b/web/api/queries/des/des.c @@ -6,132 +6,3 @@ // ---------------------------------------------------------------------------- // single exponential smoothing - -struct grouping_des { - NETDATA_DOUBLE alpha; - NETDATA_DOUBLE alpha_other; - NETDATA_DOUBLE beta; - NETDATA_DOUBLE beta_other; - - NETDATA_DOUBLE level; - NETDATA_DOUBLE trend; - - size_t count; -}; - -static size_t max_window_size = 15; - -void grouping_init_des(void) { - long long ret = config_get_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size); - if(ret <= 1) { - config_set_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size); - } - else { - max_window_size = (size_t) ret; - } -} - -static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_des *g) { - (void)g; - - NETDATA_DOUBLE points; - if(r->group == 1) { - // provide a running DES - points = (NETDATA_DOUBLE)r->internal.points_wanted; - } - else { - // provide a SES with flush points - points = (NETDATA_DOUBLE)r->group; - } - - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - // A commonly used value for alpha is 2 / (N + 1) - return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points; -} - -static inline void set_alpha(RRDR *r, struct grouping_des *g) { - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - // A commonly used value for alpha is 2 / (N + 1) - - g->alpha = 2.0 / (window(r, g) + 1.0); - g->alpha_other = 1.0 - g->alpha; - - //info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha); -} - -static inline void set_beta(RRDR *r, struct grouping_des *g) { - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - // A commonly used value for alpha is 2 / (N + 1) - - g->beta = 2.0 / (window(r, g) + 1.0); - g->beta_other = 1.0 - g->beta; - - //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta); -} - -void grouping_create_des(RRDR *r, const char *options __maybe_unused) { - struct grouping_des *g = (struct grouping_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct grouping_des)); - set_alpha(r, g); - set_beta(r, g); - g->level = 0.0; - g->trend = 0.0; - g->count = 0; - r->internal.grouping_data = g; -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_des(RRDR *r) { - struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; - g->level = 0.0; - g->trend = 0.0; - g->count = 0; - - // fprintf(stderr, "\nDES: "); - -} - -void grouping_free_des(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_des(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; - - if(likely(g->count > 0)) { - // we have at least a number so far - - if(unlikely(g->count == 1)) { - // the second value we got - g->trend = value - g->trend; - g->level = value; - } - - // for the values, except the first - NETDATA_DOUBLE last_level = g->level; - g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); - g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); - } - else { - // the first value we got - g->level = g->trend = value; - } - - g->count++; - - //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend); -} - -NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data; - - if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - return 0.0; - } - - //fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level); - - return g->level; -} diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h index 05fa01b3..3153d497 100644 --- a/web/api/queries/des/des.h +++ b/web/api/queries/des/des.h @@ -6,12 +6,133 @@ #include "../query.h" #include "../rrdr.h" -void grouping_init_des(void); +struct tg_des { + NETDATA_DOUBLE alpha; + NETDATA_DOUBLE alpha_other; + NETDATA_DOUBLE beta; + NETDATA_DOUBLE beta_other; -void grouping_create_des(RRDR *r, const char *options __maybe_unused); -void grouping_reset_des(RRDR *r); -void grouping_free_des(RRDR *r); -void grouping_add_des(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + NETDATA_DOUBLE level; + NETDATA_DOUBLE trend; + + size_t count; +}; + +static size_t tg_des_max_window_size = 15; + +static inline void tg_des_init(void) { + long long ret = config_get_number(CONFIG_SECTION_WEB, "des max tg_des_window", (long long)tg_des_max_window_size); + if(ret <= 1) { + config_set_number(CONFIG_SECTION_WEB, "des max tg_des_window", (long long)tg_des_max_window_size); + } + else { + tg_des_max_window_size = (size_t) ret; + } +} + +static inline NETDATA_DOUBLE tg_des_window(RRDR *r, struct tg_des *g) { + (void)g; + + NETDATA_DOUBLE points; + if(r->view.group == 1) { + // provide a running DES + points = (NETDATA_DOUBLE)r->time_grouping.points_wanted; + } + else { + // provide a SES with flush points + points = (NETDATA_DOUBLE)r->view.group; + } + + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + // A commonly used value for alpha is 2 / (N + 1) + return (points > (NETDATA_DOUBLE)tg_des_max_window_size) ? (NETDATA_DOUBLE)tg_des_max_window_size : points; +} + +static inline void tg_des_set_alpha(RRDR *r, struct tg_des *g) { + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + // A commonly used value for alpha is 2 / (N + 1) + + g->alpha = 2.0 / (tg_des_window(r, g) + 1.0); + g->alpha_other = 1.0 - g->alpha; + + //info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha); +} + +static inline void tg_des_set_beta(RRDR *r, struct tg_des *g) { + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + // A commonly used value for alpha is 2 / (N + 1) + + g->beta = 2.0 / (tg_des_window(r, g) + 1.0); + g->beta_other = 1.0 - g->beta; + + //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta); +} + +static inline void tg_des_create(RRDR *r, const char *options __maybe_unused) { + struct tg_des *g = (struct tg_des *)onewayalloc_mallocz(r->internal.owa, sizeof(struct tg_des)); + tg_des_set_alpha(r, g); + tg_des_set_beta(r, g); + g->level = 0.0; + g->trend = 0.0; + g->count = 0; + r->time_grouping.data = g; +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_des_reset(RRDR *r) { + struct tg_des *g = (struct tg_des *)r->time_grouping.data; + g->level = 0.0; + g->trend = 0.0; + g->count = 0; + + // fprintf(stderr, "\nDES: "); + +} + +static inline void tg_des_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_des_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_des *g = (struct tg_des *)r->time_grouping.data; + + if(likely(g->count > 0)) { + // we have at least a number so far + + if(unlikely(g->count == 1)) { + // the second value we got + g->trend = value - g->trend; + g->level = value; + } + + // for the values, except the first + NETDATA_DOUBLE last_level = g->level; + g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend)); + g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend); + } + else { + // the first value we got + g->level = g->trend = value; + } + + g->count++; + + //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend); +} + +static inline NETDATA_DOUBLE tg_des_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_des *g = (struct tg_des *)r->time_grouping.data; + + if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + return 0.0; + } + + //fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level); + + return g->level; +} #endif //NETDATA_API_QUERIES_DES_H diff --git a/web/api/queries/incremental_sum/README.md b/web/api/queries/incremental_sum/README.md index 44301172..9b89f318 100644 --- a/web/api/queries/incremental_sum/README.md +++ b/web/api/queries/incremental_sum/README.md @@ -1,6 +1,10 @@ <!-- title: "Incremental Sum (`incremental_sum`)" +sidebar_label: "Incremental Sum (`incremental_sum`)" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/incremental_sum/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Incremental Sum (`incremental_sum`) diff --git a/web/api/queries/incremental_sum/incremental_sum.c b/web/api/queries/incremental_sum/incremental_sum.c index afca530c..88072f29 100644 --- a/web/api/queries/incremental_sum/incremental_sum.c +++ b/web/api/queries/incremental_sum/incremental_sum.c @@ -5,62 +5,3 @@ // ---------------------------------------------------------------------------- // incremental sum -struct grouping_incremental_sum { - NETDATA_DOUBLE first; - NETDATA_DOUBLE last; - size_t count; -}; - -void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused) { - r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_incremental_sum)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_incremental_sum(RRDR *r) { - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; - g->first = 0; - g->last = 0; - g->count = 0; -} - -void grouping_free_incremental_sum(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; - - if(unlikely(!g->count)) { - g->first = value; - g->count++; - } - else { - g->last = value; - g->count++; - } -} - -NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_incremental_sum *g = (struct grouping_incremental_sum *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else if(unlikely(g->count == 1)) { - value = 0.0; - } - else { - value = g->last - g->first; - } - - g->first = 0.0; - g->last = 0.0; - g->count = 0; - - return value; -} diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h index c24507fc..dd6483b2 100644 --- a/web/api/queries/incremental_sum/incremental_sum.h +++ b/web/api/queries/incremental_sum/incremental_sum.h @@ -6,10 +6,64 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused); -void grouping_reset_incremental_sum(RRDR *r); -void grouping_free_incremental_sum(RRDR *r); -void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_incremental_sum { + NETDATA_DOUBLE first; + NETDATA_DOUBLE last; + size_t count; +}; + +static inline void tg_incremental_sum_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_incremental_sum)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_incremental_sum_reset(RRDR *r) { + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; + g->first = 0; + g->last = 0; + g->count = 0; +} + +static inline void tg_incremental_sum_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_incremental_sum_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; + + if(unlikely(!g->count)) { + g->first = value; + g->count++; + } + else { + g->last = value; + g->count++; + } +} + +static inline NETDATA_DOUBLE tg_incremental_sum_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(unlikely(g->count == 1)) { + value = 0.0; + } + else { + value = g->last - g->first; + } + + g->first = 0.0; + g->last = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H diff --git a/web/api/queries/max/README.md b/web/api/queries/max/README.md index 48da7cf0..82749c4a 100644 --- a/web/api/queries/max/README.md +++ b/web/api/queries/max/README.md @@ -1,6 +1,10 @@ <!-- title: "Max" +sidebar_label: "Max" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/max/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Max diff --git a/web/api/queries/max/max.c b/web/api/queries/max/max.c index 73cf9fa6..cc5999a2 100644 --- a/web/api/queries/max/max.c +++ b/web/api/queries/max/max.c @@ -5,53 +5,3 @@ // ---------------------------------------------------------------------------- // max -struct grouping_max { - NETDATA_DOUBLE max; - size_t count; -}; - -void grouping_create_max(RRDR *r, const char *options __maybe_unused) { - r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_max)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_max(RRDR *r) { - struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; - g->max = 0; - g->count = 0; -} - -void grouping_free_max(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_max(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; - - if(!g->count || fabsndd(value) > fabsndd(g->max)) { - g->max = value; - g->count++; - } -} - -NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_max *g = (struct grouping_max *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - value = g->max; - } - - g->max = 0.0; - g->count = 0; - - return value; -} - diff --git a/web/api/queries/max/max.h b/web/api/queries/max/max.h index e2427d26..c26bb79a 100644 --- a/web/api/queries/max/max.h +++ b/web/api/queries/max/max.h @@ -6,10 +6,54 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_max(RRDR *r, const char *options __maybe_unused); -void grouping_reset_max(RRDR *r); -void grouping_free_max(RRDR *r); -void grouping_add_max(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_max { + NETDATA_DOUBLE max; + size_t count; +}; + +static inline void tg_max_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_max)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_max_reset(RRDR *r) { + struct tg_max *g = (struct tg_max *)r->time_grouping.data; + g->max = 0; + g->count = 0; +} + +static inline void tg_max_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_max_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_max *g = (struct tg_max *)r->time_grouping.data; + + if(!g->count || fabsndd(value) > fabsndd(g->max)) { + g->max = value; + g->count++; + } +} + +static inline NETDATA_DOUBLE tg_max_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_max *g = (struct tg_max *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = g->max; + } + + g->max = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_MAX_H diff --git a/web/api/queries/median/README.md b/web/api/queries/median/README.md index 5600284c..15549b3b 100644 --- a/web/api/queries/median/README.md +++ b/web/api/queries/median/README.md @@ -1,7 +1,11 @@ <!-- title: "Median" +sidebar_label: "Median" description: "Use median in API queries and health entities to find the 'middle' value from a sample, eliminating any unwanted spikes in the returned metrics." custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/median/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Median diff --git a/web/api/queries/median/median.c b/web/api/queries/median/median.c index 40fd4ec3..9865b485 100644 --- a/web/api/queries/median/median.c +++ b/web/api/queries/median/median.c @@ -4,137 +4,3 @@ // ---------------------------------------------------------------------------- // median - -struct grouping_median { - size_t series_size; - size_t next_pos; - NETDATA_DOUBLE percent; - - NETDATA_DOUBLE *series; -}; - -void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { - long entries = r->group; - if(entries < 10) entries = 10; - - struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median)); - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); - g->series_size = (size_t)entries; - - g->percent = def; - if(options && *options) { - g->percent = str2ndd(options, NULL); - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; - if(g->percent < 0.0) g->percent = 0.0; - if(g->percent > 50.0) g->percent = 50.0; - } - - g->percent = g->percent / 100.0; - r->internal.grouping_data = g; -} - -void grouping_create_median(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 0.0); -} -void grouping_create_trimmed_median1(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 1.0); -} -void grouping_create_trimmed_median2(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 2.0); -} -void grouping_create_trimmed_median3(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 3.0); -} -void grouping_create_trimmed_median5(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 5.0); -} -void grouping_create_trimmed_median10(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 10.0); -} -void grouping_create_trimmed_median15(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 15.0); -} -void grouping_create_trimmed_median20(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 20.0); -} -void grouping_create_trimmed_median25(RRDR *r, const char *options) { - grouping_create_median_internal(r, options, 25.0); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_median(RRDR *r) { - struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; - g->next_pos = 0; -} - -void grouping_free_median(RRDR *r) { - struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; - if(g) onewayalloc_freez(r->internal.owa, g->series); - - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; - - if(unlikely(g->next_pos >= g->series_size)) { - g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); - g->series_size *= 2; - } - - g->series[g->next_pos++] = value; -} - -NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; - - size_t available_slots = g->next_pos; - NETDATA_DOUBLE value; - - if(unlikely(!available_slots)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else if(available_slots == 1) { - value = g->series[0]; - } - else { - sort_series(g->series, available_slots); - - size_t start_slot = 0; - size_t end_slot = available_slots - 1; - - if(g->percent > 0.0) { - NETDATA_DOUBLE min = g->series[0]; - NETDATA_DOUBLE max = g->series[available_slots - 1]; - NETDATA_DOUBLE delta = (max - min) * g->percent; - - NETDATA_DOUBLE wanted_min = min + delta; - NETDATA_DOUBLE wanted_max = max - delta; - - for (start_slot = 0; start_slot < available_slots; start_slot++) - if (g->series[start_slot] >= wanted_min) break; - - for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--) - if (g->series[end_slot] <= wanted_max) break; - } - - if(start_slot == end_slot) - value = g->series[start_slot]; - else - value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1); - } - - if(unlikely(!netdata_double_isnumber(value))) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - - //log_series_to_stderr(g->series, g->next_pos, value, "median"); - - g->next_pos = 0; - - return value; -} diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h index 9fc159db..3d6d3592 100644 --- a/web/api/queries/median/median.h +++ b/web/api/queries/median/median.h @@ -6,18 +6,138 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_median(RRDR *r, const char *options); -void grouping_create_trimmed_median1(RRDR *r, const char *options); -void grouping_create_trimmed_median2(RRDR *r, const char *options); -void grouping_create_trimmed_median3(RRDR *r, const char *options); -void grouping_create_trimmed_median5(RRDR *r, const char *options); -void grouping_create_trimmed_median10(RRDR *r, const char *options); -void grouping_create_trimmed_median15(RRDR *r, const char *options); -void grouping_create_trimmed_median20(RRDR *r, const char *options); -void grouping_create_trimmed_median25(RRDR *r, const char *options); -void grouping_reset_median(RRDR *r); -void grouping_free_median(RRDR *r); -void grouping_add_median(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_median { + size_t series_size; + size_t next_pos; + NETDATA_DOUBLE percent; + + NETDATA_DOUBLE *series; +}; + +static inline void tg_median_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { + long entries = r->view.group; + if(entries < 10) entries = 10; + + struct tg_median *g = (struct tg_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_median)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); + g->series_size = (size_t)entries; + + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 50.0) g->percent = 50.0; + } + + g->percent = g->percent / 100.0; + r->time_grouping.data = g; +} + +static inline void tg_median_create(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 0.0); +} +static inline void tg_median_create_trimmed_1(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 1.0); +} +static inline void tg_median_create_trimmed_2(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 2.0); +} +static inline void tg_median_create_trimmed_3(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 3.0); +} +static inline void tg_median_create_trimmed_5(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 5.0); +} +static inline void tg_median_create_trimmed_10(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 10.0); +} +static inline void tg_median_create_trimmed_15(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 15.0); +} +static inline void tg_median_create_trimmed_20(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 20.0); +} +static inline void tg_median_create_trimmed_25(RRDR *r, const char *options) { + tg_median_create_internal(r, options, 25.0); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_median_reset(RRDR *r) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + g->next_pos = 0; +} + +static inline void tg_median_free(RRDR *r) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_median_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + + if(unlikely(g->next_pos >= g->series_size)) { + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; + } + + g->series[g->next_pos++] = value; +} + +static inline NETDATA_DOUBLE tg_median_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_median *g = (struct tg_median *)r->time_grouping.data; + + size_t available_slots = g->next_pos; + NETDATA_DOUBLE value; + + if(unlikely(!available_slots)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(available_slots == 1) { + value = g->series[0]; + } + else { + sort_series(g->series, available_slots); + + size_t start_slot = 0; + size_t end_slot = available_slots - 1; + + if(g->percent > 0.0) { + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + NETDATA_DOUBLE delta = (max - min) * g->percent; + + NETDATA_DOUBLE wanted_min = min + delta; + NETDATA_DOUBLE wanted_max = max - delta; + + for (start_slot = 0; start_slot < available_slots; start_slot++) + if (g->series[start_slot] >= wanted_min) break; + + for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--) + if (g->series[end_slot] <= wanted_max) break; + } + + if(start_slot == end_slot) + value = g->series[start_slot]; + else + value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1); + } + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "median"); + + g->next_pos = 0; + + return value; +} #endif //NETDATA_API_QUERIES_MEDIAN_H diff --git a/web/api/queries/min/README.md b/web/api/queries/min/README.md index 495523c0..cf63aaa0 100644 --- a/web/api/queries/min/README.md +++ b/web/api/queries/min/README.md @@ -1,6 +1,10 @@ <!-- title: "Min" +sidebar_label: "Min" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/min/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Min diff --git a/web/api/queries/min/min.c b/web/api/queries/min/min.c index 1752e9e0..cefa7cf3 100644 --- a/web/api/queries/min/min.c +++ b/web/api/queries/min/min.c @@ -5,53 +5,3 @@ // ---------------------------------------------------------------------------- // min -struct grouping_min { - NETDATA_DOUBLE min; - size_t count; -}; - -void grouping_create_min(RRDR *r, const char *options __maybe_unused) { - r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_min)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_min(RRDR *r) { - struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data; - g->min = 0; - g->count = 0; -} - -void grouping_free_min(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_min(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data; - - if(!g->count || fabsndd(value) < fabsndd(g->min)) { - g->min = value; - g->count++; - } -} - -NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_min *g = (struct grouping_min *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - value = g->min; - } - - g->min = 0.0; - g->count = 0; - - return value; -} - diff --git a/web/api/queries/min/min.h b/web/api/queries/min/min.h index dcdfe252..3c53dfd1 100644 --- a/web/api/queries/min/min.h +++ b/web/api/queries/min/min.h @@ -6,10 +6,54 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_min(RRDR *r, const char *options __maybe_unused); -void grouping_reset_min(RRDR *r); -void grouping_free_min(RRDR *r); -void grouping_add_min(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_min { + NETDATA_DOUBLE min; + size_t count; +}; + +static inline void tg_min_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_min)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_min_reset(RRDR *r) { + struct tg_min *g = (struct tg_min *)r->time_grouping.data; + g->min = 0; + g->count = 0; +} + +static inline void tg_min_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_min_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_min *g = (struct tg_min *)r->time_grouping.data; + + if(!g->count || fabsndd(value) < fabsndd(g->min)) { + g->min = value; + g->count++; + } +} + +static inline NETDATA_DOUBLE tg_min_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_min *g = (struct tg_min *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = g->min; + } + + g->min = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_MIN_H diff --git a/web/api/queries/percentile/README.md b/web/api/queries/percentile/README.md index 70afc742..19ec81ed 100644 --- a/web/api/queries/percentile/README.md +++ b/web/api/queries/percentile/README.md @@ -1,7 +1,11 @@ <!-- title: "Percentile" +sidebar_label: "Percentile" description: "Use percentile in API queries and health entities to find the 'percentile' value from a sample, eliminating any unwanted spikes in the returned metrics." custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/percentile/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Percentile diff --git a/web/api/queries/percentile/percentile.c b/web/api/queries/percentile/percentile.c index 88f8600d..da3b3269 100644 --- a/web/api/queries/percentile/percentile.c +++ b/web/api/queries/percentile/percentile.c @@ -4,166 +4,3 @@ // ---------------------------------------------------------------------------- // median - -struct grouping_percentile { - size_t series_size; - size_t next_pos; - NETDATA_DOUBLE percent; - - NETDATA_DOUBLE *series; -}; - -static void grouping_create_percentile_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { - long entries = r->group; - if(entries < 10) entries = 10; - - struct grouping_percentile *g = (struct grouping_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_percentile)); - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); - g->series_size = (size_t)entries; - - g->percent = def; - if(options && *options) { - g->percent = str2ndd(options, NULL); - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; - if(g->percent < 0.0) g->percent = 0.0; - if(g->percent > 100.0) g->percent = 100.0; - } - - g->percent = g->percent / 100.0; - r->internal.grouping_data = g; -} - -void grouping_create_percentile25(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 25.0); -} -void grouping_create_percentile50(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 50.0); -} -void grouping_create_percentile75(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 75.0); -} -void grouping_create_percentile80(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 80.0); -} -void grouping_create_percentile90(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 90.0); -} -void grouping_create_percentile95(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 95.0); -} -void grouping_create_percentile97(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 97.0); -} -void grouping_create_percentile98(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 98.0); -} -void grouping_create_percentile99(RRDR *r, const char *options) { - grouping_create_percentile_internal(r, options, 99.0); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_percentile(RRDR *r) { - struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; - g->next_pos = 0; -} - -void grouping_free_percentile(RRDR *r) { - struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; - if(g) onewayalloc_freez(r->internal.owa, g->series); - - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; - - if(unlikely(g->next_pos >= g->series_size)) { - g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); - g->series_size *= 2; - } - - g->series[g->next_pos++] = value; -} - -NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_percentile *g = (struct grouping_percentile *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - size_t available_slots = g->next_pos; - - if(unlikely(!available_slots)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else if(available_slots == 1) { - value = g->series[0]; - } - else { - sort_series(g->series, available_slots); - - NETDATA_DOUBLE min = g->series[0]; - NETDATA_DOUBLE max = g->series[available_slots - 1]; - - if (min != max) { - size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent); - if(!slots_to_use) slots_to_use = 1; - - NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots; - NETDATA_DOUBLE percent_delta = g->percent - percent_to_use; - - NETDATA_DOUBLE percent_interpolation_slot = 0.0; - NETDATA_DOUBLE percent_last_slot = 0.0; - if(percent_delta > 0.0) { - NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots; - NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use; - - percent_interpolation_slot = percent_delta / percent_1slot; - percent_last_slot = 1 - percent_interpolation_slot; - } - - int start_slot, stop_slot, step, last_slot, interpolation_slot; - if(min >= 0.0 && max >= 0.0) { - start_slot = 0; - stop_slot = start_slot + (int)slots_to_use; - last_slot = stop_slot - 1; - interpolation_slot = stop_slot; - step = 1; - } - else { - start_slot = (int)available_slots - 1; - stop_slot = start_slot - (int)slots_to_use; - last_slot = stop_slot + 1; - interpolation_slot = stop_slot; - step = -1; - } - - value = 0.0; - for(int slot = start_slot; slot != stop_slot ; slot += step) - value += g->series[slot]; - - size_t counted = slots_to_use; - if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) { - value += g->series[interpolation_slot] * percent_interpolation_slot; - value += g->series[last_slot] * percent_last_slot; - counted++; - } - - value = value / (NETDATA_DOUBLE)counted; - } - else - value = min; - } - - if(unlikely(!netdata_double_isnumber(value))) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - - //log_series_to_stderr(g->series, g->next_pos, value, "percentile"); - - g->next_pos = 0; - - return value; -} diff --git a/web/api/queries/percentile/percentile.h b/web/api/queries/percentile/percentile.h index 65e335c1..0532f9d3 100644 --- a/web/api/queries/percentile/percentile.h +++ b/web/api/queries/percentile/percentile.h @@ -6,18 +6,167 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_percentile25(RRDR *r, const char *options); -void grouping_create_percentile50(RRDR *r, const char *options); -void grouping_create_percentile75(RRDR *r, const char *options); -void grouping_create_percentile80(RRDR *r, const char *options); -void grouping_create_percentile90(RRDR *r, const char *options); -void grouping_create_percentile95(RRDR *r, const char *options); -void grouping_create_percentile97(RRDR *r, const char *options); -void grouping_create_percentile98(RRDR *r, const char *options); -void grouping_create_percentile99(RRDR *r, const char *options ); -void grouping_reset_percentile(RRDR *r); -void grouping_free_percentile(RRDR *r); -void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_percentile { + size_t series_size; + size_t next_pos; + NETDATA_DOUBLE percent; + + NETDATA_DOUBLE *series; +}; + +static inline void tg_percentile_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { + long entries = r->view.group; + if(entries < 10) entries = 10; + + struct tg_percentile *g = (struct tg_percentile *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_percentile)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); + g->series_size = (size_t)entries; + + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 100.0) g->percent = 100.0; + } + + g->percent = g->percent / 100.0; + r->time_grouping.data = g; +} + +static inline void tg_percentile_create_25(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 25.0); +} +static inline void tg_percentile_create_50(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 50.0); +} +static inline void tg_percentile_create_75(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 75.0); +} +static inline void tg_percentile_create_80(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 80.0); +} +static inline void tg_percentile_create_90(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 90.0); +} +static inline void tg_percentile_create_95(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 95.0); +} +static inline void tg_percentile_create_97(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 97.0); +} +static inline void tg_percentile_create_98(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 98.0); +} +static inline void tg_percentile_create_99(RRDR *r, const char *options) { + tg_percentile_create_internal(r, options, 99.0); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_percentile_reset(RRDR *r) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + g->next_pos = 0; +} + +static inline void tg_percentile_free(RRDR *r) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_percentile_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + + if(unlikely(g->next_pos >= g->series_size)) { + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; + } + + g->series[g->next_pos++] = value; +} + +static inline NETDATA_DOUBLE tg_percentile_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_percentile *g = (struct tg_percentile *)r->time_grouping.data; + + NETDATA_DOUBLE value; + size_t available_slots = g->next_pos; + + if(unlikely(!available_slots)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(available_slots == 1) { + value = g->series[0]; + } + else { + sort_series(g->series, available_slots); + + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + + if (min != max) { + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent); + if(!slots_to_use) slots_to_use = 1; + + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use; + + NETDATA_DOUBLE percent_interpolation_slot = 0.0; + NETDATA_DOUBLE percent_last_slot = 0.0; + if(percent_delta > 0.0) { + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use; + + percent_interpolation_slot = percent_delta / percent_1slot; + percent_last_slot = 1 - percent_interpolation_slot; + } + + int start_slot, stop_slot, step, last_slot, interpolation_slot; + if(min >= 0.0 && max >= 0.0) { + start_slot = 0; + stop_slot = start_slot + (int)slots_to_use; + last_slot = stop_slot - 1; + interpolation_slot = stop_slot; + step = 1; + } + else { + start_slot = (int)available_slots - 1; + stop_slot = start_slot - (int)slots_to_use; + last_slot = stop_slot + 1; + interpolation_slot = stop_slot; + step = -1; + } + + value = 0.0; + for(int slot = start_slot; slot != stop_slot ; slot += step) + value += g->series[slot]; + + size_t counted = slots_to_use; + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) { + value += g->series[interpolation_slot] * percent_interpolation_slot; + value += g->series[last_slot] * percent_last_slot; + counted++; + } + + value = value / (NETDATA_DOUBLE)counted; + } + else + value = min; + } + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "percentile"); + + g->next_pos = 0; + + return value; +} #endif //NETDATA_API_QUERIES_PERCENTILE_H diff --git a/web/api/queries/query.c b/web/api/queries/query.c index df7e0979..3770d477 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -24,7 +24,8 @@ static struct { const char *name; uint32_t hash; - RRDR_GROUPING value; + RRDR_TIME_GROUPING value; + RRDR_TIME_GROUPING add_flush; // One time initialization for the module. // This is called once, when netdata starts. @@ -59,397 +60,445 @@ static struct { {.name = "average", .hash = 0, .value = RRDR_GROUPING_AVERAGE, + .add_flush = RRDR_GROUPING_AVERAGE, .init = NULL, - .create= grouping_create_average, - .reset = grouping_reset_average, - .free = grouping_free_average, - .add = grouping_add_average, - .flush = grouping_flush_average, + .create= tg_average_create, + .reset = tg_average_reset, + .free = tg_average_free, + .add = tg_average_add, + .flush = tg_average_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, - {.name = "mean", // alias on 'average' + {.name = "avg", // alias on 'average' .hash = 0, .value = RRDR_GROUPING_AVERAGE, + .add_flush = RRDR_GROUPING_AVERAGE, .init = NULL, - .create= grouping_create_average, - .reset = grouping_reset_average, - .free = grouping_free_average, - .add = grouping_add_average, - .flush = grouping_flush_average, + .create= tg_average_create, + .reset = tg_average_reset, + .free = tg_average_free, + .add = tg_average_add, + .flush = tg_average_flush, + .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE + }, + {.name = "mean", // alias on 'average' + .hash = 0, + .value = RRDR_GROUPING_AVERAGE, + .add_flush = RRDR_GROUPING_AVERAGE, + .init = NULL, + .create= tg_average_create, + .reset = tg_average_reset, + .free = tg_average_free, + .add = tg_average_add, + .flush = tg_average_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean1", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEAN1, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean1, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_1, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean2", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEAN2, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean2, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_2, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean3", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEAN3, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean3, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_3, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean5", .hash = 0, - .value = RRDR_GROUPING_TRIMMED_MEAN5, + .value = RRDR_GROUPING_TRIMMED_MEAN, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean5, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_5, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean10", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEAN10, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean10, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_10, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean15", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEAN15, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean15, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_15, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean20", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEAN20, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean20, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_20, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean25", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEAN25, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean25, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_25, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-mean", .hash = 0, - .value = RRDR_GROUPING_TRIMMED_MEAN5, + .value = RRDR_GROUPING_TRIMMED_MEAN, + .add_flush = RRDR_GROUPING_TRIMMED_MEAN, .init = NULL, - .create= grouping_create_trimmed_mean5, - .reset = grouping_reset_trimmed_mean, - .free = grouping_free_trimmed_mean, - .add = grouping_add_trimmed_mean, - .flush = grouping_flush_trimmed_mean, + .create= tg_trimmed_mean_create_5, + .reset = tg_trimmed_mean_reset, + .free = tg_trimmed_mean_free, + .add = tg_trimmed_mean_add, + .flush = tg_trimmed_mean_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "incremental_sum", .hash = 0, .value = RRDR_GROUPING_INCREMENTAL_SUM, + .add_flush = RRDR_GROUPING_INCREMENTAL_SUM, .init = NULL, - .create= grouping_create_incremental_sum, - .reset = grouping_reset_incremental_sum, - .free = grouping_free_incremental_sum, - .add = grouping_add_incremental_sum, - .flush = grouping_flush_incremental_sum, + .create= tg_incremental_sum_create, + .reset = tg_incremental_sum_reset, + .free = tg_incremental_sum_free, + .add = tg_incremental_sum_add, + .flush = tg_incremental_sum_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "incremental-sum", .hash = 0, .value = RRDR_GROUPING_INCREMENTAL_SUM, + .add_flush = RRDR_GROUPING_INCREMENTAL_SUM, .init = NULL, - .create= grouping_create_incremental_sum, - .reset = grouping_reset_incremental_sum, - .free = grouping_free_incremental_sum, - .add = grouping_add_incremental_sum, - .flush = grouping_flush_incremental_sum, + .create= tg_incremental_sum_create, + .reset = tg_incremental_sum_reset, + .free = tg_incremental_sum_free, + .add = tg_incremental_sum_add, + .flush = tg_incremental_sum_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "median", .hash = 0, .value = RRDR_GROUPING_MEDIAN, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_median, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median1", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN1, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median1, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_1, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median2", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN2, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median2, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_2, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median3", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN3, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median3, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_3, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median5", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN5, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median5, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_5, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median10", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN10, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median10, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_10, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median15", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN15, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median15, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_15, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median20", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN20, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median20, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_20, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median25", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN25, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median25, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_25, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "trimmed-median", .hash = 0, .value = RRDR_GROUPING_TRIMMED_MEDIAN5, + .add_flush = RRDR_GROUPING_MEDIAN, .init = NULL, - .create= grouping_create_trimmed_median5, - .reset = grouping_reset_median, - .free = grouping_free_median, - .add = grouping_add_median, - .flush = grouping_flush_median, + .create= tg_median_create_trimmed_5, + .reset = tg_median_reset, + .free = tg_median_free, + .add = tg_median_add, + .flush = tg_median_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile25", .hash = 0, .value = RRDR_GROUPING_PERCENTILE25, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile25, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_25, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile50", .hash = 0, .value = RRDR_GROUPING_PERCENTILE50, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile50, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_50, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile75", .hash = 0, .value = RRDR_GROUPING_PERCENTILE75, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile75, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_75, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile80", .hash = 0, .value = RRDR_GROUPING_PERCENTILE80, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile80, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_80, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile90", .hash = 0, .value = RRDR_GROUPING_PERCENTILE90, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile90, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_90, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile95", .hash = 0, - .value = RRDR_GROUPING_PERCENTILE95, + .value = RRDR_GROUPING_PERCENTILE, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile95, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_95, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile97", .hash = 0, .value = RRDR_GROUPING_PERCENTILE97, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile97, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_97, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile98", .hash = 0, .value = RRDR_GROUPING_PERCENTILE98, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile98, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_98, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile99", .hash = 0, .value = RRDR_GROUPING_PERCENTILE99, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile99, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_99, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "percentile", .hash = 0, - .value = RRDR_GROUPING_PERCENTILE95, + .value = RRDR_GROUPING_PERCENTILE, + .add_flush = RRDR_GROUPING_PERCENTILE, .init = NULL, - .create= grouping_create_percentile95, - .reset = grouping_reset_percentile, - .free = grouping_free_percentile, - .add = grouping_add_percentile, - .flush = grouping_flush_percentile, + .create= tg_percentile_create_95, + .reset = tg_percentile_reset, + .free = tg_percentile_free, + .add = tg_percentile_add, + .flush = tg_percentile_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "min", .hash = 0, .value = RRDR_GROUPING_MIN, + .add_flush = RRDR_GROUPING_MIN, .init = NULL, - .create= grouping_create_min, - .reset = grouping_reset_min, - .free = grouping_free_min, - .add = grouping_add_min, - .flush = grouping_flush_min, + .create= tg_min_create, + .reset = tg_min_reset, + .free = tg_min_free, + .add = tg_min_add, + .flush = tg_min_flush, .tier_query_fetch = TIER_QUERY_FETCH_MIN }, {.name = "max", .hash = 0, .value = RRDR_GROUPING_MAX, + .add_flush = RRDR_GROUPING_MAX, .init = NULL, - .create= grouping_create_max, - .reset = grouping_reset_max, - .free = grouping_free_max, - .add = grouping_add_max, - .flush = grouping_flush_max, + .create= tg_max_create, + .reset = tg_max_reset, + .free = tg_max_free, + .add = tg_max_add, + .flush = tg_max_flush, .tier_query_fetch = TIER_QUERY_FETCH_MAX }, {.name = "sum", .hash = 0, .value = RRDR_GROUPING_SUM, + .add_flush = RRDR_GROUPING_SUM, .init = NULL, - .create= grouping_create_sum, - .reset = grouping_reset_sum, - .free = grouping_free_sum, - .add = grouping_add_sum, - .flush = grouping_flush_sum, + .create= tg_sum_create, + .reset = tg_sum_reset, + .free = tg_sum_free, + .add = tg_sum_add, + .flush = tg_sum_flush, .tier_query_fetch = TIER_QUERY_FETCH_SUM }, @@ -457,97 +506,75 @@ static struct { {.name = "stddev", .hash = 0, .value = RRDR_GROUPING_STDDEV, + .add_flush = RRDR_GROUPING_STDDEV, .init = NULL, - .create= grouping_create_stddev, - .reset = grouping_reset_stddev, - .free = grouping_free_stddev, - .add = grouping_add_stddev, - .flush = grouping_flush_stddev, + .create= tg_stddev_create, + .reset = tg_stddev_reset, + .free = tg_stddev_free, + .add = tg_stddev_add, + .flush = tg_stddev_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "cv", // coefficient variation is calculated by stddev .hash = 0, .value = RRDR_GROUPING_CV, + .add_flush = RRDR_GROUPING_CV, .init = NULL, - .create= grouping_create_stddev, // not an error, stddev calculates this too - .reset = grouping_reset_stddev, // not an error, stddev calculates this too - .free = grouping_free_stddev, // not an error, stddev calculates this too - .add = grouping_add_stddev, // not an error, stddev calculates this too - .flush = grouping_flush_coefficient_of_variation, + .create= tg_stddev_create, // not an error, stddev calculates this too + .reset = tg_stddev_reset, // not an error, stddev calculates this too + .free = tg_stddev_free, // not an error, stddev calculates this too + .add = tg_stddev_add, // not an error, stddev calculates this too + .flush = tg_stddev_coefficient_of_variation_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "rsd", // alias of 'cv' .hash = 0, .value = RRDR_GROUPING_CV, + .add_flush = RRDR_GROUPING_CV, .init = NULL, - .create= grouping_create_stddev, // not an error, stddev calculates this too - .reset = grouping_reset_stddev, // not an error, stddev calculates this too - .free = grouping_free_stddev, // not an error, stddev calculates this too - .add = grouping_add_stddev, // not an error, stddev calculates this too - .flush = grouping_flush_coefficient_of_variation, - .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE - }, - - /* - {.name = "mean", // same as average, no need to define it again - .hash = 0, - .value = RRDR_GROUPING_MEAN, - .setup = NULL, - .create= grouping_create_stddev, - .reset = grouping_reset_stddev, - .free = grouping_free_stddev, - .add = grouping_add_stddev, - .flush = grouping_flush_mean, + .create= tg_stddev_create, // not an error, stddev calculates this too + .reset = tg_stddev_reset, // not an error, stddev calculates this too + .free = tg_stddev_free, // not an error, stddev calculates this too + .add = tg_stddev_add, // not an error, stddev calculates this too + .flush = tg_stddev_coefficient_of_variation_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, - */ - - /* - {.name = "variance", // meaningless to offer - .hash = 0, - .value = RRDR_GROUPING_VARIANCE, - .setup = NULL, - .create= grouping_create_stddev, - .reset = grouping_reset_stddev, - .free = grouping_free_stddev, - .add = grouping_add_stddev, - .flush = grouping_flush_variance, - .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE - }, - */ // single exponential smoothing {.name = "ses", .hash = 0, .value = RRDR_GROUPING_SES, - .init = grouping_init_ses, - .create= grouping_create_ses, - .reset = grouping_reset_ses, - .free = grouping_free_ses, - .add = grouping_add_ses, - .flush = grouping_flush_ses, + .add_flush = RRDR_GROUPING_SES, + .init = tg_ses_init, + .create= tg_ses_create, + .reset = tg_ses_reset, + .free = tg_ses_free, + .add = tg_ses_add, + .flush = tg_ses_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "ema", // alias for 'ses' .hash = 0, .value = RRDR_GROUPING_SES, + .add_flush = RRDR_GROUPING_SES, .init = NULL, - .create= grouping_create_ses, - .reset = grouping_reset_ses, - .free = grouping_free_ses, - .add = grouping_add_ses, - .flush = grouping_flush_ses, + .create= tg_ses_create, + .reset = tg_ses_reset, + .free = tg_ses_free, + .add = tg_ses_add, + .flush = tg_ses_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "ewma", // alias for ses .hash = 0, .value = RRDR_GROUPING_SES, + .add_flush = RRDR_GROUPING_SES, .init = NULL, - .create= grouping_create_ses, - .reset = grouping_reset_ses, - .free = grouping_free_ses, - .add = grouping_add_ses, - .flush = grouping_flush_ses, + .create= tg_ses_create, + .reset = tg_ses_reset, + .free = tg_ses_free, + .add = tg_ses_add, + .flush = tg_ses_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, @@ -555,24 +582,26 @@ static struct { {.name = "des", .hash = 0, .value = RRDR_GROUPING_DES, - .init = grouping_init_des, - .create= grouping_create_des, - .reset = grouping_reset_des, - .free = grouping_free_des, - .add = grouping_add_des, - .flush = grouping_flush_des, + .add_flush = RRDR_GROUPING_DES, + .init = tg_des_init, + .create= tg_des_create, + .reset = tg_des_reset, + .free = tg_des_free, + .add = tg_des_add, + .flush = tg_des_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, {.name = "countif", .hash = 0, .value = RRDR_GROUPING_COUNTIF, + .add_flush = RRDR_GROUPING_COUNTIF, .init = NULL, - .create= grouping_create_countif, - .reset = grouping_reset_countif, - .free = grouping_free_countif, - .add = grouping_add_countif, - .flush = grouping_flush_countif, + .create= tg_countif_create, + .reset = tg_countif_reset, + .free = tg_countif_free, + .add = tg_countif_add, + .flush = tg_countif_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE }, @@ -580,17 +609,18 @@ static struct { {.name = NULL, .hash = 0, .value = RRDR_GROUPING_UNDEFINED, + .add_flush = RRDR_GROUPING_AVERAGE, .init = NULL, - .create= grouping_create_average, - .reset = grouping_reset_average, - .free = grouping_free_average, - .add = grouping_add_average, - .flush = grouping_flush_average, + .create= tg_average_create, + .reset = tg_average_reset, + .free = tg_average_free, + .add = tg_average_add, + .flush = tg_average_flush, .tier_query_fetch = TIER_QUERY_FETCH_AVERAGE } }; -void web_client_api_v1_init_grouping(void) { +void time_grouping_init(void) { int i; for(i = 0; api_v1_data_groups[i].name ; i++) { @@ -601,7 +631,7 @@ void web_client_api_v1_init_grouping(void) { } } -const char *group_method2string(RRDR_GROUPING group) { +const char *time_grouping_method2string(RRDR_TIME_GROUPING group) { int i; for(i = 0; api_v1_data_groups[i].name ; i++) { @@ -613,7 +643,7 @@ const char *group_method2string(RRDR_GROUPING group) { return "unknown-group-method"; } -RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def) { +RRDR_TIME_GROUPING time_grouping_parse(const char *name, RRDR_TIME_GROUPING def) { int i; uint32_t hash = simple_hash(name); @@ -624,7 +654,7 @@ RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPI return def; } -const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group) { +const char *time_grouping_tostring(RRDR_TIME_GROUPING group) { int i; for(i = 0; api_v1_data_groups[i].name ; i++) @@ -634,28 +664,242 @@ const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group) return "unknown"; } -static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) { +static void rrdr_set_grouping_function(RRDR *r, RRDR_TIME_GROUPING group_method) { int i, found = 0; for(i = 0; !found && api_v1_data_groups[i].name ;i++) { if(api_v1_data_groups[i].value == group_method) { - r->internal.grouping_create = api_v1_data_groups[i].create; - r->internal.grouping_reset = api_v1_data_groups[i].reset; - r->internal.grouping_free = api_v1_data_groups[i].free; - r->internal.grouping_add = api_v1_data_groups[i].add; - r->internal.grouping_flush = api_v1_data_groups[i].flush; - r->internal.tier_query_fetch = api_v1_data_groups[i].tier_query_fetch; + r->time_grouping.create = api_v1_data_groups[i].create; + r->time_grouping.reset = api_v1_data_groups[i].reset; + r->time_grouping.free = api_v1_data_groups[i].free; + r->time_grouping.add = api_v1_data_groups[i].add; + r->time_grouping.flush = api_v1_data_groups[i].flush; + r->time_grouping.tier_query_fetch = api_v1_data_groups[i].tier_query_fetch; + r->time_grouping.add_flush = api_v1_data_groups[i].add_flush; found = 1; } } if(!found) { errno = 0; internal_error(true, "QUERY: grouping method %u not found. Using 'average'", (unsigned int)group_method); - r->internal.grouping_create = grouping_create_average; - r->internal.grouping_reset = grouping_reset_average; - r->internal.grouping_free = grouping_free_average; - r->internal.grouping_add = grouping_add_average; - r->internal.grouping_flush = grouping_flush_average; - r->internal.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE; + r->time_grouping.create = tg_average_create; + r->time_grouping.reset = tg_average_reset; + r->time_grouping.free = tg_average_free; + r->time_grouping.add = tg_average_add; + r->time_grouping.flush = tg_average_flush; + r->time_grouping.tier_query_fetch = TIER_QUERY_FETCH_AVERAGE; + r->time_grouping.add_flush = RRDR_GROUPING_AVERAGE; + } +} + +static inline void time_grouping_add(RRDR *r, NETDATA_DOUBLE value, const RRDR_TIME_GROUPING add_flush) { + switch(add_flush) { + case RRDR_GROUPING_AVERAGE: + tg_average_add(r, value); + break; + + case RRDR_GROUPING_MAX: + tg_max_add(r, value); + break; + + case RRDR_GROUPING_MIN: + tg_min_add(r, value); + break; + + case RRDR_GROUPING_MEDIAN: + tg_median_add(r, value); + break; + + case RRDR_GROUPING_STDDEV: + case RRDR_GROUPING_CV: + tg_stddev_add(r, value); + break; + + case RRDR_GROUPING_SUM: + tg_sum_add(r, value); + break; + + case RRDR_GROUPING_COUNTIF: + tg_countif_add(r, value); + break; + + case RRDR_GROUPING_TRIMMED_MEAN: + tg_trimmed_mean_add(r, value); + break; + + case RRDR_GROUPING_PERCENTILE: + tg_percentile_add(r, value); + break; + + case RRDR_GROUPING_SES: + tg_ses_add(r, value); + break; + + case RRDR_GROUPING_DES: + tg_des_add(r, value); + break; + + case RRDR_GROUPING_INCREMENTAL_SUM: + tg_incremental_sum_add(r, value); + break; + + default: + r->time_grouping.add(r, value); + break; + } +} + +static inline NETDATA_DOUBLE time_grouping_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr, const RRDR_TIME_GROUPING add_flush) { + switch(add_flush) { + case RRDR_GROUPING_AVERAGE: + return tg_average_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_MAX: + return tg_max_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_MIN: + return tg_min_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_MEDIAN: + return tg_median_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_STDDEV: + return tg_stddev_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_CV: + return tg_stddev_coefficient_of_variation_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_SUM: + return tg_sum_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_COUNTIF: + return tg_countif_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_TRIMMED_MEAN: + return tg_trimmed_mean_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_PERCENTILE: + return tg_percentile_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_SES: + return tg_ses_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_DES: + return tg_des_flush(r, rrdr_value_options_ptr); + + case RRDR_GROUPING_INCREMENTAL_SUM: + return tg_incremental_sum_flush(r, rrdr_value_options_ptr); + + default: + return r->time_grouping.flush(r, rrdr_value_options_ptr); + } +} + +RRDR_GROUP_BY group_by_parse(char *s) { + RRDR_GROUP_BY group_by = RRDR_GROUP_BY_NONE; + + while(s) { + char *key = strsep_skip_consecutive_separators(&s, ",| "); + if (!key || !*key) continue; + + if (strcmp(key, "selected") == 0) + group_by |= RRDR_GROUP_BY_SELECTED; + + if (strcmp(key, "dimension") == 0) + group_by |= RRDR_GROUP_BY_DIMENSION; + + if (strcmp(key, "instance") == 0) + group_by |= RRDR_GROUP_BY_INSTANCE; + + if (strcmp(key, "percentage-of-instance") == 0) + group_by |= RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE; + + if (strcmp(key, "label") == 0) + group_by |= RRDR_GROUP_BY_LABEL; + + if (strcmp(key, "node") == 0) + group_by |= RRDR_GROUP_BY_NODE; + + if (strcmp(key, "context") == 0) + group_by |= RRDR_GROUP_BY_CONTEXT; + + if (strcmp(key, "units") == 0) + group_by |= RRDR_GROUP_BY_UNITS; + } + + if((group_by & RRDR_GROUP_BY_SELECTED) && (group_by & ~RRDR_GROUP_BY_SELECTED)) { + internal_error(true, "group-by given by query has 'selected' together with more groupings"); + group_by = RRDR_GROUP_BY_SELECTED; // remove all other groupings + } + + if(group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + group_by = RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE; // remove all other groupings + + return group_by; +} + +void buffer_json_group_by_to_array(BUFFER *wb, RRDR_GROUP_BY group_by) { + if(group_by == RRDR_GROUP_BY_NONE) + buffer_json_add_array_item_string(wb, "none"); + else { + if (group_by & RRDR_GROUP_BY_DIMENSION) + buffer_json_add_array_item_string(wb, "dimension"); + + if (group_by & RRDR_GROUP_BY_INSTANCE) + buffer_json_add_array_item_string(wb, "instance"); + + if (group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + buffer_json_add_array_item_string(wb, "percentage-of-instance"); + + if (group_by & RRDR_GROUP_BY_LABEL) + buffer_json_add_array_item_string(wb, "label"); + + if (group_by & RRDR_GROUP_BY_NODE) + buffer_json_add_array_item_string(wb, "node"); + + if (group_by & RRDR_GROUP_BY_CONTEXT) + buffer_json_add_array_item_string(wb, "context"); + + if (group_by & RRDR_GROUP_BY_UNITS) + buffer_json_add_array_item_string(wb, "units"); + + if (group_by & RRDR_GROUP_BY_SELECTED) + buffer_json_add_array_item_string(wb, "selected"); + } +} + +RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s) { + if(strcmp(s, "average") == 0) + return RRDR_GROUP_BY_FUNCTION_AVERAGE; + + if(strcmp(s, "avg") == 0) + return RRDR_GROUP_BY_FUNCTION_AVERAGE; + + if(strcmp(s, "min") == 0) + return RRDR_GROUP_BY_FUNCTION_MIN; + + if(strcmp(s, "max") == 0) + return RRDR_GROUP_BY_FUNCTION_MAX; + + if(strcmp(s, "sum") == 0) + return RRDR_GROUP_BY_FUNCTION_SUM; + + return RRDR_GROUP_BY_FUNCTION_AVERAGE; +} + +const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_by_function) { + switch(group_by_function) { + default: + case RRDR_GROUP_BY_FUNCTION_AVERAGE: + return "average"; + + case RRDR_GROUP_BY_FUNCTION_MIN: + return "min"; + + case RRDR_GROUP_BY_FUNCTION_MAX: + return "max"; + + case RRDR_GROUP_BY_FUNCTION_SUM: + return "sum"; } } @@ -670,28 +914,20 @@ static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rr return &r->v[ rrdr_line * r->d ]; } -static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) { +static inline long rrdr_line_init(RRDR *r __maybe_unused, time_t t __maybe_unused, long rrdr_line) { rrdr_line++; - internal_error(rrdr_line >= (long)r->n, + internal_fatal(rrdr_line >= (long)r->n, "QUERY: requested to step above RRDR size for query '%s'", r->internal.qt->id); - internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t, - "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of query '%s'", - (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->internal.qt->id); - - // save the time - r->t[rrdr_line] = t; + internal_fatal(r->t[rrdr_line] != t, + "QUERY: wrong timestamp at RRDR line %ld, expected %ld, got %ld, of query '%s'", + rrdr_line, r->t[rrdr_line], t, r->internal.qt->id); return rrdr_line; } -static inline void rrdr_done(RRDR *r, long rrdr_line) { - r->rows = rrdr_line + 1; -} - - // ---------------------------------------------------------------------------- // tier management @@ -822,7 +1058,7 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after // find the db time-range for this tier for all metrics for(size_t i = 0, used = qt->query.used; i < used ; i++) { - QUERY_METRIC *qm = &qt->query.array[i]; + QUERY_METRIC *qm = query_metric(qt, i); time_t first_time_s = qm->tiers[tier].db_first_time_s; time_t last_time_s = qm->tiers[tier].db_last_time_s; @@ -872,7 +1108,7 @@ static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, t // find the db minimum update every for this tier for all metrics time_t common_update_every_s = default_rrd_update_every; for(size_t i = 0, used = qt->query.used; i < used ; i++) { - QUERY_METRIC *qm = &qt->query.array[i]; + QUERY_METRIC *qm = query_metric(qt, i); time_t update_every_s = qm->tiers[best_tier].db_update_every_s; @@ -889,24 +1125,20 @@ static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, t // query ops typedef struct query_point { - time_t end_time; - time_t start_time; + STORAGE_POINT sp; NETDATA_DOUBLE value; - NETDATA_DOUBLE anomaly; - SN_FLAGS flags; + bool added; #ifdef NETDATA_INTERNAL_CHECKS size_t id; #endif } QUERY_POINT; QUERY_POINT QUERY_POINT_EMPTY = { - .end_time = 0, - .start_time = 0, - .value = NAN, - .anomaly = 0, - .flags = SN_FLAG_NONE, + .sp = STORAGE_POINT_UNSET, + .value = NAN, + .added = false, #ifdef NETDATA_INTERNAL_CHECKS - .id = 0, + .id = 0, #endif }; @@ -934,21 +1166,27 @@ typedef struct query_engine_ops { size_t tier; struct query_metric_tier *tier_ptr; struct storage_engine_query_handle *handle; - STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle); - int (*is_finished)(struct storage_engine_query_handle *handle); - void (*finalize)(struct storage_engine_query_handle *handle); // aggregating points over time - void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value); - NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); size_t group_points_non_zero; size_t group_points_added; - NETDATA_DOUBLE group_anomaly_rate; + STORAGE_POINT group_point; // aggregates min, max, sum, count, anomaly count for each group point + STORAGE_POINT query_point; // aggregates min, max, sum, count, anomaly count across the whole query RRDR_VALUE_FLAGS group_value_flags; // statistics size_t db_total_points_read; size_t db_points_read_per_tier[RRD_STORAGE_TIERS]; + + struct { + time_t expanded_after; + time_t expanded_before; + struct storage_engine_query_handle handle; + bool initialized; + bool finalized; + } plans[QUERY_PLANS_MAX]; + + struct query_engine_ops *next; } QUERY_ENGINE_OPS; @@ -1005,40 +1243,28 @@ static void query_planer_initialize_plans(QUERY_ENGINE_OPS *ops) { time_t after = qm->plan.array[p].after - (time_t)(update_every * points_to_add_to_after); time_t before = qm->plan.array[p].before + (time_t)(update_every * points_to_add_to_before); - qm->plan.array[p].expanded_after = after; - qm->plan.array[p].expanded_before = before; + ops->plans[p].expanded_after = after; + ops->plans[p].expanded_before = before; + + ops->r->internal.qt->db.tiers[tier].queries++; struct query_metric_tier *tier_ptr = &qm->tiers[tier]; - tier_ptr->eng->api.query_ops.init( - tier_ptr->db_metric_handle, - &qm->plan.array[p].handle, - after, before, - ops->r->internal.qt->request.priority); - - qm->plan.array[p].next_metric = tier_ptr->eng->api.query_ops.next_metric; - qm->plan.array[p].is_finished = tier_ptr->eng->api.query_ops.is_finished; - qm->plan.array[p].finalize = tier_ptr->eng->api.query_ops.finalize; - qm->plan.array[p].initialized = true; - qm->plan.array[p].finalized = false; + STORAGE_ENGINE *eng = query_metric_storage_engine(ops->r->internal.qt, qm, tier); + storage_engine_query_init(eng->backend, tier_ptr->db_metric_handle, &ops->plans[p].handle, + after, before, ops->r->internal.qt->request.priority); + + ops->plans[p].initialized = true; + ops->plans[p].finalized = false; } } static void query_planer_finalize_plan(QUERY_ENGINE_OPS *ops, size_t plan_id) { - QUERY_METRIC *qm = ops->qm; - - if(qm->plan.array[plan_id].initialized && !qm->plan.array[plan_id].finalized) { - qm->plan.array[plan_id].finalize(&qm->plan.array[plan_id].handle); - qm->plan.array[plan_id].initialized = false; - qm->plan.array[plan_id].finalized = true; - qm->plan.array[plan_id].next_metric = NULL; - qm->plan.array[plan_id].is_finished = NULL; - qm->plan.array[plan_id].finalize = NULL; + // QUERY_METRIC *qm = ops->qm; - if(ops->current_plan == plan_id) { - ops->next_metric = NULL; - ops->is_finished = NULL; - ops->finalize = NULL; - } + if(ops->plans[plan_id].initialized && !ops->plans[plan_id].finalized) { + storage_engine_query_finalize(&ops->plans[plan_id].handle); + ops->plans[plan_id].initialized = false; + ops->plans[plan_id].finalized = true; } } @@ -1053,17 +1279,14 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti QUERY_METRIC *qm = ops->qm; internal_fatal(plan_id >= qm->plan.used, "QUERY: invalid plan_id given"); - internal_fatal(!qm->plan.array[plan_id].initialized, "QUERY: plan has not been initialized"); - internal_fatal(qm->plan.array[plan_id].finalized, "QUERY: plan has been finalized"); + internal_fatal(!ops->plans[plan_id].initialized, "QUERY: plan has not been initialized"); + internal_fatal(ops->plans[plan_id].finalized, "QUERY: plan has been finalized"); internal_fatal(qm->plan.array[plan_id].after > qm->plan.array[plan_id].before, "QUERY: flipped after/before"); ops->tier = qm->plan.array[plan_id].tier; ops->tier_ptr = &qm->tiers[ops->tier]; - ops->handle = &qm->plan.array[plan_id].handle; - ops->next_metric = qm->plan.array[plan_id].next_metric; - ops->is_finished = qm->plan.array[plan_id].is_finished; - ops->finalize = qm->plan.array[plan_id].finalize; + ops->handle = &ops->plans[plan_id].handle; ops->current_plan = plan_id; if(plan_id + 1 < qm->plan.used && qm->plan.array[plan_id + 1].after < qm->plan.array[plan_id].before) @@ -1071,8 +1294,8 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti else ops->current_plan_expire_time = qm->plan.array[plan_id].before; - ops->plan_expanded_after = qm->plan.array[plan_id].expanded_after; - ops->plan_expanded_before = qm->plan.array[plan_id].expanded_before; + ops->plan_expanded_after = ops->plans[plan_id].expanded_after; + ops->plan_expanded_before = ops->plans[plan_id].expanded_before; } static bool query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t last_point_end_time) { @@ -1117,18 +1340,17 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before // put our selected tier as the first plan size_t selected_tier; + bool switch_tiers = true; - if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER + if((ops->r->internal.qt->window.options & RRDR_OPTION_SELECTED_TIER) && ops->r->internal.qt->window.tier < storage_tiers && query_metric_is_valid_tier(qm, ops->r->internal.qt->window.tier)) { selected_tier = ops->r->internal.qt->window.tier; + switch_tiers = false; } else { selected_tier = query_metric_best_tier_for_timeframe(qm, after_wanted, before_wanted, points_wanted); - if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER) - ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER; - if(!query_metric_is_valid_tier(qm, selected_tier)) return false; @@ -1142,7 +1364,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before qm->plan.array[0].after = (qm->tiers[selected_tier].db_first_time_s < after_wanted) ? after_wanted : qm->tiers[selected_tier].db_first_time_s; qm->plan.array[0].before = (qm->tiers[selected_tier].db_last_time_s > before_wanted) ? before_wanted : qm->tiers[selected_tier].db_last_time_s; - if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) { + if(switch_tiers) { // the selected tier time_t selected_tier_first_time_s = qm->plan.array[0].after; time_t selected_tier_last_time_s = qm->plan.array[0].before; @@ -1150,7 +1372,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before // check if our selected tier can start the query if (selected_tier_first_time_s > after_wanted) { // we need some help from other tiers - for (size_t tr = (int)selected_tier + 1; tr < storage_tiers; tr++) { + for (size_t tr = (int)selected_tier + 1; tr < storage_tiers && qm->plan.used < QUERY_PLANS_MAX ; tr++) { if(!query_metric_is_valid_tier(qm, tr)) continue; @@ -1164,9 +1386,9 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before .tier = tr, .after = (tier_first_time_s < after_wanted) ? after_wanted : tier_first_time_s, .before = selected_tier_first_time_s, - .initialized = false, - .finalized = false, }; + ops->plans[qm->plan.used].initialized = false; + ops->plans[qm->plan.used].finalized = false; qm->plan.array[qm->plan.used++] = t; internal_fatal(!t.after || !t.before, "QUERY: invalid plan selected"); @@ -1183,7 +1405,7 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before // check if our selected tier can finish the query if (selected_tier_last_time_s < before_wanted) { // we need some help from other tiers - for (int tr = (int)selected_tier - 1; tr >= 0; tr--) { + for (int tr = (int)selected_tier - 1; tr >= 0 && qm->plan.used < QUERY_PLANS_MAX ; tr--) { if(!query_metric_is_valid_tier(qm, tr)) continue; @@ -1199,9 +1421,9 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before .tier = tr, .after = selected_tier_last_time_s, .before = (tier_last_time_s > before_wanted) ? before_wanted : tier_last_time_s, - .initialized = false, - .finalized = false, }; + ops->plans[qm->plan.used].initialized = false; + ops->plans[qm->plan.used].finalized = false; qm->plan.array[qm->plan.used++] = t; // prepare for the tier @@ -1244,60 +1466,102 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before #define query_interpolate_point(this_point, last_point, now) do { \ if(likely( \ /* the point to interpolate is more than 1s wide */ \ - (this_point).end_time - (this_point).start_time > 1 \ + (this_point).sp.end_time_s - (this_point).sp.start_time_s > 1 \ \ /* the two points are exactly next to each other */ \ - && (last_point).end_time == (this_point).start_time \ + && (last_point).sp.end_time_s == (this_point).sp.start_time_s \ \ /* both points are valid numbers */ \ && netdata_double_isnumber((this_point).value) \ && netdata_double_isnumber((last_point).value) \ \ )) { \ - (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).end_time - (now)) / (NETDATA_DOUBLE)((this_point).end_time - (this_point).start_time)); \ - (this_point).end_time = now; \ + (this_point).value = (last_point).value + ((this_point).value - (last_point).value) * (1.0 - (NETDATA_DOUBLE)((this_point).sp.end_time_s - (now)) / (NETDATA_DOUBLE)((this_point).sp.end_time_s - (this_point).sp.start_time_s)); \ + (this_point).sp.end_time_s = now; \ } \ } while(0) -#define query_add_point_to_group(r, point, ops) do { \ +#define query_add_point_to_group(r, point, ops, add_flush) do { \ if(likely(netdata_double_isnumber((point).value))) { \ if(likely(fpclassify((point).value) != FP_ZERO)) \ (ops)->group_points_non_zero++; \ \ - if(unlikely((point).flags & SN_FLAG_RESET)) \ + if(unlikely((point).sp.flags & SN_FLAG_RESET)) \ (ops)->group_value_flags |= RRDR_VALUE_RESET; \ \ - (ops)->grouping_add(r, (point).value); \ + time_grouping_add(r, (point).value, add_flush); \ + \ + storage_point_merge_to((ops)->group_point, (point).sp); \ + if(!(point).added) \ + storage_point_merge_to((ops)->query_point, (point).sp); \ } \ \ (ops)->group_points_added++; \ - (ops)->group_anomaly_rate += (point).anomaly; \ } while(0) -static QUERY_ENGINE_OPS *rrd2rrdr_query_prep(RRDR *r, size_t dim_id_in_rrdr) { +static __thread QUERY_ENGINE_OPS *released_ops = NULL; + +static void rrd2rrdr_query_ops_freeall(RRDR *r __maybe_unused) { + while(released_ops) { + QUERY_ENGINE_OPS *ops = released_ops; + released_ops = ops->next; + + onewayalloc_freez(r->internal.owa, ops); + } +} + +static void rrd2rrdr_query_ops_release(QUERY_ENGINE_OPS *ops) { + if(!ops) return; + + ops->next = released_ops; + released_ops = ops; +} + +static QUERY_ENGINE_OPS *rrd2rrdr_query_ops_get(RRDR *r) { + QUERY_ENGINE_OPS *ops; + if(released_ops) { + ops = released_ops; + released_ops = ops->next; + } + else { + ops = onewayalloc_mallocz(r->internal.owa, sizeof(QUERY_ENGINE_OPS)); + } + + memset(ops, 0, sizeof(*ops)); + return ops; +} + +static QUERY_ENGINE_OPS *rrd2rrdr_query_ops_prep(RRDR *r, size_t query_metric_id) { QUERY_TARGET *qt = r->internal.qt; - QUERY_ENGINE_OPS *ops = onewayalloc_mallocz(r->internal.owa, sizeof(QUERY_ENGINE_OPS)); + QUERY_ENGINE_OPS *ops = rrd2rrdr_query_ops_get(r); *ops = (QUERY_ENGINE_OPS) { .r = r, - .qm = &qt->query.array[dim_id_in_rrdr], - .grouping_add = r->internal.grouping_add, - .grouping_flush = r->internal.grouping_flush, - .tier_query_fetch = r->internal.tier_query_fetch, - .view_update_every = r->update_every, - .query_granularity = (time_t)(r->update_every / r->group), + .qm = query_metric(qt, query_metric_id), + .tier_query_fetch = r->time_grouping.tier_query_fetch, + .view_update_every = r->view.update_every, + .query_granularity = (time_t)(r->view.update_every / r->view.group), .group_value_flags = RRDR_VALUE_NOTHING, }; - if(!query_plan(ops, qt->window.after, qt->window.before, qt->window.points)) + if(!query_plan(ops, qt->window.after, qt->window.before, qt->window.points)) { + rrd2rrdr_query_ops_release(ops); return NULL; + } return ops; } static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_OPS *ops) { QUERY_TARGET *qt = r->internal.qt; - QUERY_METRIC *qm = &qt->query.array[dim_id_in_rrdr]; (void)qm; + QUERY_METRIC *qm = ops->qm; + + const RRDR_TIME_GROUPING add_flush = r->time_grouping.add_flush; + + ops->group_point = STORAGE_POINT_UNSET; + ops->query_point = STORAGE_POINT_UNSET; + + RRDR_OPTIONS options = qt->window.options; size_t points_wanted = qt->window.points; time_t after_wanted = qt->window.after; time_t before_wanted = qt->window.before; (void)before_wanted; @@ -1306,15 +1570,12 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ // if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0) // debug_this = true; - time_t max_date = 0, - min_date = 0; - size_t points_added = 0; long rrdr_line = -1; - bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false; + bool use_anomaly_bit_as_value = (r->internal.qt->window.options & RRDR_OPTION_ANOMALY_BIT) ? true : false; - NETDATA_DOUBLE min = r->min, max = r->max; + NETDATA_DOUBLE min = r->view.min, max = r->view.max; QUERY_POINT last2_point = QUERY_POINT_EMPTY; QUERY_POINT last1_point = QUERY_POINT_EMPTY; @@ -1329,12 +1590,14 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ time_t now_end_time = after_wanted + ops->view_update_every - ops->query_granularity; size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch; + size_t query_is_finished_counter = 0; // The main loop, based on the query granularity we need - for( ; points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops->view_update_every) { + for( ; points_added < points_wanted && query_is_finished_counter <= 10 ; + now_start_time = now_end_time, now_end_time += ops->view_update_every) { if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) { - query_planer_next_plan(ops, now_end_time, new_point.end_time); + query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s); db_points_read_since_plan_switch = 0; } @@ -1347,26 +1610,35 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ last1_point = new_point; } - if(unlikely(ops->is_finished(ops->handle))) { + if(unlikely(storage_engine_query_is_finished(ops->handle))) { + query_is_finished_counter++; + if(count_same_end_time != 0) { last2_point = last1_point; last1_point = new_point; } new_point = QUERY_POINT_EMPTY; - new_point.start_time = last1_point.end_time; - new_point.end_time = now_end_time; + new_point.sp.start_time_s = last1_point.sp.end_time_s; + new_point.sp.end_time_s = now_end_time; // // if(debug_this) info("QUERY: is finished() returned true"); // break; } + else + query_is_finished_counter = 0; // fetch the new point { STORAGE_POINT sp; if(likely(storage_point_is_unset(next1_point))) { db_points_read_since_plan_switch++; - sp = ops->next_metric(ops->handle); + sp = storage_engine_query_next_metric(ops->handle); + ops->db_points_read_per_tier[ops->tier]++; + ops->db_total_points_read++; + + if(unlikely(options & RRDR_OPTION_ABSOLUTE)) + storage_point_make_positive(sp); } else { // ONE POINT READ-AHEAD @@ -1377,7 +1649,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ // ONE POINT READ-AHEAD if(unlikely(query_plan_should_switch_plan(ops, sp.end_time_s) && - query_planer_next_plan(ops, now_end_time, new_point.end_time))) { + query_planer_next_plan(ops, now_end_time, new_point.sp.end_time_s))) { // The end time of the current point, crosses our plans (tiers) // so, we switched plan (tier) @@ -1387,7 +1659,12 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ // A. the entire point of the previous plan is to the future of point from the next plan // B. part of the point of the previous plan overlaps with the point from the next plan - STORAGE_POINT sp2 = ops->next_metric(ops->handle); + STORAGE_POINT sp2 = storage_engine_query_next_metric(ops->handle); + ops->db_points_read_per_tier[ops->tier]++; + ops->db_total_points_read++; + + if(unlikely(options & RRDR_OPTION_ABSOLUTE)) + storage_point_make_positive(sp); if(sp.start_time_s > sp2.start_time_s) // the point from the previous plan is useless @@ -1399,12 +1676,8 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ next1_point = sp2; } - ops->db_points_read_per_tier[ops->tier]++; - ops->db_total_points_read++; - - new_point.start_time = sp.start_time_s; - new_point.end_time = sp.end_time_s; - new_point.anomaly = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0; + new_point.sp = sp; + new_point.added = false; query_point_set_id(new_point, ops->db_total_points_read); // if(debug_this) @@ -1415,13 +1688,13 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ if(likely(!storage_point_is_unset(sp) && !storage_point_is_gap(sp))) { if(unlikely(use_anomaly_bit_as_value)) - new_point.value = new_point.anomaly; + new_point.value = storage_point_anomaly_rate(new_point.sp); else { switch (ops->tier_query_fetch) { default: case TIER_QUERY_FETCH_AVERAGE: - new_point.value = sp.sum / sp.count; + new_point.value = sp.sum / (NETDATA_DOUBLE)sp.count; break; case TIER_QUERY_FETCH_MIN: @@ -1438,36 +1711,34 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ }; } } - else { + else new_point.value = NAN; - new_point.flags = SN_FLAG_NONE; - } } // check if the db is giving us zero duration points if(unlikely(db_points_read_since_plan_switch > 1 && - new_point.start_time == new_point.end_time)) { + new_point.sp.start_time_s == new_point.sp.end_time_s)) { internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned " "point %zu from %ld to %ld, that are both equal", - qt->id, string2str(qm->dimension.id), - new_point.id, new_point.start_time, new_point.end_time); + qt->id, query_metric_id(qt, qm), + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s); - new_point.start_time = new_point.end_time - ops->tier_ptr->db_update_every_s; + new_point.sp.start_time_s = new_point.sp.end_time_s - ops->tier_ptr->db_update_every_s; } // check if the db is advancing the query if(unlikely(db_points_read_since_plan_switch > 1 && - new_point.end_time <= last1_point.end_time)) { + new_point.sp.end_time_s <= last1_point.sp.end_time_s)) { internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned " "point %zu from %ld to %ld, before the " "last point %zu from %ld to %ld, " "now is %ld to %ld", - qt->id, string2str(qm->dimension.id), - new_point.id, new_point.start_time, new_point.end_time, - last1_point.id, last1_point.start_time, last1_point.end_time, + qt->id, query_metric_id(qt, qm), + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s, + last1_point.id, last1_point.sp.start_time_s, last1_point.sp.end_time_s, now_start_time, now_end_time); count_same_end_time++; @@ -1476,13 +1747,14 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ count_same_end_time = 0; // decide how to use this point - if(likely(new_point.end_time < now_end_time)) { // likely to favor tier0 + if(likely(new_point.sp.end_time_s < now_end_time)) { // likely to favor tier0 // this db point ends before our now_end_time - if(likely(new_point.end_time >= now_start_time)) { // likely to favor tier0 + if(likely(new_point.sp.end_time_s >= now_start_time)) { // likely to favor tier0 // this db point ends after our now_start time - query_add_point_to_group(r, new_point, ops); + query_add_point_to_group(r, new_point, ops, add_flush); + new_point.added = true; } else { // we don't need this db point @@ -1493,14 +1765,14 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ // at exactly the time we will want // we only log if this is not point 1 - internal_error(new_point.end_time < ops->plan_expanded_after && + internal_error(new_point.sp.end_time_s < ops->plan_expanded_after && db_points_read_since_plan_switch > 1, "QUERY: '%s', dimension '%s' next_metric() " "returned point %zu from %ld time %ld, " "which is entirely before our current timeframe %ld to %ld " "(and before the entire query, after %ld, before %ld)", - qt->id, string2str(qm->dimension.id), - new_point.id, new_point.start_time, new_point.end_time, + qt->id, query_metric_id(qt, qm), + new_point.id, new_point.sp.start_time_s, new_point.sp.end_time_s, now_start_time, now_end_time, ops->plan_expanded_after, ops->plan_expanded_before); } @@ -1518,15 +1790,15 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ "QUERY: '%s', dimension '%s', the database does not advance the query," " it returned an end time less or equal to the end time of the last " "point we got %ld, %zu times", - qt->id, string2str(qm->dimension.id), - last1_point.end_time, count_same_end_time); + qt->id, query_metric_id(qt, qm), + last1_point.sp.end_time_s, count_same_end_time); - if(unlikely(new_point.end_time <= last1_point.end_time)) - new_point.end_time = now_end_time; + if(unlikely(new_point.sp.end_time_s <= last1_point.sp.end_time_s)) + new_point.sp.end_time_s = now_end_time; } - time_t stop_time = new_point.end_time; - if(unlikely(!storage_point_is_unset(next1_point))) { + time_t stop_time = new_point.sp.end_time_s; + if(unlikely(!storage_point_is_unset(next1_point) && next1_point.start_time_s >= now_end_time)) { // ONE POINT READ-AHEAD // the point crosses the start time of the // read ahead storage point we have read @@ -1537,18 +1809,20 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ // we have 3 points in memory: last2, last1, new // we select the one to use based on their timestamps - size_t iterations = 0; - for ( ; now_end_time <= stop_time && points_added < points_wanted ; - now_end_time += ops->view_update_every, iterations++) { + internal_fatal(now_end_time > stop_time || points_added >= points_wanted, + "QUERY: first part of query provides invalid point to interpolate (now_end_time %ld, stop_time %ld", + now_end_time, stop_time); + do { // now_start_time is wrong in this loop // but, we don't need it QUERY_POINT current_point; - if(likely(now_end_time > new_point.start_time)) { + if(likely(now_end_time > new_point.sp.start_time_s)) { // it is time for our NEW point to be used current_point = new_point; + new_point.added = true; // first copy, then set it, so that new_point will not be added again query_interpolate_point(current_point, last1_point, now_end_time); // internal_error(current_point.id > 0 @@ -1564,9 +1838,10 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ // current_point.id, current_point.start_time, current_point.end_time, // now_end_time); } - else if(likely(now_end_time <= last1_point.end_time)) { + else if(likely(now_end_time <= last1_point.sp.end_time_s)) { // our LAST point is still valid current_point = last1_point; + last1_point.added = true; // first copy, then set it, so that last1_point will not be added again query_interpolate_point(current_point, last2_point, now_end_time); // internal_error(current_point.id > 0 @@ -1586,14 +1861,11 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ current_point = QUERY_POINT_EMPTY; } - query_add_point_to_group(r, current_point, ops); + query_add_point_to_group(r, current_point, ops, add_flush); rrdr_line = rrdr_line_init(r, now_end_time, rrdr_line); size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr; - if(unlikely(!min_date)) min_date = now_end_time; - max_date = now_end_time; - // find the place to store our values RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index]; @@ -1605,15 +1877,12 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ *rrdr_value_options_ptr = ops->group_value_flags; // store the group value - NETDATA_DOUBLE group_value = ops->grouping_flush(r, rrdr_value_options_ptr); + NETDATA_DOUBLE group_value = time_grouping_flush(r, rrdr_value_options_ptr, add_flush); r->v[rrdr_o_v_index] = group_value; - // we only store uint8_t anomaly rates, - // so let's get double precision by storing - // anomaly rates in the range 0 - 200 - r->ar[rrdr_o_v_index] = ops->group_anomaly_rate / (NETDATA_DOUBLE)ops->group_points_added; + r->ar[rrdr_o_v_index] = storage_point_anomaly_rate(ops->group_point); - if(likely(points_added || dim_id_in_rrdr)) { + if(likely(points_added || r->internal.queries_count)) { // find the min/max across all dimensions if(unlikely(group_value < min)) min = group_value; @@ -1621,7 +1890,7 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ } else { - // runs only when dim_id_in_rrdr == 0 && points_added == 0 + // runs only when r->internal.queries_count == 0 && points_added == 0 // so, on the first point added for the query. min = max = group_value; } @@ -1630,31 +1899,38 @@ static void rrd2rrdr_query_execute(RRDR *r, size_t dim_id_in_rrdr, QUERY_ENGINE_ ops->group_points_added = 0; ops->group_value_flags = RRDR_VALUE_NOTHING; ops->group_points_non_zero = 0; - ops->group_anomaly_rate = 0; - } - // the loop above increased "now" by query_granularity, + ops->group_point = STORAGE_POINT_UNSET; + + now_end_time += ops->view_update_every; + } while(now_end_time <= stop_time && points_added < points_wanted); + + // the loop above increased "now" by ops->view_update_every, // but the main loop will increase it too, // so, let's undo the last iteration of this loop - if(iterations) - now_end_time -= ops->view_update_every; + now_end_time -= ops->view_update_every; } query_planer_finalize_remaining_plans(ops); - r->internal.result_points_generated += points_added; - r->internal.db_points_read += ops->db_total_points_read; + qm->query_points = ops->query_point; + + // fill the rest of the points with empty values + while (points_added < points_wanted) { + rrdr_line++; + size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr; + r->o[rrdr_o_v_index] = RRDR_VALUE_EMPTY; + r->v[rrdr_o_v_index] = 0.0; + r->ar[rrdr_o_v_index] = 0.0; + points_added++; + } + + r->internal.queries_count++; + r->view.min = min; + r->view.max = max; + + r->stats.result_points_generated += points_added; + r->stats.db_points_read += ops->db_total_points_read; for(size_t tr = 0; tr < storage_tiers ; tr++) - r->internal.tier_points_read[tr] += ops->db_points_read_per_tier[tr]; - - r->min = min; - r->max = max; - r->before = max_date; - r->after = min_date - ops->view_update_every + ops->query_granularity; - rrdr_done(r, rrdr_line); - - internal_error(points_added != points_wanted, - "QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).", - qt->id, string2str(qm->dimension.id), - (size_t)points_wanted, (size_t)points_added, ops->db_total_points_read); + qt->db.tiers[tr].points += ops->db_points_read_per_tier[tr]; } // ---------------------------------------------------------------------------- @@ -1669,7 +1945,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s struct rrddim_tier *t = &rd->tiers[tier]; if(unlikely(!t)) return; - time_t latest_time_s = t->query_ops->latest_time_s(t->db_metric_handle); + time_t latest_time_s = storage_engine_latest_time_s(t->backend, t->db_metric_handle); time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every; time_t time_diff = now_s - latest_time_s; @@ -1683,21 +1959,21 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s // for each lower tier for(int read_tier = (int)tier - 1; read_tier >= 0 ; read_tier--){ - time_t smaller_tier_first_time = rd->tiers[read_tier].query_ops->oldest_time_s(rd->tiers[read_tier].db_metric_handle); - time_t smaller_tier_last_time = rd->tiers[read_tier].query_ops->latest_time_s(rd->tiers[read_tier].db_metric_handle); + time_t smaller_tier_first_time = storage_engine_oldest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle); + time_t smaller_tier_last_time = storage_engine_latest_time_s(rd->tiers[read_tier].backend, rd->tiers[read_tier].db_metric_handle); if(smaller_tier_last_time <= latest_time_s) continue; // it is as bad as we are long after_wanted = (latest_time_s < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_s; long before_wanted = smaller_tier_last_time; struct rrddim_tier *tmp = &rd->tiers[read_tier]; - tmp->query_ops->init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH); + storage_engine_query_init(tmp->backend, tmp->db_metric_handle, &handle, after_wanted, before_wanted, STORAGE_PRIORITY_HIGH); size_t points_read = 0; - while(!tmp->query_ops->is_finished(&handle)) { + while(!storage_engine_query_is_finished(&handle)) { - STORAGE_POINT sp = tmp->query_ops->next_metric(&handle); + STORAGE_POINT sp = storage_engine_query_next_metric(&handle); points_read++; if(sp.end_time_s > latest_time_s) { @@ -1706,7 +1982,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s } } - tmp->query_ops->finalize(&handle); + storage_engine_query_finalize(&handle); store_metric_collection_completed(); global_statistics_backfill_query_completed(points_read); @@ -1721,7 +1997,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s #ifdef NETDATA_INTERNAL_CHECKS static void rrd2rrdr_log_request_response_metadata(RRDR *r , RRDR_OPTIONS options __maybe_unused - , RRDR_GROUPING group_method + , RRDR_TIME_GROUPING group_method , bool aligned , size_t group , time_t resampling_time @@ -1737,8 +2013,9 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r , const char *msg ) { - time_t first_entry_s = r->internal.qt->db.first_time_s; - time_t last_entry_s = r->internal.qt->db.last_time_s; + QUERY_TARGET *qt = r->internal.qt; + time_t first_entry_s = qt->db.first_time_s; + time_t last_entry_s = qt->db.last_time_s; internal_error( true, @@ -1748,33 +2025,33 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r "duration (got: %ld, want: %ld, req: %ld, db: %ld), " "points (got: %zu, want: %zu, req: %zu), " "%s" - , r->internal.qt->id - , r->internal.qt->window.query_granularity + , qt->id + , qt->window.query_granularity // grouping , (aligned) ? "aligned" : "unaligned" - , group_method2string(group_method) + , time_grouping_method2string(group_method) , group , resampling_time , resampling_group // after - , r->after + , r->view.after , after_wanted , after_requested , first_entry_s // before - , r->before + , r->view.before , before_wanted , before_requested , last_entry_s // duration - , (long)(r->before - r->after + r->internal.qt->window.query_granularity) - , (long)(before_wanted - after_wanted + r->internal.qt->window.query_granularity) + , (long)(r->view.before - r->view.after + qt->window.query_granularity) + , (long)(before_wanted - after_wanted + qt->window.query_granularity) , (long)before_requested - after_requested - , (long)((last_entry_s - first_entry_s) + r->internal.qt->window.query_granularity) + , (long)((last_entry_s - first_entry_s) + qt->window.query_granularity) // points , r->rows @@ -1788,9 +2065,12 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r #endif // NETDATA_INTERNAL_CHECKS // Returns 1 if an absolute period was requested or 0 if it was a relative period -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before) { +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr) { time_t now = now_realtime_sec() - 1; + if(now_ptr) + *now_ptr = now; + int absolute_period_requested = -1; long long after_requested, before_requested; @@ -1890,11 +2170,11 @@ bool query_target_calculate_window(QUERY_TARGET *qt) { size_t points_requested = (long)qt->request.points; time_t after_requested = qt->request.after; time_t before_requested = qt->request.before; - RRDR_GROUPING group_method = qt->request.group_method; + RRDR_TIME_GROUPING group_method = qt->request.time_group_method; time_t resampling_time_requested = qt->request.resampling_time; - RRDR_OPTIONS options = qt->request.options; + RRDR_OPTIONS options = qt->window.options; size_t tier = qt->request.tier; - time_t update_every = qt->db.minimum_latest_update_every_s; + time_t update_every = qt->db.minimum_latest_update_every_s ? qt->db.minimum_latest_update_every_s : 1; // RULES // points_requested = 0 @@ -1953,27 +2233,36 @@ bool query_target_calculate_window(QUERY_TARGET *qt) { time_t last_entry_s = qt->db.last_time_s; if (first_entry_s == 0 || last_entry_s == 0) { - internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld", qt->id, first_entry_s, last_entry_s); - query_debug_log_free(); - return false; - } + internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld)", qt->id, first_entry_s, last_entry_s); + after_wanted = qt->window.after; + before_wanted = qt->window.before; - query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_s, last_entry_s); + if(after_wanted == before_wanted) + after_wanted = before_wanted - update_every; - if (after_wanted == 0) { - after_wanted = first_entry_s; - query_debug_log(":zero after_wanted %ld", after_wanted); + if (points_wanted == 0) { + points_wanted = (before_wanted - after_wanted) / update_every; + query_debug_log(":zero points_wanted %zu", points_wanted); + } } + else { + query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_s, last_entry_s); - if (before_wanted == 0) { - before_wanted = last_entry_s; - before_is_aligned_to_db_end = true; - query_debug_log(":zero before_wanted %ld", before_wanted); - } + if (after_wanted == 0) { + after_wanted = first_entry_s; + query_debug_log(":zero after_wanted %ld", after_wanted); + } - if (points_wanted == 0) { - points_wanted = (last_entry_s - first_entry_s) / update_every; - query_debug_log(":zero points_wanted %zu", points_wanted); + if (before_wanted == 0) { + before_wanted = last_entry_s; + before_is_aligned_to_db_end = true; + query_debug_log(":zero before_wanted %ld", before_wanted); + } + + if (points_wanted == 0) { + points_wanted = (last_entry_s - first_entry_s) / update_every; + query_debug_log(":zero points_wanted %zu", points_wanted); + } } } @@ -1983,7 +2272,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) { } // convert our before_wanted and after_wanted to absolute - rrdr_relative_window_to_absolute(&after_wanted, &before_wanted); + rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL); query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted); if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) { @@ -2145,8 +2434,8 @@ bool query_target_calculate_window(QUERY_TARGET *qt) { qt->window.relative = relative_period_requested; qt->window.points = points_wanted; qt->window.group = group; - qt->window.group_method = group_method; - qt->window.group_options = qt->request.group_options; + qt->window.time_group_method = group_method; + qt->window.time_group_options = qt->request.time_group_options; qt->window.query_granularity = query_granularity; qt->window.resampling_group = resampling_group; qt->window.resampling_divisor = resampling_divisor; @@ -2157,80 +2446,1081 @@ bool query_target_calculate_window(QUERY_TARGET *qt) { return true; } +// ---------------------------------------------------------------------------- +// group by + +struct group_by_label_key { + DICTIONARY *values; +}; + +static void group_by_label_key_insert_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + // add the key to our r->label_keys global keys dictionary + DICTIONARY *label_keys = data; + dictionary_set(label_keys, dictionary_acquired_item_name(item), NULL, 0); + + // create a dictionary for the values of this key + struct group_by_label_key *k = value; + k->values = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, 0); +} + +static void group_by_label_key_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct group_by_label_key *k = value; + dictionary_destroy(k->values); +} + +static int rrdlabels_traversal_cb_to_group_by_label_key(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) { + DICTIONARY *dl = data; + struct group_by_label_key *k = dictionary_set(dl, name, NULL, sizeof(struct group_by_label_key)); + dictionary_set(k->values, value, NULL, 0); + return 1; +} + +void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) { + if(!r->label_keys || !r->dl) + return; + + buffer_json_member_add_object(wb, key); + + void *t; + dfe_start_read(r->label_keys, t) { + buffer_json_member_add_array(wb, t_dfe.name); + + for(size_t d = 0; d < r->d ;d++) { + if(!rrdr_dimension_should_be_exposed(r->od[d], options)) + continue; + + struct group_by_label_key *k = dictionary_get(r->dl[d], t_dfe.name); + if(k) { + buffer_json_add_array_item_array(wb); + void *tt; + dfe_start_read(k->values, tt) { + buffer_json_add_array_item_string(wb, tt_dfe.name); + } + dfe_done(tt); + buffer_json_array_close(wb); + } + else + buffer_json_add_array_item_string(wb, NULL); + } + + buffer_json_array_close(wb); + } + dfe_done(t); + + buffer_json_object_close(wb); // key +} + +static int group_by_label_is_space(char c) { + if(c == ',' || c == '|') + return 1; + + return 0; +} + +static void rrd2rrdr_set_timestamps(RRDR *r) { + QUERY_TARGET *qt = r->internal.qt; + + internal_fatal(qt->window.points != r->n, "QUERY: mismatch to the number of points in qt and r"); + + r->view.group = qt->window.group; + r->view.update_every = (int) query_view_update_every(qt); + r->view.before = qt->window.before; + r->view.after = qt->window.after; + + r->time_grouping.points_wanted = qt->window.points; + r->time_grouping.resampling_group = qt->window.resampling_group; + r->time_grouping.resampling_divisor = qt->window.resampling_divisor; + + r->rows = qt->window.points; + + size_t points_wanted = qt->window.points; + time_t after_wanted = qt->window.after; + time_t before_wanted = qt->window.before; (void)before_wanted; + + time_t view_update_every = r->view.update_every; + time_t query_granularity = (time_t)(r->view.update_every / r->view.group); + + size_t rrdr_line = 0; + time_t first_point_end_time = after_wanted + view_update_every - query_granularity; + time_t now_end_time = first_point_end_time; + + while (rrdr_line < points_wanted) { + r->t[rrdr_line++] = now_end_time; + now_end_time += view_update_every; + } + + internal_fatal(r->t[0] != first_point_end_time, "QUERY: wrong first timestamp in the query"); + internal_error(r->t[points_wanted - 1] != before_wanted, + "QUERY: wrong last timestamp in the query, expected %ld, found %ld", + before_wanted, r->t[points_wanted - 1]); +} + +static void query_group_by_make_dimension_key(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) { + buffer_flush(key); + if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) { + buffer_strcat(key, "__hidden_dimensions__"); + } + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) { + buffer_strcat(key, "selected"); + } + else { + if (group_by & RRDR_GROUP_BY_DIMENSION) { + buffer_fast_strcat(key, "|", 1); + buffer_strcat(key, query_metric_name(qt, qm)); + } + + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) { + buffer_fast_strcat(key, "|", 1); + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version))); + } + + if (group_by & RRDR_GROUP_BY_LABEL) { + DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria); + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) { + buffer_fast_strcat(key, "|", 1); + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]"); + } + } + + if (group_by & RRDR_GROUP_BY_NODE) { + buffer_fast_strcat(key, "|", 1); + buffer_strcat(key, qn->rrdhost->machine_guid); + } + + if (group_by & RRDR_GROUP_BY_CONTEXT) { + buffer_fast_strcat(key, "|", 1); + buffer_strcat(key, rrdcontext_acquired_id(qc->rca)); + } + + if (group_by & RRDR_GROUP_BY_UNITS) { + buffer_fast_strcat(key, "|", 1); + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria)); + } + } +} + +static void query_group_by_make_dimension_id(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) { + buffer_flush(key); + if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) { + buffer_strcat(key, "__hidden_dimensions__"); + } + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) { + buffer_strcat(key, "selected"); + } + else { + if (group_by & RRDR_GROUP_BY_DIMENSION) { + buffer_strcat(key, query_metric_name(qt, qm)); + } + + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + if (group_by & RRDR_GROUP_BY_NODE) + buffer_strcat(key, rrdinstance_acquired_id(qi->ria)); + else + buffer_strcat(key, string2str(query_instance_id_fqdn(qi, qt->request.version))); + } + + if (group_by & RRDR_GROUP_BY_LABEL) { + DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria); + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]"); + } + } + + if (group_by & RRDR_GROUP_BY_NODE) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + buffer_strcat(key, qn->rrdhost->machine_guid); + } + + if (group_by & RRDR_GROUP_BY_CONTEXT) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + buffer_strcat(key, rrdcontext_acquired_id(qc->rca)); + } + + if (group_by & RRDR_GROUP_BY_UNITS) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria)); + } + } +} + +static void query_group_by_make_dimension_name(BUFFER *key, RRDR_GROUP_BY group_by, size_t group_by_id, QUERY_TARGET *qt, QUERY_NODE *qn, QUERY_CONTEXT *qc, QUERY_INSTANCE *qi, QUERY_DIMENSION *qd __maybe_unused, QUERY_METRIC *qm, bool query_has_percentage_of_instance) { + buffer_flush(key); + if(unlikely(!query_has_percentage_of_instance && qm->status & RRDR_DIMENSION_HIDDEN)) { + buffer_strcat(key, "__hidden_dimensions__"); + } + else if(unlikely(group_by & RRDR_GROUP_BY_SELECTED)) { + buffer_strcat(key, "selected"); + } + else { + if (group_by & RRDR_GROUP_BY_DIMENSION) { + buffer_strcat(key, query_metric_name(qt, qm)); + } + + if (group_by & (RRDR_GROUP_BY_INSTANCE|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE)) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + if (group_by & RRDR_GROUP_BY_NODE) + buffer_strcat(key, rrdinstance_acquired_name(qi->ria)); + else + buffer_strcat(key, string2str(query_instance_name_fqdn(qi, qt->request.version))); + } + + if (group_by & RRDR_GROUP_BY_LABEL) { + DICTIONARY *labels = rrdinstance_acquired_labels(qi->ria); + for (size_t l = 0; l < qt->group_by[group_by_id].used; l++) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + rrdlabels_get_value_to_buffer_or_unset(labels, key, qt->group_by[group_by_id].label_keys[l], "[unset]"); + } + } + + if (group_by & RRDR_GROUP_BY_NODE) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + buffer_strcat(key, rrdhost_hostname(qn->rrdhost)); + } + + if (group_by & RRDR_GROUP_BY_CONTEXT) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + buffer_strcat(key, rrdcontext_acquired_id(qc->rca)); + } + + if (group_by & RRDR_GROUP_BY_UNITS) { + if (buffer_strlen(key) != 0) + buffer_fast_strcat(key, ",", 1); + + buffer_strcat(key, query_target_has_percentage_units(qt) ? "%" : rrdinstance_acquired_units(qi->ria)); + } + } +} + +struct rrdr_group_by_entry { + size_t priority; + size_t count; + STRING *id; + STRING *name; + STRING *units; + RRDR_DIMENSION_FLAGS od; + DICTIONARY *dl; +}; + +static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { + RRDR *r_tmp = NULL; + RRDR_OPTIONS options = qt->window.options; + + if(qt->request.version < 2) { + // v1 query + RRDR *r = rrdr_create(owa, qt, qt->query.used, qt->window.points); + if(unlikely(!r)) { + internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, dimensions=%u, points=%zu", + qt->id, qt->window.after, qt->window.before, qt->query.used, qt->window.points); + return NULL; + } + r->group_by.r = NULL; + + for(size_t d = 0; d < qt->query.used ; d++) { + QUERY_METRIC *qm = query_metric(qt, d); + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); + r->di[d] = rrdmetric_acquired_id_dup(qd->rma); + r->dn[d] = rrdmetric_acquired_name_dup(qd->rma); + } + + rrd2rrdr_set_timestamps(r); + return r; + } + // v2 query + + // parse all the group-by label keys + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + if (qt->request.group_by[g].group_by & RRDR_GROUP_BY_LABEL && + qt->request.group_by[g].group_by_label && *qt->request.group_by[g].group_by_label) + qt->group_by[g].used = quoted_strings_splitter( + qt->request.group_by[g].group_by_label, qt->group_by[g].label_keys, + GROUP_BY_MAX_LABEL_KEYS, group_by_label_is_space); + + if (!qt->group_by[g].used) + qt->request.group_by[g].group_by &= ~RRDR_GROUP_BY_LABEL; + } + + // make sure there are valid group-by methods + bool query_has_percentage_of_instance = false; + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) { + if(!(qt->request.group_by[g].group_by & SUPPORTED_GROUP_BY_METHODS)) + qt->request.group_by[g].group_by = (g == 0) ? RRDR_GROUP_BY_DIMENSION : RRDR_GROUP_BY_NONE; + + if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + query_has_percentage_of_instance = true; + } + + // merge all group-by options to upper levels + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES - 1 ;g++) { + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_NONE) + continue; + + if(qt->request.group_by[g].group_by == RRDR_GROUP_BY_SELECTED) { + for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++) + qt->request.group_by[r].group_by = RRDR_GROUP_BY_NONE; + } + else { + for (size_t r = g + 1; r < MAX_QUERY_GROUP_BY_PASSES; r++) { + if (qt->request.group_by[r].group_by == RRDR_GROUP_BY_NONE) + continue; + + if (qt->request.group_by[r].group_by != RRDR_GROUP_BY_SELECTED) { + if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + qt->request.group_by[g].group_by |= RRDR_GROUP_BY_INSTANCE; + else + qt->request.group_by[g].group_by |= qt->request.group_by[r].group_by; + + if(qt->request.group_by[r].group_by & RRDR_GROUP_BY_LABEL) { + for (size_t lr = 0; lr < qt->group_by[r].used; lr++) { + bool found = false; + for (size_t lg = 0; lg < qt->group_by[g].used; lg++) { + if (strcmp(qt->group_by[g].label_keys[lg], qt->group_by[r].label_keys[lr]) == 0) { + found = true; + break; + } + } + + if (!found && qt->group_by[g].used < GROUP_BY_MAX_LABEL_KEYS * MAX_QUERY_GROUP_BY_PASSES) + qt->group_by[g].label_keys[qt->group_by[g].used++] = qt->group_by[r].label_keys[lr]; + } + } + } + } + } + } + + int added = 0; + RRDR *first_r = NULL, *last_r = NULL; + BUFFER *key = buffer_create(0, NULL); + struct rrdr_group_by_entry *entries = onewayalloc_mallocz(owa, qt->query.used * sizeof(struct rrdr_group_by_entry)); + DICTIONARY *groups = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + DICTIONARY *label_keys = NULL; + + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) { + RRDR_GROUP_BY group_by = qt->request.group_by[g].group_by; + + if(group_by == RRDR_GROUP_BY_NONE) + break; + + memset(entries, 0, qt->query.used * sizeof(struct rrdr_group_by_entry)); + dictionary_flush(groups); + added = 0; + + size_t hidden_dimensions = 0; + bool final_grouping = (g == MAX_QUERY_GROUP_BY_PASSES - 1 || qt->request.group_by[g + 1].group_by == RRDR_GROUP_BY_NONE) ? true : false; + + if (final_grouping && (options & RRDR_OPTION_GROUP_BY_LABELS)) + label_keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, 0); + + QUERY_INSTANCE *last_qi = NULL; + size_t priority = 0; + time_t update_every_max = 0; + for (size_t d = 0; d < qt->query.used; d++) { + QUERY_METRIC *qm = query_metric(qt, d); + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id); + QUERY_NODE *qn = query_node(qt, qm->link.query_node_id); + + if (qi != last_qi) { + last_qi = qi; + + time_t update_every = rrdinstance_acquired_update_every(qi->ria); + if (update_every > update_every_max) + update_every_max = update_every; + } + + priority = qd->priority; + + if(qm->status & RRDR_DIMENSION_HIDDEN) + hidden_dimensions++; + + // -------------------------------------------------------------------- + // generate the group by key + + query_group_by_make_dimension_key(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance); + + // lookup the key in the dictionary + + int pos = -1; + int *set = dictionary_set(groups, buffer_tostring(key), &pos, sizeof(pos)); + if (*set == -1) { + // the key just added to the dictionary + + *set = pos = added++; + + // ---------------------------------------------------------------- + // generate the dimension id + + query_group_by_make_dimension_id(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance); + entries[pos].id = string_strdupz(buffer_tostring(key)); + + // ---------------------------------------------------------------- + // generate the dimension name + + query_group_by_make_dimension_name(key, group_by, g, qt, qn, qc, qi, qd, qm, query_has_percentage_of_instance); + entries[pos].name = string_strdupz(buffer_tostring(key)); + + // add the rest of the info + entries[pos].units = rrdinstance_acquired_units_dup(qi->ria); + entries[pos].priority = priority; + + if (label_keys) { + entries[pos].dl = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE, + NULL, sizeof(struct group_by_label_key)); + dictionary_register_insert_callback(entries[pos].dl, group_by_label_key_insert_cb, label_keys); + dictionary_register_delete_callback(entries[pos].dl, group_by_label_key_delete_cb, label_keys); + } + } else { + // the key found in the dictionary + pos = *set; + } + + entries[pos].count++; + + if (unlikely(priority < entries[pos].priority)) + entries[pos].priority = priority; + + if(g > 0) + last_r->dgbs[qm->grouped_as.slot] = pos; + else + qm->grouped_as.first_slot = pos; + + qm->grouped_as.slot = pos; + qm->grouped_as.id = entries[pos].id; + qm->grouped_as.name = entries[pos].name; + qm->grouped_as.units = entries[pos].units; + + // copy the dimension flags decided by the query target + // we need this, because if a dimension is explicitly selected + // the query target adds to it the non-zero flag + qm->status |= RRDR_DIMENSION_GROUPED; + + if(query_has_percentage_of_instance) + // when the query has percentage of instance + // there will be no hidden dimensions in the final query + // so we have to remove the hidden flag from all dimensions + entries[pos].od |= qm->status & ~RRDR_DIMENSION_HIDDEN; + else + entries[pos].od |= qm->status; + + if (entries[pos].dl) + rrdlabels_walkthrough_read(rrdinstance_acquired_labels(qi->ria), + rrdlabels_traversal_cb_to_group_by_label_key, entries[pos].dl); + } + + RRDR *r = rrdr_create(owa, qt, added, qt->window.points); + if (!r) { + internal_error(true, + "QUERY: cannot create group by RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu", + qt->id, qt->window.after, qt->window.before, added, qt->window.points); + goto cleanup; + } + + bool hidden_dimension_on_percentage_of_instance = hidden_dimensions && (group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE); + + // prevent double cleanup in case of error + added = 0; + + if(!last_r) + first_r = last_r = r; + else + last_r->group_by.r = r; + + last_r = r; + + rrd2rrdr_set_timestamps(r); + r->dp = onewayalloc_callocz(owa, r->d, sizeof(*r->dp)); + r->dview = onewayalloc_callocz(owa, r->d, sizeof(*r->dview)); + r->dgbc = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbc)); + r->gbc = onewayalloc_callocz(owa, r->n * r->d, sizeof(*r->gbc)); + r->dqp = onewayalloc_callocz(owa, r->d, sizeof(STORAGE_POINT)); + + if(hidden_dimension_on_percentage_of_instance) + // this is where we are going to group the hidden dimensions + r->vh = onewayalloc_mallocz(owa, r->n * r->d * sizeof(*r->vh)); + + if(!final_grouping) + // this is where we are going to store the slot in the next RRDR + // that we are going to group by the dimension of this RRDR + r->dgbs = onewayalloc_callocz(owa, r->d, sizeof(*r->dgbs)); + + if (label_keys) { + r->dl = onewayalloc_callocz(owa, r->d, sizeof(DICTIONARY *)); + r->label_keys = label_keys; + label_keys = NULL; + } + + // zero r (dimension options, names, and ids) + // this is required, because group-by may lead to empty dimensions + for (size_t d = 0; d < r->d; d++) { + r->di[d] = entries[d].id; + r->dn[d] = entries[d].name; + + r->od[d] = entries[d].od; + r->du[d] = entries[d].units; + r->dp[d] = entries[d].priority; + r->dgbc[d] = entries[d].count; + + if (r->dl) + r->dl[d] = entries[d].dl; + } + + // initialize partial trimming + r->partial_data_trimming.max_update_every = update_every_max; + r->partial_data_trimming.expected_after = + (!(qt->window.options & RRDR_OPTION_RETURN_RAW) && + qt->window.before >= qt->window.now - update_every_max) ? + qt->window.before - update_every_max : + qt->window.before; + r->partial_data_trimming.trimmed_after = qt->window.before; + + // make all values empty + for (size_t i = 0; i != r->n; i++) { + NETDATA_DOUBLE *cn = &r->v[i * r->d]; + RRDR_VALUE_FLAGS *co = &r->o[i * r->d]; + NETDATA_DOUBLE *ar = &r->ar[i * r->d]; + NETDATA_DOUBLE *vh = r->vh ? &r->vh[i * r->d] : NULL; + + for (size_t d = 0; d < r->d; d++) { + cn[d] = NAN; + ar[d] = 0.0; + co[d] = RRDR_VALUE_EMPTY; + + if(vh) + *vh = NAN; + } + } + } + + if(!first_r || !last_r) + goto cleanup; + + r_tmp = rrdr_create(owa, qt, 1, qt->window.points); + if (!r_tmp) { + internal_error(true, + "QUERY: cannot create group by temporary RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu", + qt->id, qt->window.after, qt->window.before, 1, qt->window.points); + goto cleanup; + } + rrd2rrdr_set_timestamps(r_tmp); + r_tmp->group_by.r = first_r; + +cleanup: + if(!first_r || !last_r || !r_tmp) { + if(r_tmp) { + r_tmp->group_by.r = NULL; + rrdr_free(owa, r_tmp); + } + + if(first_r) { + RRDR *r = first_r; + while (r) { + r_tmp = r->group_by.r; + r->group_by.r = NULL; + rrdr_free(owa, r); + r = r_tmp; + } + } + + if(entries && added) { + for (int d = 0; d < added; d++) { + string_freez(entries[d].id); + string_freez(entries[d].name); + string_freez(entries[d].units); + dictionary_destroy(entries[d].dl); + } + } + dictionary_destroy(label_keys); + + first_r = last_r = r_tmp = NULL; + } + + buffer_free(key); + onewayalloc_freez(owa, entries); + dictionary_destroy(groups); + + return r_tmp; +} + +static void rrd2rrdr_group_by_add_metric(RRDR *r_dst, size_t d_dst, RRDR *r_tmp, size_t d_tmp, + RRDR_GROUP_BY_FUNCTION group_by_aggregate_function, + STORAGE_POINT *query_points, size_t pass __maybe_unused) { + if(!r_tmp || r_dst == r_tmp || !(r_tmp->od[d_tmp] & RRDR_DIMENSION_QUERIED)) + return; + + internal_fatal(r_dst->n != r_tmp->n, "QUERY: group-by source and destination do not have the same number of rows"); + internal_fatal(d_dst >= r_dst->d, "QUERY: group-by destination dimension number exceeds destination RRDR size"); + internal_fatal(d_tmp >= r_tmp->d, "QUERY: group-by source dimension number exceeds source RRDR size"); + internal_fatal(!r_dst->dqp, "QUERY: group-by destination is not properly prepared (missing dqp array)"); + internal_fatal(!r_dst->gbc, "QUERY: group-by destination is not properly prepared (missing gbc array)"); + + bool hidden_dimension_on_percentage_of_instance = (r_tmp->od[d_tmp] & RRDR_DIMENSION_HIDDEN) && r_dst->vh; + + if(!hidden_dimension_on_percentage_of_instance) { + r_dst->od[d_dst] |= r_tmp->od[d_tmp]; + storage_point_merge_to(r_dst->dqp[d_dst], *query_points); + } + + // do the group_by + for(size_t i = 0; i != rrdr_rows(r_tmp) ; i++) { + + size_t idx_tmp = i * r_tmp->d + d_tmp; + NETDATA_DOUBLE n_tmp = r_tmp->v[ idx_tmp ]; + RRDR_VALUE_FLAGS o_tmp = r_tmp->o[ idx_tmp ]; + NETDATA_DOUBLE ar_tmp = r_tmp->ar[ idx_tmp ]; + + if(o_tmp & RRDR_VALUE_EMPTY) + continue; + + size_t idx_dst = i * r_dst->d + d_dst; + NETDATA_DOUBLE *cn = (hidden_dimension_on_percentage_of_instance) ? &r_dst->vh[ idx_dst ] : &r_dst->v[ idx_dst ]; + RRDR_VALUE_FLAGS *co = &r_dst->o[ idx_dst ]; + NETDATA_DOUBLE *ar = &r_dst->ar[ idx_dst ]; + uint32_t *gbc = &r_dst->gbc[ idx_dst ]; + + switch(group_by_aggregate_function) { + default: + case RRDR_GROUP_BY_FUNCTION_AVERAGE: + case RRDR_GROUP_BY_FUNCTION_SUM: + if(isnan(*cn)) + *cn = n_tmp; + else + *cn += n_tmp; + break; + + case RRDR_GROUP_BY_FUNCTION_MIN: + if(isnan(*cn) || n_tmp < *cn) + *cn = n_tmp; + break; + + case RRDR_GROUP_BY_FUNCTION_MAX: + if(isnan(*cn) || n_tmp > *cn) + *cn = n_tmp; + break; + } + + if(!hidden_dimension_on_percentage_of_instance) { + *co &= ~RRDR_VALUE_EMPTY; + *co |= (o_tmp & (RRDR_VALUE_RESET | RRDR_VALUE_PARTIAL)); + *ar += ar_tmp; + (*gbc)++; + } + } +} + +static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) { + time_t trimmable_after = r->partial_data_trimming.expected_after; + + // find the point just before the trimmable ones + ssize_t i = (ssize_t)r->n - 1; + for( ; i >= 0 ;i--) { + if (r->t[i] < trimmable_after) + break; + } + + if(unlikely(i < 0)) + return; + + size_t last_row_gbc = 0; + for (; i < (ssize_t)r->n; i++) { + size_t row_gbc = 0; + for (size_t d = 0; d < r->d; d++) { + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) + continue; + + row_gbc += r->gbc[ i * r->d + d ]; + } + + if (unlikely(r->t[i] >= trimmable_after && row_gbc < last_row_gbc)) { + // discard the rest of the points + r->partial_data_trimming.trimmed_after = r->t[i]; + r->rows = i; + break; + } + else + last_row_gbc = row_gbc; + } +} + +static void rrdr2rrdr_group_by_calculate_percentage_of_instance(RRDR *r) { + if(!r->vh) + return; + + for(size_t i = 0; i < r->n ;i++) { + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *ch = &r->vh[ i * r->d ]; + + for(size_t d = 0; d < r->d ;d++) { + NETDATA_DOUBLE n = cn[d]; + NETDATA_DOUBLE h = ch[d]; + + if(isnan(n)) + cn[d] = 0.0; + + else if(isnan(h)) + cn[d] = 100.0; + + else + cn[d] = n * 100.0 / (n + h); + } + } +} + +static void rrd2rrdr_convert_to_percentage(RRDR *r) { + size_t global_min_max_values = 0; + NETDATA_DOUBLE global_min = NAN, global_max = NAN; + + for(size_t i = 0; i != r->n ;i++) { + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ]; + + NETDATA_DOUBLE total = 0; + for (size_t d = 0; d < r->d; d++) { + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) + continue; + + if(co[d] & RRDR_VALUE_EMPTY) + continue; + + total += cn[d]; + } + + if(total == 0.0) + total = 1.0; + + for (size_t d = 0; d < r->d; d++) { + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) + continue; + + if(co[d] & RRDR_VALUE_EMPTY) + continue; + + NETDATA_DOUBLE n = cn[d]; + n = cn[d] = n * 100.0 / total; + + if(unlikely(!global_min_max_values++)) + global_min = global_max = n; + else { + if(n < global_min) + global_min = n; + if(n > global_max) + global_max = n; + } + } + } + + r->view.min = global_min; + r->view.max = global_max; + + if(!r->dview) + // v1 query + return; + + // v2 query + + for (size_t d = 0; d < r->d; d++) { + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) + continue; + + size_t count = 0; + NETDATA_DOUBLE min = 0.0, max = 0.0, sum = 0.0, ars = 0.0; + for(size_t i = 0; i != r->rows ;i++) { // we use r->rows to respect trimming + size_t idx = i * r->d + d; + + RRDR_VALUE_FLAGS o = r->o[ idx ]; + + if (o & RRDR_VALUE_EMPTY) + continue; + + NETDATA_DOUBLE ar = r->ar[ idx ]; + ars += ar; + + NETDATA_DOUBLE n = r->v[ idx ]; + sum += n; + + if(!count++) + min = max = n; + else { + if(n < min) + min = n; + if(n > max) + max = n; + } + } + + r->dview[d] = (STORAGE_POINT) { + .sum = sum, + .count = count, + .min = min, + .max = max, + .anomaly_count = (size_t)(ars * (NETDATA_DOUBLE)count), + }; + } +} + +static RRDR *rrd2rrdr_group_by_finalize(RRDR *r_tmp) { + QUERY_TARGET *qt = r_tmp->internal.qt; + RRDR_OPTIONS options = qt->window.options; + + if(!r_tmp->group_by.r) { + // v1 query + if(options & RRDR_OPTION_PERCENTAGE) + rrd2rrdr_convert_to_percentage(r_tmp); + return r_tmp; + } + // v2 query + + // do the additional passes on RRDRs + RRDR *last_r = r_tmp->group_by.r; + rrdr2rrdr_group_by_calculate_percentage_of_instance(last_r); + + RRDR *r = last_r->group_by.r; + size_t pass = 0; + while(r) { + pass++; + for(size_t d = 0; d < last_r->d ;d++) { + rrd2rrdr_group_by_add_metric(r, last_r->dgbs[d], last_r, d, + qt->request.group_by[pass].aggregation, + &last_r->dqp[d], pass); + } + rrdr2rrdr_group_by_calculate_percentage_of_instance(r); + + last_r = r; + r = last_r->group_by.r; + } + + // free all RRDRs except the last one + r = r_tmp; + while(r != last_r) { + r_tmp = r->group_by.r; + r->group_by.r = NULL; + rrdr_free(r->internal.owa, r); + r = r_tmp; + } + r = last_r; + + // find the final aggregation + RRDR_GROUP_BY_FUNCTION aggregation = qt->request.group_by[0].aggregation; + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) + if(qt->request.group_by[g].group_by != RRDR_GROUP_BY_NONE) + aggregation = qt->request.group_by[g].aggregation; + + if(!(options & RRDR_OPTION_RETURN_RAW) && r->partial_data_trimming.expected_after < qt->window.before) + rrdr2rrdr_group_by_partial_trimming(r); + + // apply averaging, remove RRDR_VALUE_EMPTY, find the non-zero dimensions, min and max + size_t global_min_max_values = 0; + size_t dimensions_nonzero = 0; + NETDATA_DOUBLE global_min = NAN, global_max = NAN; + for (size_t d = 0; d < r->d; d++) { + if (unlikely(!(r->od[d] & RRDR_DIMENSION_QUERIED))) + continue; + + size_t points_nonzero = 0; + NETDATA_DOUBLE min = 0, max = 0, sum = 0, ars = 0; + size_t count = 0; + + for(size_t i = 0; i != r->n ;i++) { + size_t idx = i * r->d + d; + + NETDATA_DOUBLE *cn = &r->v[ idx ]; + RRDR_VALUE_FLAGS *co = &r->o[ idx ]; + NETDATA_DOUBLE *ar = &r->ar[ idx ]; + uint32_t gbc = r->gbc[ idx ]; + + if(likely(gbc)) { + *co &= ~RRDR_VALUE_EMPTY; + + if(gbc != r->dgbc[d]) + *co |= RRDR_VALUE_PARTIAL; + + NETDATA_DOUBLE n; + + sum += *cn; + ars += *ar; + + if(aggregation == RRDR_GROUP_BY_FUNCTION_AVERAGE && !query_target_aggregatable(qt)) + n = (*cn /= gbc); + else + n = *cn; + + if(!query_target_aggregatable(qt)) + *ar /= gbc; + + if(islessgreater(n, 0.0)) + points_nonzero++; + + if(unlikely(!count)) + min = max = n; + else { + if(n < min) + min = n; + + if(n > max) + max = n; + } + + if(unlikely(!global_min_max_values++)) + global_min = global_max = n; + else { + if(n < global_min) + global_min = n; + + if(n > global_max) + global_max = n; + } + + count += gbc; + } + } + + if(points_nonzero) { + r->od[d] |= RRDR_DIMENSION_NONZERO; + dimensions_nonzero++; + } + + r->dview[d] = (STORAGE_POINT) { + .sum = sum, + .count = count, + .min = min, + .max = max, + .anomaly_count = (size_t)(ars * RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER / 100.0), + }; + } + + r->view.min = global_min; + r->view.max = global_max; + + if(!dimensions_nonzero && (qt->window.options & RRDR_OPTION_NONZERO)) { + // all dimensions are zero + // remove the nonzero option + qt->window.options &= ~RRDR_OPTION_NONZERO; + } + + if(options & RRDR_OPTION_PERCENTAGE && !(options & RRDR_OPTION_RETURN_RAW)) + rrd2rrdr_convert_to_percentage(r); + + // update query instance counts in query host and query context + { + size_t h = 0, c = 0, i = 0; + for(; h < qt->nodes.used ; h++) { + QUERY_NODE *qn = &qt->nodes.array[h]; + + for(; c < qt->contexts.used ;c++) { + QUERY_CONTEXT *qc = &qt->contexts.array[c]; + + if(!rrdcontext_acquired_belongs_to_host(qc->rca, qn->rrdhost)) + break; + + for(; i < qt->instances.used ;i++) { + QUERY_INSTANCE *qi = &qt->instances.array[i]; + + if(!rrdinstance_acquired_belongs_to_context(qi->ria, qc->rca)) + break; + + if(qi->metrics.queried) { + qc->instances.queried++; + qn->instances.queried++; + } + else if(qi->metrics.failed) { + qc->instances.failed++; + qn->instances.failed++; + } + } + } + } + } + + return r; +} + +// ---------------------------------------------------------------------------- +// query entry point + RRDR *rrd2rrdr_legacy( ONEWAYALLOC *owa, RRDSET *st, size_t points, time_t after, time_t before, - RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions, - const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source, + RRDR_TIME_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions, + const char *group_options, time_t timeout_ms, size_t tier, QUERY_SOURCE query_source, STORAGE_PRIORITY priority) { QUERY_TARGET_REQUEST qtr = { + .version = 1, .st = st, .points = points, .after = after, .before = before, - .group_method = group_method, + .time_group_method = group_method, .resampling_time = resampling_time, .options = options, .dimensions = dimensions, - .group_options = group_options, - .timeout = timeout, + .time_group_options = group_options, + .timeout_ms = timeout_ms, .tier = tier, .query_source = query_source, .priority = priority, }; - return rrd2rrdr(owa, query_target_create(&qtr)); + QUERY_TARGET *qt = query_target_create(&qtr); + RRDR *r = rrd2rrdr(owa, qt); + if(!r) { + query_target_release(qt); + return NULL; + } + + r->internal.release_with_rrdr_qt = qt; + return r; } RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) { - if(!qt) + if(!qt || !owa) return NULL; - if(!owa) { - query_target_release(qt); - return NULL; - } - // qt.window members are the WANTED ones. // qt.request members are the REQUESTED ones. - RRDR *r = rrdr_create(owa, qt); - if(unlikely(!r)) { - internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, points=%zu", - qt->id, qt->window.after, qt->window.before, qt->window.points); + RRDR *r_tmp = rrd2rrdr_group_by_initialize(owa, qt); + if(!r_tmp) return NULL; - } - if(unlikely(!r->d || !qt->window.points)) { - internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%ld, before=%ld, points=%zu", - qt->id, qt->window.after, qt->window.before, qt->window.points); - return r; - } + // the RRDR we group-by at + RRDR *r = (r_tmp->group_by.r) ? r_tmp->group_by.r : r_tmp; + + // the final RRDR to return to callers + RRDR *last_r = r_tmp; + while(last_r->group_by.r) + last_r = last_r->group_by.r; if(qt->window.relative) - r->result_options |= RRDR_RESULT_OPTION_RELATIVE; + last_r->view.flags |= RRDR_RESULT_FLAG_RELATIVE; else - r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; - - // ------------------------------------------------------------------------- - // initialize RRDR - - r->group = qt->window.group; - r->update_every = (int) (qt->window.group * qt->window.query_granularity); - r->before = qt->window.before; - r->after = qt->window.after; - r->internal.points_wanted = qt->window.points; - r->internal.resampling_group = qt->window.resampling_group; - r->internal.resampling_divisor = qt->window.resampling_divisor; - r->internal.query_options = qt->window.options; + last_r->view.flags |= RRDR_RESULT_FLAG_ABSOLUTE; // ------------------------------------------------------------------------- // assign the processor functions - rrdr_set_grouping_function(r, qt->window.group_method); + rrdr_set_grouping_function(r_tmp, qt->window.time_group_method); // allocate any memory required by the grouping method - r->internal.grouping_create(r, qt->window.group_options); + r_tmp->time_grouping.create(r_tmp, qt->window.time_group_options); // ------------------------------------------------------------------------- // do the work for each dimension @@ -2239,122 +3529,207 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) { size_t max_rows = 0; long dimensions_used = 0, dimensions_nonzero = 0; - struct timeval query_start_time; - struct timeval query_current_time; - if (qt->request.timeout) - now_realtime_timeval(&query_start_time); - size_t last_db_points_read = 0; size_t last_result_points_generated = 0; - QUERY_ENGINE_OPS **ops = onewayalloc_callocz(r->internal.owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *)); + internal_fatal(released_ops, "QUERY: released_ops should be NULL when the query starts"); - size_t capacity = libuv_worker_threads * 2; + QUERY_ENGINE_OPS **ops = NULL; + if(qt->query.used) + ops = onewayalloc_callocz(owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *)); + + size_t capacity = libuv_worker_threads * 10; size_t max_queries_to_prepare = (qt->query.used > (capacity - 1)) ? (capacity - 1) : qt->query.used; size_t queries_prepared = 0; while(queries_prepared < max_queries_to_prepare) { // preload another query - ops[queries_prepared] = rrd2rrdr_query_prep(r, queries_prepared); + ops[queries_prepared] = rrd2rrdr_query_ops_prep(r_tmp, queries_prepared); queries_prepared++; } - for(size_t c = 0, max = qt->query.used; c < max ; c++) { + QUERY_NODE *last_qn = NULL; + usec_t last_ut = now_monotonic_usec(); + usec_t last_qn_ut = last_ut; + + for(size_t d = 0; d < qt->query.used ; d++) { + QUERY_METRIC *qm = query_metric(qt, d); + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); + QUERY_INSTANCE *qi = query_instance(qt, qm->link.query_instance_id); + QUERY_CONTEXT *qc = query_context(qt, qm->link.query_context_id); + QUERY_NODE *qn = query_node(qt, qm->link.query_node_id); - if(queries_prepared < max) { + usec_t now_ut = last_ut; + if(qn != last_qn) { + if(last_qn) + last_qn->duration_ut = now_ut - last_qn_ut; + + last_qn = qn; + last_qn_ut = now_ut; + } + + if(queries_prepared < qt->query.used) { // preload another query - ops[queries_prepared] = rrd2rrdr_query_prep(r, queries_prepared); + ops[queries_prepared] = rrd2rrdr_query_ops_prep(r_tmp, queries_prepared); queries_prepared++; } + size_t dim_in_rrdr_tmp = (r_tmp != r) ? 0 : d; + // set the query target dimension options to rrdr - r->od[c] = qt->query.array[c].dimension.options; + r_tmp->od[dim_in_rrdr_tmp] = qm->status; // reset the grouping for the new dimension - r->internal.grouping_reset(r); + r_tmp->time_grouping.reset(r_tmp); - if(ops[c]) { - r->od[c] |= RRDR_DIMENSION_QUERIED; - rrd2rrdr_query_execute(r, c, ops[c]); + if(ops[d]) { + rrd2rrdr_query_execute(r_tmp, dim_in_rrdr_tmp, ops[d]); + r_tmp->od[dim_in_rrdr_tmp] |= RRDR_DIMENSION_QUERIED; + + now_ut = now_monotonic_usec(); + qm->duration_ut = now_ut - last_ut; + last_ut = now_ut; + + if(r_tmp != r) { + // copy back whatever got updated from the temporary r + + // the query updates RRDR_DIMENSION_NONZERO + qm->status = r_tmp->od[dim_in_rrdr_tmp]; + + // the query updates these + r->view.min = r_tmp->view.min; + r->view.max = r_tmp->view.max; + r->view.after = r_tmp->view.after; + r->view.before = r_tmp->view.before; + r->rows = r_tmp->rows; + + rrd2rrdr_group_by_add_metric(r, qm->grouped_as.first_slot, r_tmp, dim_in_rrdr_tmp, + qt->request.group_by[0].aggregation, &qm->query_points, 0); + } + + rrd2rrdr_query_ops_release(ops[d]); // reuse this ops allocation + ops[d] = NULL; + + qi->metrics.queried++; + qc->metrics.queried++; + qn->metrics.queried++; + + qd->status |= QUERY_STATUS_QUERIED; + qm->status |= RRDR_DIMENSION_QUERIED; + + if(qt->request.version >= 2) { + // we need to make the query points positive now + // since we will aggregate it across multiple dimensions + storage_point_make_positive(qm->query_points); + storage_point_merge_to(qi->query_points, qm->query_points); + storage_point_merge_to(qc->query_points, qm->query_points); + storage_point_merge_to(qn->query_points, qm->query_points); + storage_point_merge_to(qt->query_points, qm->query_points); + } } - else + else { + qi->metrics.failed++; + qc->metrics.failed++; + qn->metrics.failed++; + + qd->status |= QUERY_STATUS_FAILED; + qm->status |= RRDR_DIMENSION_FAILED; + continue; + } global_statistics_rrdr_query_completed( 1, - r->internal.db_points_read - last_db_points_read, - r->internal.result_points_generated - last_result_points_generated, + r_tmp->stats.db_points_read - last_db_points_read, + r_tmp->stats.result_points_generated - last_result_points_generated, qt->request.query_source); - last_db_points_read = r->internal.db_points_read; - last_result_points_generated = r->internal.result_points_generated; - - if (qt->request.timeout) - now_realtime_timeval(&query_current_time); + last_db_points_read = r_tmp->stats.db_points_read; + last_result_points_generated = r_tmp->stats.result_points_generated; - if(r->od[c] & RRDR_DIMENSION_NONZERO) + if(qm->status & RRDR_DIMENSION_NONZERO) dimensions_nonzero++; // verify all dimensions are aligned if(unlikely(!dimensions_used)) { - min_before = r->before; - max_after = r->after; + min_before = r->view.before; + max_after = r->view.after; max_rows = r->rows; } else { - if(r->after != max_after) { + if(r->view.after != max_after) { internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - string2str(qt->query.array[c].dimension.id), (size_t)max_after, string2str(qt->query.array[c].dimension.name), (size_t)r->after); + rrdinstance_acquired_id(qi->ria), (size_t)max_after, rrdmetric_acquired_id(qd->rma), (size_t)r->view.after); - r->after = (r->after > max_after) ? r->after : max_after; + r->view.after = (r->view.after > max_after) ? r->view.after : max_after; } - if(r->before != min_before) { + if(r->view.before != min_before) { internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - string2str(qt->query.array[c].dimension.id), (size_t)min_before, string2str(qt->query.array[c].dimension.name), (size_t)r->before); + rrdinstance_acquired_id(qi->ria), (size_t)min_before, rrdmetric_acquired_id(qd->rma), (size_t)r->view.before); - r->before = (r->before < min_before) ? r->before : min_before; + r->view.before = (r->view.before < min_before) ? r->view.before : min_before; } if(r->rows != max_rows) { internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - string2str(qt->query.array[c].dimension.id), (size_t)max_rows, string2str(qt->query.array[c].dimension.name), (size_t)r->rows); + rrdinstance_acquired_id(qi->ria), (size_t)max_rows, rrdmetric_acquired_id(qd->rma), (size_t)r->rows); r->rows = (r->rows > max_rows) ? r->rows : max_rows; } } dimensions_used++; - if (qt->request.timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout) { + + bool cancel = false; + if (qt->request.interrupt_callback && qt->request.interrupt_callback(qt->request.interrupt_callback_data)) { + cancel = true; + log_access("QUERY INTERRUPTED"); + } + + if (qt->request.timeout_ms && ((NETDATA_DOUBLE)(now_ut - qt->timings.received_ut) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout_ms) { + cancel = true; log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)", - (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout); - r->result_options |= RRDR_RESULT_OPTION_CANCEL; + (NETDATA_DOUBLE)(now_ut - qt->timings.received_ut) / 1000.0, (long long)qt->request.timeout_ms); + } - for(size_t i = c + 1; i < queries_prepared ; i++) { - if(ops[i]) + if(cancel) { + r->view.flags |= RRDR_RESULT_FLAG_CANCEL; + + for(size_t i = d + 1; i < queries_prepared ; i++) { + if(ops[i]) { query_planer_finalize_remaining_plans(ops[i]); + rrd2rrdr_query_ops_release(ops[i]); + ops[i] = NULL; + } } break; } } + // free all resources used by the grouping method + r_tmp->time_grouping.free(r_tmp); + + // get the final RRDR to send to the caller + r = rrd2rrdr_group_by_finalize(r_tmp); + #ifdef NETDATA_INTERNAL_CHECKS - if (dimensions_used) { + if (dimensions_used && !(r->view.flags & RRDR_RESULT_FLAG_CANCEL)) { if(r->internal.log) - rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.time_group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, qt->request.after, qt->window.before, qt->request.before, qt->request.points, qt->window.points, /*after_slot, before_slot,*/ r->internal.log); if(r->rows != qt->window.points) - rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.time_group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, qt->request.after, qt->window.before, qt->request.before, qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'"); - if(qt->window.aligned && (r->before % (qt->window.group * qt->window.query_granularity)) != 0) - rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, - qt->window.after, qt->request.after, qt->window.before,qt->request.before, + if(qt->window.aligned && (r->view.before % query_view_update_every(qt)) != 0) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.time_group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + qt->window.after, qt->request.after, qt->window.before, qt->request.before, qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required"); @@ -2362,21 +3737,21 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) { //if(qt->window.aligned && (r->after % group) != 0) // rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); - if(r->before != qt->window.before) - rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + if(r->view.before != qt->window.before) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.time_group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, qt->request.after, qt->window.before, qt->request.before, qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'"); - if(r->before != qt->window.before) - rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + if(r->view.before != qt->window.before) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.time_group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, qt->request.after, qt->window.before, qt->request.before, qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'"); // reported 'after' varies, depending on group - if(r->after != qt->window.after) - rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + if(r->view.after != qt->window.after) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.time_group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, qt->request.after, qt->window.before, qt->request.before, qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'"); @@ -2384,26 +3759,21 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) { } #endif - // free all resources used by the grouping method - r->internal.grouping_free(r); + // free the query pipelining ops + for(size_t d = 0; d < qt->query.used ; d++) { + rrd2rrdr_query_ops_release(ops[d]); + ops[d] = NULL; + } + rrd2rrdr_query_ops_freeall(r); + internal_fatal(released_ops, "QUERY: released_ops should be NULL when the query ends"); - if(likely(dimensions_used)) { + onewayalloc_freez(owa, ops); + + if(likely(dimensions_used && (qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero)) // when all the dimensions are zero, we should return all of them - if (unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero && - !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) { - // all the dimensions are zero - // mark them as NONZERO to send them all - for (size_t c = 0, max = qt->query.used; c < max; c++) { - if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; - if (unlikely(!(r->od[c] & RRDR_DIMENSION_QUERIED))) continue; - r->od[c] |= RRDR_DIMENSION_NONZERO; - } - } + qt->window.options &= ~RRDR_OPTION_NONZERO; - return r; - } + qt->timings.executed_ut = now_monotonic_usec(); - // we couldn't query any dimension - rrdr_free(owa, r); - return NULL; + return r; } diff --git a/web/api/queries/query.h b/web/api/queries/query.h index ebad5a1f..e6fdcfbe 100644 --- a/web/api/queries/query.h +++ b/web/api/queries/query.h @@ -7,7 +7,7 @@ extern "C" { #endif -typedef enum rrdr_grouping { +typedef enum rrdr_time_grouping { RRDR_GROUPING_UNDEFINED = 0, RRDR_GROUPING_AVERAGE, RRDR_GROUPING_MIN, @@ -17,7 +17,7 @@ typedef enum rrdr_grouping { RRDR_GROUPING_TRIMMED_MEAN1, RRDR_GROUPING_TRIMMED_MEAN2, RRDR_GROUPING_TRIMMED_MEAN3, - RRDR_GROUPING_TRIMMED_MEAN5, + RRDR_GROUPING_TRIMMED_MEAN, RRDR_GROUPING_TRIMMED_MEAN10, RRDR_GROUPING_TRIMMED_MEAN15, RRDR_GROUPING_TRIMMED_MEAN20, @@ -36,7 +36,7 @@ typedef enum rrdr_grouping { RRDR_GROUPING_PERCENTILE75, RRDR_GROUPING_PERCENTILE80, RRDR_GROUPING_PERCENTILE90, - RRDR_GROUPING_PERCENTILE95, + RRDR_GROUPING_PERCENTILE, RRDR_GROUPING_PERCENTILE97, RRDR_GROUPING_PERCENTILE98, RRDR_GROUPING_PERCENTILE99, @@ -45,12 +45,50 @@ typedef enum rrdr_grouping { RRDR_GROUPING_SES, RRDR_GROUPING_DES, RRDR_GROUPING_COUNTIF, -} RRDR_GROUPING; +} RRDR_TIME_GROUPING; -const char *group_method2string(RRDR_GROUPING group); -void web_client_api_v1_init_grouping(void); -RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def); -const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group); +const char *time_grouping_method2string(RRDR_TIME_GROUPING group); +void time_grouping_init(void); +RRDR_TIME_GROUPING time_grouping_parse(const char *name, RRDR_TIME_GROUPING def); +const char *time_grouping_tostring(RRDR_TIME_GROUPING group); + +typedef enum rrdr_group_by { + RRDR_GROUP_BY_NONE = 0, + RRDR_GROUP_BY_SELECTED = (1 << 0), + RRDR_GROUP_BY_DIMENSION = (1 << 1), + RRDR_GROUP_BY_INSTANCE = (1 << 2), + RRDR_GROUP_BY_LABEL = (1 << 3), + RRDR_GROUP_BY_NODE = (1 << 4), + RRDR_GROUP_BY_CONTEXT = (1 << 5), + RRDR_GROUP_BY_UNITS = (1 << 6), + RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE = (1 << 7), +} RRDR_GROUP_BY; + +#define SUPPORTED_GROUP_BY_METHODS (\ + RRDR_GROUP_BY_SELECTED |\ + RRDR_GROUP_BY_DIMENSION |\ + RRDR_GROUP_BY_INSTANCE |\ + RRDR_GROUP_BY_LABEL |\ + RRDR_GROUP_BY_NODE |\ + RRDR_GROUP_BY_CONTEXT |\ + RRDR_GROUP_BY_UNITS |\ + RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE \ +) + +struct web_buffer; + +RRDR_GROUP_BY group_by_parse(char *s); +void buffer_json_group_by_to_array(struct web_buffer *wb, RRDR_GROUP_BY group_by); + +typedef enum rrdr_group_by_function { + RRDR_GROUP_BY_FUNCTION_AVERAGE = 0, + RRDR_GROUP_BY_FUNCTION_MIN, + RRDR_GROUP_BY_FUNCTION_MAX, + RRDR_GROUP_BY_FUNCTION_SUM, +} RRDR_GROUP_BY_FUNCTION; + +RRDR_GROUP_BY_FUNCTION group_by_aggregate_function_parse(const char *s); +const char *group_by_aggregate_function_to_string(RRDR_GROUP_BY_FUNCTION group_by_function); #ifdef __cplusplus } diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c index 676224c9..2a001689 100644 --- a/web/api/queries/rrdr.c +++ b/web/api/queries/rrdr.c @@ -61,41 +61,86 @@ static void rrdr_dump(RRDR *r) inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) { if(unlikely(!r)) return; - query_target_release(r->internal.qt); + for(size_t d = 0; d < r->d ;d++) { + string_freez(r->di[d]); + string_freez(r->dn[d]); + string_freez(r->du[d]); + } + + query_target_release(r->internal.release_with_rrdr_qt); + onewayalloc_freez(owa, r->t); onewayalloc_freez(owa, r->v); + onewayalloc_freez(owa, r->vh); onewayalloc_freez(owa, r->o); onewayalloc_freez(owa, r->od); + onewayalloc_freez(owa, r->di); + onewayalloc_freez(owa, r->dn); + onewayalloc_freez(owa, r->du); + onewayalloc_freez(owa, r->dp); + onewayalloc_freez(owa, r->dview); + onewayalloc_freez(owa, r->dqp); onewayalloc_freez(owa, r->ar); + onewayalloc_freez(owa, r->gbc); + onewayalloc_freez(owa, r->dgbc); + onewayalloc_freez(owa, r->dgbs); + + if(r->dl) { + for(size_t d = 0; d < r->d ;d++) + dictionary_destroy(r->dl[d]); + + onewayalloc_freez(owa, r->dl); + } + + dictionary_destroy(r->label_keys); + + if(r->group_by.r) { + // prevent accidental infinite recursion + r->group_by.r->group_by.r = NULL; + + // do not release qt twice + r->group_by.r->internal.qt = NULL; + + rrdr_free(owa, r->group_by.r); + } + onewayalloc_freez(owa, r); } -RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt) { - if(unlikely(!qt || !qt->query.used || !qt->window.points)) +RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt, size_t dimensions, size_t points) { + if(unlikely(!qt)) return NULL; - size_t dimensions = qt->query.used; - size_t points = qt->window.points; - // create the rrdr RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR)); r->internal.owa = owa; r->internal.qt = qt; - r->before = qt->window.before; - r->after = qt->window.after; - r->internal.points_wanted = qt->window.points; + r->view.before = qt->window.before; + r->view.after = qt->window.after; + r->time_grouping.points_wanted = points; r->d = (int)dimensions; r->n = (int)points; - r->t = onewayalloc_callocz(owa, points, sizeof(time_t)); - r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE)); - r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS)); - r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE)); - r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS)); + if(points && dimensions) { + r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE)); + r->o = onewayalloc_mallocz(owa, points * dimensions * sizeof(RRDR_VALUE_FLAGS)); + r->ar = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE)); + } + + if(points) { + r->t = onewayalloc_callocz(owa, points, sizeof(time_t)); + } + + if(dimensions) { + r->od = onewayalloc_mallocz(owa, dimensions * sizeof(RRDR_DIMENSION_FLAGS)); + r->di = onewayalloc_callocz(owa, dimensions, sizeof(STRING *)); + r->dn = onewayalloc_callocz(owa, dimensions, sizeof(STRING *)); + r->du = onewayalloc_callocz(owa, dimensions, sizeof(STRING *)); + } - r->group = 1; - r->update_every = 1; + r->view.group = 1; + r->view.update_every = 1; return r; } diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index 2d982b13..c57be67f 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -18,111 +18,152 @@ typedef enum tier_query_fetch { } TIER_QUERY_FETCH; typedef enum rrdr_options { - RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values - RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest) - RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing - RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum - RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates - RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates - RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros - RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array - RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs - RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result - RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes - RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total - RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes - RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign - RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs - RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names - RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database - RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points - RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number - RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes - RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap - RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query - RRDR_OPTION_ALL_DIMENSIONS = 0x00800000, // Return the full dimensions list - RRDR_OPTION_SHOW_PLAN = 0x01000000, // Return the query plan in jsonwrap + RRDR_OPTION_NONZERO = (1 << 0), // don't output dimensions with just zero values + RRDR_OPTION_REVERSED = (1 << 1), // output the rows in reverse order (oldest to newest) + RRDR_OPTION_ABSOLUTE = (1 << 2), // values positive, for DATASOURCE_SSV before summing + RRDR_OPTION_MIN2MAX = (1 << 3), // when adding dimensions, use max - min, instead of sum + RRDR_OPTION_SECONDS = (1 << 4), // output seconds, instead of dates + RRDR_OPTION_MILLISECONDS = (1 << 5), // output milliseconds, instead of dates + RRDR_OPTION_NULL2ZERO = (1 << 6), // do not show nulls, convert them to zeros + RRDR_OPTION_OBJECTSROWS = (1 << 7), // each row of values should be an object, not an array + RRDR_OPTION_GOOGLE_JSON = (1 << 8), // comply with google JSON/JSONP specs + RRDR_OPTION_JSON_WRAP = (1 << 9), // wrap the response in a JSON header with info about the result + RRDR_OPTION_LABEL_QUOTES = (1 << 10), // in CSV output, wrap header labels in double quotes + RRDR_OPTION_PERCENTAGE = (1 << 11), // give values as percentage of total + RRDR_OPTION_NOT_ALIGNED = (1 << 12), // do not align charts for persistent timeframes + RRDR_OPTION_DISPLAY_ABS = (1 << 13), // for badges, display the absolute value, but calculate colors with sign + RRDR_OPTION_MATCH_IDS = (1 << 14), // when filtering dimensions, match only IDs + RRDR_OPTION_MATCH_NAMES = (1 << 15), // when filtering dimensions, match only names + RRDR_OPTION_NATURAL_POINTS = (1 << 16), // return the natural points of the database + RRDR_OPTION_VIRTUAL_POINTS = (1 << 17), // return virtual points + RRDR_OPTION_ANOMALY_BIT = (1 << 18), // Return the anomaly bit stored in each collected_number + RRDR_OPTION_RETURN_RAW = (1 << 19), // Return raw data for aggregating across multiple nodes + RRDR_OPTION_RETURN_JWAR = (1 << 20), // Return anomaly rates in jsonwrap + RRDR_OPTION_SELECTED_TIER = (1 << 21), // Use the selected tier for the query + RRDR_OPTION_ALL_DIMENSIONS = (1 << 22), // Return the full dimensions list + RRDR_OPTION_SHOW_DETAILS = (1 << 23), // v2 returns detailed object tree + RRDR_OPTION_DEBUG = (1 << 24), // v2 returns request description + RRDR_OPTION_MINIFY = (1 << 25), // remove JSON spaces and newlines from JSON output + RRDR_OPTION_GROUP_BY_LABELS = (1 << 26), // v2 returns flattened labels per dimension of the chart // internal ones - not to be exposed to the API - RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate - RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION + RRDR_OPTION_HEALTH_RSRVD1 = (1 << 30), // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION + RRDR_OPTION_INTERNAL_AR = (1 << 31), // internal use only, to let the formatters know we want to render the anomaly rate } RRDR_OPTIONS; -typedef enum rrdr_value_flag { - RRDR_VALUE_NOTHING = 0x00, // no flag set (a good default) - RRDR_VALUE_EMPTY = 0x01, // the database value is empty - RRDR_VALUE_RESET = 0x02, // the database value is marked as reset (overflown) +typedef enum __attribute__ ((__packed__)) rrdr_value_flag { + + // IMPORTANT: + // THIS IS AN AGREED BIT MAP BETWEEN AGENT, CLOUD FRONT-END AND CLOUD BACK-END + // DO NOT CHANGE THE MAPPINGS ! + + RRDR_VALUE_NOTHING = 0, // no flag set (a good default) + RRDR_VALUE_EMPTY = (1 << 0), // the database value is empty + RRDR_VALUE_RESET = (1 << 1), // the database value is marked as reset (overflown) + RRDR_VALUE_PARTIAL = (1 << 2), // the database provides partial data about this point (used in group-by) } RRDR_VALUE_FLAGS; -typedef enum rrdr_dimension_flag { - RRDR_DIMENSION_DEFAULT = 0x00, - RRDR_DIMENSION_HIDDEN = 0x04, // the dimension is hidden (not to be presented to callers) - RRDR_DIMENSION_NONZERO = 0x08, // the dimension is non zero (contains non-zero values) - RRDR_DIMENSION_QUERIED = 0x10, // the dimension is selected for evaluation in this RRDR +typedef enum __attribute__ ((__packed__)) rrdr_dimension_flag { + RRDR_DIMENSION_DEFAULT = 0, + RRDR_DIMENSION_HIDDEN = (1 << 0), // the dimension is hidden (not to be presented to callers) + RRDR_DIMENSION_NONZERO = (1 << 1), // the dimension is non zero (contains non-zero values) + RRDR_DIMENSION_SELECTED = (1 << 2), // the dimension has been selected for query + RRDR_DIMENSION_QUERIED = (1 << 3), // the dimension has been queried + RRDR_DIMENSION_FAILED = (1 << 4), // the dimension failed to be queried + RRDR_DIMENSION_GROUPED = (1 << 5), // the dimension has been grouped in this RRDR } RRDR_DIMENSION_FLAGS; // RRDR result options -typedef enum rrdr_result_flags { - RRDR_RESULT_OPTION_ABSOLUTE = 0x00000001, // the query uses absolute time-frames - // (can be cached by browsers and proxies) - RRDR_RESULT_OPTION_RELATIVE = 0x00000002, // the query uses relative time-frames - // (should not to be cached by browsers and proxies) - RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames - RRDR_RESULT_OPTION_CANCEL = 0x00000008, // the query needs to be cancelled -} RRDR_RESULT_OPTIONS; +typedef enum __attribute__ ((__packed__)) rrdr_result_flags { + RRDR_RESULT_FLAG_ABSOLUTE = (1 << 0), // the query uses absolute time-frames + // (can be cached by browsers and proxies) + RRDR_RESULT_FLAG_RELATIVE = (1 << 1), // the query uses relative time-frames + // (should not to be cached by browsers and proxies) + RRDR_RESULT_FLAG_CANCEL = (1 << 2), // the query needs to be cancelled +} RRDR_RESULT_FLAGS; -typedef struct rrdresult { - RRDR_RESULT_OPTIONS result_options; // RRDR_RESULT_OPTION_* +#define RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER 1000.0 +typedef struct rrdresult { size_t d; // the number of dimensions - size_t n; // the number of values in the arrays - size_t rows; // the number of rows used + size_t n; // the number of values in the arrays (number of points per dimension) + size_t rows; // the number of actual rows used RRDR_DIMENSION_FLAGS *od; // the options for the dimensions + STRING **di; // array of d dimension ids + STRING **dn; // array of d dimension names + STRING **du; // array of d dimension units + uint32_t *dgbs; // array of d dimension group by slots - NOT ALLOCATED when RRDR is created + uint32_t *dgbc; // array of d dimension group by counts - NOT ALLOCATED when RRDR is created + uint32_t *dp; // array of d dimension priority - NOT ALLOCATED when RRDR is created + DICTIONARY **dl; // array of d dimension labels - NOT ALLOCATED when RRDR is created + STORAGE_POINT *dqp; // array of d dimensions query points - NOT ALLOCATED when RRDR is created + STORAGE_POINT *dview; // array of d dimensions group by view - NOT ALLOCATED when RRDR is created + NETDATA_DOUBLE *vh; // array of n x d hidden values, while grouping - NOT ALLOCATED when RRDR is created + + DICTIONARY *label_keys; + time_t *t; // array of n timestamps NETDATA_DOUBLE *v; // array n x d values RRDR_VALUE_FLAGS *o; // array n x d options for each value returned NETDATA_DOUBLE *ar; // array n x d of anomaly rates (0 - 100) + uint32_t *gbc; // array n x d of group by count - NOT ALLOCATED when RRDR is created - size_t group; // how many collected values were grouped for each row - time_t update_every; // what is the suggested update frequency in seconds - - NETDATA_DOUBLE min; - NETDATA_DOUBLE max; + struct { + size_t group; // how many collected values were grouped for each row - NEEDED BY GROUPING FUNCTIONS + time_t after; + time_t before; + time_t update_every; // what is the suggested update frequency in seconds + NETDATA_DOUBLE min; + NETDATA_DOUBLE max; + RRDR_RESULT_FLAGS flags; // RRDR_RESULT_FLAG_* + } view; - time_t before; - time_t after; + struct { + size_t db_points_read; + size_t result_points_generated; + } stats; - // internal rrd2rrdr() members below this point struct { - ONEWAYALLOC *owa; // the allocator used - struct query_target *qt; // the QUERY_TARGET + void *data; // the internal data of the grouping function - RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query) + // grouping function pointers + RRDR_TIME_GROUPING add_flush; + void (*create)(struct rrdresult *r, const char *options); + void (*reset)(struct rrdresult *r); + void (*free)(struct rrdresult *r); + void (*add)(struct rrdresult *r, NETDATA_DOUBLE value); + NETDATA_DOUBLE (*flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + + TIER_QUERY_FETCH tier_query_fetch; // which value to use from STORAGE_POINT size_t points_wanted; // used by SES and DES size_t resampling_group; // used by AVERAGE NETDATA_DOUBLE resampling_divisor; // used by AVERAGE + } time_grouping; - // grouping function pointers - void (*grouping_create)(struct rrdresult *r, const char *options); - void (*grouping_reset)(struct rrdresult *r); - void (*grouping_free)(struct rrdresult *r); - void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value); - NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); + struct { + struct rrdresult *r; + } group_by; - TIER_QUERY_FETCH tier_query_fetch; // which value to use from STORAGE_POINT - void *grouping_data; // the internal data of the grouping function + struct { + time_t max_update_every; + time_t expected_after; + time_t trimmed_after; + } partial_data_trimming; + + struct { + ONEWAYALLOC *owa; // the allocator used + struct query_target *qt; // the QUERY_TARGET + size_t contexts; // temp needed between json_wrapper_begin2() and json_wrapper_end2() + size_t queries_count; // temp needed to know if a query is the first executed #ifdef NETDATA_INTERNAL_CHECKS const char *log; #endif - // statistics - size_t db_points_read; - size_t result_points_generated; - size_t tier_points_read[RRD_STORAGE_TIERS]; + struct query_target *release_with_rrdr_qt; } internal; } RRDR; @@ -130,7 +171,7 @@ typedef struct rrdresult { #include "database/rrd.h" void rrdr_free(ONEWAYALLOC *owa, RRDR *r); -RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt); +RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt, size_t dimensions, size_t points); #include "../web_api_v1.h" #include "web/api/queries/query.h" @@ -138,14 +179,14 @@ RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt); RRDR *rrd2rrdr_legacy( ONEWAYALLOC *owa, RRDSET *st, size_t points, time_t after, time_t before, - RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions, - const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source, + RRDR_TIME_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions, + const char *group_options, time_t timeout_ms, size_t tier, QUERY_SOURCE query_source, STORAGE_PRIORITY priority); RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt); bool query_target_calculate_window(struct query_target *qt); -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before); +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr); #ifdef __cplusplus } diff --git a/web/api/queries/ses/README.md b/web/api/queries/ses/README.md index b835b812..56634d36 100644 --- a/web/api/queries/ses/README.md +++ b/web/api/queries/ses/README.md @@ -1,6 +1,10 @@ <!-- title: "Single (or Simple) Exponential Smoothing (`ses`)" +sidebar_label: "Single (or Simple) Exponential Smoothing (`ses`)" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/ses/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Single (or Simple) Exponential Smoothing (`ses`) diff --git a/web/api/queries/ses/ses.c b/web/api/queries/ses/ses.c index 5e94002c..39eb445a 100644 --- a/web/api/queries/ses/ses.c +++ b/web/api/queries/ses/ses.c @@ -6,85 +6,3 @@ // ---------------------------------------------------------------------------- // single exponential smoothing -struct grouping_ses { - NETDATA_DOUBLE alpha; - NETDATA_DOUBLE alpha_other; - NETDATA_DOUBLE level; - size_t count; -}; - -static size_t max_window_size = 15; - -void grouping_init_ses(void) { - long long ret = config_get_number(CONFIG_SECTION_WEB, "ses max window", (long long)max_window_size); - if(ret <= 1) { - config_set_number(CONFIG_SECTION_WEB, "ses max window", (long long)max_window_size); - } - else { - max_window_size = (size_t) ret; - } -} - -static inline NETDATA_DOUBLE window(RRDR *r, struct grouping_ses *g) { - (void)g; - - NETDATA_DOUBLE points; - if(r->group == 1) { - // provide a running DES - points = (NETDATA_DOUBLE)r->internal.points_wanted; - } - else { - // provide a SES with flush points - points = (NETDATA_DOUBLE)r->group; - } - - return (points > (NETDATA_DOUBLE)max_window_size) ? (NETDATA_DOUBLE)max_window_size : points; -} - -static inline void set_alpha(RRDR *r, struct grouping_ses *g) { - // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - // A commonly used value for alpha is 2 / (N + 1) - g->alpha = 2.0 / (window(r, g) + 1.0); - g->alpha_other = 1.0 - g->alpha; -} - -void grouping_create_ses(RRDR *r, const char *options __maybe_unused) { - struct grouping_ses *g = (struct grouping_ses *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_ses)); - set_alpha(r, g); - g->level = 0.0; - r->internal.grouping_data = g; -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_ses(RRDR *r) { - struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data; - g->level = 0.0; - g->count = 0; -} - -void grouping_free_ses(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data; - - if(unlikely(!g->count)) - g->level = value; - - g->level = g->alpha * value + g->alpha_other * g->level; - g->count++; -} - -NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_ses *g = (struct grouping_ses *)r->internal.grouping_data; - - if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - return 0.0; - } - - return g->level; -} diff --git a/web/api/queries/ses/ses.h b/web/api/queries/ses/ses.h index 79b09fbd..de8645ff 100644 --- a/web/api/queries/ses/ses.h +++ b/web/api/queries/ses/ses.h @@ -6,12 +6,87 @@ #include "../query.h" #include "../rrdr.h" -void grouping_init_ses(void); +struct tg_ses { + NETDATA_DOUBLE alpha; + NETDATA_DOUBLE alpha_other; + NETDATA_DOUBLE level; + size_t count; +}; -void grouping_create_ses(RRDR *r, const char *options __maybe_unused); -void grouping_reset_ses(RRDR *r); -void grouping_free_ses(RRDR *r); -void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +static size_t tg_ses_max_window_size = 15; + +static inline void tg_ses_init(void) { + long long ret = config_get_number(CONFIG_SECTION_WEB, "ses max tg_des_window", (long long)tg_ses_max_window_size); + if(ret <= 1) { + config_set_number(CONFIG_SECTION_WEB, "ses max tg_des_window", (long long)tg_ses_max_window_size); + } + else { + tg_ses_max_window_size = (size_t) ret; + } +} + +static inline NETDATA_DOUBLE tg_ses_window(RRDR *r, struct tg_ses *g) { + (void)g; + + NETDATA_DOUBLE points; + if(r->view.group == 1) { + // provide a running DES + points = (NETDATA_DOUBLE)r->time_grouping.points_wanted; + } + else { + // provide a SES with flush points + points = (NETDATA_DOUBLE)r->view.group; + } + + return (points > (NETDATA_DOUBLE)tg_ses_max_window_size) ? (NETDATA_DOUBLE)tg_ses_max_window_size : points; +} + +static inline void tg_ses_set_alpha(RRDR *r, struct tg_ses *g) { + // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + // A commonly used value for alpha is 2 / (N + 1) + g->alpha = 2.0 / (tg_ses_window(r, g) + 1.0); + g->alpha_other = 1.0 - g->alpha; +} + +static inline void tg_ses_create(RRDR *r, const char *options __maybe_unused) { + struct tg_ses *g = (struct tg_ses *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_ses)); + tg_ses_set_alpha(r, g); + g->level = 0.0; + r->time_grouping.data = g; +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_ses_reset(RRDR *r) { + struct tg_ses *g = (struct tg_ses *)r->time_grouping.data; + g->level = 0.0; + g->count = 0; +} + +static inline void tg_ses_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_ses_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_ses *g = (struct tg_ses *)r->time_grouping.data; + + if(unlikely(!g->count)) + g->level = value; + + g->level = g->alpha * value + g->alpha_other * g->level; + g->count++; +} + +static inline NETDATA_DOUBLE tg_ses_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_ses *g = (struct tg_ses *)r->time_grouping.data; + + if(unlikely(!g->count || !netdata_double_isnumber(g->level))) { + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + return 0.0; + } + + return g->level; +} #endif //NETDATA_API_QUERIES_SES_H diff --git a/web/api/queries/stddev/README.md b/web/api/queries/stddev/README.md index 2fca47d5..f0586a06 100644 --- a/web/api/queries/stddev/README.md +++ b/web/api/queries/stddev/README.md @@ -1,6 +1,10 @@ <!-- title: "standard deviation (`stddev`)" +sidebar_label: "standard deviation (`stddev`)" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/stddev/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # standard deviation (`stddev`) diff --git a/web/api/queries/stddev/stddev.c b/web/api/queries/stddev/stddev.c index 92a67b42..8f543119 100644 --- a/web/api/queries/stddev/stddev.c +++ b/web/api/queries/stddev/stddev.c @@ -6,123 +6,11 @@ // ---------------------------------------------------------------------------- // stddev -// this implementation comes from: -// https://www.johndcook.com/blog/standard_deviation/ - -struct grouping_stddev { - long count; - NETDATA_DOUBLE m_oldM, m_newM, m_oldS, m_newS; -}; - -void grouping_create_stddev(RRDR *r, const char *options __maybe_unused) { - r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_stddev)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_stddev(RRDR *r) { - struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - g->count = 0; -} - -void grouping_free_stddev(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - - g->count++; - - // See Knuth TAOCP vol 2, 3rd edition, page 232 - if (g->count == 1) { - g->m_oldM = g->m_newM = value; - g->m_oldS = 0.0; - } - else { - g->m_newM = g->m_oldM + (value - g->m_oldM) / g->count; - g->m_newS = g->m_oldS + (value - g->m_oldM) * (value - g->m_newM); - - // set up for next iteration - g->m_oldM = g->m_newM; - g->m_oldS = g->m_newS; - } -} - -static inline NETDATA_DOUBLE mean(struct grouping_stddev *g) { - return (g->count > 0) ? g->m_newM : 0.0; -} - -static inline NETDATA_DOUBLE variance(struct grouping_stddev *g) { - return ( (g->count > 1) ? g->m_newS/(NETDATA_DOUBLE)(g->count - 1) : 0.0 ); -} -static inline NETDATA_DOUBLE stddev(struct grouping_stddev *g) { - return sqrtndd(variance(g)); -} - -NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(likely(g->count > 1)) { - value = stddev(g); - - if(!netdata_double_isnumber(value)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - } - else if(g->count == 1) { - value = 0.0; - } - else { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - - grouping_reset_stddev(r); - - return value; -} - -// https://en.wikipedia.org/wiki/Coefficient_of_variation -NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(likely(g->count > 1)) { - NETDATA_DOUBLE m = mean(g); - value = 100.0 * stddev(g) / ((m < 0)? -m : m); - - if(unlikely(!netdata_double_isnumber(value))) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - } - else if(g->count == 1) { - // one value collected - value = 0.0; - } - else { - // no values collected - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - - grouping_reset_stddev(r); - - return value; -} - - /* * Mean = average * NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; + struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.grouping_data; NETDATA_DOUBLE value; @@ -149,7 +37,7 @@ NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options * It is not advised to use this version of variance directly * NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_stddev *g = (struct grouping_stddev *)r->internal.grouping_data; + struct grouping_stddev *g = (struct grouping_stddev *)r->grouping.grouping_data; NETDATA_DOUBLE value; diff --git a/web/api/queries/stddev/stddev.h b/web/api/queries/stddev/stddev.h index 4b8ffcd5..f7a1a06c 100644 --- a/web/api/queries/stddev/stddev.h +++ b/web/api/queries/stddev/stddev.h @@ -6,13 +6,115 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_stddev(RRDR *r, const char *options __maybe_unused); -void grouping_reset_stddev(RRDR *r); -void grouping_free_stddev(RRDR *r); -void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -// NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -// NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +// this implementation comes from: +// https://www.johndcook.com/blog/standard_deviation/ + +struct tg_stddev { + long count; + NETDATA_DOUBLE m_oldM, m_newM, m_oldS, m_newS; +}; + +static inline void tg_stddev_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_stddev)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_stddev_reset(RRDR *r) { + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data; + g->count = 0; +} + +static inline void tg_stddev_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_stddev_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data; + + g->count++; + + // See Knuth TAOCP vol 2, 3rd edition, page 232 + if (g->count == 1) { + g->m_oldM = g->m_newM = value; + g->m_oldS = 0.0; + } + else { + g->m_newM = g->m_oldM + (value - g->m_oldM) / g->count; + g->m_newS = g->m_oldS + (value - g->m_oldM) * (value - g->m_newM); + + // set up for next iteration + g->m_oldM = g->m_newM; + g->m_oldS = g->m_newS; + } +} + +static inline NETDATA_DOUBLE tg_stddev_mean(struct tg_stddev *g) { + return (g->count > 0) ? g->m_newM : 0.0; +} + +static inline NETDATA_DOUBLE tg_stddev_variance(struct tg_stddev *g) { + return ( (g->count > 1) ? g->m_newS/(NETDATA_DOUBLE)(g->count - 1) : 0.0 ); +} +static inline NETDATA_DOUBLE tg_stddev_stddev(struct tg_stddev *g) { + return sqrtndd(tg_stddev_variance(g)); +} + +static inline NETDATA_DOUBLE tg_stddev_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(likely(g->count > 1)) { + value = tg_stddev_stddev(g); + + if(!netdata_double_isnumber(value)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + } + else if(g->count == 1) { + value = 0.0; + } + else { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + tg_stddev_reset(r); + + return value; +} + +// https://en.wikipedia.org/wiki/Coefficient_of_variation +static inline NETDATA_DOUBLE tg_stddev_coefficient_of_variation_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_stddev *g = (struct tg_stddev *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(likely(g->count > 1)) { + NETDATA_DOUBLE m = tg_stddev_mean(g); + value = 100.0 * tg_stddev_stddev(g) / ((m < 0)? -m : m); + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + } + else if(g->count == 1) { + // one value collected + value = 0.0; + } + else { + // no values collected + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + tg_stddev_reset(r); + + return value; +} #endif //NETDATA_API_QUERIES_STDDEV_H diff --git a/web/api/queries/sum/README.md b/web/api/queries/sum/README.md index d4465bd8..62e18aca 100644 --- a/web/api/queries/sum/README.md +++ b/web/api/queries/sum/README.md @@ -1,6 +1,10 @@ <!-- title: "Sum" +sidebar_label: "Sum" custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/sum/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Sum diff --git a/web/api/queries/sum/sum.c b/web/api/queries/sum/sum.c index eec6e2ad..cf448421 100644 --- a/web/api/queries/sum/sum.c +++ b/web/api/queries/sum/sum.c @@ -5,51 +5,5 @@ // ---------------------------------------------------------------------------- // sum -struct grouping_sum { - NETDATA_DOUBLE sum; - size_t count; -}; - -void grouping_create_sum(RRDR *r, const char *options __maybe_unused) { - r->internal.grouping_data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_sum)); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_sum(RRDR *r) { - struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data; - g->sum = 0; - g->count = 0; -} - -void grouping_free_sum(RRDR *r) { - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data; - g->sum += value; - g->count++; -} - -NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_sum *g = (struct grouping_sum *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - - if(unlikely(!g->count)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else { - value = g->sum; - } - - g->sum = 0.0; - g->count = 0; - - return value; -} diff --git a/web/api/queries/sum/sum.h b/web/api/queries/sum/sum.h index 89878277..5e07f45d 100644 --- a/web/api/queries/sum/sum.h +++ b/web/api/queries/sum/sum.h @@ -6,10 +6,51 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_sum(RRDR *r, const char *options __maybe_unused); -void grouping_reset_sum(RRDR *r); -void grouping_free_sum(RRDR *r); -void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_sum { + NETDATA_DOUBLE sum; + size_t count; +}; + +static inline void tg_sum_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_sum)); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_sum_reset(RRDR *r) { + struct tg_sum *g = (struct tg_sum *)r->time_grouping.data; + g->sum = 0; + g->count = 0; +} + +static inline void tg_sum_free(RRDR *r) { + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_sum_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_sum *g = (struct tg_sum *)r->time_grouping.data; + g->sum += value; + g->count++; +} + +static inline NETDATA_DOUBLE tg_sum_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_sum *g = (struct tg_sum *)r->time_grouping.data; + + NETDATA_DOUBLE value; + + if(unlikely(!g->count)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else { + value = g->sum; + } + + g->sum = 0.0; + g->count = 0; + + return value; +} #endif //NETDATA_API_QUERY_SUM_H diff --git a/web/api/queries/trimmed_mean/README.md b/web/api/queries/trimmed_mean/README.md index 71cdb85d..08a32b83 100644 --- a/web/api/queries/trimmed_mean/README.md +++ b/web/api/queries/trimmed_mean/README.md @@ -1,7 +1,11 @@ <!-- title: "Trimmed Mean" +sidebar_label: "Trimmed Mean" description: "Use trimmed-mean in API queries and health entities to find the average value from a sample, eliminating any unwanted spikes in the returned metrics." custom_edit_url: https://github.com/netdata/netdata/edit/master/web/api/queries/trimmed_mean/README.md +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Developers/Web/Api/Queries" --> # Trimmed Mean diff --git a/web/api/queries/trimmed_mean/trimmed_mean.c b/web/api/queries/trimmed_mean/trimmed_mean.c index 2277208a..c50db7ed 100644 --- a/web/api/queries/trimmed_mean/trimmed_mean.c +++ b/web/api/queries/trimmed_mean/trimmed_mean.c @@ -5,162 +5,3 @@ // ---------------------------------------------------------------------------- // median -struct grouping_trimmed_mean { - size_t series_size; - size_t next_pos; - NETDATA_DOUBLE percent; - - NETDATA_DOUBLE *series; -}; - -static void grouping_create_trimmed_mean_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { - long entries = r->group; - if(entries < 10) entries = 10; - - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_trimmed_mean)); - g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); - g->series_size = (size_t)entries; - - g->percent = def; - if(options && *options) { - g->percent = str2ndd(options, NULL); - if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; - if(g->percent < 0.0) g->percent = 0.0; - if(g->percent > 50.0) g->percent = 50.0; - } - - g->percent = 1.0 - ((g->percent / 100.0) * 2.0); - r->internal.grouping_data = g; -} - -void grouping_create_trimmed_mean1(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 1.0); -} -void grouping_create_trimmed_mean2(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 2.0); -} -void grouping_create_trimmed_mean3(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 3.0); -} -void grouping_create_trimmed_mean5(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 5.0); -} -void grouping_create_trimmed_mean10(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 10.0); -} -void grouping_create_trimmed_mean15(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 15.0); -} -void grouping_create_trimmed_mean20(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 20.0); -} -void grouping_create_trimmed_mean25(RRDR *r, const char *options) { - grouping_create_trimmed_mean_internal(r, options, 25.0); -} - -// resets when switches dimensions -// so, clear everything to restart -void grouping_reset_trimmed_mean(RRDR *r) { - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; - g->next_pos = 0; -} - -void grouping_free_trimmed_mean(RRDR *r) { - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; - if(g) onewayalloc_freez(r->internal.owa, g->series); - - onewayalloc_freez(r->internal.owa, r->internal.grouping_data); - r->internal.grouping_data = NULL; -} - -void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value) { - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; - - if(unlikely(g->next_pos >= g->series_size)) { - g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); - g->series_size *= 2; - } - - g->series[g->next_pos++] = value; -} - -NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { - struct grouping_trimmed_mean *g = (struct grouping_trimmed_mean *)r->internal.grouping_data; - - NETDATA_DOUBLE value; - size_t available_slots = g->next_pos; - - if(unlikely(!available_slots)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - else if(available_slots == 1) { - value = g->series[0]; - } - else { - sort_series(g->series, available_slots); - - NETDATA_DOUBLE min = g->series[0]; - NETDATA_DOUBLE max = g->series[available_slots - 1]; - - if (min != max) { - size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent); - if(!slots_to_use) slots_to_use = 1; - - NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots; - NETDATA_DOUBLE percent_delta = g->percent - percent_to_use; - - NETDATA_DOUBLE percent_interpolation_slot = 0.0; - NETDATA_DOUBLE percent_last_slot = 0.0; - if(percent_delta > 0.0) { - NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots; - NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use; - - percent_interpolation_slot = percent_delta / percent_1slot; - percent_last_slot = 1 - percent_interpolation_slot; - } - - int start_slot, stop_slot, step, last_slot, interpolation_slot; - if(min >= 0.0 && max >= 0.0) { - start_slot = (int)((available_slots - slots_to_use) / 2); - stop_slot = start_slot + (int)slots_to_use; - last_slot = stop_slot - 1; - interpolation_slot = stop_slot; - step = 1; - } - else { - start_slot = (int)available_slots - 1 - (int)((available_slots - slots_to_use) / 2); - stop_slot = start_slot - (int)slots_to_use; - last_slot = stop_slot + 1; - interpolation_slot = stop_slot; - step = -1; - } - - value = 0.0; - for(int slot = start_slot; slot != stop_slot ; slot += step) - value += g->series[slot]; - - size_t counted = slots_to_use; - if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) { - value += g->series[interpolation_slot] * percent_interpolation_slot; - value += g->series[last_slot] * percent_last_slot; - counted++; - } - - value = value / (NETDATA_DOUBLE)counted; - } - else - value = min; - } - - if(unlikely(!netdata_double_isnumber(value))) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; - } - - //log_series_to_stderr(g->series, g->next_pos, value, "trimmed_mean"); - - g->next_pos = 0; - - return value; -} diff --git a/web/api/queries/trimmed_mean/trimmed_mean.h b/web/api/queries/trimmed_mean/trimmed_mean.h index e66d9254..3c09015b 100644 --- a/web/api/queries/trimmed_mean/trimmed_mean.h +++ b/web/api/queries/trimmed_mean/trimmed_mean.h @@ -6,17 +6,164 @@ #include "../query.h" #include "../rrdr.h" -void grouping_create_trimmed_mean1(RRDR *r, const char *options); -void grouping_create_trimmed_mean2(RRDR *r, const char *options); -void grouping_create_trimmed_mean3(RRDR *r, const char *options); -void grouping_create_trimmed_mean5(RRDR *r, const char *options); -void grouping_create_trimmed_mean10(RRDR *r, const char *options); -void grouping_create_trimmed_mean15(RRDR *r, const char *options); -void grouping_create_trimmed_mean20(RRDR *r, const char *options); -void grouping_create_trimmed_mean25(RRDR *r, const char *options); -void grouping_reset_trimmed_mean(RRDR *r); -void grouping_free_trimmed_mean(RRDR *r); -void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value); -NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +struct tg_trimmed_mean { + size_t series_size; + size_t next_pos; + NETDATA_DOUBLE percent; + + NETDATA_DOUBLE *series; +}; + +static inline void tg_trimmed_mean_create_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { + long entries = r->view.group; + if(entries < 10) entries = 10; + + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_trimmed_mean)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); + g->series_size = (size_t)entries; + + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 50.0) g->percent = 50.0; + } + + g->percent = 1.0 - ((g->percent / 100.0) * 2.0); + r->time_grouping.data = g; +} + +static inline void tg_trimmed_mean_create_1(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 1.0); +} +static inline void tg_trimmed_mean_create_2(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 2.0); +} +static inline void tg_trimmed_mean_create_3(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 3.0); +} +static inline void tg_trimmed_mean_create_5(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 5.0); +} +static inline void tg_trimmed_mean_create_10(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 10.0); +} +static inline void tg_trimmed_mean_create_15(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 15.0); +} +static inline void tg_trimmed_mean_create_20(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 20.0); +} +static inline void tg_trimmed_mean_create_25(RRDR *r, const char *options) { + tg_trimmed_mean_create_internal(r, options, 25.0); +} + +// resets when switches dimensions +// so, clear everything to restart +static inline void tg_trimmed_mean_reset(RRDR *r) { + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data; + g->next_pos = 0; +} + +static inline void tg_trimmed_mean_free(RRDR *r) { + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->time_grouping.data); + r->time_grouping.data = NULL; +} + +static inline void tg_trimmed_mean_add(RRDR *r, NETDATA_DOUBLE value) { + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data; + + if(unlikely(g->next_pos >= g->series_size)) { + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; + } + + g->series[g->next_pos++] = value; +} + +static inline NETDATA_DOUBLE tg_trimmed_mean_flush(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { + struct tg_trimmed_mean *g = (struct tg_trimmed_mean *)r->time_grouping.data; + + NETDATA_DOUBLE value; + size_t available_slots = g->next_pos; + + if(unlikely(!available_slots)) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + else if(available_slots == 1) { + value = g->series[0]; + } + else { + sort_series(g->series, available_slots); + + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + + if (min != max) { + size_t slots_to_use = (size_t)((NETDATA_DOUBLE)available_slots * g->percent); + if(!slots_to_use) slots_to_use = 1; + + NETDATA_DOUBLE percent_to_use = (NETDATA_DOUBLE)slots_to_use / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_delta = g->percent - percent_to_use; + + NETDATA_DOUBLE percent_interpolation_slot = 0.0; + NETDATA_DOUBLE percent_last_slot = 0.0; + if(percent_delta > 0.0) { + NETDATA_DOUBLE percent_to_use_plus_1_slot = (NETDATA_DOUBLE)(slots_to_use + 1) / (NETDATA_DOUBLE)available_slots; + NETDATA_DOUBLE percent_1slot = percent_to_use_plus_1_slot - percent_to_use; + + percent_interpolation_slot = percent_delta / percent_1slot; + percent_last_slot = 1 - percent_interpolation_slot; + } + + int start_slot, stop_slot, step, last_slot, interpolation_slot; + if(min >= 0.0 && max >= 0.0) { + start_slot = (int)((available_slots - slots_to_use) / 2); + stop_slot = start_slot + (int)slots_to_use; + last_slot = stop_slot - 1; + interpolation_slot = stop_slot; + step = 1; + } + else { + start_slot = (int)available_slots - 1 - (int)((available_slots - slots_to_use) / 2); + stop_slot = start_slot - (int)slots_to_use; + last_slot = stop_slot + 1; + interpolation_slot = stop_slot; + step = -1; + } + + value = 0.0; + for(int slot = start_slot; slot != stop_slot ; slot += step) + value += g->series[slot]; + + size_t counted = slots_to_use; + if(percent_interpolation_slot > 0.0 && interpolation_slot >= 0 && interpolation_slot < (int)available_slots) { + value += g->series[interpolation_slot] * percent_interpolation_slot; + value += g->series[last_slot] * percent_last_slot; + counted++; + } + + value = value / (NETDATA_DOUBLE)counted; + } + else + value = min; + } + + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "trimmed_mean"); + + g->next_pos = 0; + + return value; +} #endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index 485aaca2..0830a969 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -24,10 +24,11 @@ static struct { const char *name; WEIGHTS_METHOD value; } weights_methods[] = { - { "ks2" , WEIGHTS_METHOD_MC_KS2} - , { "volume" , WEIGHTS_METHOD_MC_VOLUME} - , { "anomaly-rate" , WEIGHTS_METHOD_ANOMALY_RATE} - , { NULL , 0 } + { "ks2" , WEIGHTS_METHOD_MC_KS2} + , { "volume" , WEIGHTS_METHOD_MC_VOLUME} + , { "anomaly-rate" , WEIGHTS_METHOD_ANOMALY_RATE} + , { "value" , WEIGHTS_METHOD_VALUE} + , { NULL , 0 } }; WEIGHTS_METHOD weights_string_to_method(const char *method) { @@ -56,14 +57,18 @@ typedef enum { struct register_result { RESULT_FLAGS flags; + RRDHOST *host; RRDCONTEXT_ACQUIRED *rca; RRDINSTANCE_ACQUIRED *ria; RRDMETRIC_ACQUIRED *rma; NETDATA_DOUBLE value; + STORAGE_POINT highlighted; + STORAGE_POINT baseline; + usec_t duration_ut; }; static DICTIONARY *register_result_init() { - DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED); + DICTIONARY *results = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct register_result)); return results; } @@ -71,14 +76,10 @@ static void register_result_destroy(DICTIONARY *results) { dictionary_destroy(results); } -static void register_result(DICTIONARY *results, - RRDCONTEXT_ACQUIRED *rca, - RRDINSTANCE_ACQUIRED *ria, - RRDMETRIC_ACQUIRED *rma, - NETDATA_DOUBLE value, - RESULT_FLAGS flags, - WEIGHTS_STATS *stats, - bool register_zero) { +static void register_result(DICTIONARY *results, RRDHOST *host, RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, + RRDMETRIC_ACQUIRED *rma, NETDATA_DOUBLE value, RESULT_FLAGS flags, + STORAGE_POINT *highlighted, STORAGE_POINT *baseline, WEIGHTS_STATS *stats, + bool register_zero, usec_t duration_ut) { if(!netdata_double_isnumber(value)) return; @@ -90,17 +91,25 @@ static void register_result(DICTIONARY *results, return; // keep track of the max of the baseline / highlight ratio - if(flags & RESULT_IS_BASE_HIGH_RATIO && v > stats->max_base_high_ratio) + if((flags & RESULT_IS_BASE_HIGH_RATIO) && v > stats->max_base_high_ratio) stats->max_base_high_ratio = v; struct register_result t = { .flags = flags, + .host = host, .rca = rca, .ria = ria, .rma = rma, - .value = v + .value = v, + .duration_ut = duration_ut, }; + if(highlighted) + t.highlighted = *highlighted; + + if(baseline) + t.baseline = *baseline; + // we can use the pointer address or RMA as a unique key for each metric char buf[20 + 1]; ssize_t len = snprintfz(buf, 20, "%p", rma); @@ -114,112 +123,92 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w time_t after, time_t before, time_t baseline_after, time_t baseline_before, size_t points, WEIGHTS_METHOD method, - RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, size_t examined_dimensions __maybe_unused, usec_t duration, WEIGHTS_STATS *stats) { - buffer_sprintf(wb, "{\n" - "\t\"after\": %lld,\n" - "\t\"before\": %lld,\n" - "\t\"duration\": %lld,\n" - "\t\"points\": %zu,\n", - (long long)after, - (long long)before, - (long long)(before - after), - points - ); - - if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) - buffer_sprintf(wb, "" - "\t\"baseline_after\": %lld,\n" - "\t\"baseline_before\": %lld,\n" - "\t\"baseline_duration\": %lld,\n" - "\t\"baseline_points\": %zu,\n", - (long long)baseline_after, - (long long)baseline_before, - (long long)(baseline_before - baseline_after), - points << shifts - ); - - buffer_sprintf(wb, "" - "\t\"statistics\": {\n" - "\t\t\"query_time_ms\": %f,\n" - "\t\t\"db_queries\": %zu,\n" - "\t\t\"query_result_points\": %zu,\n" - "\t\t\"binary_searches\": %zu,\n" - "\t\t\"db_points_read\": %zu,\n" - "\t\t\"db_points_per_tier\": [ ", - (double)duration / (double)USEC_PER_MS, - stats->db_queries, - stats->result_points, - stats->binary_searches, - stats->db_points - ); - - for(size_t tier = 0; tier < storage_tiers ;tier++) - buffer_sprintf(wb, "%s%zu", tier?", ":"", stats->db_points_per_tier[tier]); - - buffer_sprintf(wb, " ]\n" - "\t},\n" - "\t\"group\": \"%s\",\n" - "\t\"method\": \"%s\",\n" - "\t\"options\": \"", - web_client_api_request_v1_data_group_to_string(group), - weights_method_to_string(method) - ); - - web_client_api_request_v1_data_options_to_buffer(wb, options); + buffer_json_member_add_time_t(wb, "after", after); + buffer_json_member_add_time_t(wb, "before", before); + buffer_json_member_add_time_t(wb, "duration", before - after); + buffer_json_member_add_uint64(wb, "points", points); + + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) { + buffer_json_member_add_time_t(wb, "baseline_after", baseline_after); + buffer_json_member_add_time_t(wb, "baseline_before", baseline_before); + buffer_json_member_add_time_t(wb, "baseline_duration", baseline_before - baseline_after); + buffer_json_member_add_uint64(wb, "baseline_points", points << shifts); + } + + buffer_json_member_add_object(wb, "statistics"); + { + buffer_json_member_add_double(wb, "query_time_ms", (double) duration / (double) USEC_PER_MS); + buffer_json_member_add_uint64(wb, "db_queries", stats->db_queries); + buffer_json_member_add_uint64(wb, "query_result_points", stats->result_points); + buffer_json_member_add_uint64(wb, "binary_searches", stats->binary_searches); + buffer_json_member_add_uint64(wb, "db_points_read", stats->db_points); + + buffer_json_member_add_array(wb, "db_points_per_tier"); + { + for (size_t tier = 0; tier < storage_tiers; tier++) + buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]); + } + buffer_json_array_close(wb); + } + buffer_json_object_close(wb); + + buffer_json_member_add_string(wb, "group", time_grouping_tostring(group)); + buffer_json_member_add_string(wb, "method", weights_method_to_string(method)); + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options); } static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb, time_t after, time_t before, time_t baseline_after, time_t baseline_before, size_t points, WEIGHTS_METHOD method, - RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, size_t examined_dimensions, usec_t duration, WEIGHTS_STATS *stats) { + buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + results_header_to_json(results, wb, after, before, baseline_after, baseline_before, points, method, group, options, shifts, examined_dimensions, duration, stats); - buffer_strcat(wb, "\",\n\t\"correlated_charts\": {\n"); + buffer_json_member_add_object(wb, "correlated_charts"); - size_t charts = 0, chart_dims = 0, total_dimensions = 0; + size_t charts = 0, total_dimensions = 0; struct register_result *t; RRDINSTANCE_ACQUIRED *last_ria = NULL; // never access this - we use it only for comparison dfe_start_read(results, t) { if(t->ria != last_ria) { last_ria = t->ria; - if(charts) buffer_strcat(wb, "\n\t\t\t}\n\t\t},\n"); - buffer_strcat(wb, "\t\t\""); - buffer_strcat(wb, rrdinstance_acquired_id(t->ria)); - buffer_strcat(wb, "\": {\n"); - buffer_strcat(wb, "\t\t\t\"context\": \""); - buffer_strcat(wb, rrdcontext_acquired_id(t->rca)); - buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n"); + if(charts) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // chart:id + } + + buffer_json_member_add_object(wb, rrdinstance_acquired_id(t->ria)); + buffer_json_member_add_string(wb, "context", rrdcontext_acquired_id(t->rca)); + buffer_json_member_add_object(wb, "dimensions"); charts++; - chart_dims = 0; } - if (chart_dims) buffer_sprintf(wb, ",\n"); - buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value); - chart_dims++; + buffer_json_member_add_double(wb, rrdmetric_acquired_name(t->rma), t->value); total_dimensions++; } dfe_done(t); // close dimensions and chart - if (total_dimensions) - buffer_strcat(wb, "\n\t\t\t}\n\t\t}\n"); - - // close correlated_charts - buffer_sprintf(wb, "\t},\n" - "\t\"correlated_dimensions\": %zu,\n" - "\t\"total_dimensions_count\": %zu\n" - "}\n", - total_dimensions, - examined_dimensions - ); + if (total_dimensions) { + buffer_json_object_close(wb); // dimensions + buffer_json_object_close(wb); // chart:id + } + + buffer_json_object_close(wb); + + buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions); + buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions); + buffer_json_finalize(wb); return total_dimensions; } @@ -228,14 +217,16 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w time_t after, time_t before, time_t baseline_after, time_t baseline_before, size_t points, WEIGHTS_METHOD method, - RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, size_t examined_dimensions, usec_t duration, WEIGHTS_STATS *stats) { + buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + results_header_to_json(results, wb, after, before, baseline_after, baseline_before, points, method, group, options, shifts, examined_dimensions, duration, stats); - buffer_strcat(wb, "\",\n\t\"contexts\": {\n"); + buffer_json_member_add_object(wb, "contexts"); size_t contexts = 0, charts = 0, total_dimensions = 0, context_dims = 0, chart_dims = 0; NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0; @@ -247,18 +238,17 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w if(t->rca != last_rca) { last_rca = t->rca; - if(contexts) - buffer_sprintf(wb, "\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" - "\t\t\t\t}\n\t\t\t},\n" - "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n" - , charts_total_weight / (double)chart_dims - , contexts_total_weight / (double)context_dims); + if(contexts) { + buffer_json_object_close(wb); // dimensions + buffer_json_member_add_double(wb, "weight", charts_total_weight / (double) chart_dims); + buffer_json_object_close(wb); // chart:id + buffer_json_object_close(wb); // charts + buffer_json_member_add_double(wb, "weight", contexts_total_weight / (double) context_dims); + buffer_json_object_close(wb); // context + } - buffer_strcat(wb, "\t\t\""); - buffer_strcat(wb, rrdcontext_acquired_id(t->rca)); - buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n"); + buffer_json_member_add_object(wb, rrdcontext_acquired_id(t->rca)); + buffer_json_member_add_object(wb, "charts"); contexts++; charts = 0; @@ -271,25 +261,21 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w if(t->ria != last_ria) { last_ria = t->ria; - if(charts) - buffer_sprintf(wb, "\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" - "\t\t\t\t},\n" - , charts_total_weight / (double)chart_dims); + if(charts) { + buffer_json_object_close(wb); // dimensions + buffer_json_member_add_double(wb, "weight", charts_total_weight / (double) chart_dims); + buffer_json_object_close(wb); // chart:id + } - buffer_strcat(wb, "\t\t\t\t\""); - buffer_strcat(wb, rrdinstance_acquired_id(t->ria)); - buffer_strcat(wb, "\": {\n"); - buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n"); + buffer_json_member_add_object(wb, rrdinstance_acquired_id(t->ria)); + buffer_json_member_add_object(wb, "dimensions"); charts++; chart_dims = 0; charts_total_weight = 0.0; } - if (chart_dims) buffer_sprintf(wb, ",\n"); - buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value); + buffer_json_member_add_double(wb, rrdmetric_acquired_name(t->rma), t->value); charts_total_weight += t->value; contexts_total_weight += t->value; chart_dims++; @@ -299,25 +285,794 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w dfe_done(t); // close dimensions and chart - if (total_dimensions) - buffer_sprintf(wb, "\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" - "\t\t\t\t}\n" - "\t\t\t},\n" - "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" - "\t\t}\n" - , charts_total_weight / (double)chart_dims - , contexts_total_weight / (double)context_dims); - - // close correlated_charts - buffer_sprintf(wb, "\t},\n" - "\t\"weighted_dimensions\": %zu,\n" - "\t\"total_dimensions_count\": %zu\n" - "}\n", - total_dimensions, - examined_dimensions - ); + if (total_dimensions) { + buffer_json_object_close(wb); // dimensions + buffer_json_member_add_double(wb, "weight", charts_total_weight / (double) chart_dims); + buffer_json_object_close(wb); // chart:id + buffer_json_object_close(wb); // charts + buffer_json_member_add_double(wb, "weight", contexts_total_weight / (double) context_dims); + buffer_json_object_close(wb); // context + } + + buffer_json_object_close(wb); + + buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions); + buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions); + buffer_json_finalize(wb); + + return total_dimensions; +} + +struct query_weights_data { + QUERY_WEIGHTS_REQUEST *qwr; + + SIMPLE_PATTERN *scope_nodes_sp; + SIMPLE_PATTERN *scope_contexts_sp; + SIMPLE_PATTERN *nodes_sp; + SIMPLE_PATTERN *contexts_sp; + SIMPLE_PATTERN *instances_sp; + SIMPLE_PATTERN *dimensions_sp; + SIMPLE_PATTERN *labels_sp; + SIMPLE_PATTERN *alerts_sp; + + usec_t timeout_us; + bool timed_out; + bool interrupted; + + struct query_timings timings; + + size_t examined_dimensions; + bool register_zero; + + DICTIONARY *results; + WEIGHTS_STATS stats; + + uint32_t shifts; + + struct query_versions versions; +}; + +#define AGGREGATED_WEIGHT_EMPTY (struct aggregated_weight) { \ + .min = NAN, \ + .max = NAN, \ + .sum = NAN, \ + .count = 0, \ + .hsp = STORAGE_POINT_UNSET, \ + .bsp = STORAGE_POINT_UNSET, \ +} + +#define merge_into_aw(aw, t) do { \ + if(!(aw).count) { \ + (aw).count = 1; \ + (aw).min = (aw).max = (aw).sum = (t)->value; \ + (aw).hsp = (t)->highlighted; \ + if(baseline) \ + (aw).bsp = (t)->baseline; \ + } \ + else { \ + (aw).count++; \ + (aw).sum += (t)->value; \ + if((t)->value < (aw).min) \ + (aw).min = (t)->value; \ + if((t)->value > (aw).max) \ + (aw).max = (t)->value; \ + storage_point_merge_to((aw).hsp, (t)->highlighted); \ + if(baseline) \ + storage_point_merge_to((aw).bsp, (t)->baseline); \ + } \ +} while(0) + +static void results_header_to_json_v2(DICTIONARY *results __maybe_unused, BUFFER *wb, struct query_weights_data *qwd, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions __maybe_unused, usec_t duration __maybe_unused, + WEIGHTS_STATS *stats, bool group_by) { + + buffer_json_member_add_object(wb, "request"); + buffer_json_member_add_string(wb, "method", weights_method_to_string(method)); + web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options); + + buffer_json_member_add_object(wb, "scope"); + buffer_json_member_add_string(wb, "scope_nodes", qwd->qwr->scope_nodes ? qwd->qwr->scope_nodes : "*"); + buffer_json_member_add_string(wb, "scope_contexts", qwd->qwr->scope_contexts ? qwd->qwr->scope_contexts : "*"); + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "selectors"); + buffer_json_member_add_string(wb, "nodes", qwd->qwr->nodes ? qwd->qwr->nodes : "*"); + buffer_json_member_add_string(wb, "contexts", qwd->qwr->contexts ? qwd->qwr->contexts : "*"); + buffer_json_member_add_string(wb, "instances", qwd->qwr->instances ? qwd->qwr->instances : "*"); + buffer_json_member_add_string(wb, "dimensions", qwd->qwr->dimensions ? qwd->qwr->dimensions : "*"); + buffer_json_member_add_string(wb, "labels", qwd->qwr->labels ? qwd->qwr->labels : "*"); + buffer_json_member_add_string(wb, "alerts", qwd->qwr->alerts ? qwd->qwr->alerts : "*"); + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "window"); + buffer_json_member_add_time_t(wb, "after", qwd->qwr->after); + buffer_json_member_add_time_t(wb, "before", qwd->qwr->before); + buffer_json_member_add_uint64(wb, "points", qwd->qwr->points); + if(qwd->qwr->options & RRDR_OPTION_SELECTED_TIER) + buffer_json_member_add_uint64(wb, "tier", qwd->qwr->tier); + else + buffer_json_member_add_string(wb, "tier", NULL); + buffer_json_object_close(wb); + + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) { + buffer_json_member_add_object(wb, "baseline"); + buffer_json_member_add_time_t(wb, "baseline_after", qwd->qwr->baseline_after); + buffer_json_member_add_time_t(wb, "baseline_before", qwd->qwr->baseline_before); + buffer_json_object_close(wb); + } + + buffer_json_member_add_object(wb, "aggregations"); + buffer_json_member_add_object(wb, "time"); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qwd->qwr->time_group_method)); + buffer_json_member_add_string(wb, "time_group_options", qwd->qwr->time_group_options); + buffer_json_object_close(wb); // time + + buffer_json_member_add_array(wb, "metrics"); + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_array(wb, "group_by"); + buffer_json_group_by_to_array(wb, qwd->qwr->group_by.group_by); + buffer_json_array_close(wb); + +// buffer_json_member_add_array(wb, "group_by_label"); +// buffer_json_array_close(wb); + + buffer_json_member_add_string(wb, "aggregation", group_by_aggregate_function_to_string(qwd->qwr->group_by.aggregation)); + } + buffer_json_object_close(wb); // 1st group by + buffer_json_array_close(wb); // array + buffer_json_object_close(wb); // aggregations + + buffer_json_member_add_uint64(wb, "timeout", qwd->qwr->timeout_ms); + buffer_json_object_close(wb); // request + + buffer_json_member_add_object(wb, "view"); + buffer_json_member_add_string(wb, "format", (group_by)?"grouped":"full"); + buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(group)); + + buffer_json_member_add_object(wb, "window"); + buffer_json_member_add_time_t(wb, "after", after); + buffer_json_member_add_time_t(wb, "before", before); + buffer_json_member_add_time_t(wb, "duration", before - after); + buffer_json_member_add_uint64(wb, "points", points); + buffer_json_object_close(wb); + + if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) { + buffer_json_member_add_object(wb, "baseline"); + buffer_json_member_add_time_t(wb, "after", baseline_after); + buffer_json_member_add_time_t(wb, "before", baseline_before); + buffer_json_member_add_time_t(wb, "duration", baseline_before - baseline_after); + buffer_json_member_add_uint64(wb, "points", points << shifts); + buffer_json_object_close(wb); + } + + buffer_json_object_close(wb); // view + + buffer_json_member_add_object(wb, "db"); + { + buffer_json_member_add_uint64(wb, "db_queries", stats->db_queries); + buffer_json_member_add_uint64(wb, "query_result_points", stats->result_points); + buffer_json_member_add_uint64(wb, "binary_searches", stats->binary_searches); + buffer_json_member_add_uint64(wb, "db_points_read", stats->db_points); + + buffer_json_member_add_array(wb, "db_points_per_tier"); + { + for (size_t tier = 0; tier < storage_tiers; tier++) + buffer_json_add_array_item_uint64(wb, stats->db_points_per_tier[tier]); + } + buffer_json_array_close(wb); + } + buffer_json_object_close(wb); // db +} + +typedef enum { + WPT_DIMENSION = 0, + WPT_INSTANCE = 1, + WPT_CONTEXT = 2, + WPT_NODE = 3, + WPT_GROUP = 4, +} WEIGHTS_POINT_TYPE; + +struct aggregated_weight { + const char *name; + NETDATA_DOUBLE min; + NETDATA_DOUBLE max; + NETDATA_DOUBLE sum; + size_t count; + STORAGE_POINT hsp; + STORAGE_POINT bsp; +}; + +static inline void storage_point_to_json(BUFFER *wb, WEIGHTS_POINT_TYPE type, ssize_t di, ssize_t ii, ssize_t ci, ssize_t ni, struct aggregated_weight *aw, RRDR_OPTIONS options __maybe_unused, bool baseline) { + if(type != WPT_GROUP) { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_uint64(wb, type); // "type" + buffer_json_add_array_item_int64(wb, ni); + if (type != WPT_NODE) { + buffer_json_add_array_item_int64(wb, ci); + if (type != WPT_CONTEXT) { + buffer_json_add_array_item_int64(wb, ii); + if (type != WPT_INSTANCE) + buffer_json_add_array_item_int64(wb, di); + else + buffer_json_add_array_item_string(wb, NULL); + } + else { + buffer_json_add_array_item_string(wb, NULL); + buffer_json_add_array_item_string(wb, NULL); + } + } + else { + buffer_json_add_array_item_string(wb, NULL); + buffer_json_add_array_item_string(wb, NULL); + buffer_json_add_array_item_string(wb, NULL); + } + buffer_json_add_array_item_double(wb, (aw->count) ? aw->sum / (NETDATA_DOUBLE)aw->count : 0.0); // "weight" + } + else { + buffer_json_member_add_array(wb, "v"); + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_double(wb, aw->min); // "min" + buffer_json_add_array_item_double(wb, (aw->count) ? aw->sum / (NETDATA_DOUBLE)aw->count : 0.0); // "avg" + buffer_json_add_array_item_double(wb, aw->max); // "max" + buffer_json_add_array_item_double(wb, aw->sum); // "sum" + buffer_json_add_array_item_uint64(wb, aw->count); // "count" + buffer_json_array_close(wb); + } + + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_double(wb, aw->hsp.min); // "min" + buffer_json_add_array_item_double(wb, (aw->hsp.count) ? aw->hsp.sum / (NETDATA_DOUBLE) aw->hsp.count : 0.0); // "avg" + buffer_json_add_array_item_double(wb, aw->hsp.max); // "max" + buffer_json_add_array_item_double(wb, aw->hsp.sum); // "sum" + buffer_json_add_array_item_uint64(wb, aw->hsp.count); // "count" + buffer_json_add_array_item_uint64(wb, aw->hsp.anomaly_count); // "anomaly_count" + buffer_json_array_close(wb); + + if(baseline) { + buffer_json_add_array_item_array(wb); + buffer_json_add_array_item_double(wb, aw->bsp.min); // "min" + buffer_json_add_array_item_double(wb, (aw->bsp.count) ? aw->bsp.sum / (NETDATA_DOUBLE) aw->bsp.count : 0.0); // "avg" + buffer_json_add_array_item_double(wb, aw->bsp.max); // "max" + buffer_json_add_array_item_double(wb, aw->bsp.sum); // "sum" + buffer_json_add_array_item_uint64(wb, aw->bsp.count); // "count" + buffer_json_add_array_item_uint64(wb, aw->bsp.anomaly_count); // "anomaly_count" + buffer_json_array_close(wb); + } + + buffer_json_array_close(wb); +} + +static void multinode_data_schema(BUFFER *wb, RRDR_OPTIONS options __maybe_unused, const char *key, bool baseline, bool group_by) { + buffer_json_member_add_object(wb, key); // schema + + buffer_json_member_add_string(wb, "type", "array"); + buffer_json_member_add_array(wb, "items"); + + if(group_by) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "weight"); + buffer_json_member_add_string(wb, "type", "array"); + buffer_json_member_add_array(wb, "labels"); + { + buffer_json_add_array_item_string(wb, "min"); + buffer_json_add_array_item_string(wb, "avg"); + buffer_json_add_array_item_string(wb, "max"); + buffer_json_add_array_item_string(wb, "sum"); + buffer_json_add_array_item_string(wb, "count"); + } + buffer_json_array_close(wb); + } + buffer_json_object_close(wb); + } + else { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "name", "row_type"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_array(wb, "value"); + buffer_json_add_array_item_string(wb, "dimension"); + buffer_json_add_array_item_string(wb, "instance"); + buffer_json_add_array_item_string(wb, "context"); + buffer_json_add_array_item_string(wb, "node"); + buffer_json_array_close(wb); + buffer_json_object_close(wb); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "ni"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "nodes"); + } + buffer_json_object_close(wb); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "ci"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "contexts"); + } + buffer_json_object_close(wb); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "ii"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "instances"); + } + buffer_json_object_close(wb); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "di"); + buffer_json_member_add_string(wb, "type", "integer"); + buffer_json_member_add_string(wb, "dictionary", "dimensions"); + } + buffer_json_object_close(wb); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "weight"); + buffer_json_member_add_string(wb, "type", "number"); + } + buffer_json_object_close(wb); + } + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "timeframe"); + buffer_json_member_add_string(wb, "type", "array"); + buffer_json_member_add_array(wb, "labels"); + { + buffer_json_add_array_item_string(wb, "min"); + buffer_json_add_array_item_string(wb, "avg"); + buffer_json_add_array_item_string(wb, "max"); + buffer_json_add_array_item_string(wb, "sum"); + buffer_json_add_array_item_string(wb, "count"); + buffer_json_add_array_item_string(wb, "anomaly_count"); + } + buffer_json_array_close(wb); + buffer_json_member_add_object(wb, "calculations"); + buffer_json_member_add_string(wb, "anomaly rate", "anomaly_count * 100 / count"); + buffer_json_object_close(wb); + } + buffer_json_object_close(wb); + + if(baseline) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", "baseline timeframe"); + buffer_json_member_add_string(wb, "type", "array"); + buffer_json_member_add_array(wb, "labels"); + { + buffer_json_add_array_item_string(wb, "min"); + buffer_json_add_array_item_string(wb, "avg"); + buffer_json_add_array_item_string(wb, "max"); + buffer_json_add_array_item_string(wb, "sum"); + buffer_json_add_array_item_string(wb, "count"); + buffer_json_add_array_item_string(wb, "anomaly_count"); + } + buffer_json_array_close(wb); + buffer_json_member_add_object(wb, "calculations"); + buffer_json_member_add_string(wb, "anomaly rate", "anomaly_count * 100 / count"); + buffer_json_object_close(wb); + } + buffer_json_object_close(wb); + } + + buffer_json_array_close(wb); // items + buffer_json_object_close(wb); // schema +} + +struct dict_unique_node { + bool existing; + bool exposed; + uint32_t i; + RRDHOST *host; + usec_t duration_ut; +}; + +struct dict_unique_name_units { + bool existing; + bool exposed; + uint32_t i; + const char *units; +}; + +struct dict_unique_id_name { + bool existing; + bool exposed; + uint32_t i; + const char *id; + const char *name; +}; + +static inline struct dict_unique_node *dict_unique_node_add(DICTIONARY *dict, RRDHOST *host, ssize_t *max_id) { + struct dict_unique_node *dun = dictionary_set(dict, host->machine_guid, NULL, sizeof(struct dict_unique_node)); + if(!dun->existing) { + dun->existing = true; + dun->host = host; + dun->i = *max_id; + (*max_id)++; + } + + return dun; +} + +static inline struct dict_unique_name_units *dict_unique_name_units_add(DICTIONARY *dict, const char *name, const char *units, ssize_t *max_id) { + struct dict_unique_name_units *dun = dictionary_set(dict, name, NULL, sizeof(struct dict_unique_name_units)); + if(!dun->existing) { + dun->units = units; + dun->existing = true; + dun->i = *max_id; + (*max_id)++; + } + + return dun; +} + +static inline struct dict_unique_id_name *dict_unique_id_name_add(DICTIONARY *dict, const char *id, const char *name, ssize_t *max_id) { + char key[1024 + 1]; + snprintfz(key, 1024, "%s:%s", id, name); + struct dict_unique_id_name *dun = dictionary_set(dict, key, NULL, sizeof(struct dict_unique_id_name)); + if(!dun->existing) { + dun->existing = true; + dun->i = *max_id; + (*max_id)++; + dun->id = id; + dun->name = name; + } + + return dun; +} + +static size_t registered_results_to_json_multinode_no_group_by( + DICTIONARY *results, BUFFER *wb, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions, struct query_weights_data *qwd, + WEIGHTS_STATS *stats, + struct query_versions *versions) { + buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_member_add_uint64(wb, "api", 2); + + results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before, + points, method, group, options, shifts, examined_dimensions, + qwd->timings.executed_ut - qwd->timings.received_ut, stats, false); + + version_hashes_api_v2(wb, versions); + + bool baseline = method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME; + multinode_data_schema(wb, options, "schema", baseline, false); + + DICTIONARY *dict_nodes = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_node)); + DICTIONARY *dict_contexts = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_name_units)); + DICTIONARY *dict_instances = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_id_name)); + DICTIONARY *dict_dimensions = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dict_unique_id_name)); + + buffer_json_member_add_array(wb, "result"); + + struct aggregated_weight node_aw = AGGREGATED_WEIGHT_EMPTY, context_aw = AGGREGATED_WEIGHT_EMPTY, instance_aw = AGGREGATED_WEIGHT_EMPTY; + struct register_result *t; + RRDHOST *last_host = NULL; + RRDCONTEXT_ACQUIRED *last_rca = NULL; + RRDINSTANCE_ACQUIRED *last_ria = NULL; + struct dict_unique_name_units *context_dun = NULL; + struct dict_unique_node *node_dun = NULL; + struct dict_unique_id_name *instance_dun = NULL; + struct dict_unique_id_name *dimension_dun = NULL; + ssize_t di = -1, ii = -1, ci = -1, ni = -1; + ssize_t di_max = 0, ii_max = 0, ci_max = 0, ni_max = 0; + size_t total_dimensions = 0; + dfe_start_read(results, t) { + + // close instance + if(t->ria != last_ria && last_ria) { + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, &instance_aw, options, baseline); + instance_dun->exposed = true; + last_ria = NULL; + instance_aw = AGGREGATED_WEIGHT_EMPTY; + } + + // close context + if(t->rca != last_rca && last_rca) { + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, &context_aw, options, baseline); + context_dun->exposed = true; + last_rca = NULL; + context_aw = AGGREGATED_WEIGHT_EMPTY; + } + + // close node + if(t->host != last_host && last_host) { + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, &node_aw, options, baseline); + node_dun->exposed = true; + last_host = NULL; + node_aw = AGGREGATED_WEIGHT_EMPTY; + } + + // open node + if(t->host != last_host) { + last_host = t->host; + node_dun = dict_unique_node_add(dict_nodes, t->host, &ni_max); + ni = node_dun->i; + } + + // open context + if(t->rca != last_rca) { + last_rca = t->rca; + context_dun = dict_unique_name_units_add(dict_contexts, rrdcontext_acquired_id(t->rca), + rrdcontext_acquired_units(t->rca), &ci_max); + ci = context_dun->i; + } + + // open instance + if(t->ria != last_ria) { + last_ria = t->ria; + instance_dun = dict_unique_id_name_add(dict_instances, rrdinstance_acquired_id(t->ria), rrdinstance_acquired_name(t->ria), &ii_max); + ii = instance_dun->i; + } + + dimension_dun = dict_unique_id_name_add(dict_dimensions, rrdmetric_acquired_id(t->rma), rrdmetric_acquired_name(t->rma), &di_max); + di = dimension_dun->i; + + struct aggregated_weight aw = { + .min = t->value, + .max = t->value, + .sum = t->value, + .count = 1, + .hsp = t->highlighted, + .bsp = t->baseline, + }; + + storage_point_to_json(wb, WPT_DIMENSION, di, ii, ci, ni, &aw, options, baseline); + node_dun->exposed = true; + context_dun->exposed = true; + instance_dun->exposed = true; + dimension_dun->exposed = true; + + merge_into_aw(instance_aw, t); + merge_into_aw(context_aw, t); + merge_into_aw(node_aw, t); + + node_dun->duration_ut += t->duration_ut; + total_dimensions++; + } + dfe_done(t); + + // close instance + if(last_ria) { + storage_point_to_json(wb, WPT_INSTANCE, di, ii, ci, ni, &instance_aw, options, baseline); + instance_dun->exposed = true; + } + + // close context + if(last_rca) { + storage_point_to_json(wb, WPT_CONTEXT, di, ii, ci, ni, &context_aw, options, baseline); + context_dun->exposed = true; + } + + // close node + if(last_host) { + storage_point_to_json(wb, WPT_NODE, di, ii, ci, ni, &node_aw, options, baseline); + node_dun->exposed = true; + } + + buffer_json_array_close(wb); // points + + buffer_json_member_add_object(wb, "dictionaries"); + buffer_json_member_add_array(wb, "nodes"); + { + struct dict_unique_node *dun; + dfe_start_read(dict_nodes, dun) { + if(!dun->exposed) + continue; + + buffer_json_add_array_item_object(wb); + buffer_json_node_add_v2(wb, dun->host, dun->i, dun->duration_ut); + buffer_json_object_close(wb); + } + dfe_done(dun); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "contexts"); + { + struct dict_unique_name_units *dun; + dfe_start_read(dict_contexts, dun) { + if(!dun->exposed) + continue; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", dun_dfe.name); + buffer_json_member_add_string(wb, "units", dun->units); + buffer_json_member_add_int64(wb, "ci", dun->i); + buffer_json_object_close(wb); + } + dfe_done(dun); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "instances"); + { + struct dict_unique_id_name *dun; + dfe_start_read(dict_instances, dun) { + if(!dun->exposed) + continue; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", dun->id); + if(dun->id != dun->name) + buffer_json_member_add_string(wb, "nm", dun->name); + buffer_json_member_add_int64(wb, "ii", dun->i); + buffer_json_object_close(wb); + } + dfe_done(dun); + } + buffer_json_array_close(wb); + + buffer_json_member_add_array(wb, "dimensions"); + { + struct dict_unique_id_name *dun; + dfe_start_read(dict_dimensions, dun) { + if(!dun->exposed) + continue; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", dun->id); + if(dun->id != dun->name) + buffer_json_member_add_string(wb, "nm", dun->name); + buffer_json_member_add_int64(wb, "di", dun->i); + buffer_json_object_close(wb); + } + dfe_done(dun); + } + buffer_json_array_close(wb); + + buffer_json_object_close(wb); //dictionaries + + buffer_json_agents_array_v2(wb, &qwd->timings, 0); + buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions); + buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions); + buffer_json_finalize(wb); + + dictionary_destroy(dict_nodes); + dictionary_destroy(dict_contexts); + dictionary_destroy(dict_instances); + dictionary_destroy(dict_dimensions); + + return total_dimensions; +} + +static size_t registered_results_to_json_multinode_group_by( + DICTIONARY *results, BUFFER *wb, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, + RRDR_TIME_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, + size_t examined_dimensions, struct query_weights_data *qwd, + WEIGHTS_STATS *stats, + struct query_versions *versions) { + buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_member_add_uint64(wb, "api", 2); + + results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before, + points, method, group, options, shifts, examined_dimensions, + qwd->timings.executed_ut - qwd->timings.received_ut, stats, true); + + version_hashes_api_v2(wb, versions); + + bool baseline = method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME; + multinode_data_schema(wb, options, "v_schema", baseline, true); + + DICTIONARY *group_by = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(struct aggregated_weight)); + + struct register_result *t; + size_t total_dimensions = 0; + BUFFER *key = buffer_create(0, NULL); + BUFFER *name = buffer_create(0, NULL); + dfe_start_read(results, t) { + + buffer_flush(key); + buffer_flush(name); + + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_DIMENSION) { + buffer_strcat(key, rrdmetric_acquired_name(t->rma)); + buffer_strcat(name, rrdmetric_acquired_name(t->rma)); + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_INSTANCE) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, rrdinstance_acquired_id(t->ria)); + buffer_strcat(name, rrdinstance_acquired_name(t->ria)); + + if(!(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_NODE)) { + buffer_fast_strcat(key, "@", 1); + buffer_fast_strcat(name, "@", 1); + buffer_strcat(key, t->host->machine_guid); + buffer_strcat(name, rrdhost_hostname(t->host)); + } + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_NODE) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, t->host->machine_guid); + buffer_strcat(name, rrdhost_hostname(t->host)); + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_CONTEXT) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, rrdcontext_acquired_id(t->rca)); + buffer_strcat(name, rrdcontext_acquired_id(t->rca)); + } + if(qwd->qwr->group_by.group_by & RRDR_GROUP_BY_UNITS) { + if(buffer_strlen(key)) { + buffer_fast_strcat(key, ",", 1); + buffer_fast_strcat(name, ",", 1); + } + + buffer_strcat(key, rrdcontext_acquired_units(t->rca)); + buffer_strcat(name, rrdcontext_acquired_units(t->rca)); + } + + struct aggregated_weight *aw = dictionary_set(group_by, buffer_tostring(key), NULL, sizeof(struct aggregated_weight)); + if(!aw->name) { + aw->name = strdupz(buffer_tostring(name)); + aw->min = aw->max = aw->sum = t->value; + aw->count = 1; + aw->hsp = t->highlighted; + aw->bsp = t->baseline; + } + else + merge_into_aw(*aw, t); + + total_dimensions++; + } + dfe_done(t); + buffer_free(key); key = NULL; + buffer_free(name); name = NULL; + + struct aggregated_weight *aw; + buffer_json_member_add_array(wb, "result"); + dfe_start_read(group_by, aw) { + const char *k = aw_dfe.name; + const char *n = aw->name; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", k); + + if(strcmp(k, n) != 0) + buffer_json_member_add_string(wb, "nm", n); + + storage_point_to_json(wb, WPT_GROUP, 0, 0, 0, 0, aw, options, baseline); + buffer_json_object_close(wb); + + freez((void *)aw->name); + } + dfe_done(aw); + buffer_json_array_close(wb); // result + + buffer_json_agents_array_v2(wb, &qwd->timings, 0); + buffer_json_member_add_uint64(wb, "correlated_dimensions", total_dimensions); + buffer_json_member_add_uint64(wb, "total_dimensions_count", examined_dimensions); + buffer_json_finalize(wb); + + dictionary_destroy(group_by); return total_dimensions; } @@ -500,14 +1255,16 @@ NETDATA_DOUBLE *rrd2rrdr_ks2( ONEWAYALLOC *owa, RRDHOST *host, RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma, time_t after, time_t before, size_t points, RRDR_OPTIONS options, - RRDR_GROUPING group_method, const char *group_options, size_t tier, + RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier, WEIGHTS_STATS *stats, - size_t *entries + size_t *entries, + STORAGE_POINT *sp ) { NETDATA_DOUBLE *ret = NULL; QUERY_TARGET_REQUEST qtr = { + .version = 1, .host = host, .rca = rca, .ria = ria, @@ -516,25 +1273,27 @@ NETDATA_DOUBLE *rrd2rrdr_ks2( .before = before, .points = points, .options = options, - .group_method = group_method, - .group_options = group_options, + .time_group_method = time_group_method, + .time_group_options = time_group_options, .tier = tier, .query_source = QUERY_SOURCE_API_WEIGHTS, - .priority = STORAGE_PRIORITY_NORMAL, + .priority = STORAGE_PRIORITY_SYNCHRONOUS, }; - RRDR *r = rrd2rrdr(owa, query_target_create(&qtr)); + QUERY_TARGET *qt = query_target_create(&qtr); + RRDR *r = rrd2rrdr(owa, qt); if(!r) goto cleanup; stats->db_queries++; - stats->result_points += r->internal.result_points_generated; - stats->db_points += r->internal.db_points_read; + stats->result_points += r->stats.result_points_generated; + stats->db_points += r->stats.db_points_read; for(size_t tr = 0; tr < storage_tiers ; tr++) - stats->db_points_per_tier[tr] += r->internal.tier_points_read[tr]; + stats->db_points_per_tier[tr] += r->internal.qt->db.tiers[tr].points; - if(r->d != 1) { - error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu", r->internal.qt->id, r->d); + if(r->d != 1 || r->internal.qt->query.used != 1) { + error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu r->d and %zu qt->query.used", + r->internal.qt->id, r->d, (size_t)r->internal.qt->query.used); goto cleanup; } @@ -553,6 +1312,9 @@ NETDATA_DOUBLE *rrd2rrdr_ks2( *entries = rrdr_rows(r); ret = onewayalloc_mallocz(owa, sizeof(NETDATA_DOUBLE) * rrdr_rows(r)); + if(sp) + *sp = r->internal.qt->query.array[0].query_points; + // copy the points of the dimension to a contiguous array // there is no need to check for empty values, since empty values are already zero // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43 @@ -560,6 +1322,7 @@ NETDATA_DOUBLE *rrd2rrdr_ks2( cleanup: rrdr_free(owa, r); + query_target_release(qt); return ret; } @@ -570,27 +1333,30 @@ static void rrdset_metric_correlations_ks2( time_t baseline_after, time_t baseline_before, time_t after, time_t before, size_t points, RRDR_OPTIONS options, - RRDR_GROUPING group_method, const char *group_options, size_t tier, + RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier, uint32_t shifts, WEIGHTS_STATS *stats, bool register_zero ) { options |= RRDR_OPTION_NATURAL_POINTS; + usec_t started_ut = now_monotonic_usec(); ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); size_t high_points = 0; + STORAGE_POINT highlighted_sp; NETDATA_DOUBLE *highlight = rrd2rrdr_ks2( owa, host, rca, ria, rma, after, before, points, - options, group_method, group_options, tier, stats, &high_points); + options, time_group_method, time_group_options, tier, stats, &high_points, &highlighted_sp); if(!highlight) goto cleanup; size_t base_points = 0; + STORAGE_POINT baseline_sp; NETDATA_DOUBLE *baseline = rrd2rrdr_ks2( owa, host, rca, ria, rma, baseline_after, baseline_before, high_points << shifts, - options, group_method, group_options, tier, stats, &base_points); + options, time_group_method, time_group_options, tier, stats, &base_points, &baseline_sp); if(!baseline) goto cleanup; @@ -610,9 +1376,12 @@ static void rrdset_metric_correlations_ks2( prob = 1.0; } + usec_t ended_ut = now_monotonic_usec(); + // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated // so, we flip the result of kstwo() - register_result(results, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero); + register_result(results, host, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, &highlighted_sp, + &baseline_sp, stats, register_zero, ended_ut - started_ut); } cleanup: @@ -622,8 +1391,8 @@ cleanup: // ---------------------------------------------------------------------------- // VOLUME algorithm functions -static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats) { - stats->db_queries++; +static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats, size_t queries) { + stats->db_queries += queries; stats->result_points += qv->result_points; stats->db_points += qv->points_read; for(size_t tier = 0; tier < storage_tiers ; tier++) @@ -636,16 +1405,16 @@ static void rrdset_metric_correlations_volume( DICTIONARY *results, time_t baseline_after, time_t baseline_before, time_t after, time_t before, - RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier, WEIGHTS_STATS *stats, bool register_zero) { options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS; QUERY_VALUE baseline_average = rrdmetric2value(host, rca, ria, rma, baseline_after, baseline_before, - options, group_method, group_options, tier, 0, - QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_NORMAL); - merge_query_value_to_stats(&baseline_average, stats); + options, time_group_method, time_group_options, tier, 0, + QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS); + merge_query_value_to_stats(&baseline_average, stats, 1); if(!netdata_double_isnumber(baseline_average.value)) { // this means no data for the baseline window, but we may have data for the highlighted one - assume zero @@ -653,9 +1422,9 @@ static void rrdset_metric_correlations_volume( } QUERY_VALUE highlight_average = rrdmetric2value(host, rca, ria, rma, after, before, - options, group_method, group_options, tier, 0, - QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_NORMAL); - merge_query_value_to_stats(&highlight_average, stats); + options, time_group_method, time_group_options, tier, 0, + QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS); + merge_query_value_to_stats(&highlight_average, stats, 1); if(!netdata_double_isnumber(highlight_average.value)) return; @@ -665,12 +1434,17 @@ static void rrdset_metric_correlations_volume( return; } + if((options & RRDR_OPTION_ANOMALY_BIT) && highlight_average.value < baseline_average.value) { + // when working on anomaly bits, we are looking for an increase in the anomaly rate + return; + } + char highlight_countif_options[50 + 1]; snprintfz(highlight_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average.value < baseline_average.value ? "<" : ">", baseline_average.value); QUERY_VALUE highlight_countif = rrdmetric2value(host, rca, ria, rma, after, before, options, RRDR_GROUPING_COUNTIF, highlight_countif_options, tier, 0, - QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_NORMAL); - merge_query_value_to_stats(&highlight_countif, stats); + QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS); + merge_query_value_to_stats(&highlight_countif, stats, 1); if(!netdata_double_isnumber(highlight_countif.value)) { info("WEIGHTS: highlighted countif query failed, but highlighted average worked - strange..."); @@ -693,31 +1467,104 @@ static void rrdset_metric_correlations_volume( pcent = highlight_countif.value; } - register_result(results, rca, ria, rma, pcent, flags, stats, register_zero); + register_result(results, host, rca, ria, rma, pcent, flags, &highlight_average.sp, &baseline_average.sp, stats, + register_zero, baseline_average.duration_ut + highlight_average.duration_ut + highlight_countif.duration_ut); } // ---------------------------------------------------------------------------- -// ANOMALY RATE algorithm functions +// VALUE / ANOMALY RATE algorithm functions -static void rrdset_weights_anomaly_rate( +static void rrdset_weights_value( RRDHOST *host, RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma, DICTIONARY *results, time_t after, time_t before, - RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + RRDR_OPTIONS options, RRDR_TIME_GROUPING time_group_method, const char *time_group_options, size_t tier, WEIGHTS_STATS *stats, bool register_zero) { - options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ANOMALY_BIT | RRDR_OPTION_NATURAL_POINTS; + options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_NATURAL_POINTS; QUERY_VALUE qv = rrdmetric2value(host, rca, ria, rma, after, before, - options, group_method, group_options, tier, 0, - QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_NORMAL); + options, time_group_method, time_group_options, tier, 0, + QUERY_SOURCE_API_WEIGHTS, STORAGE_PRIORITY_SYNCHRONOUS); - merge_query_value_to_stats(&qv, stats); + merge_query_value_to_stats(&qv, stats, 1); if(netdata_double_isnumber(qv.value)) - register_result(results, rca, ria, rma, qv.value, 0, stats, register_zero); + register_result(results, host, rca, ria, rma, qv.value, 0, &qv.sp, NULL, stats, register_zero, qv.duration_ut); +} + +static void rrdset_weights_multi_dimensional_value(struct query_weights_data *qwd) { + QUERY_TARGET_REQUEST qtr = { + .version = 1, + .scope_nodes = qwd->qwr->scope_nodes, + .scope_contexts = qwd->qwr->scope_contexts, + .nodes = qwd->qwr->nodes, + .contexts = qwd->qwr->contexts, + .instances = qwd->qwr->instances, + .dimensions = qwd->qwr->dimensions, + .labels = qwd->qwr->labels, + .alerts = qwd->qwr->alerts, + .after = qwd->qwr->after, + .before = qwd->qwr->before, + .points = 1, + .options = qwd->qwr->options | RRDR_OPTION_NATURAL_POINTS, + .time_group_method = qwd->qwr->time_group_method, + .time_group_options = qwd->qwr->time_group_options, + .tier = qwd->qwr->tier, + .timeout_ms = qwd->qwr->timeout_ms, + .query_source = QUERY_SOURCE_API_WEIGHTS, + .priority = STORAGE_PRIORITY_NORMAL, + }; + + ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); + QUERY_TARGET *qt = query_target_create(&qtr); + RRDR *r = rrd2rrdr(owa, qt); + + if(!r || rrdr_rows(r) != 1 || !r->d || r->d != r->internal.qt->query.used) + goto cleanup; + + QUERY_VALUE qv = { + .after = r->view.after, + .before = r->view.before, + .points_read = r->stats.db_points_read, + .result_points = r->stats.result_points_generated, + }; + + size_t queries = 0; + for(size_t d = 0; d < r->d ;d++) { + if(!rrdr_dimension_should_be_exposed(r->od[d], qwd->qwr->options)) + continue; + + long i = 0; // only one row + NETDATA_DOUBLE *cn = &r->v[ i * r->d ]; + NETDATA_DOUBLE *ar = &r->ar[ i * r->d ]; + + qv.value = cn[d]; + qv.anomaly_rate = ar[d]; + storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points); + + if(netdata_double_isnumber(qv.value)) { + QUERY_METRIC *qm = query_metric(r->internal.qt, d); + QUERY_DIMENSION *qd = query_dimension(r->internal.qt, qm->link.query_dimension_id); + QUERY_INSTANCE *qi = query_instance(r->internal.qt, qm->link.query_instance_id); + QUERY_CONTEXT *qc = query_context(r->internal.qt, qm->link.query_context_id); + QUERY_NODE *qn = query_node(r->internal.qt, qm->link.query_node_id); + + register_result(qwd->results, qn->rrdhost, qc->rca, qi->ria, qd->rma, qv.value, 0, &qv.sp, + NULL, &qwd->stats, qwd->register_zero, qm->duration_ut); + } + + queries++; + } + + merge_query_value_to_stats(&qv, &qwd->stats, queries); + +cleanup: + rrdr_free(owa, r); + query_target_release(qt); + onewayalloc_destroy(owa); } // ---------------------------------------------------------------------------- @@ -765,13 +1612,15 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) { NETDATA_DOUBLE slots[dimensions]; dimensions = 0; dfe_start_read(results, t) { - if(t->flags & (RESULT_IS_PERCENTAGE_OF_TIME)) + if(t->flags & RESULT_IS_PERCENTAGE_OF_TIME) t->value = t->value * stats->max_base_high_ratio; slots[dimensions++] = t->value; } dfe_done(t); + if(!dimensions) return 0; // Coverity fix + // sort the array with the values of all dimensions qsort(slots, dimensions, sizeof(NETDATA_DOUBLE), compare_netdata_doubles); @@ -805,60 +1654,184 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) { // ---------------------------------------------------------------------------- // The main function -int web_api_v1_weights( - RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, - RRDR_GROUPING group, const char *group_options, - time_t baseline_after, time_t baseline_before, - time_t after, time_t before, - size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout) { +static ssize_t weights_for_rrdmetric(void *data, RRDHOST *host, RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma) { + struct query_weights_data *qwd = data; + QUERY_WEIGHTS_REQUEST *qwr = qwd->qwr; + + if(qwd->qwr->interrupt_callback && qwd->qwr->interrupt_callback(qwd->qwr->interrupt_callback_data)) { + qwd->interrupted = true; + return -1; + } + + qwd->examined_dimensions++; + + switch(qwr->method) { + case WEIGHTS_METHOD_VALUE: + rrdset_weights_value( + host, rca, ria, rma, + qwd->results, + qwr->after, qwr->before, + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier, + &qwd->stats, qwd->register_zero + ); + break; + + case WEIGHTS_METHOD_ANOMALY_RATE: + qwr->options |= RRDR_OPTION_ANOMALY_BIT; + rrdset_weights_value( + host, rca, ria, rma, + qwd->results, + qwr->after, qwr->before, + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier, + &qwd->stats, qwd->register_zero + ); + break; + + case WEIGHTS_METHOD_MC_VOLUME: + rrdset_metric_correlations_volume( + host, rca, ria, rma, + qwd->results, + qwr->baseline_after, qwr->baseline_before, + qwr->after, qwr->before, + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier, + &qwd->stats, qwd->register_zero + ); + break; + + default: + case WEIGHTS_METHOD_MC_KS2: + rrdset_metric_correlations_ks2( + host, rca, ria, rma, + qwd->results, + qwr->baseline_after, qwr->baseline_before, + qwr->after, qwr->before, qwr->points, + qwr->options, qwr->time_group_method, qwr->time_group_options, qwr->tier, qwd->shifts, + &qwd->stats, qwd->register_zero + ); + break; + } + + qwd->timings.executed_ut = now_monotonic_usec(); + if(qwd->timings.executed_ut - qwd->timings.received_ut > qwd->timeout_us) { + qwd->timed_out = true; + return -1; + } + + return 1; +} + +static ssize_t weights_do_context_callback(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context) { + if(!queryable_context) + return false; + + struct query_weights_data *qwd = data; + + bool has_retention = false; + switch(qwd->qwr->method) { + case WEIGHTS_METHOD_VALUE: + case WEIGHTS_METHOD_ANOMALY_RATE: + has_retention = rrdcontext_retention_match(rca, qwd->qwr->after, qwd->qwr->before); + break; + + case WEIGHTS_METHOD_MC_KS2: + case WEIGHTS_METHOD_MC_VOLUME: + has_retention = rrdcontext_retention_match(rca, qwd->qwr->after, qwd->qwr->before); + if(has_retention) + has_retention = rrdcontext_retention_match(rca, qwd->qwr->baseline_after, qwd->qwr->baseline_before); + break; + } + + if(!has_retention) + return 0; + + ssize_t ret = weights_foreach_rrdmetric_in_context(rca, + qwd->instances_sp, + NULL, + qwd->labels_sp, + qwd->alerts_sp, + qwd->dimensions_sp, + true, true, qwd->qwr->version, + weights_for_rrdmetric, qwd); + return ret; +} + +ssize_t weights_do_node_callback(void *data, RRDHOST *host, bool queryable) { + if(!queryable) + return 0; - WEIGHTS_STATS stats = {}; + struct query_weights_data *qwd = data; + + ssize_t ret = query_scope_foreach_context(host, qwd->qwr->scope_contexts, + qwd->scope_contexts_sp, qwd->contexts_sp, + weights_do_context_callback, queryable, qwd); + + return ret; +} + +int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) { - DICTIONARY *results = register_result_init(); - DICTIONARY *metrics = NULL; char *error = NULL; int resp = HTTP_RESP_OK; // if the user didn't give a timeout // assume 60 seconds - if(!timeout) - timeout = 60 * MSEC_PER_SEC; + if(!qwr->timeout_ms) + qwr->timeout_ms = 5 * 60 * MSEC_PER_SEC; // if the timeout is less than 1 second // make it at least 1 second - if(timeout < (long)(1 * MSEC_PER_SEC)) - timeout = 1 * MSEC_PER_SEC; - - usec_t timeout_usec = timeout * USEC_PER_MS; - usec_t started_usec = now_realtime_usec(); + if(qwr->timeout_ms < (long)(1 * MSEC_PER_SEC)) + qwr->timeout_ms = 1 * MSEC_PER_SEC; + + struct query_weights_data qwd = { + .qwr = qwr, + + .scope_nodes_sp = string_to_simple_pattern(qwr->scope_nodes), + .scope_contexts_sp = string_to_simple_pattern(qwr->scope_contexts), + .nodes_sp = string_to_simple_pattern(qwr->nodes), + .contexts_sp = string_to_simple_pattern(qwr->contexts), + .instances_sp = string_to_simple_pattern(qwr->instances), + .dimensions_sp = string_to_simple_pattern(qwr->dimensions), + .labels_sp = string_to_simple_pattern(qwr->labels), + .alerts_sp = string_to_simple_pattern(qwr->alerts), + .timeout_us = qwr->timeout_ms * USEC_PER_MS, + .timed_out = false, + .examined_dimensions = 0, + .register_zero = true, + .results = register_result_init(), + .stats = {}, + .shifts = 0, + .timings = { + .received_ut = now_monotonic_usec(), + } + }; - if(!rrdr_relative_window_to_absolute(&after, &before)) + if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL)) buffer_no_cacheable(wb); - if (before <= after) { + if (qwr->before <= qwr->after) { resp = HTTP_RESP_BAD_REQUEST; error = "Invalid selected time-range."; goto cleanup; } - uint32_t shifts = 0; - if(method == WEIGHTS_METHOD_MC_KS2 || method == WEIGHTS_METHOD_MC_VOLUME) { - if(!points) points = 500; + if(qwr->method == WEIGHTS_METHOD_MC_KS2 || qwr->method == WEIGHTS_METHOD_MC_VOLUME) { + if(!qwr->points) qwr->points = 500; - if(baseline_before <= API_RELATIVE_TIME_MAX) - baseline_before += after; + if(qwr->baseline_before <= API_RELATIVE_TIME_MAX) + qwr->baseline_before += qwr->after; - rrdr_relative_window_to_absolute(&baseline_after, &baseline_before); + rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL); - if (baseline_before <= baseline_after) { + if (qwr->baseline_before <= qwr->baseline_after) { resp = HTTP_RESP_BAD_REQUEST; error = "Invalid baseline time-range."; goto cleanup; } // baseline should be a power of two multiple of highlight - long long base_delta = baseline_before - baseline_after; - long long high_delta = before - after; + long long base_delta = qwr->baseline_before - qwr->baseline_after; + long long high_delta = qwr->before - qwr->after; uint32_t multiplier = (uint32_t)round((double)base_delta / (double)high_delta); // check if the multiplier is a power of two @@ -880,138 +1853,146 @@ int web_api_v1_weights( // we need to do, to divide baseline numbers to match // the highlight ones while(multiplier > 1) { - shifts++; + qwd.shifts++; multiplier = multiplier >> 1; } // if the baseline size will not comply to MAX_POINTS // lower the window of the baseline - while(shifts && (points << shifts) > MAX_POINTS) - shifts--; + while(qwd.shifts && (qwr->points << qwd.shifts) > MAX_POINTS) + qwd.shifts--; // if the baseline size still does not comply to MAX_POINTS // lower the resolution of the highlight and the baseline - while((points << shifts) > MAX_POINTS) - points = points >> 1; + while((qwr->points << qwd.shifts) > MAX_POINTS) + qwr->points = qwr->points >> 1; - if(points < 15) { + if(qwr->points < 15) { resp = HTTP_RESP_BAD_REQUEST; error = "Too few points available, at least 15 are needed."; goto cleanup; } // adjust the baseline to be multiplier times bigger than the highlight - baseline_after = baseline_before - (high_delta << shifts); + qwr->baseline_after = qwr->baseline_before - (high_delta << qwd.shifts); } - size_t examined_dimensions = 0; + if(qwr->options & RRDR_OPTION_NONZERO) { + qwd.register_zero = false; - bool register_zero = true; - if(options & RRDR_OPTION_NONZERO) { - register_zero = false; - options &= ~RRDR_OPTION_NONZERO; + // remove it to run the queries without it + qwr->options &= ~RRDR_OPTION_NONZERO; } - metrics = rrdcontext_all_metrics_to_dict(host, contexts); - struct metric_entry *me; - - // for every metric_entry in the dictionary - dfe_start_read(metrics, me) { - usec_t now_usec = now_realtime_usec(); - if(now_usec - started_usec > timeout_usec) { - error = "timed out"; - resp = HTTP_RESP_GATEWAY_TIMEOUT; - goto cleanup; + if(qwr->host && qwr->version == 1) + weights_do_node_callback(&qwd, qwr->host, true); + else { + if((qwd.qwr->method == WEIGHTS_METHOD_VALUE || qwd.qwr->method == WEIGHTS_METHOD_ANOMALY_RATE) && (qwd.contexts_sp || qwd.scope_contexts_sp)) { + rrdset_weights_multi_dimensional_value(&qwd); } - - examined_dimensions++; - - switch(method) { - case WEIGHTS_METHOD_ANOMALY_RATE: - options |= RRDR_OPTION_ANOMALY_BIT; - rrdset_weights_anomaly_rate( - host, - me->rca, me->ria, me->rma, - results, - after, before, - options, group, group_options, tier, - &stats, register_zero - ); - break; - - case WEIGHTS_METHOD_MC_VOLUME: - rrdset_metric_correlations_volume( - host, - me->rca, me->ria, me->rma, - results, - baseline_after, baseline_before, - after, before, - options, group, group_options, tier, - &stats, register_zero - ); - break; - - default: - case WEIGHTS_METHOD_MC_KS2: - rrdset_metric_correlations_ks2( - host, - me->rca, me->ria, me->rma, - results, - baseline_after, baseline_before, - after, before, points, - options, group, group_options, tier, shifts, - &stats, register_zero - ); - break; + else { + query_scope_foreach_host(qwd.scope_nodes_sp, qwd.nodes_sp, + weights_do_node_callback, &qwd, + &qwd.versions, + NULL); } } - dfe_done(me); - if(!register_zero) - options |= RRDR_OPTION_NONZERO; + if(!qwd.register_zero) { + // put it back, to show it in the response + qwr->options |= RRDR_OPTION_NONZERO; + } + + if(qwd.timed_out) { + error = "timed out"; + resp = HTTP_RESP_GATEWAY_TIMEOUT; + goto cleanup; + } + + if(qwd.interrupted) { + error = "interrupted"; + resp = HTTP_RESP_BACKEND_FETCH_FAILED; + goto cleanup; + } + + if(!qwd.register_zero) + qwr->options |= RRDR_OPTION_NONZERO; - if(!(options & RRDR_OPTION_RETURN_RAW)) - spread_results_evenly(results, &stats); + if(!(qwr->options & RRDR_OPTION_RETURN_RAW) && qwr->method != WEIGHTS_METHOD_VALUE) + spread_results_evenly(qwd.results, &qwd.stats); - usec_t ended_usec = now_realtime_usec(); + usec_t ended_usec = qwd.timings.executed_ut = now_monotonic_usec(); // generate the json output we need buffer_flush(wb); size_t added_dimensions = 0; - switch(format) { + switch(qwr->format) { case WEIGHTS_FORMAT_CHARTS: added_dimensions = registered_results_to_json_charts( - results, wb, - after, before, - baseline_after, baseline_before, - points, method, group, options, shifts, - examined_dimensions, - ended_usec - started_usec, &stats); + qwd.results, wb, + qwr->after, qwr->before, + qwr->baseline_after, qwr->baseline_before, + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts, + qwd.examined_dimensions, + ended_usec - qwd.timings.received_ut, &qwd.stats); break; - default: case WEIGHTS_FORMAT_CONTEXTS: added_dimensions = registered_results_to_json_contexts( - results, wb, - after, before, - baseline_after, baseline_before, - points, method, group, options, shifts, - examined_dimensions, - ended_usec - started_usec, &stats); + qwd.results, wb, + qwr->after, qwr->before, + qwr->baseline_after, qwr->baseline_before, + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts, + qwd.examined_dimensions, + ended_usec - qwd.timings.received_ut, &qwd.stats); + break; + + default: + case WEIGHTS_FORMAT_MULTINODE: + // we don't support these groupings in weights + qwr->group_by.group_by &= ~(RRDR_GROUP_BY_LABEL|RRDR_GROUP_BY_SELECTED|RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE); + if(qwr->group_by.group_by == RRDR_GROUP_BY_NONE) { + added_dimensions = + registered_results_to_json_multinode_no_group_by( + qwd.results, wb, + qwr->after, qwr->before, + qwr->baseline_after, qwr->baseline_before, + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts, + qwd.examined_dimensions, + &qwd, &qwd.stats, &qwd.versions); + } + else { + added_dimensions = + registered_results_to_json_multinode_group_by( + qwd.results, wb, + qwr->after, qwr->before, + qwr->baseline_after, qwr->baseline_before, + qwr->points, qwr->method, qwr->time_group_method, qwr->options, qwd.shifts, + qwd.examined_dimensions, + &qwd, &qwd.stats, &qwd.versions); + } break; } - if(!added_dimensions) { + if(!added_dimensions && qwr->version < 2) { error = "no results produced."; resp = HTTP_RESP_NOT_FOUND; } cleanup: - if(metrics) dictionary_destroy(metrics); - if(results) register_result_destroy(results); + simple_pattern_free(qwd.scope_nodes_sp); + simple_pattern_free(qwd.scope_contexts_sp); + simple_pattern_free(qwd.nodes_sp); + simple_pattern_free(qwd.contexts_sp); + simple_pattern_free(qwd.instances_sp); + simple_pattern_free(qwd.dimensions_sp); + simple_pattern_free(qwd.labels_sp); + simple_pattern_free(qwd.alerts_sp); + + register_result_destroy(qwd.results); if(error) { buffer_flush(wb); diff --git a/web/api/queries/weights.h b/web/api/queries/weights.h index 50d8634e..66bea6ab 100644 --- a/web/api/queries/weights.h +++ b/web/api/queries/weights.h @@ -9,22 +9,57 @@ typedef enum { WEIGHTS_METHOD_MC_KS2 = 1, WEIGHTS_METHOD_MC_VOLUME = 2, WEIGHTS_METHOD_ANOMALY_RATE = 3, + WEIGHTS_METHOD_VALUE = 4, } WEIGHTS_METHOD; typedef enum { WEIGHTS_FORMAT_CHARTS = 1, WEIGHTS_FORMAT_CONTEXTS = 2, + WEIGHTS_FORMAT_MULTINODE = 3, } WEIGHTS_FORMAT; extern int enable_metric_correlations; extern int metric_correlations_version; extern WEIGHTS_METHOD default_metric_correlations_method; -int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, - RRDR_GROUPING group, const char *group_options, - time_t baseline_after, time_t baseline_before, - time_t after, time_t before, - size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout); +typedef bool (*weights_interrupt_callback_t)(void *data); + +typedef struct query_weights_request { + size_t version; + RRDHOST *host; + const char *scope_nodes; + const char *scope_contexts; + const char *nodes; + const char *contexts; + const char *instances; + const char *dimensions; + const char *labels; + const char *alerts; + + struct { + RRDR_GROUP_BY group_by; + char *group_by_label; + RRDR_GROUP_BY_FUNCTION aggregation; + } group_by; + + WEIGHTS_METHOD method; + WEIGHTS_FORMAT format; + RRDR_TIME_GROUPING time_group_method; + const char *time_group_options; + time_t baseline_after; + time_t baseline_before; + time_t after; + time_t before; + size_t points; + RRDR_OPTIONS options; + size_t tier; + time_t timeout_ms; + + weights_interrupt_callback_t interrupt_callback; + void *interrupt_callback_data; +} QUERY_WEIGHTS_REQUEST; + +int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr); WEIGHTS_METHOD weights_string_to_method(const char *method); const char *weights_method_to_string(WEIGHTS_METHOD method); |