From 7877a98bd9c00db5e81dd2f8c734cba2bab20be7 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 12 Aug 2022 09:26:17 +0200 Subject: Merging upstream version 1.36.0. Signed-off-by: Daniel Baumann --- web/api/queries/median/README.md | 17 +++++- web/api/queries/median/median.c | 111 ++++++++++++++++++++++++++++++--------- web/api/queries/median/median.h | 14 +++-- 3 files changed, 114 insertions(+), 28 deletions(-) (limited to 'web/api/queries/median') diff --git a/web/api/queries/median/README.md b/web/api/queries/median/README.md index bb7d4c66b..5600284c2 100644 --- a/web/api/queries/median/README.md +++ b/web/api/queries/median/README.md @@ -13,6 +13,20 @@ The median is the value separating the higher half from the lower half of a data `median` is not an accurate average. However, it eliminates all spikes, by sorting all the values in a period, and selecting the value in the middle of the sorted array. +Netdata also supports `trimmed-median`, which trims a percentage of the smaller and bigger values prior to finding the +median. The following `trimmed-median` functions are defined: + +- `trimmed-median1` +- `trimmed-median2` +- `trimmed-median3` +- `trimmed-median5` +- `trimmed-median10` +- `trimmed-median15` +- `trimmed-median20` +- `trimmed-median25` + +The function `trimmed-median` is an alias for `trimmed-median5`. + ## how to use Use it in alarms like this: @@ -27,7 +41,8 @@ lookup: median -1m unaligned of my_dimension `median` does not change the units. For example, if the chart units is `requests/sec`, the result will be again expressed in the same units. -It can also be used in APIs and badges as `&group=median` in the URL. +It can also be used in APIs and badges as `&group=median` in the URL. Additionally, a percentage may be given with +`&group_options=` to trim all small and big values before finding the median. ## Examples diff --git a/web/api/queries/median/median.c b/web/api/queries/median/median.c index bffcee12f..40fd4ec3a 100644 --- a/web/api/queries/median/median.c +++ b/web/api/queries/median/median.c @@ -2,27 +2,65 @@ #include "median.h" - // ---------------------------------------------------------------------------- // median struct grouping_median { size_t series_size; size_t next_pos; + NETDATA_DOUBLE percent; - LONG_DOUBLE series[]; + NETDATA_DOUBLE *series; }; -void grouping_create_median(RRDR *r) { +void grouping_create_median_internal(RRDR *r, const char *options, NETDATA_DOUBLE def) { long entries = r->group; - if(entries < 0) entries = 0; + if(entries < 10) entries = 10; - struct grouping_median *g = (struct grouping_median *)callocz(1, sizeof(struct grouping_median) + entries * sizeof(LONG_DOUBLE)); + struct grouping_median *g = (struct grouping_median *)onewayalloc_callocz(r->internal.owa, 1, sizeof(struct grouping_median)); + g->series = onewayalloc_mallocz(r->internal.owa, entries * sizeof(NETDATA_DOUBLE)); g->series_size = (size_t)entries; + g->percent = def; + if(options && *options) { + g->percent = str2ndd(options, NULL); + if(!netdata_double_isnumber(g->percent)) g->percent = 0.0; + if(g->percent < 0.0) g->percent = 0.0; + if(g->percent > 50.0) g->percent = 50.0; + } + + g->percent = g->percent / 100.0; r->internal.grouping_data = g; } +void grouping_create_median(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 0.0); +} +void grouping_create_trimmed_median1(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 1.0); +} +void grouping_create_trimmed_median2(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 2.0); +} +void grouping_create_trimmed_median3(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 3.0); +} +void grouping_create_trimmed_median5(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 5.0); +} +void grouping_create_trimmed_median10(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 10.0); +} +void grouping_create_trimmed_median15(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 15.0); +} +void grouping_create_trimmed_median20(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 20.0); +} +void grouping_create_trimmed_median25(RRDR *r, const char *options) { + grouping_create_median_internal(r, options, 25.0); +} + // resets when switches dimensions // so, clear everything to restart void grouping_reset_median(RRDR *r) { @@ -31,47 +69,72 @@ void grouping_reset_median(RRDR *r) { } void grouping_free_median(RRDR *r) { - freez(r->internal.grouping_data); + struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; + if(g) onewayalloc_freez(r->internal.owa, g->series); + + onewayalloc_freez(r->internal.owa, r->internal.grouping_data); r->internal.grouping_data = NULL; } -void grouping_add_median(RRDR *r, calculated_number value) { +void grouping_add_median(RRDR *r, NETDATA_DOUBLE value) { struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; if(unlikely(g->next_pos >= g->series_size)) { - error("INTERNAL ERROR: median buffer overflow on chart '%s' - next_pos = %zu, series_size = %zu, r->group = %ld.", r->st->name, g->next_pos, g->series_size, r->group); + g->series = onewayalloc_doublesize( r->internal.owa, g->series, g->series_size * sizeof(NETDATA_DOUBLE)); + g->series_size *= 2; } - else - g->series[g->next_pos++] = (LONG_DOUBLE)value; + + g->series[g->next_pos++] = value; } -calculated_number grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { +NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) { struct grouping_median *g = (struct grouping_median *)r->internal.grouping_data; - calculated_number value; + size_t available_slots = g->next_pos; + NETDATA_DOUBLE value; - if(unlikely(!g->next_pos)) { + if(unlikely(!available_slots)) { value = 0.0; *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; } + else if(available_slots == 1) { + value = g->series[0]; + } else { - if(g->next_pos > 1) { - sort_series(g->series, g->next_pos); - value = (calculated_number)median_on_sorted_series(g->series, g->next_pos); - } - else - value = (calculated_number)g->series[0]; + sort_series(g->series, available_slots); + + size_t start_slot = 0; + size_t end_slot = available_slots - 1; + + if(g->percent > 0.0) { + NETDATA_DOUBLE min = g->series[0]; + NETDATA_DOUBLE max = g->series[available_slots - 1]; + NETDATA_DOUBLE delta = (max - min) * g->percent; + + NETDATA_DOUBLE wanted_min = min + delta; + NETDATA_DOUBLE wanted_max = max - delta; + + for (start_slot = 0; start_slot < available_slots; start_slot++) + if (g->series[start_slot] >= wanted_min) break; - if(!calculated_number_isnumber(value)) { - value = 0.0; - *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + for (end_slot = available_slots - 1; end_slot > start_slot; end_slot--) + if (g->series[end_slot] <= wanted_max) break; } - //log_series_to_stderr(g->series, g->next_pos, value, "median"); + if(start_slot == end_slot) + value = g->series[start_slot]; + else + value = median_on_sorted_series(&g->series[start_slot], end_slot - start_slot + 1); } + if(unlikely(!netdata_double_isnumber(value))) { + value = 0.0; + *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; + } + + //log_series_to_stderr(g->series, g->next_pos, value, "median"); + g->next_pos = 0; return value; } - diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h index 28d52b31e..dd1b3de61 100644 --- a/web/api/queries/median/median.h +++ b/web/api/queries/median/median.h @@ -6,10 +6,18 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_median(RRDR *r); +extern void grouping_create_median(RRDR *r, const char *options); +extern void grouping_create_trimmed_median1(RRDR *r, const char *options); +extern void grouping_create_trimmed_median2(RRDR *r, const char *options); +extern void grouping_create_trimmed_median3(RRDR *r, const char *options); +extern void grouping_create_trimmed_median5(RRDR *r, const char *options); +extern void grouping_create_trimmed_median10(RRDR *r, const char *options); +extern void grouping_create_trimmed_median15(RRDR *r, const char *options); +extern void grouping_create_trimmed_median20(RRDR *r, const char *options); +extern void grouping_create_trimmed_median25(RRDR *r, const char *options); extern void grouping_reset_median(RRDR *r); extern void grouping_free_median(RRDR *r); -extern void grouping_add_median(RRDR *r, calculated_number value); -extern calculated_number grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +extern void grouping_add_median(RRDR *r, NETDATA_DOUBLE value); +extern NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_MEDIAN_H -- cgit v1.2.3