From a2d7dede737947d7c6afa20a88e1f0c64e0eb96c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 10 Aug 2023 11:18:52 +0200 Subject: Merging upstream version 1.42.0. Signed-off-by: Daniel Baumann --- web/api/queries/incremental_sum/incremental_sum.h | 28 +++--- web/api/queries/query.c | 102 ++++------------------ web/api/queries/rrdr.h | 2 - web/api/queries/weights.c | 14 +-- 4 files changed, 38 insertions(+), 108 deletions(-) (limited to 'web/api/queries') diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h index dd6483b2c..f110c5861 100644 --- a/web/api/queries/incremental_sum/incremental_sum.h +++ b/web/api/queries/incremental_sum/incremental_sum.h @@ -12,19 +12,20 @@ struct tg_incremental_sum { size_t count; }; -static inline void tg_incremental_sum_create(RRDR *r, const char *options __maybe_unused) { - r->time_grouping.data = onewayalloc_callocz(r->internal.owa, 1, sizeof(struct tg_incremental_sum)); -} - // resets when switches dimensions // so, clear everything to restart static inline void tg_incremental_sum_reset(RRDR *r) { struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; - g->first = 0; - g->last = 0; + g->first = NAN; + g->last = NAN; g->count = 0; } +static inline void tg_incremental_sum_create(RRDR *r, const char *options __maybe_unused) { + r->time_grouping.data = onewayalloc_mallocz(r->internal.owa, sizeof(struct tg_incremental_sum)); + tg_incremental_sum_reset(r); +} + static inline void tg_incremental_sum_free(RRDR *r) { onewayalloc_freez(r->internal.owa, r->time_grouping.data); r->time_grouping.data = NULL; @@ -34,7 +35,11 @@ static inline void tg_incremental_sum_add(RRDR *r, NETDATA_DOUBLE value) { struct tg_incremental_sum *g = (struct tg_incremental_sum *)r->time_grouping.data; if(unlikely(!g->count)) { - g->first = value; + if(isnan(g->first)) + g->first = value; + else + g->last = value; + g->count++; } else { @@ -48,19 +53,16 @@ static inline NETDATA_DOUBLE tg_incremental_sum_flush(RRDR *r, RRDR_VALUE_FLAGS NETDATA_DOUBLE value; - if(unlikely(!g->count)) { + if(unlikely(!g->count || isnan(g->first) || isnan(g->last))) { value = 0.0; *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY; } - else if(unlikely(g->count == 1)) { - value = 0.0; - } else { value = g->last - g->first; } - g->first = 0.0; - g->last = 0.0; + g->first = g->last; + g->last = NAN; g->count = 0; return value; diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 74a925bc3..6f3cbd5fd 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -17,6 +17,7 @@ #include "percentile/percentile.h" #include "trimmed_mean/trimmed_mean.h" +#define QUERY_PLAN_MIN_POINTS 10 #define POINTS_TO_EXPAND_QUERY 5 // ---------------------------------------------------------------------------- @@ -996,6 +997,10 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) return query_metric_first_working_tier(qm); + if(points_wanted < QUERY_PLAN_MIN_POINTS) + // when selecting tiers, aim for a resolution of at least QUERY_PLAN_MIN_POINTS points + points_wanted = (before_wanted - after_wanted) > QUERY_PLAN_MIN_POINTS ? QUERY_PLAN_MIN_POINTS : before_wanted - after_wanted; + time_t min_first_time_s = 0; time_t max_last_time_s = 0; @@ -2070,88 +2075,6 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r } #endif // NETDATA_INTERNAL_CHECKS -// Returns 1 if an absolute period was requested or 0 if it was a relative period -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr) { - time_t now = now_realtime_sec() - 1; - - if(now_ptr) - *now_ptr = now; - - int absolute_period_requested = -1; - long long after_requested, before_requested; - - before_requested = *before; - after_requested = *after; - - // allow relative for before (smaller than API_RELATIVE_TIME_MAX) - if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) { - // if the user asked for a positive relative time, - // flip it to a negative - if(before_requested > 0) - before_requested = -before_requested; - - before_requested = now + before_requested; - absolute_period_requested = 0; - } - - // allow relative for after (smaller than API_RELATIVE_TIME_MAX) - if(ABS(after_requested) <= API_RELATIVE_TIME_MAX) { - if(after_requested > 0) - after_requested = -after_requested; - - // if the user didn't give an after, use the number of points - // to give a sane default - if(after_requested == 0) - after_requested = -600; - - // since the query engine now returns inclusive timestamps - // it is awkward to return 6 points when after=-5 is given - // so for relative queries we add 1 second, to give - // more predictable results to users. - after_requested = before_requested + after_requested + 1; - absolute_period_requested = 0; - } - - if(absolute_period_requested == -1) - absolute_period_requested = 1; - - // check if the parameters are flipped - if(after_requested > before_requested) { - long long t = before_requested; - before_requested = after_requested; - after_requested = t; - } - - // if the query requests future data - // shift the query back to be in the present time - // (this may also happen because of the rules above) - if(before_requested > now) { - long long delta = before_requested - now; - before_requested -= delta; - after_requested -= delta; - } - - time_t absolute_minimum_time = now - (10 * 365 * 86400); - time_t absolute_maximum_time = now + (1 * 365 * 86400); - - if (after_requested < absolute_minimum_time && !unittest_running) - after_requested = absolute_minimum_time; - - if (after_requested > absolute_maximum_time && !unittest_running) - after_requested = absolute_maximum_time; - - if (before_requested < absolute_minimum_time && !unittest_running) - before_requested = absolute_minimum_time; - - if (before_requested > absolute_maximum_time && !unittest_running) - before_requested = absolute_maximum_time; - - *before = before_requested; - *after = after_requested; - - return (absolute_period_requested != 1); -} - // #define DEBUG_QUERY_LOGIC 1 #ifdef DEBUG_QUERY_LOGIC @@ -2278,7 +2201,7 @@ bool query_target_calculate_window(QUERY_TARGET *qt) { } // convert our before_wanted and after_wanted to absolute - rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL); + rrdr_relative_window_to_absolute(&after_wanted, &before_wanted, NULL, unittest_running); query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted); if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) { @@ -2983,11 +2906,11 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) { } // initialize partial trimming - r->partial_data_trimming.max_update_every = update_every_max; + r->partial_data_trimming.max_update_every = update_every_max * 2; r->partial_data_trimming.expected_after = (!query_target_aggregatable(qt) && - qt->window.before >= qt->window.now - update_every_max) ? - qt->window.before - update_every_max : + qt->window.before >= qt->window.now - r->partial_data_trimming.max_update_every) ? + qt->window.before - r->partial_data_trimming.max_update_every : qt->window.before; r->partial_data_trimming.trimmed_after = qt->window.before; @@ -3139,6 +3062,8 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) { if(unlikely(i < 0)) return; + // internal_error(true, "Found trimmable index %zd (from 0 to %zu)", i, r->n - 1); + size_t last_row_gbc = 0; for (; i < (ssize_t)r->n; i++) { size_t row_gbc = 0; @@ -3149,8 +3074,11 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) { row_gbc += r->gbc[ i * r->d + d ]; } - if (unlikely(r->t[i] >= trimmable_after && row_gbc < last_row_gbc)) { + // internal_error(true, "GBC of index %zd is %zu", i, row_gbc); + + if (unlikely(r->t[i] >= trimmable_after && (row_gbc < last_row_gbc || !row_gbc))) { // discard the rest of the points + // internal_error(true, "Discarding points %zd to %zu", i, r->n - 1); r->partial_data_trimming.trimmed_after = r->t[i]; r->rows = i; break; diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index c4a1f83f2..e02e00675 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -206,8 +206,6 @@ RRDR *rrd2rrdr_legacy( RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt); bool query_target_calculate_window(struct query_target *qt); -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr); - #ifdef __cplusplus } #endif diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index 8ffd8951a..2782aef60 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -169,7 +169,7 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb, size_t examined_dimensions, usec_t duration, WEIGHTS_STATS *stats) { - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); results_header_to_json(results, wb, after, before, baseline_after, baseline_before, points, method, group, options, shifts, examined_dimensions, duration, stats); @@ -221,7 +221,7 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w size_t examined_dimensions, usec_t duration, WEIGHTS_STATS *stats) { - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); results_header_to_json(results, wb, after, before, baseline_after, baseline_before, points, method, group, options, shifts, examined_dimensions, duration, stats); @@ -739,7 +739,7 @@ static size_t registered_results_to_json_multinode_no_group_by( size_t examined_dimensions, struct query_weights_data *qwd, WEIGHTS_STATS *stats, struct query_versions *versions) { - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); buffer_json_member_add_uint64(wb, "api", 2); results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before, @@ -958,7 +958,7 @@ static size_t registered_results_to_json_multinode_group_by( size_t examined_dimensions, struct query_weights_data *qwd, WEIGHTS_STATS *stats, struct query_versions *versions) { - buffer_json_initialize(wb, "\"", "\"", 0, true, options & RRDR_OPTION_MINIFY); + buffer_json_initialize(wb, "\"", "\"", 0, true, (options & RRDR_OPTION_MINIFY) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); buffer_json_member_add_uint64(wb, "api", 2); results_header_to_json_v2(results, wb, qwd, after, before, baseline_after, baseline_before, @@ -1806,8 +1806,10 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) { } }; - if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL)) + if(!rrdr_relative_window_to_absolute(&qwr->after, &qwr->before, NULL, false)) buffer_no_cacheable(wb); + else + buffer_cacheable(wb); if (qwr->before <= qwr->after) { resp = HTTP_RESP_BAD_REQUEST; @@ -1821,7 +1823,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) { if(qwr->baseline_before <= API_RELATIVE_TIME_MAX) qwr->baseline_before += qwr->after; - rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL); + rrdr_relative_window_to_absolute(&qwr->baseline_after, &qwr->baseline_before, NULL, false); if (qwr->baseline_before <= qwr->baseline_after) { resp = HTTP_RESP_BAD_REQUEST; -- cgit v1.2.3