From 00151562145df50cc65e9902d52d5fa77f89fe50 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 9 Jun 2022 06:52:47 +0200 Subject: Merging upstream version 1.35.0. Signed-off-by: Daniel Baumann --- web/api/queries/query.c | 134 ++++++++++++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 49 deletions(-) (limited to 'web/api/queries/query.c') diff --git a/web/api/queries/query.c b/web/api/queries/query.c index c55a97060..5c6c70411 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -3,6 +3,7 @@ #include "query.h" #include "web/api/formatters/rrd2json.h" #include "rrdr.h" +#include "database/ram/rrddim_mem.h" #include "average/average.h" #include "incremental_sum/incremental_sum.h" @@ -27,7 +28,7 @@ static struct { // Allocate all required structures for a query. // This is called once for each netdata query. - void *(*create)(struct rrdresult *r); + void (*create)(struct rrdresult *r); // Cleanup collected values, but don't destroy the structures. // This is called when the query engine switches dimensions, @@ -464,7 +465,9 @@ static inline void do_dimension_variablestep( } } // add this value to grouping - r->internal.grouping_add(r, value); + if(likely(!isnan(value))) + r->internal.grouping_add(r, value); + values_in_group++; db_points_read++; } @@ -537,25 +540,18 @@ static inline void do_dimension_fixedstep( , time_t before_wanted , uint32_t options ){ -#ifdef NETDATA_INTERNAL_CHECKS - RRDSET *st = r->st; -#endif - - time_t - now = after_wanted, + time_t now = after_wanted, dt = r->update_every / r->group, /* usually is st->update_every */ max_date = 0, min_date = 0; - long - group_size = r->group, + long group_size = r->group, points_added = 0, values_in_group = 0, values_in_group_non_zero = 0, rrdr_line = -1; - RRDR_VALUE_FLAGS - group_value_flags = RRDR_VALUE_NOTHING; + RRDR_VALUE_FLAGS group_value_flags = RRDR_VALUE_NOTHING; struct rrddim_query_handle handle; @@ -563,6 +559,13 @@ static inline void do_dimension_fixedstep( size_t db_points_read = 0; time_t db_now = now; time_t first_time_t = rrddim_first_entry_t(rd); + + // cache the function pointers we need in the loop + storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time) = rd->state->query_ops.next_metric; + void (*grouping_add)(struct rrdresult *r, calculated_number value) = r->internal.grouping_add; + calculated_number (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) = r->internal.grouping_flush; + RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode; + for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) { // make sure we return data in the proper time range if(unlikely(now > before_wanted)) { @@ -571,44 +574,74 @@ static inline void do_dimension_fixedstep( #endif break; } + if(unlikely(now < after_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS r->internal.log = "skipped, because attempted to access the db before 'wanted after'"; #endif continue; } + // read the value from the database //storage_number n = rd->values[slot]; + #ifdef NETDATA_INTERNAL_CHECKS - if ((rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) && - (rrdset_time2slot(st, now) != (long unsigned)handle.slotted.slot)) { - error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)handle.slotted.slot, rrdset_time2slot(st, now)); + struct mem_query_handle* mem_handle = (struct mem_query_handle*)handle.handle; + if ((rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) && + (rrdset_time2slot(r->st, now) != (long unsigned)(mem_handle->slot))) { + error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)mem_handle->slot, rrdset_time2slot(r->st, now)); } #endif + db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it + storage_number n; - if (rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t) + calculated_number value; + + if (unlikely(rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && now <= first_time_t)) { n = SN_EMPTY_SLOT; - else - n = rd->state->query_ops.next_metric(&handle, &db_now); + value = NAN; + } + else { + // load the metric value + n = next_metric(&handle, &db_now); + db_points_read++; + + // and unpack it + if(likely(does_storage_number_exist(n))) { + if (options & RRDR_OPTION_ANOMALY_BIT) + value = (n & SN_ANOMALY_BIT) ? 0.0 : 100.0; + else + value = unpack_storage_number(n); + } + else + value = NAN; + } + if(unlikely(db_now > before_wanted)) { #ifdef NETDATA_INTERNAL_CHECKS r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; #endif break; } - for ( ; now <= db_now ; now += dt) { - calculated_number value = NAN; - if(likely(now >= db_now && does_storage_number_exist(n))) { + + // this loop exists only to fill nulls + // so, if there is a value already, we use it for the first iteration + // but the following iterations will just fill nulls to the destination + for ( ; now <= db_now ; now += dt, value = NAN, n = SN_EMPTY_SLOT) { + if(likely(does_storage_number_exist(n))) { + #if defined(NETDATA_INTERNAL_CHECKS) && defined(ENABLE_DBENGINE) - if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != handle.rrdeng.now)) { - error("INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", rd->id, (long)handle.rrdeng.now, (long)now); + if(now >= db_now) { + struct rrdeng_query_handle *rrd_handle = (struct rrdeng_query_handle *)handle.handle; + if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != rrd_handle->now)) + error( + "INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", + rd->id, + (long)rrd_handle->now, + (long)now); } #endif - if (options & RRDR_OPTION_ANOMALY_BIT) - value = (n & SN_ANOMALY_BIT) ? 0.0 : 100.0; - else - value = unpack_storage_number(n); if(likely(value != 0.0)) values_in_group_non_zero++; @@ -616,21 +649,21 @@ static inline void do_dimension_fixedstep( if(unlikely(did_storage_number_reset(n))) group_value_flags |= RRDR_VALUE_RESET; + grouping_add(r, value); } // add this value for grouping - r->internal.grouping_add(r, value); values_in_group++; - db_points_read++; if(unlikely(values_in_group == group_size)) { rrdr_line = rrdr_line_init(r, now, rrdr_line); + size_t rrdr_o_v_index = rrdr_line * r->d + dim_id_in_rrdr; if(unlikely(!min_date)) min_date = now; max_date = now; // find the place to store our values - RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr]; + RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_o_v_index]; // update the dimension options if(likely(values_in_group_non_zero)) @@ -639,21 +672,21 @@ static inline void do_dimension_fixedstep( // store the specific point options *rrdr_value_options_ptr = group_value_flags; - // store the value - calculated_number value = r->internal.grouping_flush(r, rrdr_value_options_ptr); - r->v[rrdr_line * r->d + dim_id_in_rrdr] = value; + // store the group value + calculated_number group_value = grouping_flush(r, rrdr_value_options_ptr); + r->v[rrdr_o_v_index] = group_value; if(likely(points_added || dim_id_in_rrdr)) { // find the min/max across all dimensions - if(unlikely(value < min)) min = value; - if(unlikely(value > max)) max = value; + if(unlikely(group_value < min)) min = group_value; + if(unlikely(group_value > max)) max = group_value; } else { // runs only when dim_id_in_rrdr == 0 && points_added == 0 // so, on the first point added for the query. - min = max = value; + min = max = group_value; } points_added++; @@ -831,7 +864,8 @@ static int rrdr_convert_before_after_to_absolute( } static RRDR *rrd2rrdr_fixedstep( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , long points_requested , long long after_requested , long long before_requested @@ -855,7 +889,7 @@ static RRDR *rrd2rrdr_fixedstep( RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; if(duration <= 0 || available_points <= 0) - return rrdr_create(st, 1, context_param_list); + return rrdr_create(owa, st, 1, context_param_list); // check the number of wanted points in the result if(unlikely(points_requested < 0)) points_requested = -points_requested; @@ -1013,7 +1047,7 @@ static RRDR *rrd2rrdr_fixedstep( // initialize our result set // this also locks the chart for us - RRDR *r = rrdr_create(st, points_wanted, context_param_list); + RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); if(unlikely(!r)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); @@ -1077,7 +1111,7 @@ static RRDR *rrd2rrdr_fixedstep( } // allocate any memory required by the grouping method - r->internal.grouping_data = r->internal.grouping_create(r); + r->internal.grouping_create(r); // ------------------------------------------------------------------------- @@ -1216,7 +1250,8 @@ static RRDR *rrd2rrdr_fixedstep( #ifdef ENABLE_DBENGINE static RRDR *rrd2rrdr_variablestep( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , long points_requested , long long after_requested , long long before_requested @@ -1242,7 +1277,7 @@ static RRDR *rrd2rrdr_variablestep( if(duration <= 0 || available_points <= 0) { freez(region_info_array); - return rrdr_create(st, 1, context_param_list); + return rrdr_create(owa, st, 1, context_param_list); } // check the number of wanted points in the result @@ -1401,7 +1436,7 @@ static RRDR *rrd2rrdr_variablestep( // initialize our result set // this also locks the chart for us - RRDR *r = rrdr_create(st, points_wanted, context_param_list); + RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); if(unlikely(!r)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); @@ -1468,7 +1503,7 @@ static RRDR *rrd2rrdr_variablestep( } // allocate any memory required by the grouping method - r->internal.grouping_data = r->internal.grouping_create(r); + r->internal.grouping_create(r); // ------------------------------------------------------------------------- @@ -1608,7 +1643,8 @@ static RRDR *rrd2rrdr_variablestep( #endif //#ifdef ENABLE_DBENGINE RRDR *rrd2rrdr( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , long points_requested , long long after_requested , long long before_requested @@ -1644,7 +1680,7 @@ RRDR *rrd2rrdr( first_entry_t = after_requested; if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { - rebuild_context_param_list(context_param_list, after_requested); + rebuild_context_param_list(owa, context_param_list, after_requested); st = context_param_list->rd ? context_param_list->rd->rrdset : NULL; if (unlikely(!st)) return NULL; @@ -1669,7 +1705,7 @@ RRDR *rrd2rrdr( } freez(region_info_array); } - return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, + return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); } else { @@ -1680,13 +1716,13 @@ RRDR *rrd2rrdr( rrd_update_every, first_entry_t, last_entry_t, options); } - return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method, + return rrd2rrdr_variablestep(owa, st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list, timeout); } } #endif - return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, + return rrd2rrdr_fixedstep(owa, st, points_requested, after_requested, before_requested, group_method, resampling_time_requested, options, dimensions, rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list, timeout); } -- cgit v1.2.3