diff options
Diffstat (limited to 'web/api/queries/query.c')
| -rw-r--r-- | web/api/queries/query.c | 876 |
1 files changed, 746 insertions, 130 deletions
diff --git a/web/api/queries/query.c b/web/api/queries/query.c index e90dc8afe..af3bcfe38 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -376,7 +376,7 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) { // ---------------------------------------------------------------------------- // fill RRDR for a single dimension -static inline void do_dimension( +static inline void do_dimension_variablestep( RRDR *r , long points_wanted , RRDDIM *rd @@ -384,16 +384,16 @@ static inline void do_dimension( , time_t after_wanted , time_t before_wanted ){ - RRDSET *st = r->st; +// RRDSET *st = r->st; time_t now = after_wanted, - dt = st->update_every, + dt = r->update_every, max_date = 0, min_date = 0; long - group_size = r->group, +// group_size = r->group, points_added = 0, values_in_group = 0, values_in_group_non_zero = 0, @@ -403,102 +403,240 @@ static inline void do_dimension( group_value_flags = RRDR_VALUE_NOTHING; struct rrddim_query_handle handle; - uint8_t initialized_query; calculated_number min = r->min, max = r->max; size_t db_points_read = 0; + time_t db_now = now; + storage_number n_curr, n_prev = SN_EMPTY_SLOT; + calculated_number value; + + for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) { + // make sure we return data in the proper time range + if (unlikely(now > before_wanted)) { +#ifdef NETDATA_INTERNAL_CHECKS + r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; +#endif + break; + } + if (unlikely(now < after_wanted)) { +#ifdef NETDATA_INTERNAL_CHECKS + r->internal.log = "skipped, because attempted to access the db before 'wanted after'"; +#endif + continue; + } + + while (now >= db_now && (!rd->state->query_ops.is_finished(&handle) || + does_storage_number_exist(n_prev))) { + value = NAN; + if (does_storage_number_exist(n_prev)) { + // use the previously read database value + n_curr = n_prev; + } else { + // read the value from the database + n_curr = rd->state->query_ops.next_metric(&handle, &db_now); + } + n_prev = SN_EMPTY_SLOT; + // db_now has a different value than above + if (likely(now >= db_now)) { + if (likely(does_storage_number_exist(n_curr))) { + value = unpack_storage_number(n_curr); + if (likely(value != 0.0)) + values_in_group_non_zero++; + + if (unlikely(did_storage_number_reset(n_curr))) + group_value_flags |= RRDR_VALUE_RESET; + } + } else { + // We must postpone processing the value and fill the result with gaps instead + if (likely(does_storage_number_exist(n_curr))) { + n_prev = n_curr; + } + } + // add this value to grouping + r->internal.grouping_add(r, value); + values_in_group++; + db_points_read++; + } + + if (0 == values_in_group) { + // add NAN to grouping + r->internal.grouping_add(r, NAN); + } + + rrdr_line = rrdr_line_init(r, now, rrdr_line); + + if(unlikely(!min_date)) min_date = now; + max_date = now; + + // find the place to store our values + RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr]; + + // update the dimension options + if(likely(values_in_group_non_zero)) + r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO; + + // store the specific point options + *rrdr_value_options_ptr = group_value_flags; + + // store the value + value = r->internal.grouping_flush(r, rrdr_value_options_ptr); + r->v[rrdr_line * r->d + dim_id_in_rrdr] = value; + + if(likely(points_added || dim_id_in_rrdr)) { + // find the min/max across all dimensions + + if(unlikely(value < min)) min = value; + if(unlikely(value > max)) max = value; + + } + else { + // runs only when dim_id_in_rrdr == 0 && points_added == 0 + // so, on the first point added for the query. + min = max = value; + } + + points_added++; + values_in_group = 0; + group_value_flags = RRDR_VALUE_NOTHING; + values_in_group_non_zero = 0; + } + rd->state->query_ops.finalize(&handle); + + r->internal.db_points_read += db_points_read; + r->internal.result_points_generated += points_added; + + r->min = min; + r->max = max; + r->before = max_date; + r->after = min_date - (r->group - 1) * dt; + rrdr_done(r, rrdr_line); + + #ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(r->rows != points_added)) + error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows); + #endif +} - for(initialized_query = 0 ; points_added < points_wanted ; now += dt) { +static inline void do_dimension_fixedstep( + RRDR *r + , long points_wanted + , RRDDIM *rd + , long dim_id_in_rrdr + , time_t after_wanted + , time_t before_wanted +){ + RRDSET *st = r->st; + + time_t + now = after_wanted, + dt = r->update_every / r->group, /* usually is st->update_every */ + max_date = 0, + min_date = 0; + long + group_size = r->group, + points_added = 0, + values_in_group = 0, + values_in_group_non_zero = 0, + rrdr_line = -1; + + RRDR_VALUE_FLAGS + group_value_flags = RRDR_VALUE_NOTHING; + + struct rrddim_query_handle handle; + + calculated_number min = r->min, max = r->max; + size_t db_points_read = 0; + time_t db_now = now; + + for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) { // make sure we return data in the proper time range if(unlikely(now > before_wanted)) { - #ifdef NETDATA_INTERNAL_CHECKS +#ifdef NETDATA_INTERNAL_CHECKS r->internal.log = "stopped, because attempted to access the db after 'wanted before'"; - #endif +#endif break; } if(unlikely(now < after_wanted)) { - #ifdef NETDATA_INTERNAL_CHECKS +#ifdef NETDATA_INTERNAL_CHECKS r->internal.log = "skipped, because attempted to access the db before 'wanted after'"; - #endif +#endif continue; } - - if (unlikely(!initialized_query)) { - rd->state->query_ops.init(rd, &handle, now, before_wanted); - initialized_query = 1; - } // read the value from the database //storage_number n = rd->values[slot]; #ifdef NETDATA_INTERNAL_CHECKS - if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { -#ifdef ENABLE_DBENGINE - if (now != handle.rrdeng.now) - error("INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", rd->id, (long)handle.rrdeng.now, (long)now); -#endif - } else if (rrdset_time2slot(st, now) != (long unsigned)handle.slotted.slot) { + if ((rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) && + (rrdset_time2slot(st, now) != (long unsigned)handle.slotted.slot)) { error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)handle.slotted.slot, rrdset_time2slot(st, now)); } #endif - storage_number n = rd->state->query_ops.next_metric(&handle); - calculated_number value = NAN; - if(likely(does_storage_number_exist(n))) { + db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it + storage_number n = rd->state->query_ops.next_metric(&handle, &db_now); + for ( ; now <= db_now ; now += dt) { + calculated_number value = NAN; + if(likely(now >= db_now && does_storage_number_exist(n))) { +#if defined(NETDATA_INTERNAL_CHECKS) && defined(ENABLE_DBENGINE) + if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != handle.rrdeng.now)) { + error("INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", rd->id, (long)handle.rrdeng.now, (long)now); + } +#endif + value = unpack_storage_number(n); + if(likely(value != 0.0)) + values_in_group_non_zero++; - value = unpack_storage_number(n); - if(likely(value != 0.0)) - values_in_group_non_zero++; + if(unlikely(did_storage_number_reset(n))) + group_value_flags |= RRDR_VALUE_RESET; - if(unlikely(did_storage_number_reset(n))) - group_value_flags |= RRDR_VALUE_RESET; + } - } + // add this value for grouping + r->internal.grouping_add(r, value); + values_in_group++; + db_points_read++; - // add this value for grouping - r->internal.grouping_add(r, value); - values_in_group++; - db_points_read++; + if(unlikely(values_in_group == group_size)) { + rrdr_line = rrdr_line_init(r, now, rrdr_line); - if(unlikely(values_in_group == group_size)) { - rrdr_line = rrdr_line_init(r, now, rrdr_line); + if(unlikely(!min_date)) min_date = now; + max_date = now; - if(unlikely(!min_date)) min_date = now; - max_date = now; + // find the place to store our values + RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr]; - // find the place to store our values - RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr]; + // update the dimension options + if(likely(values_in_group_non_zero)) + r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO; - // update the dimension options - if(likely(values_in_group_non_zero)) - r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO; + // store the specific point options + *rrdr_value_options_ptr = group_value_flags; - // store the specific point options - *rrdr_value_options_ptr = group_value_flags; + // store the value + calculated_number value = r->internal.grouping_flush(r, rrdr_value_options_ptr); + r->v[rrdr_line * r->d + dim_id_in_rrdr] = value; - // store the value - calculated_number value = r->internal.grouping_flush(r, rrdr_value_options_ptr); - r->v[rrdr_line * r->d + dim_id_in_rrdr] = value; + if(likely(points_added || dim_id_in_rrdr)) { + // find the min/max across all dimensions - if(likely(points_added || dim_id_in_rrdr)) { - // find the min/max across all dimensions + if(unlikely(value < min)) min = value; + if(unlikely(value > max)) max = value; - if(unlikely(value < min)) min = value; - if(unlikely(value > max)) max = value; + } + else { + // runs only when dim_id_in_rrdr == 0 && points_added == 0 + // so, on the first point added for the query. + min = max = value; + } + points_added++; + values_in_group = 0; + group_value_flags = RRDR_VALUE_NOTHING; + values_in_group_non_zero = 0; } - else { - // runs only when dim_id_in_rrdr == 0 && points_added == 0 - // so, on the first point added for the query. - min = max = value; - } - - points_added++; - values_in_group = 0; - group_value_flags = RRDR_VALUE_NOTHING; - values_in_group_non_zero = 0; } + now = db_now; } - if (likely(initialized_query)) - rd->state->query_ops.finalize(&handle); + rd->state->query_ops.finalize(&handle); r->internal.db_points_read += db_points_read; r->internal.result_points_generated += points_added; @@ -506,13 +644,13 @@ static inline void do_dimension( r->min = min; r->max = max; r->before = max_date; - r->after = min_date - (r->group - 1) * r->st->update_every; + r->after = min_date - (r->group - 1) * dt; rrdr_done(r, rrdr_line); - #ifdef NETDATA_INTERNAL_CHECKS +#ifdef NETDATA_INTERNAL_CHECKS if(unlikely(r->rows != points_added)) error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows); - #endif +#endif } // ---------------------------------------------------------------------------- @@ -589,22 +727,18 @@ static void rrd2rrdr_log_request_response_metdata(RRDR *r } #endif // NETDATA_INTERNAL_CHECKS -RRDR *rrd2rrdr( - RRDSET *st - , long points_requested - , long long after_requested - , long long before_requested - , RRDR_GROUPING group_method - , long resampling_time_requested - , RRDR_OPTIONS options - , const char *dimensions +// Returns 1 if an absolute period was requested or 0 if it was a relative period +static int rrdr_convert_before_after_to_absolute( + long long *after_requestedp + , long long *before_requestedp + , time_t first_entry_t + , time_t last_entry_t ) { - int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); - int absolute_period_requested = -1; + long long after_requested, before_requested; - time_t first_entry_t = rrdset_first_entry_t(st); - time_t last_entry_t = rrdset_last_entry_t(st); + before_requested = *before_requestedp; + after_requested = *after_requestedp; if(before_requested == 0 && after_requested == 0) { // dump the all the data @@ -614,25 +748,15 @@ RRDR *rrd2rrdr( } // allow relative for before (smaller than API_RELATIVE_TIME_MAX) - if(((before_requested < 0)?-before_requested:before_requested) <= API_RELATIVE_TIME_MAX) { - if(abs(before_requested) % st->update_every) { - // make sure it is multiple of st->update_every - if(before_requested < 0) before_requested = before_requested - st->update_every - before_requested % st->update_every; - else before_requested = before_requested + st->update_every - before_requested % st->update_every; - } + if(abs(before_requested) <= API_RELATIVE_TIME_MAX) { if(before_requested > 0) before_requested = first_entry_t + before_requested; - else before_requested = last_entry_t + before_requested; + else before_requested = last_entry_t + before_requested; //last_entry_t is not really now_t + //TODO: fix before_requested to be relative to now_t absolute_period_requested = 0; } // allow relative for after (smaller than API_RELATIVE_TIME_MAX) - if(((after_requested < 0)?-after_requested:after_requested) <= API_RELATIVE_TIME_MAX) { - if(after_requested == 0) after_requested = -st->update_every; - if(abs(after_requested) % st->update_every) { - // make sure it is multiple of st->update_every - if(after_requested < 0) after_requested = after_requested - st->update_every - after_requested % st->update_every; - else after_requested = after_requested + st->update_every - after_requested % st->update_every; - } + if(abs(after_requested) <= API_RELATIVE_TIME_MAX) { after_requested = before_requested + after_requested; absolute_period_requested = 0; } @@ -654,9 +778,53 @@ RRDR *rrd2rrdr( after_requested = tmp; } + *before_requestedp = before_requested; + *after_requestedp = after_requested; + + return absolute_period_requested; +} + +static RRDR *rrd2rrdr_fixedstep( + RRDSET *st + , long points_requested + , long long after_requested + , long long before_requested + , RRDR_GROUPING group_method + , long resampling_time_requested + , RRDR_OPTIONS options + , const char *dimensions + , int update_every + , time_t first_entry_t + , time_t last_entry_t + , int absolute_period_requested +) { + int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); + + if(!absolute_period_requested) { + if(before_requested % update_every) { + // make sure it is multiple of update_every + if(before_requested > 0) + before_requested = before_requested - update_every + before_requested % update_every; + #ifdef NETDATA_INTERNAL_CHECKS + else + error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero before_requested", st->name); + #endif + } + if(after_requested % update_every) { + // make sure it is multiple of update_every + if(after_requested < 0) + after_requested = after_requested - update_every + after_requested % update_every; + #ifdef NETDATA_INTERNAL_CHECKS + else + error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero after_requested", st->name); + #endif + } + if(after_requested == before_requested) after_requested -= update_every; + } + // the duration of the chart time_t duration = before_requested - after_requested; - long available_points = duration / st->update_every; + long available_points = duration / update_every; if(duration <= 0 || available_points <= 0) return rrdr_create(st, 1); @@ -674,7 +842,7 @@ RRDR *rrd2rrdr( // resampling_time_requested enforces a certain grouping multiple calculated_number resampling_divisor = 1.0; long resampling_group = 1; - if(unlikely(resampling_time_requested > st->update_every)) { + if(unlikely(resampling_time_requested > update_every)) { if (unlikely(resampling_time_requested > duration)) { // group_time is above the available duration @@ -684,7 +852,7 @@ RRDR *rrd2rrdr( after_requested = before_requested - resampling_time_requested; duration = before_requested - after_requested; - available_points = duration / st->update_every; + available_points = duration / update_every; group = available_points / points_requested; } @@ -696,16 +864,16 @@ RRDR *rrd2rrdr( if(delta > resampling_time_requested / 10) { after_requested -= resampling_time_requested - delta; duration = before_requested - after_requested; - available_points = duration / st->update_every; + available_points = duration / update_every; group = available_points / points_requested; } } // the points we should group to satisfy gtime - resampling_group = resampling_time_requested / st->update_every; - if(unlikely(resampling_time_requested % st->update_every)) { + resampling_group = resampling_time_requested / update_every; + if(unlikely(resampling_time_requested % update_every)) { #ifdef NETDATA_INTERNAL_CHECKS - info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, st->update_every); + info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every); #endif resampling_group++; @@ -716,7 +884,7 @@ RRDR *rrd2rrdr( if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group //resampling_divisor = group / resampling_group; - resampling_divisor = (calculated_number)(group * st->update_every) / (calculated_number)resampling_time_requested; + resampling_divisor = (calculated_number)(group * update_every) / (calculated_number)resampling_time_requested; } // now that we have group, @@ -724,8 +892,8 @@ RRDR *rrd2rrdr( if(aligned) { // alignement has been requested, so align the values - before_requested -= (before_requested % group); - after_requested -= (after_requested % group); + before_requested -= before_requested % (group * update_every); + after_requested -= after_requested % (group * update_every); } // we align the request on requested_before @@ -735,28 +903,28 @@ RRDR *rrd2rrdr( error("INTERNAL ERROR: rrd2rrdr() on %s, before_wanted is after db max", st->name); #endif - before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * st->update_every )); + before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * update_every )); } //size_t before_slot = rrdset_time2slot(st, before_wanted); // we need to estimate the number of points, for having // an integer number of values per point - long points_wanted = (before_wanted - after_requested) / (st->update_every * group); + long points_wanted = (before_wanted - after_requested) / (update_every * group); - time_t after_wanted = before_wanted - (points_wanted * group * st->update_every) + st->update_every; + time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; if(unlikely(after_wanted < first_entry_t)) { // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning points_wanted = (before_wanted - first_entry_t) / group; // recalculate after wanted with the new number of points - after_wanted = before_wanted - (points_wanted * group * st->update_every) + st->update_every; + after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; if(unlikely(after_wanted < first_entry_t)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name); #endif - after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * st->update_every )) + ( ((aligned)?group:1) * st->update_every ); + after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every ); } } //size_t after_slot = rrdset_time2slot(st, after_wanted); @@ -772,7 +940,7 @@ RRDR *rrd2rrdr( } // recalculate points_wanted using the final time-frame - points_wanted = (before_wanted - after_wanted) / st->update_every / group + 1; + points_wanted = (before_wanted - after_wanted) / update_every / group + 1; if(unlikely(points_wanted < 0)) { #ifdef NETDATA_INTERNAL_CHECKS error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted); @@ -803,8 +971,8 @@ RRDR *rrd2rrdr( error("INTERNAL CHECK: after_slot is invalid %zu, expected 0 to %ld", after_slot, st->entries - 1); */ - if(points_wanted > (before_wanted - after_wanted) / group / st->update_every + 1) - error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / st->update_every + 1); + if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1) + error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1); if(group < resampling_group) error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group); @@ -844,7 +1012,7 @@ RRDR *rrd2rrdr( // initialize RRDR r->group = group; - r->update_every = (int)group * st->update_every; + r->update_every = (int)group * update_every; r->before = before_wanted; r->after = after_wanted; r->internal.points_wanted = points_wanted; @@ -913,7 +1081,7 @@ RRDR *rrd2rrdr( // reset the grouping for the new dimension r->internal.grouping_reset(r); - do_dimension( + do_dimension_fixedstep( r , points_wanted , rd @@ -961,30 +1129,426 @@ RRDR *rrd2rrdr( } #ifdef NETDATA_INTERNAL_CHECKS + if (dimensions_used) { + if(r->internal.log) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log); + + if(r->rows != points_wanted) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'"); + + if(aligned && (r->before % group) != 0) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required"); + + // 'after' should not be aligned, since we start inside the first group + //if(aligned && (r->after % group) != 0) + // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); + + if(r->before != before_requested) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'"); + + if(r->before != before_wanted) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'"); + + // reported 'after' varies, depending on group + if(r->after != after_wanted) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'"); + } + #endif + + // free all resources used by the grouping method + r->internal.grouping_free(r); + + // when all the dimensions are zero, we should return all of them + if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) { + // all the dimensions are zero + // mark them as NONZERO to send them all + for(rd = st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { + if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; + r->od[c] |= RRDR_DIMENSION_NONZERO; + } + } + + rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated); + return r; +} + +#ifdef ENABLE_DBENGINE +static RRDR *rrd2rrdr_variablestep( + RRDSET *st + , long points_requested + , long long after_requested + , long long before_requested + , RRDR_GROUPING group_method + , long resampling_time_requested + , RRDR_OPTIONS options + , const char *dimensions + , int update_every + , time_t first_entry_t + , time_t last_entry_t + , int absolute_period_requested + , struct rrdeng_region_info *region_info_array +) { + int aligned = !(options & RRDR_OPTION_NOT_ALIGNED); + + if(!absolute_period_requested) { + if(before_requested % update_every) { + // make sure it is multiple of update_every + if(before_requested > 0) + before_requested = before_requested - before_requested % update_every; + #ifdef NETDATA_INTERNAL_CHECKS + else + error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero before_requested", st->name); + #endif + } + if(after_requested % update_every) { + // make sure it is multiple of update_every + if(after_requested < 0) + after_requested = after_requested - after_requested % update_every; + #ifdef NETDATA_INTERNAL_CHECKS + else + error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero after_requested", st->name); + #endif + } + if(after_requested == before_requested) after_requested -= update_every; + } + + // the duration of the chart + time_t duration = before_requested - after_requested; + long available_points = duration / update_every; + + if(duration <= 0 || available_points <= 0) { + freez(region_info_array); + return rrdr_create(st, 1); + } + + // check the number of wanted points in the result + if(unlikely(points_requested < 0)) points_requested = -points_requested; + if(unlikely(points_requested > available_points)) points_requested = available_points; + if(unlikely(points_requested == 0)) points_requested = available_points; + + // calculate the desired grouping of source data points + long group = available_points / points_requested; + if(unlikely(group <= 0)) group = 1; + if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer + + // resampling_time_requested enforces a certain grouping multiple + calculated_number resampling_divisor = 1.0; + long resampling_group = 1; + if(unlikely(resampling_time_requested > update_every)) { + if (unlikely(resampling_time_requested > duration)) { + // group_time is above the available duration + + #ifdef NETDATA_INTERNAL_CHECKS + info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, resampling_time_requested, duration); + #endif + + after_requested = before_requested - resampling_time_requested; + duration = before_requested - after_requested; + available_points = duration / update_every; + group = available_points / points_requested; + } + + // if the duration is not aligned to resampling time + // extend the duration to the past, to avoid a gap at the chart + // only when the missing duration is above 1/10th of a point + if(duration % resampling_time_requested) { + time_t delta = duration % resampling_time_requested; + if(delta > resampling_time_requested / 10) { + after_requested -= resampling_time_requested - delta; + duration = before_requested - after_requested; + available_points = duration / update_every; + group = available_points / points_requested; + } + } + + // the points we should group to satisfy gtime + resampling_group = resampling_time_requested / update_every; + if(unlikely(resampling_time_requested % update_every)) { + #ifdef NETDATA_INTERNAL_CHECKS + info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every); + #endif + + resampling_group++; + } + + // adapt group according to resampling_group + if(unlikely(group < resampling_group)) group = resampling_group; // do not allow grouping below the desired one + if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group + + //resampling_divisor = group / resampling_group; + resampling_divisor = (calculated_number)(group * update_every) / (calculated_number)resampling_time_requested; + } + + // now that we have group, + // align the requested timeframe to fit it. + + if(aligned) { + // alignement has been requested, so align the values + before_requested -= before_requested % (group * update_every); + after_requested -= after_requested % (group * update_every); + } + + // we align the request on requested_before + time_t before_wanted = before_requested; + if(likely(before_wanted > last_entry_t)) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: rrd2rrdr() on %s, before_wanted is after db max", st->name); + #endif + + before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * update_every )); + } + //size_t before_slot = rrdset_time2slot(st, before_wanted); + + // we need to estimate the number of points, for having + // an integer number of values per point + long points_wanted = (before_wanted - after_requested) / (update_every * group); + + time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; + if(unlikely(after_wanted < first_entry_t)) { + // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning + points_wanted = (before_wanted - first_entry_t) / group; + + // recalculate after wanted with the new number of points + after_wanted = before_wanted - (points_wanted * group * update_every) + update_every; + + if(unlikely(after_wanted < first_entry_t)) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name); + #endif + + after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every ); + } + } + //size_t after_slot = rrdset_time2slot(st, after_wanted); + + // check if they are reversed + if(unlikely(after_wanted > before_wanted)) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: rrd2rrdr() on %s, reversed wanted after/before", st->name); + #endif + time_t tmp = before_wanted; + before_wanted = after_wanted; + after_wanted = tmp; + } - if(r->internal.log) - rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log); + // recalculate points_wanted using the final time-frame + points_wanted = (before_wanted - after_wanted) / update_every / group + 1; + if(unlikely(points_wanted < 0)) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted); + #endif + points_wanted = 0; + } + +#ifdef NETDATA_INTERNAL_CHECKS + duration = before_wanted - after_wanted; - if(r->rows != points_wanted) - rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'"); + if(after_wanted < first_entry_t) + error("INTERNAL CHECK: after_wanted %u is too small, minimum %u", (uint32_t)after_wanted, (uint32_t)first_entry_t); - if(aligned && (r->before % group) != 0) - rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required"); + if(after_wanted > last_entry_t) + error("INTERNAL CHECK: after_wanted %u is too big, maximum %u", (uint32_t)after_wanted, (uint32_t)last_entry_t); - // 'after' should not be aligned, since we start inside the first group - //if(aligned && (r->after % group) != 0) - // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); + if(before_wanted < first_entry_t) + error("INTERNAL CHECK: before_wanted %u is too small, minimum %u", (uint32_t)before_wanted, (uint32_t)first_entry_t); - if(r->before != before_requested) - rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'"); + if(before_wanted > last_entry_t) + error("INTERNAL CHECK: before_wanted %u is too big, maximum %u", (uint32_t)before_wanted, (uint32_t)last_entry_t); - if(r->before != before_wanted) - rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'"); +/* + if(before_slot >= (size_t)st->entries) + error("INTERNAL CHECK: before_slot is invalid %zu, expected 0 to %ld", before_slot, st->entries - 1); - // reported 'after' varies, depending on group - if(r->after != after_wanted) - rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'"); + if(after_slot >= (size_t)st->entries) + error("INTERNAL CHECK: after_slot is invalid %zu, expected 0 to %ld", after_slot, st->entries - 1); +*/ + if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1) + error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1); + + if(group < resampling_group) + error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group); + + if(group > resampling_group && group % resampling_group) + error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, resampling_group); +#endif + + // ------------------------------------------------------------------------- + // initialize our result set + // this also locks the chart for us + + RRDR *r = rrdr_create(st, points_wanted); + if(unlikely(!r)) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); + #endif + freez(region_info_array); + return NULL; + } + + if(unlikely(!r->d || !points_wanted)) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); + #endif + freez(region_info_array); + return r; + } + + r->result_options |= RRDR_RESULT_OPTION_VARIABLE_STEP; + if(unlikely(absolute_period_requested == 1)) + r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; + else + r->result_options |= RRDR_RESULT_OPTION_RELATIVE; + + // find how many dimensions we have + long dimensions_count = r->d; + + // ------------------------------------------------------------------------- + // initialize RRDR + + r->group = group; + r->update_every = (int)group * update_every; + r->before = before_wanted; + r->after = after_wanted; + r->internal.points_wanted = points_wanted; + r->internal.resampling_group = resampling_group; + r->internal.resampling_divisor = resampling_divisor; + + + // ------------------------------------------------------------------------- + // assign the processor functions + + { + int i, found = 0; + for(i = 0; !found && api_v1_data_groups[i].name ;i++) { + if(api_v1_data_groups[i].value == group_method) { + r->internal.grouping_create= api_v1_data_groups[i].create; + r->internal.grouping_reset = api_v1_data_groups[i].reset; + r->internal.grouping_free = api_v1_data_groups[i].free; + r->internal.grouping_add = api_v1_data_groups[i].add; + r->internal.grouping_flush = api_v1_data_groups[i].flush; + found = 1; + } + } + if(!found) { + errno = 0; + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name); + #endif + r->internal.grouping_create= grouping_create_average; + r->internal.grouping_reset = grouping_reset_average; + r->internal.grouping_free = grouping_free_average; + r->internal.grouping_add = grouping_add_average; + r->internal.grouping_flush = grouping_flush_average; + } + } + + // allocate any memory required by the grouping method + r->internal.grouping_data = r->internal.grouping_create(r); + + + // ------------------------------------------------------------------------- + // disable the not-wanted dimensions + + rrdset_check_rdlock(st); + + if(dimensions) + rrdr_disable_not_selected_dimensions(r, options, dimensions); + + + // ------------------------------------------------------------------------- + // do the work for each dimension + + time_t max_after = 0, min_before = 0; + long max_rows = 0; + + RRDDIM *rd; + long c, dimensions_used = 0, dimensions_nonzero = 0; + for(rd = st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { + + // if we need a percentage, we need to calculate all dimensions + if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) { + if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED; + continue; + } + r->od[c] |= RRDR_DIMENSION_SELECTED; + + // reset the grouping for the new dimension + r->internal.grouping_reset(r); + + do_dimension_variablestep( + r + , points_wanted + , rd + , c + , after_wanted + , before_wanted + ); + + if(r->od[c] & RRDR_DIMENSION_NONZERO) + dimensions_nonzero++; + + // verify all dimensions are aligned + if(unlikely(!dimensions_used)) { + min_before = r->before; + max_after = r->after; + max_rows = r->rows; + } + else { + if(r->after != max_after) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", + st->name, (size_t)max_after, rd->name, (size_t)r->after); + #endif + r->after = (r->after > max_after) ? r->after : max_after; + } + + if(r->before != min_before) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", + st->name, (size_t)min_before, rd->name, (size_t)r->before); + #endif + r->before = (r->before < min_before) ? r->before : min_before; + } + + if(r->rows != max_rows) { + #ifdef NETDATA_INTERNAL_CHECKS + error("INTERNAL ERROR: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", + st->name, (size_t)max_rows, rd->name, (size_t)r->rows); + #endif + r->rows = (r->rows > max_rows) ? r->rows : max_rows; + } + } + + dimensions_used++; + } + + #ifdef NETDATA_INTERNAL_CHECKS + + if (dimensions_used) { + if(r->internal.log) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log); + + if(r->rows != points_wanted) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'"); + + if(aligned && (r->before % group) != 0) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required"); + + // 'after' should not be aligned, since we start inside the first group + //if(aligned && (r->after % group) != 0) + // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); + + if(r->before != before_requested) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'"); + + if(r->before != before_wanted) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'"); + + // reported 'after' varies, depending on group + if(r->after != after_wanted) + rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'"); + } #endif // free all resources used by the grouping method @@ -1001,5 +1565,57 @@ RRDR *rrd2rrdr( } rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated); + freez(region_info_array); return r; } +#endif //#ifdef ENABLE_DBENGINE + +RRDR *rrd2rrdr( + RRDSET *st + , long points_requested + , long long after_requested + , long long before_requested + , RRDR_GROUPING group_method + , long resampling_time_requested + , RRDR_OPTIONS options + , const char *dimensions +) { + int rrd_update_every; + int absolute_period_requested; + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); + + absolute_period_requested = rrdr_convert_before_after_to_absolute(&after_requested, &before_requested, + first_entry_t, last_entry_t); + +#ifdef ENABLE_DBENGINE + if ((st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)) { + struct rrdeng_region_info *region_info_array; + unsigned regions, max_interval; + + /* This call takes the chart read-lock */ + regions = rrdeng_variable_step_boundaries(st, after_requested, before_requested, + ®ion_info_array, &max_interval); + if (1 == regions) { + if (region_info_array) + rrd_update_every = region_info_array[0].update_every; + else + rrd_update_every = st->update_every; + if (region_info_array) + freez(region_info_array); + return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, + resampling_time_requested, options, dimensions, rrd_update_every, + first_entry_t, last_entry_t, absolute_period_requested); + } else { + rrd_update_every = (uint16_t)max_interval; + return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method, + resampling_time_requested, options, dimensions, rrd_update_every, + first_entry_t, last_entry_t, absolute_period_requested, region_info_array); + } + } +#endif + rrd_update_every = st->update_every; + return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method, + resampling_time_requested, options, dimensions, + rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested); +}
\ No newline at end of file |