summaryrefslogtreecommitdiffstats
path: root/web/api/queries/query.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--web/api/queries/query.c876
1 files changed, 746 insertions, 130 deletions
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index e90dc8afe..af3bcfe38 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -376,7 +376,7 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) {
// ----------------------------------------------------------------------------
// fill RRDR for a single dimension
-static inline void do_dimension(
+static inline void do_dimension_variablestep(
RRDR *r
, long points_wanted
, RRDDIM *rd
@@ -384,16 +384,16 @@ static inline void do_dimension(
, time_t after_wanted
, time_t before_wanted
){
- RRDSET *st = r->st;
+// RRDSET *st = r->st;
time_t
now = after_wanted,
- dt = st->update_every,
+ dt = r->update_every,
max_date = 0,
min_date = 0;
long
- group_size = r->group,
+// group_size = r->group,
points_added = 0,
values_in_group = 0,
values_in_group_non_zero = 0,
@@ -403,102 +403,240 @@ static inline void do_dimension(
group_value_flags = RRDR_VALUE_NOTHING;
struct rrddim_query_handle handle;
- uint8_t initialized_query;
calculated_number min = r->min, max = r->max;
size_t db_points_read = 0;
+ time_t db_now = now;
+ storage_number n_curr, n_prev = SN_EMPTY_SLOT;
+ calculated_number value;
+
+ for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) {
+ // make sure we return data in the proper time range
+ if (unlikely(now > before_wanted)) {
+#ifdef NETDATA_INTERNAL_CHECKS
+ r->internal.log = "stopped, because attempted to access the db after 'wanted before'";
+#endif
+ break;
+ }
+ if (unlikely(now < after_wanted)) {
+#ifdef NETDATA_INTERNAL_CHECKS
+ r->internal.log = "skipped, because attempted to access the db before 'wanted after'";
+#endif
+ continue;
+ }
+
+ while (now >= db_now && (!rd->state->query_ops.is_finished(&handle) ||
+ does_storage_number_exist(n_prev))) {
+ value = NAN;
+ if (does_storage_number_exist(n_prev)) {
+ // use the previously read database value
+ n_curr = n_prev;
+ } else {
+ // read the value from the database
+ n_curr = rd->state->query_ops.next_metric(&handle, &db_now);
+ }
+ n_prev = SN_EMPTY_SLOT;
+ // db_now has a different value than above
+ if (likely(now >= db_now)) {
+ if (likely(does_storage_number_exist(n_curr))) {
+ value = unpack_storage_number(n_curr);
+ if (likely(value != 0.0))
+ values_in_group_non_zero++;
+
+ if (unlikely(did_storage_number_reset(n_curr)))
+ group_value_flags |= RRDR_VALUE_RESET;
+ }
+ } else {
+ // We must postpone processing the value and fill the result with gaps instead
+ if (likely(does_storage_number_exist(n_curr))) {
+ n_prev = n_curr;
+ }
+ }
+ // add this value to grouping
+ r->internal.grouping_add(r, value);
+ values_in_group++;
+ db_points_read++;
+ }
+
+ if (0 == values_in_group) {
+ // add NAN to grouping
+ r->internal.grouping_add(r, NAN);
+ }
+
+ rrdr_line = rrdr_line_init(r, now, rrdr_line);
+
+ if(unlikely(!min_date)) min_date = now;
+ max_date = now;
+
+ // find the place to store our values
+ RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr];
+
+ // update the dimension options
+ if(likely(values_in_group_non_zero))
+ r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
+
+ // store the specific point options
+ *rrdr_value_options_ptr = group_value_flags;
+
+ // store the value
+ value = r->internal.grouping_flush(r, rrdr_value_options_ptr);
+ r->v[rrdr_line * r->d + dim_id_in_rrdr] = value;
+
+ if(likely(points_added || dim_id_in_rrdr)) {
+ // find the min/max across all dimensions
+
+ if(unlikely(value < min)) min = value;
+ if(unlikely(value > max)) max = value;
+
+ }
+ else {
+ // runs only when dim_id_in_rrdr == 0 && points_added == 0
+ // so, on the first point added for the query.
+ min = max = value;
+ }
+
+ points_added++;
+ values_in_group = 0;
+ group_value_flags = RRDR_VALUE_NOTHING;
+ values_in_group_non_zero = 0;
+ }
+ rd->state->query_ops.finalize(&handle);
+
+ r->internal.db_points_read += db_points_read;
+ r->internal.result_points_generated += points_added;
+
+ r->min = min;
+ r->max = max;
+ r->before = max_date;
+ r->after = min_date - (r->group - 1) * dt;
+ rrdr_done(r, rrdr_line);
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(r->rows != points_added))
+ error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows);
+ #endif
+}
- for(initialized_query = 0 ; points_added < points_wanted ; now += dt) {
+static inline void do_dimension_fixedstep(
+ RRDR *r
+ , long points_wanted
+ , RRDDIM *rd
+ , long dim_id_in_rrdr
+ , time_t after_wanted
+ , time_t before_wanted
+){
+ RRDSET *st = r->st;
+
+ time_t
+ now = after_wanted,
+ dt = r->update_every / r->group, /* usually is st->update_every */
+ max_date = 0,
+ min_date = 0;
+ long
+ group_size = r->group,
+ points_added = 0,
+ values_in_group = 0,
+ values_in_group_non_zero = 0,
+ rrdr_line = -1;
+
+ RRDR_VALUE_FLAGS
+ group_value_flags = RRDR_VALUE_NOTHING;
+
+ struct rrddim_query_handle handle;
+
+ calculated_number min = r->min, max = r->max;
+ size_t db_points_read = 0;
+ time_t db_now = now;
+
+ for(rd->state->query_ops.init(rd, &handle, now, before_wanted) ; points_added < points_wanted ; now += dt) {
// make sure we return data in the proper time range
if(unlikely(now > before_wanted)) {
- #ifdef NETDATA_INTERNAL_CHECKS
+#ifdef NETDATA_INTERNAL_CHECKS
r->internal.log = "stopped, because attempted to access the db after 'wanted before'";
- #endif
+#endif
break;
}
if(unlikely(now < after_wanted)) {
- #ifdef NETDATA_INTERNAL_CHECKS
+#ifdef NETDATA_INTERNAL_CHECKS
r->internal.log = "skipped, because attempted to access the db before 'wanted after'";
- #endif
+#endif
continue;
}
-
- if (unlikely(!initialized_query)) {
- rd->state->query_ops.init(rd, &handle, now, before_wanted);
- initialized_query = 1;
- }
// read the value from the database
//storage_number n = rd->values[slot];
#ifdef NETDATA_INTERNAL_CHECKS
- if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-#ifdef ENABLE_DBENGINE
- if (now != handle.rrdeng.now)
- error("INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", rd->id, (long)handle.rrdeng.now, (long)now);
-#endif
- } else if (rrdset_time2slot(st, now) != (long unsigned)handle.slotted.slot) {
+ if ((rd->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) &&
+ (rrdset_time2slot(st, now) != (long unsigned)handle.slotted.slot)) {
error("INTERNAL CHECK: Unaligned query for %s, database slot: %lu, expected slot: %lu", rd->id, (long unsigned)handle.slotted.slot, rrdset_time2slot(st, now));
}
#endif
- storage_number n = rd->state->query_ops.next_metric(&handle);
- calculated_number value = NAN;
- if(likely(does_storage_number_exist(n))) {
+ db_now = now; // this is needed to set db_now in case the next_metric implementation does not set it
+ storage_number n = rd->state->query_ops.next_metric(&handle, &db_now);
+ for ( ; now <= db_now ; now += dt) {
+ calculated_number value = NAN;
+ if(likely(now >= db_now && does_storage_number_exist(n))) {
+#if defined(NETDATA_INTERNAL_CHECKS) && defined(ENABLE_DBENGINE)
+ if ((rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && (now != handle.rrdeng.now)) {
+ error("INTERNAL CHECK: Unaligned query for %s, database time: %ld, expected time: %ld", rd->id, (long)handle.rrdeng.now, (long)now);
+ }
+#endif
+ value = unpack_storage_number(n);
+ if(likely(value != 0.0))
+ values_in_group_non_zero++;
- value = unpack_storage_number(n);
- if(likely(value != 0.0))
- values_in_group_non_zero++;
+ if(unlikely(did_storage_number_reset(n)))
+ group_value_flags |= RRDR_VALUE_RESET;
- if(unlikely(did_storage_number_reset(n)))
- group_value_flags |= RRDR_VALUE_RESET;
+ }
- }
+ // add this value for grouping
+ r->internal.grouping_add(r, value);
+ values_in_group++;
+ db_points_read++;
- // add this value for grouping
- r->internal.grouping_add(r, value);
- values_in_group++;
- db_points_read++;
+ if(unlikely(values_in_group == group_size)) {
+ rrdr_line = rrdr_line_init(r, now, rrdr_line);
- if(unlikely(values_in_group == group_size)) {
- rrdr_line = rrdr_line_init(r, now, rrdr_line);
+ if(unlikely(!min_date)) min_date = now;
+ max_date = now;
- if(unlikely(!min_date)) min_date = now;
- max_date = now;
+ // find the place to store our values
+ RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr];
- // find the place to store our values
- RRDR_VALUE_FLAGS *rrdr_value_options_ptr = &r->o[rrdr_line * r->d + dim_id_in_rrdr];
+ // update the dimension options
+ if(likely(values_in_group_non_zero))
+ r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
- // update the dimension options
- if(likely(values_in_group_non_zero))
- r->od[dim_id_in_rrdr] |= RRDR_DIMENSION_NONZERO;
+ // store the specific point options
+ *rrdr_value_options_ptr = group_value_flags;
- // store the specific point options
- *rrdr_value_options_ptr = group_value_flags;
+ // store the value
+ calculated_number value = r->internal.grouping_flush(r, rrdr_value_options_ptr);
+ r->v[rrdr_line * r->d + dim_id_in_rrdr] = value;
- // store the value
- calculated_number value = r->internal.grouping_flush(r, rrdr_value_options_ptr);
- r->v[rrdr_line * r->d + dim_id_in_rrdr] = value;
+ if(likely(points_added || dim_id_in_rrdr)) {
+ // find the min/max across all dimensions
- if(likely(points_added || dim_id_in_rrdr)) {
- // find the min/max across all dimensions
+ if(unlikely(value < min)) min = value;
+ if(unlikely(value > max)) max = value;
- if(unlikely(value < min)) min = value;
- if(unlikely(value > max)) max = value;
+ }
+ else {
+ // runs only when dim_id_in_rrdr == 0 && points_added == 0
+ // so, on the first point added for the query.
+ min = max = value;
+ }
+ points_added++;
+ values_in_group = 0;
+ group_value_flags = RRDR_VALUE_NOTHING;
+ values_in_group_non_zero = 0;
}
- else {
- // runs only when dim_id_in_rrdr == 0 && points_added == 0
- // so, on the first point added for the query.
- min = max = value;
- }
-
- points_added++;
- values_in_group = 0;
- group_value_flags = RRDR_VALUE_NOTHING;
- values_in_group_non_zero = 0;
}
+ now = db_now;
}
- if (likely(initialized_query))
- rd->state->query_ops.finalize(&handle);
+ rd->state->query_ops.finalize(&handle);
r->internal.db_points_read += db_points_read;
r->internal.result_points_generated += points_added;
@@ -506,13 +644,13 @@ static inline void do_dimension(
r->min = min;
r->max = max;
r->before = max_date;
- r->after = min_date - (r->group - 1) * r->st->update_every;
+ r->after = min_date - (r->group - 1) * dt;
rrdr_done(r, rrdr_line);
- #ifdef NETDATA_INTERNAL_CHECKS
+#ifdef NETDATA_INTERNAL_CHECKS
if(unlikely(r->rows != points_added))
error("INTERNAL ERROR: %s.%s added %zu rows, but RRDR says I added %zu.", r->st->name, rd->name, (size_t)points_added, (size_t)r->rows);
- #endif
+#endif
}
// ----------------------------------------------------------------------------
@@ -589,22 +727,18 @@ static void rrd2rrdr_log_request_response_metdata(RRDR *r
}
#endif // NETDATA_INTERNAL_CHECKS
-RRDR *rrd2rrdr(
- RRDSET *st
- , long points_requested
- , long long after_requested
- , long long before_requested
- , RRDR_GROUPING group_method
- , long resampling_time_requested
- , RRDR_OPTIONS options
- , const char *dimensions
+// Returns 1 if an absolute period was requested or 0 if it was a relative period
+static int rrdr_convert_before_after_to_absolute(
+ long long *after_requestedp
+ , long long *before_requestedp
+ , time_t first_entry_t
+ , time_t last_entry_t
) {
- int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
-
int absolute_period_requested = -1;
+ long long after_requested, before_requested;
- time_t first_entry_t = rrdset_first_entry_t(st);
- time_t last_entry_t = rrdset_last_entry_t(st);
+ before_requested = *before_requestedp;
+ after_requested = *after_requestedp;
if(before_requested == 0 && after_requested == 0) {
// dump the all the data
@@ -614,25 +748,15 @@ RRDR *rrd2rrdr(
}
// allow relative for before (smaller than API_RELATIVE_TIME_MAX)
- if(((before_requested < 0)?-before_requested:before_requested) <= API_RELATIVE_TIME_MAX) {
- if(abs(before_requested) % st->update_every) {
- // make sure it is multiple of st->update_every
- if(before_requested < 0) before_requested = before_requested - st->update_every - before_requested % st->update_every;
- else before_requested = before_requested + st->update_every - before_requested % st->update_every;
- }
+ if(abs(before_requested) <= API_RELATIVE_TIME_MAX) {
if(before_requested > 0) before_requested = first_entry_t + before_requested;
- else before_requested = last_entry_t + before_requested;
+ else before_requested = last_entry_t + before_requested; //last_entry_t is not really now_t
+ //TODO: fix before_requested to be relative to now_t
absolute_period_requested = 0;
}
// allow relative for after (smaller than API_RELATIVE_TIME_MAX)
- if(((after_requested < 0)?-after_requested:after_requested) <= API_RELATIVE_TIME_MAX) {
- if(after_requested == 0) after_requested = -st->update_every;
- if(abs(after_requested) % st->update_every) {
- // make sure it is multiple of st->update_every
- if(after_requested < 0) after_requested = after_requested - st->update_every - after_requested % st->update_every;
- else after_requested = after_requested + st->update_every - after_requested % st->update_every;
- }
+ if(abs(after_requested) <= API_RELATIVE_TIME_MAX) {
after_requested = before_requested + after_requested;
absolute_period_requested = 0;
}
@@ -654,9 +778,53 @@ RRDR *rrd2rrdr(
after_requested = tmp;
}
+ *before_requestedp = before_requested;
+ *after_requestedp = after_requested;
+
+ return absolute_period_requested;
+}
+
+static RRDR *rrd2rrdr_fixedstep(
+ RRDSET *st
+ , long points_requested
+ , long long after_requested
+ , long long before_requested
+ , RRDR_GROUPING group_method
+ , long resampling_time_requested
+ , RRDR_OPTIONS options
+ , const char *dimensions
+ , int update_every
+ , time_t first_entry_t
+ , time_t last_entry_t
+ , int absolute_period_requested
+) {
+ int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
+
+ if(!absolute_period_requested) {
+ if(before_requested % update_every) {
+ // make sure it is multiple of update_every
+ if(before_requested > 0)
+ before_requested = before_requested - update_every + before_requested % update_every;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ else
+ error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero before_requested", st->name);
+ #endif
+ }
+ if(after_requested % update_every) {
+ // make sure it is multiple of update_every
+ if(after_requested < 0)
+ after_requested = after_requested - update_every + after_requested % update_every;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ else
+ error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero after_requested", st->name);
+ #endif
+ }
+ if(after_requested == before_requested) after_requested -= update_every;
+ }
+
// the duration of the chart
time_t duration = before_requested - after_requested;
- long available_points = duration / st->update_every;
+ long available_points = duration / update_every;
if(duration <= 0 || available_points <= 0)
return rrdr_create(st, 1);
@@ -674,7 +842,7 @@ RRDR *rrd2rrdr(
// resampling_time_requested enforces a certain grouping multiple
calculated_number resampling_divisor = 1.0;
long resampling_group = 1;
- if(unlikely(resampling_time_requested > st->update_every)) {
+ if(unlikely(resampling_time_requested > update_every)) {
if (unlikely(resampling_time_requested > duration)) {
// group_time is above the available duration
@@ -684,7 +852,7 @@ RRDR *rrd2rrdr(
after_requested = before_requested - resampling_time_requested;
duration = before_requested - after_requested;
- available_points = duration / st->update_every;
+ available_points = duration / update_every;
group = available_points / points_requested;
}
@@ -696,16 +864,16 @@ RRDR *rrd2rrdr(
if(delta > resampling_time_requested / 10) {
after_requested -= resampling_time_requested - delta;
duration = before_requested - after_requested;
- available_points = duration / st->update_every;
+ available_points = duration / update_every;
group = available_points / points_requested;
}
}
// the points we should group to satisfy gtime
- resampling_group = resampling_time_requested / st->update_every;
- if(unlikely(resampling_time_requested % st->update_every)) {
+ resampling_group = resampling_time_requested / update_every;
+ if(unlikely(resampling_time_requested % update_every)) {
#ifdef NETDATA_INTERNAL_CHECKS
- info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, st->update_every);
+ info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every);
#endif
resampling_group++;
@@ -716,7 +884,7 @@ RRDR *rrd2rrdr(
if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
//resampling_divisor = group / resampling_group;
- resampling_divisor = (calculated_number)(group * st->update_every) / (calculated_number)resampling_time_requested;
+ resampling_divisor = (calculated_number)(group * update_every) / (calculated_number)resampling_time_requested;
}
// now that we have group,
@@ -724,8 +892,8 @@ RRDR *rrd2rrdr(
if(aligned) {
// alignement has been requested, so align the values
- before_requested -= (before_requested % group);
- after_requested -= (after_requested % group);
+ before_requested -= before_requested % (group * update_every);
+ after_requested -= after_requested % (group * update_every);
}
// we align the request on requested_before
@@ -735,28 +903,28 @@ RRDR *rrd2rrdr(
error("INTERNAL ERROR: rrd2rrdr() on %s, before_wanted is after db max", st->name);
#endif
- before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * st->update_every ));
+ before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * update_every ));
}
//size_t before_slot = rrdset_time2slot(st, before_wanted);
// we need to estimate the number of points, for having
// an integer number of values per point
- long points_wanted = (before_wanted - after_requested) / (st->update_every * group);
+ long points_wanted = (before_wanted - after_requested) / (update_every * group);
- time_t after_wanted = before_wanted - (points_wanted * group * st->update_every) + st->update_every;
+ time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;
if(unlikely(after_wanted < first_entry_t)) {
// hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning
points_wanted = (before_wanted - first_entry_t) / group;
// recalculate after wanted with the new number of points
- after_wanted = before_wanted - (points_wanted * group * st->update_every) + st->update_every;
+ after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;
if(unlikely(after_wanted < first_entry_t)) {
#ifdef NETDATA_INTERNAL_CHECKS
error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name);
#endif
- after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * st->update_every )) + ( ((aligned)?group:1) * st->update_every );
+ after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every );
}
}
//size_t after_slot = rrdset_time2slot(st, after_wanted);
@@ -772,7 +940,7 @@ RRDR *rrd2rrdr(
}
// recalculate points_wanted using the final time-frame
- points_wanted = (before_wanted - after_wanted) / st->update_every / group + 1;
+ points_wanted = (before_wanted - after_wanted) / update_every / group + 1;
if(unlikely(points_wanted < 0)) {
#ifdef NETDATA_INTERNAL_CHECKS
error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted);
@@ -803,8 +971,8 @@ RRDR *rrd2rrdr(
error("INTERNAL CHECK: after_slot is invalid %zu, expected 0 to %ld", after_slot, st->entries - 1);
*/
- if(points_wanted > (before_wanted - after_wanted) / group / st->update_every + 1)
- error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / st->update_every + 1);
+ if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1)
+ error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1);
if(group < resampling_group)
error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group);
@@ -844,7 +1012,7 @@ RRDR *rrd2rrdr(
// initialize RRDR
r->group = group;
- r->update_every = (int)group * st->update_every;
+ r->update_every = (int)group * update_every;
r->before = before_wanted;
r->after = after_wanted;
r->internal.points_wanted = points_wanted;
@@ -913,7 +1081,7 @@ RRDR *rrd2rrdr(
// reset the grouping for the new dimension
r->internal.grouping_reset(r);
- do_dimension(
+ do_dimension_fixedstep(
r
, points_wanted
, rd
@@ -961,30 +1129,426 @@ RRDR *rrd2rrdr(
}
#ifdef NETDATA_INTERNAL_CHECKS
+ if (dimensions_used) {
+ if(r->internal.log)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log);
+
+ if(r->rows != points_wanted)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'");
+
+ if(aligned && (r->before % group) != 0)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required");
+
+ // 'after' should not be aligned, since we start inside the first group
+ //if(aligned && (r->after % group) != 0)
+ // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
+
+ if(r->before != before_requested)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'");
+
+ if(r->before != before_wanted)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'");
+
+ // reported 'after' varies, depending on group
+ if(r->after != after_wanted)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'");
+ }
+ #endif
+
+ // free all resources used by the grouping method
+ r->internal.grouping_free(r);
+
+ // when all the dimensions are zero, we should return all of them
+ if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero)) {
+ // all the dimensions are zero
+ // mark them as NONZERO to send them all
+ for(rd = st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
+ if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
+ r->od[c] |= RRDR_DIMENSION_NONZERO;
+ }
+ }
+
+ rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);
+ return r;
+}
+
+#ifdef ENABLE_DBENGINE
+static RRDR *rrd2rrdr_variablestep(
+ RRDSET *st
+ , long points_requested
+ , long long after_requested
+ , long long before_requested
+ , RRDR_GROUPING group_method
+ , long resampling_time_requested
+ , RRDR_OPTIONS options
+ , const char *dimensions
+ , int update_every
+ , time_t first_entry_t
+ , time_t last_entry_t
+ , int absolute_period_requested
+ , struct rrdeng_region_info *region_info_array
+) {
+ int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
+
+ if(!absolute_period_requested) {
+ if(before_requested % update_every) {
+ // make sure it is multiple of update_every
+ if(before_requested > 0)
+ before_requested = before_requested - before_requested % update_every;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ else
+ error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero before_requested", st->name);
+ #endif
+ }
+ if(after_requested % update_every) {
+ // make sure it is multiple of update_every
+ if(after_requested < 0)
+ after_requested = after_requested - after_requested % update_every;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ else
+ error("INTERNAL ERROR: rrd2rrdr() on %s, negative or zero after_requested", st->name);
+ #endif
+ }
+ if(after_requested == before_requested) after_requested -= update_every;
+ }
+
+ // the duration of the chart
+ time_t duration = before_requested - after_requested;
+ long available_points = duration / update_every;
+
+ if(duration <= 0 || available_points <= 0) {
+ freez(region_info_array);
+ return rrdr_create(st, 1);
+ }
+
+ // check the number of wanted points in the result
+ if(unlikely(points_requested < 0)) points_requested = -points_requested;
+ if(unlikely(points_requested > available_points)) points_requested = available_points;
+ if(unlikely(points_requested == 0)) points_requested = available_points;
+
+ // calculate the desired grouping of source data points
+ long group = available_points / points_requested;
+ if(unlikely(group <= 0)) group = 1;
+ if(unlikely(available_points % points_requested > points_requested / 2)) group++; // rounding to the closest integer
+
+ // resampling_time_requested enforces a certain grouping multiple
+ calculated_number resampling_divisor = 1.0;
+ long resampling_group = 1;
+ if(unlikely(resampling_time_requested > update_every)) {
+ if (unlikely(resampling_time_requested > duration)) {
+ // group_time is above the available duration
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+ info("INTERNAL CHECK: %s: requested gtime %ld secs, is greater than the desired duration %ld secs", st->id, resampling_time_requested, duration);
+ #endif
+
+ after_requested = before_requested - resampling_time_requested;
+ duration = before_requested - after_requested;
+ available_points = duration / update_every;
+ group = available_points / points_requested;
+ }
+
+ // if the duration is not aligned to resampling time
+ // extend the duration to the past, to avoid a gap at the chart
+ // only when the missing duration is above 1/10th of a point
+ if(duration % resampling_time_requested) {
+ time_t delta = duration % resampling_time_requested;
+ if(delta > resampling_time_requested / 10) {
+ after_requested -= resampling_time_requested - delta;
+ duration = before_requested - after_requested;
+ available_points = duration / update_every;
+ group = available_points / points_requested;
+ }
+ }
+
+ // the points we should group to satisfy gtime
+ resampling_group = resampling_time_requested / update_every;
+ if(unlikely(resampling_time_requested % update_every)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ info("INTERNAL CHECK: %s: requested gtime %ld secs, is not a multiple of the chart's data collection frequency %d secs", st->id, resampling_time_requested, update_every);
+ #endif
+
+ resampling_group++;
+ }
+
+ // adapt group according to resampling_group
+ if(unlikely(group < resampling_group)) group = resampling_group; // do not allow grouping below the desired one
+ if(unlikely(group % resampling_group)) group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
+
+ //resampling_divisor = group / resampling_group;
+ resampling_divisor = (calculated_number)(group * update_every) / (calculated_number)resampling_time_requested;
+ }
+
+ // now that we have group,
+ // align the requested timeframe to fit it.
+
+ if(aligned) {
+ // alignement has been requested, so align the values
+ before_requested -= before_requested % (group * update_every);
+ after_requested -= after_requested % (group * update_every);
+ }
+
+ // we align the request on requested_before
+ time_t before_wanted = before_requested;
+ if(likely(before_wanted > last_entry_t)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: rrd2rrdr() on %s, before_wanted is after db max", st->name);
+ #endif
+
+ before_wanted = last_entry_t - (last_entry_t % ( ((aligned)?group:1) * update_every ));
+ }
+ //size_t before_slot = rrdset_time2slot(st, before_wanted);
+
+ // we need to estimate the number of points, for having
+ // an integer number of values per point
+ long points_wanted = (before_wanted - after_requested) / (update_every * group);
+
+ time_t after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;
+ if(unlikely(after_wanted < first_entry_t)) {
+ // hm... we go to the past, calculate again points_wanted using all the db from before_wanted to the beginning
+ points_wanted = (before_wanted - first_entry_t) / group;
+
+ // recalculate after wanted with the new number of points
+ after_wanted = before_wanted - (points_wanted * group * update_every) + update_every;
+
+ if(unlikely(after_wanted < first_entry_t)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: rrd2rrdr() on %s, after_wanted is before db min", st->name);
+ #endif
+
+ after_wanted = first_entry_t - (first_entry_t % ( ((aligned)?group:1) * update_every )) + ( ((aligned)?group:1) * update_every );
+ }
+ }
+ //size_t after_slot = rrdset_time2slot(st, after_wanted);
+
+ // check if they are reversed
+ if(unlikely(after_wanted > before_wanted)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: rrd2rrdr() on %s, reversed wanted after/before", st->name);
+ #endif
+ time_t tmp = before_wanted;
+ before_wanted = after_wanted;
+ after_wanted = tmp;
+ }
- if(r->internal.log)
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log);
+ // recalculate points_wanted using the final time-frame
+ points_wanted = (before_wanted - after_wanted) / update_every / group + 1;
+ if(unlikely(points_wanted < 0)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: rrd2rrdr() on %s, points_wanted is %ld", st->name, points_wanted);
+ #endif
+ points_wanted = 0;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ duration = before_wanted - after_wanted;
- if(r->rows != points_wanted)
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'");
+ if(after_wanted < first_entry_t)
+ error("INTERNAL CHECK: after_wanted %u is too small, minimum %u", (uint32_t)after_wanted, (uint32_t)first_entry_t);
- if(aligned && (r->before % group) != 0)
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required");
+ if(after_wanted > last_entry_t)
+ error("INTERNAL CHECK: after_wanted %u is too big, maximum %u", (uint32_t)after_wanted, (uint32_t)last_entry_t);
- // 'after' should not be aligned, since we start inside the first group
- //if(aligned && (r->after % group) != 0)
- // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
+ if(before_wanted < first_entry_t)
+ error("INTERNAL CHECK: before_wanted %u is too small, minimum %u", (uint32_t)before_wanted, (uint32_t)first_entry_t);
- if(r->before != before_requested)
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'");
+ if(before_wanted > last_entry_t)
+ error("INTERNAL CHECK: before_wanted %u is too big, maximum %u", (uint32_t)before_wanted, (uint32_t)last_entry_t);
- if(r->before != before_wanted)
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'");
+/*
+ if(before_slot >= (size_t)st->entries)
+ error("INTERNAL CHECK: before_slot is invalid %zu, expected 0 to %ld", before_slot, st->entries - 1);
- // reported 'after' varies, depending on group
- if(r->after != after_wanted)
- rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'");
+ if(after_slot >= (size_t)st->entries)
+ error("INTERNAL CHECK: after_slot is invalid %zu, expected 0 to %ld", after_slot, st->entries - 1);
+*/
+ if(points_wanted > (before_wanted - after_wanted) / group / update_every + 1)
+ error("INTERNAL CHECK: points_wanted %ld is more than points %ld", points_wanted, (before_wanted - after_wanted) / group / update_every + 1);
+
+ if(group < resampling_group)
+ error("INTERNAL CHECK: group %ld is less than the desired group points %ld", group, resampling_group);
+
+ if(group > resampling_group && group % resampling_group)
+ error("INTERNAL CHECK: group %ld is not a multiple of the desired group points %ld", group, resampling_group);
+#endif
+
+ // -------------------------------------------------------------------------
+ // initialize our result set
+ // this also locks the chart for us
+
+ RRDR *r = rrdr_create(st, points_wanted);
+ if(unlikely(!r)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
+ #endif
+ freez(region_info_array);
+ return NULL;
+ }
+
+ if(unlikely(!r->d || !points_wanted)) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL CHECK: Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
+ #endif
+ freez(region_info_array);
+ return r;
+ }
+
+ r->result_options |= RRDR_RESULT_OPTION_VARIABLE_STEP;
+ if(unlikely(absolute_period_requested == 1))
+ r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
+ else
+ r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
+
+ // find how many dimensions we have
+ long dimensions_count = r->d;
+
+ // -------------------------------------------------------------------------
+ // initialize RRDR
+
+ r->group = group;
+ r->update_every = (int)group * update_every;
+ r->before = before_wanted;
+ r->after = after_wanted;
+ r->internal.points_wanted = points_wanted;
+ r->internal.resampling_group = resampling_group;
+ r->internal.resampling_divisor = resampling_divisor;
+
+
+ // -------------------------------------------------------------------------
+ // assign the processor functions
+
+ {
+ int i, found = 0;
+ for(i = 0; !found && api_v1_data_groups[i].name ;i++) {
+ if(api_v1_data_groups[i].value == group_method) {
+ r->internal.grouping_create= api_v1_data_groups[i].create;
+ r->internal.grouping_reset = api_v1_data_groups[i].reset;
+ r->internal.grouping_free = api_v1_data_groups[i].free;
+ r->internal.grouping_add = api_v1_data_groups[i].add;
+ r->internal.grouping_flush = api_v1_data_groups[i].flush;
+ found = 1;
+ }
+ }
+ if(!found) {
+ errno = 0;
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: grouping method %u not found for chart '%s'. Using 'average'", (unsigned int)group_method, r->st->name);
+ #endif
+ r->internal.grouping_create= grouping_create_average;
+ r->internal.grouping_reset = grouping_reset_average;
+ r->internal.grouping_free = grouping_free_average;
+ r->internal.grouping_add = grouping_add_average;
+ r->internal.grouping_flush = grouping_flush_average;
+ }
+ }
+
+ // allocate any memory required by the grouping method
+ r->internal.grouping_data = r->internal.grouping_create(r);
+
+
+ // -------------------------------------------------------------------------
+ // disable the not-wanted dimensions
+
+ rrdset_check_rdlock(st);
+
+ if(dimensions)
+ rrdr_disable_not_selected_dimensions(r, options, dimensions);
+
+
+ // -------------------------------------------------------------------------
+ // do the work for each dimension
+
+ time_t max_after = 0, min_before = 0;
+ long max_rows = 0;
+
+ RRDDIM *rd;
+ long c, dimensions_used = 0, dimensions_nonzero = 0;
+ for(rd = st->dimensions, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
+
+ // if we need a percentage, we need to calculate all dimensions
+ if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) {
+ if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;
+ continue;
+ }
+ r->od[c] |= RRDR_DIMENSION_SELECTED;
+
+ // reset the grouping for the new dimension
+ r->internal.grouping_reset(r);
+
+ do_dimension_variablestep(
+ r
+ , points_wanted
+ , rd
+ , c
+ , after_wanted
+ , before_wanted
+ );
+
+ if(r->od[c] & RRDR_DIMENSION_NONZERO)
+ dimensions_nonzero++;
+
+ // verify all dimensions are aligned
+ if(unlikely(!dimensions_used)) {
+ min_before = r->before;
+ max_after = r->after;
+ max_rows = r->rows;
+ }
+ else {
+ if(r->after != max_after) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
+ st->name, (size_t)max_after, rd->name, (size_t)r->after);
+ #endif
+ r->after = (r->after > max_after) ? r->after : max_after;
+ }
+
+ if(r->before != min_before) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
+ st->name, (size_t)min_before, rd->name, (size_t)r->before);
+ #endif
+ r->before = (r->before < min_before) ? r->before : min_before;
+ }
+
+ if(r->rows != max_rows) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("INTERNAL ERROR: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
+ st->name, (size_t)max_rows, rd->name, (size_t)r->rows);
+ #endif
+ r->rows = (r->rows > max_rows) ? r->rows : max_rows;
+ }
+ }
+
+ dimensions_used++;
+ }
+
+ #ifdef NETDATA_INTERNAL_CHECKS
+
+ if (dimensions_used) {
+ if(r->internal.log)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ r->internal.log);
+
+ if(r->rows != points_wanted)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'");
+
+ if(aligned && (r->before % group) != 0)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required");
+
+ // 'after' should not be aligned, since we start inside the first group
+ //if(aligned && (r->after % group) != 0)
+ // rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
+
+ if(r->before != before_requested)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'");
+
+ if(r->before != before_wanted)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'");
+
+ // reported 'after' varies, depending on group
+ if(r->after != after_wanted)
+ rrd2rrdr_log_request_response_metdata(r, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'");
+ }
#endif
// free all resources used by the grouping method
@@ -1001,5 +1565,57 @@ RRDR *rrd2rrdr(
}
rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);
+ freez(region_info_array);
return r;
}
+#endif //#ifdef ENABLE_DBENGINE
+
+RRDR *rrd2rrdr(
+ RRDSET *st
+ , long points_requested
+ , long long after_requested
+ , long long before_requested
+ , RRDR_GROUPING group_method
+ , long resampling_time_requested
+ , RRDR_OPTIONS options
+ , const char *dimensions
+) {
+ int rrd_update_every;
+ int absolute_period_requested;
+ time_t first_entry_t = rrdset_first_entry_t(st);
+ time_t last_entry_t = rrdset_last_entry_t(st);
+
+ absolute_period_requested = rrdr_convert_before_after_to_absolute(&after_requested, &before_requested,
+ first_entry_t, last_entry_t);
+
+#ifdef ENABLE_DBENGINE
+ if ((st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)) {
+ struct rrdeng_region_info *region_info_array;
+ unsigned regions, max_interval;
+
+ /* This call takes the chart read-lock */
+ regions = rrdeng_variable_step_boundaries(st, after_requested, before_requested,
+ &region_info_array, &max_interval);
+ if (1 == regions) {
+ if (region_info_array)
+ rrd_update_every = region_info_array[0].update_every;
+ else
+ rrd_update_every = st->update_every;
+ if (region_info_array)
+ freez(region_info_array);
+ return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
+ resampling_time_requested, options, dimensions, rrd_update_every,
+ first_entry_t, last_entry_t, absolute_period_requested);
+ } else {
+ rrd_update_every = (uint16_t)max_interval;
+ return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method,
+ resampling_time_requested, options, dimensions, rrd_update_every,
+ first_entry_t, last_entry_t, absolute_period_requested, region_info_array);
+ }
+ }
+#endif
+ rrd_update_every = st->update_every;
+ return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
+ resampling_time_requested, options, dimensions,
+ rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested);
+} \ No newline at end of file