diff options
Diffstat (limited to 'web/api/queries')
| -rw-r--r-- | web/api/queries/average/average.h | 10 | ||||
| -rw-r--r-- | web/api/queries/countif/countif.h | 10 | ||||
| -rw-r--r-- | web/api/queries/des/des.h | 12 | ||||
| -rw-r--r-- | web/api/queries/incremental_sum/incremental_sum.h | 10 | ||||
| -rw-r--r-- | web/api/queries/max/max.h | 10 | ||||
| -rw-r--r-- | web/api/queries/median/median.h | 26 | ||||
| -rw-r--r-- | web/api/queries/min/min.h | 10 | ||||
| -rw-r--r-- | web/api/queries/percentile/percentile.h | 26 | ||||
| -rw-r--r-- | web/api/queries/query.c | 1012 | ||||
| -rw-r--r-- | web/api/queries/query.h | 8 | ||||
| -rw-r--r-- | web/api/queries/rrdr.c | 60 | ||||
| -rw-r--r-- | web/api/queries/rrdr.h | 111 | ||||
| -rw-r--r-- | web/api/queries/ses/ses.h | 12 | ||||
| -rw-r--r-- | web/api/queries/stddev/stddev.h | 16 | ||||
| -rw-r--r-- | web/api/queries/sum/sum.h | 10 | ||||
| -rw-r--r-- | web/api/queries/trimmed_mean/trimmed_mean.h | 24 | ||||
| -rw-r--r-- | web/api/queries/weights.c | 763 | ||||
| -rw-r--r-- | web/api/queries/weights.h | 14 |
18 files changed, 1031 insertions, 1113 deletions
diff --git a/web/api/queries/average/average.h b/web/api/queries/average/average.h index 55c51722c..b31966886 100644 --- a/web/api/queries/average/average.h +++ b/web/api/queries/average/average.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_average(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_average(RRDR *r); -extern void grouping_free_average(RRDR *r); -extern void grouping_add_average(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_average(RRDR *r, const char *options __maybe_unused); +void grouping_reset_average(RRDR *r); +void grouping_free_average(RRDR *r); +void grouping_add_average(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_AVERAGE_H diff --git a/web/api/queries/countif/countif.h b/web/api/queries/countif/countif.h index 0c7d2d7d1..dfe805658 100644 --- a/web/api/queries/countif/countif.h +++ b/web/api/queries/countif/countif.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_countif(RRDR *r); -extern void grouping_free_countif(RRDR *r); -extern void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_countif(RRDR *r, const char *options __maybe_unused); +void grouping_reset_countif(RRDR *r); +void grouping_free_countif(RRDR *r); +void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_COUNTIF_H diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h index 8906d14eb..05fa01b34 100644 --- a/web/api/queries/des/des.h +++ b/web/api/queries/des/des.h @@ -6,12 +6,12 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_init_des(void); +void grouping_init_des(void); -extern void grouping_create_des(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_des(RRDR *r); -extern void grouping_free_des(RRDR *r); -extern void grouping_add_des(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_des(RRDR *r, const char *options __maybe_unused); +void grouping_reset_des(RRDR *r); +void grouping_free_des(RRDR *r); +void grouping_add_des(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_DES_H diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h index 6d908cef6..c24507fcf 100644 --- a/web/api/queries/incremental_sum/incremental_sum.h +++ b/web/api/queries/incremental_sum/incremental_sum.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_incremental_sum(RRDR *r); -extern void grouping_free_incremental_sum(RRDR *r); -extern void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused); +void grouping_reset_incremental_sum(RRDR *r); +void grouping_free_incremental_sum(RRDR *r); +void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H diff --git a/web/api/queries/max/max.h b/web/api/queries/max/max.h index 28913686b..e2427d26d 100644 --- a/web/api/queries/max/max.h +++ b/web/api/queries/max/max.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_max(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_max(RRDR *r); -extern void grouping_free_max(RRDR *r); -extern void grouping_add_max(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_max(RRDR *r, const char *options __maybe_unused); +void grouping_reset_max(RRDR *r); +void grouping_free_max(RRDR *r); +void grouping_add_max(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_MAX_H diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h index dd1b3de61..9fc159db4 100644 --- a/web/api/queries/median/median.h +++ b/web/api/queries/median/median.h @@ -6,18 +6,18 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_median(RRDR *r, const char *options); -extern void grouping_create_trimmed_median1(RRDR *r, const char *options); -extern void grouping_create_trimmed_median2(RRDR *r, const char *options); -extern void grouping_create_trimmed_median3(RRDR *r, const char *options); -extern void grouping_create_trimmed_median5(RRDR *r, const char *options); -extern void grouping_create_trimmed_median10(RRDR *r, const char *options); -extern void grouping_create_trimmed_median15(RRDR *r, const char *options); -extern void grouping_create_trimmed_median20(RRDR *r, const char *options); -extern void grouping_create_trimmed_median25(RRDR *r, const char *options); -extern void grouping_reset_median(RRDR *r); -extern void grouping_free_median(RRDR *r); -extern void grouping_add_median(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_median(RRDR *r, const char *options); +void grouping_create_trimmed_median1(RRDR *r, const char *options); +void grouping_create_trimmed_median2(RRDR *r, const char *options); +void grouping_create_trimmed_median3(RRDR *r, const char *options); +void grouping_create_trimmed_median5(RRDR *r, const char *options); +void grouping_create_trimmed_median10(RRDR *r, const char *options); +void grouping_create_trimmed_median15(RRDR *r, const char *options); +void grouping_create_trimmed_median20(RRDR *r, const char *options); +void grouping_create_trimmed_median25(RRDR *r, const char *options); +void grouping_reset_median(RRDR *r); +void grouping_free_median(RRDR *r); +void grouping_add_median(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_MEDIAN_H diff --git a/web/api/queries/min/min.h b/web/api/queries/min/min.h index b8627f667..dcdfe252f 100644 --- a/web/api/queries/min/min.h +++ b/web/api/queries/min/min.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_min(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_min(RRDR *r); -extern void grouping_free_min(RRDR *r); -extern void grouping_add_min(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_min(RRDR *r, const char *options __maybe_unused); +void grouping_reset_min(RRDR *r); +void grouping_free_min(RRDR *r); +void grouping_add_min(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_MIN_H diff --git a/web/api/queries/percentile/percentile.h b/web/api/queries/percentile/percentile.h index 709717ebd..65e335c11 100644 --- a/web/api/queries/percentile/percentile.h +++ b/web/api/queries/percentile/percentile.h @@ -6,18 +6,18 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_percentile25(RRDR *r, const char *options); -extern void grouping_create_percentile50(RRDR *r, const char *options); -extern void grouping_create_percentile75(RRDR *r, const char *options); -extern void grouping_create_percentile80(RRDR *r, const char *options); -extern void grouping_create_percentile90(RRDR *r, const char *options); -extern void grouping_create_percentile95(RRDR *r, const char *options); -extern void grouping_create_percentile97(RRDR *r, const char *options); -extern void grouping_create_percentile98(RRDR *r, const char *options); -extern void grouping_create_percentile99(RRDR *r, const char *options ); -extern void grouping_reset_percentile(RRDR *r); -extern void grouping_free_percentile(RRDR *r); -extern void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_percentile25(RRDR *r, const char *options); +void grouping_create_percentile50(RRDR *r, const char *options); +void grouping_create_percentile75(RRDR *r, const char *options); +void grouping_create_percentile80(RRDR *r, const char *options); +void grouping_create_percentile90(RRDR *r, const char *options); +void grouping_create_percentile95(RRDR *r, const char *options); +void grouping_create_percentile97(RRDR *r, const char *options); +void grouping_create_percentile98(RRDR *r, const char *options); +void grouping_create_percentile99(RRDR *r, const char *options ); +void grouping_reset_percentile(RRDR *r); +void grouping_free_percentile(RRDR *r); +void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_PERCENTILE_H diff --git a/web/api/queries/query.c b/web/api/queries/query.c index d776f6d11..ccd195135 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -658,71 +658,6 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) { } // ---------------------------------------------------------------------------- - -static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims, - struct context_param *context_param_list) -{ - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; - int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)); - - if (should_lock) - rrdset_check_rdlock(r->st); - - if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return; - - int match_ids = 0, match_names = 0; - - if(unlikely(options & RRDR_OPTION_MATCH_IDS)) - match_ids = 1; - if(unlikely(options & RRDR_OPTION_MATCH_NAMES)) - match_names = 1; - - if(likely(!match_ids && !match_names)) - match_ids = match_names = 1; - - SIMPLE_PATTERN *pattern = simple_pattern_create(dims, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT); - - RRDDIM *d; - long c, dims_selected = 0, dims_not_hidden_not_zero = 0; - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) { - if( (match_ids && simple_pattern_matches(pattern, d->id)) - || (match_names && simple_pattern_matches(pattern, d->name)) - ) { - r->od[c] |= RRDR_DIMENSION_SELECTED; - if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) r->od[c] &= ~RRDR_DIMENSION_HIDDEN; - dims_selected++; - - // since the user needs this dimension - // make it appear as NONZERO, to return it - // even if the dimension has only zeros - // unless option non_zero is set - if(unlikely(!(options & RRDR_OPTION_NONZERO))) - r->od[c] |= RRDR_DIMENSION_NONZERO; - - // count the visible dimensions - if(likely(r->od[c] & RRDR_DIMENSION_NONZERO)) - dims_not_hidden_not_zero++; - } - else { - r->od[c] |= RRDR_DIMENSION_HIDDEN; - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED; - } - } - simple_pattern_free(pattern); - - // check if all dimensions are hidden - if(unlikely(!dims_not_hidden_not_zero && dims_selected)) { - // there are a few selected dimensions, - // but they are all zero - // enable the selected ones - // to avoid returning an empty chart - for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) - r->od[c] |= RRDR_DIMENSION_NONZERO; - } -} - -// ---------------------------------------------------------------------------- // helpers to find our way in RRDR static inline RRDR_VALUE_FLAGS *UNUSED_FUNCTION(rrdr_line_options)(RRDR *r, long rrdr_line) { @@ -736,13 +671,13 @@ static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rr static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) { rrdr_line++; - internal_error(rrdr_line >= r->n, - "QUERY: requested to step above RRDR size for chart '%s'", - r->st->name); + internal_error(rrdr_line >= (long)r->n, + "QUERY: requested to step above RRDR size for query '%s'", + r->internal.qt->id); internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t, - "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'", - (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name); + "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of query '%s'", + (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->internal.qt->id); // save the time r->t[rrdr_line] = t; @@ -758,68 +693,137 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) { // ---------------------------------------------------------------------------- // tier management -static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, time_t before_wanted, long points_wanted) { +static bool query_metric_is_valid_tier(QUERY_METRIC *qm, size_t tier) { + if(!qm->tiers[tier].db_metric_handle || !qm->tiers[tier].db_first_time_t || !qm->tiers[tier].db_last_time_t || !qm->tiers[tier].db_update_every) + return false; + + return true; +} + +static size_t query_metric_first_working_tier(QUERY_METRIC *qm) { + for(size_t tier = 0; tier < storage_tiers ; tier++) { + + // find the db time-range for this tier for all metrics + STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle; + time_t first_t = qm->tiers[tier].db_first_time_t; + time_t last_t = qm->tiers[tier].db_last_time_t; + time_t update_every = qm->tiers[tier].db_update_every; + + if(!db_metric_handle || !first_t || !last_t || !update_every) + continue; + + return tier; + } + + return 0; +} + +static long query_plan_points_coverage_weight(time_t db_first_t, time_t db_last_t, time_t db_update_every, time_t after_wanted, time_t before_wanted, size_t points_wanted, size_t tier __maybe_unused) { + if(db_first_t == 0 || db_last_t == 0 || db_update_every == 0) + return -LONG_MAX; + + time_t common_first_t = MAX(db_first_t, after_wanted); + time_t common_last_t = MIN(db_last_t, before_wanted); + + long time_coverage = (common_last_t - common_first_t) * 1000000 / (before_wanted - after_wanted); + size_t points_wanted_in_coverage = points_wanted * time_coverage / 1000000; + + long points_available = (common_last_t - common_first_t) / db_update_every; + long points_delta = (long)(points_available - points_wanted_in_coverage); + long points_coverage = (points_delta < 0) ? (long)(points_available * time_coverage / points_wanted_in_coverage) : time_coverage; + + // a way to benefit higher tiers + // points_coverage += (long)tier * 10000; + + if(points_available <= 0) + return -LONG_MAX; + + return points_coverage; +} + +static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t after_wanted, time_t before_wanted, size_t points_wanted) { if(unlikely(storage_tiers < 2)) return 0; - if(unlikely(after_wanted == before_wanted || points_wanted <= 0 || !rd || !rd->rrdset)) { + if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) + return query_metric_first_working_tier(qm); - if(!rd) - internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation"); - else - internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation", - (rd->rrdset)?rd->rrdset->name:"unknown", rd->name); + long weight[storage_tiers]; - return 0; + for(size_t tier = 0; tier < storage_tiers ; tier++) { + + // find the db time-range for this tier for all metrics + STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle; + time_t first_t = qm->tiers[tier].db_first_time_t; + time_t last_t = qm->tiers[tier].db_last_time_t; + time_t update_every = qm->tiers[tier].db_update_every; + + if(!db_metric_handle || !first_t || !last_t || !update_every) { + weight[tier] = -LONG_MAX; + continue; + } + + weight[tier] = query_plan_points_coverage_weight(first_t, last_t, update_every, after_wanted, before_wanted, points_wanted, tier); } - //BUFFER *wb = buffer_create(1000); - //buffer_sprintf(wb, "Best tier for chart '%s', dim '%s', from %ld to %ld (dur %ld, every %d), points %ld", - // rd->rrdset->name, rd->name, after_wanted, before_wanted, before_wanted - after_wanted, rd->update_every, points_wanted); + size_t best_tier = 0; + for(size_t tier = 1; tier < storage_tiers ; tier++) { + if(weight[tier] >= weight[best_tier]) + best_tier = tier; + } + + return best_tier; +} + +static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted) { + if(unlikely(storage_tiers < 2)) + return 0; + + if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) { + internal_error(true, "QUERY: '%s' has invalid params to tier calculation", qt->id); + return 0; + } long weight[storage_tiers]; - for(int tier = 0; tier < storage_tiers ; tier++) { - if(unlikely(!rd->tiers[tier])) { - internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized", - tier, rd->rrdset->name, rd->name); - // buffer_free(wb); - return 0; - } + for(size_t tier = 0; tier < storage_tiers ; tier++) { - time_t first_t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle); - time_t last_t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle); + time_t common_first_t = 0; + time_t common_last_t = 0; + time_t common_update_every = 0; - time_t common_after = MAX(first_t, after_wanted); - time_t common_before = MIN(last_t, before_wanted); + // find the db time-range for this tier for all metrics + for(size_t i = 0, used = qt->query.used; i < used ; i++) { + QUERY_METRIC *qm = &qt->query.array[i]; - long time_coverage = (common_before - common_after) * 1000 / (before_wanted - after_wanted); - if(time_coverage < 0) time_coverage = 0; + time_t first_t = qm->tiers[tier].db_first_time_t; + time_t last_t = qm->tiers[tier].db_last_time_t; + time_t update_every = qm->tiers[tier].db_update_every; - int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every; - if(unlikely(update_every == 0)) { - internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d", - tier, rd->rrdset->name, rd->name, rd->tiers[tier]->tier_grouping, rd->update_every); - // buffer_free(wb); - return 0; - } + if(!first_t || !last_t || !update_every) + continue; - long points_available = (before_wanted - after_wanted) / update_every; - long points_delta = points_available - points_wanted; - long points_coverage = (points_delta < 0) ? points_available * 1000 / points_wanted: 1000; + if(!common_first_t) + common_first_t = first_t; + else + common_first_t = MIN(first_t, common_first_t); - if(points_available <= 0) - weight[tier] = -LONG_MAX; - else - weight[tier] = points_coverage; + if(!common_last_t) + common_last_t = last_t; + else + common_last_t = MAX(last_t, common_last_t); - // buffer_sprintf(wb, ": tier %d, first %ld, last %ld (dur %ld, tg %d, every %d), points %ld, tcoverage %ld, pcoverage %ld, weight %ld", - // tier, first_t, last_t, last_t - first_t, rd->tiers[tier]->tier_grouping, update_every, - // points_available, time_coverage, points_coverage, weight[tier]); + if(!common_update_every) + common_update_every = update_every; + else + common_update_every = MIN(update_every, common_update_every); + } + + weight[tier] = query_plan_points_coverage_weight(common_first_t, common_last_t, common_update_every, after_wanted, before_wanted, points_wanted, tier); } - int best_tier = 0; - for(int tier = 1; tier < storage_tiers ; tier++) { + size_t best_tier = 0; + for(size_t tier = 1; tier < storage_tiers ; tier++) { if(weight[tier] >= weight[best_tier]) best_tier = tier; } @@ -827,47 +831,30 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, if(weight[best_tier] == -LONG_MAX) best_tier = 0; - //buffer_sprintf(wb, ": final best tier %d", best_tier); - //internal_error(true, "%s", buffer_tostring(wb)); - //buffer_free(wb); - return best_tier; } -static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) { - int ret = st->update_every; - - if(unlikely(!st->dimensions)) - return ret; - - rrdset_rdlock(st); - int best_tier; - - if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers) +static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted, RRDR_OPTIONS options, size_t tier) { + size_t best_tier; + if((options & RRDR_OPTION_SELECTED_TIER) && tier < storage_tiers) best_tier = tier; else - best_tier = rrddim_find_best_tier_for_timeframe(st->dimensions, after_wanted, before_wanted, points_wanted); + best_tier = rrddim_find_best_tier_for_timeframe(qt, after_wanted, before_wanted, points_wanted); - if(!st->dimensions->tiers[best_tier]) { - internal_error( - true, - "QUERY: tier %d on chart '%s', is not initialized", best_tier, st->name); - } - else { - ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every; - if(unlikely(!ret)) { - internal_error( - true, - "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d", - st->name, st->dimensions->tiers[best_tier]->tier_grouping, st->update_every); - - ret = st->update_every; - } - } + // find the db minimum update every for this tier for all metrics + time_t common_update_every = default_rrd_update_every; + for(size_t i = 0, used = qt->query.used; i < used ; i++) { + QUERY_METRIC *qm = &qt->query.array[i]; - rrdset_unlock(st); + time_t update_every = qm->tiers[best_tier].db_update_every; - return ret; + if(!i) + common_update_every = update_every; + else + common_update_every = MIN(update_every, common_update_every); + } + + return common_update_every; } // ---------------------------------------------------------------------------- @@ -915,7 +902,7 @@ typedef struct query_plan { typedef struct query_engine_ops { // configuration RRDR *r; - RRDDIM *rd; + QUERY_METRIC *qm; time_t view_update_every; time_t query_granularity; TIER_QUERY_FETCH tier_query_fetch; @@ -927,11 +914,11 @@ typedef struct query_engine_ops { // storage queries size_t tier; - struct rrddim_tier *tier_ptr; - struct rrddim_query_handle handle; - STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle); - int (*is_finished)(struct rrddim_query_handle *handle); - void (*finalize)(struct rrddim_query_handle *handle); + struct query_metric_tier *tier_ptr; + struct storage_engine_query_handle handle; + STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle); + int (*is_finished)(struct storage_engine_query_handle *handle); + void (*finalize)(struct storage_engine_query_handle *handle); // aggregating points over time void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value); @@ -963,11 +950,11 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti after = overwrite_after; ops->tier = ops->plan.data[plan_id].tier; - ops->tier_ptr = ops->rd->tiers[ops->tier]; - ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch); - ops->next_metric = ops->tier_ptr->query_ops.next_metric; - ops->is_finished = ops->tier_ptr->query_ops.is_finished; - ops->finalize = ops->tier_ptr->query_ops.finalize; + ops->tier_ptr = &ops->qm->tiers[ops->tier]; + ops->tier_ptr->eng->api.query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before); + ops->next_metric = ops->tier_ptr->eng->api.query_ops.next_metric; + ops->is_finished = ops->tier_ptr->eng->api.query_ops.is_finished; + ops->finalize = ops->tier_ptr->eng->api.query_ops.finalize; ops->current_plan = plan_id; ops->current_plan_expire_time = ops->plan.data[plan_id].before; } @@ -976,26 +963,38 @@ static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t las internal_error(now < ops->current_plan_expire_time && now < ops->plan.data[ops->current_plan].before, "QUERY: switching query plan too early!"); + size_t old_plan = ops->current_plan; + time_t next_plan_before_time; do { ops->current_plan++; if (ops->current_plan >= ops->plan.entries) { - ops->current_plan = ops->plan.entries - 1; + ops->current_plan = old_plan; + ops->current_plan_expire_time = ops->r->internal.qt->window.before; + // let the query run with current plan + // we will not switch it return; } next_plan_before_time = ops->plan.data[ops->current_plan].before; } while(now >= next_plan_before_time || last_point_end_time >= next_plan_before_time); + if(!query_metric_is_valid_tier(ops->qm, ops->plan.data[ops->current_plan].tier)) { + ops->current_plan = old_plan; + ops->current_plan_expire_time = ops->r->internal.qt->window.before; + return; + } + if(ops->finalize) { ops->finalize(&ops->handle); ops->finalize = NULL; + ops->is_finished = NULL; } - query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time)); - // internal_error(true, "QUERY: switched plan to %zu (all is %zu), previous expiration was %ld, this starts at %ld, now is %ld, last_point_end_time %ld", ops->current_plan, ops->plan.entries, ops->plan.data[ops->current_plan-1].before, ops->plan.data[ops->current_plan].after, now, last_point_end_time); + + query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time)); } static int compare_query_plan_entries_on_start_time(const void *a, const void *b) { @@ -1004,21 +1003,20 @@ static int compare_query_plan_entries_on_start_time(const void *a, const void *b return (p1->after < p2->after)?-1:1; } -static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, long points_wanted) { - RRDDIM *rd = ops->rd; - +static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, size_t points_wanted) { //BUFFER *wb = buffer_create(1000); //buffer_sprintf(wb, "QUERY PLAN for chart '%s' dimension '%s', from %ld to %ld:", rd->rrdset->name, rd->name, after_wanted, before_wanted); // put our selected tier as the first plan size_t selected_tier; - if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER && ops->r->internal.query_tier >= 0 && ops->r->internal.query_tier < storage_tiers) { - selected_tier = ops->r->internal.query_tier; + if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER + && ops->r->internal.qt->window.tier < storage_tiers + && query_metric_is_valid_tier(ops->qm, ops->r->internal.qt->window.tier)) { + selected_tier = ops->r->internal.qt->window.tier; } else { - - selected_tier = rrddim_find_best_tier_for_timeframe(rd, after_wanted, before_wanted, points_wanted); + selected_tier = query_metric_best_tier_for_timeframe(ops->qm, after_wanted, before_wanted, points_wanted); if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER) ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER; @@ -1026,8 +1024,8 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before ops->plan.entries = 1; ops->plan.data[0].tier = selected_tier; - ops->plan.data[0].after = rd->tiers[selected_tier]->query_ops.oldest_time(rd->tiers[selected_tier]->db_metric_handle); - ops->plan.data[0].before = rd->tiers[selected_tier]->query_ops.latest_time(rd->tiers[selected_tier]->db_metric_handle); + ops->plan.data[0].after = ops->qm->tiers[selected_tier].db_first_time_t; + ops->plan.data[0].before = ops->qm->tiers[selected_tier].db_last_time_t; if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) { // the selected tier @@ -1039,9 +1037,12 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before // check if our selected tier can start the query if (selected_tier_first_time_t > after_wanted) { // we need some help from other tiers - for (int tr = (int)selected_tier + 1; tr < storage_tiers; tr++) { + for (size_t tr = (int)selected_tier + 1; tr < storage_tiers; tr++) { + if(!query_metric_is_valid_tier(ops->qm, tr)) + continue; + // find the first time of this tier - time_t first_time_t = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle); + time_t first_time_t = ops->qm->tiers[tr].db_first_time_t; //buffer_sprintf(wb, ": EVAL AFTER tier %d, %ld", tier, first_time_t); @@ -1067,8 +1068,11 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before if (selected_tier_last_time_t < before_wanted) { // we need some help from other tiers for (int tr = (int)selected_tier - 1; tr >= 0; tr--) { + if(!query_metric_is_valid_tier(ops->qm, tr)) + continue; + // find the last time of this tier - time_t last_time_t = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle); + time_t last_time_t = ops->qm->tiers[tr].db_last_time_t; //buffer_sprintf(wb, ": EVAL BEFORE tier %d, %ld", tier, last_time_t); @@ -1096,8 +1100,11 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before qsort(&ops->plan.data, ops->plan.entries, sizeof(QUERY_PLAN_ENTRY), compare_query_plan_entries_on_start_time); // make sure it has the whole timeframe we need - ops->plan.data[0].after = after_wanted; - ops->plan.data[ops->plan.entries - 1].before = before_wanted; + if(ops->plan.data[0].after < after_wanted) + ops->plan.data[0].after = after_wanted; + + if(ops->plan.data[ops->plan.entries - 1].before > before_wanted) + ops->plan.data[ops->plan.entries - 1].before = before_wanted; //buffer_sprintf(wb, ": FINAL STEPS %zu", ops->plan.entries); @@ -1106,7 +1113,12 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before //internal_error(true, "%s", buffer_tostring(wb)); + if(!query_metric_is_valid_tier(ops->qm, ops->plan.data[0].tier)) + return false; + query_planer_activate_plan(ops, 0, 0); + + return true; } @@ -1146,14 +1158,17 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before (ops).group_anomaly_rate += (point).anomaly; \ } while(0) -static inline void rrd2rrdr_do_dimension( - RRDR *r - , long points_wanted - , RRDDIM *rd - , long dim_id_in_rrdr - , time_t after_wanted - , time_t before_wanted -){ +static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) { + QUERY_TARGET *qt = r->internal.qt; + QUERY_METRIC *qm = &qt->query.array[dim_id_in_rrdr]; + size_t points_wanted = qt->window.points; + time_t after_wanted = qt->window.after; + time_t before_wanted = qt->window.before; + +// bool debug_this = false; +// if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0) +// debug_this = true; + time_t max_date = 0, min_date = 0; @@ -1161,19 +1176,20 @@ static inline void rrd2rrdr_do_dimension( QUERY_ENGINE_OPS ops = { .r = r, - .rd = rd, + .qm = qm, .grouping_add = r->internal.grouping_add, .grouping_flush = r->internal.grouping_flush, .tier_query_fetch = r->internal.tier_query_fetch, .view_update_every = r->update_every, - .query_granularity = r->update_every / r->group, + .query_granularity = (time_t)(r->update_every / r->group), .group_value_flags = RRDR_VALUE_NOTHING }; long rrdr_line = -1; bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false; - query_plan(&ops, after_wanted, before_wanted, points_wanted); + if(!query_plan(&ops, after_wanted, before_wanted, points_wanted)) + return; NETDATA_DOUBLE min = r->min, max = r->max; @@ -1184,15 +1200,18 @@ static inline void rrd2rrdr_do_dimension( time_t now_start_time = after_wanted - ops.query_granularity; time_t now_end_time = after_wanted + ops.view_update_every - ops.query_granularity; + size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch; + // The main loop, based on the query granularity we need - for( ; (long)points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) { + for( ; points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) { - if(query_plan_should_switch_plan(ops, now_end_time)) + if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) { query_planer_next_plan(&ops, now_end_time, new_point.end_time); + db_points_read_since_plan_switch = 0; + } // read all the points of the db, prior to the time we need (now_end_time) - size_t count_same_end_time = 0; while(count_same_end_time < 100) { if(likely(count_same_end_time == 0)) { @@ -1208,11 +1227,15 @@ static inline void rrd2rrdr_do_dimension( new_point = QUERY_POINT_EMPTY; new_point.start_time = last1_point.end_time; new_point.end_time = now_end_time; +// +// if(debug_this) info("QUERY: is finished() returned true"); +// break; } // fetch the new point { + db_points_read_since_plan_switch++; STORAGE_POINT sp = ops.next_metric(&ops.handle); ops.db_points_read_per_tier[ops.tier]++; @@ -1223,6 +1246,10 @@ static inline void rrd2rrdr_do_dimension( new_point.anomaly = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0; query_point_set_id(new_point, ops.db_total_points_read); +// if(debug_this) +// info("QUERY: got point %zu, from time %ld to %ld // now from %ld to %ld // query from %ld to %ld", +// new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted); +// // set the right value to the point we got if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) { @@ -1258,17 +1285,18 @@ static inline void rrd2rrdr_do_dimension( // check if the db is giving us zero duration points if(unlikely(new_point.start_time == new_point.end_time)) { - internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal", - rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time); + internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned point %zu start time %ld, end time %ld, that are both equal", + qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time); - new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every); + new_point.start_time = new_point.end_time - ops.tier_ptr->db_update_every; } // check if the db is advancing the query if(unlikely(new_point.end_time <= last1_point.end_time)) { - internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld", - rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time, - last1_point.id, last1_point.end_time, now_start_time, now_end_time); + internal_error(db_points_read_since_plan_switch > 1, + "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld to %ld, before the last point %zu from %ld to %ld, now is %ld to %ld", + qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time, + last1_point.id, last1_point.start_time, last1_point.end_time, now_start_time, now_end_time); count_same_end_time++; continue; @@ -1294,8 +1322,8 @@ static inline void rrd2rrdr_do_dimension( // we only log if this is not point 1 internal_error(new_point.end_time < after_wanted && new_point.id > 1, - "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)", - rd->rrdset->name, rd->name, + "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)", + qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted); @@ -1311,8 +1339,8 @@ static inline void rrd2rrdr_do_dimension( if(unlikely(count_same_end_time)) { internal_error(true, - "QUERY: the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times", - last1_point.end_time, count_same_end_time); + "QUERY: '%s', dimension '%s', the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times", + qt->id, string2str(qm->dimension.id), last1_point.end_time, count_same_end_time); if(unlikely(new_point.end_time <= last1_point.end_time)) new_point.end_time = now_end_time; @@ -1323,7 +1351,7 @@ static inline void rrd2rrdr_do_dimension( // we select the one to use based on their timestamps size_t iterations = 0; - for ( ; now_end_time <= new_point.end_time && (long)points_added < points_wanted ; + for ( ; now_end_time <= new_point.end_time && points_added < points_wanted ; now_end_time += ops.view_update_every, iterations++) { // now_start_time is wrong in this loop @@ -1336,22 +1364,35 @@ static inline void rrd2rrdr_do_dimension( current_point = new_point; query_interpolate_point(current_point, last1_point, now_end_time); - internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time, - "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld," - " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.", - rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, - current_point.id, current_point.start_time, current_point.end_time, now_end_time); +// internal_error(current_point.id > 0 +// && last1_point.id == 0 +// && current_point.end_time > after_wanted +// && current_point.end_time > now_end_time, +// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld," +// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld," +// " but we could really favor by having last_point1 in this query.", +// qt->id, string2str(qm->dimension.id), +// after_wanted, before_wanted, +// ops.view_update_every, ops.query_granularity, +// current_point.id, current_point.start_time, current_point.end_time, +// now_end_time); } else if(likely(now_end_time <= last1_point.end_time)) { // our LAST point is still valid current_point = last1_point; query_interpolate_point(current_point, last2_point, now_end_time); - internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time, - "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld," - " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.", - rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, - current_point.id, current_point.start_time, current_point.end_time, now_end_time); +// internal_error(current_point.id > 0 +// && last2_point.id == 0 +// && current_point.end_time > after_wanted +// && current_point.end_time > now_end_time, +// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld," +// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld," +// " but we could really favor by having last_point2 in this query.", +// qt->id, string2str(qm->dimension.id), +// after_wanted, before_wanted, ops.view_update_every, ops.query_granularity, +// current_point.id, current_point.start_time, current_point.end_time, +// now_end_time); } else { // a GAP, we don't have a value this time @@ -1414,7 +1455,7 @@ static inline void rrd2rrdr_do_dimension( r->internal.result_points_generated += points_added; r->internal.db_points_read += ops.db_total_points_read; - for(int tr = 0; tr < storage_tiers ; tr++) + for(size_t tr = 0; tr < storage_tiers ; tr++) r->internal.tier_points_read[tr] += ops.db_points_read_per_tier[tr]; r->min = min; @@ -1423,24 +1464,26 @@ static inline void rrd2rrdr_do_dimension( r->after = min_date - ops.view_update_every + ops.query_granularity; rrdr_done(r, rrdr_line); - internal_error((long)points_added != points_wanted, - "QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).", - r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read); + internal_error(points_added != points_wanted, + "QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).", + qt->id, string2str(qm->dimension.id), + (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read); } // ---------------------------------------------------------------------------- // fill the gap of a tier -extern void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut); +void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut); +void store_metric_collection_completed(void); -void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) { - if(unlikely(tier < 0 || tier >= storage_tiers)) return; +void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now) { + if(unlikely(tier >= storage_tiers)) return; if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return; struct rrddim_tier *t = rd->tiers[tier]; if(unlikely(!t)) return; - time_t latest_time_t = t->query_ops.latest_time(t->db_metric_handle); + time_t latest_time_t = t->query_ops->latest_time(t->db_metric_handle); time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every; time_t time_diff = now - latest_time_t; @@ -1450,43 +1493,40 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) { // there is really nothing we can do if(now <= latest_time_t || time_diff < granularity) return; - struct rrddim_query_handle handle; - - size_t all_points_read = 0; + struct storage_engine_query_handle handle; // for each lower tier - for(int tr = tier - 1; tr >= 0 ;tr--){ - time_t smaller_tier_first_time = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle); - time_t smaller_tier_last_time = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle); + for(int tr = (int)tier - 1; tr >= 0 ;tr--){ + time_t smaller_tier_first_time = rd->tiers[tr]->query_ops->oldest_time(rd->tiers[tr]->db_metric_handle); + time_t smaller_tier_last_time = rd->tiers[tr]->query_ops->latest_time(rd->tiers[tr]->db_metric_handle); if(smaller_tier_last_time <= latest_time_t) continue; // it is as bad as we are long after_wanted = (latest_time_t < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_t; long before_wanted = smaller_tier_last_time; struct rrddim_tier *tmp = rd->tiers[tr]; - tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE); + tmp->query_ops->init(tmp->db_metric_handle, &handle, after_wanted, before_wanted); - size_t points = 0; + size_t points_read = 0; - while(!tmp->query_ops.is_finished(&handle)) { + while(!tmp->query_ops->is_finished(&handle)) { - STORAGE_POINT sp = tmp->query_ops.next_metric(&handle); + STORAGE_POINT sp = tmp->query_ops->next_metric(&handle); + points_read++; if(sp.end_time > latest_time_t) { latest_time_t = sp.end_time; - store_metric_at_tier(rd, t, sp, sp.end_time * USEC_PER_SEC); - points++; + store_metric_at_tier(rd, tr, t, sp, sp.end_time * USEC_PER_SEC); } } - all_points_read += points; - tmp->query_ops.finalize(&handle); + tmp->query_ops->finalize(&handle); + store_metric_collection_completed(); + global_statistics_backfill_query_completed(points_read); //internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d", // rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr); } - - rrdr_query_completed(all_points_read, all_points_read); } // ---------------------------------------------------------------------------- @@ -1497,29 +1537,33 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r , RRDR_OPTIONS options __maybe_unused , RRDR_GROUPING group_method , bool aligned - , long group - , long resampling_time - , long resampling_group + , size_t group + , time_t resampling_time + , size_t resampling_group , time_t after_wanted , time_t after_requested , time_t before_wanted , time_t before_requested - , long points_requested - , long points_wanted + , size_t points_requested + , size_t points_wanted //, size_t after_slot //, size_t before_slot , const char *msg ) { - netdata_rwlock_rdlock(&r->st->rrdset_rwlock); - info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), " - "after (got: %zu, want: %zu, req: %ld, db: %zu), " - "before (got: %zu, want: %zu, req: %ld, db: %zu), " - "duration (got: %zu, want: %zu, req: %ld, db: %zu), " - //"slot (after: %zu, before: %zu, delta: %zu), " - "points (got: %ld, want: %ld, req: %ld, db: %ld), " + + time_t first_entry_t = r->internal.qt->db.first_time_t; + time_t last_entry_t = r->internal.qt->db.last_time_t; + + internal_error( + true, + "rrd2rrdr() on %s update every %ld with %s grouping %s (group: %zu, resampling_time: %ld, resampling_group: %zu), " + "after (got: %ld, want: %ld, req: %ld, db: %ld), " + "before (got: %ld, want: %ld, req: %ld, db: %ld), " + "duration (got: %ld, want: %ld, req: %ld, db: %ld), " + "points (got: %zu, want: %zu, req: %zu), " "%s" - , r->st->name - , r->st->update_every + , r->internal.qt->id + , r->internal.qt->window.query_granularity // grouping , (aligned) ? "aligned" : "unaligned" @@ -1529,45 +1573,36 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r , resampling_group // after - , (size_t)r->after - , (size_t)after_wanted + , r->after + , after_wanted , after_requested - , (size_t)rrdset_first_entry_t_nolock(r->st) + , first_entry_t // before - , (size_t)r->before - , (size_t)before_wanted + , r->before + , before_wanted , before_requested - , (size_t)rrdset_last_entry_t_nolock(r->st) + , last_entry_t // duration - , (size_t)(r->before - r->after + r->st->update_every) - , (size_t)(before_wanted - after_wanted + r->st->update_every) - , before_requested - after_requested - , (size_t)((rrdset_last_entry_t_nolock(r->st) - rrdset_first_entry_t_nolock(r->st)) + r->st->update_every) - - // slot - /* - , after_slot - , before_slot - , (after_slot > before_slot) ? (r->st->entries - after_slot + before_slot) : (before_slot - after_slot) - */ + , (long)(r->before - r->after + r->internal.qt->window.query_granularity) + , (long)(before_wanted - after_wanted + r->internal.qt->window.query_granularity) + , (long)before_requested - after_requested + , (long)((last_entry_t - first_entry_t) + r->internal.qt->window.query_granularity) // points , r->rows , points_wanted , points_requested - , r->st->entries // message , msg ); - netdata_rwlock_unlock(&r->st->rrdset_rwlock); } #endif // NETDATA_INTERNAL_CHECKS // Returns 1 if an absolute period was requested or 0 if it was a relative period -int rrdr_relative_window_to_absolute(long long *after, long long *before) { +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before) { time_t now = now_realtime_sec() - 1; int absolute_period_requested = -1; @@ -1624,10 +1659,25 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) { after_requested -= delta; } + time_t absolute_minimum_time = now - (10 * 365 * 86400); + time_t absolute_maximum_time = now + (1 * 365 * 86400); + + if (after_requested < absolute_minimum_time && !unittest_running) + after_requested = absolute_minimum_time; + + if (after_requested > absolute_maximum_time && !unittest_running) + after_requested = absolute_maximum_time; + + if (before_requested < absolute_minimum_time && !unittest_running) + before_requested = absolute_minimum_time; + + if (before_requested > absolute_maximum_time && !unittest_running) + before_requested = absolute_maximum_time; + *before = before_requested; *after = after_requested; - return absolute_period_requested; + return (absolute_period_requested != 1); } // #define DEBUG_QUERY_LOGIC 1 @@ -1636,7 +1686,7 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) { #define query_debug_log_init() BUFFER *debug_log = buffer_create(1000) #define query_debug_log(args...) buffer_sprintf(debug_log, ##args) #define query_debug_log_fin() { \ - info("QUERY: chart '%s', after:%lld, before:%lld, duration:%lld, points:%ld, res:%ld - wanted => after:%lld, before:%lld, points:%ld, group:%ld, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", st->name, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \ + info("QUERY: '%s', after:%ld, before:%ld, duration:%ld, points:%zu, res:%ld - wanted => after:%ld, before:%ld, points:%zu, group:%zu, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", qt->id, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \ buffer_free(debug_log); \ debug_log = NULL; \ } @@ -1648,21 +1698,18 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) { #define query_debug_log_free() debug_dummy() #endif -RRDR *rrd2rrdr( - ONEWAYALLOC *owa - , RRDSET *st - , long points_requested - , long long after_requested - , long long before_requested - , RRDR_GROUPING group_method - , long resampling_time_requested - , RRDR_OPTIONS options - , const char *dimensions - , struct context_param *context_param_list - , const char *group_options - , int timeout - , int tier -) { +bool query_target_calculate_window(QUERY_TARGET *qt) { + if (unlikely(!qt)) return false; + + size_t points_requested = (long)qt->request.points; + time_t after_requested = qt->request.after; + time_t before_requested = qt->request.before; + RRDR_GROUPING group_method = qt->request.group_method; + time_t resampling_time_requested = qt->request.resampling_time; + RRDR_OPTIONS options = qt->request.options; + size_t tier = qt->request.tier; + time_t update_every = qt->db.minimum_latest_update_every; + // RULES // points_requested = 0 // the user wants all the natural points the database has @@ -1676,10 +1723,9 @@ RRDR *rrd2rrdr( // when natural points are wanted, the query has to be aligned to the update_every // of the database - long points_wanted = points_requested; - long long after_wanted = after_requested; - long long before_wanted = before_requested; - int update_every = st->update_every; + size_t points_wanted = points_requested; + time_t after_wanted = after_requested; + time_t before_wanted = before_requested; bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED); bool automatic_natural_points = (points_wanted == 0); @@ -1689,13 +1735,7 @@ RRDR *rrd2rrdr( query_debug_log_init(); - // make sure points_wanted is positive - if(points_wanted < 0) { - points_wanted = -points_wanted; - query_debug_log(":-points_wanted %ld", points_wanted); - } - - if(ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) { + if (ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) { relative_period_requested = true; natural_points = true; options |= RRDR_OPTION_NATURAL_POINTS; @@ -1703,105 +1743,93 @@ RRDR *rrd2rrdr( } // if the user wants virtual points, make sure we do it - if(options & RRDR_OPTION_VIRTUAL_POINTS) + if (options & RRDR_OPTION_VIRTUAL_POINTS) natural_points = false; // set the right flag about natural and virtual points - if(natural_points) { + if (natural_points) { options |= RRDR_OPTION_NATURAL_POINTS; - if(options & RRDR_OPTION_VIRTUAL_POINTS) + if (options & RRDR_OPTION_VIRTUAL_POINTS) options &= ~RRDR_OPTION_VIRTUAL_POINTS; } else { options |= RRDR_OPTION_VIRTUAL_POINTS; - if(options & RRDR_OPTION_NATURAL_POINTS) + if (options & RRDR_OPTION_NATURAL_POINTS) options &= ~RRDR_OPTION_NATURAL_POINTS; } - if(after_wanted == 0 || before_wanted == 0) { - // for non-context queries we have to find the duration of the database - // for context queries we will assume 600 seconds duration - - if(!context_param_list) { - relative_period_requested = true; - - rrdset_rdlock(st); - time_t first_entry_t = rrdset_first_entry_t_nolock(st); - time_t last_entry_t = rrdset_last_entry_t_nolock(st); - rrdset_unlock(st); - - if(first_entry_t == 0 || last_entry_t == 0) { - internal_error(true, "QUERY: chart without data detected on '%s'", st->name); - query_debug_log_free(); - return NULL; - } + if (after_wanted == 0 || before_wanted == 0) { + relative_period_requested = true; - query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t); + time_t first_entry_t = qt->db.first_time_t; + time_t last_entry_t = qt->db.last_time_t; - if (after_wanted == 0) { - after_wanted = first_entry_t; - query_debug_log(":zero after_wanted %lld", after_wanted); - } + if (first_entry_t == 0 || last_entry_t == 0) { + internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld", qt->id, first_entry_t, last_entry_t); + query_debug_log_free(); + return false; + } - if (before_wanted == 0) { - before_wanted = last_entry_t; - before_is_aligned_to_db_end = true; - query_debug_log(":zero before_wanted %lld", before_wanted); - } + query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t); - if(points_wanted == 0) { - points_wanted = (last_entry_t - first_entry_t) / update_every; - query_debug_log(":zero points_wanted %ld", points_wanted); - } + if (after_wanted == 0) { + after_wanted = first_entry_t; + query_debug_log(":zero after_wanted %ld", after_wanted); } - // if they are still zero, assume 600 + if (before_wanted == 0) { + before_wanted = last_entry_t; + before_is_aligned_to_db_end = true; + query_debug_log(":zero before_wanted %ld", before_wanted); + } - if(after_wanted == 0) { - after_wanted = -600; - query_debug_log(":zero600 after_wanted %lld", after_wanted); + if (points_wanted == 0) { + points_wanted = (last_entry_t - first_entry_t) / update_every; + query_debug_log(":zero points_wanted %zu", points_wanted); } } - if(points_wanted == 0) { + if (points_wanted == 0) { points_wanted = 600; - query_debug_log(":zero600 points_wanted %ld", points_wanted); + query_debug_log(":zero600 points_wanted %zu", points_wanted); } // convert our before_wanted and after_wanted to absolute rrdr_relative_window_to_absolute(&after_wanted, &before_wanted); - query_debug_log(":relative2absolute after %lld, before %lld", after_wanted, before_wanted); + query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted); - if(natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) { - update_every = rrdset_find_natural_update_every_for_timeframe(st, after_wanted, before_wanted, points_wanted, options, tier); - if(update_every <= 0) update_every = st->update_every; - query_debug_log(":natural update every %d", update_every); + if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) { + update_every = rrdset_find_natural_update_every_for_timeframe( + qt, after_wanted, before_wanted, points_wanted, options, tier); + + if (update_every <= 0) update_every = qt->db.minimum_latest_update_every; + query_debug_log(":natural update every %ld", update_every); } // this is the update_every of the query // it may be different to the update_every of the database - time_t query_granularity = (natural_points)?update_every:1; - if(query_granularity <= 0) query_granularity = 1; + time_t query_granularity = (natural_points) ? update_every : 1; + if (query_granularity <= 0) query_granularity = 1; query_debug_log(":query_granularity %ld", query_granularity); // align before_wanted and after_wanted to query_granularity if (before_wanted % query_granularity) { before_wanted -= before_wanted % query_granularity; - query_debug_log(":granularity align before_wanted %lld", before_wanted); + query_debug_log(":granularity align before_wanted %ld", before_wanted); } if (after_wanted % query_granularity) { after_wanted -= after_wanted % query_granularity; - query_debug_log(":granularity align after_wanted %lld", after_wanted); + query_debug_log(":granularity align after_wanted %ld", after_wanted); } // automatic_natural_points is set when the user wants all the points available in the database - if(automatic_natural_points) { + if (automatic_natural_points) { points_wanted = (before_wanted - after_wanted + 1) / query_granularity; - if(unlikely(points_wanted <= 0)) points_wanted = 1; - query_debug_log(":auto natural points_wanted %ld", points_wanted); + if (unlikely(points_wanted <= 0)) points_wanted = 1; + query_debug_log(":auto natural points_wanted %zu", points_wanted); } time_t duration = before_wanted - after_wanted; @@ -1810,42 +1838,47 @@ RRDR *rrd2rrdr( if (unlikely(resampling_time_requested > duration)) { after_wanted = before_wanted - resampling_time_requested; duration = before_wanted - after_wanted; - query_debug_log(":resampling after_wanted %lld", after_wanted); + query_debug_log(":resampling after_wanted %ld", after_wanted); } // if the duration is not aligned to resampling time // extend the duration to the past, to avoid a gap at the chart // only when the missing duration is above 1/10th of a point - if(resampling_time_requested > query_granularity && duration % resampling_time_requested) { + if (resampling_time_requested > query_granularity && duration % resampling_time_requested) { time_t delta = duration % resampling_time_requested; - if(delta > resampling_time_requested / 10) { + if (delta > resampling_time_requested / 10) { after_wanted -= resampling_time_requested - delta; duration = before_wanted - after_wanted; - query_debug_log(":resampling2 after_wanted %lld", after_wanted); + query_debug_log(":resampling2 after_wanted %ld", after_wanted); } } // the available points of the query - long points_available = (duration + 1) / query_granularity; - if(unlikely(points_available <= 0)) points_available = 1; - query_debug_log(":points_available %ld", points_available); + size_t points_available = (duration + 1) / query_granularity; + if (unlikely(points_available <= 0)) points_available = 1; + query_debug_log(":points_available %zu", points_available); - if(points_wanted > points_available) { + if (points_wanted > points_available) { points_wanted = points_available; - query_debug_log(":max points_wanted %ld", points_wanted); + query_debug_log(":max points_wanted %zu", points_wanted); + } + + if(points_wanted > 86400 && !unittest_running) { + points_wanted = 86400; + query_debug_log(":absolute max points_wanted %zu", points_wanted); } // calculate the desired grouping of source data points - long group = points_available / points_wanted; - if(group <= 0) group = 1; + size_t group = points_available / points_wanted; + if (group == 0) group = 1; // round "group" to the closest integer - if(points_available % points_wanted > points_wanted / 2) + if (points_available % points_wanted > points_wanted / 2) group++; - query_debug_log(":group %ld", group); + query_debug_log(":group %zu", group); - if(points_wanted * group * query_granularity < duration) { + if (points_wanted * group * query_granularity < (size_t)duration) { // the grouping we are going to do, is not enough // to cover the entire duration requested, so // we have to change the number of points, to make sure we will @@ -1854,162 +1887,186 @@ RRDR *rrd2rrdr( // let's see how many points are the optimal points_wanted = points_available / group; - if(points_wanted * group < points_available) + if (points_wanted * group < points_available) points_wanted++; - if(unlikely(points_wanted <= 0)) + if (unlikely(points_wanted == 0)) points_wanted = 1; - query_debug_log(":optimal points %ld", points_wanted); + query_debug_log(":optimal points %zu", points_wanted); } // resampling_time_requested enforces a certain grouping multiple NETDATA_DOUBLE resampling_divisor = 1.0; - long resampling_group = 1; - if(unlikely(resampling_time_requested > query_granularity)) { + size_t resampling_group = 1; + if (unlikely(resampling_time_requested > query_granularity)) { // the points we should group to satisfy gtime resampling_group = resampling_time_requested / query_granularity; - if(unlikely(resampling_time_requested % query_granularity)) + if (unlikely(resampling_time_requested % query_granularity)) resampling_group++; - query_debug_log(":resampling group %ld", resampling_group); + query_debug_log(":resampling group %zu", resampling_group); // adapt group according to resampling_group - if(unlikely(group < resampling_group)) { - group = resampling_group; // do not allow grouping below the desired one - query_debug_log(":group less res %ld", group); + if (unlikely(group < resampling_group)) { + group = resampling_group; // do not allow grouping below the desired one + query_debug_log(":group less res %zu", group); } - if(unlikely(group % resampling_group)) { + if (unlikely(group % resampling_group)) { group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group - query_debug_log(":group mod res %ld", group); + query_debug_log(":group mod res %zu", group); } // resampling_divisor = group / resampling_group; - resampling_divisor = (NETDATA_DOUBLE)(group * query_granularity) / (NETDATA_DOUBLE)resampling_time_requested; + resampling_divisor = (NETDATA_DOUBLE) (group * query_granularity) / (NETDATA_DOUBLE) resampling_time_requested; query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor); } // now that we have group, align the requested timeframe to fit it. - if(aligned && before_wanted % (group * query_granularity)) { - if(before_is_aligned_to_db_end) - before_wanted -= before_wanted % (group * query_granularity); + if (aligned && before_wanted % (group * query_granularity)) { + if (before_is_aligned_to_db_end) + before_wanted -= before_wanted % (time_t)(group * query_granularity); else - before_wanted += (group * query_granularity) - before_wanted % (group * query_granularity); - query_debug_log(":align before_wanted %lld", before_wanted); + before_wanted += (time_t)(group * query_granularity) - before_wanted % (time_t)(group * query_granularity); + query_debug_log(":align before_wanted %ld", before_wanted); } - after_wanted = before_wanted - (points_wanted * group * query_granularity) + query_granularity; - query_debug_log(":final after_wanted %lld", after_wanted); + after_wanted = before_wanted - (time_t)(points_wanted * group * query_granularity) + query_granularity; + query_debug_log(":final after_wanted %ld", after_wanted); duration = before_wanted - after_wanted; query_debug_log(":final duration %ld", duration + 1); - // check the context query based on the starting time of the query - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { - rebuild_context_param_list(owa, context_param_list, after_wanted); - st = context_param_list->rd ? context_param_list->rd->rrdset : NULL; - - if(unlikely(!st)) - return NULL; - } + query_debug_log_fin(); internal_error(points_wanted != duration / (query_granularity * group) + 1, - "QUERY: points_wanted %ld is not points %ld", - points_wanted, duration / (query_granularity * group) + 1); + "QUERY: points_wanted %zu is not points %zu", + points_wanted, (size_t)(duration / (query_granularity * group) + 1)); internal_error(group < resampling_group, - "QUERY: group %ld is less than the desired group points %ld", + "QUERY: group %zu is less than the desired group points %zu", group, resampling_group); internal_error(group > resampling_group && group % resampling_group, - "QUERY: group %ld is not a multiple of the desired group points %ld", + "QUERY: group %zu is not a multiple of the desired group points %zu", group, resampling_group); // ------------------------------------------------------------------------- - // initialize our result set - // this also locks the chart for us + // update QUERY_TARGET with our calculations + + qt->window.after = after_wanted; + qt->window.before = before_wanted; + qt->window.relative = relative_period_requested; + qt->window.points = points_wanted; + qt->window.group = group; + qt->window.group_method = group_method; + qt->window.group_options = qt->request.group_options; + qt->window.query_granularity = query_granularity; + qt->window.resampling_group = resampling_group; + qt->window.resampling_divisor = resampling_divisor; + qt->window.options = options; + qt->window.tier = tier; + qt->window.aligned = aligned; + + return true; +} + +RRDR *rrd2rrdr_legacy( + ONEWAYALLOC *owa, + RRDSET *st, size_t points, time_t after, time_t before, + RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions, + const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source) { + + QUERY_TARGET_REQUEST qtr = { + .st = st, + .points = points, + .after = after, + .before = before, + .group_method = group_method, + .resampling_time = resampling_time, + .options = options, + .dimensions = dimensions, + .group_options = group_options, + .timeout = timeout, + .tier = tier, + .query_source = query_source, + }; + + return rrd2rrdr(owa, query_target_create(&qtr)); +} - RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list); +RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) { + if(!qt) + return NULL; + + if(!owa) { + query_target_release(qt); + return NULL; + } + + // qt.window members are the WANTED ones. + // qt.request members are the REQUESTED ones. + + RRDR *r = rrdr_create(owa, qt); if(unlikely(!r)) { - internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", - st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted); + internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, points=%zu", + qt->id, qt->window.after, qt->window.before, qt->window.points); return NULL; } - if(unlikely(!r->d || !points_wanted)) { - internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld", - st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted); + if(unlikely(!r->d || !qt->window.points)) { + internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%ld, before=%ld, points=%zu", + qt->id, qt->window.after, qt->window.before, qt->window.points); return r; } - if(relative_period_requested) + if(qt->window.relative) r->result_options |= RRDR_RESULT_OPTION_RELATIVE; else r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; - // find how many dimensions we have - long dimensions_count = r->d; - // ------------------------------------------------------------------------- // initialize RRDR - r->group = group; - r->update_every = (int)(group * query_granularity); - r->before = before_wanted; - r->after = after_wanted; - r->internal.points_wanted = points_wanted; - r->internal.resampling_group = resampling_group; - r->internal.resampling_divisor = resampling_divisor; - r->internal.query_options = options; - r->internal.query_tier = tier; + r->group = qt->window.group; + r->update_every = (int) (qt->window.group * qt->window.query_granularity); + r->before = qt->window.before; + r->after = qt->window.after; + r->internal.points_wanted = qt->window.points; + r->internal.resampling_group = qt->window.resampling_group; + r->internal.resampling_divisor = qt->window.resampling_divisor; + r->internal.query_options = qt->window.options; // ------------------------------------------------------------------------- // assign the processor functions - rrdr_set_grouping_function(r, group_method); + rrdr_set_grouping_function(r, qt->window.group_method); // allocate any memory required by the grouping method - r->internal.grouping_create(r, group_options); - - - // ------------------------------------------------------------------------- - // disable the not-wanted dimensions - - if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) - rrdset_check_rdlock(st); - - if(dimensions) - rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list); - - - query_debug_log_fin(); + r->internal.grouping_create(r, qt->window.group_options); // ------------------------------------------------------------------------- // do the work for each dimension time_t max_after = 0, min_before = 0; - long max_rows = 0; + size_t max_rows = 0; - RRDDIM *first_rd = context_param_list ? context_param_list->rd : st->dimensions; - RRDDIM *rd; - long c, dimensions_used = 0, dimensions_nonzero = 0; + long dimensions_used = 0, dimensions_nonzero = 0; struct timeval query_start_time; struct timeval query_current_time; - if (timeout) now_realtime_timeval(&query_start_time); + if (qt->request.timeout) + now_realtime_timeval(&query_start_time); - for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { + for(size_t c = 0, max = qt->query.used; c < max ; c++) { + // set the query target dimension options to rrdr + r->od[c] = qt->query.array[c].dimension.options; - // if we need a percentage, we need to calculate all dimensions - if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) { - if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED; - continue; - } r->od[c] |= RRDR_DIMENSION_SELECTED; // reset the grouping for the new dimension r->internal.grouping_reset(r); - rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted); - if (timeout) + rrd2rrdr_do_dimension(r, c); + if (qt->request.timeout) now_realtime_timeval(&query_current_time); if(r->od[c] & RRDR_DIMENSION_NONZERO) @@ -2024,30 +2081,30 @@ RRDR *rrd2rrdr( else { if(r->after != max_after) { internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_after, rd->name, (size_t)r->after); + string2str(qt->query.array[c].dimension.id), (size_t)max_after, string2str(qt->query.array[c].dimension.name), (size_t)r->after); r->after = (r->after > max_after) ? r->after : max_after; } if(r->before != min_before) { internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)min_before, rd->name, (size_t)r->before); + string2str(qt->query.array[c].dimension.id), (size_t)min_before, string2str(qt->query.array[c].dimension.name), (size_t)r->before); r->before = (r->before < min_before) ? r->before : min_before; } if(r->rows != max_rows) { internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu", - st->name, (size_t)max_rows, rd->name, (size_t)r->rows); + string2str(qt->query.array[c].dimension.id), (size_t)max_rows, string2str(qt->query.array[c].dimension.name), (size_t)r->rows); r->rows = (r->rows > max_rows) ? r->rows : max_rows; } } dimensions_used++; - if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) { - log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)", - (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout); + if (qt->request.timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout) { + log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)", + (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout); r->result_options |= RRDR_RESULT_OPTION_CANCEL; break; } @@ -2056,44 +2113,44 @@ RRDR *rrd2rrdr( #ifdef NETDATA_INTERNAL_CHECKS if (dimensions_used) { if(r->internal.log) - rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, - after_wanted, after_requested, before_wanted, before_requested, - points_requested, points_wanted, /*after_slot, before_slot,*/ + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + qt->window.after, qt->request.after, qt->window.before, qt->request.before, + qt->request.points, qt->window.points, /*after_slot, before_slot,*/ r->internal.log); - if(r->rows != points_wanted) - rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, - after_wanted, after_requested, before_wanted, before_requested, - points_requested, points_wanted, /*after_slot, before_slot,*/ + if(r->rows != qt->window.points) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + qt->window.after, qt->request.after, qt->window.before, qt->request.before, + qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "got 'points' is not wanted 'points'"); - if(aligned && (r->before % (group * query_granularity)) != 0) - rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, - after_wanted, after_requested, before_wanted,before_wanted, - points_requested, points_wanted, /*after_slot, before_slot,*/ + if(qt->window.aligned && (r->before % (qt->window.group * qt->window.query_granularity)) != 0) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + qt->window.after, qt->request.after, qt->window.before,qt->request.before, + qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "'before' is not aligned but alignment is required"); // 'after' should not be aligned, since we start inside the first group - //if(aligned && (r->after % group) != 0) - // rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); + //if(qt->window.aligned && (r->after % group) != 0) + // rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required"); - if(r->before != before_wanted) - rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, - after_wanted, after_requested, before_wanted, before_requested, - points_requested, points_wanted, /*after_slot, before_slot,*/ + if(r->before != qt->window.before) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + qt->window.after, qt->request.after, qt->window.before, qt->request.before, + qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "chart is not aligned to requested 'before'"); - if(r->before != before_wanted) - rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, - after_wanted, after_requested, before_wanted, before_requested, - points_requested, points_wanted, /*after_slot, before_slot,*/ + if(r->before != qt->window.before) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + qt->window.after, qt->request.after, qt->window.before, qt->request.before, + qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "got 'before' is not wanted 'before'"); // reported 'after' varies, depending on group - if(r->after != after_wanted) - rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, - after_wanted, after_requested, before_wanted, before_requested, - points_requested, points_wanted, /*after_slot, before_slot,*/ + if(r->after != qt->window.after) + rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, + qt->window.after, qt->request.after, qt->window.before, qt->request.before, + qt->request.points, qt->window.points, /*after_slot, before_slot,*/ "got 'after' is not wanted 'after'"); } @@ -2103,15 +2160,16 @@ RRDR *rrd2rrdr( r->internal.grouping_free(r); // when all the dimensions are zero, we should return all of them - if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) { + if(unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) { // all the dimensions are zero // mark them as NONZERO to send them all - for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) { + for(size_t c = 0, max = qt->query.used; c < max ; c++) { if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue; r->od[c] |= RRDR_DIMENSION_NONZERO; } } - rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated); + global_statistics_rrdr_query_completed(dimensions_used, r->internal.db_points_read, + r->internal.result_points_generated, qt->request.query_source); return r; } diff --git a/web/api/queries/query.h b/web/api/queries/query.h index df876d9ac..ebad5a1f8 100644 --- a/web/api/queries/query.h +++ b/web/api/queries/query.h @@ -47,10 +47,10 @@ typedef enum rrdr_grouping { RRDR_GROUPING_COUNTIF, } RRDR_GROUPING; -extern const char *group_method2string(RRDR_GROUPING group); -extern void web_client_api_v1_init_grouping(void); -extern RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def); -extern const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group); +const char *group_method2string(RRDR_GROUPING group); +void web_client_api_v1_init_grouping(void); +RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def); +const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group); #ifdef __cplusplus } diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c index ecf4ca2ac..676224c9d 100644 --- a/web/api/queries/rrdr.c +++ b/web/api/queries/rrdr.c @@ -61,9 +61,7 @@ static void rrdr_dump(RRDR *r) inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) { if(unlikely(!r)) return; - if(likely(r->st_locked_by_rrdr_create)) - rrdset_unlock(r->st); - + query_target_release(r->internal.qt); onewayalloc_freez(owa, r->t); onewayalloc_freez(owa, r->v); onewayalloc_freez(owa, r->o); @@ -72,12 +70,23 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) { onewayalloc_freez(owa, r); } -RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) { +RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt) { + if(unlikely(!qt || !qt->query.used || !qt->window.points)) + return NULL; + + size_t dimensions = qt->query.used; + size_t points = qt->window.points; + + // create the rrdr RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR)); r->internal.owa = owa; + r->internal.qt = qt; - r->d = dimensions; - r->n = points; + r->before = qt->window.before; + r->after = qt->window.after; + r->internal.points_wanted = qt->window.points; + r->d = (int)dimensions; + r->n = (int)points; r->t = onewayalloc_callocz(owa, points, sizeof(time_t)); r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE)); @@ -90,42 +99,3 @@ RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points return r; } - -RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) { - if (unlikely(!st)) return NULL; - - bool st_locked_by_rrdr_create = false; - if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) { - rrdset_rdlock(st); - st_locked_by_rrdr_create = true; - } - - // count the number of dimensions - int dimensions = 0; - RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL; - RRDDIM *rd; - if (temp_rd) { - RRDDIM *t = temp_rd; - while (t) { - dimensions++; - t = t->next; - } - } else - rrddim_foreach_read(rd, st) dimensions++; - - // create the rrdr - RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n); - r->st = st; - r->st_locked_by_rrdr_create = st_locked_by_rrdr_create; - - // set the hidden flag on hidden dimensions - int c; - for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) { - if (unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN))) - r->od[c] = RRDR_DIMENSION_HIDDEN; - else - r->od[c] = RRDR_DIMENSION_DEFAULT; - } - - return r; -} diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h index 1c80e103f..6151cddc7 100644 --- a/web/api/queries/rrdr.h +++ b/web/api/queries/rrdr.h @@ -18,32 +18,33 @@ typedef enum tier_query_fetch { } TIER_QUERY_FETCH; typedef enum rrdr_options { - RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values - RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest) - RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing - RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum - RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates - RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates - RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros - RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array - RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs - RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result - RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes - RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total - RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes - RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign - RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs - RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names - RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response - RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database - RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points - RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number - RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes - RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap - RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query + RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values + RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest) + RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing + RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum + RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates + RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates + RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros + RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array + RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs + RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result + RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes + RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total + RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes + RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign + RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs + RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names + RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database + RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points + RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number + RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes + RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap + RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query + RRDR_OPTION_ALL_DIMENSIONS = 0x00800000, // Return the full dimensions list // internal ones - not to be exposed to the API - RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate + RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate + RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION } RRDR_OPTIONS; typedef enum rrdr_value_flag { @@ -67,16 +68,14 @@ typedef enum rrdr_result_flags { // (should not to be cached by browsers and proxies) RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames RRDR_RESULT_OPTION_CANCEL = 0x00000008, // the query needs to be cancelled -} RRDR_RESULT_FLAGS; +} RRDR_RESULT_OPTIONS; typedef struct rrdresult { - struct rrdset *st; // the chart this result refers to + RRDR_RESULT_OPTIONS result_options; // RRDR_RESULT_OPTION_* - RRDR_RESULT_FLAGS result_options; // RRDR_RESULT_OPTION_* - - int d; // the number of dimensions - long n; // the number of values in the arrays - long rows; // the number of rows used + size_t d; // the number of dimensions + size_t n; // the number of values in the arrays + size_t rows; // the number of rows used RRDR_DIMENSION_FLAGS *od; // the options for the dimensions @@ -85,8 +84,8 @@ typedef struct rrdresult { RRDR_VALUE_FLAGS *o; // array n x d options for each value returned NETDATA_DOUBLE *ar; // array n x d of anomaly rates (0 - 100) - long group; // how many collected values were grouped for each row - int update_every; // what is the suggested update frequency in seconds + size_t group; // how many collected values were grouped for each row + time_t update_every; // what is the suggested update frequency in seconds NETDATA_DOUBLE min; NETDATA_DOUBLE max; @@ -94,53 +93,57 @@ typedef struct rrdresult { time_t before; time_t after; - bool st_locked_by_rrdr_create; // if st is read locked by us - // internal rrd2rrdr() members below this point struct { - int query_tier; // the selected tier - RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query) + ONEWAYALLOC *owa; // the allocator used + struct query_target *qt; // the QUERY_TARGET + + RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query) - long points_wanted; - long resampling_group; - NETDATA_DOUBLE resampling_divisor; + size_t points_wanted; // used by SES and DES + size_t resampling_group; // used by AVERAGE + NETDATA_DOUBLE resampling_divisor; // used by AVERAGE + // grouping function pointers void (*grouping_create)(struct rrdresult *r, const char *options); void (*grouping_reset)(struct rrdresult *r); void (*grouping_free)(struct rrdresult *r); void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value); NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); - void *grouping_data; - TIER_QUERY_FETCH tier_query_fetch; - #ifdef NETDATA_INTERNAL_CHECKS + TIER_QUERY_FETCH tier_query_fetch; // which value to use from STORAGE_POINT + void *grouping_data; // the internal data of the grouping function + +#ifdef NETDATA_INTERNAL_CHECKS const char *log; - #endif +#endif + // statistics size_t db_points_read; size_t result_points_generated; size_t tier_points_read[RRD_STORAGE_TIERS]; - ONEWAYALLOC *owa; } internal; } RRDR; #define rrdr_rows(r) ((r)->rows) #include "database/rrd.h" -extern void rrdr_free(ONEWAYALLOC *owa, RRDR *r); -extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list); -extern RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points); +void rrdr_free(ONEWAYALLOC *owa, RRDR *r); +RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt); #include "../web_api_v1.h" #include "web/api/queries/query.h" -extern RRDR *rrd2rrdr( - ONEWAYALLOC *owa, - RRDSET *st, long points_wanted, long long after_wanted, long long before_wanted, - RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions, - struct context_param *context_param_list, const char *group_options, int timeout, int tier); +RRDR *rrd2rrdr_legacy( + ONEWAYALLOC *owa, + RRDSET *st, size_t points, time_t after, time_t before, + RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions, + const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source); + +RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt); +bool query_target_calculate_window(struct query_target *qt); -extern int rrdr_relative_window_to_absolute(long long *after, long long *before); +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before); #ifdef __cplusplus } diff --git a/web/api/queries/ses/ses.h b/web/api/queries/ses/ses.h index 094b8de3f..79b09fbdf 100644 --- a/web/api/queries/ses/ses.h +++ b/web/api/queries/ses/ses.h @@ -6,12 +6,12 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_init_ses(void); +void grouping_init_ses(void); -extern void grouping_create_ses(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_ses(RRDR *r); -extern void grouping_free_ses(RRDR *r); -extern void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_ses(RRDR *r, const char *options __maybe_unused); +void grouping_reset_ses(RRDR *r); +void grouping_free_ses(RRDR *r); +void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_SES_H diff --git a/web/api/queries/stddev/stddev.h b/web/api/queries/stddev/stddev.h index c5c91f88d..4b8ffcd53 100644 --- a/web/api/queries/stddev/stddev.h +++ b/web/api/queries/stddev/stddev.h @@ -6,13 +6,13 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_stddev(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_stddev(RRDR *r); -extern void grouping_free_stddev(RRDR *r); -extern void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -extern NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -// extern NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); -// extern NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_stddev(RRDR *r, const char *options __maybe_unused); +void grouping_reset_stddev(RRDR *r); +void grouping_free_stddev(RRDR *r); +void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +// NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +// NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_STDDEV_H diff --git a/web/api/queries/sum/sum.h b/web/api/queries/sum/sum.h index 4e7e396e9..898782775 100644 --- a/web/api/queries/sum/sum.h +++ b/web/api/queries/sum/sum.h @@ -6,10 +6,10 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_sum(RRDR *r, const char *options __maybe_unused); -extern void grouping_reset_sum(RRDR *r); -extern void grouping_free_sum(RRDR *r); -extern void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_sum(RRDR *r, const char *options __maybe_unused); +void grouping_reset_sum(RRDR *r); +void grouping_free_sum(RRDR *r); +void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERY_SUM_H diff --git a/web/api/queries/trimmed_mean/trimmed_mean.h b/web/api/queries/trimmed_mean/trimmed_mean.h index 1a4f63e9c..e66d92541 100644 --- a/web/api/queries/trimmed_mean/trimmed_mean.h +++ b/web/api/queries/trimmed_mean/trimmed_mean.h @@ -6,17 +6,17 @@ #include "../query.h" #include "../rrdr.h" -extern void grouping_create_trimmed_mean1(RRDR *r, const char *options); -extern void grouping_create_trimmed_mean2(RRDR *r, const char *options); -extern void grouping_create_trimmed_mean3(RRDR *r, const char *options); -extern void grouping_create_trimmed_mean5(RRDR *r, const char *options); -extern void grouping_create_trimmed_mean10(RRDR *r, const char *options); -extern void grouping_create_trimmed_mean15(RRDR *r, const char *options); -extern void grouping_create_trimmed_mean20(RRDR *r, const char *options); -extern void grouping_create_trimmed_mean25(RRDR *r, const char *options); -extern void grouping_reset_trimmed_mean(RRDR *r); -extern void grouping_free_trimmed_mean(RRDR *r); -extern void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value); -extern NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); +void grouping_create_trimmed_mean1(RRDR *r, const char *options); +void grouping_create_trimmed_mean2(RRDR *r, const char *options); +void grouping_create_trimmed_mean3(RRDR *r, const char *options); +void grouping_create_trimmed_mean5(RRDR *r, const char *options); +void grouping_create_trimmed_mean10(RRDR *r, const char *options); +void grouping_create_trimmed_mean15(RRDR *r, const char *options); +void grouping_create_trimmed_mean20(RRDR *r, const char *options); +void grouping_create_trimmed_mean25(RRDR *r, const char *options); +void grouping_reset_trimmed_mean(RRDR *r); +void grouping_free_trimmed_mean(RRDR *r); +void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value); +NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr); #endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index 97a00f91c..a9555a66b 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -56,40 +56,14 @@ typedef enum { struct register_result { RESULT_FLAGS flags; - RRDSET *st; - const char *chart_id; - const char *context; - const char *dim_name; + RRDCONTEXT_ACQUIRED *rca; + RRDINSTANCE_ACQUIRED *ria; + RRDMETRIC_ACQUIRED *rma; NETDATA_DOUBLE value; - - struct register_result *next; // used to link contexts together }; -static void register_result_insert_callback(const char *name, void *value, void *data) { - (void)name; - (void)data; - - struct register_result *t = (struct register_result *)value; - - if(t->chart_id) t->chart_id = strdupz(t->chart_id); - if(t->context) t->context = strdupz(t->context); - if(t->dim_name) t->dim_name = strdupz(t->dim_name); -} - -static void register_result_delete_callback(const char *name, void *value, void *data) { - (void)name; - (void)data; - struct register_result *t = (struct register_result *)value; - - freez((void *)t->chart_id); - freez((void *)t->context); - freez((void *)t->dim_name); -} - static DICTIONARY *register_result_init() { - DICTIONARY *results = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); - dictionary_register_insert_callback(results, register_result_insert_callback, results); - dictionary_register_delete_callback(results, register_result_delete_callback, results); + DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED); return results; } @@ -98,8 +72,9 @@ static void register_result_destroy(DICTIONARY *results) { } static void register_result(DICTIONARY *results, - RRDSET *st, - RRDDIM *d, + RRDCONTEXT_ACQUIRED *rca, + RRDINSTANCE_ACQUIRED *ria, + RRDMETRIC_ACQUIRED *rma, NETDATA_DOUBLE value, RESULT_FLAGS flags, WEIGHTS_STATS *stats, @@ -120,25 +95,25 @@ static void register_result(DICTIONARY *results, struct register_result t = { .flags = flags, - .st = st, - .chart_id = st->id, - .context = st->context, - .dim_name = d->name, + .rca = rca, + .ria = ria, + .rma = rma, .value = v }; - char buf[5000 + 1]; - snprintfz(buf, 5000, "%s:%s", st->id, d->name); - dictionary_set(results, buf, &t, sizeof(struct register_result)); + // we can use the pointer address or RMA as a unique key for each metric + char buf[20 + 1]; + ssize_t len = snprintfz(buf, 20, "%p", rma); + dictionary_set_advanced(results, buf, len + 1, &t, sizeof(struct register_result), NULL); } // ---------------------------------------------------------------------------- // Generation of JSON output for the results static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *wb, - long long after, long long before, - long long baseline_after, long long baseline_before, - long points, WEIGHTS_METHOD method, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, size_t examined_dimensions __maybe_unused, usec_t duration, WEIGHTS_STATS *stats) { @@ -147,10 +122,10 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w "\t\"after\": %lld,\n" "\t\"before\": %lld,\n" "\t\"duration\": %lld,\n" - "\t\"points\": %ld,\n", - after, - before, - before - after, + "\t\"points\": %zu,\n", + (long long)after, + (long long)before, + (long long)(before - after), points ); @@ -159,10 +134,10 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w "\t\"baseline_after\": %lld,\n" "\t\"baseline_before\": %lld,\n" "\t\"baseline_duration\": %lld,\n" - "\t\"baseline_points\": %ld,\n", - baseline_after, - baseline_before, - baseline_before - baseline_after, + "\t\"baseline_points\": %zu,\n", + (long long)baseline_after, + (long long)baseline_before, + (long long)(baseline_before - baseline_after), points << shifts ); @@ -181,7 +156,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w stats->db_points ); - for(int tier = 0; tier < storage_tiers ;tier++) + for(size_t tier = 0; tier < storage_tiers ;tier++) buffer_sprintf(wb, "%s%zu", tier?", ":"", stats->db_points_per_tier[tier]); buffer_sprintf(wb, " ]\n" @@ -193,13 +168,13 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w weights_method_to_string(method) ); - web_client_api_request_v1_data_options_to_string(wb, options); + web_client_api_request_v1_data_options_to_buffer(wb, options); } static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb, - long long after, long long before, - long long baseline_after, long long baseline_before, - long points, WEIGHTS_METHOD method, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, size_t examined_dimensions, usec_t duration, WEIGHTS_STATS *stats) { @@ -211,23 +186,23 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb, size_t charts = 0, chart_dims = 0, total_dimensions = 0; struct register_result *t; - RRDSET *last_st = NULL; // never access this - we use it only for comparison + RRDINSTANCE_ACQUIRED *last_ria = NULL; // never access this - we use it only for comparison dfe_start_read(results, t) { - if(!last_st || t->st != last_st) { - last_st = t->st; + if(t->ria != last_ria) { + last_ria = t->ria; if(charts) buffer_strcat(wb, "\n\t\t\t}\n\t\t},\n"); buffer_strcat(wb, "\t\t\""); - buffer_strcat(wb, t->chart_id); + buffer_strcat(wb, rrdinstance_acquired_id(t->ria)); buffer_strcat(wb, "\": {\n"); buffer_strcat(wb, "\t\t\t\"context\": \""); - buffer_strcat(wb, t->context); + buffer_strcat(wb, rrdcontext_acquired_id(t->rca)); buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n"); charts++; chart_dims = 0; } if (chart_dims) buffer_sprintf(wb, ",\n"); - buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, t->dim_name, t->value); + buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value); chart_dims++; total_dimensions++; } @@ -250,9 +225,9 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb, } static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *wb, - long long after, long long before, - long long baseline_after, long long baseline_before, - long points, WEIGHTS_METHOD method, + time_t after, time_t before, + time_t baseline_after, time_t baseline_before, + size_t points, WEIGHTS_METHOD method, RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts, size_t examined_dimensions, usec_t duration, WEIGHTS_STATS *stats) { @@ -260,78 +235,80 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w results_header_to_json(results, wb, after, before, baseline_after, baseline_before, points, method, group, options, shifts, examined_dimensions, duration, stats); - DICTIONARY *context_results = dictionary_create( - DICTIONARY_FLAG_SINGLE_THREADED - |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE - |DICTIONARY_FLAG_NAME_LINK_DONT_CLONE - |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE - ); + buffer_strcat(wb, "\",\n\t\"contexts\": {\n"); + size_t contexts = 0, charts = 0, total_dimensions = 0, context_dims = 0, chart_dims = 0; + NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0; struct register_result *t; + RRDCONTEXT_ACQUIRED *last_rca = NULL; + RRDINSTANCE_ACQUIRED *last_ria = NULL; dfe_start_read(results, t) { - struct register_result *tc = dictionary_set(context_results, t->context, t, sizeof(*t)); - if(tc == t) - t->next = NULL; - else { - t->next = tc->next; - tc->next = t; + + if(t->rca != last_rca) { + last_rca = t->rca; + + if(contexts) + buffer_sprintf(wb, "\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" + "\t\t\t\t}\n\t\t\t},\n" + "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n" + , charts_total_weight / (double)chart_dims + , contexts_total_weight / (double)context_dims); + + buffer_strcat(wb, "\t\t\""); + buffer_strcat(wb, rrdcontext_acquired_id(t->rca)); + buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n"); + + contexts++; + charts = 0; + context_dims = 0; + contexts_total_weight = 0.0; + + last_ria = NULL; } - } - dfe_done(t); - buffer_strcat(wb, "\",\n\t\"contexts\": {\n"); + if(t->ria != last_ria) { + last_ria = t->ria; - size_t contexts = 0, total_dimensions = 0, charts = 0, context_dims = 0, chart_dims = 0; - NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0; - RRDSET *last_st = NULL; // never access this - we use it only for comparison - dfe_start_read(context_results, t) { - - if(contexts) - buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims); - - contexts++; - context_dims = 0; - contexts_total_weight = 0.0; - - buffer_strcat(wb, "\t\t\""); - buffer_strcat(wb, t->context); - buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n"); - - charts = 0; - chart_dims = 0; - struct register_result *tt; - for(tt = t; tt ; tt = tt->next) { - if(!last_st || tt->st != last_st) { - last_st = tt->st; - - if(charts) - buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t},\n", charts_total_weight / chart_dims); - - buffer_strcat(wb, "\t\t\t\t\""); - buffer_strcat(wb, tt->chart_id); - buffer_strcat(wb, "\": {\n"); - buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n"); - charts++; - chart_dims = 0; - charts_total_weight = 0.0; - } - - if (chart_dims) buffer_sprintf(wb, ",\n"); - buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, tt->dim_name, tt->value); - charts_total_weight += tt->value; - contexts_total_weight += tt->value; - chart_dims++; - context_dims++; - total_dimensions++; + if(charts) + buffer_sprintf(wb, "\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" + "\t\t\t\t},\n" + , charts_total_weight / (double)chart_dims); + + buffer_strcat(wb, "\t\t\t\t\""); + buffer_strcat(wb, rrdinstance_acquired_id(t->ria)); + buffer_strcat(wb, "\": {\n"); + buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n"); + + charts++; + chart_dims = 0; + charts_total_weight = 0.0; } + + if (chart_dims) buffer_sprintf(wb, ",\n"); + buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value); + charts_total_weight += t->value; + contexts_total_weight += t->value; + chart_dims++; + context_dims++; + total_dimensions++; } dfe_done(t); - dictionary_destroy(context_results); - // close dimensions and chart if (total_dimensions) - buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t}\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims); + buffer_sprintf(wb, "\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" + "\t\t\t\t}\n" + "\t\t\t},\n" + "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n" + "\t\t}\n" + , charts_total_weight / (double)chart_dims + , contexts_total_weight / (double)context_dims); // close correlated_charts buffer_sprintf(wb, "\t},\n" @@ -391,7 +368,10 @@ static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, NETDATA_DOUBLE *arr, si return added; } -static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMBERS highlight_diffs[], int high_size, uint32_t base_shifts) { +static double ks_2samp( + DIFFS_NUMBERS baseline_diffs[], int base_size, + DIFFS_NUMBERS highlight_diffs[], int high_size, + uint32_t base_shifts) { qsort(baseline_diffs, base_size, sizeof(DIFFS_NUMBERS), compare_diffs); qsort(highlight_diffs, high_size, sizeof(DIFFS_NUMBERS), compare_diffs); @@ -414,7 +394,7 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB // This would require a lot of multiplications and divisions. // // To speed it up, we do the binary search to find the index of each number - // but then we divide the base index by the power of two number (shifts) it + // but, then we divide the base index by the power of two number (shifts) it // is bigger than high index. So the 2 indexes are now comparable. // We also keep track of the original indexes with min and max, to properly // calculate their percentages once the loops finish. @@ -495,7 +475,9 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB static double kstwo( NETDATA_DOUBLE baseline[], int baseline_points, - NETDATA_DOUBLE highlight[], int highlight_points, uint32_t base_shifts) { + NETDATA_DOUBLE highlight[], int highlight_points, + uint32_t base_shifts) { + // -1 in size, since the calculate_pairs_diffs() returns one less point DIFFS_NUMBERS baseline_diffs[baseline_points - 1]; DIFFS_NUMBERS highlight_diffs[highlight_points - 1]; @@ -514,308 +496,215 @@ static double kstwo( return ks_2samp(baseline_diffs, base_size, highlight_diffs, high_size, base_shifts); } +NETDATA_DOUBLE *rrd2rrdr_ks2( + ONEWAYALLOC *owa, RRDHOST *host, + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma, + time_t after, time_t before, size_t points, RRDR_OPTIONS options, + RRDR_GROUPING group_method, const char *group_options, size_t tier, + WEIGHTS_STATS *stats, + size_t *entries + ) { + + NETDATA_DOUBLE *ret = NULL; + + QUERY_TARGET_REQUEST qtr = { + .host = host, + .rca = rca, + .ria = ria, + .rma = rma, + .after = after, + .before = before, + .points = points, + .options = options, + .group_method = group_method, + .group_options = group_options, + .tier = tier, + .query_source = QUERY_SOURCE_API_WEIGHTS, + }; -static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results, - long long baseline_after, long long baseline_before, - long long after, long long before, - long long points, RRDR_OPTIONS options, - RRDR_GROUPING group, const char *group_options, int tier, - uint32_t shifts, int timeout, - WEIGHTS_STATS *stats, bool register_zero) { - options |= RRDR_OPTION_NATURAL_POINTS; - - long group_time = 0; - struct context_param *context_param_list = NULL; - - int examined_dimensions = 0; - - RRDR *high_rrdr = NULL; - RRDR *base_rrdr = NULL; - - // get first the highlight to find the number of points available - stats->db_queries++; - usec_t started_usec = now_realtime_usec(); - ONEWAYALLOC *owa = onewayalloc_create(0); - high_rrdr = rrd2rrdr(owa, st, points, - after, before, group, - group_time, options, NULL, context_param_list, group_options, - timeout, tier); - if(!high_rrdr) { - info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name); + RRDR *r = rrd2rrdr(owa, query_target_create(&qtr)); + if(!r) goto cleanup; - } - for(int i = 0; i < storage_tiers ;i++) - stats->db_points_per_tier[i] += high_rrdr->internal.tier_points_read[i]; + stats->db_queries++; + stats->result_points += r->internal.result_points_generated; + stats->db_points += r->internal.db_points_read; + for(size_t tr = 0; tr < storage_tiers ; tr++) + stats->db_points_per_tier[tr] += r->internal.tier_points_read[tr]; - stats->db_points += high_rrdr->internal.db_points_read; - stats->result_points += high_rrdr->internal.result_points_generated; - if(!high_rrdr->d) { - info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name); + if(r->d != 1) { + error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu", r->internal.qt->id, r->d); goto cleanup; } - if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) { - info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", st->name); + + if(unlikely(r->od[0] & RRDR_DIMENSION_HIDDEN)) goto cleanup; - } - int high_points = rrdr_rows(high_rrdr); - usec_t now_usec = now_realtime_usec(); - if(now_usec - started_usec > timeout * USEC_PER_MS) + if(unlikely(!(r->od[0] & RRDR_DIMENSION_NONZERO))) goto cleanup; - // get the baseline, requesting the same number of points as the highlight - stats->db_queries++; - base_rrdr = rrd2rrdr(owa, st,high_points << shifts, - baseline_after, baseline_before, group, - group_time, options, NULL, context_param_list, group_options, - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), tier); - if(!base_rrdr) { - info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name); + if(rrdr_rows(r) < 2) goto cleanup; - } - for(int i = 0; i < storage_tiers ;i++) - stats->db_points_per_tier[i] += base_rrdr->internal.tier_points_read[i]; + *entries = rrdr_rows(r); + ret = onewayalloc_mallocz(owa, sizeof(NETDATA_DOUBLE) * rrdr_rows(r)); - stats->db_points += base_rrdr->internal.db_points_read; - stats->result_points += base_rrdr->internal.result_points_generated; - if(!base_rrdr->d) { - info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name); - goto cleanup; - } - if (base_rrdr->d != high_rrdr->d) { - info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", st->name); - goto cleanup; - } - if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) { - info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", st->name); - goto cleanup; - } - int base_points = rrdr_rows(base_rrdr); + // copy the points of the dimension to a contiguous array + // there is no need to check for empty values, since empty values are already zero + // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43 + memcpy(ret, r->v, rrdr_rows(r) * sizeof(NETDATA_DOUBLE)); - now_usec = now_realtime_usec(); - if(now_usec - started_usec > timeout * USEC_PER_MS) - goto cleanup; +cleanup: + rrdr_free(owa, r); + return ret; +} + +static void rrdset_metric_correlations_ks2( + RRDHOST *host, + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma, + DICTIONARY *results, + time_t baseline_after, time_t baseline_before, + time_t after, time_t before, + size_t points, RRDR_OPTIONS options, + RRDR_GROUPING group_method, const char *group_options, size_t tier, + uint32_t shifts, + WEIGHTS_STATS *stats, bool register_zero + ) { + + options |= RRDR_OPTION_NATURAL_POINTS; - // we need at least 2 points to do the job - if(base_points < 2 || high_points < 2) + ONEWAYALLOC *owa = onewayalloc_create(16 * 1024); + + size_t high_points = 0; + NETDATA_DOUBLE *highlight = rrd2rrdr_ks2( + owa, host, rca, ria, rma, after, before, points, + options, group_method, group_options, tier, stats, &high_points); + + if(!highlight) goto cleanup; - // for each dimension - RRDDIM *d; - int i; - for(i = 0, d = base_rrdr->st->dimensions ; d && i < base_rrdr->d; i++, d = d->next) { + size_t base_points = 0; + NETDATA_DOUBLE *baseline = rrd2rrdr_ks2( + owa, host, rca, ria, rma, baseline_after, baseline_before, high_points << shifts, + options, group_method, group_options, tier, stats, &base_points); + + if(!baseline) + goto cleanup; - // skip the not evaluated ones - if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN)) - continue; + stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1); - examined_dimensions++; + double prob = kstwo(baseline, (int)base_points, highlight, (int)high_points, shifts); + if(!isnan(prob) && !isinf(prob)) { - // skip the dimensions that are just zero for both the baseline and the highlight - if(unlikely(!(base_rrdr->od[i] & RRDR_DIMENSION_NONZERO) && !(high_rrdr->od[i] & RRDR_DIMENSION_NONZERO))) - continue; - - // copy the baseline points of the dimension to a contiguous array - // there is no need to check for empty values, since empty are already zero - NETDATA_DOUBLE baseline[base_points]; - for(int c = 0; c < base_points; c++) - baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ]; - - // copy the highlight points of the dimension to a contiguous array - // there is no need to check for empty values, since empty values are already zero - // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43 - NETDATA_DOUBLE highlight[high_points]; - for(int c = 0; c < high_points; c++) - highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ]; - - stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1); - - double prob = kstwo(baseline, base_points, highlight, high_points, shifts); - if(!isnan(prob) && !isinf(prob)) { - - // these conditions should never happen, but still let's check - if(unlikely(prob < 0.0)) { - error("Metric correlations: kstwo() returned a negative number: %f", prob); - prob = -prob; - } - if(unlikely(prob > 1.0)) { - error("Metric correlations: kstwo() returned a number above 1.0: %f", prob); - prob = 1.0; - } - - // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated - // so we flip the result of kstwo() - register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero); + // these conditions should never happen, but still let's check + if(unlikely(prob < 0.0)) { + error("Metric correlations: kstwo() returned a negative number: %f", prob); + prob = -prob; + } + if(unlikely(prob > 1.0)) { + error("Metric correlations: kstwo() returned a number above 1.0: %f", prob); + prob = 1.0; } + + // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated + // so, we flip the result of kstwo() + register_result(results, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero); } cleanup: - rrdr_free(owa, high_rrdr); - rrdr_free(owa, base_rrdr); onewayalloc_destroy(owa); - return examined_dimensions; } // ---------------------------------------------------------------------------- // VOLUME algorithm functions -static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results, - long long baseline_after, long long baseline_before, - long long after, long long before, - RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options, - int tier, int timeout, - WEIGHTS_STATS *stats, bool register_zero) { +static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats) { + stats->db_queries++; + stats->result_points += qv->result_points; + stats->db_points += qv->points_read; + for(size_t tier = 0; tier < storage_tiers ; tier++) + stats->db_points_per_tier[tier] += qv->storage_points_per_tier[tier]; +} + +static void rrdset_metric_correlations_volume( + RRDHOST *host, + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma, + DICTIONARY *results, + time_t baseline_after, time_t baseline_before, + time_t after, time_t before, + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + size_t tier, + WEIGHTS_STATS *stats, bool register_zero) { options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS; - long group_time = 0; - - int examined_dimensions = 0; - int ret, value_is_null; - usec_t started_usec = now_realtime_usec(); - RRDDIM *d; - for(d = st->dimensions; d ; d = d->next) { - usec_t now_usec = now_realtime_usec(); - if(now_usec - started_usec > timeout * USEC_PER_MS) - return examined_dimensions; + QUERY_VALUE baseline_average = rrdmetric2value(host, rca, ria, rma, baseline_after, baseline_before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS); + merge_query_value_to_stats(&baseline_average, stats); - // we count how many metrics we evaluated - examined_dimensions++; + if(!netdata_double_isnumber(baseline_average.value)) { + // this means no data for the baseline window, but we may have data for the highlighted one - assume zero + baseline_average.value = 0.0; + } - // there is no point to pass a timeout to these queries - // since the query engine checks for a timeout between - // dimensions, and we query a single dimension at a time. - - stats->db_queries++; - NETDATA_DOUBLE baseline_average = NAN; - NETDATA_DOUBLE base_anomaly_rate = 0; - value_is_null = 1; - ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1, - baseline_after, baseline_before, - group, group_options, group_time, options, - NULL, NULL, - &stats->db_points, stats->db_points_per_tier, - &stats->result_points, - &value_is_null, &base_anomaly_rate, 0, tier); - - if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) { - // this means no data for the baseline window, but we may have data for the highlighted one - assume zero - baseline_average = 0.0; - } + QUERY_VALUE highlight_average = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS); + merge_query_value_to_stats(&highlight_average, stats); - stats->db_queries++; - NETDATA_DOUBLE highlight_average = NAN; - NETDATA_DOUBLE high_anomaly_rate = 0; - value_is_null = 1; - ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1, - after, before, - group, group_options, group_time, options, - NULL, NULL, - &stats->db_points, stats->db_points_per_tier, - &stats->result_points, - &value_is_null, &high_anomaly_rate, 0, tier); - - if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) { - // this means no data for the highlighted duration - so skip it - continue; - } + if(!netdata_double_isnumber(highlight_average.value)) + return; - if(baseline_average == highlight_average) { - // they are the same - let's move on - continue; - } + if(baseline_average.value == highlight_average.value) { + // they are the same - let's move on + return; + } - stats->db_queries++; - NETDATA_DOUBLE highlight_countif = NAN; - value_is_null = 1; - - char highlighted_countif_options[50 + 1]; - snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average); - - ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1, - after, before, - RRDR_GROUPING_COUNTIF,highlighted_countif_options, - group_time, options, - NULL, NULL, - &stats->db_points, stats->db_points_per_tier, - &stats->result_points, - &value_is_null, NULL, 0, tier); - - if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) { - info("MC: highlighted countif query failed, but highlighted average worked - strange..."); - continue; - } + char highlight_countif_options[50 + 1]; + snprintfz(highlight_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average.value < baseline_average.value ? "<" : ">", baseline_average.value); + QUERY_VALUE highlight_countif = rrdmetric2value(host, rca, ria, rma, after, before, options, RRDR_GROUPING_COUNTIF, highlight_countif_options, tier, 0, QUERY_SOURCE_API_WEIGHTS); + merge_query_value_to_stats(&highlight_countif, stats); - // this represents the percentage of time - // the highlighted window was above/below the baseline window - // (above or below depending on their averages) - highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0 + if(!netdata_double_isnumber(highlight_countif.value)) { + info("WEIGHTS: highlighted countif query failed, but highlighted average worked - strange..."); + return; + } - RESULT_FLAGS flags; - NETDATA_DOUBLE pcent = NAN; - if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) { - flags = RESULT_IS_BASE_HIGH_RATIO; - pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif; - } - else { - flags = RESULT_IS_PERCENTAGE_OF_TIME; - pcent = highlight_countif; - } + // this represents the percentage of time + // the highlighted window was above/below the baseline window + // (above or below depending on their averages) + highlight_countif.value = highlight_countif.value / 100.0; // countif returns 0 - 100.0 - register_result(results, st, d, pcent, flags, stats, register_zero); + RESULT_FLAGS flags; + NETDATA_DOUBLE pcent = NAN; + if(isgreater(baseline_average.value, 0.0) || isless(baseline_average.value, 0.0)) { + flags = RESULT_IS_BASE_HIGH_RATIO; + pcent = (highlight_average.value - baseline_average.value) / baseline_average.value * highlight_countif.value; + } + else { + flags = RESULT_IS_PERCENTAGE_OF_TIME; + pcent = highlight_countif.value; } - return examined_dimensions; + register_result(results, rca, ria, rma, pcent, flags, stats, register_zero); } // ---------------------------------------------------------------------------- // ANOMALY RATE algorithm functions -static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results, - long long after, long long before, - RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options, - int tier, int timeout, - WEIGHTS_STATS *stats, bool register_zero) { +static void rrdset_weights_anomaly_rate( + RRDHOST *host, + RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma, + DICTIONARY *results, + time_t after, time_t before, + RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options, + size_t tier, + WEIGHTS_STATS *stats, bool register_zero) { options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ANOMALY_BIT | RRDR_OPTION_NATURAL_POINTS; - long group_time = 0; - int examined_dimensions = 0; - int ret, value_is_null; - usec_t started_usec = now_realtime_usec(); - - RRDDIM *d; - for(d = st->dimensions; d ; d = d->next) { - usec_t now_usec = now_realtime_usec(); - if(now_usec - started_usec > timeout * USEC_PER_MS) - return examined_dimensions; + QUERY_VALUE qv = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS); + merge_query_value_to_stats(&qv, stats); - // we count how many metrics we evaluated - examined_dimensions++; - - // there is no point to pass a timeout to these queries - // since the query engine checks for a timeout between - // dimensions, and we query a single dimension at a time. - - stats->db_queries++; - NETDATA_DOUBLE average = NAN; - NETDATA_DOUBLE anomaly_rate = 0; - value_is_null = 1; - ret = rrdset2value_api_v1(st, NULL, &average, d->id, 1, - after, before, - group, group_options, group_time, options, - NULL, NULL, - &stats->db_points, stats->db_points_per_tier, - &stats->result_points, - &value_is_null, &anomaly_rate, 0, tier); - - if(ret == HTTP_RESP_OK || !value_is_null || netdata_double_isnumber(average)) - register_result(results, st, d, average, 0, stats, register_zero); - } - - return examined_dimensions; + if(netdata_double_isnumber(qv.value)) + register_result(results, rca, ria, rma, qv.value, 0, stats, register_zero); } // ---------------------------------------------------------------------------- @@ -853,7 +742,7 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) { struct register_result *t; // count the dimensions - size_t dimensions = dictionary_stats_entries(results); + size_t dimensions = dictionary_entries(results); if(!dimensions) return 0; if(stats->max_base_high_ratio == 0.0) @@ -903,15 +792,17 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) { // ---------------------------------------------------------------------------- // The main function -int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, - RRDR_GROUPING group, const char *group_options, - long long baseline_after, long long baseline_before, - long long after, long long before, - long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout) { +int web_api_v1_weights( + RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, + RRDR_GROUPING group, const char *group_options, + time_t baseline_after, time_t baseline_before, + time_t after, time_t before, + size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout) { + WEIGHTS_STATS stats = {}; DICTIONARY *results = register_result_init(); - DICTIONARY *charts = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);; + DICTIONARY *metrics = NULL; char *error = NULL; int resp = HTTP_RESP_OK; @@ -1000,20 +891,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS baseline_after = baseline_before - (high_delta << shifts); } - // dont lock here and wait for results - // get the charts and run mc after - RRDSET *st; - rrdhost_rdlock(host); - rrdset_foreach_read(st, host) { - if (rrdset_is_available_for_viewers(st)) { - if(!contexts || simple_pattern_matches(contexts, st->context)) - dictionary_set(charts, st->name, NULL, 0); - } - } - rrdhost_unlock(host); - size_t examined_dimensions = 0; - void *ptr; bool register_zero = true; if(options & RRDR_OPTION_NONZERO) { @@ -1021,8 +899,11 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS options &= ~RRDR_OPTION_NONZERO; } - // for every chart in the dictionary - dfe_start_read(charts, ptr) { + metrics = rrdcontext_all_metrics_to_dict(host, contexts); + struct metric_entry *me; + + // for every metric_entry in the dictionary + dfe_start_read(metrics, me) { usec_t now_usec = now_realtime_usec(); if(now_usec - started_usec > timeout_usec) { error = "timed out"; @@ -1030,46 +911,48 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS goto cleanup; } - st = rrdset_find_byname(host, ptr_name); - if(!st) continue; - - rrdset_rdlock(st); + examined_dimensions++; switch(method) { case WEIGHTS_METHOD_ANOMALY_RATE: options |= RRDR_OPTION_ANOMALY_BIT; - points = 1; - examined_dimensions += rrdset_weights_anomaly_rate(st, results, - after, before, - options, group, group_options, tier, - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), - &stats, register_zero); + rrdset_weights_anomaly_rate( + host, + me->rca, me->ria, me->rma, + results, + after, before, + options, group, group_options, tier, + &stats, register_zero + ); break; case WEIGHTS_METHOD_MC_VOLUME: - points = 1; - examined_dimensions += rrdset_metric_correlations_volume(st, results, - baseline_after, baseline_before, - after, before, - options, group, group_options, tier, - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), - &stats, register_zero); + rrdset_metric_correlations_volume( + host, + me->rca, me->ria, me->rma, + results, + baseline_after, baseline_before, + after, before, + options, group, group_options, tier, + &stats, register_zero + ); break; default: case WEIGHTS_METHOD_MC_KS2: - examined_dimensions += rrdset_metric_correlations_ks2(st, results, - baseline_after, baseline_before, - after, before, - points, options, group, group_options, tier, shifts, - (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), - &stats, register_zero); + rrdset_metric_correlations_ks2( + host, + me->rca, me->ria, me->rma, + results, + baseline_after, baseline_before, + after, before, points, + options, group, group_options, tier, shifts, + &stats, register_zero + ); break; } - - rrdset_unlock(st); } - dfe_done(ptr); + dfe_done(me); if(!register_zero) options |= RRDR_OPTION_NONZERO; @@ -1085,22 +968,26 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS size_t added_dimensions = 0; switch(format) { case WEIGHTS_FORMAT_CHARTS: - added_dimensions = registered_results_to_json_charts(results, wb, - after, before, - baseline_after, baseline_before, - points, method, group, options, shifts, - examined_dimensions, - ended_usec - started_usec, &stats); + added_dimensions = + registered_results_to_json_charts( + results, wb, + after, before, + baseline_after, baseline_before, + points, method, group, options, shifts, + examined_dimensions, + ended_usec - started_usec, &stats); break; default: case WEIGHTS_FORMAT_CONTEXTS: - added_dimensions = registered_results_to_json_contexts(results, wb, - after, before, - baseline_after, baseline_before, - points, method, group, options, shifts, - examined_dimensions, - ended_usec - started_usec, &stats); + added_dimensions = + registered_results_to_json_contexts( + results, wb, + after, before, + baseline_after, baseline_before, + points, method, group, options, shifts, + examined_dimensions, + ended_usec - started_usec, &stats); break; } @@ -1110,7 +997,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS } cleanup: - if(charts) dictionary_destroy(charts); + if(metrics) dictionary_destroy(metrics); if(results) register_result_destroy(results); if(error) { diff --git a/web/api/queries/weights.h b/web/api/queries/weights.h index f88a134f2..50d8634ef 100644 --- a/web/api/queries/weights.h +++ b/web/api/queries/weights.h @@ -20,14 +20,14 @@ extern int enable_metric_correlations; extern int metric_correlations_version; extern WEIGHTS_METHOD default_metric_correlations_method; -extern int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, +int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, RRDR_GROUPING group, const char *group_options, - long long baseline_after, long long baseline_before, - long long after, long long before, - long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout); + time_t baseline_after, time_t baseline_before, + time_t after, time_t before, + size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout); -extern WEIGHTS_METHOD weights_string_to_method(const char *method); -extern const char *weights_method_to_string(WEIGHTS_METHOD method); -extern int mc_unittest(void); +WEIGHTS_METHOD weights_string_to_method(const char *method); +const char *weights_method_to_string(WEIGHTS_METHOD method); +int mc_unittest(void); #endif //NETDATA_API_WEIGHTS_H |