summaryrefslogtreecommitdiffstats
path: root/web/api/queries
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2022-11-30 18:47:00 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2022-11-30 18:47:00 +0000
commit03bf87dcb06f7021bfb2df2fa8691593c6148aff (patch)
treee16b06711a2ed77cafb4b7754be0220c3d14a9d7 /web/api/queries
parentAdding upstream version 1.36.1. (diff)
downloadnetdata-03bf87dcb06f7021bfb2df2fa8691593c6148aff.tar.xz
netdata-03bf87dcb06f7021bfb2df2fa8691593c6148aff.zip
Adding upstream version 1.37.0.upstream/1.37.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'web/api/queries')
-rw-r--r--web/api/queries/average/average.h10
-rw-r--r--web/api/queries/countif/countif.h10
-rw-r--r--web/api/queries/des/des.h12
-rw-r--r--web/api/queries/incremental_sum/incremental_sum.h10
-rw-r--r--web/api/queries/max/max.h10
-rw-r--r--web/api/queries/median/median.h26
-rw-r--r--web/api/queries/min/min.h10
-rw-r--r--web/api/queries/percentile/percentile.h26
-rw-r--r--web/api/queries/query.c1012
-rw-r--r--web/api/queries/query.h8
-rw-r--r--web/api/queries/rrdr.c60
-rw-r--r--web/api/queries/rrdr.h111
-rw-r--r--web/api/queries/ses/ses.h12
-rw-r--r--web/api/queries/stddev/stddev.h16
-rw-r--r--web/api/queries/sum/sum.h10
-rw-r--r--web/api/queries/trimmed_mean/trimmed_mean.h24
-rw-r--r--web/api/queries/weights.c763
-rw-r--r--web/api/queries/weights.h14
18 files changed, 1031 insertions, 1113 deletions
diff --git a/web/api/queries/average/average.h b/web/api/queries/average/average.h
index 55c51722c..b31966886 100644
--- a/web/api/queries/average/average.h
+++ b/web/api/queries/average/average.h
@@ -6,10 +6,10 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_average(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_average(RRDR *r);
-extern void grouping_free_average(RRDR *r);
-extern void grouping_add_average(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_average(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_average(RRDR *r);
+void grouping_free_average(RRDR *r);
+void grouping_add_average(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_AVERAGE_H
diff --git a/web/api/queries/countif/countif.h b/web/api/queries/countif/countif.h
index 0c7d2d7d1..dfe805658 100644
--- a/web/api/queries/countif/countif.h
+++ b/web/api/queries/countif/countif.h
@@ -6,10 +6,10 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_countif(RRDR *r);
-extern void grouping_free_countif(RRDR *r);
-extern void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_countif(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_countif(RRDR *r);
+void grouping_free_countif(RRDR *r);
+void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_COUNTIF_H
diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h
index 8906d14eb..05fa01b34 100644
--- a/web/api/queries/des/des.h
+++ b/web/api/queries/des/des.h
@@ -6,12 +6,12 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_init_des(void);
+void grouping_init_des(void);
-extern void grouping_create_des(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_des(RRDR *r);
-extern void grouping_free_des(RRDR *r);
-extern void grouping_add_des(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_des(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_des(RRDR *r);
+void grouping_free_des(RRDR *r);
+void grouping_add_des(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_DES_H
diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h
index 6d908cef6..c24507fcf 100644
--- a/web/api/queries/incremental_sum/incremental_sum.h
+++ b/web/api/queries/incremental_sum/incremental_sum.h
@@ -6,10 +6,10 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_incremental_sum(RRDR *r);
-extern void grouping_free_incremental_sum(RRDR *r);
-extern void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_incremental_sum(RRDR *r);
+void grouping_free_incremental_sum(RRDR *r);
+void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H
diff --git a/web/api/queries/max/max.h b/web/api/queries/max/max.h
index 28913686b..e2427d26d 100644
--- a/web/api/queries/max/max.h
+++ b/web/api/queries/max/max.h
@@ -6,10 +6,10 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_max(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_max(RRDR *r);
-extern void grouping_free_max(RRDR *r);
-extern void grouping_add_max(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_max(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_max(RRDR *r);
+void grouping_free_max(RRDR *r);
+void grouping_add_max(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_MAX_H
diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h
index dd1b3de61..9fc159db4 100644
--- a/web/api/queries/median/median.h
+++ b/web/api/queries/median/median.h
@@ -6,18 +6,18 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_median(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median1(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median2(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median3(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median5(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median10(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median15(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median20(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median25(RRDR *r, const char *options);
-extern void grouping_reset_median(RRDR *r);
-extern void grouping_free_median(RRDR *r);
-extern void grouping_add_median(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_median(RRDR *r, const char *options);
+void grouping_create_trimmed_median1(RRDR *r, const char *options);
+void grouping_create_trimmed_median2(RRDR *r, const char *options);
+void grouping_create_trimmed_median3(RRDR *r, const char *options);
+void grouping_create_trimmed_median5(RRDR *r, const char *options);
+void grouping_create_trimmed_median10(RRDR *r, const char *options);
+void grouping_create_trimmed_median15(RRDR *r, const char *options);
+void grouping_create_trimmed_median20(RRDR *r, const char *options);
+void grouping_create_trimmed_median25(RRDR *r, const char *options);
+void grouping_reset_median(RRDR *r);
+void grouping_free_median(RRDR *r);
+void grouping_add_median(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_MEDIAN_H
diff --git a/web/api/queries/min/min.h b/web/api/queries/min/min.h
index b8627f667..dcdfe252f 100644
--- a/web/api/queries/min/min.h
+++ b/web/api/queries/min/min.h
@@ -6,10 +6,10 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_min(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_min(RRDR *r);
-extern void grouping_free_min(RRDR *r);
-extern void grouping_add_min(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_min(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_min(RRDR *r);
+void grouping_free_min(RRDR *r);
+void grouping_add_min(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_MIN_H
diff --git a/web/api/queries/percentile/percentile.h b/web/api/queries/percentile/percentile.h
index 709717ebd..65e335c11 100644
--- a/web/api/queries/percentile/percentile.h
+++ b/web/api/queries/percentile/percentile.h
@@ -6,18 +6,18 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_percentile25(RRDR *r, const char *options);
-extern void grouping_create_percentile50(RRDR *r, const char *options);
-extern void grouping_create_percentile75(RRDR *r, const char *options);
-extern void grouping_create_percentile80(RRDR *r, const char *options);
-extern void grouping_create_percentile90(RRDR *r, const char *options);
-extern void grouping_create_percentile95(RRDR *r, const char *options);
-extern void grouping_create_percentile97(RRDR *r, const char *options);
-extern void grouping_create_percentile98(RRDR *r, const char *options);
-extern void grouping_create_percentile99(RRDR *r, const char *options );
-extern void grouping_reset_percentile(RRDR *r);
-extern void grouping_free_percentile(RRDR *r);
-extern void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_percentile25(RRDR *r, const char *options);
+void grouping_create_percentile50(RRDR *r, const char *options);
+void grouping_create_percentile75(RRDR *r, const char *options);
+void grouping_create_percentile80(RRDR *r, const char *options);
+void grouping_create_percentile90(RRDR *r, const char *options);
+void grouping_create_percentile95(RRDR *r, const char *options);
+void grouping_create_percentile97(RRDR *r, const char *options);
+void grouping_create_percentile98(RRDR *r, const char *options);
+void grouping_create_percentile99(RRDR *r, const char *options );
+void grouping_reset_percentile(RRDR *r);
+void grouping_free_percentile(RRDR *r);
+void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_PERCENTILE_H
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index d776f6d11..ccd195135 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -658,71 +658,6 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) {
}
// ----------------------------------------------------------------------------
-
-static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,
- struct context_param *context_param_list)
-{
- RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
- int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
-
- if (should_lock)
- rrdset_check_rdlock(r->st);
-
- if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
-
- int match_ids = 0, match_names = 0;
-
- if(unlikely(options & RRDR_OPTION_MATCH_IDS))
- match_ids = 1;
- if(unlikely(options & RRDR_OPTION_MATCH_NAMES))
- match_names = 1;
-
- if(likely(!match_ids && !match_names))
- match_ids = match_names = 1;
-
- SIMPLE_PATTERN *pattern = simple_pattern_create(dims, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
-
- RRDDIM *d;
- long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
- for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) {
- if( (match_ids && simple_pattern_matches(pattern, d->id))
- || (match_names && simple_pattern_matches(pattern, d->name))
- ) {
- r->od[c] |= RRDR_DIMENSION_SELECTED;
- if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) r->od[c] &= ~RRDR_DIMENSION_HIDDEN;
- dims_selected++;
-
- // since the user needs this dimension
- // make it appear as NONZERO, to return it
- // even if the dimension has only zeros
- // unless option non_zero is set
- if(unlikely(!(options & RRDR_OPTION_NONZERO)))
- r->od[c] |= RRDR_DIMENSION_NONZERO;
-
- // count the visible dimensions
- if(likely(r->od[c] & RRDR_DIMENSION_NONZERO))
- dims_not_hidden_not_zero++;
- }
- else {
- r->od[c] |= RRDR_DIMENSION_HIDDEN;
- if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;
- }
- }
- simple_pattern_free(pattern);
-
- // check if all dimensions are hidden
- if(unlikely(!dims_not_hidden_not_zero && dims_selected)) {
- // there are a few selected dimensions,
- // but they are all zero
- // enable the selected ones
- // to avoid returning an empty chart
- for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next)
- if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED))
- r->od[c] |= RRDR_DIMENSION_NONZERO;
- }
-}
-
-// ----------------------------------------------------------------------------
// helpers to find our way in RRDR
static inline RRDR_VALUE_FLAGS *UNUSED_FUNCTION(rrdr_line_options)(RRDR *r, long rrdr_line) {
@@ -736,13 +671,13 @@ static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rr
static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) {
rrdr_line++;
- internal_error(rrdr_line >= r->n,
- "QUERY: requested to step above RRDR size for chart '%s'",
- r->st->name);
+ internal_error(rrdr_line >= (long)r->n,
+ "QUERY: requested to step above RRDR size for query '%s'",
+ r->internal.qt->id);
internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t,
- "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'",
- (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name);
+ "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of query '%s'",
+ (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->internal.qt->id);
// save the time
r->t[rrdr_line] = t;
@@ -758,68 +693,137 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) {
// ----------------------------------------------------------------------------
// tier management
-static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, time_t before_wanted, long points_wanted) {
+static bool query_metric_is_valid_tier(QUERY_METRIC *qm, size_t tier) {
+ if(!qm->tiers[tier].db_metric_handle || !qm->tiers[tier].db_first_time_t || !qm->tiers[tier].db_last_time_t || !qm->tiers[tier].db_update_every)
+ return false;
+
+ return true;
+}
+
+static size_t query_metric_first_working_tier(QUERY_METRIC *qm) {
+ for(size_t tier = 0; tier < storage_tiers ; tier++) {
+
+ // find the db time-range for this tier for all metrics
+ STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
+ time_t first_t = qm->tiers[tier].db_first_time_t;
+ time_t last_t = qm->tiers[tier].db_last_time_t;
+ time_t update_every = qm->tiers[tier].db_update_every;
+
+ if(!db_metric_handle || !first_t || !last_t || !update_every)
+ continue;
+
+ return tier;
+ }
+
+ return 0;
+}
+
+static long query_plan_points_coverage_weight(time_t db_first_t, time_t db_last_t, time_t db_update_every, time_t after_wanted, time_t before_wanted, size_t points_wanted, size_t tier __maybe_unused) {
+ if(db_first_t == 0 || db_last_t == 0 || db_update_every == 0)
+ return -LONG_MAX;
+
+ time_t common_first_t = MAX(db_first_t, after_wanted);
+ time_t common_last_t = MIN(db_last_t, before_wanted);
+
+ long time_coverage = (common_last_t - common_first_t) * 1000000 / (before_wanted - after_wanted);
+ size_t points_wanted_in_coverage = points_wanted * time_coverage / 1000000;
+
+ long points_available = (common_last_t - common_first_t) / db_update_every;
+ long points_delta = (long)(points_available - points_wanted_in_coverage);
+ long points_coverage = (points_delta < 0) ? (long)(points_available * time_coverage / points_wanted_in_coverage) : time_coverage;
+
+ // a way to benefit higher tiers
+ // points_coverage += (long)tier * 10000;
+
+ if(points_available <= 0)
+ return -LONG_MAX;
+
+ return points_coverage;
+}
+
+static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
if(unlikely(storage_tiers < 2))
return 0;
- if(unlikely(after_wanted == before_wanted || points_wanted <= 0 || !rd || !rd->rrdset)) {
+ if(unlikely(after_wanted == before_wanted || points_wanted <= 0))
+ return query_metric_first_working_tier(qm);
- if(!rd)
- internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation");
- else
- internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation",
- (rd->rrdset)?rd->rrdset->name:"unknown", rd->name);
+ long weight[storage_tiers];
- return 0;
+ for(size_t tier = 0; tier < storage_tiers ; tier++) {
+
+ // find the db time-range for this tier for all metrics
+ STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
+ time_t first_t = qm->tiers[tier].db_first_time_t;
+ time_t last_t = qm->tiers[tier].db_last_time_t;
+ time_t update_every = qm->tiers[tier].db_update_every;
+
+ if(!db_metric_handle || !first_t || !last_t || !update_every) {
+ weight[tier] = -LONG_MAX;
+ continue;
+ }
+
+ weight[tier] = query_plan_points_coverage_weight(first_t, last_t, update_every, after_wanted, before_wanted, points_wanted, tier);
}
- //BUFFER *wb = buffer_create(1000);
- //buffer_sprintf(wb, "Best tier for chart '%s', dim '%s', from %ld to %ld (dur %ld, every %d), points %ld",
- // rd->rrdset->name, rd->name, after_wanted, before_wanted, before_wanted - after_wanted, rd->update_every, points_wanted);
+ size_t best_tier = 0;
+ for(size_t tier = 1; tier < storage_tiers ; tier++) {
+ if(weight[tier] >= weight[best_tier])
+ best_tier = tier;
+ }
+
+ return best_tier;
+}
+
+static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
+ if(unlikely(storage_tiers < 2))
+ return 0;
+
+ if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) {
+ internal_error(true, "QUERY: '%s' has invalid params to tier calculation", qt->id);
+ return 0;
+ }
long weight[storage_tiers];
- for(int tier = 0; tier < storage_tiers ; tier++) {
- if(unlikely(!rd->tiers[tier])) {
- internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized",
- tier, rd->rrdset->name, rd->name);
- // buffer_free(wb);
- return 0;
- }
+ for(size_t tier = 0; tier < storage_tiers ; tier++) {
- time_t first_t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
- time_t last_t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
+ time_t common_first_t = 0;
+ time_t common_last_t = 0;
+ time_t common_update_every = 0;
- time_t common_after = MAX(first_t, after_wanted);
- time_t common_before = MIN(last_t, before_wanted);
+ // find the db time-range for this tier for all metrics
+ for(size_t i = 0, used = qt->query.used; i < used ; i++) {
+ QUERY_METRIC *qm = &qt->query.array[i];
- long time_coverage = (common_before - common_after) * 1000 / (before_wanted - after_wanted);
- if(time_coverage < 0) time_coverage = 0;
+ time_t first_t = qm->tiers[tier].db_first_time_t;
+ time_t last_t = qm->tiers[tier].db_last_time_t;
+ time_t update_every = qm->tiers[tier].db_update_every;
- int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every;
- if(unlikely(update_every == 0)) {
- internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d",
- tier, rd->rrdset->name, rd->name, rd->tiers[tier]->tier_grouping, rd->update_every);
- // buffer_free(wb);
- return 0;
- }
+ if(!first_t || !last_t || !update_every)
+ continue;
- long points_available = (before_wanted - after_wanted) / update_every;
- long points_delta = points_available - points_wanted;
- long points_coverage = (points_delta < 0) ? points_available * 1000 / points_wanted: 1000;
+ if(!common_first_t)
+ common_first_t = first_t;
+ else
+ common_first_t = MIN(first_t, common_first_t);
- if(points_available <= 0)
- weight[tier] = -LONG_MAX;
- else
- weight[tier] = points_coverage;
+ if(!common_last_t)
+ common_last_t = last_t;
+ else
+ common_last_t = MAX(last_t, common_last_t);
- // buffer_sprintf(wb, ": tier %d, first %ld, last %ld (dur %ld, tg %d, every %d), points %ld, tcoverage %ld, pcoverage %ld, weight %ld",
- // tier, first_t, last_t, last_t - first_t, rd->tiers[tier]->tier_grouping, update_every,
- // points_available, time_coverage, points_coverage, weight[tier]);
+ if(!common_update_every)
+ common_update_every = update_every;
+ else
+ common_update_every = MIN(update_every, common_update_every);
+ }
+
+ weight[tier] = query_plan_points_coverage_weight(common_first_t, common_last_t, common_update_every, after_wanted, before_wanted, points_wanted, tier);
}
- int best_tier = 0;
- for(int tier = 1; tier < storage_tiers ; tier++) {
+ size_t best_tier = 0;
+ for(size_t tier = 1; tier < storage_tiers ; tier++) {
if(weight[tier] >= weight[best_tier])
best_tier = tier;
}
@@ -827,47 +831,30 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted,
if(weight[best_tier] == -LONG_MAX)
best_tier = 0;
- //buffer_sprintf(wb, ": final best tier %d", best_tier);
- //internal_error(true, "%s", buffer_tostring(wb));
- //buffer_free(wb);
-
return best_tier;
}
-static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) {
- int ret = st->update_every;
-
- if(unlikely(!st->dimensions))
- return ret;
-
- rrdset_rdlock(st);
- int best_tier;
-
- if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers)
+static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted, RRDR_OPTIONS options, size_t tier) {
+ size_t best_tier;
+ if((options & RRDR_OPTION_SELECTED_TIER) && tier < storage_tiers)
best_tier = tier;
else
- best_tier = rrddim_find_best_tier_for_timeframe(st->dimensions, after_wanted, before_wanted, points_wanted);
+ best_tier = rrddim_find_best_tier_for_timeframe(qt, after_wanted, before_wanted, points_wanted);
- if(!st->dimensions->tiers[best_tier]) {
- internal_error(
- true,
- "QUERY: tier %d on chart '%s', is not initialized", best_tier, st->name);
- }
- else {
- ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every;
- if(unlikely(!ret)) {
- internal_error(
- true,
- "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d",
- st->name, st->dimensions->tiers[best_tier]->tier_grouping, st->update_every);
-
- ret = st->update_every;
- }
- }
+ // find the db minimum update every for this tier for all metrics
+ time_t common_update_every = default_rrd_update_every;
+ for(size_t i = 0, used = qt->query.used; i < used ; i++) {
+ QUERY_METRIC *qm = &qt->query.array[i];
- rrdset_unlock(st);
+ time_t update_every = qm->tiers[best_tier].db_update_every;
- return ret;
+ if(!i)
+ common_update_every = update_every;
+ else
+ common_update_every = MIN(update_every, common_update_every);
+ }
+
+ return common_update_every;
}
// ----------------------------------------------------------------------------
@@ -915,7 +902,7 @@ typedef struct query_plan {
typedef struct query_engine_ops {
// configuration
RRDR *r;
- RRDDIM *rd;
+ QUERY_METRIC *qm;
time_t view_update_every;
time_t query_granularity;
TIER_QUERY_FETCH tier_query_fetch;
@@ -927,11 +914,11 @@ typedef struct query_engine_ops {
// storage queries
size_t tier;
- struct rrddim_tier *tier_ptr;
- struct rrddim_query_handle handle;
- STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
- int (*is_finished)(struct rrddim_query_handle *handle);
- void (*finalize)(struct rrddim_query_handle *handle);
+ struct query_metric_tier *tier_ptr;
+ struct storage_engine_query_handle handle;
+ STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
+ int (*is_finished)(struct storage_engine_query_handle *handle);
+ void (*finalize)(struct storage_engine_query_handle *handle);
// aggregating points over time
void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
@@ -963,11 +950,11 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
after = overwrite_after;
ops->tier = ops->plan.data[plan_id].tier;
- ops->tier_ptr = ops->rd->tiers[ops->tier];
- ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch);
- ops->next_metric = ops->tier_ptr->query_ops.next_metric;
- ops->is_finished = ops->tier_ptr->query_ops.is_finished;
- ops->finalize = ops->tier_ptr->query_ops.finalize;
+ ops->tier_ptr = &ops->qm->tiers[ops->tier];
+ ops->tier_ptr->eng->api.query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before);
+ ops->next_metric = ops->tier_ptr->eng->api.query_ops.next_metric;
+ ops->is_finished = ops->tier_ptr->eng->api.query_ops.is_finished;
+ ops->finalize = ops->tier_ptr->eng->api.query_ops.finalize;
ops->current_plan = plan_id;
ops->current_plan_expire_time = ops->plan.data[plan_id].before;
}
@@ -976,26 +963,38 @@ static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t las
internal_error(now < ops->current_plan_expire_time && now < ops->plan.data[ops->current_plan].before,
"QUERY: switching query plan too early!");
+ size_t old_plan = ops->current_plan;
+
time_t next_plan_before_time;
do {
ops->current_plan++;
if (ops->current_plan >= ops->plan.entries) {
- ops->current_plan = ops->plan.entries - 1;
+ ops->current_plan = old_plan;
+ ops->current_plan_expire_time = ops->r->internal.qt->window.before;
+ // let the query run with current plan
+ // we will not switch it
return;
}
next_plan_before_time = ops->plan.data[ops->current_plan].before;
} while(now >= next_plan_before_time || last_point_end_time >= next_plan_before_time);
+ if(!query_metric_is_valid_tier(ops->qm, ops->plan.data[ops->current_plan].tier)) {
+ ops->current_plan = old_plan;
+ ops->current_plan_expire_time = ops->r->internal.qt->window.before;
+ return;
+ }
+
if(ops->finalize) {
ops->finalize(&ops->handle);
ops->finalize = NULL;
+ ops->is_finished = NULL;
}
- query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
-
// internal_error(true, "QUERY: switched plan to %zu (all is %zu), previous expiration was %ld, this starts at %ld, now is %ld, last_point_end_time %ld", ops->current_plan, ops->plan.entries, ops->plan.data[ops->current_plan-1].before, ops->plan.data[ops->current_plan].after, now, last_point_end_time);
+
+ query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
}
static int compare_query_plan_entries_on_start_time(const void *a, const void *b) {
@@ -1004,21 +1003,20 @@ static int compare_query_plan_entries_on_start_time(const void *a, const void *b
return (p1->after < p2->after)?-1:1;
}
-static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, long points_wanted) {
- RRDDIM *rd = ops->rd;
-
+static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
//BUFFER *wb = buffer_create(1000);
//buffer_sprintf(wb, "QUERY PLAN for chart '%s' dimension '%s', from %ld to %ld:", rd->rrdset->name, rd->name, after_wanted, before_wanted);
// put our selected tier as the first plan
size_t selected_tier;
- if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER && ops->r->internal.query_tier >= 0 && ops->r->internal.query_tier < storage_tiers) {
- selected_tier = ops->r->internal.query_tier;
+ if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER
+ && ops->r->internal.qt->window.tier < storage_tiers
+ && query_metric_is_valid_tier(ops->qm, ops->r->internal.qt->window.tier)) {
+ selected_tier = ops->r->internal.qt->window.tier;
}
else {
-
- selected_tier = rrddim_find_best_tier_for_timeframe(rd, after_wanted, before_wanted, points_wanted);
+ selected_tier = query_metric_best_tier_for_timeframe(ops->qm, after_wanted, before_wanted, points_wanted);
if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)
ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER;
@@ -1026,8 +1024,8 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
ops->plan.entries = 1;
ops->plan.data[0].tier = selected_tier;
- ops->plan.data[0].after = rd->tiers[selected_tier]->query_ops.oldest_time(rd->tiers[selected_tier]->db_metric_handle);
- ops->plan.data[0].before = rd->tiers[selected_tier]->query_ops.latest_time(rd->tiers[selected_tier]->db_metric_handle);
+ ops->plan.data[0].after = ops->qm->tiers[selected_tier].db_first_time_t;
+ ops->plan.data[0].before = ops->qm->tiers[selected_tier].db_last_time_t;
if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) {
// the selected tier
@@ -1039,9 +1037,12 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
// check if our selected tier can start the query
if (selected_tier_first_time_t > after_wanted) {
// we need some help from other tiers
- for (int tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
+ for (size_t tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
+ if(!query_metric_is_valid_tier(ops->qm, tr))
+ continue;
+
// find the first time of this tier
- time_t first_time_t = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
+ time_t first_time_t = ops->qm->tiers[tr].db_first_time_t;
//buffer_sprintf(wb, ": EVAL AFTER tier %d, %ld", tier, first_time_t);
@@ -1067,8 +1068,11 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
if (selected_tier_last_time_t < before_wanted) {
// we need some help from other tiers
for (int tr = (int)selected_tier - 1; tr >= 0; tr--) {
+ if(!query_metric_is_valid_tier(ops->qm, tr))
+ continue;
+
// find the last time of this tier
- time_t last_time_t = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
+ time_t last_time_t = ops->qm->tiers[tr].db_last_time_t;
//buffer_sprintf(wb, ": EVAL BEFORE tier %d, %ld", tier, last_time_t);
@@ -1096,8 +1100,11 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
qsort(&ops->plan.data, ops->plan.entries, sizeof(QUERY_PLAN_ENTRY), compare_query_plan_entries_on_start_time);
// make sure it has the whole timeframe we need
- ops->plan.data[0].after = after_wanted;
- ops->plan.data[ops->plan.entries - 1].before = before_wanted;
+ if(ops->plan.data[0].after < after_wanted)
+ ops->plan.data[0].after = after_wanted;
+
+ if(ops->plan.data[ops->plan.entries - 1].before > before_wanted)
+ ops->plan.data[ops->plan.entries - 1].before = before_wanted;
//buffer_sprintf(wb, ": FINAL STEPS %zu", ops->plan.entries);
@@ -1106,7 +1113,12 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
//internal_error(true, "%s", buffer_tostring(wb));
+ if(!query_metric_is_valid_tier(ops->qm, ops->plan.data[0].tier))
+ return false;
+
query_planer_activate_plan(ops, 0, 0);
+
+ return true;
}
@@ -1146,14 +1158,17 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
(ops).group_anomaly_rate += (point).anomaly; \
} while(0)
-static inline void rrd2rrdr_do_dimension(
- RRDR *r
- , long points_wanted
- , RRDDIM *rd
- , long dim_id_in_rrdr
- , time_t after_wanted
- , time_t before_wanted
-){
+static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) {
+ QUERY_TARGET *qt = r->internal.qt;
+ QUERY_METRIC *qm = &qt->query.array[dim_id_in_rrdr];
+ size_t points_wanted = qt->window.points;
+ time_t after_wanted = qt->window.after;
+ time_t before_wanted = qt->window.before;
+
+// bool debug_this = false;
+// if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
+// debug_this = true;
+
time_t max_date = 0,
min_date = 0;
@@ -1161,19 +1176,20 @@ static inline void rrd2rrdr_do_dimension(
QUERY_ENGINE_OPS ops = {
.r = r,
- .rd = rd,
+ .qm = qm,
.grouping_add = r->internal.grouping_add,
.grouping_flush = r->internal.grouping_flush,
.tier_query_fetch = r->internal.tier_query_fetch,
.view_update_every = r->update_every,
- .query_granularity = r->update_every / r->group,
+ .query_granularity = (time_t)(r->update_every / r->group),
.group_value_flags = RRDR_VALUE_NOTHING
};
long rrdr_line = -1;
bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
- query_plan(&ops, after_wanted, before_wanted, points_wanted);
+ if(!query_plan(&ops, after_wanted, before_wanted, points_wanted))
+ return;
NETDATA_DOUBLE min = r->min, max = r->max;
@@ -1184,15 +1200,18 @@ static inline void rrd2rrdr_do_dimension(
time_t now_start_time = after_wanted - ops.query_granularity;
time_t now_end_time = after_wanted + ops.view_update_every - ops.query_granularity;
+ size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch;
+
// The main loop, based on the query granularity we need
- for( ; (long)points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {
+ for( ; points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {
- if(query_plan_should_switch_plan(ops, now_end_time))
+ if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) {
query_planer_next_plan(&ops, now_end_time, new_point.end_time);
+ db_points_read_since_plan_switch = 0;
+ }
// read all the points of the db, prior to the time we need (now_end_time)
-
size_t count_same_end_time = 0;
while(count_same_end_time < 100) {
if(likely(count_same_end_time == 0)) {
@@ -1208,11 +1227,15 @@ static inline void rrd2rrdr_do_dimension(
new_point = QUERY_POINT_EMPTY;
new_point.start_time = last1_point.end_time;
new_point.end_time = now_end_time;
+//
+// if(debug_this) info("QUERY: is finished() returned true");
+//
break;
}
// fetch the new point
{
+ db_points_read_since_plan_switch++;
STORAGE_POINT sp = ops.next_metric(&ops.handle);
ops.db_points_read_per_tier[ops.tier]++;
@@ -1223,6 +1246,10 @@ static inline void rrd2rrdr_do_dimension(
new_point.anomaly = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0;
query_point_set_id(new_point, ops.db_total_points_read);
+// if(debug_this)
+// info("QUERY: got point %zu, from time %ld to %ld // now from %ld to %ld // query from %ld to %ld",
+// new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
+//
// set the right value to the point we got
if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
@@ -1258,17 +1285,18 @@ static inline void rrd2rrdr_do_dimension(
// check if the db is giving us zero duration points
if(unlikely(new_point.start_time == new_point.end_time)) {
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",
- rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time);
+ internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned point %zu start time %ld, end time %ld, that are both equal",
+ qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time);
- new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every);
+ new_point.start_time = new_point.end_time - ops.tier_ptr->db_update_every;
}
// check if the db is advancing the query
if(unlikely(new_point.end_time <= last1_point.end_time)) {
- internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld",
- rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time,
- last1_point.id, last1_point.end_time, now_start_time, now_end_time);
+ internal_error(db_points_read_since_plan_switch > 1,
+ "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld to %ld, before the last point %zu from %ld to %ld, now is %ld to %ld",
+ qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time,
+ last1_point.id, last1_point.start_time, last1_point.end_time, now_start_time, now_end_time);
count_same_end_time++;
continue;
@@ -1294,8 +1322,8 @@ static inline void rrd2rrdr_do_dimension(
// we only log if this is not point 1
internal_error(new_point.end_time < after_wanted && new_point.id > 1,
- "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",
- rd->rrdset->name, rd->name,
+ "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",
+ qt->id, string2str(qm->dimension.id),
new_point.id, new_point.start_time, new_point.end_time,
now_start_time, now_end_time,
after_wanted, before_wanted);
@@ -1311,8 +1339,8 @@ static inline void rrd2rrdr_do_dimension(
if(unlikely(count_same_end_time)) {
internal_error(true,
- "QUERY: the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",
- last1_point.end_time, count_same_end_time);
+ "QUERY: '%s', dimension '%s', the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",
+ qt->id, string2str(qm->dimension.id), last1_point.end_time, count_same_end_time);
if(unlikely(new_point.end_time <= last1_point.end_time))
new_point.end_time = now_end_time;
@@ -1323,7 +1351,7 @@ static inline void rrd2rrdr_do_dimension(
// we select the one to use based on their timestamps
size_t iterations = 0;
- for ( ; now_end_time <= new_point.end_time && (long)points_added < points_wanted ;
+ for ( ; now_end_time <= new_point.end_time && points_added < points_wanted ;
now_end_time += ops.view_update_every, iterations++) {
// now_start_time is wrong in this loop
@@ -1336,22 +1364,35 @@ static inline void rrd2rrdr_do_dimension(
current_point = new_point;
query_interpolate_point(current_point, last1_point, now_end_time);
- internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
- "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
- " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.",
- rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
- current_point.id, current_point.start_time, current_point.end_time, now_end_time);
+// internal_error(current_point.id > 0
+// && last1_point.id == 0
+// && current_point.end_time > after_wanted
+// && current_point.end_time > now_end_time,
+// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
+// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
+// " but we could really favor by having last_point1 in this query.",
+// qt->id, string2str(qm->dimension.id),
+// after_wanted, before_wanted,
+// ops.view_update_every, ops.query_granularity,
+// current_point.id, current_point.start_time, current_point.end_time,
+// now_end_time);
}
else if(likely(now_end_time <= last1_point.end_time)) {
// our LAST point is still valid
current_point = last1_point;
query_interpolate_point(current_point, last2_point, now_end_time);
- internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
- "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
- " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.",
- rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
- current_point.id, current_point.start_time, current_point.end_time, now_end_time);
+// internal_error(current_point.id > 0
+// && last2_point.id == 0
+// && current_point.end_time > after_wanted
+// && current_point.end_time > now_end_time,
+// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
+// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
+// " but we could really favor by having last_point2 in this query.",
+// qt->id, string2str(qm->dimension.id),
+// after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
+// current_point.id, current_point.start_time, current_point.end_time,
+// now_end_time);
}
else {
// a GAP, we don't have a value this time
@@ -1414,7 +1455,7 @@ static inline void rrd2rrdr_do_dimension(
r->internal.result_points_generated += points_added;
r->internal.db_points_read += ops.db_total_points_read;
- for(int tr = 0; tr < storage_tiers ; tr++)
+ for(size_t tr = 0; tr < storage_tiers ; tr++)
r->internal.tier_points_read[tr] += ops.db_points_read_per_tier[tr];
r->min = min;
@@ -1423,24 +1464,26 @@ static inline void rrd2rrdr_do_dimension(
r->after = min_date - ops.view_update_every + ops.query_granularity;
rrdr_done(r, rrdr_line);
- internal_error((long)points_added != points_wanted,
- "QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).",
- r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
+ internal_error(points_added != points_wanted,
+ "QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).",
+ qt->id, string2str(qm->dimension.id),
+ (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
}
// ----------------------------------------------------------------------------
// fill the gap of a tier
-extern void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
+void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
+void store_metric_collection_completed(void);
-void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
- if(unlikely(tier < 0 || tier >= storage_tiers)) return;
+void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now) {
+ if(unlikely(tier >= storage_tiers)) return;
if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;
struct rrddim_tier *t = rd->tiers[tier];
if(unlikely(!t)) return;
- time_t latest_time_t = t->query_ops.latest_time(t->db_metric_handle);
+ time_t latest_time_t = t->query_ops->latest_time(t->db_metric_handle);
time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every;
time_t time_diff = now - latest_time_t;
@@ -1450,43 +1493,40 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
// there is really nothing we can do
if(now <= latest_time_t || time_diff < granularity) return;
- struct rrddim_query_handle handle;
-
- size_t all_points_read = 0;
+ struct storage_engine_query_handle handle;
// for each lower tier
- for(int tr = tier - 1; tr >= 0 ;tr--){
- time_t smaller_tier_first_time = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
- time_t smaller_tier_last_time = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
+ for(int tr = (int)tier - 1; tr >= 0 ;tr--){
+ time_t smaller_tier_first_time = rd->tiers[tr]->query_ops->oldest_time(rd->tiers[tr]->db_metric_handle);
+ time_t smaller_tier_last_time = rd->tiers[tr]->query_ops->latest_time(rd->tiers[tr]->db_metric_handle);
if(smaller_tier_last_time <= latest_time_t) continue; // it is as bad as we are
long after_wanted = (latest_time_t < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_t;
long before_wanted = smaller_tier_last_time;
struct rrddim_tier *tmp = rd->tiers[tr];
- tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE);
+ tmp->query_ops->init(tmp->db_metric_handle, &handle, after_wanted, before_wanted);
- size_t points = 0;
+ size_t points_read = 0;
- while(!tmp->query_ops.is_finished(&handle)) {
+ while(!tmp->query_ops->is_finished(&handle)) {
- STORAGE_POINT sp = tmp->query_ops.next_metric(&handle);
+ STORAGE_POINT sp = tmp->query_ops->next_metric(&handle);
+ points_read++;
if(sp.end_time > latest_time_t) {
latest_time_t = sp.end_time;
- store_metric_at_tier(rd, t, sp, sp.end_time * USEC_PER_SEC);
- points++;
+ store_metric_at_tier(rd, tr, t, sp, sp.end_time * USEC_PER_SEC);
}
}
- all_points_read += points;
- tmp->query_ops.finalize(&handle);
+ tmp->query_ops->finalize(&handle);
+ store_metric_collection_completed();
+ global_statistics_backfill_query_completed(points_read);
//internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d",
// rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr);
}
-
- rrdr_query_completed(all_points_read, all_points_read);
}
// ----------------------------------------------------------------------------
@@ -1497,29 +1537,33 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
, RRDR_OPTIONS options __maybe_unused
, RRDR_GROUPING group_method
, bool aligned
- , long group
- , long resampling_time
- , long resampling_group
+ , size_t group
+ , time_t resampling_time
+ , size_t resampling_group
, time_t after_wanted
, time_t after_requested
, time_t before_wanted
, time_t before_requested
- , long points_requested
- , long points_wanted
+ , size_t points_requested
+ , size_t points_wanted
//, size_t after_slot
//, size_t before_slot
, const char *msg
) {
- netdata_rwlock_rdlock(&r->st->rrdset_rwlock);
- info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), "
- "after (got: %zu, want: %zu, req: %ld, db: %zu), "
- "before (got: %zu, want: %zu, req: %ld, db: %zu), "
- "duration (got: %zu, want: %zu, req: %ld, db: %zu), "
- //"slot (after: %zu, before: %zu, delta: %zu), "
- "points (got: %ld, want: %ld, req: %ld, db: %ld), "
+
+ time_t first_entry_t = r->internal.qt->db.first_time_t;
+ time_t last_entry_t = r->internal.qt->db.last_time_t;
+
+ internal_error(
+ true,
+ "rrd2rrdr() on %s update every %ld with %s grouping %s (group: %zu, resampling_time: %ld, resampling_group: %zu), "
+ "after (got: %ld, want: %ld, req: %ld, db: %ld), "
+ "before (got: %ld, want: %ld, req: %ld, db: %ld), "
+ "duration (got: %ld, want: %ld, req: %ld, db: %ld), "
+ "points (got: %zu, want: %zu, req: %zu), "
"%s"
- , r->st->name
- , r->st->update_every
+ , r->internal.qt->id
+ , r->internal.qt->window.query_granularity
// grouping
, (aligned) ? "aligned" : "unaligned"
@@ -1529,45 +1573,36 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
, resampling_group
// after
- , (size_t)r->after
- , (size_t)after_wanted
+ , r->after
+ , after_wanted
, after_requested
- , (size_t)rrdset_first_entry_t_nolock(r->st)
+ , first_entry_t
// before
- , (size_t)r->before
- , (size_t)before_wanted
+ , r->before
+ , before_wanted
, before_requested
- , (size_t)rrdset_last_entry_t_nolock(r->st)
+ , last_entry_t
// duration
- , (size_t)(r->before - r->after + r->st->update_every)
- , (size_t)(before_wanted - after_wanted + r->st->update_every)
- , before_requested - after_requested
- , (size_t)((rrdset_last_entry_t_nolock(r->st) - rrdset_first_entry_t_nolock(r->st)) + r->st->update_every)
-
- // slot
- /*
- , after_slot
- , before_slot
- , (after_slot > before_slot) ? (r->st->entries - after_slot + before_slot) : (before_slot - after_slot)
- */
+ , (long)(r->before - r->after + r->internal.qt->window.query_granularity)
+ , (long)(before_wanted - after_wanted + r->internal.qt->window.query_granularity)
+ , (long)before_requested - after_requested
+ , (long)((last_entry_t - first_entry_t) + r->internal.qt->window.query_granularity)
// points
, r->rows
, points_wanted
, points_requested
- , r->st->entries
// message
, msg
);
- netdata_rwlock_unlock(&r->st->rrdset_rwlock);
}
#endif // NETDATA_INTERNAL_CHECKS
// Returns 1 if an absolute period was requested or 0 if it was a relative period
-int rrdr_relative_window_to_absolute(long long *after, long long *before) {
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before) {
time_t now = now_realtime_sec() - 1;
int absolute_period_requested = -1;
@@ -1624,10 +1659,25 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
after_requested -= delta;
}
+ time_t absolute_minimum_time = now - (10 * 365 * 86400);
+ time_t absolute_maximum_time = now + (1 * 365 * 86400);
+
+ if (after_requested < absolute_minimum_time && !unittest_running)
+ after_requested = absolute_minimum_time;
+
+ if (after_requested > absolute_maximum_time && !unittest_running)
+ after_requested = absolute_maximum_time;
+
+ if (before_requested < absolute_minimum_time && !unittest_running)
+ before_requested = absolute_minimum_time;
+
+ if (before_requested > absolute_maximum_time && !unittest_running)
+ before_requested = absolute_maximum_time;
+
*before = before_requested;
*after = after_requested;
- return absolute_period_requested;
+ return (absolute_period_requested != 1);
}
// #define DEBUG_QUERY_LOGIC 1
@@ -1636,7 +1686,7 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
#define query_debug_log_init() BUFFER *debug_log = buffer_create(1000)
#define query_debug_log(args...) buffer_sprintf(debug_log, ##args)
#define query_debug_log_fin() { \
- info("QUERY: chart '%s', after:%lld, before:%lld, duration:%lld, points:%ld, res:%ld - wanted => after:%lld, before:%lld, points:%ld, group:%ld, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", st->name, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
+ info("QUERY: '%s', after:%ld, before:%ld, duration:%ld, points:%zu, res:%ld - wanted => after:%ld, before:%ld, points:%zu, group:%zu, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", qt->id, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
buffer_free(debug_log); \
debug_log = NULL; \
}
@@ -1648,21 +1698,18 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
#define query_debug_log_free() debug_dummy()
#endif
-RRDR *rrd2rrdr(
- ONEWAYALLOC *owa
- , RRDSET *st
- , long points_requested
- , long long after_requested
- , long long before_requested
- , RRDR_GROUPING group_method
- , long resampling_time_requested
- , RRDR_OPTIONS options
- , const char *dimensions
- , struct context_param *context_param_list
- , const char *group_options
- , int timeout
- , int tier
-) {
+bool query_target_calculate_window(QUERY_TARGET *qt) {
+ if (unlikely(!qt)) return false;
+
+ size_t points_requested = (long)qt->request.points;
+ time_t after_requested = qt->request.after;
+ time_t before_requested = qt->request.before;
+ RRDR_GROUPING group_method = qt->request.group_method;
+ time_t resampling_time_requested = qt->request.resampling_time;
+ RRDR_OPTIONS options = qt->request.options;
+ size_t tier = qt->request.tier;
+ time_t update_every = qt->db.minimum_latest_update_every;
+
// RULES
// points_requested = 0
// the user wants all the natural points the database has
@@ -1676,10 +1723,9 @@ RRDR *rrd2rrdr(
// when natural points are wanted, the query has to be aligned to the update_every
// of the database
- long points_wanted = points_requested;
- long long after_wanted = after_requested;
- long long before_wanted = before_requested;
- int update_every = st->update_every;
+ size_t points_wanted = points_requested;
+ time_t after_wanted = after_requested;
+ time_t before_wanted = before_requested;
bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
bool automatic_natural_points = (points_wanted == 0);
@@ -1689,13 +1735,7 @@ RRDR *rrd2rrdr(
query_debug_log_init();
- // make sure points_wanted is positive
- if(points_wanted < 0) {
- points_wanted = -points_wanted;
- query_debug_log(":-points_wanted %ld", points_wanted);
- }
-
- if(ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
+ if (ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
relative_period_requested = true;
natural_points = true;
options |= RRDR_OPTION_NATURAL_POINTS;
@@ -1703,105 +1743,93 @@ RRDR *rrd2rrdr(
}
// if the user wants virtual points, make sure we do it
- if(options & RRDR_OPTION_VIRTUAL_POINTS)
+ if (options & RRDR_OPTION_VIRTUAL_POINTS)
natural_points = false;
// set the right flag about natural and virtual points
- if(natural_points) {
+ if (natural_points) {
options |= RRDR_OPTION_NATURAL_POINTS;
- if(options & RRDR_OPTION_VIRTUAL_POINTS)
+ if (options & RRDR_OPTION_VIRTUAL_POINTS)
options &= ~RRDR_OPTION_VIRTUAL_POINTS;
}
else {
options |= RRDR_OPTION_VIRTUAL_POINTS;
- if(options & RRDR_OPTION_NATURAL_POINTS)
+ if (options & RRDR_OPTION_NATURAL_POINTS)
options &= ~RRDR_OPTION_NATURAL_POINTS;
}
- if(after_wanted == 0 || before_wanted == 0) {
- // for non-context queries we have to find the duration of the database
- // for context queries we will assume 600 seconds duration
-
- if(!context_param_list) {
- relative_period_requested = true;
-
- rrdset_rdlock(st);
- time_t first_entry_t = rrdset_first_entry_t_nolock(st);
- time_t last_entry_t = rrdset_last_entry_t_nolock(st);
- rrdset_unlock(st);
-
- if(first_entry_t == 0 || last_entry_t == 0) {
- internal_error(true, "QUERY: chart without data detected on '%s'", st->name);
- query_debug_log_free();
- return NULL;
- }
+ if (after_wanted == 0 || before_wanted == 0) {
+ relative_period_requested = true;
- query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
+ time_t first_entry_t = qt->db.first_time_t;
+ time_t last_entry_t = qt->db.last_time_t;
- if (after_wanted == 0) {
- after_wanted = first_entry_t;
- query_debug_log(":zero after_wanted %lld", after_wanted);
- }
+ if (first_entry_t == 0 || last_entry_t == 0) {
+ internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld", qt->id, first_entry_t, last_entry_t);
+ query_debug_log_free();
+ return false;
+ }
- if (before_wanted == 0) {
- before_wanted = last_entry_t;
- before_is_aligned_to_db_end = true;
- query_debug_log(":zero before_wanted %lld", before_wanted);
- }
+ query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
- if(points_wanted == 0) {
- points_wanted = (last_entry_t - first_entry_t) / update_every;
- query_debug_log(":zero points_wanted %ld", points_wanted);
- }
+ if (after_wanted == 0) {
+ after_wanted = first_entry_t;
+ query_debug_log(":zero after_wanted %ld", after_wanted);
}
- // if they are still zero, assume 600
+ if (before_wanted == 0) {
+ before_wanted = last_entry_t;
+ before_is_aligned_to_db_end = true;
+ query_debug_log(":zero before_wanted %ld", before_wanted);
+ }
- if(after_wanted == 0) {
- after_wanted = -600;
- query_debug_log(":zero600 after_wanted %lld", after_wanted);
+ if (points_wanted == 0) {
+ points_wanted = (last_entry_t - first_entry_t) / update_every;
+ query_debug_log(":zero points_wanted %zu", points_wanted);
}
}
- if(points_wanted == 0) {
+ if (points_wanted == 0) {
points_wanted = 600;
- query_debug_log(":zero600 points_wanted %ld", points_wanted);
+ query_debug_log(":zero600 points_wanted %zu", points_wanted);
}
// convert our before_wanted and after_wanted to absolute
rrdr_relative_window_to_absolute(&after_wanted, &before_wanted);
- query_debug_log(":relative2absolute after %lld, before %lld", after_wanted, before_wanted);
+ query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
- if(natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
- update_every = rrdset_find_natural_update_every_for_timeframe(st, after_wanted, before_wanted, points_wanted, options, tier);
- if(update_every <= 0) update_every = st->update_every;
- query_debug_log(":natural update every %d", update_every);
+ if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
+ update_every = rrdset_find_natural_update_every_for_timeframe(
+ qt, after_wanted, before_wanted, points_wanted, options, tier);
+
+ if (update_every <= 0) update_every = qt->db.minimum_latest_update_every;
+ query_debug_log(":natural update every %ld", update_every);
}
// this is the update_every of the query
// it may be different to the update_every of the database
- time_t query_granularity = (natural_points)?update_every:1;
- if(query_granularity <= 0) query_granularity = 1;
+ time_t query_granularity = (natural_points) ? update_every : 1;
+ if (query_granularity <= 0) query_granularity = 1;
query_debug_log(":query_granularity %ld", query_granularity);
// align before_wanted and after_wanted to query_granularity
if (before_wanted % query_granularity) {
before_wanted -= before_wanted % query_granularity;
- query_debug_log(":granularity align before_wanted %lld", before_wanted);
+ query_debug_log(":granularity align before_wanted %ld", before_wanted);
}
if (after_wanted % query_granularity) {
after_wanted -= after_wanted % query_granularity;
- query_debug_log(":granularity align after_wanted %lld", after_wanted);
+ query_debug_log(":granularity align after_wanted %ld", after_wanted);
}
// automatic_natural_points is set when the user wants all the points available in the database
- if(automatic_natural_points) {
+ if (automatic_natural_points) {
points_wanted = (before_wanted - after_wanted + 1) / query_granularity;
- if(unlikely(points_wanted <= 0)) points_wanted = 1;
- query_debug_log(":auto natural points_wanted %ld", points_wanted);
+ if (unlikely(points_wanted <= 0)) points_wanted = 1;
+ query_debug_log(":auto natural points_wanted %zu", points_wanted);
}
time_t duration = before_wanted - after_wanted;
@@ -1810,42 +1838,47 @@ RRDR *rrd2rrdr(
if (unlikely(resampling_time_requested > duration)) {
after_wanted = before_wanted - resampling_time_requested;
duration = before_wanted - after_wanted;
- query_debug_log(":resampling after_wanted %lld", after_wanted);
+ query_debug_log(":resampling after_wanted %ld", after_wanted);
}
// if the duration is not aligned to resampling time
// extend the duration to the past, to avoid a gap at the chart
// only when the missing duration is above 1/10th of a point
- if(resampling_time_requested > query_granularity && duration % resampling_time_requested) {
+ if (resampling_time_requested > query_granularity && duration % resampling_time_requested) {
time_t delta = duration % resampling_time_requested;
- if(delta > resampling_time_requested / 10) {
+ if (delta > resampling_time_requested / 10) {
after_wanted -= resampling_time_requested - delta;
duration = before_wanted - after_wanted;
- query_debug_log(":resampling2 after_wanted %lld", after_wanted);
+ query_debug_log(":resampling2 after_wanted %ld", after_wanted);
}
}
// the available points of the query
- long points_available = (duration + 1) / query_granularity;
- if(unlikely(points_available <= 0)) points_available = 1;
- query_debug_log(":points_available %ld", points_available);
+ size_t points_available = (duration + 1) / query_granularity;
+ if (unlikely(points_available <= 0)) points_available = 1;
+ query_debug_log(":points_available %zu", points_available);
- if(points_wanted > points_available) {
+ if (points_wanted > points_available) {
points_wanted = points_available;
- query_debug_log(":max points_wanted %ld", points_wanted);
+ query_debug_log(":max points_wanted %zu", points_wanted);
+ }
+
+ if(points_wanted > 86400 && !unittest_running) {
+ points_wanted = 86400;
+ query_debug_log(":absolute max points_wanted %zu", points_wanted);
}
// calculate the desired grouping of source data points
- long group = points_available / points_wanted;
- if(group <= 0) group = 1;
+ size_t group = points_available / points_wanted;
+ if (group == 0) group = 1;
// round "group" to the closest integer
- if(points_available % points_wanted > points_wanted / 2)
+ if (points_available % points_wanted > points_wanted / 2)
group++;
- query_debug_log(":group %ld", group);
+ query_debug_log(":group %zu", group);
- if(points_wanted * group * query_granularity < duration) {
+ if (points_wanted * group * query_granularity < (size_t)duration) {
// the grouping we are going to do, is not enough
// to cover the entire duration requested, so
// we have to change the number of points, to make sure we will
@@ -1854,162 +1887,186 @@ RRDR *rrd2rrdr(
// let's see how many points are the optimal
points_wanted = points_available / group;
- if(points_wanted * group < points_available)
+ if (points_wanted * group < points_available)
points_wanted++;
- if(unlikely(points_wanted <= 0))
+ if (unlikely(points_wanted == 0))
points_wanted = 1;
- query_debug_log(":optimal points %ld", points_wanted);
+ query_debug_log(":optimal points %zu", points_wanted);
}
// resampling_time_requested enforces a certain grouping multiple
NETDATA_DOUBLE resampling_divisor = 1.0;
- long resampling_group = 1;
- if(unlikely(resampling_time_requested > query_granularity)) {
+ size_t resampling_group = 1;
+ if (unlikely(resampling_time_requested > query_granularity)) {
// the points we should group to satisfy gtime
resampling_group = resampling_time_requested / query_granularity;
- if(unlikely(resampling_time_requested % query_granularity))
+ if (unlikely(resampling_time_requested % query_granularity))
resampling_group++;
- query_debug_log(":resampling group %ld", resampling_group);
+ query_debug_log(":resampling group %zu", resampling_group);
// adapt group according to resampling_group
- if(unlikely(group < resampling_group)) {
- group = resampling_group; // do not allow grouping below the desired one
- query_debug_log(":group less res %ld", group);
+ if (unlikely(group < resampling_group)) {
+ group = resampling_group; // do not allow grouping below the desired one
+ query_debug_log(":group less res %zu", group);
}
- if(unlikely(group % resampling_group)) {
+ if (unlikely(group % resampling_group)) {
group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
- query_debug_log(":group mod res %ld", group);
+ query_debug_log(":group mod res %zu", group);
}
// resampling_divisor = group / resampling_group;
- resampling_divisor = (NETDATA_DOUBLE)(group * query_granularity) / (NETDATA_DOUBLE)resampling_time_requested;
+ resampling_divisor = (NETDATA_DOUBLE) (group * query_granularity) / (NETDATA_DOUBLE) resampling_time_requested;
query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor);
}
// now that we have group, align the requested timeframe to fit it.
- if(aligned && before_wanted % (group * query_granularity)) {
- if(before_is_aligned_to_db_end)
- before_wanted -= before_wanted % (group * query_granularity);
+ if (aligned && before_wanted % (group * query_granularity)) {
+ if (before_is_aligned_to_db_end)
+ before_wanted -= before_wanted % (time_t)(group * query_granularity);
else
- before_wanted += (group * query_granularity) - before_wanted % (group * query_granularity);
- query_debug_log(":align before_wanted %lld", before_wanted);
+ before_wanted += (time_t)(group * query_granularity) - before_wanted % (time_t)(group * query_granularity);
+ query_debug_log(":align before_wanted %ld", before_wanted);
}
- after_wanted = before_wanted - (points_wanted * group * query_granularity) + query_granularity;
- query_debug_log(":final after_wanted %lld", after_wanted);
+ after_wanted = before_wanted - (time_t)(points_wanted * group * query_granularity) + query_granularity;
+ query_debug_log(":final after_wanted %ld", after_wanted);
duration = before_wanted - after_wanted;
query_debug_log(":final duration %ld", duration + 1);
- // check the context query based on the starting time of the query
- if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
- rebuild_context_param_list(owa, context_param_list, after_wanted);
- st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
-
- if(unlikely(!st))
- return NULL;
- }
+ query_debug_log_fin();
internal_error(points_wanted != duration / (query_granularity * group) + 1,
- "QUERY: points_wanted %ld is not points %ld",
- points_wanted, duration / (query_granularity * group) + 1);
+ "QUERY: points_wanted %zu is not points %zu",
+ points_wanted, (size_t)(duration / (query_granularity * group) + 1));
internal_error(group < resampling_group,
- "QUERY: group %ld is less than the desired group points %ld",
+ "QUERY: group %zu is less than the desired group points %zu",
group, resampling_group);
internal_error(group > resampling_group && group % resampling_group,
- "QUERY: group %ld is not a multiple of the desired group points %ld",
+ "QUERY: group %zu is not a multiple of the desired group points %zu",
group, resampling_group);
// -------------------------------------------------------------------------
- // initialize our result set
- // this also locks the chart for us
+ // update QUERY_TARGET with our calculations
+
+ qt->window.after = after_wanted;
+ qt->window.before = before_wanted;
+ qt->window.relative = relative_period_requested;
+ qt->window.points = points_wanted;
+ qt->window.group = group;
+ qt->window.group_method = group_method;
+ qt->window.group_options = qt->request.group_options;
+ qt->window.query_granularity = query_granularity;
+ qt->window.resampling_group = resampling_group;
+ qt->window.resampling_divisor = resampling_divisor;
+ qt->window.options = options;
+ qt->window.tier = tier;
+ qt->window.aligned = aligned;
+
+ return true;
+}
+
+RRDR *rrd2rrdr_legacy(
+ ONEWAYALLOC *owa,
+ RRDSET *st, size_t points, time_t after, time_t before,
+ RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,
+ const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source) {
+
+ QUERY_TARGET_REQUEST qtr = {
+ .st = st,
+ .points = points,
+ .after = after,
+ .before = before,
+ .group_method = group_method,
+ .resampling_time = resampling_time,
+ .options = options,
+ .dimensions = dimensions,
+ .group_options = group_options,
+ .timeout = timeout,
+ .tier = tier,
+ .query_source = query_source,
+ };
+
+ return rrd2rrdr(owa, query_target_create(&qtr));
+}
- RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);
+RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
+ if(!qt)
+ return NULL;
+
+ if(!owa) {
+ query_target_release(qt);
+ return NULL;
+ }
+
+ // qt.window members are the WANTED ones.
+ // qt.request members are the REQUESTED ones.
+
+ RRDR *r = rrdr_create(owa, qt);
if(unlikely(!r)) {
- internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld",
- st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
+ internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, points=%zu",
+ qt->id, qt->window.after, qt->window.before, qt->window.points);
return NULL;
}
- if(unlikely(!r->d || !points_wanted)) {
- internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld",
- st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
+ if(unlikely(!r->d || !qt->window.points)) {
+ internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%ld, before=%ld, points=%zu",
+ qt->id, qt->window.after, qt->window.before, qt->window.points);
return r;
}
- if(relative_period_requested)
+ if(qt->window.relative)
r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
else
r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
- // find how many dimensions we have
- long dimensions_count = r->d;
-
// -------------------------------------------------------------------------
// initialize RRDR
- r->group = group;
- r->update_every = (int)(group * query_granularity);
- r->before = before_wanted;
- r->after = after_wanted;
- r->internal.points_wanted = points_wanted;
- r->internal.resampling_group = resampling_group;
- r->internal.resampling_divisor = resampling_divisor;
- r->internal.query_options = options;
- r->internal.query_tier = tier;
+ r->group = qt->window.group;
+ r->update_every = (int) (qt->window.group * qt->window.query_granularity);
+ r->before = qt->window.before;
+ r->after = qt->window.after;
+ r->internal.points_wanted = qt->window.points;
+ r->internal.resampling_group = qt->window.resampling_group;
+ r->internal.resampling_divisor = qt->window.resampling_divisor;
+ r->internal.query_options = qt->window.options;
// -------------------------------------------------------------------------
// assign the processor functions
- rrdr_set_grouping_function(r, group_method);
+ rrdr_set_grouping_function(r, qt->window.group_method);
// allocate any memory required by the grouping method
- r->internal.grouping_create(r, group_options);
-
-
- // -------------------------------------------------------------------------
- // disable the not-wanted dimensions
-
- if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
- rrdset_check_rdlock(st);
-
- if(dimensions)
- rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
-
-
- query_debug_log_fin();
+ r->internal.grouping_create(r, qt->window.group_options);
// -------------------------------------------------------------------------
// do the work for each dimension
time_t max_after = 0, min_before = 0;
- long max_rows = 0;
+ size_t max_rows = 0;
- RRDDIM *first_rd = context_param_list ? context_param_list->rd : st->dimensions;
- RRDDIM *rd;
- long c, dimensions_used = 0, dimensions_nonzero = 0;
+ long dimensions_used = 0, dimensions_nonzero = 0;
struct timeval query_start_time;
struct timeval query_current_time;
- if (timeout) now_realtime_timeval(&query_start_time);
+ if (qt->request.timeout)
+ now_realtime_timeval(&query_start_time);
- for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
+ for(size_t c = 0, max = qt->query.used; c < max ; c++) {
+ // set the query target dimension options to rrdr
+ r->od[c] = qt->query.array[c].dimension.options;
- // if we need a percentage, we need to calculate all dimensions
- if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) {
- if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;
- continue;
- }
r->od[c] |= RRDR_DIMENSION_SELECTED;
// reset the grouping for the new dimension
r->internal.grouping_reset(r);
- rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted);
- if (timeout)
+ rrd2rrdr_do_dimension(r, c);
+ if (qt->request.timeout)
now_realtime_timeval(&query_current_time);
if(r->od[c] & RRDR_DIMENSION_NONZERO)
@@ -2024,30 +2081,30 @@ RRDR *rrd2rrdr(
else {
if(r->after != max_after) {
internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
- st->name, (size_t)max_after, rd->name, (size_t)r->after);
+ string2str(qt->query.array[c].dimension.id), (size_t)max_after, string2str(qt->query.array[c].dimension.name), (size_t)r->after);
r->after = (r->after > max_after) ? r->after : max_after;
}
if(r->before != min_before) {
internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
- st->name, (size_t)min_before, rd->name, (size_t)r->before);
+ string2str(qt->query.array[c].dimension.id), (size_t)min_before, string2str(qt->query.array[c].dimension.name), (size_t)r->before);
r->before = (r->before < min_before) ? r->before : min_before;
}
if(r->rows != max_rows) {
internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
- st->name, (size_t)max_rows, rd->name, (size_t)r->rows);
+ string2str(qt->query.array[c].dimension.id), (size_t)max_rows, string2str(qt->query.array[c].dimension.name), (size_t)r->rows);
r->rows = (r->rows > max_rows) ? r->rows : max_rows;
}
}
dimensions_used++;
- if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
- log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
- (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
+ if (qt->request.timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout) {
+ log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)",
+ (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout);
r->result_options |= RRDR_RESULT_OPTION_CANCEL;
break;
}
@@ -2056,44 +2113,44 @@ RRDR *rrd2rrdr(
#ifdef NETDATA_INTERNAL_CHECKS
if (dimensions_used) {
if(r->internal.log)
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
- after_wanted, after_requested, before_wanted, before_requested,
- points_requested, points_wanted, /*after_slot, before_slot,*/
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
r->internal.log);
- if(r->rows != points_wanted)
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
- after_wanted, after_requested, before_wanted, before_requested,
- points_requested, points_wanted, /*after_slot, before_slot,*/
+ if(r->rows != qt->window.points)
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
"got 'points' is not wanted 'points'");
- if(aligned && (r->before % (group * query_granularity)) != 0)
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
- after_wanted, after_requested, before_wanted,before_wanted,
- points_requested, points_wanted, /*after_slot, before_slot,*/
+ if(qt->window.aligned && (r->before % (qt->window.group * qt->window.query_granularity)) != 0)
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+ qt->window.after, qt->request.after, qt->window.before,qt->request.before,
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
"'before' is not aligned but alignment is required");
// 'after' should not be aligned, since we start inside the first group
- //if(aligned && (r->after % group) != 0)
- // rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
+ //if(qt->window.aligned && (r->after % group) != 0)
+ // rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
- if(r->before != before_wanted)
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
- after_wanted, after_requested, before_wanted, before_requested,
- points_requested, points_wanted, /*after_slot, before_slot,*/
+ if(r->before != qt->window.before)
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
"chart is not aligned to requested 'before'");
- if(r->before != before_wanted)
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
- after_wanted, after_requested, before_wanted, before_requested,
- points_requested, points_wanted, /*after_slot, before_slot,*/
+ if(r->before != qt->window.before)
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
"got 'before' is not wanted 'before'");
// reported 'after' varies, depending on group
- if(r->after != after_wanted)
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
- after_wanted, after_requested, before_wanted, before_requested,
- points_requested, points_wanted, /*after_slot, before_slot,*/
+ if(r->after != qt->window.after)
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
"got 'after' is not wanted 'after'");
}
@@ -2103,15 +2160,16 @@ RRDR *rrd2rrdr(
r->internal.grouping_free(r);
// when all the dimensions are zero, we should return all of them
- if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
+ if(unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
// all the dimensions are zero
// mark them as NONZERO to send them all
- for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
+ for(size_t c = 0, max = qt->query.used; c < max ; c++) {
if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
r->od[c] |= RRDR_DIMENSION_NONZERO;
}
}
- rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);
+ global_statistics_rrdr_query_completed(dimensions_used, r->internal.db_points_read,
+ r->internal.result_points_generated, qt->request.query_source);
return r;
}
diff --git a/web/api/queries/query.h b/web/api/queries/query.h
index df876d9ac..ebad5a1f8 100644
--- a/web/api/queries/query.h
+++ b/web/api/queries/query.h
@@ -47,10 +47,10 @@ typedef enum rrdr_grouping {
RRDR_GROUPING_COUNTIF,
} RRDR_GROUPING;
-extern const char *group_method2string(RRDR_GROUPING group);
-extern void web_client_api_v1_init_grouping(void);
-extern RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def);
-extern const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group);
+const char *group_method2string(RRDR_GROUPING group);
+void web_client_api_v1_init_grouping(void);
+RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def);
+const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group);
#ifdef __cplusplus
}
diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c
index ecf4ca2ac..676224c9d 100644
--- a/web/api/queries/rrdr.c
+++ b/web/api/queries/rrdr.c
@@ -61,9 +61,7 @@ static void rrdr_dump(RRDR *r)
inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
if(unlikely(!r)) return;
- if(likely(r->st_locked_by_rrdr_create))
- rrdset_unlock(r->st);
-
+ query_target_release(r->internal.qt);
onewayalloc_freez(owa, r->t);
onewayalloc_freez(owa, r->v);
onewayalloc_freez(owa, r->o);
@@ -72,12 +70,23 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
onewayalloc_freez(owa, r);
}
-RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) {
+RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
+ if(unlikely(!qt || !qt->query.used || !qt->window.points))
+ return NULL;
+
+ size_t dimensions = qt->query.used;
+ size_t points = qt->window.points;
+
+ // create the rrdr
RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
r->internal.owa = owa;
+ r->internal.qt = qt;
- r->d = dimensions;
- r->n = points;
+ r->before = qt->window.before;
+ r->after = qt->window.after;
+ r->internal.points_wanted = qt->window.points;
+ r->d = (int)dimensions;
+ r->n = (int)points;
r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
@@ -90,42 +99,3 @@ RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points
return r;
}
-
-RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) {
- if (unlikely(!st)) return NULL;
-
- bool st_locked_by_rrdr_create = false;
- if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
- rrdset_rdlock(st);
- st_locked_by_rrdr_create = true;
- }
-
- // count the number of dimensions
- int dimensions = 0;
- RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
- RRDDIM *rd;
- if (temp_rd) {
- RRDDIM *t = temp_rd;
- while (t) {
- dimensions++;
- t = t->next;
- }
- } else
- rrddim_foreach_read(rd, st) dimensions++;
-
- // create the rrdr
- RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n);
- r->st = st;
- r->st_locked_by_rrdr_create = st_locked_by_rrdr_create;
-
- // set the hidden flag on hidden dimensions
- int c;
- for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) {
- if (unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)))
- r->od[c] = RRDR_DIMENSION_HIDDEN;
- else
- r->od[c] = RRDR_DIMENSION_DEFAULT;
- }
-
- return r;
-}
diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h
index 1c80e103f..6151cddc7 100644
--- a/web/api/queries/rrdr.h
+++ b/web/api/queries/rrdr.h
@@ -18,32 +18,33 @@ typedef enum tier_query_fetch {
} TIER_QUERY_FETCH;
typedef enum rrdr_options {
- RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
- RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest)
- RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing
- RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum
- RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates
- RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates
- RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros
- RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array
- RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs
- RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result
- RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes
- RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total
- RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes
- RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
- RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs
- RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
- RRDR_OPTION_CUSTOM_VARS = 0x00010000, // when wrapping response in a JSON, return custom variables in response
- RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
- RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points
- RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number
- RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes
- RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap
- RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query
+ RRDR_OPTION_NONZERO = 0x00000001, // don't output dimensions with just zero values
+ RRDR_OPTION_REVERSED = 0x00000002, // output the rows in reverse order (oldest to newest)
+ RRDR_OPTION_ABSOLUTE = 0x00000004, // values positive, for DATASOURCE_SSV before summing
+ RRDR_OPTION_MIN2MAX = 0x00000008, // when adding dimensions, use max - min, instead of sum
+ RRDR_OPTION_SECONDS = 0x00000010, // output seconds, instead of dates
+ RRDR_OPTION_MILLISECONDS = 0x00000020, // output milliseconds, instead of dates
+ RRDR_OPTION_NULL2ZERO = 0x00000040, // do not show nulls, convert them to zeros
+ RRDR_OPTION_OBJECTSROWS = 0x00000080, // each row of values should be an object, not an array
+ RRDR_OPTION_GOOGLE_JSON = 0x00000100, // comply with google JSON/JSONP specs
+ RRDR_OPTION_JSON_WRAP = 0x00000200, // wrap the response in a JSON header with info about the result
+ RRDR_OPTION_LABEL_QUOTES = 0x00000400, // in CSV output, wrap header labels in double quotes
+ RRDR_OPTION_PERCENTAGE = 0x00000800, // give values as percentage of total
+ RRDR_OPTION_NOT_ALIGNED = 0x00001000, // do not align charts for persistent timeframes
+ RRDR_OPTION_DISPLAY_ABS = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
+ RRDR_OPTION_MATCH_IDS = 0x00004000, // when filtering dimensions, match only IDs
+ RRDR_OPTION_MATCH_NAMES = 0x00008000, // when filtering dimensions, match only names
+ RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
+ RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points
+ RRDR_OPTION_ANOMALY_BIT = 0x00080000, // Return the anomaly bit stored in each collected_number
+ RRDR_OPTION_RETURN_RAW = 0x00100000, // Return raw data for aggregating across multiple nodes
+ RRDR_OPTION_RETURN_JWAR = 0x00200000, // Return anomaly rates in jsonwrap
+ RRDR_OPTION_SELECTED_TIER = 0x00400000, // Use the selected tier for the query
+ RRDR_OPTION_ALL_DIMENSIONS = 0x00800000, // Return the full dimensions list
// internal ones - not to be exposed to the API
- RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
+ RRDR_OPTION_INTERNAL_AR = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
+ RRDR_OPTION_HEALTH_RSRVD1 = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
} RRDR_OPTIONS;
typedef enum rrdr_value_flag {
@@ -67,16 +68,14 @@ typedef enum rrdr_result_flags {
// (should not to be cached by browsers and proxies)
RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames
RRDR_RESULT_OPTION_CANCEL = 0x00000008, // the query needs to be cancelled
-} RRDR_RESULT_FLAGS;
+} RRDR_RESULT_OPTIONS;
typedef struct rrdresult {
- struct rrdset *st; // the chart this result refers to
+ RRDR_RESULT_OPTIONS result_options; // RRDR_RESULT_OPTION_*
- RRDR_RESULT_FLAGS result_options; // RRDR_RESULT_OPTION_*
-
- int d; // the number of dimensions
- long n; // the number of values in the arrays
- long rows; // the number of rows used
+ size_t d; // the number of dimensions
+ size_t n; // the number of values in the arrays
+ size_t rows; // the number of rows used
RRDR_DIMENSION_FLAGS *od; // the options for the dimensions
@@ -85,8 +84,8 @@ typedef struct rrdresult {
RRDR_VALUE_FLAGS *o; // array n x d options for each value returned
NETDATA_DOUBLE *ar; // array n x d of anomaly rates (0 - 100)
- long group; // how many collected values were grouped for each row
- int update_every; // what is the suggested update frequency in seconds
+ size_t group; // how many collected values were grouped for each row
+ time_t update_every; // what is the suggested update frequency in seconds
NETDATA_DOUBLE min;
NETDATA_DOUBLE max;
@@ -94,53 +93,57 @@ typedef struct rrdresult {
time_t before;
time_t after;
- bool st_locked_by_rrdr_create; // if st is read locked by us
-
// internal rrd2rrdr() members below this point
struct {
- int query_tier; // the selected tier
- RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query)
+ ONEWAYALLOC *owa; // the allocator used
+ struct query_target *qt; // the QUERY_TARGET
+
+ RRDR_OPTIONS query_options; // RRDR_OPTION_* (as run by the query)
- long points_wanted;
- long resampling_group;
- NETDATA_DOUBLE resampling_divisor;
+ size_t points_wanted; // used by SES and DES
+ size_t resampling_group; // used by AVERAGE
+ NETDATA_DOUBLE resampling_divisor; // used by AVERAGE
+ // grouping function pointers
void (*grouping_create)(struct rrdresult *r, const char *options);
void (*grouping_reset)(struct rrdresult *r);
void (*grouping_free)(struct rrdresult *r);
void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
- void *grouping_data;
- TIER_QUERY_FETCH tier_query_fetch;
- #ifdef NETDATA_INTERNAL_CHECKS
+ TIER_QUERY_FETCH tier_query_fetch; // which value to use from STORAGE_POINT
+ void *grouping_data; // the internal data of the grouping function
+
+#ifdef NETDATA_INTERNAL_CHECKS
const char *log;
- #endif
+#endif
+ // statistics
size_t db_points_read;
size_t result_points_generated;
size_t tier_points_read[RRD_STORAGE_TIERS];
- ONEWAYALLOC *owa;
} internal;
} RRDR;
#define rrdr_rows(r) ((r)->rows)
#include "database/rrd.h"
-extern void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
-extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list);
-extern RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points);
+void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
+RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt);
#include "../web_api_v1.h"
#include "web/api/queries/query.h"
-extern RRDR *rrd2rrdr(
- ONEWAYALLOC *owa,
- RRDSET *st, long points_wanted, long long after_wanted, long long before_wanted,
- RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
- struct context_param *context_param_list, const char *group_options, int timeout, int tier);
+RRDR *rrd2rrdr_legacy(
+ ONEWAYALLOC *owa,
+ RRDSET *st, size_t points, time_t after, time_t before,
+ RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,
+ const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source);
+
+RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
+bool query_target_calculate_window(struct query_target *qt);
-extern int rrdr_relative_window_to_absolute(long long *after, long long *before);
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before);
#ifdef __cplusplus
}
diff --git a/web/api/queries/ses/ses.h b/web/api/queries/ses/ses.h
index 094b8de3f..79b09fbdf 100644
--- a/web/api/queries/ses/ses.h
+++ b/web/api/queries/ses/ses.h
@@ -6,12 +6,12 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_init_ses(void);
+void grouping_init_ses(void);
-extern void grouping_create_ses(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_ses(RRDR *r);
-extern void grouping_free_ses(RRDR *r);
-extern void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_ses(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_ses(RRDR *r);
+void grouping_free_ses(RRDR *r);
+void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_SES_H
diff --git a/web/api/queries/stddev/stddev.h b/web/api/queries/stddev/stddev.h
index c5c91f88d..4b8ffcd53 100644
--- a/web/api/queries/stddev/stddev.h
+++ b/web/api/queries/stddev/stddev.h
@@ -6,13 +6,13 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_stddev(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_stddev(RRDR *r);
-extern void grouping_free_stddev(RRDR *r);
-extern void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
-extern NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
-// extern NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
-// extern NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_stddev(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_stddev(RRDR *r);
+void grouping_free_stddev(RRDR *r);
+void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+// NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+// NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_STDDEV_H
diff --git a/web/api/queries/sum/sum.h b/web/api/queries/sum/sum.h
index 4e7e396e9..898782775 100644
--- a/web/api/queries/sum/sum.h
+++ b/web/api/queries/sum/sum.h
@@ -6,10 +6,10 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_sum(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_sum(RRDR *r);
-extern void grouping_free_sum(RRDR *r);
-extern void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_sum(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_sum(RRDR *r);
+void grouping_free_sum(RRDR *r);
+void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERY_SUM_H
diff --git a/web/api/queries/trimmed_mean/trimmed_mean.h b/web/api/queries/trimmed_mean/trimmed_mean.h
index 1a4f63e9c..e66d92541 100644
--- a/web/api/queries/trimmed_mean/trimmed_mean.h
+++ b/web/api/queries/trimmed_mean/trimmed_mean.h
@@ -6,17 +6,17 @@
#include "../query.h"
#include "../rrdr.h"
-extern void grouping_create_trimmed_mean1(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean2(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean3(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean5(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean10(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean15(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean20(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean25(RRDR *r, const char *options);
-extern void grouping_reset_trimmed_mean(RRDR *r);
-extern void grouping_free_trimmed_mean(RRDR *r);
-extern void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_trimmed_mean1(RRDR *r, const char *options);
+void grouping_create_trimmed_mean2(RRDR *r, const char *options);
+void grouping_create_trimmed_mean3(RRDR *r, const char *options);
+void grouping_create_trimmed_mean5(RRDR *r, const char *options);
+void grouping_create_trimmed_mean10(RRDR *r, const char *options);
+void grouping_create_trimmed_mean15(RRDR *r, const char *options);
+void grouping_create_trimmed_mean20(RRDR *r, const char *options);
+void grouping_create_trimmed_mean25(RRDR *r, const char *options);
+void grouping_reset_trimmed_mean(RRDR *r);
+void grouping_free_trimmed_mean(RRDR *r);
+void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
#endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H
diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c
index 97a00f91c..a9555a66b 100644
--- a/web/api/queries/weights.c
+++ b/web/api/queries/weights.c
@@ -56,40 +56,14 @@ typedef enum {
struct register_result {
RESULT_FLAGS flags;
- RRDSET *st;
- const char *chart_id;
- const char *context;
- const char *dim_name;
+ RRDCONTEXT_ACQUIRED *rca;
+ RRDINSTANCE_ACQUIRED *ria;
+ RRDMETRIC_ACQUIRED *rma;
NETDATA_DOUBLE value;
-
- struct register_result *next; // used to link contexts together
};
-static void register_result_insert_callback(const char *name, void *value, void *data) {
- (void)name;
- (void)data;
-
- struct register_result *t = (struct register_result *)value;
-
- if(t->chart_id) t->chart_id = strdupz(t->chart_id);
- if(t->context) t->context = strdupz(t->context);
- if(t->dim_name) t->dim_name = strdupz(t->dim_name);
-}
-
-static void register_result_delete_callback(const char *name, void *value, void *data) {
- (void)name;
- (void)data;
- struct register_result *t = (struct register_result *)value;
-
- freez((void *)t->chart_id);
- freez((void *)t->context);
- freez((void *)t->dim_name);
-}
-
static DICTIONARY *register_result_init() {
- DICTIONARY *results = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
- dictionary_register_insert_callback(results, register_result_insert_callback, results);
- dictionary_register_delete_callback(results, register_result_delete_callback, results);
+ DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED);
return results;
}
@@ -98,8 +72,9 @@ static void register_result_destroy(DICTIONARY *results) {
}
static void register_result(DICTIONARY *results,
- RRDSET *st,
- RRDDIM *d,
+ RRDCONTEXT_ACQUIRED *rca,
+ RRDINSTANCE_ACQUIRED *ria,
+ RRDMETRIC_ACQUIRED *rma,
NETDATA_DOUBLE value,
RESULT_FLAGS flags,
WEIGHTS_STATS *stats,
@@ -120,25 +95,25 @@ static void register_result(DICTIONARY *results,
struct register_result t = {
.flags = flags,
- .st = st,
- .chart_id = st->id,
- .context = st->context,
- .dim_name = d->name,
+ .rca = rca,
+ .ria = ria,
+ .rma = rma,
.value = v
};
- char buf[5000 + 1];
- snprintfz(buf, 5000, "%s:%s", st->id, d->name);
- dictionary_set(results, buf, &t, sizeof(struct register_result));
+ // we can use the pointer address or RMA as a unique key for each metric
+ char buf[20 + 1];
+ ssize_t len = snprintfz(buf, 20, "%p", rma);
+ dictionary_set_advanced(results, buf, len + 1, &t, sizeof(struct register_result), NULL);
}
// ----------------------------------------------------------------------------
// Generation of JSON output for the results
static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *wb,
- long long after, long long before,
- long long baseline_after, long long baseline_before,
- long points, WEIGHTS_METHOD method,
+ time_t after, time_t before,
+ time_t baseline_after, time_t baseline_before,
+ size_t points, WEIGHTS_METHOD method,
RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
size_t examined_dimensions __maybe_unused, usec_t duration,
WEIGHTS_STATS *stats) {
@@ -147,10 +122,10 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
"\t\"after\": %lld,\n"
"\t\"before\": %lld,\n"
"\t\"duration\": %lld,\n"
- "\t\"points\": %ld,\n",
- after,
- before,
- before - after,
+ "\t\"points\": %zu,\n",
+ (long long)after,
+ (long long)before,
+ (long long)(before - after),
points
);
@@ -159,10 +134,10 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
"\t\"baseline_after\": %lld,\n"
"\t\"baseline_before\": %lld,\n"
"\t\"baseline_duration\": %lld,\n"
- "\t\"baseline_points\": %ld,\n",
- baseline_after,
- baseline_before,
- baseline_before - baseline_after,
+ "\t\"baseline_points\": %zu,\n",
+ (long long)baseline_after,
+ (long long)baseline_before,
+ (long long)(baseline_before - baseline_after),
points << shifts
);
@@ -181,7 +156,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
stats->db_points
);
- for(int tier = 0; tier < storage_tiers ;tier++)
+ for(size_t tier = 0; tier < storage_tiers ;tier++)
buffer_sprintf(wb, "%s%zu", tier?", ":"", stats->db_points_per_tier[tier]);
buffer_sprintf(wb, " ]\n"
@@ -193,13 +168,13 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
weights_method_to_string(method)
);
- web_client_api_request_v1_data_options_to_string(wb, options);
+ web_client_api_request_v1_data_options_to_buffer(wb, options);
}
static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
- long long after, long long before,
- long long baseline_after, long long baseline_before,
- long points, WEIGHTS_METHOD method,
+ time_t after, time_t before,
+ time_t baseline_after, time_t baseline_before,
+ size_t points, WEIGHTS_METHOD method,
RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
size_t examined_dimensions, usec_t duration,
WEIGHTS_STATS *stats) {
@@ -211,23 +186,23 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
size_t charts = 0, chart_dims = 0, total_dimensions = 0;
struct register_result *t;
- RRDSET *last_st = NULL; // never access this - we use it only for comparison
+ RRDINSTANCE_ACQUIRED *last_ria = NULL; // never access this - we use it only for comparison
dfe_start_read(results, t) {
- if(!last_st || t->st != last_st) {
- last_st = t->st;
+ if(t->ria != last_ria) {
+ last_ria = t->ria;
if(charts) buffer_strcat(wb, "\n\t\t\t}\n\t\t},\n");
buffer_strcat(wb, "\t\t\"");
- buffer_strcat(wb, t->chart_id);
+ buffer_strcat(wb, rrdinstance_acquired_id(t->ria));
buffer_strcat(wb, "\": {\n");
buffer_strcat(wb, "\t\t\t\"context\": \"");
- buffer_strcat(wb, t->context);
+ buffer_strcat(wb, rrdcontext_acquired_id(t->rca));
buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n");
charts++;
chart_dims = 0;
}
if (chart_dims) buffer_sprintf(wb, ",\n");
- buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, t->dim_name, t->value);
+ buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value);
chart_dims++;
total_dimensions++;
}
@@ -250,9 +225,9 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
}
static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *wb,
- long long after, long long before,
- long long baseline_after, long long baseline_before,
- long points, WEIGHTS_METHOD method,
+ time_t after, time_t before,
+ time_t baseline_after, time_t baseline_before,
+ size_t points, WEIGHTS_METHOD method,
RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
size_t examined_dimensions, usec_t duration,
WEIGHTS_STATS *stats) {
@@ -260,78 +235,80 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w
results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
points, method, group, options, shifts, examined_dimensions, duration, stats);
- DICTIONARY *context_results = dictionary_create(
- DICTIONARY_FLAG_SINGLE_THREADED
- |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE
- |DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
- |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
- );
+ buffer_strcat(wb, "\",\n\t\"contexts\": {\n");
+ size_t contexts = 0, charts = 0, total_dimensions = 0, context_dims = 0, chart_dims = 0;
+ NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0;
struct register_result *t;
+ RRDCONTEXT_ACQUIRED *last_rca = NULL;
+ RRDINSTANCE_ACQUIRED *last_ria = NULL;
dfe_start_read(results, t) {
- struct register_result *tc = dictionary_set(context_results, t->context, t, sizeof(*t));
- if(tc == t)
- t->next = NULL;
- else {
- t->next = tc->next;
- tc->next = t;
+
+ if(t->rca != last_rca) {
+ last_rca = t->rca;
+
+ if(contexts)
+ buffer_sprintf(wb, "\n"
+ "\t\t\t\t\t},\n"
+ "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+ "\t\t\t\t}\n\t\t\t},\n"
+ "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n"
+ , charts_total_weight / (double)chart_dims
+ , contexts_total_weight / (double)context_dims);
+
+ buffer_strcat(wb, "\t\t\"");
+ buffer_strcat(wb, rrdcontext_acquired_id(t->rca));
+ buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n");
+
+ contexts++;
+ charts = 0;
+ context_dims = 0;
+ contexts_total_weight = 0.0;
+
+ last_ria = NULL;
}
- }
- dfe_done(t);
- buffer_strcat(wb, "\",\n\t\"contexts\": {\n");
+ if(t->ria != last_ria) {
+ last_ria = t->ria;
- size_t contexts = 0, total_dimensions = 0, charts = 0, context_dims = 0, chart_dims = 0;
- NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0;
- RRDSET *last_st = NULL; // never access this - we use it only for comparison
- dfe_start_read(context_results, t) {
-
- if(contexts)
- buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims);
-
- contexts++;
- context_dims = 0;
- contexts_total_weight = 0.0;
-
- buffer_strcat(wb, "\t\t\"");
- buffer_strcat(wb, t->context);
- buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n");
-
- charts = 0;
- chart_dims = 0;
- struct register_result *tt;
- for(tt = t; tt ; tt = tt->next) {
- if(!last_st || tt->st != last_st) {
- last_st = tt->st;
-
- if(charts)
- buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t},\n", charts_total_weight / chart_dims);
-
- buffer_strcat(wb, "\t\t\t\t\"");
- buffer_strcat(wb, tt->chart_id);
- buffer_strcat(wb, "\": {\n");
- buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n");
- charts++;
- chart_dims = 0;
- charts_total_weight = 0.0;
- }
-
- if (chart_dims) buffer_sprintf(wb, ",\n");
- buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, tt->dim_name, tt->value);
- charts_total_weight += tt->value;
- contexts_total_weight += tt->value;
- chart_dims++;
- context_dims++;
- total_dimensions++;
+ if(charts)
+ buffer_sprintf(wb, "\n"
+ "\t\t\t\t\t},\n"
+ "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+ "\t\t\t\t},\n"
+ , charts_total_weight / (double)chart_dims);
+
+ buffer_strcat(wb, "\t\t\t\t\"");
+ buffer_strcat(wb, rrdinstance_acquired_id(t->ria));
+ buffer_strcat(wb, "\": {\n");
+ buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n");
+
+ charts++;
+ chart_dims = 0;
+ charts_total_weight = 0.0;
}
+
+ if (chart_dims) buffer_sprintf(wb, ",\n");
+ buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value);
+ charts_total_weight += t->value;
+ contexts_total_weight += t->value;
+ chart_dims++;
+ context_dims++;
+ total_dimensions++;
}
dfe_done(t);
- dictionary_destroy(context_results);
-
// close dimensions and chart
if (total_dimensions)
- buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t}\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims);
+ buffer_sprintf(wb, "\n"
+ "\t\t\t\t\t},\n"
+ "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+ "\t\t\t\t}\n"
+ "\t\t\t},\n"
+ "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+ "\t\t}\n"
+ , charts_total_weight / (double)chart_dims
+ , contexts_total_weight / (double)context_dims);
// close correlated_charts
buffer_sprintf(wb, "\t},\n"
@@ -391,7 +368,10 @@ static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, NETDATA_DOUBLE *arr, si
return added;
}
-static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMBERS highlight_diffs[], int high_size, uint32_t base_shifts) {
+static double ks_2samp(
+ DIFFS_NUMBERS baseline_diffs[], int base_size,
+ DIFFS_NUMBERS highlight_diffs[], int high_size,
+ uint32_t base_shifts) {
qsort(baseline_diffs, base_size, sizeof(DIFFS_NUMBERS), compare_diffs);
qsort(highlight_diffs, high_size, sizeof(DIFFS_NUMBERS), compare_diffs);
@@ -414,7 +394,7 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
// This would require a lot of multiplications and divisions.
//
// To speed it up, we do the binary search to find the index of each number
- // but then we divide the base index by the power of two number (shifts) it
+ // but, then we divide the base index by the power of two number (shifts) it
// is bigger than high index. So the 2 indexes are now comparable.
// We also keep track of the original indexes with min and max, to properly
// calculate their percentages once the loops finish.
@@ -495,7 +475,9 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
static double kstwo(
NETDATA_DOUBLE baseline[], int baseline_points,
- NETDATA_DOUBLE highlight[], int highlight_points, uint32_t base_shifts) {
+ NETDATA_DOUBLE highlight[], int highlight_points,
+ uint32_t base_shifts) {
+
// -1 in size, since the calculate_pairs_diffs() returns one less point
DIFFS_NUMBERS baseline_diffs[baseline_points - 1];
DIFFS_NUMBERS highlight_diffs[highlight_points - 1];
@@ -514,308 +496,215 @@ static double kstwo(
return ks_2samp(baseline_diffs, base_size, highlight_diffs, high_size, base_shifts);
}
+NETDATA_DOUBLE *rrd2rrdr_ks2(
+ ONEWAYALLOC *owa, RRDHOST *host,
+ RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+ time_t after, time_t before, size_t points, RRDR_OPTIONS options,
+ RRDR_GROUPING group_method, const char *group_options, size_t tier,
+ WEIGHTS_STATS *stats,
+ size_t *entries
+ ) {
+
+ NETDATA_DOUBLE *ret = NULL;
+
+ QUERY_TARGET_REQUEST qtr = {
+ .host = host,
+ .rca = rca,
+ .ria = ria,
+ .rma = rma,
+ .after = after,
+ .before = before,
+ .points = points,
+ .options = options,
+ .group_method = group_method,
+ .group_options = group_options,
+ .tier = tier,
+ .query_source = QUERY_SOURCE_API_WEIGHTS,
+ };
-static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
- long long baseline_after, long long baseline_before,
- long long after, long long before,
- long long points, RRDR_OPTIONS options,
- RRDR_GROUPING group, const char *group_options, int tier,
- uint32_t shifts, int timeout,
- WEIGHTS_STATS *stats, bool register_zero) {
- options |= RRDR_OPTION_NATURAL_POINTS;
-
- long group_time = 0;
- struct context_param *context_param_list = NULL;
-
- int examined_dimensions = 0;
-
- RRDR *high_rrdr = NULL;
- RRDR *base_rrdr = NULL;
-
- // get first the highlight to find the number of points available
- stats->db_queries++;
- usec_t started_usec = now_realtime_usec();
- ONEWAYALLOC *owa = onewayalloc_create(0);
- high_rrdr = rrd2rrdr(owa, st, points,
- after, before, group,
- group_time, options, NULL, context_param_list, group_options,
- timeout, tier);
- if(!high_rrdr) {
- info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name);
+ RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
+ if(!r)
goto cleanup;
- }
- for(int i = 0; i < storage_tiers ;i++)
- stats->db_points_per_tier[i] += high_rrdr->internal.tier_points_read[i];
+ stats->db_queries++;
+ stats->result_points += r->internal.result_points_generated;
+ stats->db_points += r->internal.db_points_read;
+ for(size_t tr = 0; tr < storage_tiers ; tr++)
+ stats->db_points_per_tier[tr] += r->internal.tier_points_read[tr];
- stats->db_points += high_rrdr->internal.db_points_read;
- stats->result_points += high_rrdr->internal.result_points_generated;
- if(!high_rrdr->d) {
- info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
+ if(r->d != 1) {
+ error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu", r->internal.qt->id, r->d);
goto cleanup;
}
- if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
- info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", st->name);
+
+ if(unlikely(r->od[0] & RRDR_DIMENSION_HIDDEN))
goto cleanup;
- }
- int high_points = rrdr_rows(high_rrdr);
- usec_t now_usec = now_realtime_usec();
- if(now_usec - started_usec > timeout * USEC_PER_MS)
+ if(unlikely(!(r->od[0] & RRDR_DIMENSION_NONZERO)))
goto cleanup;
- // get the baseline, requesting the same number of points as the highlight
- stats->db_queries++;
- base_rrdr = rrd2rrdr(owa, st,high_points << shifts,
- baseline_after, baseline_before, group,
- group_time, options, NULL, context_param_list, group_options,
- (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), tier);
- if(!base_rrdr) {
- info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name);
+ if(rrdr_rows(r) < 2)
goto cleanup;
- }
- for(int i = 0; i < storage_tiers ;i++)
- stats->db_points_per_tier[i] += base_rrdr->internal.tier_points_read[i];
+ *entries = rrdr_rows(r);
+ ret = onewayalloc_mallocz(owa, sizeof(NETDATA_DOUBLE) * rrdr_rows(r));
- stats->db_points += base_rrdr->internal.db_points_read;
- stats->result_points += base_rrdr->internal.result_points_generated;
- if(!base_rrdr->d) {
- info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
- goto cleanup;
- }
- if (base_rrdr->d != high_rrdr->d) {
- info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", st->name);
- goto cleanup;
- }
- if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
- info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", st->name);
- goto cleanup;
- }
- int base_points = rrdr_rows(base_rrdr);
+ // copy the points of the dimension to a contiguous array
+ // there is no need to check for empty values, since empty values are already zero
+ // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
+ memcpy(ret, r->v, rrdr_rows(r) * sizeof(NETDATA_DOUBLE));
- now_usec = now_realtime_usec();
- if(now_usec - started_usec > timeout * USEC_PER_MS)
- goto cleanup;
+cleanup:
+ rrdr_free(owa, r);
+ return ret;
+}
+
+static void rrdset_metric_correlations_ks2(
+ RRDHOST *host,
+ RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+ DICTIONARY *results,
+ time_t baseline_after, time_t baseline_before,
+ time_t after, time_t before,
+ size_t points, RRDR_OPTIONS options,
+ RRDR_GROUPING group_method, const char *group_options, size_t tier,
+ uint32_t shifts,
+ WEIGHTS_STATS *stats, bool register_zero
+ ) {
+
+ options |= RRDR_OPTION_NATURAL_POINTS;
- // we need at least 2 points to do the job
- if(base_points < 2 || high_points < 2)
+ ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
+
+ size_t high_points = 0;
+ NETDATA_DOUBLE *highlight = rrd2rrdr_ks2(
+ owa, host, rca, ria, rma, after, before, points,
+ options, group_method, group_options, tier, stats, &high_points);
+
+ if(!highlight)
goto cleanup;
- // for each dimension
- RRDDIM *d;
- int i;
- for(i = 0, d = base_rrdr->st->dimensions ; d && i < base_rrdr->d; i++, d = d->next) {
+ size_t base_points = 0;
+ NETDATA_DOUBLE *baseline = rrd2rrdr_ks2(
+ owa, host, rca, ria, rma, baseline_after, baseline_before, high_points << shifts,
+ options, group_method, group_options, tier, stats, &base_points);
+
+ if(!baseline)
+ goto cleanup;
- // skip the not evaluated ones
- if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN))
- continue;
+ stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1);
- examined_dimensions++;
+ double prob = kstwo(baseline, (int)base_points, highlight, (int)high_points, shifts);
+ if(!isnan(prob) && !isinf(prob)) {
- // skip the dimensions that are just zero for both the baseline and the highlight
- if(unlikely(!(base_rrdr->od[i] & RRDR_DIMENSION_NONZERO) && !(high_rrdr->od[i] & RRDR_DIMENSION_NONZERO)))
- continue;
-
- // copy the baseline points of the dimension to a contiguous array
- // there is no need to check for empty values, since empty are already zero
- NETDATA_DOUBLE baseline[base_points];
- for(int c = 0; c < base_points; c++)
- baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ];
-
- // copy the highlight points of the dimension to a contiguous array
- // there is no need to check for empty values, since empty values are already zero
- // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
- NETDATA_DOUBLE highlight[high_points];
- for(int c = 0; c < high_points; c++)
- highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ];
-
- stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1);
-
- double prob = kstwo(baseline, base_points, highlight, high_points, shifts);
- if(!isnan(prob) && !isinf(prob)) {
-
- // these conditions should never happen, but still let's check
- if(unlikely(prob < 0.0)) {
- error("Metric correlations: kstwo() returned a negative number: %f", prob);
- prob = -prob;
- }
- if(unlikely(prob > 1.0)) {
- error("Metric correlations: kstwo() returned a number above 1.0: %f", prob);
- prob = 1.0;
- }
-
- // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated
- // so we flip the result of kstwo()
- register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero);
+ // these conditions should never happen, but still let's check
+ if(unlikely(prob < 0.0)) {
+ error("Metric correlations: kstwo() returned a negative number: %f", prob);
+ prob = -prob;
+ }
+ if(unlikely(prob > 1.0)) {
+ error("Metric correlations: kstwo() returned a number above 1.0: %f", prob);
+ prob = 1.0;
}
+
+ // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated
+ // so, we flip the result of kstwo()
+ register_result(results, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero);
}
cleanup:
- rrdr_free(owa, high_rrdr);
- rrdr_free(owa, base_rrdr);
onewayalloc_destroy(owa);
- return examined_dimensions;
}
// ----------------------------------------------------------------------------
// VOLUME algorithm functions
-static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
- long long baseline_after, long long baseline_before,
- long long after, long long before,
- RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,
- int tier, int timeout,
- WEIGHTS_STATS *stats, bool register_zero) {
+static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats) {
+ stats->db_queries++;
+ stats->result_points += qv->result_points;
+ stats->db_points += qv->points_read;
+ for(size_t tier = 0; tier < storage_tiers ; tier++)
+ stats->db_points_per_tier[tier] += qv->storage_points_per_tier[tier];
+}
+
+static void rrdset_metric_correlations_volume(
+ RRDHOST *host,
+ RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+ DICTIONARY *results,
+ time_t baseline_after, time_t baseline_before,
+ time_t after, time_t before,
+ RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
+ size_t tier,
+ WEIGHTS_STATS *stats, bool register_zero) {
options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS;
- long group_time = 0;
-
- int examined_dimensions = 0;
- int ret, value_is_null;
- usec_t started_usec = now_realtime_usec();
- RRDDIM *d;
- for(d = st->dimensions; d ; d = d->next) {
- usec_t now_usec = now_realtime_usec();
- if(now_usec - started_usec > timeout * USEC_PER_MS)
- return examined_dimensions;
+ QUERY_VALUE baseline_average = rrdmetric2value(host, rca, ria, rma, baseline_after, baseline_before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+ merge_query_value_to_stats(&baseline_average, stats);
- // we count how many metrics we evaluated
- examined_dimensions++;
+ if(!netdata_double_isnumber(baseline_average.value)) {
+ // this means no data for the baseline window, but we may have data for the highlighted one - assume zero
+ baseline_average.value = 0.0;
+ }
- // there is no point to pass a timeout to these queries
- // since the query engine checks for a timeout between
- // dimensions, and we query a single dimension at a time.
-
- stats->db_queries++;
- NETDATA_DOUBLE baseline_average = NAN;
- NETDATA_DOUBLE base_anomaly_rate = 0;
- value_is_null = 1;
- ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,
- baseline_after, baseline_before,
- group, group_options, group_time, options,
- NULL, NULL,
- &stats->db_points, stats->db_points_per_tier,
- &stats->result_points,
- &value_is_null, &base_anomaly_rate, 0, tier);
-
- if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) {
- // this means no data for the baseline window, but we may have data for the highlighted one - assume zero
- baseline_average = 0.0;
- }
+ QUERY_VALUE highlight_average = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+ merge_query_value_to_stats(&highlight_average, stats);
- stats->db_queries++;
- NETDATA_DOUBLE highlight_average = NAN;
- NETDATA_DOUBLE high_anomaly_rate = 0;
- value_is_null = 1;
- ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1,
- after, before,
- group, group_options, group_time, options,
- NULL, NULL,
- &stats->db_points, stats->db_points_per_tier,
- &stats->result_points,
- &value_is_null, &high_anomaly_rate, 0, tier);
-
- if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) {
- // this means no data for the highlighted duration - so skip it
- continue;
- }
+ if(!netdata_double_isnumber(highlight_average.value))
+ return;
- if(baseline_average == highlight_average) {
- // they are the same - let's move on
- continue;
- }
+ if(baseline_average.value == highlight_average.value) {
+ // they are the same - let's move on
+ return;
+ }
- stats->db_queries++;
- NETDATA_DOUBLE highlight_countif = NAN;
- value_is_null = 1;
-
- char highlighted_countif_options[50 + 1];
- snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);
-
- ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1,
- after, before,
- RRDR_GROUPING_COUNTIF,highlighted_countif_options,
- group_time, options,
- NULL, NULL,
- &stats->db_points, stats->db_points_per_tier,
- &stats->result_points,
- &value_is_null, NULL, 0, tier);
-
- if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) {
- info("MC: highlighted countif query failed, but highlighted average worked - strange...");
- continue;
- }
+ char highlight_countif_options[50 + 1];
+ snprintfz(highlight_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average.value < baseline_average.value ? "<" : ">", baseline_average.value);
+ QUERY_VALUE highlight_countif = rrdmetric2value(host, rca, ria, rma, after, before, options, RRDR_GROUPING_COUNTIF, highlight_countif_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+ merge_query_value_to_stats(&highlight_countif, stats);
- // this represents the percentage of time
- // the highlighted window was above/below the baseline window
- // (above or below depending on their averages)
- highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0
+ if(!netdata_double_isnumber(highlight_countif.value)) {
+ info("WEIGHTS: highlighted countif query failed, but highlighted average worked - strange...");
+ return;
+ }
- RESULT_FLAGS flags;
- NETDATA_DOUBLE pcent = NAN;
- if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) {
- flags = RESULT_IS_BASE_HIGH_RATIO;
- pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif;
- }
- else {
- flags = RESULT_IS_PERCENTAGE_OF_TIME;
- pcent = highlight_countif;
- }
+ // this represents the percentage of time
+ // the highlighted window was above/below the baseline window
+ // (above or below depending on their averages)
+ highlight_countif.value = highlight_countif.value / 100.0; // countif returns 0 - 100.0
- register_result(results, st, d, pcent, flags, stats, register_zero);
+ RESULT_FLAGS flags;
+ NETDATA_DOUBLE pcent = NAN;
+ if(isgreater(baseline_average.value, 0.0) || isless(baseline_average.value, 0.0)) {
+ flags = RESULT_IS_BASE_HIGH_RATIO;
+ pcent = (highlight_average.value - baseline_average.value) / baseline_average.value * highlight_countif.value;
+ }
+ else {
+ flags = RESULT_IS_PERCENTAGE_OF_TIME;
+ pcent = highlight_countif.value;
}
- return examined_dimensions;
+ register_result(results, rca, ria, rma, pcent, flags, stats, register_zero);
}
// ----------------------------------------------------------------------------
// ANOMALY RATE algorithm functions
-static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results,
- long long after, long long before,
- RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,
- int tier, int timeout,
- WEIGHTS_STATS *stats, bool register_zero) {
+static void rrdset_weights_anomaly_rate(
+ RRDHOST *host,
+ RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+ DICTIONARY *results,
+ time_t after, time_t before,
+ RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
+ size_t tier,
+ WEIGHTS_STATS *stats, bool register_zero) {
options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ANOMALY_BIT | RRDR_OPTION_NATURAL_POINTS;
- long group_time = 0;
- int examined_dimensions = 0;
- int ret, value_is_null;
- usec_t started_usec = now_realtime_usec();
-
- RRDDIM *d;
- for(d = st->dimensions; d ; d = d->next) {
- usec_t now_usec = now_realtime_usec();
- if(now_usec - started_usec > timeout * USEC_PER_MS)
- return examined_dimensions;
+ QUERY_VALUE qv = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+ merge_query_value_to_stats(&qv, stats);
- // we count how many metrics we evaluated
- examined_dimensions++;
-
- // there is no point to pass a timeout to these queries
- // since the query engine checks for a timeout between
- // dimensions, and we query a single dimension at a time.
-
- stats->db_queries++;
- NETDATA_DOUBLE average = NAN;
- NETDATA_DOUBLE anomaly_rate = 0;
- value_is_null = 1;
- ret = rrdset2value_api_v1(st, NULL, &average, d->id, 1,
- after, before,
- group, group_options, group_time, options,
- NULL, NULL,
- &stats->db_points, stats->db_points_per_tier,
- &stats->result_points,
- &value_is_null, &anomaly_rate, 0, tier);
-
- if(ret == HTTP_RESP_OK || !value_is_null || netdata_double_isnumber(average))
- register_result(results, st, d, average, 0, stats, register_zero);
- }
-
- return examined_dimensions;
+ if(netdata_double_isnumber(qv.value))
+ register_result(results, rca, ria, rma, qv.value, 0, stats, register_zero);
}
// ----------------------------------------------------------------------------
@@ -853,7 +742,7 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) {
struct register_result *t;
// count the dimensions
- size_t dimensions = dictionary_stats_entries(results);
+ size_t dimensions = dictionary_entries(results);
if(!dimensions) return 0;
if(stats->max_base_high_ratio == 0.0)
@@ -903,15 +792,17 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) {
// ----------------------------------------------------------------------------
// The main function
-int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
- RRDR_GROUPING group, const char *group_options,
- long long baseline_after, long long baseline_before,
- long long after, long long before,
- long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout) {
+int web_api_v1_weights(
+ RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
+ RRDR_GROUPING group, const char *group_options,
+ time_t baseline_after, time_t baseline_before,
+ time_t after, time_t before,
+ size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout) {
+
WEIGHTS_STATS stats = {};
DICTIONARY *results = register_result_init();
- DICTIONARY *charts = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);;
+ DICTIONARY *metrics = NULL;
char *error = NULL;
int resp = HTTP_RESP_OK;
@@ -1000,20 +891,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
baseline_after = baseline_before - (high_delta << shifts);
}
- // dont lock here and wait for results
- // get the charts and run mc after
- RRDSET *st;
- rrdhost_rdlock(host);
- rrdset_foreach_read(st, host) {
- if (rrdset_is_available_for_viewers(st)) {
- if(!contexts || simple_pattern_matches(contexts, st->context))
- dictionary_set(charts, st->name, NULL, 0);
- }
- }
- rrdhost_unlock(host);
-
size_t examined_dimensions = 0;
- void *ptr;
bool register_zero = true;
if(options & RRDR_OPTION_NONZERO) {
@@ -1021,8 +899,11 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
options &= ~RRDR_OPTION_NONZERO;
}
- // for every chart in the dictionary
- dfe_start_read(charts, ptr) {
+ metrics = rrdcontext_all_metrics_to_dict(host, contexts);
+ struct metric_entry *me;
+
+ // for every metric_entry in the dictionary
+ dfe_start_read(metrics, me) {
usec_t now_usec = now_realtime_usec();
if(now_usec - started_usec > timeout_usec) {
error = "timed out";
@@ -1030,46 +911,48 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
goto cleanup;
}
- st = rrdset_find_byname(host, ptr_name);
- if(!st) continue;
-
- rrdset_rdlock(st);
+ examined_dimensions++;
switch(method) {
case WEIGHTS_METHOD_ANOMALY_RATE:
options |= RRDR_OPTION_ANOMALY_BIT;
- points = 1;
- examined_dimensions += rrdset_weights_anomaly_rate(st, results,
- after, before,
- options, group, group_options, tier,
- (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
- &stats, register_zero);
+ rrdset_weights_anomaly_rate(
+ host,
+ me->rca, me->ria, me->rma,
+ results,
+ after, before,
+ options, group, group_options, tier,
+ &stats, register_zero
+ );
break;
case WEIGHTS_METHOD_MC_VOLUME:
- points = 1;
- examined_dimensions += rrdset_metric_correlations_volume(st, results,
- baseline_after, baseline_before,
- after, before,
- options, group, group_options, tier,
- (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
- &stats, register_zero);
+ rrdset_metric_correlations_volume(
+ host,
+ me->rca, me->ria, me->rma,
+ results,
+ baseline_after, baseline_before,
+ after, before,
+ options, group, group_options, tier,
+ &stats, register_zero
+ );
break;
default:
case WEIGHTS_METHOD_MC_KS2:
- examined_dimensions += rrdset_metric_correlations_ks2(st, results,
- baseline_after, baseline_before,
- after, before,
- points, options, group, group_options, tier, shifts,
- (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
- &stats, register_zero);
+ rrdset_metric_correlations_ks2(
+ host,
+ me->rca, me->ria, me->rma,
+ results,
+ baseline_after, baseline_before,
+ after, before, points,
+ options, group, group_options, tier, shifts,
+ &stats, register_zero
+ );
break;
}
-
- rrdset_unlock(st);
}
- dfe_done(ptr);
+ dfe_done(me);
if(!register_zero)
options |= RRDR_OPTION_NONZERO;
@@ -1085,22 +968,26 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
size_t added_dimensions = 0;
switch(format) {
case WEIGHTS_FORMAT_CHARTS:
- added_dimensions = registered_results_to_json_charts(results, wb,
- after, before,
- baseline_after, baseline_before,
- points, method, group, options, shifts,
- examined_dimensions,
- ended_usec - started_usec, &stats);
+ added_dimensions =
+ registered_results_to_json_charts(
+ results, wb,
+ after, before,
+ baseline_after, baseline_before,
+ points, method, group, options, shifts,
+ examined_dimensions,
+ ended_usec - started_usec, &stats);
break;
default:
case WEIGHTS_FORMAT_CONTEXTS:
- added_dimensions = registered_results_to_json_contexts(results, wb,
- after, before,
- baseline_after, baseline_before,
- points, method, group, options, shifts,
- examined_dimensions,
- ended_usec - started_usec, &stats);
+ added_dimensions =
+ registered_results_to_json_contexts(
+ results, wb,
+ after, before,
+ baseline_after, baseline_before,
+ points, method, group, options, shifts,
+ examined_dimensions,
+ ended_usec - started_usec, &stats);
break;
}
@@ -1110,7 +997,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
}
cleanup:
- if(charts) dictionary_destroy(charts);
+ if(metrics) dictionary_destroy(metrics);
if(results) register_result_destroy(results);
if(error) {
diff --git a/web/api/queries/weights.h b/web/api/queries/weights.h
index f88a134f2..50d8634ef 100644
--- a/web/api/queries/weights.h
+++ b/web/api/queries/weights.h
@@ -20,14 +20,14 @@ extern int enable_metric_correlations;
extern int metric_correlations_version;
extern WEIGHTS_METHOD default_metric_correlations_method;
-extern int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
+int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
RRDR_GROUPING group, const char *group_options,
- long long baseline_after, long long baseline_before,
- long long after, long long before,
- long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout);
+ time_t baseline_after, time_t baseline_before,
+ time_t after, time_t before,
+ size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout);
-extern WEIGHTS_METHOD weights_string_to_method(const char *method);
-extern const char *weights_method_to_string(WEIGHTS_METHOD method);
-extern int mc_unittest(void);
+WEIGHTS_METHOD weights_string_to_method(const char *method);
+const char *weights_method_to_string(WEIGHTS_METHOD method);
+int mc_unittest(void);
#endif //NETDATA_API_WEIGHTS_H