Adding upstream version 1.37.0.upstream/1.37.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

46 files changed, 1768 insertions, 1896 deletions

diff --git a/web/api/badges/web_buffer_svg.c b/web/api/badges/web_buffer_svg.c
index 00b4ad650..080f2240f 100644
--- a/web/api/badges/web_buffer_svg.c
+++ b/web/api/badges/web_buffer_svg.c
@@ -893,6 +893,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
     int group = RRDR_GROUPING_AVERAGE;
     uint32_t options = 0x00000000;
 
+    const RRDCALC_ACQUIRED *rca = NULL;
+    RRDCALC *rc = NULL;
+    RRDSET *st = NULL;
+
     while(url) {
         char *value = mystrsep(&url, "&");
         if(!value || !*value) continue;
@@ -957,7 +961,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
 
     int scale = (scale_str && *scale_str)?str2i(scale_str):100;
 
-    RRDSET *st = rrdset_find(host, chart);
+    st = rrdset_find(host, chart);
     if(!st) st = rrdset_find_byname(host, chart);
     if(!st) {
         buffer_no_cacheable(w->response.data);
@@ -967,9 +971,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
     }
     st->last_accessed_time = now_realtime_sec();
 
-    RRDCALC *rc = NULL;
     if(alarm) {
-        rc = rrdcalc_find(st, alarm);
+        rca = rrdcalc_from_rrdset_get(st, alarm);
+        rc = rrdcalc_acquired_to_rrdcalc(rca);
+
         if (!rc) {
             buffer_no_cacheable(w->response.data);
             buffer_svg(w->response.data, "alarm not found", NAN, "", NULL, NULL, -1, scale, 0, -1, -1, NULL, NULL);
@@ -1020,19 +1025,19 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
             label = dim;
         }
         else
-            label = st->name;
+            label = rrdset_name(st);
     }
     if(!units) {
         if(alarm) {
             if(rc->units)
-                units = rc->units;
+                units = rrdcalc_units(rc);
             else
                 units = "";
         }
         else if(options & RRDR_OPTION_PERCENTAGE)
             units = "%";
         else
-            units = st->units;
+            units = rrdset_units(st);
     }
 
     debug(D_WEB_CLIENT, "%llu: API command 'badge.svg' for chart '%s', alarm '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', options '0x%08x'"
@@ -1111,7 +1116,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
                                       points, after, before, group, group_options, 0, options,
                                       NULL, &latest_timestamp,
                                       NULL, NULL, NULL,
-                                      &value_is_null, NULL, 0, 0);
+                                      &value_is_null, NULL, 0, 0,
+                                      QUERY_SOURCE_API_BADGE);
 
         // if the value cannot be calculated, show empty badge
         if (ret != HTTP_RESP_OK) {
@@ -1143,7 +1149,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
         );
     }
 
-    cleanup:
+cleanup:
+    rrdcalc_from_rrdset_release(st, rca);
     buffer_free(dimensions);
     return ret;
 }
diff --git a/web/api/badges/web_buffer_svg.h b/web/api/badges/web_buffer_svg.h
index 4853a8864..71857811f 100644
--- a/web/api/badges/web_buffer_svg.h
+++ b/web/api/badges/web_buffer_svg.h
@@ -6,12 +6,12 @@
 #include "libnetdata/libnetdata.h"
 #include "web/server/web_client.h"
 
-extern void buffer_svg(BUFFER *wb, const char *label,
+void buffer_svg(BUFFER *wb, const char *label,
     NETDATA_DOUBLE value, const char *units, const char *label_color, const char *value_color, int precision, int scale, uint32_t options, int fixed_width_lbl, int fixed_width_val, const char* text_color_lbl, const char* text_color_val);
-extern char *format_value_and_unit(char *value_string, size_t value_string_len,
+char *format_value_and_unit(char *value_string, size_t value_string_len,
     NETDATA_DOUBLE value, const char *units, int precision);
 
-extern int web_client_api_request_v1_badge(struct rrdhost *host, struct web_client *w, char *url);
+int web_client_api_request_v1_badge(struct rrdhost *host, struct web_client *w, char *url);
 
 #include "web/api/web_api_v1.h"
 
diff --git a/web/api/exporters/allmetrics.h b/web/api/exporters/allmetrics.h
index f076ff0d5..3afc42e28 100644
--- a/web/api/exporters/allmetrics.h
+++ b/web/api/exporters/allmetrics.h
@@ -7,6 +7,6 @@
 #include "shell/allmetrics_shell.h"
 #include "web/server/web_client.h"
 
-extern int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url);
 
 #endif //NETDATA_API_ALLMETRICS_H
diff --git a/web/api/exporters/shell/allmetrics_shell.c b/web/api/exporters/shell/allmetrics_shell.c
index 615aab43c..0ffbac67b 100644
--- a/web/api/exporters/shell/allmetrics_shell.c
+++ b/web/api/exporters/shell/allmetrics_shell.c
@@ -25,73 +25,71 @@ static inline size_t shell_name_copy(char *d, const char *s, size_t usable) {
 void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb) {
     analytics_log_shell();
     SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT);
-    rrdhost_rdlock(host);
 
     // for each chart
     RRDSET *st;
     rrdset_foreach_read(st, host) {
-        if (filter && !simple_pattern_matches(filter, st->name))
+        if (filter && !simple_pattern_matches(filter, rrdset_name(st)))
             continue;
 
         NETDATA_DOUBLE total = 0.0;
         char chart[SHELL_ELEMENT_MAX + 1];
-        shell_name_copy(chart, st->name?st->name:st->id, SHELL_ELEMENT_MAX);
+        shell_name_copy(chart, st->name?rrdset_name(st):rrdset_id(st), SHELL_ELEMENT_MAX);
 
-        buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name);
+        buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", rrdset_id(st), rrdset_name(st));
         if(rrdset_is_available_for_viewers(st)) {
-            rrdset_rdlock(st);
-
             // for each dimension
             RRDDIM *rd;
             rrddim_foreach_read(rd, st) {
                 if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
                     char dimension[SHELL_ELEMENT_MAX + 1];
-                    shell_name_copy(dimension, rd->name?rd->name:rd->id, SHELL_ELEMENT_MAX);
+                    shell_name_copy(dimension, rd->name?rrddim_name(rd):rrddim_id(rd), SHELL_ELEMENT_MAX);
 
                     NETDATA_DOUBLE n = rd->last_stored_value;
 
                     if(isnan(n) || isinf(n))
-                        buffer_sprintf(wb, "NETDATA_%s_%s=\"\"      # %s\n", chart, dimension, st->units);
+                        buffer_sprintf(wb, "NETDATA_%s_%s=\"\"      # %s\n", chart, dimension, rrdset_units(st));
                     else {
                         if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
                         n = roundndd(n);
-                        if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
-                        buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\"      # %s\n", chart, dimension, n, st->units);
+                        if(!rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)) total += n;
+                        buffer_sprintf(wb, "NETDATA_%s_%s=\"" NETDATA_DOUBLE_FORMAT_ZERO "\"      # %s\n", chart, dimension, n, rrdset_units(st));
                     }
                 }
             }
+            rrddim_foreach_done(rd);
 
             total = roundndd(total);
-            buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\"      # %s\n", chart, total, st->units);
-            rrdset_unlock(st);
+            buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"" NETDATA_DOUBLE_FORMAT_ZERO "\"      # %s\n", chart, total, rrdset_units(st));
         }
     }
+    rrdset_foreach_done(st);
 
     buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n");
 
     RRDCALC *rc;
-    for(rc = host->alarms; rc ;rc = rc->next) {
+    foreach_rrdcalc_in_rrdhost_read(host, rc) {
         if(!rc->rrdset) continue;
 
         char chart[SHELL_ELEMENT_MAX + 1];
-        shell_name_copy(chart, rc->rrdset->name?rc->rrdset->name:rc->rrdset->id, SHELL_ELEMENT_MAX);
+        shell_name_copy(chart, rc->rrdset->name?rrdset_name(rc->rrdset):rrdset_id(rc->rrdset), SHELL_ELEMENT_MAX);
 
         char alarm[SHELL_ELEMENT_MAX + 1];
-        shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
+        shell_name_copy(alarm, rrdcalc_name(rc), SHELL_ELEMENT_MAX);
 
         NETDATA_DOUBLE n = rc->value;
 
         if(isnan(n) || isinf(n))
-            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\"      # %s\n", chart, alarm, rc->units);
+            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\"      # %s\n", chart, alarm, rrdcalc_units(rc));
         else {
             n = roundndd(n);
-            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\"      # %s\n", chart, alarm, n, rc->units);
+            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"" NETDATA_DOUBLE_FORMAT_ZERO "\"      # %s\n", chart, alarm, n, rrdcalc_units(rc));
         }
 
         buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
     }
+    foreach_rrdcalc_in_rrdhost_done(rc);
 
-    rrdhost_unlock(host);
     simple_pattern_free(filter);
 }
 
@@ -100,7 +98,6 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_
 void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb) {
     analytics_log_json();
     SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT);
-    rrdhost_rdlock(host);
 
     buffer_strcat(wb, "{");
 
@@ -110,12 +107,10 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
     // for each chart
     RRDSET *st;
     rrdset_foreach_read(st, host) {
-        if (filter && !(simple_pattern_matches(filter, st->id) || simple_pattern_matches(filter, st->name)))
+        if (filter && !(simple_pattern_matches(filter, rrdset_id(st)) || simple_pattern_matches(filter, rrdset_name(st))))
             continue;
 
         if(rrdset_is_available_for_viewers(st)) {
-            rrdset_rdlock(st);
-
             buffer_sprintf(
                 wb,
                 "%s\n"
@@ -127,12 +122,12 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
                 "\t\t\"last_updated\": %"PRId64",\n"
                 "\t\t\"dimensions\": {",
                 chart_counter ? "," : "",
-                st->id,
-                st->name,
-                st->family,
-                st->context,
-                st->units,
-                (int64_t)rrdset_last_entry_t_nolock(st));
+                rrdset_id(st),
+                rrdset_name(st),
+                rrdset_family(st),
+                rrdset_context(st),
+                rrdset_units(st),
+                (int64_t)rrdset_last_entry_t(st));
 
             chart_counter++;
             dimension_counter = 0;
@@ -148,8 +143,8 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
                         "\t\t\t\t\"name\": \"%s\",\n"
                         "\t\t\t\t\"value\": ",
                         dimension_counter ? "," : "",
-                        rd->id,
-                        rd->name);
+                        rrddim_id(rd),
+                        rrddim_name(rd));
 
                     if(isnan(rd->last_stored_value))
                         buffer_strcat(wb, "null");
@@ -161,14 +156,14 @@ void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_s
                     dimension_counter++;
                 }
             }
+            rrddim_foreach_done(rd);
 
             buffer_strcat(wb, "\n\t\t}\n\t}");
-            rrdset_unlock(st);
         }
     }
+    rrdset_foreach_done(st);
 
     buffer_strcat(wb, "\n}");
-    rrdhost_unlock(host);
     simple_pattern_free(filter);
 }
 
diff --git a/web/api/exporters/shell/allmetrics_shell.h b/web/api/exporters/shell/allmetrics_shell.h
index 1ee9aa717..d6598e08d 100644
--- a/web/api/exporters/shell/allmetrics_shell.h
+++ b/web/api/exporters/shell/allmetrics_shell.h
@@ -15,7 +15,7 @@
 #define ALLMETRICS_JSON                         3
 #define ALLMETRICS_PROMETHEUS_ALL_HOSTS         4
 
-extern void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb);
-extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb);
+void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, const char *filter_string, BUFFER *wb);
+void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, const char *filter_string, BUFFER *wb);
 
 #endif //NETDATA_API_ALLMETRICS_SHELL_H
diff --git a/web/api/formatters/charts2json.c b/web/api/formatters/charts2json.c
index 4325b6530..1fc20b493 100644
--- a/web/api/formatters/charts2json.c
+++ b/web/api/formatters/charts2json.c
@@ -57,11 +57,11 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
                        ",\n\t\"memory_mode\": \"%s\""
                        ",\n\t\"custom_info\": \"%s\""
                        ",\n\t\"charts\": {"
-                   , host->hostname
-                   , host->program_version
+                   , rrdhost_hostname(host)
+                   , rrdhost_program_version(host)
                    , get_release_channel()
-                   , host->os
-                   , host->timezone
+                   , rrdhost_os(host)
+                   , rrdhost_timezone(host)
                    , host->rrd_update_every
                    , host->rrd_history_entries
                    , rrd_memory_mode_name(host->rrd_memory_mode)
@@ -69,12 +69,11 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
     );
 
     c = 0;
-    rrdhost_rdlock(host);
     rrdset_foreach_read(st, host) {
         if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) {
             if(c) buffer_strcat(wb, ",");
             buffer_strcat(wb, "\n\t\t\"");
-            buffer_strcat(wb, st->id);
+            buffer_strcat(wb, rrdset_id(st));
             buffer_strcat(wb, "\": ");
             rrdset2json(st, wb, &dimensions, &memory, skip_volatile);
 
@@ -82,13 +81,14 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
             st->last_accessed_time = now;
         }
     }
+    rrdset_foreach_done(st);
 
     RRDCALC *rc;
-    for(rc = host->alarms; rc ; rc = rc->next) {
+    foreach_rrdcalc_in_rrdhost_read(host, rc) {
         if(rc->rrdset)
             alarms++;
     }
-    rrdhost_unlock(host);
+    foreach_rrdcalc_in_rrdhost_done(rc);
 
     buffer_sprintf(wb
                    , "\n\t}"
@@ -117,7 +117,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
                                  "\n\t\t\t\"hostname\": \"%s\""
                                  "\n\t\t}"
                                , (found > 0) ? "," : ""
-                               , h->hostname
+                               , rrdhost_hostname(h)
                 );
 
                 found++;
@@ -131,7 +131,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
                        , "\n\t\t{"
                          "\n\t\t\t\"hostname\": \"%s\""
                          "\n\t\t}"
-                       , host->hostname
+                       , rrdhost_hostname(host)
         );
     }
 
@@ -141,8 +141,8 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived
 // generate collectors list for the api/v1/info call
 
 struct collector {
-    char *plugin;
-    char *module;
+    const char *plugin;
+    const char *module;
 };
 
 struct array_printer {
@@ -150,9 +150,7 @@ struct array_printer {
     BUFFER *wb;
 };
 
-static int print_collector_callback(const char *name, void *entry, void *data) {
-    (void)name;
-
+static int print_collector_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) {
     struct array_printer *ap = (struct array_printer *)data;
     BUFFER *wb = ap->wb;
     struct collector *col=(struct collector *) entry;
@@ -167,24 +165,23 @@ static int print_collector_callback(const char *name, void *entry, void *data) {
 }
 
 void chartcollectors2json(RRDHOST *host, BUFFER *wb) {
-    DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
+    DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
     RRDSET *st;
     char name[500];
 
     time_t now = now_realtime_sec();
-    rrdhost_rdlock(host);
     rrdset_foreach_read(st, host) {
         if (rrdset_is_available_for_viewers(st)) {
             struct collector col = {
-                    .plugin = st->plugin_name ? st->plugin_name : "",
-                    .module = st->module_name ? st->module_name : ""
+                    .plugin = rrdset_plugin_name(st),
+                    .module = rrdset_module_name(st)
             };
             sprintf(name, "%s:%s", col.plugin, col.module);
             dictionary_set(dict, name, &col, sizeof(struct collector));
             st->last_accessed_time = now;
         }
     }
-    rrdhost_unlock(host);
+    rrdset_foreach_done(st);
     struct array_printer ap = {
             .c = 0,
             .wb = wb
diff --git a/web/api/formatters/charts2json.h b/web/api/formatters/charts2json.h
index 2d8cce310..d4b04af58 100644
--- a/web/api/formatters/charts2json.h
+++ b/web/api/formatters/charts2json.h
@@ -5,8 +5,8 @@
 
 #include "rrd2json.h"
 
-extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived);
-extern void chartcollectors2json(RRDHOST *host, BUFFER *wb);
-extern const char* get_release_channel();
+void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived);
+void chartcollectors2json(RRDHOST *host, BUFFER *wb);
+const char* get_release_channel();
 
 #endif //NETDATA_API_FORMATTER_CHARTS2JSON_H
diff --git a/web/api/formatters/csv/csv.c b/web/api/formatters/csv/csv.c
index 6d87ca374..603a17169 100644
--- a/web/api/formatters/csv/csv.c
+++ b/web/api/formatters/csv/csv.c
@@ -3,15 +3,14 @@
 #include "libnetdata/libnetdata.h"
 #include "csv.h"
 
-void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd) {
-    rrdset_check_rdlock(r->st);
-
+void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines) {
     //info("RRD2CSV(): %s: BEGIN", r->st->id);
+    QUERY_TARGET *qt = r->internal.qt;
     long c, i;
-    RRDDIM *d;
+    const long used = qt->query.used;
 
     // print the csv header
-    for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+    for(c = 0, i = 0; c < used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -23,7 +22,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         }
         buffer_strcat(wb, separator);
         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
-        buffer_strcat(wb, d->name);
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
         i++;
     }
@@ -31,7 +30,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
 
     if(format == DATASOURCE_CSV_MARKDOWN) {
         // print the --- line after header
-        for(c = 0, i = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+        for(c = 0, i = 0; c < used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -89,7 +88,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         int set_min_max = 0;
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+            for(c = 0; c < used ;c++) {
                 NETDATA_DOUBLE n = cn[c];
 
                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
@@ -103,7 +102,7 @@ void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const
         }
 
         // for each dimension
-        for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d && c < r->d ;c++, d = d->next) {
+        for(c = 0; c < used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
diff --git a/web/api/formatters/csv/csv.h b/web/api/formatters/csv/csv.h
index cf6020de4..666d4c660 100644
--- a/web/api/formatters/csv/csv.h
+++ b/web/api/formatters/csv/csv.h
@@ -5,7 +5,7 @@
 
 #include "web/api/queries/rrdr.h"
 
-extern void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines, RRDDIM *temp_rd);
+void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, const char *startline, const char *separator, const char *endline, const char *betweenlines);
 
 #include "../rrd2json.h"
 
diff --git a/web/api/formatters/json/json.c b/web/api/formatters/json/json.c
index 6f07b9aa4..608150cba 100644
--- a/web/api/formatters/json/json.c
+++ b/web/api/formatters/json/json.c
@@ -5,15 +5,7 @@
 #define JSON_DATES_JS 1
 #define JSON_DATES_TIMESTAMP 2
 
-void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct context_param *context_param_list)
-{
-    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
-
-    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
-
-    if (should_lock)
-        rrdset_check_rdlock(r->st);
-
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
     //info("RRD2JSON(): %s: BEGIN", r->st->id);
     int row_annotations = 0, dates, dates_with_new = 0;
     char kq[2] = "",                        // key quote
@@ -112,21 +104,21 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
     // -------------------------------------------------------------------------
     // print the JSON header
 
+    QUERY_TARGET *qt = r->internal.qt;
     long c, i;
-    RRDDIM *rd;
+    const long used = qt->query.used;
 
     // print the header lines
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         buffer_fast_strcat(wb, pre_label, pre_label_len);
-        buffer_strcat(wb, rd->name);
-//        buffer_strcat(wb, ".");
-//        buffer_strcat(wb, rd->rrdset->name);
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
         buffer_fast_strcat(wb, post_label, post_label_len);
         i++;
     }
+
     if(!i) {
         buffer_fast_strcat(wb, pre_label, pre_label_len);
         buffer_fast_strcat(wb, "no data", 7);
@@ -134,7 +126,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
     }
     size_t total_number_of_dimensions = i;
 
-    // print the begin of row data
+    // print the beginning of row data
     buffer_strcat(wb, data_begin);
 
     // if all dimensions are hidden, print a null
@@ -187,7 +179,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
             if(unlikely(row_annotations)) {
                 // google supports one annotation per row
                 int annotation_found = 0;
-                for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd ;c++, rd = rd->next) {
+                for(c = 0; c < used ; c++) {
                     if(unlikely(!(r->od[c] & RRDR_DIMENSION_SELECTED))) continue;
 
                     if(unlikely(co[c] & RRDR_VALUE_RESET)) {
@@ -222,7 +214,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
         int set_min_max = 0;
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+            for(c = 0; c < used ;c++) {
                 NETDATA_DOUBLE n;
                 if(unlikely(options & RRDR_OPTION_INTERNAL_AR))
                     n = ar[c];
@@ -240,7 +232,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
         }
 
         // for each dimension
-        for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+        for(c = 0; c < used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -253,7 +245,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable,  struct
             buffer_fast_strcat(wb, pre_value, pre_value_len);
 
             if(unlikely( options & RRDR_OPTION_OBJECTSROWS ))
-                buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
+                buffer_sprintf(wb, "%s%s%s: ", kq, string2str(qt->query.array[c].dimension.name), kq);
 
             if(co[c] & RRDR_VALUE_EMPTY && !(options & RRDR_OPTION_INTERNAL_AR)) {
                 if(unlikely(options & RRDR_OPTION_NULL2ZERO))
diff --git a/web/api/formatters/json/json.h b/web/api/formatters/json/json.h
index 5c4e11371..fb59e5c9a 100644
--- a/web/api/formatters/json/json.h
+++ b/web/api/formatters/json/json.h
@@ -5,6 +5,6 @@
 
 #include "../rrd2json.h"
 
-extern void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable, struct context_param *context_param_list);
+void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable);
 
 #endif //NETDATA_API_FORMATTER_JSON_H
diff --git a/web/api/formatters/json_wrapper.c b/web/api/formatters/json_wrapper.c
index 04cace2fb..8b9b7522c 100644
--- a/web/api/formatters/json_wrapper.c
+++ b/web/api/formatters/json_wrapper.c
@@ -7,9 +7,7 @@ struct value_output {
     BUFFER *wb;
 };
 
-static int value_list_output(const char *name, void *entry, void *data) {
-    (void)name;
-
+static int value_list_output_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data) {
     struct value_output *ap = (struct value_output *)data;
     BUFFER *wb = ap->wb;
     char *output = (char *) entry;
@@ -35,25 +33,17 @@ static int fill_formatted_callback(const char *name, const char *value, RRDLABEL
 }
 
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-    RRDR_GROUPING group_method, QUERY_PARAMS *rrdset_query_data)
+    RRDR_GROUPING group_method)
 {
-    struct context_param *context_param_list = rrdset_query_data->context_param_list;
-    char *chart_label_key = rrdset_query_data->chart_label_key;
-
-    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
-    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
-    uint8_t context_mode = (!context_param_list || (context_param_list->flags & CONTEXT_FLAGS_CONTEXT));
-
-    if (should_lock)
-        rrdset_check_rdlock(r->st);
+    QUERY_TARGET *qt = r->internal.qt;
 
     long rows = rrdr_rows(r);
     long c, i;
-    RRDDIM *rd;
+    const long query_used = qt->query.used;
 
     //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
     char kq[2] = "",                    // key quote
-            sq[2] = "";                     // string quote
+         sq[2] = "";                    // string quote
 
     if( options & RRDR_OPTION_GOOGLE_JSON ) {
         kq[0] = '\0';
@@ -64,52 +54,47 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
         sq[0] = '"';
     }
 
-    if (should_lock)
-        rrdset_rdlock(r->st);
     buffer_sprintf(wb, "{\n"
                        "   %sapi%s: 1,\n"
                        "   %sid%s: %s%s%s,\n"
                        "   %sname%s: %s%s%s,\n"
-                       "   %sview_update_every%s: %d,\n"
-                       "   %supdate_every%s: %d,\n"
-                       "   %sfirst_entry%s: %u,\n"
-                       "   %slast_entry%s: %u,\n"
-                       "   %sbefore%s: %u,\n"
-                       "   %safter%s: %u,\n"
+                       "   %sview_update_every%s: %lld,\n"
+                       "   %supdate_every%s: %lld,\n"
+                       "   %sfirst_entry%s: %lld,\n"
+                       "   %slast_entry%s: %lld,\n"
+                       "   %sbefore%s: %lld,\n"
+                       "   %safter%s: %lld,\n"
                        "   %sgroup%s: %s%s%s,\n"
                        "   %soptions%s: %s"
                    , kq, kq
-                   , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->id, sq
-                   , kq, kq, sq, context_mode && temp_rd?r->st->context:r->st->name, sq
-                   , kq, kq, r->update_every
-                   , kq, kq, r->st->update_every
-                   , kq, kq, (uint32_t) (context_param_list ? context_param_list->first_entry_t : rrdset_first_entry_t_nolock(r->st))
-                   , kq, kq, (uint32_t) (context_param_list ? context_param_list->last_entry_t : rrdset_last_entry_t_nolock(r->st))
-                   , kq, kq, (uint32_t)r->before
-                   , kq, kq, (uint32_t)r->after
+                   , kq, kq, sq, qt->id, sq
+                   , kq, kq, sq, qt->id, sq
+                   , kq, kq, (long long)r->update_every
+                   , kq, kq, (long long)qt->db.minimum_latest_update_every
+                   , kq, kq, (long long)qt->db.first_time_t
+                   , kq, kq, (long long)qt->db.last_time_t
+                   , kq, kq, (long long)r->before
+                   , kq, kq, (long long)r->after
                    , kq, kq, sq, web_client_api_request_v1_data_group_to_string(group_method), sq
                    , kq, kq, sq);
 
-    web_client_api_request_v1_data_options_to_string(wb, r->internal.query_options);
+    web_client_api_request_v1_data_options_to_buffer(wb, r->internal.query_options);
 
     buffer_sprintf(wb, "%s,\n   %sdimension_names%s: [", sq, kq, kq);
 
-    if (should_lock)
-        rrdset_unlock(r->st);
-
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < query_used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         if(i) buffer_strcat(wb, ", ");
         buffer_strcat(wb, sq);
-        buffer_strcat(wb, rd->name);
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.name));
         buffer_strcat(wb, sq);
         i++;
     }
     if(!i) {
 #ifdef NETDATA_INTERNAL_CHECKS
-        error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
+        error("QUERY: '%s', RRDR is empty, %zu dimensions, options is 0x%08x", qt->id, r->d, options);
 #endif
         rows = 0;
         buffer_strcat(wb, sq);
@@ -121,13 +106,13 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                        "   %sdimension_ids%s: ["
                    , kq, kq);
 
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < query_used ; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         if(i) buffer_strcat(wb, ", ");
         buffer_strcat(wb, sq);
-        buffer_strcat(wb, rd->id);
+        buffer_strcat(wb, string2str(qt->query.array[c].dimension.id));
         buffer_strcat(wb, sq);
         i++;
     }
@@ -139,7 +124,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
     }
     buffer_strcat(wb, "],\n");
 
-    if (rrdset_query_data->show_dimensions) {
+    if (r->internal.query_options & RRDR_OPTION_ALL_DIMENSIONS) {
         buffer_sprintf(wb, "   %sfull_dimension_list%s: [", kq, kq);
 
         char name[RRD_ID_LENGTH_MAX * 2 + 2];
@@ -147,58 +132,97 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
 
         struct value_output co = {.c = 0, .wb = wb};
 
-        DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
-        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
-            snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->id, rd->name);
-            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->id, rd->name);
-            dictionary_set(dict, name, output, len+1);
+        DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+        for (c = 0; c < (long)qt->metrics.used ;c++) {
+            snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+                      rrdmetric_acquired_id(qt->metrics.array[c]),
+                      rrdmetric_acquired_name(qt->metrics.array[c]));
+
+            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
+                                rrdmetric_acquired_id(qt->metrics.array[c]),
+                                rrdmetric_acquired_name(qt->metrics.array[c]));
+
+            dictionary_set(dict, name, output, len + 1);
         }
-        dictionary_walkthrough_read(dict, value_list_output, &co);
+        dictionary_walkthrough_read(dict, value_list_output_callback, &co);
         dictionary_destroy(dict);
 
         co.c = 0;
         buffer_sprintf(wb, "],\n   %sfull_chart_list%s: [", kq, kq);
-        dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
-        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
-            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]", rd->rrdset->id, rd->rrdset->name);
-            snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", rd->rrdset->id, rd->rrdset->name);
+        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+        for (c = 0; c < (long)qt->instances.used ; c++) {
+            RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+
+            snprintfz(name, RRD_ID_LENGTH_MAX * 2 + 1, "%s:%s",
+                      rrdinstance_acquired_id(ria),
+                      rrdinstance_acquired_name(ria));
+
+            int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\",\"%s\"]",
+                                rrdinstance_acquired_id(ria),
+                                rrdinstance_acquired_name(ria));
+
             dictionary_set(dict, name, output, len + 1);
         }
-
-        dictionary_walkthrough_read(dict, value_list_output, &co);
+        dictionary_walkthrough_read(dict, value_list_output_callback, &co);
         dictionary_destroy(dict);
 
-        RRDSET *st;
         co.c = 0;
         buffer_sprintf(wb, "],\n   %sfull_chart_labels%s: [", kq, kq);
-        dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
-        for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
-            st = rd->rrdset;
-            if (st->state && st->state->chart_labels)
-                rrdlabels_walkthrough_read(st->state->chart_labels, fill_formatted_callback, dict);
+        dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+        for (c = 0; c < (long)qt->instances.used ; c++) {
+            RRDINSTANCE_ACQUIRED *ria = qt->instances.array[c];
+            rrdlabels_walkthrough_read(rrdinstance_acquired_labels(ria), fill_formatted_callback, dict);
         }
-        dictionary_walkthrough_read(dict, value_list_output, &co);
+        dictionary_walkthrough_read(dict, value_list_output_callback, &co);
         dictionary_destroy(dict);
         buffer_strcat(wb, "],\n");
     }
 
-    // Composite charts
-    if (context_mode && temp_rd) {
+    // functions
+    {
+        DICTIONARY *funcs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+        RRDINSTANCE_ACQUIRED *ria = NULL;
+        for (c = 0; c < query_used ; c++) {
+            QUERY_METRIC *qm = &qt->query.array[c];
+            if(qm->link.ria == ria)
+                continue;
+
+            ria = qm->link.ria;
+            chart_functions_to_dict(rrdinstance_acquired_functions(ria), funcs);
+        }
+
+        buffer_sprintf(wb, "   %sfunctions%s: [", kq, kq);
+        void *t; (void)t;
+        dfe_start_read(funcs, t) {
+            const char *comma = "";
+            if(t_dfe.counter) comma = ", ";
+            buffer_sprintf(wb, "%s%s%s%s", comma, sq, t_dfe.name, sq);
+        }
+        dfe_done(t);
+        dictionary_destroy(funcs);
+        buffer_strcat(wb, "],\n");
+    }
+
+    // context query
+    if (!qt->request.st) {
         buffer_sprintf(
             wb,
             "   %schart_ids%s: [",
             kq, kq);
 
-        for (c = 0, i = 0, rd = temp_rd ; rd && c < r->d; c++, rd = rd->next) {
+        for (c = 0, i = 0; c < query_used; c++) {
+            QUERY_METRIC *qm = &qt->query.array[c];
+
             if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
                 continue;
+
             if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
                 continue;
 
             if (i)
                 buffer_strcat(wb, ", ");
             buffer_strcat(wb, sq);
-            buffer_strcat(wb, rd->rrdset->id);
+            buffer_strcat(wb, string2str(qm->chart.id));
             buffer_strcat(wb, sq);
             i++;
         }
@@ -209,29 +233,29 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
             buffer_strcat(wb, sq);
         }
         buffer_strcat(wb, "],\n");
-        if (chart_label_key) {
+        if (qt->instances.chart_label_key_pattern) {
             buffer_sprintf(wb, "   %schart_labels%s: { ", kq, kq);
 
-            SIMPLE_PATTERN *pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
-            SIMPLE_PATTERN *original_pattern = pattern;
+            SIMPLE_PATTERN *pattern = qt->instances.chart_label_key_pattern;
             char *label_key = NULL;
             int keys = 0;
             while (pattern && (label_key = simple_pattern_iterate(&pattern))) {
-
                 if (keys)
                     buffer_strcat(wb, ", ");
                 buffer_sprintf(wb, "%s%s%s : [", kq, label_key, kq);
                 keys++;
 
-                for (c = 0, i = 0, rd = temp_rd; rd && c < r->d; c++, rd = rd->next) {
+                for (c = 0, i = 0; c < query_used; c++) {
+                    QUERY_METRIC *qm = &qt->query.array[c];
+
                     if (unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN))
                         continue;
                     if (unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO)))
                         continue;
+
                     if (i)
                         buffer_strcat(wb, ", ");
-
-                    rrdlabels_get_value_to_buffer_or_null(rd->rrdset->state->chart_labels, wb, label_key, sq, "null");
+                    rrdlabels_get_value_to_buffer_or_null(rrdinstance_acquired_labels(qm->link.ria), wb, label_key, sq, "null");
                     i++;
                 }
                 if (!i) {
@@ -243,33 +267,26 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                 buffer_strcat(wb, "]");
             }
             buffer_strcat(wb, "},\n");
-            simple_pattern_free(original_pattern);
         }
     }
 
     buffer_sprintf(wb, "   %slatest_values%s: ["
                    , kq, kq);
 
-    for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+    for(c = 0, i = 0; c < query_used ;c++) {
+        QUERY_METRIC *qm = &qt->query.array[c];
+
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
         if(i) buffer_strcat(wb, ", ");
         i++;
 
-        NETDATA_DOUBLE value = rd->last_stored_value;
+        NETDATA_DOUBLE value = rrdmetric_acquired_last_stored_value(qm->link.rma);
         if (NAN == value)
             buffer_strcat(wb, "null");
         else
             buffer_rrd_value(wb, value);
-        /*
-        storage_number n = rd->values[rrdset_last_slot(r->st)];
-
-        if(!does_storage_number_exist(n))
-            buffer_strcat(wb, "null");
-        else
-            buffer_rrd_value(wb, unpack_storage_number(n));
-        */
     }
     if(!i) {
         rows = 0;
@@ -286,7 +303,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
 
         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
             total = 0;
-            for(c = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+            for(c = 0; c < query_used ;c++) {
                 NETDATA_DOUBLE *cn = &r->v[ (rrdr_rows(r) - 1) * r->d ];
                 NETDATA_DOUBLE n = cn[c];
 
@@ -299,7 +316,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
             if(total == 0) total = 1;
         }
 
-        for(c = 0, i = 0, rd = temp_rd?temp_rd:r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
+        for(c = 0, i = 0; c < query_used ;c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -349,16 +366,11 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
                    , kq, kq
                    );
 
-    for(int tier = 0; tier < storage_tiers ; tier++)
+    for(size_t tier = 0; tier < storage_tiers ; tier++)
         buffer_sprintf(wb, "%s%zu", tier>0?", ":"", r->internal.tier_points_read[tier]);
 
     buffer_strcat(wb, " ]");
 
-    if((options & RRDR_OPTION_CUSTOM_VARS) && (options & RRDR_OPTION_JSON_WRAP)) {
-        buffer_sprintf(wb, ",\n   %schart_variables%s: ", kq, kq);
-        health_api_v1_chart_custom_variables2json(r->st, wb);
-    }
-
     buffer_sprintf(wb, ",\n   %sresult%s: ", kq, kq);
 
     if(string_value) buffer_strcat(wb, sq);
diff --git a/web/api/formatters/json_wrapper.h b/web/api/formatters/json_wrapper.h
index bfadc883e..91c1475c5 100644
--- a/web/api/formatters/json_wrapper.h
+++ b/web/api/formatters/json_wrapper.h
@@ -7,9 +7,9 @@
 #include "web/api/queries/query.h"
 
 
-extern void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
-    RRDR_GROUPING group_method, QUERY_PARAMS *query_params);
-extern void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
-extern void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
+void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
+    RRDR_GROUPING group_method);
+void rrdr_json_wrapper_anomaly_rates(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
+void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value);
 
 #endif //NETDATA_API_FORMATTER_JSON_WRAPPER_H
diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c
index 7aa478d95..8bf547192 100644
--- a/web/api/formatters/rrd2json.c
+++ b/web/api/formatters/rrd2json.c
@@ -3,129 +3,6 @@
 #include "web/api/web_api_v1.h"
 #include "database/storage_engine.h"
 
-static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
-{
-    if (unlikely(!temp_rd))
-        return;
-
-    onewayalloc_freez(owa, (char *)temp_rd->id);
-
-    if (unlikely(archive_mode)) {
-        temp_rd->rrdset->counter--;
-        if (!temp_rd->rrdset->counter) {
-            onewayalloc_freez(owa, (char *)temp_rd->rrdset->name);
-            onewayalloc_freez(owa, temp_rd->rrdset->context);
-            onewayalloc_freez(owa, temp_rd->rrdset);
-        }
-    }
-
-    for(int tier = 0; tier < storage_tiers ;tier++) {
-        if(!temp_rd->tiers[tier]) continue;
-
-        if(archive_mode) {
-            STORAGE_ENGINE *eng = storage_engine_get(temp_rd->tiers[tier]->mode);
-            if (eng)
-                eng->api.free(temp_rd->tiers[tier]->db_metric_handle);
-        }
-
-        onewayalloc_freez(owa, temp_rd->tiers[tier]);
-    }
-
-    onewayalloc_freez(owa, temp_rd);
-}
-
-static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode)
-{
-    if (unlikely(!temp_rd))
-        return;
-
-    RRDDIM *t;
-    while (temp_rd) {
-        t = temp_rd->next;
-        free_single_rrdrim(owa, temp_rd, archive_mode);
-        temp_rd = t;
-    }
-}
-
-void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list)
-{
-    if (unlikely(!param_list || !*param_list))
-        return;
-
-    free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE);
-    onewayalloc_freez(owa, (*param_list));
-    *param_list = NULL;
-}
-
-void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested)
-{
-    RRDDIM *temp_rd = context_param_list->rd;
-    RRDDIM *new_rd_list = NULL, *t;
-    int is_archived = (context_param_list->flags & CONTEXT_FLAGS_ARCHIVE);
-
-    RRDSET *st = temp_rd->rrdset;
-    RRDSET *last_st = st;
-    time_t last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
-    time_t last_last_entry_t = last_entry_t;
-    while (temp_rd) {
-        t = temp_rd->next;
-
-        st = temp_rd->rrdset;
-        if (st == last_st) {
-            last_entry_t = last_last_entry_t;
-        }else {
-            last_entry_t = is_archived ? st->last_entry_t : rrdset_last_entry_t(st);
-            last_last_entry_t = last_entry_t;
-            last_st = st;
-        }
-
-        if (last_entry_t >= after_requested) {
-            temp_rd->next = new_rd_list;
-            new_rd_list = temp_rd;
-        } else
-            free_single_rrdrim(owa, temp_rd, is_archived);
-        temp_rd = t;
-    }
-    context_param_list->rd = new_rd_list;
-};
-
-void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st)
-{
-    if (unlikely(!param_list || !st))
-        return;
-
-    if (unlikely(!(*param_list))) {
-        *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
-        (*param_list)->first_entry_t = LONG_MAX;
-        (*param_list)->last_entry_t = 0;
-        (*param_list)->flags = CONTEXT_FLAGS_CONTEXT;
-        (*param_list)->rd = NULL;
-    }
-
-    RRDDIM *rd1;
-    st->last_accessed_time = now_realtime_sec();
-    rrdset_rdlock(st);
-
-    (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t_nolock(st));
-    (*param_list)->last_entry_t  = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st));
-
-    rrddim_foreach_read(rd1, st) {
-        RRDDIM *rd = onewayalloc_memdupz(owa, rd1, sizeof(RRDDIM));
-        rd->id = onewayalloc_strdupz(owa, rd1->id);
-        rd->name = onewayalloc_strdupz(owa, rd1->name);
-        for(int tier = 0; tier < storage_tiers ;tier++) {
-            if(rd1->tiers[tier])
-                rd->tiers[tier] = onewayalloc_memdupz(owa, rd1->tiers[tier], sizeof(*rd->tiers[tier]));
-            else
-                rd->tiers[tier] = NULL;
-        }
-        rd->next = (*param_list)->rd;
-        (*param_list)->rd = rd;
-    }
-
-    rrdset_unlock(st);
-}
-
 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
     rrdset2json(st, wb, NULL, NULL, 0);
 }
@@ -183,12 +60,12 @@ int rrdset2value_api_v1(
         , BUFFER *wb
         , NETDATA_DOUBLE *n
         , const char *dimensions
-        , long points
-        , long long after
-        , long long before
-        , int group_method
+        , size_t points
+        , time_t after
+        , time_t before
+        , RRDR_GROUPING group_method
         , const char *group_options
-        , long group_time
+        , time_t resampling_time
         , uint32_t options
         , time_t *db_after
         , time_t *db_before
@@ -197,16 +74,27 @@ int rrdset2value_api_v1(
         , size_t *result_points_generated
         , int *value_is_null
         , NETDATA_DOUBLE *anomaly_rate
-        , int timeout
-        , int tier
+        , time_t timeout
+        , size_t tier
+        , QUERY_SOURCE query_source
 ) {
     int ret = HTTP_RESP_INTERNAL_SERVER_ERROR;
 
     ONEWAYALLOC *owa = onewayalloc_create(0);
-
-    RRDR *r = rrd2rrdr(owa, st, points, after, before,
-                       group_method, group_time, options, dimensions, NULL,
-                       group_options, timeout, tier);
+    RRDR *r = rrd2rrdr_legacy(
+            owa,
+            st,
+            points,
+            after,
+            before,
+            group_method,
+            resampling_time,
+            options,
+            dimensions,
+            group_options,
+            timeout,
+            tier,
+            query_source);
 
     if(!r) {
         if(value_is_null) *value_is_null = 1;
@@ -218,7 +106,7 @@ int rrdset2value_api_v1(
         *db_points_read += r->internal.db_points_read;
 
     if(db_points_per_tier) {
-        for(int t = 0; t < storage_tiers ;t++)
+        for(size_t t = 0; t < storage_tiers ;t++)
             db_points_per_tier[t] += r->internal.tier_points_read[t];
     }
 
@@ -244,50 +132,19 @@ int rrdset2value_api_v1(
     if(db_after)  *db_after  = r->after;
     if(db_before) *db_before = r->before;
 
-    long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
-    *n = rrdr2value(r, i, options, value_is_null, anomaly_rate, NULL);
+    long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
+    *n = rrdr2value(r, i, options, value_is_null, anomaly_rate);
     ret = HTTP_RESP_OK;
 
 cleanup:
-    if(r) rrdr_free(owa, r);
+    rrdr_free(owa, r);
     onewayalloc_destroy(owa);
     return ret;
 }
 
-int rrdset2anything_api_v1(
-          ONEWAYALLOC *owa
-        , RRDSET *st
-        , QUERY_PARAMS *query_params
-        , BUFFER *dimensions
-        , uint32_t format
-        , long points
-        , long long after
-        , long long before
-        , int group_method
-        , const char *group_options
-        , long group_time
-        , uint32_t options
-        , time_t *latest_timestamp
-        , int tier
-)
-{
-    BUFFER *wb = query_params->wb;
-    if (query_params->context_param_list && !(query_params->context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
-        st->last_accessed_time = now_realtime_sec();
-
-    RRDR *r = rrd2rrdr(
-        owa,
-        st,
-        points,
-        after,
-        before,
-        group_method,
-        group_time,
-        options,
-        dimensions ? buffer_tostring(dimensions) : NULL,
-        query_params->context_param_list,
-        group_options,
-        query_params->timeout, tier);
+int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *latest_timestamp) {
+
+    RRDR *r = rrd2rrdr(owa, qt);
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
         return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -298,11 +155,6 @@ int rrdset2anything_api_v1(
         return HTTP_RESP_BACKEND_FETCH_FAILED;
     }
 
-    if (st->state && st->state->is_ar_chart)
-        ml_process_rrdr(r, query_params->max_anomaly_rates);
-
-    RRDDIM *temp_rd = query_params->context_param_list ? query_params->context_param_list->rd : NULL;
-
     if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
         buffer_no_cacheable(wb);
     else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
@@ -311,85 +163,89 @@ int rrdset2anything_api_v1(
     if(latest_timestamp && rrdr_rows(r) > 0)
         *latest_timestamp = r->before;
 
+    DATASOURCE_FORMAT format = qt->request.format;
+    RRDR_OPTIONS options = qt->request.options;
+    RRDR_GROUPING group_method = qt->request.group_method;
+
     switch(format) {
     case DATASOURCE_SSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
-            rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
+            rrdr2ssv(r, wb, options, "", " ", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2ssv(r, wb, options, "", " ", "", temp_rd);
+            rrdr2ssv(r, wb, options, "", " ", "");
         }
         break;
 
     case DATASOURCE_SSV_COMMA:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
-            rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
+            rrdr2ssv(r, wb, options, "", ",", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2ssv(r, wb, options, "", ",", "", temp_rd);
+            rrdr2ssv(r, wb, options, "", ",", "");
         }
         break;
 
     case DATASOURCE_JS_ARRAY:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
-            rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
+            rrdr2ssv(r, wb, options, "[", ",", "]");
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
         else {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr2ssv(r, wb, options, "[", ",", "]", temp_rd);
+            rrdr2ssv(r, wb, options, "[", ",", "]");
         }
         break;
 
     case DATASOURCE_CSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
-            rrdr2csv(r, wb, format, options, "", ",", "\\n", "", temp_rd);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
+            rrdr2csv(r, wb, format, options, "", ",", "\\n", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2csv(r, wb, format, options, "", ",", "\r\n", "", temp_rd);
+            rrdr2csv(r, wb, format, options, "", ",", "\r\n", "");
         }
         break;
 
     case DATASOURCE_CSV_MARKDOWN:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
-            rrdr2csv(r, wb, format, options, "", "|", "\\n", "", temp_rd);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
+            rrdr2csv(r, wb, format, options, "", "|", "\\n", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2csv(r, wb, format, options, "", "|", "\r\n", "", temp_rd);
+            rrdr2csv(r, wb, format, options, "", "|", "\r\n", "");
         }
         break;
 
     case DATASOURCE_CSV_JSON_ARRAY:
         wb->contenttype = CT_APPLICATION_JSON;
         if(options & RRDR_OPTION_JSON_WRAP) {
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
             buffer_strcat(wb, "[\n");
-            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
+            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
             buffer_strcat(wb, "\n]");
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
         else {
             wb->contenttype = CT_APPLICATION_JSON;
             buffer_strcat(wb, "[\n");
-            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n", temp_rd);
+            rrdr2csv(r, wb, format, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
             buffer_strcat(wb, "\n]");
         }
         break;
@@ -397,29 +253,29 @@ int rrdset2anything_api_v1(
     case DATASOURCE_TSV:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
-            rrdr2csv(r, wb, format, options, "", "\t", "\\n", "", temp_rd);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
+            rrdr2csv(r, wb, format, options, "", "\t", "\\n", "");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_PLAIN;
-            rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "", temp_rd);
+            rrdr2csv(r, wb, format, options, "", "\t", "\r\n", "");
         }
         break;
 
     case DATASOURCE_HTML:
         if(options & RRDR_OPTION_JSON_WRAP) {
             wb->contenttype = CT_APPLICATION_JSON;
-            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 1, group_method);
             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
-            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "", temp_rd);
+            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
             rrdr_json_wrapper_end(r, wb, format, options, 1);
         }
         else {
             wb->contenttype = CT_TEXT_HTML;
             buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
-            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "", temp_rd);
+            rrdr2csv(r, wb, format, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
             buffer_strcat(wb, "</table>\n</center>\n</html>\n");
         }
         break;
@@ -428,9 +284,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
 
-        rrdr2json(r, wb, options, 1, query_params->context_param_list);
+        rrdr2json(r, wb, options, 1);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -440,9 +296,9 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
 
-        rrdr2json(r, wb, options, 1, query_params->context_param_list);
+        rrdr2json(r, wb, options, 1);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -451,9 +307,9 @@ int rrdset2anything_api_v1(
     case DATASOURCE_JSONP:
         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
 
-        rrdr2json(r, wb, options, 0, query_params->context_param_list);
+        rrdr2json(r, wb, options, 0);
 
         if(options & RRDR_OPTION_JSON_WRAP)
             rrdr_json_wrapper_end(r, wb, format, options, 0);
@@ -464,14 +320,14 @@ int rrdset2anything_api_v1(
         wb->contenttype = CT_APPLICATION_JSON;
 
         if(options & RRDR_OPTION_JSON_WRAP)
-            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method, query_params);
+            rrdr_json_wrapper_begin(r, wb, format, options, 0, group_method);
 
-        rrdr2json(r, wb, options, 0, query_params->context_param_list);
+        rrdr2json(r, wb, options, 0);
 
         if(options & RRDR_OPTION_JSON_WRAP) {
             if(options & RRDR_OPTION_RETURN_JWAR) {
                 rrdr_json_wrapper_anomaly_rates(r, wb, format, options, 0);
-                rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0, query_params->context_param_list);
+                rrdr2json(r, wb, options | RRDR_OPTION_INTERNAL_AR, 0);
             }
             rrdr_json_wrapper_end(r, wb, format, options, 0);
         }
diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h
index 6be53ff8a..048281d7e 100644
--- a/web/api/formatters/rrd2json.h
+++ b/web/api/formatters/rrd2json.h
@@ -5,16 +5,6 @@
 
 #include "web/api/web_api_v1.h"
 
-typedef struct query_params {
-    struct context_param *context_param_list;
-    BUFFER *wb;
-    char *chart_label_key;
-    int max_anomaly_rates;
-    int timeout;
-    int show_dimensions;
-} QUERY_PARAMS;
-
-
 #include "web/api/exporters/allmetrics.h"
 #include "web/api/queries/rrdr.h"
 
@@ -34,19 +24,20 @@ typedef struct query_params {
 #define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
 
 // type of JSON generations
-#define DATASOURCE_INVALID (-1)
-#define DATASOURCE_JSON 0
-#define DATASOURCE_DATATABLE_JSON 1
-#define DATASOURCE_DATATABLE_JSONP 2
-#define DATASOURCE_SSV 3
-#define DATASOURCE_CSV 4
-#define DATASOURCE_JSONP 5
-#define DATASOURCE_TSV 6
-#define DATASOURCE_HTML 7
-#define DATASOURCE_JS_ARRAY 8
-#define DATASOURCE_SSV_COMMA 9
-#define DATASOURCE_CSV_JSON_ARRAY 10
-#define DATASOURCE_CSV_MARKDOWN 11
+typedef enum {
+    DATASOURCE_JSON             = 0,
+    DATASOURCE_DATATABLE_JSON   = 1,
+    DATASOURCE_DATATABLE_JSONP  = 2,
+    DATASOURCE_SSV              = 3,
+    DATASOURCE_CSV              = 4,
+    DATASOURCE_JSONP            = 5,
+    DATASOURCE_TSV              = 6,
+    DATASOURCE_HTML             = 7,
+    DATASOURCE_JS_ARRAY         = 8,
+    DATASOURCE_SSV_COMMA        = 9,
+    DATASOURCE_CSV_JSON_ARRAY   = 10,
+    DATASOURCE_CSV_MARKDOWN     = 11,
+} DATASOURCE_FORMAT;
 
 #define DATASOURCE_FORMAT_JSON "json"
 #define DATASOURCE_FORMAT_DATATABLE_JSON "datatable"
@@ -61,37 +52,22 @@ typedef struct query_params {
 #define DATASOURCE_FORMAT_CSV_JSON_ARRAY "csvjsonarray"
 #define DATASOURCE_FORMAT_CSV_MARKDOWN "markdown"
 
-extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
-extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
+void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
+void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
 
-extern int rrdset2anything_api_v1(
-          ONEWAYALLOC *owa
-        , RRDSET *st
-        , QUERY_PARAMS *query_params
-        , BUFFER *dimensions
-        , uint32_t format
-        , long points
-        , long long after
-        , long long before
-        , int group_method
-        , const char *group_options
-        , long group_time
-        , uint32_t options
-        , time_t *latest_timestamp
-        , int tier
-);
+int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, struct query_target *qt, time_t *latest_timestamp);
 
-extern int rrdset2value_api_v1(
+int rrdset2value_api_v1(
           RRDSET *st
         , BUFFER *wb
         , NETDATA_DOUBLE *n
         , const char *dimensions
-        , long points
-        , long long after
-        , long long before
-        , int group_method
+        , size_t points
+        , time_t after
+        , time_t before
+        , RRDR_GROUPING group_method
         , const char *group_options
-        , long group_time
+        , time_t resampling_time
         , uint32_t options
         , time_t *db_after
         , time_t *db_before
@@ -100,12 +76,9 @@ extern int rrdset2value_api_v1(
         , size_t *result_points_generated
         , int *value_is_null
         , NETDATA_DOUBLE *anomaly_rate
-        , int timeout
-        , int tier
+        , time_t timeout
+        , size_t tier
+        , QUERY_SOURCE query_source
 );
 
-extern void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st);
-extern void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested);
-extern void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list);
-
 #endif /* NETDATA_RRD2JSON_H */
diff --git a/web/api/formatters/rrdset2json.c b/web/api/formatters/rrdset2json.c
index de8d87bae..1e8106335 100644
--- a/web/api/formatters/rrdset2json.c
+++ b/web/api/formatters/rrdset2json.c
@@ -4,7 +4,7 @@
 
 void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
 {
-    if(unlikely(!st->state || !st->state->chart_labels))
+    if(unlikely(!st->rrdlabels))
         return;
 
     char tabs[11];
@@ -18,17 +18,15 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
         indentation--;
     }
 
-    rrdlabels_to_buffer(st->state->chart_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
+    rrdlabels_to_buffer(st->rrdlabels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
     buffer_strcat(wb, "\n");
 }
 
 // generate JSON for the /api/v1/chart API call
 
 void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile) {
-    rrdset_rdlock(st);
-
-    time_t first_entry_t = rrdset_first_entry_t_nolock(st);
-    time_t last_entry_t  = rrdset_last_entry_t_nolock(st);
+    time_t first_entry_t = rrdset_first_entry_t(st);
+    time_t last_entry_t  = rrdset_last_entry_t(st);
 
     buffer_sprintf(
         wb,
@@ -45,18 +43,18 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
         "\t\t\t\"units\": \"%s\",\n"
         "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
         "\t\t\t\"chart_type\": \"%s\",\n",
-        st->id,
-        st->name,
-        st->type,
-        st->family,
-        st->context,
-        st->title,
-        st->name,
+        rrdset_id(st),
+        rrdset_name(st),
+        rrdset_parts_type(st),
+        rrdset_family(st),
+        rrdset_context(st),
+        rrdset_title(st),
+        rrdset_name(st),
         st->priority,
-        st->plugin_name ? st->plugin_name : "",
-        st->module_name ? st->module_name : "",
-        st->units,
-        st->name,
+        rrdset_plugin_name(st),
+        rrdset_module_name(st),
+        rrdset_units(st),
+        rrdset_name(st),
         rrdset_type_name(st->chart_type));
 
     if (likely(!skip_volatile))
@@ -90,7 +88,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
     size_t dimensions = 0;
     RRDDIM *rd;
     rrddim_foreach_read(rd, st) {
-        if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue;
+        if(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue;
 
         memory += sizeof(RRDDIM) + rd->memsize;
 
@@ -98,13 +96,14 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
             buffer_strcat(wb, ",\n\t\t\t\t\"");
         else
             buffer_strcat(wb, "\t\t\t\t\"");
-        buffer_strcat_jsonescape(wb, rd->id);
+        buffer_strcat_jsonescape(wb, rrddim_id(rd));
         buffer_strcat(wb, "\": { \"name\": \"");
-        buffer_strcat_jsonescape(wb, rd->name);
+        buffer_strcat_jsonescape(wb, rrddim_name(rd));
         buffer_strcat(wb, "\" }");
 
         dimensions++;
     }
+    rrddim_foreach_done(rd);
 
     if(dimensions_count) *dimensions_count += dimensions;
     if(memory_used) *memory_used += memory;
@@ -121,7 +120,8 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
         buffer_strcat(wb, ",\n\t\t\t\"alarms\": {\n");
         size_t alarms = 0;
         RRDCALC *rc;
-        for (rc = st->alarms; rc; rc = rc->rrdset_next) {
+        netdata_rwlock_rdlock(&st->alerts.rwlock);
+        DOUBLE_LINKED_LIST_FOREACH_FORWARD(st->alerts.base, rc, prev, next) {
             buffer_sprintf(
                 wb,
                 "%s"
@@ -131,23 +131,25 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor
                 "\t\t\t\t\t\"units\": \"%s\",\n"
                 "\t\t\t\t\t\"update_every\": %d\n"
                 "\t\t\t\t}",
-                (alarms) ? ",\n" : "", rc->name, rc->id, rrdcalc_status2string(rc->status), rc->units,
+                (alarms) ? ",\n" : "", rrdcalc_name(rc), rc->id, rrdcalc_status2string(rc->status), rrdcalc_units(rc),
                 rc->update_every);
 
             alarms++;
         }
+        netdata_rwlock_unlock(&st->alerts.rwlock);
         buffer_sprintf(wb,
                        "\n\t\t\t}"
         );
     }
     buffer_strcat(wb, ",\n\t\t\t\"chart_labels\": {\n");
     chart_labels2json(st, wb, 2);
-    buffer_strcat(wb, "\t\t\t}\n");
+    buffer_strcat(wb, "\t\t\t}");
 
+    buffer_strcat(wb, ",\n\t\t\t\"functions\": {\n");
+    chart_functions2json(st, wb, 4, "\"", "\"");
+    buffer_strcat(wb, "\t\t\t}");
 
     buffer_sprintf(wb,
             "\n\t\t}"
     );
-
-    rrdset_unlock(st);
 }
diff --git a/web/api/formatters/rrdset2json.h b/web/api/formatters/rrdset2json.h
index 697c84634..b2908e225 100644
--- a/web/api/formatters/rrdset2json.h
+++ b/web/api/formatters/rrdset2json.h
@@ -5,6 +5,6 @@
 
 #include "rrd2json.h"
 
-extern void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile);
+void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used, int skip_volatile);
 
 #endif //NETDATA_API_FORMATTER_RRDSET2JSON_H
diff --git a/web/api/formatters/ssv/ssv.c b/web/api/formatters/ssv/ssv.c
index 850182da1..d561980d9 100644
--- a/web/api/formatters/ssv/ssv.c
+++ b/web/api/formatters/ssv/ssv.c
@@ -2,7 +2,7 @@
 
 #include "ssv.h"
 
-void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd) {
+void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix) {
     //info("RRD2SSV(): %s: BEGIN", r->st->id);
     long i;
 
@@ -17,7 +17,7 @@ void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, con
     // for each line in the array
     for(i = start; i != end ;i += step) {
         int all_values_are_null = 0;
-        NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL, temp_rd);
+        NETDATA_DOUBLE v = rrdr2value(r, i, options, &all_values_are_null, NULL);
 
         if(likely(i != start)) {
             if(r->min > v) r->min = v;
diff --git a/web/api/formatters/ssv/ssv.h b/web/api/formatters/ssv/ssv.h
index 66716b9c9..f7d4a9548 100644
--- a/web/api/formatters/ssv/ssv.h
+++ b/web/api/formatters/ssv/ssv.h
@@ -5,6 +5,6 @@
 
 #include "../rrd2json.h"
 
-extern void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix, RRDDIM *temp_rd);
+void rrdr2ssv(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, const char *prefix, const char *separator, const char *suffix);
 
 #endif //NETDATA_API_FORMATTER_SSV_H
diff --git a/web/api/formatters/value/value.c b/web/api/formatters/value/value.c
index 30e00c068..46a71303e 100644
--- a/web/api/formatters/value/value.c
+++ b/web/api/formatters/value/value.c
@@ -3,9 +3,10 @@
 #include "value.h"
 
 
-inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd) {
+inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate) {
+    QUERY_TARGET *qt = r->internal.qt;
     long c;
-    RRDDIM *d;
+    const long used = qt->query.used;
 
     NETDATA_DOUBLE *cn = &r->v[ i * r->d ];
     RRDR_VALUE_FLAGS *co = &r->o[ i * r->d ];
@@ -20,7 +21,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
     int set_min_max = 0;
     if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
         total = 0;
-        for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) {
+        for (c = 0; c < used; c++) {
             NETDATA_DOUBLE n = cn[c];
 
             if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
@@ -34,7 +35,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
     }
 
     // for each dimension
-    for (c = 0, d = temp_rd ? temp_rd : r->st->dimensions; d && c < r->d; c++, d = d->next) {
+    for (c = 0; c < used; c++) {
         if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_DIMENSION_NONZERO))) continue;
 
@@ -80,7 +81,7 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
 
     if(anomaly_rate) {
         if(!r->d) *anomaly_rate = 0;
-        else *anomaly_rate = total_anomaly_rate / r->d;
+        else *anomaly_rate = total_anomaly_rate / (NETDATA_DOUBLE)r->d;
     }
 
     if(unlikely(all_null)) {
@@ -100,3 +101,62 @@ inline NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all
 
     return v;
 }
+
+QUERY_VALUE rrdmetric2value(RRDHOST *host,
+                            struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,
+                            time_t after, time_t before,
+                            RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
+                            size_t tier, time_t timeout, QUERY_SOURCE query_source
+) {
+    QUERY_TARGET_REQUEST qtr = {
+            .host = host,
+            .rca = rca,
+            .ria = ria,
+            .rma = rma,
+            .after = after,
+            .before = before,
+            .points = 1,
+            .options = options,
+            .group_method = group_method,
+            .group_options = group_options,
+            .tier = tier,
+            .timeout = timeout,
+            .query_source = query_source,
+    };
+
+    ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
+    RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
+
+    QUERY_VALUE qv;
+
+    if(!r || rrdr_rows(r) == 0) {
+        qv = (QUERY_VALUE) {
+                .value = NAN,
+                .anomaly_rate = NAN,
+        };
+    }
+    else {
+        qv = (QUERY_VALUE) {
+                .after = r->after,
+                .before = r->before,
+                .points_read = r->internal.db_points_read,
+                .result_points = r->internal.result_points_generated,
+        };
+
+        for(size_t t = 0; t < storage_tiers ;t++)
+            qv.storage_points_per_tier[t] = r->internal.tier_points_read[t];
+
+        long i = (!(options & RRDR_OPTION_REVERSED))?(long)rrdr_rows(r) - 1:0;
+        int all_values_are_null = 0;
+        qv.value = rrdr2value(r, i, options, &all_values_are_null, &qv.anomaly_rate);
+        if(all_values_are_null) {
+            qv.value = NAN;
+            qv.anomaly_rate = NAN;
+        }
+    }
+
+    rrdr_free(owa, r);
+    onewayalloc_destroy(owa);
+
+    return qv;
+}
diff --git a/web/api/formatters/value/value.h b/web/api/formatters/value/value.h
index fc1c7bf08..76b1869f3 100644
--- a/web/api/formatters/value/value.h
+++ b/web/api/formatters/value/value.h
@@ -5,6 +5,27 @@
 
 #include "../rrd2json.h"
 
-extern NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate, RRDDIM *temp_rd);
+typedef struct storage_value {
+    NETDATA_DOUBLE value;
+    NETDATA_DOUBLE anomaly_rate;
+    time_t after;
+    time_t before;
+    size_t points_read;
+    size_t storage_points_per_tier[RRD_STORAGE_TIERS];
+    size_t result_points;
+} QUERY_VALUE;
+
+struct rrdmetric_acquired;
+struct rrdinstance_acquired;
+struct rrdcontext_acquired;
+
+QUERY_VALUE rrdmetric2value(RRDHOST *host,
+                            struct rrdcontext_acquired *rca, struct rrdinstance_acquired *ria, struct rrdmetric_acquired *rma,
+                            time_t after, time_t before,
+                            RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
+                            size_t tier, time_t timeout, QUERY_SOURCE query_source
+);
+
+NETDATA_DOUBLE rrdr2value(RRDR *r, long i, RRDR_OPTIONS options, int *all_values_are_null, NETDATA_DOUBLE *anomaly_rate);
 
 #endif //NETDATA_API_FORMATTER_VALUE_H
diff --git a/web/api/health/health_cmdapi.h b/web/api/health/health_cmdapi.h
index d8ec6aaa0..d5309c73f 100644
--- a/web/api/health/health_cmdapi.h
+++ b/web/api/health/health_cmdapi.h
@@ -24,7 +24,7 @@
 #define HEALTH_CMDAPI_MSG_STYPEWARNING "WARNING: Added alarm selector to silence/disable alarms without a SILENCE or DISABLE command.\n"
 #define HEALTH_CMDAPI_MSG_NOSELECTORWARNING "WARNING: SILENCE or DISABLE command is ineffective without defining any alarm selectors.\n"
 
-extern int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, char *url);
 
 #include "web/api/web_api_v1.h"
 
diff --git a/web/api/netdata-swagger.json b/web/api/netdata-swagger.json
index 029783b55..cb2b4809c 100644
--- a/web/api/netdata-swagger.json
+++ b/web/api/netdata-swagger.json
@@ -573,7 +573,6 @@
                   "unaligned",
                   "match-ids",
                   "match-names",
-                  "showcustomvars",
                   "allow_past",
                   "anomaly-bit"
                 ]
@@ -1629,6 +1628,68 @@
         }
       }
     },
+    "/function": {
+      "get": {
+        "summary": "Execute a collector function.",
+        "parameters": [
+          {
+            "name": "function",
+            "in": "query",
+            "description": "The name of the function, as returned by the collector.",
+            "required": true,
+            "allowEmptyValue": false,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "timeout",
+            "in": "query",
+            "description": "The timeout in seconds to wait for the function to complete.",
+            "required": false,
+            "schema": {
+              "type": "number",
+              "format": "integer",
+              "default": 10
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "The collector function has been executed successfully. Each collector may return a different type of content."
+          },
+          "400": {
+            "description": "The request was rejected by the collector."
+          },
+          "404": {
+            "description": "The requested function is not found."
+          },
+          "500": {
+            "description": "Other internal error, getting this error means there is a bug in Netdata."
+          },
+          "503": {
+            "description": "The collector to execute the function is not currently available."
+          },
+          "504": {
+            "description": "Timeout while waiting for the collector to execute the function."
+          },
+          "591": {
+            "description": "The collector sent a response, but it was invalid or corrupted."
+          }
+        }
+      }
+    },
+    "/functions": {
+      "get": {
+        "summary": "Get a list of all registered collector functions.",
+        "description": "Collector functions are programs that can be executed on demand.",
+        "responses": {
+          "200": {
+            "description": "A JSON object containing one object per supported function."
+          }
+        }
+      }
+    },
     "/weights": {
       "get": {
         "summary": "Analyze all the metrics using an algorithm and score them accordingly",
diff --git a/web/api/netdata-swagger.yaml b/web/api/netdata-swagger.yaml
index 2e04e9f20..fced6544f 100644
--- a/web/api/netdata-swagger.yaml
+++ b/web/api/netdata-swagger.yaml
@@ -471,7 +471,6 @@ paths:
                 - unaligned
                 - match-ids
                 - match-names
-                - showcustomvars
                 - allow_past
                 - anomaly-bit
             default:
@@ -1351,6 +1350,47 @@ paths:
             that correlated the metrics did not produce any result.
         "504":
           description: Timeout - the query took too long and has been cancelled.
+  /function:
+    get:
+      summary: "Execute a collector function."
+      parameters:
+        - name: function
+          in: query
+          description: The name of the function, as returned by the collector.
+          required: true
+          allowEmptyValue: false
+          schema:
+            type: string
+        - name: timeout
+          in: query
+          description: The timeout in seconds to wait for the function to complete.
+          required: false
+          schema:
+            type: number
+            format: integer
+            default: 10
+      responses:
+        "200":
+          description: The collector function has been executed successfully. Each collector may return a different type of content.
+        "400":
+          description: The request was rejected by the collector.
+        "404":
+          description: The requested function is not found.
+        "500":
+          description: Other internal error, getting this error means there is a bug in Netdata.
+        "503":
+          description: The collector to execute the function is not currently available.
+        "504":
+          description: Timeout while waiting for the collector to execute the function.
+        "591":
+          description: The collector sent a response, but it was invalid or corrupted.
+  /functions:
+    get:
+      summary: Get a list of all registered collector functions.
+      description: Collector functions are programs that can be executed on demand.
+      responses:
+        "200":
+          description: A JSON object containing one object per supported function.
   /weights:
     get:
       summary: "Analyze all the metrics using an algorithm and score them accordingly"
diff --git a/web/api/queries/average/average.h b/web/api/queries/average/average.h
index 55c51722c..b31966886 100644
--- a/web/api/queries/average/average.h
+++ b/web/api/queries/average/average.h
@@ -6,10 +6,10 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_average(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_average(RRDR *r);
-extern void grouping_free_average(RRDR *r);
-extern void grouping_add_average(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_average(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_average(RRDR *r);
+void grouping_free_average(RRDR *r);
+void grouping_add_average(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_average(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERY_AVERAGE_H
diff --git a/web/api/queries/countif/countif.h b/web/api/queries/countif/countif.h
index 0c7d2d7d1..dfe805658 100644
--- a/web/api/queries/countif/countif.h
+++ b/web/api/queries/countif/countif.h
@@ -6,10 +6,10 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_countif(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_countif(RRDR *r);
-extern void grouping_free_countif(RRDR *r);
-extern void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_countif(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_countif(RRDR *r);
+void grouping_free_countif(RRDR *r);
+void grouping_add_countif(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_countif(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERY_COUNTIF_H
diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h
index 8906d14eb..05fa01b34 100644
--- a/web/api/queries/des/des.h
+++ b/web/api/queries/des/des.h
@@ -6,12 +6,12 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_init_des(void);
+void grouping_init_des(void);
 
-extern void grouping_create_des(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_des(RRDR *r);
-extern void grouping_free_des(RRDR *r);
-extern void grouping_add_des(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_des(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_des(RRDR *r);
+void grouping_free_des(RRDR *r);
+void grouping_add_des(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERIES_DES_H
diff --git a/web/api/queries/incremental_sum/incremental_sum.h b/web/api/queries/incremental_sum/incremental_sum.h
index 6d908cef6..c24507fcf 100644
--- a/web/api/queries/incremental_sum/incremental_sum.h
+++ b/web/api/queries/incremental_sum/incremental_sum.h
@@ -6,10 +6,10 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_incremental_sum(RRDR *r);
-extern void grouping_free_incremental_sum(RRDR *r);
-extern void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_incremental_sum(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_incremental_sum(RRDR *r);
+void grouping_free_incremental_sum(RRDR *r);
+void grouping_add_incremental_sum(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_incremental_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERY_INCREMENTAL_SUM_H
diff --git a/web/api/queries/max/max.h b/web/api/queries/max/max.h
index 28913686b..e2427d26d 100644
--- a/web/api/queries/max/max.h
+++ b/web/api/queries/max/max.h
@@ -6,10 +6,10 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_max(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_max(RRDR *r);
-extern void grouping_free_max(RRDR *r);
-extern void grouping_add_max(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_max(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_max(RRDR *r);
+void grouping_free_max(RRDR *r);
+void grouping_add_max(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_max(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERY_MAX_H
diff --git a/web/api/queries/median/median.h b/web/api/queries/median/median.h
index dd1b3de61..9fc159db4 100644
--- a/web/api/queries/median/median.h
+++ b/web/api/queries/median/median.h
@@ -6,18 +6,18 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_median(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median1(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median2(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median3(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median5(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median10(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median15(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median20(RRDR *r, const char *options);
-extern void grouping_create_trimmed_median25(RRDR *r, const char *options);
-extern void grouping_reset_median(RRDR *r);
-extern void grouping_free_median(RRDR *r);
-extern void grouping_add_median(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_median(RRDR *r, const char *options);
+void grouping_create_trimmed_median1(RRDR *r, const char *options);
+void grouping_create_trimmed_median2(RRDR *r, const char *options);
+void grouping_create_trimmed_median3(RRDR *r, const char *options);
+void grouping_create_trimmed_median5(RRDR *r, const char *options);
+void grouping_create_trimmed_median10(RRDR *r, const char *options);
+void grouping_create_trimmed_median15(RRDR *r, const char *options);
+void grouping_create_trimmed_median20(RRDR *r, const char *options);
+void grouping_create_trimmed_median25(RRDR *r, const char *options);
+void grouping_reset_median(RRDR *r);
+void grouping_free_median(RRDR *r);
+void grouping_add_median(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_median(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERIES_MEDIAN_H
diff --git a/web/api/queries/min/min.h b/web/api/queries/min/min.h
index b8627f667..dcdfe252f 100644
--- a/web/api/queries/min/min.h
+++ b/web/api/queries/min/min.h
@@ -6,10 +6,10 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_min(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_min(RRDR *r);
-extern void grouping_free_min(RRDR *r);
-extern void grouping_add_min(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_min(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_min(RRDR *r);
+void grouping_free_min(RRDR *r);
+void grouping_add_min(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_min(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERY_MIN_H
diff --git a/web/api/queries/percentile/percentile.h b/web/api/queries/percentile/percentile.h
index 709717ebd..65e335c11 100644
--- a/web/api/queries/percentile/percentile.h
+++ b/web/api/queries/percentile/percentile.h
@@ -6,18 +6,18 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_percentile25(RRDR *r, const char *options);
-extern void grouping_create_percentile50(RRDR *r, const char *options);
-extern void grouping_create_percentile75(RRDR *r, const char *options);
-extern void grouping_create_percentile80(RRDR *r, const char *options);
-extern void grouping_create_percentile90(RRDR *r, const char *options);
-extern void grouping_create_percentile95(RRDR *r, const char *options);
-extern void grouping_create_percentile97(RRDR *r, const char *options);
-extern void grouping_create_percentile98(RRDR *r, const char *options);
-extern void grouping_create_percentile99(RRDR *r, const char *options );
-extern void grouping_reset_percentile(RRDR *r);
-extern void grouping_free_percentile(RRDR *r);
-extern void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_percentile25(RRDR *r, const char *options);
+void grouping_create_percentile50(RRDR *r, const char *options);
+void grouping_create_percentile75(RRDR *r, const char *options);
+void grouping_create_percentile80(RRDR *r, const char *options);
+void grouping_create_percentile90(RRDR *r, const char *options);
+void grouping_create_percentile95(RRDR *r, const char *options);
+void grouping_create_percentile97(RRDR *r, const char *options);
+void grouping_create_percentile98(RRDR *r, const char *options);
+void grouping_create_percentile99(RRDR *r, const char *options );
+void grouping_reset_percentile(RRDR *r);
+void grouping_free_percentile(RRDR *r);
+void grouping_add_percentile(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_percentile(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERIES_PERCENTILE_H
diff --git a/web/api/queries/query.c b/web/api/queries/query.c
index d776f6d11..ccd195135 100644
--- a/web/api/queries/query.c
+++ b/web/api/queries/query.c
@@ -658,71 +658,6 @@ static void rrdr_set_grouping_function(RRDR *r, RRDR_GROUPING group_method) {
 }
 
 // ----------------------------------------------------------------------------
-
-static void rrdr_disable_not_selected_dimensions(RRDR *r, RRDR_OPTIONS options, const char *dims,
-                                                 struct context_param *context_param_list)
-{
-    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
-    int should_lock = (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE));
-
-    if (should_lock)
-        rrdset_check_rdlock(r->st);
-
-    if(unlikely(!dims || !*dims || (dims[0] == '*' && dims[1] == '\0'))) return;
-
-    int match_ids = 0, match_names = 0;
-
-    if(unlikely(options & RRDR_OPTION_MATCH_IDS))
-        match_ids = 1;
-    if(unlikely(options & RRDR_OPTION_MATCH_NAMES))
-        match_names = 1;
-
-    if(likely(!match_ids && !match_names))
-        match_ids = match_names = 1;
-
-    SIMPLE_PATTERN *pattern = simple_pattern_create(dims, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
-
-    RRDDIM *d;
-    long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
-    for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next) {
-        if(    (match_ids   && simple_pattern_matches(pattern, d->id))
-               || (match_names && simple_pattern_matches(pattern, d->name))
-                ) {
-            r->od[c] |= RRDR_DIMENSION_SELECTED;
-            if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) r->od[c] &= ~RRDR_DIMENSION_HIDDEN;
-            dims_selected++;
-
-            // since the user needs this dimension
-            // make it appear as NONZERO, to return it
-            // even if the dimension has only zeros
-            // unless option non_zero is set
-            if(unlikely(!(options & RRDR_OPTION_NONZERO)))
-                r->od[c] |= RRDR_DIMENSION_NONZERO;
-
-            // count the visible dimensions
-            if(likely(r->od[c] & RRDR_DIMENSION_NONZERO))
-                dims_not_hidden_not_zero++;
-        }
-        else {
-            r->od[c] |= RRDR_DIMENSION_HIDDEN;
-            if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;
-        }
-    }
-    simple_pattern_free(pattern);
-
-    // check if all dimensions are hidden
-    if(unlikely(!dims_not_hidden_not_zero && dims_selected)) {
-        // there are a few selected dimensions,
-        // but they are all zero
-        // enable the selected ones
-        // to avoid returning an empty chart
-        for(c = 0, d = temp_rd?temp_rd:r->st->dimensions; d ;c++, d = d->next)
-            if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED))
-                r->od[c] |= RRDR_DIMENSION_NONZERO;
-    }
-}
-
-// ----------------------------------------------------------------------------
 // helpers to find our way in RRDR
 
 static inline RRDR_VALUE_FLAGS *UNUSED_FUNCTION(rrdr_line_options)(RRDR *r, long rrdr_line) {
@@ -736,13 +671,13 @@ static inline NETDATA_DOUBLE *UNUSED_FUNCTION(rrdr_line_values)(RRDR *r, long rr
 static inline long rrdr_line_init(RRDR *r, time_t t, long rrdr_line) {
     rrdr_line++;
 
-    internal_error(rrdr_line >= r->n,
-                   "QUERY: requested to step above RRDR size for chart '%s'",
-                   r->st->name);
+    internal_error(rrdr_line >= (long)r->n,
+                   "QUERY: requested to step above RRDR size for query '%s'",
+                   r->internal.qt->id);
 
     internal_error(r->t[rrdr_line] != 0 && r->t[rrdr_line] != t,
-                   "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of chart '%s'",
-                   (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->st->name);
+                   "QUERY: overwriting the timestamp of RRDR line %zu from %zu to %zu, of query '%s'",
+                   (size_t)rrdr_line, (size_t)r->t[rrdr_line], (size_t)t, r->internal.qt->id);
 
     // save the time
     r->t[rrdr_line] = t;
@@ -758,68 +693,137 @@ static inline void rrdr_done(RRDR *r, long rrdr_line) {
 // ----------------------------------------------------------------------------
 // tier management
 
-static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted, time_t before_wanted, long points_wanted) {
+static bool query_metric_is_valid_tier(QUERY_METRIC *qm, size_t tier) {
+    if(!qm->tiers[tier].db_metric_handle || !qm->tiers[tier].db_first_time_t || !qm->tiers[tier].db_last_time_t || !qm->tiers[tier].db_update_every)
+        return false;
+
+    return true;
+}
+
+static size_t query_metric_first_working_tier(QUERY_METRIC *qm) {
+    for(size_t tier = 0; tier < storage_tiers ; tier++) {
+
+        // find the db time-range for this tier for all metrics
+        STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
+        time_t first_t = qm->tiers[tier].db_first_time_t;
+        time_t last_t  = qm->tiers[tier].db_last_time_t;
+        time_t update_every = qm->tiers[tier].db_update_every;
+
+        if(!db_metric_handle || !first_t || !last_t || !update_every)
+            continue;
+
+        return tier;
+    }
+
+    return 0;
+}
+
+static long query_plan_points_coverage_weight(time_t db_first_t, time_t db_last_t, time_t db_update_every, time_t after_wanted, time_t before_wanted, size_t points_wanted, size_t tier __maybe_unused) {
+    if(db_first_t == 0 || db_last_t == 0 || db_update_every == 0)
+        return -LONG_MAX;
+
+    time_t common_first_t = MAX(db_first_t, after_wanted);
+    time_t common_last_t = MIN(db_last_t, before_wanted);
+
+    long time_coverage = (common_last_t - common_first_t) * 1000000 / (before_wanted - after_wanted);
+    size_t points_wanted_in_coverage = points_wanted * time_coverage / 1000000;
+
+    long points_available = (common_last_t - common_first_t) / db_update_every;
+    long points_delta = (long)(points_available - points_wanted_in_coverage);
+    long points_coverage = (points_delta < 0) ? (long)(points_available * time_coverage / points_wanted_in_coverage) : time_coverage;
+
+    // a way to benefit higher tiers
+    // points_coverage += (long)tier * 10000;
+
+    if(points_available <= 0)
+        return -LONG_MAX;
+
+    return points_coverage;
+}
+
+static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
     if(unlikely(storage_tiers < 2))
         return 0;
 
-    if(unlikely(after_wanted == before_wanted || points_wanted <= 0 || !rd || !rd->rrdset)) {
+    if(unlikely(after_wanted == before_wanted || points_wanted <= 0))
+        return query_metric_first_working_tier(qm);
 
-        if(!rd)
-            internal_error(true, "QUERY: NULL dimension - invalid params to tier calculation");
-        else
-            internal_error(true, "QUERY: chart '%s' dimension '%s' invalid params to tier calculation",
-                           (rd->rrdset)?rd->rrdset->name:"unknown", rd->name);
+    long weight[storage_tiers];
 
-        return 0;
+    for(size_t tier = 0; tier < storage_tiers ; tier++) {
+
+        // find the db time-range for this tier for all metrics
+        STORAGE_METRIC_HANDLE *db_metric_handle = qm->tiers[tier].db_metric_handle;
+        time_t first_t = qm->tiers[tier].db_first_time_t;
+        time_t last_t  = qm->tiers[tier].db_last_time_t;
+        time_t update_every = qm->tiers[tier].db_update_every;
+
+        if(!db_metric_handle || !first_t || !last_t || !update_every) {
+            weight[tier] = -LONG_MAX;
+            continue;
+        }
+
+        weight[tier] = query_plan_points_coverage_weight(first_t, last_t, update_every, after_wanted, before_wanted, points_wanted, tier);
     }
 
-    //BUFFER *wb = buffer_create(1000);
-    //buffer_sprintf(wb, "Best tier for chart '%s', dim '%s', from %ld to %ld (dur %ld, every %d), points %ld",
-    //               rd->rrdset->name, rd->name, after_wanted, before_wanted, before_wanted - after_wanted, rd->update_every, points_wanted);
+    size_t best_tier = 0;
+    for(size_t tier = 1; tier < storage_tiers ; tier++) {
+        if(weight[tier] >= weight[best_tier])
+            best_tier = tier;
+    }
+
+    return best_tier;
+}
+
+static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
+    if(unlikely(storage_tiers < 2))
+        return 0;
+
+    if(unlikely(after_wanted == before_wanted || points_wanted <= 0)) {
+        internal_error(true, "QUERY: '%s' has invalid params to tier calculation", qt->id);
+        return 0;
+    }
 
     long weight[storage_tiers];
 
-    for(int tier = 0; tier < storage_tiers ; tier++) {
-        if(unlikely(!rd->tiers[tier])) {
-            internal_error(true, "QUERY: tier %d of chart '%s' dimension '%s' not initialized",
-                           tier, rd->rrdset->name, rd->name);
-    //        buffer_free(wb);
-            return 0;
-        }
+    for(size_t tier = 0; tier < storage_tiers ; tier++) {
 
-        time_t first_t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
-        time_t last_t  = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
+        time_t common_first_t = 0;
+        time_t common_last_t = 0;
+        time_t common_update_every = 0;
 
-        time_t common_after = MAX(first_t, after_wanted);
-        time_t common_before = MIN(last_t, before_wanted);
+        // find the db time-range for this tier for all metrics
+        for(size_t i = 0, used = qt->query.used; i < used ; i++) {
+            QUERY_METRIC *qm = &qt->query.array[i];
 
-        long time_coverage = (common_before - common_after) * 1000 / (before_wanted - after_wanted);
-        if(time_coverage < 0) time_coverage = 0;
+            time_t first_t = qm->tiers[tier].db_first_time_t;
+            time_t last_t  = qm->tiers[tier].db_last_time_t;
+            time_t update_every = qm->tiers[tier].db_update_every;
 
-        int update_every = (int)rd->tiers[tier]->tier_grouping * (int)rd->update_every;
-        if(unlikely(update_every == 0)) {
-            internal_error(true, "QUERY: update_every of tier %d for chart '%s' dimension '%s' is zero. tg = %d, ue = %d",
-                           tier, rd->rrdset->name, rd->name, rd->tiers[tier]->tier_grouping, rd->update_every);
-    //        buffer_free(wb);
-            return 0;
-        }
+            if(!first_t || !last_t || !update_every)
+                continue;
 
-        long points_available = (before_wanted - after_wanted) / update_every;
-        long points_delta = points_available - points_wanted;
-        long points_coverage = (points_delta < 0) ? points_available * 1000 / points_wanted: 1000;
+            if(!common_first_t)
+                common_first_t = first_t;
+            else
+                common_first_t = MIN(first_t, common_first_t);
 
-        if(points_available <= 0)
-            weight[tier] = -LONG_MAX;
-        else
-            weight[tier] = points_coverage;
+            if(!common_last_t)
+                common_last_t = last_t;
+            else
+                common_last_t = MAX(last_t, common_last_t);
 
-    //    buffer_sprintf(wb, ": tier %d, first %ld, last %ld (dur %ld, tg %d, every %d), points %ld, tcoverage %ld, pcoverage %ld, weight %ld",
-    //                   tier, first_t, last_t, last_t - first_t, rd->tiers[tier]->tier_grouping, update_every,
-    //                   points_available, time_coverage, points_coverage, weight[tier]);
+            if(!common_update_every)
+                common_update_every = update_every;
+            else
+                common_update_every = MIN(update_every, common_update_every);
+        }
+
+        weight[tier] = query_plan_points_coverage_weight(common_first_t, common_last_t, common_update_every, after_wanted, before_wanted, points_wanted, tier);
     }
 
-    int best_tier = 0;
-    for(int tier = 1; tier < storage_tiers ; tier++) {
+    size_t best_tier = 0;
+    for(size_t tier = 1; tier < storage_tiers ; tier++) {
         if(weight[tier] >= weight[best_tier])
             best_tier = tier;
     }
@@ -827,47 +831,30 @@ static int rrddim_find_best_tier_for_timeframe(RRDDIM *rd, time_t after_wanted,
     if(weight[best_tier] == -LONG_MAX)
         best_tier = 0;
 
-    //buffer_sprintf(wb, ": final best tier %d", best_tier);
-    //internal_error(true, "%s", buffer_tostring(wb));
-    //buffer_free(wb);
-
     return best_tier;
 }
 
-static int rrdset_find_natural_update_every_for_timeframe(RRDSET *st, time_t after_wanted, time_t before_wanted, long points_wanted, RRDR_OPTIONS options, int tier) {
-    int ret = st->update_every;
-
-    if(unlikely(!st->dimensions))
-        return ret;
-
-    rrdset_rdlock(st);
-    int best_tier;
-
-    if(options & RRDR_OPTION_SELECTED_TIER && tier >= 0 && tier < storage_tiers)
+static time_t rrdset_find_natural_update_every_for_timeframe(QUERY_TARGET *qt, time_t after_wanted, time_t before_wanted, size_t points_wanted, RRDR_OPTIONS options, size_t tier) {
+    size_t best_tier;
+    if((options & RRDR_OPTION_SELECTED_TIER) && tier < storage_tiers)
         best_tier = tier;
     else
-        best_tier = rrddim_find_best_tier_for_timeframe(st->dimensions, after_wanted, before_wanted, points_wanted);
+        best_tier = rrddim_find_best_tier_for_timeframe(qt, after_wanted, before_wanted, points_wanted);
 
-    if(!st->dimensions->tiers[best_tier]) {
-        internal_error(
-            true,
-            "QUERY: tier %d on chart '%s', is not initialized", best_tier, st->name);
-    }
-    else {
-        ret = (int)st->dimensions->tiers[best_tier]->tier_grouping * (int)st->update_every;
-        if(unlikely(!ret)) {
-            internal_error(
-                true,
-                "QUERY: update_every calculated to be zero on chart '%s', tier_grouping %d, update_every %d",
-                st->name, st->dimensions->tiers[best_tier]->tier_grouping, st->update_every);
-
-            ret = st->update_every;
-        }
-    }
+    // find the db minimum update every for this tier for all metrics
+    time_t common_update_every = default_rrd_update_every;
+    for(size_t i = 0, used = qt->query.used; i < used ; i++) {
+        QUERY_METRIC *qm = &qt->query.array[i];
 
-    rrdset_unlock(st);
+        time_t update_every = qm->tiers[best_tier].db_update_every;
 
-    return ret;
+        if(!i)
+            common_update_every = update_every;
+        else
+            common_update_every = MIN(update_every, common_update_every);
+    }
+
+    return common_update_every;
 }
 
 // ----------------------------------------------------------------------------
@@ -915,7 +902,7 @@ typedef struct query_plan {
 typedef struct query_engine_ops {
     // configuration
     RRDR *r;
-    RRDDIM *rd;
+    QUERY_METRIC *qm;
     time_t view_update_every;
     time_t query_granularity;
     TIER_QUERY_FETCH tier_query_fetch;
@@ -927,11 +914,11 @@ typedef struct query_engine_ops {
 
     // storage queries
     size_t tier;
-    struct rrddim_tier *tier_ptr;
-    struct rrddim_query_handle handle;
-    STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
-    int (*is_finished)(struct rrddim_query_handle *handle);
-    void (*finalize)(struct rrddim_query_handle *handle);
+    struct query_metric_tier *tier_ptr;
+    struct storage_engine_query_handle handle;
+    STORAGE_POINT (*next_metric)(struct storage_engine_query_handle *handle);
+    int (*is_finished)(struct storage_engine_query_handle *handle);
+    void (*finalize)(struct storage_engine_query_handle *handle);
 
     // aggregating points over time
     void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
@@ -963,11 +950,11 @@ static void query_planer_activate_plan(QUERY_ENGINE_OPS *ops, size_t plan_id, ti
         after = overwrite_after;
 
     ops->tier = ops->plan.data[plan_id].tier;
-    ops->tier_ptr = ops->rd->tiers[ops->tier];
-    ops->tier_ptr->query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before, ops->r->internal.tier_query_fetch);
-    ops->next_metric = ops->tier_ptr->query_ops.next_metric;
-    ops->is_finished = ops->tier_ptr->query_ops.is_finished;
-    ops->finalize = ops->tier_ptr->query_ops.finalize;
+    ops->tier_ptr = &ops->qm->tiers[ops->tier];
+    ops->tier_ptr->eng->api.query_ops.init(ops->tier_ptr->db_metric_handle, &ops->handle, after, before);
+    ops->next_metric = ops->tier_ptr->eng->api.query_ops.next_metric;
+    ops->is_finished = ops->tier_ptr->eng->api.query_ops.is_finished;
+    ops->finalize = ops->tier_ptr->eng->api.query_ops.finalize;
     ops->current_plan = plan_id;
     ops->current_plan_expire_time = ops->plan.data[plan_id].before;
 }
@@ -976,26 +963,38 @@ static void query_planer_next_plan(QUERY_ENGINE_OPS *ops, time_t now, time_t las
     internal_error(now < ops->current_plan_expire_time && now < ops->plan.data[ops->current_plan].before,
                    "QUERY: switching query plan too early!");
 
+    size_t old_plan = ops->current_plan;
+
     time_t next_plan_before_time;
     do {
         ops->current_plan++;
 
         if (ops->current_plan >= ops->plan.entries) {
-            ops->current_plan = ops->plan.entries - 1;
+            ops->current_plan = old_plan;
+            ops->current_plan_expire_time = ops->r->internal.qt->window.before;
+            // let the query run with current plan
+            // we will not switch it
             return;
         }
 
         next_plan_before_time = ops->plan.data[ops->current_plan].before;
     } while(now >= next_plan_before_time || last_point_end_time >= next_plan_before_time);
 
+    if(!query_metric_is_valid_tier(ops->qm, ops->plan.data[ops->current_plan].tier)) {
+        ops->current_plan = old_plan;
+        ops->current_plan_expire_time = ops->r->internal.qt->window.before;
+        return;
+    }
+
     if(ops->finalize) {
         ops->finalize(&ops->handle);
         ops->finalize = NULL;
+        ops->is_finished = NULL;
     }
 
-    query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
-
     // internal_error(true, "QUERY: switched plan to %zu (all is %zu), previous expiration was %ld, this starts at %ld, now is %ld, last_point_end_time %ld", ops->current_plan, ops->plan.entries, ops->plan.data[ops->current_plan-1].before, ops->plan.data[ops->current_plan].after, now, last_point_end_time);
+
+    query_planer_activate_plan(ops, ops->current_plan, MIN(now, last_point_end_time));
 }
 
 static int compare_query_plan_entries_on_start_time(const void *a, const void *b) {
@@ -1004,21 +1003,20 @@ static int compare_query_plan_entries_on_start_time(const void *a, const void *b
     return (p1->after < p2->after)?-1:1;
 }
 
-static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, long points_wanted) {
-    RRDDIM *rd = ops->rd;
-
+static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
     //BUFFER *wb = buffer_create(1000);
     //buffer_sprintf(wb, "QUERY PLAN for chart '%s' dimension '%s', from %ld to %ld:", rd->rrdset->name, rd->name, after_wanted, before_wanted);
 
     // put our selected tier as the first plan
     size_t selected_tier;
 
-    if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER && ops->r->internal.query_tier >= 0 && ops->r->internal.query_tier < storage_tiers) {
-        selected_tier = ops->r->internal.query_tier;
+    if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER
+       && ops->r->internal.qt->window.tier < storage_tiers
+       && query_metric_is_valid_tier(ops->qm, ops->r->internal.qt->window.tier)) {
+        selected_tier = ops->r->internal.qt->window.tier;
     }
     else {
-
-        selected_tier = rrddim_find_best_tier_for_timeframe(rd, after_wanted, before_wanted, points_wanted);
+        selected_tier = query_metric_best_tier_for_timeframe(ops->qm, after_wanted, before_wanted, points_wanted);
 
         if(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)
             ops->r->internal.query_options &= ~RRDR_OPTION_SELECTED_TIER;
@@ -1026,8 +1024,8 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
 
     ops->plan.entries = 1;
     ops->plan.data[0].tier = selected_tier;
-    ops->plan.data[0].after = rd->tiers[selected_tier]->query_ops.oldest_time(rd->tiers[selected_tier]->db_metric_handle);
-    ops->plan.data[0].before = rd->tiers[selected_tier]->query_ops.latest_time(rd->tiers[selected_tier]->db_metric_handle);
+    ops->plan.data[0].after = ops->qm->tiers[selected_tier].db_first_time_t;
+    ops->plan.data[0].before = ops->qm->tiers[selected_tier].db_last_time_t;
 
     if(!(ops->r->internal.query_options & RRDR_OPTION_SELECTED_TIER)) {
         // the selected tier
@@ -1039,9 +1037,12 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
         // check if our selected tier can start the query
         if (selected_tier_first_time_t > after_wanted) {
             // we need some help from other tiers
-            for (int tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
+            for (size_t tr = (int)selected_tier + 1; tr < storage_tiers; tr++) {
+                if(!query_metric_is_valid_tier(ops->qm, tr))
+                    continue;
+
                 // find the first time of this tier
-                time_t first_time_t = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
+                time_t first_time_t = ops->qm->tiers[tr].db_first_time_t;
 
                 //buffer_sprintf(wb, ": EVAL AFTER tier %d, %ld", tier, first_time_t);
 
@@ -1067,8 +1068,11 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
         if (selected_tier_last_time_t < before_wanted) {
             // we need some help from other tiers
             for (int tr = (int)selected_tier - 1; tr >= 0; tr--) {
+                if(!query_metric_is_valid_tier(ops->qm, tr))
+                    continue;
+
                 // find the last time of this tier
-                time_t last_time_t = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
+                time_t last_time_t = ops->qm->tiers[tr].db_last_time_t;
 
                 //buffer_sprintf(wb, ": EVAL BEFORE tier %d, %ld", tier, last_time_t);
 
@@ -1096,8 +1100,11 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
         qsort(&ops->plan.data, ops->plan.entries, sizeof(QUERY_PLAN_ENTRY), compare_query_plan_entries_on_start_time);
 
     // make sure it has the whole timeframe we need
-    ops->plan.data[0].after = after_wanted;
-    ops->plan.data[ops->plan.entries - 1].before = before_wanted;
+    if(ops->plan.data[0].after < after_wanted)
+        ops->plan.data[0].after = after_wanted;
+
+    if(ops->plan.data[ops->plan.entries - 1].before > before_wanted)
+        ops->plan.data[ops->plan.entries - 1].before = before_wanted;
 
     //buffer_sprintf(wb, ": FINAL STEPS %zu", ops->plan.entries);
 
@@ -1106,7 +1113,12 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
 
     //internal_error(true, "%s", buffer_tostring(wb));
 
+    if(!query_metric_is_valid_tier(ops->qm, ops->plan.data[0].tier))
+        return false;
+
     query_planer_activate_plan(ops, 0, 0);
+
+    return true;
 }
 
 
@@ -1146,14 +1158,17 @@ static void query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
     (ops).group_anomaly_rate += (point).anomaly;                        \
 } while(0)
 
-static inline void rrd2rrdr_do_dimension(
-    RRDR *r
-    , long points_wanted
-    , RRDDIM *rd
-    , long dim_id_in_rrdr
-    , time_t after_wanted
-    , time_t before_wanted
-){
+static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) {
+    QUERY_TARGET *qt = r->internal.qt;
+    QUERY_METRIC *qm = &qt->query.array[dim_id_in_rrdr];
+    size_t points_wanted = qt->window.points;
+    time_t after_wanted = qt->window.after;
+    time_t before_wanted = qt->window.before;
+
+//    bool debug_this = false;
+//    if(strcmp("user", string2str(rd->id)) == 0 && strcmp("system.cpu", string2str(rd->rrdset->id)) == 0)
+//        debug_this = true;
+
     time_t max_date = 0,
            min_date = 0;
 
@@ -1161,19 +1176,20 @@ static inline void rrd2rrdr_do_dimension(
 
     QUERY_ENGINE_OPS ops = {
         .r = r,
-        .rd = rd,
+        .qm = qm,
         .grouping_add = r->internal.grouping_add,
         .grouping_flush = r->internal.grouping_flush,
         .tier_query_fetch = r->internal.tier_query_fetch,
         .view_update_every = r->update_every,
-        .query_granularity = r->update_every / r->group,
+        .query_granularity = (time_t)(r->update_every / r->group),
         .group_value_flags = RRDR_VALUE_NOTHING
     };
 
     long rrdr_line = -1;
     bool use_anomaly_bit_as_value = (r->internal.query_options & RRDR_OPTION_ANOMALY_BIT) ? true : false;
 
-    query_plan(&ops, after_wanted, before_wanted, points_wanted);
+    if(!query_plan(&ops, after_wanted, before_wanted, points_wanted))
+        return;
 
     NETDATA_DOUBLE min = r->min, max = r->max;
 
@@ -1184,15 +1200,18 @@ static inline void rrd2rrdr_do_dimension(
     time_t now_start_time = after_wanted - ops.query_granularity;
     time_t now_end_time   = after_wanted + ops.view_update_every - ops.query_granularity;
 
+    size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch;
+
     // The main loop, based on the query granularity we need
-    for( ; (long)points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {
+    for( ; points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {
 
-        if(query_plan_should_switch_plan(ops, now_end_time))
+        if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) {
             query_planer_next_plan(&ops, now_end_time, new_point.end_time);
+            db_points_read_since_plan_switch = 0;
+        }
 
         // read all the points of the db, prior to the time we need (now_end_time)
 
-
         size_t count_same_end_time = 0;
         while(count_same_end_time < 100) {
             if(likely(count_same_end_time == 0)) {
@@ -1208,11 +1227,15 @@ static inline void rrd2rrdr_do_dimension(
                 new_point = QUERY_POINT_EMPTY;
                 new_point.start_time = last1_point.end_time;
                 new_point.end_time   = now_end_time;
+//
+//                if(debug_this) info("QUERY: is finished() returned true");
+//
                 break;
             }
 
             // fetch the new point
             {
+                db_points_read_since_plan_switch++;
                 STORAGE_POINT sp = ops.next_metric(&ops.handle);
 
                 ops.db_points_read_per_tier[ops.tier]++;
@@ -1223,6 +1246,10 @@ static inline void rrd2rrdr_do_dimension(
                 new_point.anomaly    = sp.count ? (NETDATA_DOUBLE)sp.anomaly_count * 100.0 / (NETDATA_DOUBLE)sp.count : 0.0;
                 query_point_set_id(new_point, ops.db_total_points_read);
 
+//                if(debug_this)
+//                    info("QUERY: got point %zu, from time %ld to %ld   //   now from %ld to %ld   //   query from %ld to %ld",
+//                         new_point.id, new_point.start_time, new_point.end_time, now_start_time, now_end_time, after_wanted, before_wanted);
+//
                 // set the right value to the point we got
                 if(likely(!storage_point_is_unset(sp) && !storage_point_is_empty(sp))) {
 
@@ -1258,17 +1285,18 @@ static inline void rrd2rrdr_do_dimension(
 
             // check if the db is giving us zero duration points
             if(unlikely(new_point.start_time == new_point.end_time)) {
-                internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu start time %ld, end time %ld, that are both equal",
-                               rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time);
+                internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned point %zu start time %ld, end time %ld, that are both equal",
+                               qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time);
 
-                new_point.start_time = new_point.end_time - ((time_t)ops.tier_ptr->tier_grouping * (time_t)ops.rd->update_every);
+                new_point.start_time = new_point.end_time - ops.tier_ptr->db_update_every;
             }
 
             // check if the db is advancing the query
             if(unlikely(new_point.end_time <= last1_point.end_time)) {
-                internal_error(true, "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, before the last point %zu end time %ld, now is %ld to %ld",
-                               rd->rrdset->name, rd->name, new_point.id, new_point.start_time, new_point.end_time,
-                               last1_point.id, last1_point.end_time, now_start_time, now_end_time);
+                internal_error(db_points_read_since_plan_switch > 1,
+                               "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld to %ld, before the last point %zu from %ld to %ld, now is %ld to %ld",
+                               qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time,
+                               last1_point.id, last1_point.start_time, last1_point.end_time, now_start_time, now_end_time);
 
                 count_same_end_time++;
                 continue;
@@ -1294,8 +1322,8 @@ static inline void rrd2rrdr_do_dimension(
 
                     // we only log if this is not point 1
                     internal_error(new_point.end_time < after_wanted && new_point.id > 1,
-                                   "QUERY: next_metric(%s, %s) returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",
-                                   rd->rrdset->name, rd->name,
+                                   "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld time %ld, which is entirely before our current timeframe %ld to %ld (and before the entire query, after %ld, before %ld)",
+                                   qt->id, string2str(qm->dimension.id),
                                    new_point.id, new_point.start_time, new_point.end_time,
                                    now_start_time, now_end_time,
                                    after_wanted, before_wanted);
@@ -1311,8 +1339,8 @@ static inline void rrd2rrdr_do_dimension(
 
         if(unlikely(count_same_end_time)) {
             internal_error(true,
-                           "QUERY: the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",
-                           last1_point.end_time, count_same_end_time);
+                           "QUERY: '%s', dimension '%s', the database does not advance the query, it returned an end time less or equal to the end time of the last point we got %ld, %zu times",
+                           qt->id, string2str(qm->dimension.id), last1_point.end_time, count_same_end_time);
 
             if(unlikely(new_point.end_time <= last1_point.end_time))
                 new_point.end_time = now_end_time;
@@ -1323,7 +1351,7 @@ static inline void rrd2rrdr_do_dimension(
         // we select the one to use based on their timestamps
 
         size_t iterations = 0;
-        for ( ; now_end_time <= new_point.end_time && (long)points_added < points_wanted ;
+        for ( ; now_end_time <= new_point.end_time && points_added < points_wanted ;
                 now_end_time += ops.view_update_every, iterations++) {
 
             // now_start_time is wrong in this loop
@@ -1336,22 +1364,35 @@ static inline void rrd2rrdr_do_dimension(
                 current_point = new_point;
                 query_interpolate_point(current_point, last1_point, now_end_time);
 
-                internal_error(current_point.id > 0 && last1_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
-                               "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
-                               " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point1 in this query.",
-                               rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
-                               current_point.id, current_point.start_time, current_point.end_time, now_end_time);
+//                internal_error(current_point.id > 0
+//                                && last1_point.id == 0
+//                                && current_point.end_time > after_wanted
+//                                && current_point.end_time > now_end_time,
+//                               "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
+//                               " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
+//                               " but we could really favor by having last_point1 in this query.",
+//                               qt->id, string2str(qm->dimension.id),
+//                               after_wanted, before_wanted,
+//                               ops.view_update_every, ops.query_granularity,
+//                               current_point.id, current_point.start_time, current_point.end_time,
+//                               now_end_time);
             }
             else if(likely(now_end_time <= last1_point.end_time)) {
                 // our LAST point is still valid
                 current_point = last1_point;
                 query_interpolate_point(current_point, last2_point, now_end_time);
 
-                internal_error(current_point.id > 0 && last2_point.id == 0 && current_point.end_time > after_wanted && current_point.end_time > now_end_time,
-                               "QUERY: on '%s', dim '%s', after %ld, before %ld, view update every %ld, query granularity %ld,"
-                               " interpolating point %zu (from %ld to %ld) at %ld, but we could really favor by having last_point2 in this query.",
-                               rd->rrdset->name, rd->name, after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
-                               current_point.id, current_point.start_time, current_point.end_time, now_end_time);
+//                internal_error(current_point.id > 0
+//                                && last2_point.id == 0
+//                                && current_point.end_time > after_wanted
+//                                && current_point.end_time > now_end_time,
+//                               "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
+//                               " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
+//                               " but we could really favor by having last_point2 in this query.",
+//                               qt->id, string2str(qm->dimension.id),
+//                               after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
+//                               current_point.id, current_point.start_time, current_point.end_time,
+//                               now_end_time);
             }
             else {
                 // a GAP, we don't have a value this time
@@ -1414,7 +1455,7 @@ static inline void rrd2rrdr_do_dimension(
 
     r->internal.result_points_generated += points_added;
     r->internal.db_points_read += ops.db_total_points_read;
-    for(int tr = 0; tr < storage_tiers ; tr++)
+    for(size_t tr = 0; tr < storage_tiers ; tr++)
         r->internal.tier_points_read[tr] += ops.db_points_read_per_tier[tr];
 
     r->min = min;
@@ -1423,24 +1464,26 @@ static inline void rrd2rrdr_do_dimension(
     r->after = min_date - ops.view_update_every + ops.query_granularity;
     rrdr_done(r, rrdr_line);
 
-    internal_error((long)points_added != points_wanted,
-                   "QUERY: query on %s/%s requested %zu points, but RRDR added %zu (%zu db points read).",
-                   r->st->name, rd->name, (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
+    internal_error(points_added != points_wanted,
+                   "QUERY: '%s', dimension '%s', requested %zu points, but RRDR added %zu (%zu db points read).",
+                   qt->id, string2str(qm->dimension.id),
+                   (size_t)points_wanted, (size_t)points_added, ops.db_total_points_read);
 }
 
 // ----------------------------------------------------------------------------
 // fill the gap of a tier
 
-extern void store_metric_at_tier(RRDDIM *rd, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
+void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);
+void store_metric_collection_completed(void);
 
-void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
-    if(unlikely(tier < 0 || tier >= storage_tiers)) return;
+void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now) {
+    if(unlikely(tier >= storage_tiers)) return;
     if(storage_tiers_backfill[tier] == RRD_BACKFILL_NONE) return;
 
     struct rrddim_tier *t = rd->tiers[tier];
     if(unlikely(!t)) return;
 
-    time_t latest_time_t = t->query_ops.latest_time(t->db_metric_handle);
+    time_t latest_time_t = t->query_ops->latest_time(t->db_metric_handle);
     time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every;
     time_t time_diff   = now - latest_time_t;
 
@@ -1450,43 +1493,40 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now) {
     // there is really nothing we can do
     if(now <= latest_time_t || time_diff < granularity) return;
 
-    struct rrddim_query_handle handle;
-
-    size_t all_points_read = 0;
+    struct storage_engine_query_handle handle;
 
     // for each lower tier
-    for(int tr = tier - 1; tr >= 0 ;tr--){
-        time_t smaller_tier_first_time = rd->tiers[tr]->query_ops.oldest_time(rd->tiers[tr]->db_metric_handle);
-        time_t smaller_tier_last_time = rd->tiers[tr]->query_ops.latest_time(rd->tiers[tr]->db_metric_handle);
+    for(int tr = (int)tier - 1; tr >= 0 ;tr--){
+        time_t smaller_tier_first_time = rd->tiers[tr]->query_ops->oldest_time(rd->tiers[tr]->db_metric_handle);
+        time_t smaller_tier_last_time = rd->tiers[tr]->query_ops->latest_time(rd->tiers[tr]->db_metric_handle);
         if(smaller_tier_last_time <= latest_time_t) continue;  // it is as bad as we are
 
         long after_wanted = (latest_time_t < smaller_tier_first_time) ? smaller_tier_first_time : latest_time_t;
         long before_wanted = smaller_tier_last_time;
 
         struct rrddim_tier *tmp = rd->tiers[tr];
-        tmp->query_ops.init(tmp->db_metric_handle, &handle, after_wanted, before_wanted, TIER_QUERY_FETCH_AVERAGE);
+        tmp->query_ops->init(tmp->db_metric_handle, &handle, after_wanted, before_wanted);
 
-        size_t points = 0;
+        size_t points_read = 0;
 
-        while(!tmp->query_ops.is_finished(&handle)) {
+        while(!tmp->query_ops->is_finished(&handle)) {
 
-            STORAGE_POINT sp = tmp->query_ops.next_metric(&handle);
+            STORAGE_POINT sp = tmp->query_ops->next_metric(&handle);
+            points_read++;
 
             if(sp.end_time > latest_time_t) {
                 latest_time_t = sp.end_time;
-                store_metric_at_tier(rd, t, sp, sp.end_time * USEC_PER_SEC);
-                points++;
+                store_metric_at_tier(rd, tr, t, sp, sp.end_time * USEC_PER_SEC);
             }
         }
 
-        all_points_read += points;
-        tmp->query_ops.finalize(&handle);
+        tmp->query_ops->finalize(&handle);
+        store_metric_collection_completed();
+        global_statistics_backfill_query_completed(points_read);
 
         //internal_error(true, "DBENGINE: backfilled chart '%s', dimension '%s', tier %d, from %ld to %ld, with %zu points from tier %d",
         //               rd->rrdset->name, rd->name, tier, after_wanted, before_wanted, points, tr);
     }
-
-    rrdr_query_completed(all_points_read, all_points_read);
 }
 
 // ----------------------------------------------------------------------------
@@ -1497,29 +1537,33 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
         , RRDR_OPTIONS options __maybe_unused
         , RRDR_GROUPING group_method
         , bool aligned
-        , long group
-        , long resampling_time
-        , long resampling_group
+        , size_t group
+        , time_t resampling_time
+        , size_t resampling_group
         , time_t after_wanted
         , time_t after_requested
         , time_t before_wanted
         , time_t before_requested
-        , long points_requested
-        , long points_wanted
+        , size_t points_requested
+        , size_t points_wanted
         //, size_t after_slot
         //, size_t before_slot
         , const char *msg
         ) {
-    netdata_rwlock_rdlock(&r->st->rrdset_rwlock);
-    info("INTERNAL ERROR: rrd2rrdr() on %s update every %d with %s grouping %s (group: %ld, resampling_time: %ld, resampling_group: %ld), "
-         "after (got: %zu, want: %zu, req: %ld, db: %zu), "
-         "before (got: %zu, want: %zu, req: %ld, db: %zu), "
-         "duration (got: %zu, want: %zu, req: %ld, db: %zu), "
-         //"slot (after: %zu, before: %zu, delta: %zu), "
-         "points (got: %ld, want: %ld, req: %ld, db: %ld), "
+
+    time_t first_entry_t = r->internal.qt->db.first_time_t;
+    time_t last_entry_t = r->internal.qt->db.last_time_t;
+
+    internal_error(
+         true,
+         "rrd2rrdr() on %s update every %ld with %s grouping %s (group: %zu, resampling_time: %ld, resampling_group: %zu), "
+         "after (got: %ld, want: %ld, req: %ld, db: %ld), "
+         "before (got: %ld, want: %ld, req: %ld, db: %ld), "
+         "duration (got: %ld, want: %ld, req: %ld, db: %ld), "
+         "points (got: %zu, want: %zu, req: %zu), "
          "%s"
-         , r->st->name
-         , r->st->update_every
+         , r->internal.qt->id
+         , r->internal.qt->window.query_granularity
 
          // grouping
          , (aligned) ? "aligned" : "unaligned"
@@ -1529,45 +1573,36 @@ static void rrd2rrdr_log_request_response_metadata(RRDR *r
          , resampling_group
 
          // after
-         , (size_t)r->after
-         , (size_t)after_wanted
+         , r->after
+         , after_wanted
          , after_requested
-         , (size_t)rrdset_first_entry_t_nolock(r->st)
+         , first_entry_t
 
          // before
-         , (size_t)r->before
-         , (size_t)before_wanted
+         , r->before
+         , before_wanted
          , before_requested
-         , (size_t)rrdset_last_entry_t_nolock(r->st)
+         , last_entry_t
 
          // duration
-         , (size_t)(r->before - r->after + r->st->update_every)
-         , (size_t)(before_wanted - after_wanted + r->st->update_every)
-         , before_requested - after_requested
-         , (size_t)((rrdset_last_entry_t_nolock(r->st) - rrdset_first_entry_t_nolock(r->st)) + r->st->update_every)
-
-         // slot
-         /*
-         , after_slot
-         , before_slot
-         , (after_slot > before_slot) ? (r->st->entries - after_slot + before_slot) : (before_slot - after_slot)
-          */
+         , (long)(r->before - r->after + r->internal.qt->window.query_granularity)
+         , (long)(before_wanted - after_wanted + r->internal.qt->window.query_granularity)
+         , (long)before_requested - after_requested
+         , (long)((last_entry_t - first_entry_t) + r->internal.qt->window.query_granularity)
 
          // points
          , r->rows
          , points_wanted
          , points_requested
-         , r->st->entries
 
          // message
          , msg
     );
-    netdata_rwlock_unlock(&r->st->rrdset_rwlock);
 }
 #endif // NETDATA_INTERNAL_CHECKS
 
 // Returns 1 if an absolute period was requested or 0 if it was a relative period
-int rrdr_relative_window_to_absolute(long long *after, long long *before) {
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before) {
     time_t now = now_realtime_sec() - 1;
 
     int absolute_period_requested = -1;
@@ -1624,10 +1659,25 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
         after_requested  -= delta;
     }
 
+    time_t absolute_minimum_time = now - (10 * 365 * 86400);
+    time_t absolute_maximum_time = now + (1 * 365 * 86400);
+
+    if (after_requested < absolute_minimum_time && !unittest_running)
+        after_requested = absolute_minimum_time;
+
+    if (after_requested > absolute_maximum_time && !unittest_running)
+        after_requested = absolute_maximum_time;
+
+    if (before_requested < absolute_minimum_time && !unittest_running)
+        before_requested = absolute_minimum_time;
+
+    if (before_requested > absolute_maximum_time && !unittest_running)
+        before_requested = absolute_maximum_time;
+
     *before = before_requested;
     *after = after_requested;
 
-    return absolute_period_requested;
+    return (absolute_period_requested != 1);
 }
 
 // #define DEBUG_QUERY_LOGIC 1
@@ -1636,7 +1686,7 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
 #define query_debug_log_init() BUFFER *debug_log = buffer_create(1000)
 #define query_debug_log(args...) buffer_sprintf(debug_log, ##args)
 #define query_debug_log_fin() { \
-        info("QUERY: chart '%s', after:%lld, before:%lld, duration:%lld, points:%ld, res:%ld - wanted => after:%lld, before:%lld, points:%ld, group:%ld, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", st->name, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
+        info("QUERY: '%s', after:%ld, before:%ld, duration:%ld, points:%zu, res:%ld - wanted => after:%ld, before:%ld, points:%zu, group:%zu, granularity:%ld, resgroup:%ld, resdiv:" NETDATA_DOUBLE_FORMAT_AUTO " %s", qt->id, after_requested, before_requested, before_requested - after_requested, points_requested, resampling_time_requested, after_wanted, before_wanted, points_wanted, group, query_granularity, resampling_group, resampling_divisor, buffer_tostring(debug_log)); \
         buffer_free(debug_log); \
         debug_log = NULL; \
     }
@@ -1648,21 +1698,18 @@ int rrdr_relative_window_to_absolute(long long *after, long long *before) {
 #define query_debug_log_free() debug_dummy()
 #endif
 
-RRDR *rrd2rrdr(
-          ONEWAYALLOC *owa
-        , RRDSET *st
-        , long points_requested
-        , long long after_requested
-        , long long before_requested
-        , RRDR_GROUPING group_method
-        , long resampling_time_requested
-        , RRDR_OPTIONS options
-        , const char *dimensions
-        , struct context_param *context_param_list
-        , const char *group_options
-        , int timeout
-        , int tier
-) {
+bool query_target_calculate_window(QUERY_TARGET *qt) {
+    if (unlikely(!qt)) return false;
+
+    size_t points_requested = (long)qt->request.points;
+    time_t after_requested = qt->request.after;
+    time_t before_requested = qt->request.before;
+    RRDR_GROUPING group_method = qt->request.group_method;
+    time_t resampling_time_requested = qt->request.resampling_time;
+    RRDR_OPTIONS options = qt->request.options;
+    size_t tier = qt->request.tier;
+    time_t update_every = qt->db.minimum_latest_update_every;
+
     // RULES
     // points_requested = 0
     // the user wants all the natural points the database has
@@ -1676,10 +1723,9 @@ RRDR *rrd2rrdr(
     // when natural points are wanted, the query has to be aligned to the update_every
     // of the database
 
-    long points_wanted = points_requested;
-    long long after_wanted = after_requested;
-    long long before_wanted = before_requested;
-    int update_every = st->update_every;
+    size_t points_wanted = points_requested;
+    time_t after_wanted = after_requested;
+    time_t before_wanted = before_requested;
 
     bool aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
     bool automatic_natural_points = (points_wanted == 0);
@@ -1689,13 +1735,7 @@ RRDR *rrd2rrdr(
 
     query_debug_log_init();
 
-    // make sure points_wanted is positive
-    if(points_wanted < 0) {
-        points_wanted = -points_wanted;
-        query_debug_log(":-points_wanted %ld", points_wanted);
-    }
-
-    if(ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
+    if (ABS(before_requested) <= API_RELATIVE_TIME_MAX || ABS(after_requested) <= API_RELATIVE_TIME_MAX) {
         relative_period_requested = true;
         natural_points = true;
         options |= RRDR_OPTION_NATURAL_POINTS;
@@ -1703,105 +1743,93 @@ RRDR *rrd2rrdr(
     }
 
     // if the user wants virtual points, make sure we do it
-    if(options & RRDR_OPTION_VIRTUAL_POINTS)
+    if (options & RRDR_OPTION_VIRTUAL_POINTS)
         natural_points = false;
 
     // set the right flag about natural and virtual points
-    if(natural_points) {
+    if (natural_points) {
         options |= RRDR_OPTION_NATURAL_POINTS;
 
-        if(options & RRDR_OPTION_VIRTUAL_POINTS)
+        if (options & RRDR_OPTION_VIRTUAL_POINTS)
             options &= ~RRDR_OPTION_VIRTUAL_POINTS;
     }
     else {
         options |= RRDR_OPTION_VIRTUAL_POINTS;
 
-        if(options & RRDR_OPTION_NATURAL_POINTS)
+        if (options & RRDR_OPTION_NATURAL_POINTS)
             options &= ~RRDR_OPTION_NATURAL_POINTS;
     }
 
-    if(after_wanted == 0 || before_wanted == 0) {
-        // for non-context queries we have to find the duration of the database
-        // for context queries we will assume 600 seconds duration
-
-        if(!context_param_list) {
-            relative_period_requested = true;
-
-            rrdset_rdlock(st);
-            time_t first_entry_t = rrdset_first_entry_t_nolock(st);
-            time_t last_entry_t = rrdset_last_entry_t_nolock(st);
-            rrdset_unlock(st);
-
-            if(first_entry_t == 0 || last_entry_t == 0) {
-                internal_error(true, "QUERY: chart without data detected on '%s'", st->name);
-                query_debug_log_free();
-                return NULL;
-            }
+    if (after_wanted == 0 || before_wanted == 0) {
+        relative_period_requested = true;
 
-            query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
+        time_t first_entry_t = qt->db.first_time_t;
+        time_t last_entry_t = qt->db.last_time_t;
 
-            if (after_wanted == 0) {
-                after_wanted = first_entry_t;
-                query_debug_log(":zero after_wanted %lld", after_wanted);
-            }
+        if (first_entry_t == 0 || last_entry_t == 0) {
+            internal_error(true, "QUERY: no data detected on query '%s' (db first_entry_t = %ld, last_entry_t = %ld", qt->id, first_entry_t, last_entry_t);
+            query_debug_log_free();
+            return false;
+        }
 
-            if (before_wanted == 0) {
-                before_wanted = last_entry_t;
-                before_is_aligned_to_db_end = true;
-                query_debug_log(":zero before_wanted %lld", before_wanted);
-            }
+        query_debug_log(":first_entry_t %ld, last_entry_t %ld", first_entry_t, last_entry_t);
 
-            if(points_wanted == 0) {
-                points_wanted = (last_entry_t - first_entry_t) / update_every;
-                query_debug_log(":zero points_wanted %ld", points_wanted);
-            }
+        if (after_wanted == 0) {
+            after_wanted = first_entry_t;
+            query_debug_log(":zero after_wanted %ld", after_wanted);
         }
 
-        // if they are still zero, assume 600
+        if (before_wanted == 0) {
+            before_wanted = last_entry_t;
+            before_is_aligned_to_db_end = true;
+            query_debug_log(":zero before_wanted %ld", before_wanted);
+        }
 
-        if(after_wanted == 0) {
-            after_wanted = -600;
-            query_debug_log(":zero600 after_wanted %lld", after_wanted);
+        if (points_wanted == 0) {
+            points_wanted = (last_entry_t - first_entry_t) / update_every;
+            query_debug_log(":zero points_wanted %zu", points_wanted);
         }
     }
 
-    if(points_wanted == 0) {
+    if (points_wanted == 0) {
         points_wanted = 600;
-        query_debug_log(":zero600 points_wanted %ld", points_wanted);
+        query_debug_log(":zero600 points_wanted %zu", points_wanted);
     }
 
     // convert our before_wanted and after_wanted to absolute
     rrdr_relative_window_to_absolute(&after_wanted, &before_wanted);
-    query_debug_log(":relative2absolute after %lld, before %lld", after_wanted, before_wanted);
+    query_debug_log(":relative2absolute after %ld, before %ld", after_wanted, before_wanted);
 
-    if(natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
-        update_every = rrdset_find_natural_update_every_for_timeframe(st, after_wanted, before_wanted, points_wanted, options, tier);
-        if(update_every <= 0) update_every = st->update_every;
-        query_debug_log(":natural update every %d", update_every);
+    if (natural_points && (options & RRDR_OPTION_SELECTED_TIER) && tier > 0 && storage_tiers > 1) {
+        update_every = rrdset_find_natural_update_every_for_timeframe(
+                qt, after_wanted, before_wanted, points_wanted, options, tier);
+
+        if (update_every <= 0) update_every = qt->db.minimum_latest_update_every;
+        query_debug_log(":natural update every %ld", update_every);
     }
 
     // this is the update_every of the query
     // it may be different to the update_every of the database
-    time_t query_granularity = (natural_points)?update_every:1;
-    if(query_granularity <= 0) query_granularity = 1;
+    time_t query_granularity = (natural_points) ? update_every : 1;
+    if (query_granularity <= 0) query_granularity = 1;
     query_debug_log(":query_granularity %ld", query_granularity);
 
     // align before_wanted and after_wanted to query_granularity
     if (before_wanted % query_granularity) {
         before_wanted -= before_wanted % query_granularity;
-        query_debug_log(":granularity align before_wanted %lld", before_wanted);
+        query_debug_log(":granularity align before_wanted %ld", before_wanted);
     }
 
     if (after_wanted % query_granularity) {
         after_wanted -= after_wanted % query_granularity;
-        query_debug_log(":granularity align after_wanted %lld", after_wanted);
+        query_debug_log(":granularity align after_wanted %ld", after_wanted);
     }
 
     // automatic_natural_points is set when the user wants all the points available in the database
-    if(automatic_natural_points) {
+    if (automatic_natural_points) {
         points_wanted = (before_wanted - after_wanted + 1) / query_granularity;
-        if(unlikely(points_wanted <= 0)) points_wanted = 1;
-        query_debug_log(":auto natural points_wanted %ld", points_wanted);
+        if (unlikely(points_wanted <= 0)) points_wanted = 1;
+        query_debug_log(":auto natural points_wanted %zu", points_wanted);
     }
 
     time_t duration = before_wanted - after_wanted;
@@ -1810,42 +1838,47 @@ RRDR *rrd2rrdr(
     if (unlikely(resampling_time_requested > duration)) {
         after_wanted = before_wanted - resampling_time_requested;
         duration = before_wanted - after_wanted;
-        query_debug_log(":resampling after_wanted %lld", after_wanted);
+        query_debug_log(":resampling after_wanted %ld", after_wanted);
     }
 
     // if the duration is not aligned to resampling time
     // extend the duration to the past, to avoid a gap at the chart
     // only when the missing duration is above 1/10th of a point
-    if(resampling_time_requested > query_granularity && duration % resampling_time_requested) {
+    if (resampling_time_requested > query_granularity && duration % resampling_time_requested) {
         time_t delta = duration % resampling_time_requested;
-        if(delta > resampling_time_requested / 10) {
+        if (delta > resampling_time_requested / 10) {
             after_wanted -= resampling_time_requested - delta;
             duration = before_wanted - after_wanted;
-            query_debug_log(":resampling2 after_wanted %lld", after_wanted);
+            query_debug_log(":resampling2 after_wanted %ld", after_wanted);
         }
     }
 
     // the available points of the query
-    long points_available = (duration + 1) / query_granularity;
-    if(unlikely(points_available <= 0)) points_available = 1;
-    query_debug_log(":points_available %ld", points_available);
+    size_t points_available = (duration + 1) / query_granularity;
+    if (unlikely(points_available <= 0)) points_available = 1;
+    query_debug_log(":points_available %zu", points_available);
 
-    if(points_wanted > points_available) {
+    if (points_wanted > points_available) {
         points_wanted = points_available;
-        query_debug_log(":max points_wanted %ld", points_wanted);
+        query_debug_log(":max points_wanted %zu", points_wanted);
+    }
+
+    if(points_wanted > 86400 && !unittest_running) {
+        points_wanted = 86400;
+        query_debug_log(":absolute max points_wanted %zu", points_wanted);
     }
 
     // calculate the desired grouping of source data points
-    long group = points_available / points_wanted;
-    if(group <= 0) group = 1;
+    size_t group = points_available / points_wanted;
+    if (group == 0) group = 1;
 
     // round "group" to the closest integer
-    if(points_available % points_wanted > points_wanted / 2)
+    if (points_available % points_wanted > points_wanted / 2)
         group++;
 
-    query_debug_log(":group %ld", group);
+    query_debug_log(":group %zu", group);
 
-    if(points_wanted * group * query_granularity < duration) {
+    if (points_wanted * group * query_granularity < (size_t)duration) {
         // the grouping we are going to do, is not enough
         // to cover the entire duration requested, so
         // we have to change the number of points, to make sure we will
@@ -1854,162 +1887,186 @@ RRDR *rrd2rrdr(
         // let's see how many points are the optimal
         points_wanted = points_available / group;
 
-        if(points_wanted * group < points_available)
+        if (points_wanted * group < points_available)
             points_wanted++;
 
-        if(unlikely(points_wanted <= 0))
+        if (unlikely(points_wanted == 0))
             points_wanted = 1;
 
-        query_debug_log(":optimal points %ld", points_wanted);
+        query_debug_log(":optimal points %zu", points_wanted);
     }
 
     // resampling_time_requested enforces a certain grouping multiple
     NETDATA_DOUBLE resampling_divisor = 1.0;
-    long resampling_group = 1;
-    if(unlikely(resampling_time_requested > query_granularity)) {
+    size_t resampling_group = 1;
+    if (unlikely(resampling_time_requested > query_granularity)) {
         // the points we should group to satisfy gtime
         resampling_group = resampling_time_requested / query_granularity;
-        if(unlikely(resampling_time_requested % query_granularity))
+        if (unlikely(resampling_time_requested % query_granularity))
             resampling_group++;
 
-        query_debug_log(":resampling group %ld", resampling_group);
+        query_debug_log(":resampling group %zu", resampling_group);
 
         // adapt group according to resampling_group
-        if(unlikely(group < resampling_group)) {
-            group  = resampling_group; // do not allow grouping below the desired one
-            query_debug_log(":group less res %ld", group);
+        if (unlikely(group < resampling_group)) {
+            group = resampling_group; // do not allow grouping below the desired one
+            query_debug_log(":group less res %zu", group);
         }
-        if(unlikely(group % resampling_group)) {
+        if (unlikely(group % resampling_group)) {
             group += resampling_group - (group % resampling_group); // make sure group is multiple of resampling_group
-            query_debug_log(":group mod res %ld", group);
+            query_debug_log(":group mod res %zu", group);
         }
 
         // resampling_divisor = group / resampling_group;
-        resampling_divisor = (NETDATA_DOUBLE)(group * query_granularity) / (NETDATA_DOUBLE)resampling_time_requested;
+        resampling_divisor = (NETDATA_DOUBLE) (group * query_granularity) / (NETDATA_DOUBLE) resampling_time_requested;
         query_debug_log(":resampling divisor " NETDATA_DOUBLE_FORMAT, resampling_divisor);
     }
 
     // now that we have group, align the requested timeframe to fit it.
-    if(aligned && before_wanted % (group * query_granularity)) {
-        if(before_is_aligned_to_db_end)
-            before_wanted -= before_wanted % (group * query_granularity);
+    if (aligned && before_wanted % (group * query_granularity)) {
+        if (before_is_aligned_to_db_end)
+            before_wanted -= before_wanted % (time_t)(group * query_granularity);
         else
-            before_wanted += (group * query_granularity) - before_wanted % (group * query_granularity);
-        query_debug_log(":align before_wanted %lld", before_wanted);
+            before_wanted += (time_t)(group * query_granularity) - before_wanted % (time_t)(group * query_granularity);
+        query_debug_log(":align before_wanted %ld", before_wanted);
     }
 
-    after_wanted  = before_wanted - (points_wanted * group * query_granularity) + query_granularity;
-    query_debug_log(":final after_wanted %lld", after_wanted);
+    after_wanted = before_wanted - (time_t)(points_wanted * group * query_granularity) + query_granularity;
+    query_debug_log(":final after_wanted %ld", after_wanted);
 
     duration = before_wanted - after_wanted;
     query_debug_log(":final duration %ld", duration + 1);
 
-    // check the context query based on the starting time of the query
-    if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
-        rebuild_context_param_list(owa, context_param_list, after_wanted);
-        st = context_param_list->rd ? context_param_list->rd->rrdset : NULL;
-
-        if(unlikely(!st))
-            return NULL;
-    }
+    query_debug_log_fin();
 
     internal_error(points_wanted != duration / (query_granularity * group) + 1,
-                   "QUERY: points_wanted %ld is not points %ld",
-                   points_wanted, duration / (query_granularity * group) + 1);
+                   "QUERY: points_wanted %zu is not points %zu",
+                   points_wanted, (size_t)(duration / (query_granularity * group) + 1));
 
     internal_error(group < resampling_group,
-                   "QUERY: group %ld is less than the desired group points %ld",
+                   "QUERY: group %zu is less than the desired group points %zu",
                    group, resampling_group);
 
     internal_error(group > resampling_group && group % resampling_group,
-                   "QUERY: group %ld is not a multiple of the desired group points %ld",
+                   "QUERY: group %zu is not a multiple of the desired group points %zu",
                    group, resampling_group);
 
     // -------------------------------------------------------------------------
-    // initialize our result set
-    // this also locks the chart for us
+    // update QUERY_TARGET with our calculations
+
+    qt->window.after = after_wanted;
+    qt->window.before = before_wanted;
+    qt->window.relative = relative_period_requested;
+    qt->window.points = points_wanted;
+    qt->window.group = group;
+    qt->window.group_method = group_method;
+    qt->window.group_options = qt->request.group_options;
+    qt->window.query_granularity = query_granularity;
+    qt->window.resampling_group = resampling_group;
+    qt->window.resampling_divisor = resampling_divisor;
+    qt->window.options = options;
+    qt->window.tier = tier;
+    qt->window.aligned = aligned;
+
+    return true;
+}
+
+RRDR *rrd2rrdr_legacy(
+        ONEWAYALLOC *owa,
+        RRDSET *st, size_t points, time_t after, time_t before,
+        RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,
+        const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source) {
+
+    QUERY_TARGET_REQUEST qtr = {
+            .st = st,
+            .points = points,
+            .after = after,
+            .before = before,
+            .group_method = group_method,
+            .resampling_time = resampling_time,
+            .options = options,
+            .dimensions = dimensions,
+            .group_options = group_options,
+            .timeout = timeout,
+            .tier = tier,
+            .query_source = query_source,
+    };
+
+    return rrd2rrdr(owa, query_target_create(&qtr));
+}
 
-    RRDR *r = rrdr_create(owa, st, points_wanted, context_param_list);
+RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
+    if(!qt)
+        return NULL;
+
+    if(!owa) {
+        query_target_release(qt);
+        return NULL;
+    }
+
+    // qt.window members are the WANTED ones.
+    // qt.request members are the REQUESTED ones.
+
+    RRDR *r = rrdr_create(owa, qt);
     if(unlikely(!r)) {
-        internal_error(true, "QUERY: cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld",
-                       st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
+        internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, points=%zu",
+                       qt->id, qt->window.after, qt->window.before, qt->window.points);
         return NULL;
     }
 
-    if(unlikely(!r->d || !points_wanted)) {
-        internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%zu, points=%ld",
-                       st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (size_t)duration, points_wanted);
+    if(unlikely(!r->d || !qt->window.points)) {
+        internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%ld, before=%ld, points=%zu",
+                       qt->id, qt->window.after, qt->window.before, qt->window.points);
         return r;
     }
 
-    if(relative_period_requested)
+    if(qt->window.relative)
         r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
     else
         r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
 
-    // find how many dimensions we have
-    long dimensions_count = r->d;
-
     // -------------------------------------------------------------------------
     // initialize RRDR
 
-    r->group = group;
-    r->update_every = (int)(group * query_granularity);
-    r->before = before_wanted;
-    r->after = after_wanted;
-    r->internal.points_wanted = points_wanted;
-    r->internal.resampling_group = resampling_group;
-    r->internal.resampling_divisor = resampling_divisor;
-    r->internal.query_options = options;
-    r->internal.query_tier = tier;
+    r->group = qt->window.group;
+    r->update_every = (int) (qt->window.group * qt->window.query_granularity);
+    r->before = qt->window.before;
+    r->after = qt->window.after;
+    r->internal.points_wanted = qt->window.points;
+    r->internal.resampling_group = qt->window.resampling_group;
+    r->internal.resampling_divisor = qt->window.resampling_divisor;
+    r->internal.query_options = qt->window.options;
 
     // -------------------------------------------------------------------------
     // assign the processor functions
-    rrdr_set_grouping_function(r, group_method);
+    rrdr_set_grouping_function(r, qt->window.group_method);
 
     // allocate any memory required by the grouping method
-    r->internal.grouping_create(r, group_options);
-
-
-    // -------------------------------------------------------------------------
-    // disable the not-wanted dimensions
-
-    if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE))
-        rrdset_check_rdlock(st);
-
-    if(dimensions)
-        rrdr_disable_not_selected_dimensions(r, options, dimensions, context_param_list);
-
-
-    query_debug_log_fin();
+    r->internal.grouping_create(r, qt->window.group_options);
 
     // -------------------------------------------------------------------------
     // do the work for each dimension
 
     time_t max_after = 0, min_before = 0;
-    long max_rows = 0;
+    size_t max_rows = 0;
 
-    RRDDIM *first_rd = context_param_list ? context_param_list->rd : st->dimensions;
-    RRDDIM *rd;
-    long c, dimensions_used = 0, dimensions_nonzero = 0;
+    long dimensions_used = 0, dimensions_nonzero = 0;
     struct timeval query_start_time;
     struct timeval query_current_time;
-    if (timeout) now_realtime_timeval(&query_start_time);
+    if (qt->request.timeout)
+        now_realtime_timeval(&query_start_time);
 
-    for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
+    for(size_t c = 0, max = qt->query.used; c < max ; c++) {
+        // set the query target dimension options to rrdr
+        r->od[c] = qt->query.array[c].dimension.options;
 
-        // if we need a percentage, we need to calculate all dimensions
-        if(unlikely(!(options & RRDR_OPTION_PERCENTAGE) && (r->od[c] & RRDR_DIMENSION_HIDDEN))) {
-            if(unlikely(r->od[c] & RRDR_DIMENSION_SELECTED)) r->od[c] &= ~RRDR_DIMENSION_SELECTED;
-            continue;
-        }
         r->od[c] |= RRDR_DIMENSION_SELECTED;
 
         // reset the grouping for the new dimension
         r->internal.grouping_reset(r);
 
-        rrd2rrdr_do_dimension(r, points_wanted, rd, c, after_wanted, before_wanted);
-        if (timeout)
+        rrd2rrdr_do_dimension(r, c);
+        if (qt->request.timeout)
             now_realtime_timeval(&query_current_time);
 
         if(r->od[c] & RRDR_DIMENSION_NONZERO)
@@ -2024,30 +2081,30 @@ RRDR *rrd2rrdr(
         else {
             if(r->after != max_after) {
                 internal_error(true, "QUERY: 'after' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
-                               st->name, (size_t)max_after, rd->name, (size_t)r->after);
+                               string2str(qt->query.array[c].dimension.id), (size_t)max_after, string2str(qt->query.array[c].dimension.name), (size_t)r->after);
 
                 r->after = (r->after > max_after) ? r->after : max_after;
             }
 
             if(r->before != min_before) {
                 internal_error(true, "QUERY: 'before' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
-                               st->name, (size_t)min_before, rd->name, (size_t)r->before);
+                               string2str(qt->query.array[c].dimension.id), (size_t)min_before, string2str(qt->query.array[c].dimension.name), (size_t)r->before);
 
                 r->before = (r->before < min_before) ? r->before : min_before;
             }
 
             if(r->rows != max_rows) {
                 internal_error(true, "QUERY: 'rows' mismatch between dimensions for chart '%s': max is %zu, dimension '%s' has %zu",
-                               st->name, (size_t)max_rows, rd->name, (size_t)r->rows);
+                               string2str(qt->query.array[c].dimension.id), (size_t)max_rows, string2str(qt->query.array[c].dimension.name), (size_t)r->rows);
 
                 r->rows = (r->rows > max_rows) ? r->rows : max_rows;
             }
         }
 
         dimensions_used++;
-        if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > timeout) {
-            log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %d ms)",
-                       (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, timeout);
+        if (qt->request.timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout) {
+            log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)",
+                       (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout);
             r->result_options |= RRDR_RESULT_OPTION_CANCEL;
             break;
         }
@@ -2056,44 +2113,44 @@ RRDR *rrd2rrdr(
 #ifdef NETDATA_INTERNAL_CHECKS
     if (dimensions_used) {
         if(r->internal.log)
-            rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
-                                                   after_wanted, after_requested, before_wanted, before_requested,
-                                                   points_requested, points_wanted, /*after_slot, before_slot,*/
+            rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+                                                   qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+                                                   qt->request.points, qt->window.points, /*after_slot, before_slot,*/
                                                    r->internal.log);
 
-        if(r->rows != points_wanted)
-            rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
-                                                   after_wanted, after_requested, before_wanted, before_requested,
-                                                   points_requested, points_wanted, /*after_slot, before_slot,*/
+        if(r->rows != qt->window.points)
+            rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+                                                   qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+                                                   qt->request.points, qt->window.points, /*after_slot, before_slot,*/
                                                    "got 'points' is not wanted 'points'");
 
-        if(aligned && (r->before % (group * query_granularity)) != 0)
-            rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
-                                                   after_wanted, after_requested, before_wanted,before_wanted,
-                                                   points_requested, points_wanted, /*after_slot, before_slot,*/
+        if(qt->window.aligned && (r->before % (qt->window.group * qt->window.query_granularity)) != 0)
+            rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+                                                   qt->window.after, qt->request.after, qt->window.before,qt->request.before,
+                                                   qt->request.points, qt->window.points, /*after_slot, before_slot,*/
                                                    "'before' is not aligned but alignment is required");
 
         // 'after' should not be aligned, since we start inside the first group
-        //if(aligned && (r->after % group) != 0)
-        //    rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
+        //if(qt->window.aligned && (r->after % group) != 0)
+        //    rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
 
-        if(r->before != before_wanted)
-            rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
-                                                   after_wanted, after_requested, before_wanted, before_requested,
-                                                   points_requested, points_wanted, /*after_slot, before_slot,*/
+        if(r->before != qt->window.before)
+            rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+                                                   qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+                                                   qt->request.points, qt->window.points, /*after_slot, before_slot,*/
                                                    "chart is not aligned to requested 'before'");
 
-        if(r->before != before_wanted)
-            rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
-                                                   after_wanted, after_requested, before_wanted, before_requested,
-                                                   points_requested, points_wanted, /*after_slot, before_slot,*/
+        if(r->before != qt->window.before)
+            rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+                                                   qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+                                                   qt->request.points, qt->window.points, /*after_slot, before_slot,*/
                                                    "got 'before' is not wanted 'before'");
 
         // reported 'after' varies, depending on group
-        if(r->after != after_wanted)
-            rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
-                                                   after_wanted, after_requested, before_wanted, before_requested,
-                                                   points_requested, points_wanted, /*after_slot, before_slot,*/
+        if(r->after != qt->window.after)
+            rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
+                                                   qt->window.after, qt->request.after, qt->window.before, qt->request.before,
+                                                   qt->request.points, qt->window.points, /*after_slot, before_slot,*/
                                                    "got 'after' is not wanted 'after'");
 
     }
@@ -2103,15 +2160,16 @@ RRDR *rrd2rrdr(
     r->internal.grouping_free(r);
 
     // when all the dimensions are zero, we should return all of them
-    if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
+    if(unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
         // all the dimensions are zero
         // mark them as NONZERO to send them all
-        for(rd = first_rd, c = 0 ; rd && c < dimensions_count ; rd = rd->next, c++) {
+        for(size_t c = 0, max = qt->query.used; c < max ; c++) {
             if(unlikely(r->od[c] & RRDR_DIMENSION_HIDDEN)) continue;
             r->od[c] |= RRDR_DIMENSION_NONZERO;
         }
     }
 
-    rrdr_query_completed(r->internal.db_points_read, r->internal.result_points_generated);
+    global_statistics_rrdr_query_completed(dimensions_used, r->internal.db_points_read,
+                                           r->internal.result_points_generated, qt->request.query_source);
     return r;
 }
diff --git a/web/api/queries/query.h b/web/api/queries/query.h
index df876d9ac..ebad5a1f8 100644
--- a/web/api/queries/query.h
+++ b/web/api/queries/query.h
@@ -47,10 +47,10 @@ typedef enum rrdr_grouping {
     RRDR_GROUPING_COUNTIF,
 } RRDR_GROUPING;
 
-extern const char *group_method2string(RRDR_GROUPING group);
-extern void web_client_api_v1_init_grouping(void);
-extern RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def);
-extern const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group);
+const char *group_method2string(RRDR_GROUPING group);
+void web_client_api_v1_init_grouping(void);
+RRDR_GROUPING web_client_api_request_v1_data_group(const char *name, RRDR_GROUPING def);
+const char *web_client_api_request_v1_data_group_to_string(RRDR_GROUPING group);
 
 #ifdef __cplusplus
 }
diff --git a/web/api/queries/rrdr.c b/web/api/queries/rrdr.c
index ecf4ca2ac..676224c9d 100644
--- a/web/api/queries/rrdr.c
+++ b/web/api/queries/rrdr.c
@@ -61,9 +61,7 @@ static void rrdr_dump(RRDR *r)
 inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
     if(unlikely(!r)) return;
 
-    if(likely(r->st_locked_by_rrdr_create))
-        rrdset_unlock(r->st);
-
+    query_target_release(r->internal.qt);
     onewayalloc_freez(owa, r->t);
     onewayalloc_freez(owa, r->v);
     onewayalloc_freez(owa, r->o);
@@ -72,12 +70,23 @@ inline void rrdr_free(ONEWAYALLOC *owa, RRDR *r) {
     onewayalloc_freez(owa, r);
 }
 
-RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points) {
+RRDR *rrdr_create(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
+    if(unlikely(!qt || !qt->query.used || !qt->window.points))
+        return NULL;
+
+    size_t dimensions = qt->query.used;
+    size_t points = qt->window.points;
+
+    // create the rrdr
     RRDR *r = onewayalloc_callocz(owa, 1, sizeof(RRDR));
     r->internal.owa = owa;
+    r->internal.qt = qt;
 
-    r->d = dimensions;
-    r->n = points;
+    r->before = qt->window.before;
+    r->after = qt->window.after;
+    r->internal.points_wanted = qt->window.points;
+    r->d = (int)dimensions;
+    r->n = (int)points;
 
     r->t = onewayalloc_callocz(owa, points, sizeof(time_t));
     r->v = onewayalloc_mallocz(owa, points * dimensions * sizeof(NETDATA_DOUBLE));
@@ -90,42 +99,3 @@ RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points
 
     return r;
 }
-
-RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list) {
-    if (unlikely(!st)) return NULL;
-
-    bool st_locked_by_rrdr_create = false;
-    if (!context_param_list || !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) {
-        rrdset_rdlock(st);
-        st_locked_by_rrdr_create = true;
-    }
-
-    // count the number of dimensions
-    int dimensions = 0;
-    RRDDIM *temp_rd =  context_param_list ? context_param_list->rd : NULL;
-    RRDDIM *rd;
-    if (temp_rd) {
-        RRDDIM *t = temp_rd;
-        while (t) {
-            dimensions++;
-            t = t->next;
-        }
-    } else
-        rrddim_foreach_read(rd, st) dimensions++;
-
-    // create the rrdr
-    RRDR *r = rrdr_create_for_x_dimensions(owa, dimensions, n);
-    r->st = st;
-    r->st_locked_by_rrdr_create = st_locked_by_rrdr_create;
-
-    // set the hidden flag on hidden dimensions
-    int c;
-    for (c = 0, rd = temp_rd ? temp_rd : st->dimensions; rd; c++, rd = rd->next) {
-        if (unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)))
-            r->od[c] = RRDR_DIMENSION_HIDDEN;
-        else
-            r->od[c] = RRDR_DIMENSION_DEFAULT;
-    }
-
-    return r;
-}
diff --git a/web/api/queries/rrdr.h b/web/api/queries/rrdr.h
index 1c80e103f..6151cddc7 100644
--- a/web/api/queries/rrdr.h
+++ b/web/api/queries/rrdr.h
@@ -18,32 +18,33 @@ typedef enum tier_query_fetch {
 } TIER_QUERY_FETCH;
 
 typedef enum rrdr_options {
-    RRDR_OPTION_NONZERO        = 0x00000001, // don't output dimensions with just zero values
-    RRDR_OPTION_REVERSED       = 0x00000002, // output the rows in reverse order (oldest to newest)
-    RRDR_OPTION_ABSOLUTE       = 0x00000004, // values positive, for DATASOURCE_SSV before summing
-    RRDR_OPTION_MIN2MAX        = 0x00000008, // when adding dimensions, use max - min, instead of sum
-    RRDR_OPTION_SECONDS        = 0x00000010, // output seconds, instead of dates
-    RRDR_OPTION_MILLISECONDS   = 0x00000020, // output milliseconds, instead of dates
-    RRDR_OPTION_NULL2ZERO      = 0x00000040, // do not show nulls, convert them to zeros
-    RRDR_OPTION_OBJECTSROWS    = 0x00000080, // each row of values should be an object, not an array
-    RRDR_OPTION_GOOGLE_JSON    = 0x00000100, // comply with google JSON/JSONP specs
-    RRDR_OPTION_JSON_WRAP      = 0x00000200, // wrap the response in a JSON header with info about the result
-    RRDR_OPTION_LABEL_QUOTES   = 0x00000400, // in CSV output, wrap header labels in double quotes
-    RRDR_OPTION_PERCENTAGE     = 0x00000800, // give values as percentage of total
-    RRDR_OPTION_NOT_ALIGNED    = 0x00001000, // do not align charts for persistent timeframes
-    RRDR_OPTION_DISPLAY_ABS    = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
-    RRDR_OPTION_MATCH_IDS      = 0x00004000, // when filtering dimensions, match only IDs
-    RRDR_OPTION_MATCH_NAMES    = 0x00008000, // when filtering dimensions, match only names
-    RRDR_OPTION_CUSTOM_VARS    = 0x00010000, // when wrapping response in a JSON, return custom variables in response
-    RRDR_OPTION_NATURAL_POINTS = 0x00020000, // return the natural points of the database
-    RRDR_OPTION_VIRTUAL_POINTS = 0x00040000, // return virtual points
-    RRDR_OPTION_ANOMALY_BIT    = 0x00080000, // Return the anomaly bit stored in each collected_number
-    RRDR_OPTION_RETURN_RAW     = 0x00100000, // Return raw data for aggregating across multiple nodes
-    RRDR_OPTION_RETURN_JWAR    = 0x00200000, // Return anomaly rates in jsonwrap
-    RRDR_OPTION_SELECTED_TIER  = 0x00400000, // Use the selected tier for the query
+    RRDR_OPTION_NONZERO         = 0x00000001, // don't output dimensions with just zero values
+    RRDR_OPTION_REVERSED        = 0x00000002, // output the rows in reverse order (oldest to newest)
+    RRDR_OPTION_ABSOLUTE        = 0x00000004, // values positive, for DATASOURCE_SSV before summing
+    RRDR_OPTION_MIN2MAX         = 0x00000008, // when adding dimensions, use max - min, instead of sum
+    RRDR_OPTION_SECONDS         = 0x00000010, // output seconds, instead of dates
+    RRDR_OPTION_MILLISECONDS    = 0x00000020, // output milliseconds, instead of dates
+    RRDR_OPTION_NULL2ZERO       = 0x00000040, // do not show nulls, convert them to zeros
+    RRDR_OPTION_OBJECTSROWS     = 0x00000080, // each row of values should be an object, not an array
+    RRDR_OPTION_GOOGLE_JSON     = 0x00000100, // comply with google JSON/JSONP specs
+    RRDR_OPTION_JSON_WRAP       = 0x00000200, // wrap the response in a JSON header with info about the result
+    RRDR_OPTION_LABEL_QUOTES    = 0x00000400, // in CSV output, wrap header labels in double quotes
+    RRDR_OPTION_PERCENTAGE      = 0x00000800, // give values as percentage of total
+    RRDR_OPTION_NOT_ALIGNED     = 0x00001000, // do not align charts for persistent timeframes
+    RRDR_OPTION_DISPLAY_ABS     = 0x00002000, // for badges, display the absolute value, but calculate colors with sign
+    RRDR_OPTION_MATCH_IDS       = 0x00004000, // when filtering dimensions, match only IDs
+    RRDR_OPTION_MATCH_NAMES     = 0x00008000, // when filtering dimensions, match only names
+    RRDR_OPTION_NATURAL_POINTS  = 0x00020000, // return the natural points of the database
+    RRDR_OPTION_VIRTUAL_POINTS  = 0x00040000, // return virtual points
+    RRDR_OPTION_ANOMALY_BIT     = 0x00080000, // Return the anomaly bit stored in each collected_number
+    RRDR_OPTION_RETURN_RAW      = 0x00100000, // Return raw data for aggregating across multiple nodes
+    RRDR_OPTION_RETURN_JWAR     = 0x00200000, // Return anomaly rates in jsonwrap
+    RRDR_OPTION_SELECTED_TIER   = 0x00400000, // Use the selected tier for the query
+    RRDR_OPTION_ALL_DIMENSIONS = 0x00800000, // Return the full dimensions list
 
     // internal ones - not to be exposed to the API
-    RRDR_OPTION_INTERNAL_AR    = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
+    RRDR_OPTION_INTERNAL_AR     = 0x10000000, // internal use only, to let the formatters we want to render the anomaly rate
+    RRDR_OPTION_HEALTH_RSRVD1   = 0x80000000, // reserved for RRDCALC_OPTION_NO_CLEAR_NOTIFICATION
 } RRDR_OPTIONS;
 
 typedef enum rrdr_value_flag {
@@ -67,16 +68,14 @@ typedef enum rrdr_result_flags {
                                                    // (should not to be cached by browsers and proxies)
     RRDR_RESULT_OPTION_VARIABLE_STEP = 0x00000004, // the query uses variable-step time-frames
     RRDR_RESULT_OPTION_CANCEL        = 0x00000008, // the query needs to be cancelled
-} RRDR_RESULT_FLAGS;
+} RRDR_RESULT_OPTIONS;
 
 typedef struct rrdresult {
-    struct rrdset *st;         // the chart this result refers to
+    RRDR_RESULT_OPTIONS result_options; // RRDR_RESULT_OPTION_*
 
-    RRDR_RESULT_FLAGS result_options; // RRDR_RESULT_OPTION_*
-
-    int d;                    // the number of dimensions
-    long n;                   // the number of values in the arrays
-    long rows;                // the number of rows used
+    size_t d;                 // the number of dimensions
+    size_t n;                 // the number of values in the arrays
+    size_t rows;              // the number of rows used
 
     RRDR_DIMENSION_FLAGS *od; // the options for the dimensions
 
@@ -85,8 +84,8 @@ typedef struct rrdresult {
     RRDR_VALUE_FLAGS *o;      // array n x d options for each value returned
     NETDATA_DOUBLE *ar;       // array n x d of anomaly rates (0 - 100)
 
-    long group;               // how many collected values were grouped for each row
-    int update_every;         // what is the suggested update frequency in seconds
+    size_t group;             // how many collected values were grouped for each row
+    time_t update_every;      // what is the suggested update frequency in seconds
 
     NETDATA_DOUBLE min;
     NETDATA_DOUBLE max;
@@ -94,53 +93,57 @@ typedef struct rrdresult {
     time_t before;
     time_t after;
 
-    bool st_locked_by_rrdr_create;        // if st is read locked by us
-
     // internal rrd2rrdr() members below this point
     struct {
-        int query_tier;                         // the selected tier
-        RRDR_OPTIONS query_options;       // RRDR_OPTION_* (as run by the query)
+        ONEWAYALLOC *owa;                   // the allocator used
+        struct query_target *qt;            // the QUERY_TARGET
+
+        RRDR_OPTIONS query_options;         // RRDR_OPTION_* (as run by the query)
 
-        long points_wanted;
-        long resampling_group;
-        NETDATA_DOUBLE resampling_divisor;
+        size_t points_wanted;               // used by SES and DES
+        size_t resampling_group;            // used by AVERAGE
+        NETDATA_DOUBLE resampling_divisor;  // used by AVERAGE
 
+        // grouping function pointers
         void (*grouping_create)(struct rrdresult *r, const char *options);
         void (*grouping_reset)(struct rrdresult *r);
         void (*grouping_free)(struct rrdresult *r);
         void (*grouping_add)(struct rrdresult *r, NETDATA_DOUBLE value);
         NETDATA_DOUBLE (*grouping_flush)(struct rrdresult *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
-        void *grouping_data;
 
-        TIER_QUERY_FETCH tier_query_fetch;
-        #ifdef NETDATA_INTERNAL_CHECKS
+        TIER_QUERY_FETCH tier_query_fetch;  // which value to use from STORAGE_POINT
+        void *grouping_data;                // the internal data of the grouping function
+
+#ifdef NETDATA_INTERNAL_CHECKS
         const char *log;
-        #endif
+#endif
 
+        // statistics
         size_t db_points_read;
         size_t result_points_generated;
         size_t tier_points_read[RRD_STORAGE_TIERS];
-        ONEWAYALLOC *owa;
     } internal;
 } RRDR;
 
 #define rrdr_rows(r) ((r)->rows)
 
 #include "database/rrd.h"
-extern void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
-extern RRDR *rrdr_create(ONEWAYALLOC *owa, struct rrdset *st, long n, struct context_param *context_param_list);
-extern RRDR *rrdr_create_for_x_dimensions(ONEWAYALLOC *owa, int dimensions, long points);
+void rrdr_free(ONEWAYALLOC *owa, RRDR *r);
+RRDR *rrdr_create(ONEWAYALLOC *owa, struct query_target *qt);
 
 #include "../web_api_v1.h"
 #include "web/api/queries/query.h"
 
-extern RRDR *rrd2rrdr(
-    ONEWAYALLOC *owa,
-    RRDSET *st, long points_wanted, long long after_wanted, long long before_wanted,
-    RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
-    struct context_param *context_param_list, const char *group_options, int timeout, int tier);
+RRDR *rrd2rrdr_legacy(
+        ONEWAYALLOC *owa,
+        RRDSET *st, size_t points, time_t after, time_t before,
+        RRDR_GROUPING group_method, time_t resampling_time, RRDR_OPTIONS options, const char *dimensions,
+        const char *group_options, time_t timeout, size_t tier, QUERY_SOURCE query_source);
+
+RRDR *rrd2rrdr(ONEWAYALLOC *owa, struct query_target *qt);
+bool query_target_calculate_window(struct query_target *qt);
 
-extern int rrdr_relative_window_to_absolute(long long *after, long long *before);
+bool rrdr_relative_window_to_absolute(time_t *after, time_t *before);
 
 #ifdef __cplusplus
 }
diff --git a/web/api/queries/ses/ses.h b/web/api/queries/ses/ses.h
index 094b8de3f..79b09fbdf 100644
--- a/web/api/queries/ses/ses.h
+++ b/web/api/queries/ses/ses.h
@@ -6,12 +6,12 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_init_ses(void);
+void grouping_init_ses(void);
 
-extern void grouping_create_ses(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_ses(RRDR *r);
-extern void grouping_free_ses(RRDR *r);
-extern void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_ses(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_ses(RRDR *r);
+void grouping_free_ses(RRDR *r);
+void grouping_add_ses(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_ses(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERIES_SES_H
diff --git a/web/api/queries/stddev/stddev.h b/web/api/queries/stddev/stddev.h
index c5c91f88d..4b8ffcd53 100644
--- a/web/api/queries/stddev/stddev.h
+++ b/web/api/queries/stddev/stddev.h
@@ -6,13 +6,13 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_stddev(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_stddev(RRDR *r);
-extern void grouping_free_stddev(RRDR *r);
-extern void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
-extern NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
-// extern NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
-// extern NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_stddev(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_stddev(RRDR *r);
+void grouping_free_stddev(RRDR *r);
+void grouping_add_stddev(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_stddev(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+NETDATA_DOUBLE grouping_flush_coefficient_of_variation(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+// NETDATA_DOUBLE grouping_flush_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+// NETDATA_DOUBLE grouping_flush_variance(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERIES_STDDEV_H
diff --git a/web/api/queries/sum/sum.h b/web/api/queries/sum/sum.h
index 4e7e396e9..898782775 100644
--- a/web/api/queries/sum/sum.h
+++ b/web/api/queries/sum/sum.h
@@ -6,10 +6,10 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_sum(RRDR *r, const char *options __maybe_unused);
-extern void grouping_reset_sum(RRDR *r);
-extern void grouping_free_sum(RRDR *r);
-extern void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_sum(RRDR *r, const char *options __maybe_unused);
+void grouping_reset_sum(RRDR *r);
+void grouping_free_sum(RRDR *r);
+void grouping_add_sum(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_sum(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERY_SUM_H
diff --git a/web/api/queries/trimmed_mean/trimmed_mean.h b/web/api/queries/trimmed_mean/trimmed_mean.h
index 1a4f63e9c..e66d92541 100644
--- a/web/api/queries/trimmed_mean/trimmed_mean.h
+++ b/web/api/queries/trimmed_mean/trimmed_mean.h
@@ -6,17 +6,17 @@
 #include "../query.h"
 #include "../rrdr.h"
 
-extern void grouping_create_trimmed_mean1(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean2(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean3(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean5(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean10(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean15(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean20(RRDR *r, const char *options);
-extern void grouping_create_trimmed_mean25(RRDR *r, const char *options);
-extern void grouping_reset_trimmed_mean(RRDR *r);
-extern void grouping_free_trimmed_mean(RRDR *r);
-extern void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value);
-extern NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+void grouping_create_trimmed_mean1(RRDR *r, const char *options);
+void grouping_create_trimmed_mean2(RRDR *r, const char *options);
+void grouping_create_trimmed_mean3(RRDR *r, const char *options);
+void grouping_create_trimmed_mean5(RRDR *r, const char *options);
+void grouping_create_trimmed_mean10(RRDR *r, const char *options);
+void grouping_create_trimmed_mean15(RRDR *r, const char *options);
+void grouping_create_trimmed_mean20(RRDR *r, const char *options);
+void grouping_create_trimmed_mean25(RRDR *r, const char *options);
+void grouping_reset_trimmed_mean(RRDR *r);
+void grouping_free_trimmed_mean(RRDR *r);
+void grouping_add_trimmed_mean(RRDR *r, NETDATA_DOUBLE value);
+NETDATA_DOUBLE grouping_flush_trimmed_mean(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
 
 #endif //NETDATA_API_QUERIES_TRIMMED_MEAN_H
diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c
index 97a00f91c..a9555a66b 100644
--- a/web/api/queries/weights.c
+++ b/web/api/queries/weights.c
@@ -56,40 +56,14 @@ typedef enum {
 
 struct register_result {
     RESULT_FLAGS flags;
-    RRDSET *st;
-    const char *chart_id;
-    const char *context;
-    const char *dim_name;
+    RRDCONTEXT_ACQUIRED *rca;
+    RRDINSTANCE_ACQUIRED *ria;
+    RRDMETRIC_ACQUIRED *rma;
     NETDATA_DOUBLE value;
-
-    struct register_result *next; // used to link contexts together
 };
 
-static void register_result_insert_callback(const char *name, void *value, void *data) {
-    (void)name;
-    (void)data;
-
-    struct register_result *t = (struct register_result *)value;
-
-    if(t->chart_id) t->chart_id = strdupz(t->chart_id);
-    if(t->context)  t->context  = strdupz(t->context);
-    if(t->dim_name) t->dim_name = strdupz(t->dim_name);
-}
-
-static void register_result_delete_callback(const char *name, void *value, void *data) {
-    (void)name;
-    (void)data;
-    struct register_result *t = (struct register_result *)value;
-
-    freez((void *)t->chart_id);
-    freez((void *)t->context);
-    freez((void *)t->dim_name);
-}
-
 static DICTIONARY *register_result_init() {
-    DICTIONARY *results = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
-    dictionary_register_insert_callback(results, register_result_insert_callback, results);
-    dictionary_register_delete_callback(results, register_result_delete_callback, results);
+    DICTIONARY *results = dictionary_create(DICT_OPTION_SINGLE_THREADED);
     return results;
 }
 
@@ -98,8 +72,9 @@ static void register_result_destroy(DICTIONARY *results) {
 }
 
 static void register_result(DICTIONARY *results,
-                            RRDSET *st,
-                            RRDDIM *d,
+                            RRDCONTEXT_ACQUIRED *rca,
+                            RRDINSTANCE_ACQUIRED *ria,
+                            RRDMETRIC_ACQUIRED *rma,
                             NETDATA_DOUBLE value,
                             RESULT_FLAGS flags,
                             WEIGHTS_STATS *stats,
@@ -120,25 +95,25 @@ static void register_result(DICTIONARY *results,
 
     struct register_result t = {
         .flags = flags,
-        .st = st,
-        .chart_id = st->id,
-        .context = st->context,
-        .dim_name = d->name,
+        .rca = rca,
+        .ria = ria,
+        .rma = rma,
         .value = v
     };
 
-    char buf[5000 + 1];
-    snprintfz(buf, 5000, "%s:%s", st->id, d->name);
-    dictionary_set(results, buf, &t, sizeof(struct register_result));
+    // we can use the pointer address or RMA as a unique key for each metric
+    char buf[20 + 1];
+    ssize_t len = snprintfz(buf, 20, "%p", rma);
+    dictionary_set_advanced(results, buf, len + 1, &t, sizeof(struct register_result), NULL);
 }
 
 // ----------------------------------------------------------------------------
 // Generation of JSON output for the results
 
 static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *wb,
-                                   long long after, long long before,
-                                   long long baseline_after, long long baseline_before,
-                                   long points, WEIGHTS_METHOD method,
+                                   time_t after, time_t before,
+                                   time_t baseline_after, time_t baseline_before,
+                                   size_t points, WEIGHTS_METHOD method,
                                    RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
                                    size_t examined_dimensions __maybe_unused, usec_t duration,
                                    WEIGHTS_STATS *stats) {
@@ -147,10 +122,10 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
                        "\t\"after\": %lld,\n"
                        "\t\"before\": %lld,\n"
                        "\t\"duration\": %lld,\n"
-                       "\t\"points\": %ld,\n",
-                       after,
-                       before,
-                       before - after,
+                       "\t\"points\": %zu,\n",
+                       (long long)after,
+                       (long long)before,
+                       (long long)(before - after),
                        points
                        );
 
@@ -159,10 +134,10 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
                            "\t\"baseline_after\": %lld,\n"
                            "\t\"baseline_before\": %lld,\n"
                            "\t\"baseline_duration\": %lld,\n"
-                           "\t\"baseline_points\": %ld,\n",
-                           baseline_after,
-                           baseline_before,
-                           baseline_before - baseline_after,
+                           "\t\"baseline_points\": %zu,\n",
+                           (long long)baseline_after,
+                           (long long)baseline_before,
+                           (long long)(baseline_before - baseline_after),
                            points << shifts
                        );
 
@@ -181,7 +156,7 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
                        stats->db_points
                    );
 
-    for(int tier = 0; tier < storage_tiers ;tier++)
+    for(size_t tier = 0; tier < storage_tiers ;tier++)
         buffer_sprintf(wb, "%s%zu", tier?", ":"", stats->db_points_per_tier[tier]);
 
     buffer_sprintf(wb, " ]\n"
@@ -193,13 +168,13 @@ static void results_header_to_json(DICTIONARY *results __maybe_unused, BUFFER *w
                        weights_method_to_string(method)
                    );
 
-    web_client_api_request_v1_data_options_to_string(wb, options);
+    web_client_api_request_v1_data_options_to_buffer(wb, options);
 }
 
 static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
-                                                long long after, long long before,
-                                                long long baseline_after, long long baseline_before,
-                                                long points, WEIGHTS_METHOD method,
+                                                time_t after, time_t before,
+                                                time_t baseline_after, time_t baseline_before,
+                                                size_t points, WEIGHTS_METHOD method,
                                                 RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
                                                 size_t examined_dimensions, usec_t duration,
                                                 WEIGHTS_STATS *stats) {
@@ -211,23 +186,23 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
 
     size_t charts = 0, chart_dims = 0, total_dimensions = 0;
     struct register_result *t;
-    RRDSET *last_st = NULL; // never access this - we use it only for comparison
+    RRDINSTANCE_ACQUIRED *last_ria = NULL; // never access this - we use it only for comparison
     dfe_start_read(results, t) {
-        if(!last_st || t->st != last_st) {
-            last_st = t->st;
+        if(t->ria != last_ria) {
+            last_ria = t->ria;
 
             if(charts) buffer_strcat(wb, "\n\t\t\t}\n\t\t},\n");
             buffer_strcat(wb, "\t\t\"");
-            buffer_strcat(wb, t->chart_id);
+            buffer_strcat(wb, rrdinstance_acquired_id(t->ria));
             buffer_strcat(wb, "\": {\n");
             buffer_strcat(wb, "\t\t\t\"context\": \"");
-            buffer_strcat(wb, t->context);
+            buffer_strcat(wb, rrdcontext_acquired_id(t->rca));
             buffer_strcat(wb, "\",\n\t\t\t\"dimensions\": {\n");
             charts++;
             chart_dims = 0;
         }
         if (chart_dims) buffer_sprintf(wb, ",\n");
-        buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, t->dim_name, t->value);
+        buffer_sprintf(wb, "\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value);
         chart_dims++;
         total_dimensions++;
     }
@@ -250,9 +225,9 @@ static size_t registered_results_to_json_charts(DICTIONARY *results, BUFFER *wb,
 }
 
 static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *wb,
-                                                  long long after, long long before,
-                                                  long long baseline_after, long long baseline_before,
-                                                  long points, WEIGHTS_METHOD method,
+                                                  time_t after, time_t before,
+                                                  time_t baseline_after, time_t baseline_before,
+                                                  size_t points, WEIGHTS_METHOD method,
                                                   RRDR_GROUPING group, RRDR_OPTIONS options, uint32_t shifts,
                                                   size_t examined_dimensions, usec_t duration,
                                                   WEIGHTS_STATS *stats) {
@@ -260,78 +235,80 @@ static size_t registered_results_to_json_contexts(DICTIONARY *results, BUFFER *w
     results_header_to_json(results, wb, after, before, baseline_after, baseline_before,
                            points, method, group, options, shifts, examined_dimensions, duration, stats);
 
-    DICTIONARY *context_results = dictionary_create(
-         DICTIONARY_FLAG_SINGLE_THREADED
-        |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE
-        |DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
-        |DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
-        );
+    buffer_strcat(wb, "\",\n\t\"contexts\": {\n");
 
+    size_t contexts = 0, charts = 0, total_dimensions = 0, context_dims = 0, chart_dims = 0;
+    NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0;
     struct register_result *t;
+    RRDCONTEXT_ACQUIRED *last_rca = NULL;
+    RRDINSTANCE_ACQUIRED *last_ria = NULL;
     dfe_start_read(results, t) {
-        struct register_result *tc = dictionary_set(context_results, t->context, t, sizeof(*t));
-        if(tc == t)
-            t->next = NULL;
-        else {
-            t->next = tc->next;
-            tc->next = t;
+
+        if(t->rca != last_rca) {
+            last_rca = t->rca;
+
+            if(contexts)
+                buffer_sprintf(wb, "\n"
+                                   "\t\t\t\t\t},\n"
+                                   "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+                                   "\t\t\t\t}\n\t\t\t},\n"
+                                   "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n"
+                        , charts_total_weight / (double)chart_dims
+                        , contexts_total_weight / (double)context_dims);
+
+            buffer_strcat(wb, "\t\t\"");
+            buffer_strcat(wb, rrdcontext_acquired_id(t->rca));
+            buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n");
+
+            contexts++;
+            charts = 0;
+            context_dims = 0;
+            contexts_total_weight = 0.0;
+
+            last_ria = NULL;
         }
-    }
-    dfe_done(t);
 
-    buffer_strcat(wb, "\",\n\t\"contexts\": {\n");
+        if(t->ria != last_ria) {
+            last_ria = t->ria;
 
-    size_t contexts = 0, total_dimensions = 0, charts = 0, context_dims = 0, chart_dims = 0;
-    NETDATA_DOUBLE contexts_total_weight = 0.0, charts_total_weight = 0.0;
-    RRDSET *last_st = NULL; // never access this - we use it only for comparison
-    dfe_start_read(context_results, t) {
-
-        if(contexts)
-            buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t},\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims);
-
-        contexts++;
-        context_dims = 0;
-        contexts_total_weight = 0.0;
-
-        buffer_strcat(wb, "\t\t\"");
-        buffer_strcat(wb, t->context);
-        buffer_strcat(wb, "\": {\n\t\t\t\"charts\":{\n");
-
-        charts = 0;
-        chart_dims = 0;
-        struct register_result *tt;
-        for(tt = t; tt ; tt = tt->next) {
-            if(!last_st || tt->st != last_st) {
-                last_st = tt->st;
-
-                if(charts)
-                    buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t},\n", charts_total_weight / chart_dims);
-
-                buffer_strcat(wb, "\t\t\t\t\"");
-                buffer_strcat(wb, tt->chart_id);
-                buffer_strcat(wb, "\": {\n");
-                buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n");
-                charts++;
-                chart_dims = 0;
-                charts_total_weight = 0.0;
-            }
-
-            if (chart_dims) buffer_sprintf(wb, ",\n");
-            buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, tt->dim_name, tt->value);
-            charts_total_weight += tt->value;
-            contexts_total_weight += tt->value;
-            chart_dims++;
-            context_dims++;
-            total_dimensions++;
+            if(charts)
+                buffer_sprintf(wb, "\n"
+                                   "\t\t\t\t\t},\n"
+                                   "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+                                   "\t\t\t\t},\n"
+                                   , charts_total_weight / (double)chart_dims);
+
+            buffer_strcat(wb, "\t\t\t\t\"");
+            buffer_strcat(wb, rrdinstance_acquired_id(t->ria));
+            buffer_strcat(wb, "\": {\n");
+            buffer_strcat(wb, "\t\t\t\t\t\"dimensions\": {\n");
+
+            charts++;
+            chart_dims = 0;
+            charts_total_weight = 0.0;
         }
+
+        if (chart_dims) buffer_sprintf(wb, ",\n");
+        buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": " NETDATA_DOUBLE_FORMAT, rrdmetric_acquired_name(t->rma), t->value);
+        charts_total_weight += t->value;
+        contexts_total_weight += t->value;
+        chart_dims++;
+        context_dims++;
+        total_dimensions++;
     }
     dfe_done(t);
 
-    dictionary_destroy(context_results);
-
     // close dimensions and chart
     if (total_dimensions)
-        buffer_sprintf(wb, "\n\t\t\t\t\t},\n\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t\t\t}\n\t\t\t},\n\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n\t\t}\n", charts_total_weight / chart_dims, contexts_total_weight / context_dims);
+        buffer_sprintf(wb, "\n"
+                           "\t\t\t\t\t},\n"
+                           "\t\t\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+                           "\t\t\t\t}\n"
+                           "\t\t\t},\n"
+                           "\t\t\t\"weight\":" NETDATA_DOUBLE_FORMAT "\n"
+                           "\t\t}\n"
+                           , charts_total_weight / (double)chart_dims
+                           , contexts_total_weight / (double)context_dims);
 
     // close correlated_charts
     buffer_sprintf(wb, "\t},\n"
@@ -391,7 +368,10 @@ static size_t calculate_pairs_diff(DIFFS_NUMBERS *diffs, NETDATA_DOUBLE *arr, si
     return added;
 }
 
-static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMBERS highlight_diffs[], int high_size, uint32_t base_shifts) {
+static double ks_2samp(
+        DIFFS_NUMBERS baseline_diffs[], int base_size,
+        DIFFS_NUMBERS highlight_diffs[], int high_size,
+        uint32_t base_shifts) {
 
     qsort(baseline_diffs, base_size, sizeof(DIFFS_NUMBERS), compare_diffs);
     qsort(highlight_diffs, high_size, sizeof(DIFFS_NUMBERS), compare_diffs);
@@ -414,7 +394,7 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
     // This would require a lot of multiplications and divisions.
     //
     // To speed it up, we do the binary search to find the index of each number
-    // but then we divide the base index by the power of two number (shifts) it
+    // but, then we divide the base index by the power of two number (shifts) it
     // is bigger than high index. So the 2 indexes are now comparable.
     // We also keep track of the original indexes with min and max, to properly
     // calculate their percentages once the loops finish.
@@ -495,7 +475,9 @@ static double ks_2samp(DIFFS_NUMBERS baseline_diffs[], int base_size, DIFFS_NUMB
 
 static double kstwo(
     NETDATA_DOUBLE baseline[], int baseline_points,
-    NETDATA_DOUBLE highlight[], int highlight_points, uint32_t base_shifts) {
+    NETDATA_DOUBLE highlight[], int highlight_points,
+    uint32_t base_shifts) {
+
     // -1 in size, since the calculate_pairs_diffs() returns one less point
     DIFFS_NUMBERS baseline_diffs[baseline_points - 1];
     DIFFS_NUMBERS highlight_diffs[highlight_points - 1];
@@ -514,308 +496,215 @@ static double kstwo(
     return ks_2samp(baseline_diffs, base_size, highlight_diffs, high_size, base_shifts);
 }
 
+NETDATA_DOUBLE *rrd2rrdr_ks2(
+        ONEWAYALLOC *owa, RRDHOST *host,
+        RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+        time_t after, time_t before, size_t points, RRDR_OPTIONS options,
+        RRDR_GROUPING group_method, const char *group_options, size_t tier,
+        WEIGHTS_STATS *stats,
+        size_t *entries
+        ) {
+
+    NETDATA_DOUBLE *ret = NULL;
+
+    QUERY_TARGET_REQUEST qtr = {
+            .host = host,
+            .rca = rca,
+            .ria = ria,
+            .rma = rma,
+            .after = after,
+            .before = before,
+            .points = points,
+            .options = options,
+            .group_method = group_method,
+            .group_options = group_options,
+            .tier = tier,
+            .query_source = QUERY_SOURCE_API_WEIGHTS,
+    };
 
-static int rrdset_metric_correlations_ks2(RRDSET *st, DICTIONARY *results,
-                                          long long baseline_after, long long baseline_before,
-                                          long long after, long long before,
-                                          long long points, RRDR_OPTIONS options,
-                                          RRDR_GROUPING group, const char *group_options, int tier,
-                                          uint32_t shifts, int timeout,
-                                          WEIGHTS_STATS *stats, bool register_zero) {
-    options |= RRDR_OPTION_NATURAL_POINTS;
-
-    long group_time = 0;
-    struct context_param  *context_param_list = NULL;
-
-    int examined_dimensions = 0;
-
-    RRDR *high_rrdr = NULL;
-    RRDR *base_rrdr = NULL;
-
-    // get first the highlight to find the number of points available
-    stats->db_queries++;
-    usec_t started_usec = now_realtime_usec();
-    ONEWAYALLOC *owa = onewayalloc_create(0);
-    high_rrdr = rrd2rrdr(owa, st, points,
-                         after, before, group,
-                         group_time, options, NULL, context_param_list, group_options,
-                         timeout, tier);
-    if(!high_rrdr) {
-        info("Metric correlations: rrd2rrdr() failed for the highlighted window on chart '%s'.", st->name);
+    RRDR *r = rrd2rrdr(owa, query_target_create(&qtr));
+    if(!r)
         goto cleanup;
-    }
 
-    for(int i = 0; i < storage_tiers ;i++)
-        stats->db_points_per_tier[i] += high_rrdr->internal.tier_points_read[i];
+    stats->db_queries++;
+    stats->result_points += r->internal.result_points_generated;
+    stats->db_points += r->internal.db_points_read;
+    for(size_t tr = 0; tr < storage_tiers ; tr++)
+        stats->db_points_per_tier[tr] += r->internal.tier_points_read[tr];
 
-    stats->db_points     += high_rrdr->internal.db_points_read;
-    stats->result_points += high_rrdr->internal.result_points_generated;
-    if(!high_rrdr->d) {
-        info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
+    if(r->d != 1) {
+        error("WEIGHTS: on query '%s' expected 1 dimension in RRDR but got %zu", r->internal.qt->id, r->d);
         goto cleanup;
     }
-    if(high_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
-        info("Metric correlations: rrd2rrdr() on highlighted window timed out '%s'.", st->name);
+
+    if(unlikely(r->od[0] & RRDR_DIMENSION_HIDDEN))
         goto cleanup;
-    }
-    int high_points = rrdr_rows(high_rrdr);
 
-    usec_t now_usec = now_realtime_usec();
-    if(now_usec - started_usec > timeout * USEC_PER_MS)
+    if(unlikely(!(r->od[0] & RRDR_DIMENSION_NONZERO)))
         goto cleanup;
 
-    // get the baseline, requesting the same number of points as the highlight
-    stats->db_queries++;
-    base_rrdr = rrd2rrdr(owa, st,high_points << shifts,
-                    baseline_after, baseline_before, group,
-                    group_time, options, NULL, context_param_list, group_options,
-                    (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)), tier);
-    if(!base_rrdr) {
-        info("Metric correlations: rrd2rrdr() failed for the baseline window on chart '%s'.", st->name);
+    if(rrdr_rows(r) < 2)
         goto cleanup;
-    }
 
-    for(int i = 0; i < storage_tiers ;i++)
-        stats->db_points_per_tier[i] += base_rrdr->internal.tier_points_read[i];
+    *entries = rrdr_rows(r);
+    ret = onewayalloc_mallocz(owa, sizeof(NETDATA_DOUBLE) * rrdr_rows(r));
 
-    stats->db_points     += base_rrdr->internal.db_points_read;
-    stats->result_points += base_rrdr->internal.result_points_generated;
-    if(!base_rrdr->d) {
-        info("Metric correlations: rrd2rrdr() did not return any dimensions on chart '%s'.", st->name);
-        goto cleanup;
-    }
-    if (base_rrdr->d != high_rrdr->d) {
-        info("Cannot generate metric correlations for chart '%s' when the baseline and the highlight have different number of dimensions.", st->name);
-        goto cleanup;
-    }
-    if(base_rrdr->result_options & RRDR_RESULT_OPTION_CANCEL) {
-        info("Metric correlations: rrd2rrdr() on baseline window timed out '%s'.", st->name);
-        goto cleanup;
-    }
-    int base_points = rrdr_rows(base_rrdr);
+    // copy the points of the dimension to a contiguous array
+    // there is no need to check for empty values, since empty values are already zero
+    // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
+    memcpy(ret, r->v, rrdr_rows(r) * sizeof(NETDATA_DOUBLE));
 
-    now_usec = now_realtime_usec();
-    if(now_usec - started_usec > timeout * USEC_PER_MS)
-        goto cleanup;
+cleanup:
+    rrdr_free(owa, r);
+    return ret;
+}
+
+static void rrdset_metric_correlations_ks2(
+        RRDHOST *host,
+        RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+        DICTIONARY *results,
+        time_t baseline_after, time_t baseline_before,
+        time_t after, time_t before,
+        size_t points, RRDR_OPTIONS options,
+        RRDR_GROUPING group_method, const char *group_options, size_t tier,
+        uint32_t shifts,
+        WEIGHTS_STATS *stats, bool register_zero
+        ) {
+
+    options |= RRDR_OPTION_NATURAL_POINTS;
 
-    // we need at least 2 points to do the job
-    if(base_points < 2 || high_points < 2)
+    ONEWAYALLOC *owa = onewayalloc_create(16 * 1024);
+
+    size_t high_points = 0;
+    NETDATA_DOUBLE *highlight = rrd2rrdr_ks2(
+            owa, host, rca, ria, rma, after, before, points,
+            options, group_method, group_options, tier, stats, &high_points);
+
+    if(!highlight)
         goto cleanup;
 
-    // for each dimension
-    RRDDIM *d;
-    int i;
-    for(i = 0, d = base_rrdr->st->dimensions ; d && i < base_rrdr->d; i++, d = d->next) {
+    size_t base_points = 0;
+    NETDATA_DOUBLE *baseline = rrd2rrdr_ks2(
+            owa, host, rca, ria, rma, baseline_after, baseline_before, high_points << shifts,
+            options, group_method, group_options, tier, stats, &base_points);
+
+    if(!baseline)
+        goto cleanup;
 
-        // skip the not evaluated ones
-        if(unlikely(base_rrdr->od[i] & RRDR_DIMENSION_HIDDEN) || (high_rrdr->od[i] & RRDR_DIMENSION_HIDDEN))
-            continue;
+    stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1);
 
-        examined_dimensions++;
+    double prob = kstwo(baseline, (int)base_points, highlight, (int)high_points, shifts);
+    if(!isnan(prob) && !isinf(prob)) {
 
-        // skip the dimensions that are just zero for both the baseline and the highlight
-        if(unlikely(!(base_rrdr->od[i] & RRDR_DIMENSION_NONZERO) && !(high_rrdr->od[i] & RRDR_DIMENSION_NONZERO)))
-            continue;
-
-        // copy the baseline points of the dimension to a contiguous array
-        // there is no need to check for empty values, since empty are already zero
-        NETDATA_DOUBLE baseline[base_points];
-        for(int c = 0; c < base_points; c++)
-            baseline[c] = base_rrdr->v[ c * base_rrdr->d + i ];
-
-        // copy the highlight points of the dimension to a contiguous array
-        // there is no need to check for empty values, since empty values are already zero
-        // https://github.com/netdata/netdata/blob/6e3144683a73a2024d51425b20ecfd569034c858/web/api/queries/average/average.c#L41-L43
-        NETDATA_DOUBLE highlight[high_points];
-        for(int c = 0; c < high_points; c++)
-            highlight[c] = high_rrdr->v[ c * high_rrdr->d + i ];
-
-        stats->binary_searches += 2 * (base_points - 1) + 2 * (high_points - 1);
-
-        double prob = kstwo(baseline, base_points, highlight, high_points, shifts);
-        if(!isnan(prob) && !isinf(prob)) {
-
-            // these conditions should never happen, but still let's check
-            if(unlikely(prob < 0.0)) {
-                error("Metric correlations: kstwo() returned a negative number: %f", prob);
-                prob = -prob;
-            }
-            if(unlikely(prob > 1.0)) {
-                error("Metric correlations: kstwo() returned a number above 1.0: %f", prob);
-                prob = 1.0;
-            }
-
-            // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated
-            // so we flip the result of kstwo()
-            register_result(results, base_rrdr->st, d, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero);
+        // these conditions should never happen, but still let's check
+        if(unlikely(prob < 0.0)) {
+            error("Metric correlations: kstwo() returned a negative number: %f", prob);
+            prob = -prob;
+        }
+        if(unlikely(prob > 1.0)) {
+            error("Metric correlations: kstwo() returned a number above 1.0: %f", prob);
+            prob = 1.0;
         }
+
+        // to spread the results evenly, 0.0 needs to be the less correlated and 1.0 the most correlated
+        // so, we flip the result of kstwo()
+        register_result(results, rca, ria, rma, 1.0 - prob, RESULT_IS_BASE_HIGH_RATIO, stats, register_zero);
     }
 
 cleanup:
-    rrdr_free(owa, high_rrdr);
-    rrdr_free(owa, base_rrdr);
     onewayalloc_destroy(owa);
-    return examined_dimensions;
 }
 
 // ----------------------------------------------------------------------------
 // VOLUME algorithm functions
 
-static int rrdset_metric_correlations_volume(RRDSET *st, DICTIONARY *results,
-                                             long long baseline_after, long long baseline_before,
-                                             long long after, long long before,
-                                             RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,
-                                             int tier, int timeout,
-                                             WEIGHTS_STATS *stats, bool register_zero) {
+static void merge_query_value_to_stats(QUERY_VALUE *qv, WEIGHTS_STATS *stats) {
+    stats->db_queries++;
+    stats->result_points += qv->result_points;
+    stats->db_points += qv->points_read;
+    for(size_t tier = 0; tier < storage_tiers ; tier++)
+        stats->db_points_per_tier[tier] += qv->storage_points_per_tier[tier];
+}
+
+static void rrdset_metric_correlations_volume(
+        RRDHOST *host,
+        RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+        DICTIONARY *results,
+        time_t baseline_after, time_t baseline_before,
+        time_t after, time_t before,
+        RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
+        size_t tier,
+        WEIGHTS_STATS *stats, bool register_zero) {
 
     options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ABSOLUTE | RRDR_OPTION_NATURAL_POINTS;
-    long group_time = 0;
-
-    int examined_dimensions = 0;
-    int ret, value_is_null;
-    usec_t started_usec = now_realtime_usec();
 
-    RRDDIM *d;
-    for(d = st->dimensions; d ; d = d->next) {
-        usec_t now_usec = now_realtime_usec();
-        if(now_usec - started_usec > timeout * USEC_PER_MS)
-            return examined_dimensions;
+    QUERY_VALUE baseline_average = rrdmetric2value(host, rca, ria, rma, baseline_after, baseline_before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+    merge_query_value_to_stats(&baseline_average, stats);
 
-        // we count how many metrics we evaluated
-        examined_dimensions++;
+    if(!netdata_double_isnumber(baseline_average.value)) {
+        // this means no data for the baseline window, but we may have data for the highlighted one - assume zero
+        baseline_average.value = 0.0;
+    }
 
-        // there is no point to pass a timeout to these queries
-        // since the query engine checks for a timeout between
-        // dimensions, and we query a single dimension at a time.
-
-        stats->db_queries++;
-        NETDATA_DOUBLE baseline_average = NAN;
-        NETDATA_DOUBLE base_anomaly_rate = 0;
-        value_is_null = 1;
-        ret = rrdset2value_api_v1(st, NULL, &baseline_average, d->id, 1,
-                                  baseline_after, baseline_before,
-                                  group, group_options, group_time, options,
-                                  NULL, NULL,
-                                  &stats->db_points, stats->db_points_per_tier,
-                                  &stats->result_points,
-                                  &value_is_null, &base_anomaly_rate, 0, tier);
-
-        if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(baseline_average)) {
-            // this means no data for the baseline window, but we may have data for the highlighted one - assume zero
-            baseline_average = 0.0;
-        }
+    QUERY_VALUE highlight_average = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+    merge_query_value_to_stats(&highlight_average, stats);
 
-        stats->db_queries++;
-        NETDATA_DOUBLE highlight_average = NAN;
-        NETDATA_DOUBLE high_anomaly_rate = 0;
-        value_is_null = 1;
-        ret = rrdset2value_api_v1(st, NULL, &highlight_average, d->id, 1,
-                                  after, before,
-                                  group, group_options, group_time, options,
-                                  NULL, NULL,
-                                  &stats->db_points, stats->db_points_per_tier,
-                                  &stats->result_points,
-                                  &value_is_null, &high_anomaly_rate, 0, tier);
-
-        if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_average)) {
-            // this means no data for the highlighted duration - so skip it
-            continue;
-        }
+    if(!netdata_double_isnumber(highlight_average.value))
+        return;
 
-        if(baseline_average == highlight_average) {
-            // they are the same - let's move on
-            continue;
-        }
+    if(baseline_average.value == highlight_average.value) {
+        // they are the same - let's move on
+        return;
+    }
 
-        stats->db_queries++;
-        NETDATA_DOUBLE highlight_countif = NAN;
-        value_is_null = 1;
-
-        char highlighted_countif_options[50 + 1];
-        snprintfz(highlighted_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average < baseline_average ? "<":">", baseline_average);
-
-        ret = rrdset2value_api_v1(st, NULL, &highlight_countif, d->id, 1,
-                                  after, before,
-                                  RRDR_GROUPING_COUNTIF,highlighted_countif_options,
-                                  group_time, options,
-                                  NULL, NULL,
-                                  &stats->db_points, stats->db_points_per_tier,
-                                  &stats->result_points,
-                                  &value_is_null, NULL, 0, tier);
-
-        if(ret != HTTP_RESP_OK || value_is_null || !netdata_double_isnumber(highlight_countif)) {
-            info("MC: highlighted countif query failed, but highlighted average worked - strange...");
-            continue;
-        }
+    char highlight_countif_options[50 + 1];
+    snprintfz(highlight_countif_options, 50, "%s" NETDATA_DOUBLE_FORMAT, highlight_average.value < baseline_average.value ? "<" : ">", baseline_average.value);
+    QUERY_VALUE highlight_countif = rrdmetric2value(host, rca, ria, rma, after, before, options, RRDR_GROUPING_COUNTIF, highlight_countif_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+    merge_query_value_to_stats(&highlight_countif, stats);
 
-        // this represents the percentage of time
-        // the highlighted window was above/below the baseline window
-        // (above or below depending on their averages)
-        highlight_countif = highlight_countif / 100.0; // countif returns 0 - 100.0
+    if(!netdata_double_isnumber(highlight_countif.value)) {
+        info("WEIGHTS: highlighted countif query failed, but highlighted average worked - strange...");
+        return;
+    }
 
-        RESULT_FLAGS flags;
-        NETDATA_DOUBLE pcent = NAN;
-        if(isgreater(baseline_average, 0.0) || isless(baseline_average, 0.0)) {
-            flags = RESULT_IS_BASE_HIGH_RATIO;
-            pcent = (highlight_average - baseline_average) / baseline_average * highlight_countif;
-        }
-        else {
-            flags = RESULT_IS_PERCENTAGE_OF_TIME;
-            pcent = highlight_countif;
-        }
+    // this represents the percentage of time
+    // the highlighted window was above/below the baseline window
+    // (above or below depending on their averages)
+    highlight_countif.value = highlight_countif.value / 100.0; // countif returns 0 - 100.0
 
-        register_result(results, st, d, pcent, flags, stats, register_zero);
+    RESULT_FLAGS flags;
+    NETDATA_DOUBLE pcent = NAN;
+    if(isgreater(baseline_average.value, 0.0) || isless(baseline_average.value, 0.0)) {
+        flags = RESULT_IS_BASE_HIGH_RATIO;
+        pcent = (highlight_average.value - baseline_average.value) / baseline_average.value * highlight_countif.value;
+    }
+    else {
+        flags = RESULT_IS_PERCENTAGE_OF_TIME;
+        pcent = highlight_countif.value;
     }
 
-    return examined_dimensions;
+    register_result(results, rca, ria, rma, pcent, flags, stats, register_zero);
 }
 
 // ----------------------------------------------------------------------------
 // ANOMALY RATE algorithm functions
 
-static int rrdset_weights_anomaly_rate(RRDSET *st, DICTIONARY *results,
-                                       long long after, long long before,
-                                       RRDR_OPTIONS options, RRDR_GROUPING group, const char *group_options,
-                                       int tier, int timeout,
-                                       WEIGHTS_STATS *stats, bool register_zero) {
+static void rrdset_weights_anomaly_rate(
+        RRDHOST *host,
+        RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma,
+        DICTIONARY *results,
+        time_t after, time_t before,
+        RRDR_OPTIONS options, RRDR_GROUPING group_method, const char *group_options,
+        size_t tier,
+        WEIGHTS_STATS *stats, bool register_zero) {
 
     options |= RRDR_OPTION_MATCH_IDS | RRDR_OPTION_ANOMALY_BIT | RRDR_OPTION_NATURAL_POINTS;
-    long group_time = 0;
 
-    int examined_dimensions = 0;
-    int ret, value_is_null;
-    usec_t started_usec = now_realtime_usec();
-
-    RRDDIM *d;
-    for(d = st->dimensions; d ; d = d->next) {
-        usec_t now_usec = now_realtime_usec();
-        if(now_usec - started_usec > timeout * USEC_PER_MS)
-            return examined_dimensions;
+    QUERY_VALUE qv = rrdmetric2value(host, rca, ria, rma, after, before, options, group_method, group_options, tier, 0, QUERY_SOURCE_API_WEIGHTS);
+    merge_query_value_to_stats(&qv, stats);
 
-        // we count how many metrics we evaluated
-        examined_dimensions++;
-
-        // there is no point to pass a timeout to these queries
-        // since the query engine checks for a timeout between
-        // dimensions, and we query a single dimension at a time.
-
-        stats->db_queries++;
-        NETDATA_DOUBLE average = NAN;
-        NETDATA_DOUBLE anomaly_rate = 0;
-        value_is_null = 1;
-        ret = rrdset2value_api_v1(st, NULL, &average, d->id, 1,
-                                  after, before,
-                                  group, group_options, group_time, options,
-                                  NULL, NULL,
-                                  &stats->db_points, stats->db_points_per_tier,
-                                  &stats->result_points,
-                                  &value_is_null, &anomaly_rate, 0, tier);
-
-        if(ret == HTTP_RESP_OK || !value_is_null || netdata_double_isnumber(average))
-            register_result(results, st, d, average, 0, stats, register_zero);
-    }
-
-    return examined_dimensions;
+    if(netdata_double_isnumber(qv.value))
+        register_result(results, rca, ria, rma, qv.value, 0, stats, register_zero);
 }
 
 // ----------------------------------------------------------------------------
@@ -853,7 +742,7 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) {
     struct register_result *t;
 
     // count the dimensions
-    size_t dimensions = dictionary_stats_entries(results);
+    size_t dimensions = dictionary_entries(results);
     if(!dimensions) return 0;
 
     if(stats->max_base_high_ratio == 0.0)
@@ -903,15 +792,17 @@ static size_t spread_results_evenly(DICTIONARY *results, WEIGHTS_STATS *stats) {
 // ----------------------------------------------------------------------------
 // The main function
 
-int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
-                        RRDR_GROUPING group, const char *group_options,
-                        long long baseline_after, long long baseline_before,
-                        long long after, long long before,
-                        long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout) {
+int web_api_v1_weights(
+        RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
+        RRDR_GROUPING group, const char *group_options,
+        time_t baseline_after, time_t baseline_before,
+        time_t after, time_t before,
+        size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout) {
+
     WEIGHTS_STATS stats = {};
 
     DICTIONARY *results = register_result_init();
-    DICTIONARY *charts = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);;
+    DICTIONARY *metrics = NULL;
     char *error = NULL;
     int resp = HTTP_RESP_OK;
 
@@ -1000,20 +891,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
         baseline_after = baseline_before - (high_delta << shifts);
     }
 
-    // dont lock here and wait for results
-    // get the charts and run mc after
-    RRDSET *st;
-    rrdhost_rdlock(host);
-    rrdset_foreach_read(st, host) {
-        if (rrdset_is_available_for_viewers(st)) {
-            if(!contexts || simple_pattern_matches(contexts, st->context))
-                dictionary_set(charts, st->name, NULL, 0);
-        }
-    }
-    rrdhost_unlock(host);
-
     size_t examined_dimensions = 0;
-    void *ptr;
 
     bool register_zero = true;
     if(options & RRDR_OPTION_NONZERO) {
@@ -1021,8 +899,11 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
         options &= ~RRDR_OPTION_NONZERO;
     }
 
-    // for every chart in the dictionary
-    dfe_start_read(charts, ptr) {
+    metrics = rrdcontext_all_metrics_to_dict(host, contexts);
+    struct metric_entry *me;
+
+    // for every metric_entry in the dictionary
+    dfe_start_read(metrics, me) {
         usec_t now_usec = now_realtime_usec();
         if(now_usec - started_usec > timeout_usec) {
             error = "timed out";
@@ -1030,46 +911,48 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
             goto cleanup;
         }
 
-        st = rrdset_find_byname(host, ptr_name);
-        if(!st) continue;
-
-        rrdset_rdlock(st);
+        examined_dimensions++;
 
         switch(method) {
             case WEIGHTS_METHOD_ANOMALY_RATE:
                 options |= RRDR_OPTION_ANOMALY_BIT;
-                points = 1;
-                examined_dimensions += rrdset_weights_anomaly_rate(st, results,
-                                                                   after, before,
-                                                                   options, group, group_options, tier,
-                                                                   (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
-                                                                   &stats, register_zero);
+                rrdset_weights_anomaly_rate(
+                        host,
+                        me->rca, me->ria, me->rma,
+                        results,
+                        after, before,
+                        options, group, group_options, tier,
+                        &stats, register_zero
+                        );
                 break;
 
             case WEIGHTS_METHOD_MC_VOLUME:
-                points = 1;
-                examined_dimensions += rrdset_metric_correlations_volume(st, results,
-                                                                baseline_after, baseline_before,
-                                                                after, before,
-                                                                options, group, group_options, tier,
-                                                                (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
-                                                                &stats, register_zero);
+                rrdset_metric_correlations_volume(
+                        host,
+                        me->rca, me->ria, me->rma,
+                        results,
+                        baseline_after, baseline_before,
+                        after, before,
+                        options, group, group_options, tier,
+                        &stats, register_zero
+                        );
                 break;
 
             default:
             case WEIGHTS_METHOD_MC_KS2:
-                examined_dimensions += rrdset_metric_correlations_ks2(st, results,
-                                                             baseline_after, baseline_before,
-                                                             after, before,
-                                                             points, options, group, group_options, tier, shifts,
-                                                             (int)(timeout - ((now_usec - started_usec) / USEC_PER_MS)),
-                                                             &stats, register_zero);
+                rrdset_metric_correlations_ks2(
+                        host,
+                        me->rca, me->ria, me->rma,
+                        results,
+                        baseline_after, baseline_before,
+                        after, before, points,
+                        options, group, group_options, tier, shifts,
+                        &stats, register_zero
+                        );
                 break;
         }
-
-        rrdset_unlock(st);
     }
-    dfe_done(ptr);
+    dfe_done(me);
 
     if(!register_zero)
         options |= RRDR_OPTION_NONZERO;
@@ -1085,22 +968,26 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
     size_t added_dimensions = 0;
     switch(format) {
         case WEIGHTS_FORMAT_CHARTS:
-            added_dimensions = registered_results_to_json_charts(results, wb,
-                                                                 after, before,
-                                                                 baseline_after, baseline_before,
-                                                                 points, method, group, options, shifts,
-                                                                 examined_dimensions,
-                                                                 ended_usec - started_usec, &stats);
+            added_dimensions =
+                    registered_results_to_json_charts(
+                            results, wb,
+                            after, before,
+                            baseline_after, baseline_before,
+                            points, method, group, options, shifts,
+                            examined_dimensions,
+                            ended_usec - started_usec, &stats);
             break;
 
         default:
         case WEIGHTS_FORMAT_CONTEXTS:
-            added_dimensions = registered_results_to_json_contexts(results, wb,
-                                                                   after, before,
-                                                                   baseline_after, baseline_before,
-                                                                   points, method, group, options, shifts,
-                                                                   examined_dimensions,
-                                                                   ended_usec - started_usec, &stats);
+            added_dimensions =
+                    registered_results_to_json_contexts(
+                            results, wb,
+                            after, before,
+                            baseline_after, baseline_before,
+                            points, method, group, options, shifts,
+                            examined_dimensions,
+                            ended_usec - started_usec, &stats);
             break;
     }
 
@@ -1110,7 +997,7 @@ int web_api_v1_weights(RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS
     }
 
 cleanup:
-    if(charts) dictionary_destroy(charts);
+    if(metrics) dictionary_destroy(metrics);
     if(results) register_result_destroy(results);
 
     if(error) {
diff --git a/web/api/queries/weights.h b/web/api/queries/weights.h
index f88a134f2..50d8634ef 100644
--- a/web/api/queries/weights.h
+++ b/web/api/queries/weights.h
@@ -20,14 +20,14 @@ extern int enable_metric_correlations;
 extern int metric_correlations_version;
 extern WEIGHTS_METHOD default_metric_correlations_method;
 
-extern int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
+int web_api_v1_weights (RRDHOST *host, BUFFER *wb, WEIGHTS_METHOD method, WEIGHTS_FORMAT format,
                                RRDR_GROUPING group, const char *group_options,
-                               long long baseline_after, long long baseline_before,
-                               long long after, long long before,
-                               long long points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, int tier, int timeout);
+                               time_t baseline_after, time_t baseline_before,
+                               time_t after, time_t before,
+                               size_t points, RRDR_OPTIONS options, SIMPLE_PATTERN *contexts, size_t tier, size_t timeout);
 
-extern WEIGHTS_METHOD weights_string_to_method(const char *method);
-extern const char *weights_method_to_string(WEIGHTS_METHOD method);
-extern int mc_unittest(void);
+WEIGHTS_METHOD weights_string_to_method(const char *method);
+const char *weights_method_to_string(WEIGHTS_METHOD method);
+int mc_unittest(void);
 
 #endif //NETDATA_API_WEIGHTS_H
diff --git a/web/api/tests/valid_urls.c b/web/api/tests/valid_urls.c
index 91cd19b09..8a2a87f10 100644
--- a/web/api/tests/valid_urls.c
+++ b/web/api/tests/valid_urls.c
@@ -19,19 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
     return NULL;
 }
 
-
-RRDHOST *sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
 void repr(char *result, int result_size, char const *buf, int size)
 {
     int n;
diff --git a/web/api/tests/web_api.c b/web/api/tests/web_api.c
index fd9a86ef6..93e6454ee 100644
--- a/web/api/tests/web_api.c
+++ b/web/api/tests/web_api.c
@@ -19,18 +19,6 @@ void *__wrap_free_temporary_host(RRDHOST *host)
     return NULL;
 }
 
-RRDHOST *sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
-RRDHOST *__wrap_sql_create_host_by_uuid(char *hostname)
-{
-    (void) hostname;
-    return NULL;
-}
-
 void repr(char *result, int result_size, char const *buf, int size)
 {
     int n;
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index 8bfc617fd..93f501f9e 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -34,20 +34,20 @@ static struct {
         , {"match-ids"         , 0    , RRDR_OPTION_MATCH_IDS}
         , {"match_names"       , 0    , RRDR_OPTION_MATCH_NAMES}
         , {"match-names"       , 0    , RRDR_OPTION_MATCH_NAMES}
-        , {"showcustomvars"    , 0    , RRDR_OPTION_CUSTOM_VARS}
         , {"anomaly-bit"       , 0    , RRDR_OPTION_ANOMALY_BIT}
         , {"selected-tier"     , 0    , RRDR_OPTION_SELECTED_TIER}
         , {"raw"               , 0    , RRDR_OPTION_RETURN_RAW}
         , {"jw-anomaly-rates"  , 0    , RRDR_OPTION_RETURN_JWAR}
         , {"natural-points"    , 0    , RRDR_OPTION_NATURAL_POINTS}
         , {"virtual-points"    , 0    , RRDR_OPTION_VIRTUAL_POINTS}
+        , {"all-dimensions"    , 0    , RRDR_OPTION_ALL_DIMENSIONS}
         , {NULL                , 0    , 0}
 };
 
 static struct {
     const char *name;
     uint32_t hash;
-    uint32_t value;
+    DATASOURCE_FORMAT value;
 } api_v1_data_formats[] = {
         {  DATASOURCE_FORMAT_DATATABLE_JSON , 0 , DATASOURCE_DATATABLE_JSON}
         , {DATASOURCE_FORMAT_DATATABLE_JSONP, 0 , DATASOURCE_DATATABLE_JSONP}
@@ -68,7 +68,7 @@ static struct {
 static struct {
     const char *name;
     uint32_t hash;
-    uint32_t value;
+    DATASOURCE_FORMAT value;
 } api_v1_data_google_formats[] = {
         // this is not error - when google requests json, it expects javascript
         // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source#responseformat
@@ -185,20 +185,46 @@ inline RRDR_OPTIONS web_client_api_request_v1_data_options(char *o) {
     return ret;
 }
 
-void web_client_api_request_v1_data_options_to_string(BUFFER *wb, RRDR_OPTIONS options) {
+void web_client_api_request_v1_data_options_to_buffer(BUFFER *wb, RRDR_OPTIONS options) {
     RRDR_OPTIONS used = 0; // to prevent adding duplicates
     int added = 0;
     for(int i = 0; api_v1_data_options[i].name ; i++) {
         if (unlikely((api_v1_data_options[i].value & options) && !(api_v1_data_options[i].value & used))) {
+            const char *name = api_v1_data_options[i].name;
+            used |= api_v1_data_options[i].value;
+
             if(added) buffer_strcat(wb, ",");
-            buffer_strcat(wb, api_v1_data_options[i].name);
+            buffer_strcat(wb, name);
+
+            added++;
+        }
+    }
+}
+
+void web_client_api_request_v1_data_options_to_string(char *buf, size_t size, RRDR_OPTIONS options) {
+    char *write = buf;
+    char *end = &buf[size - 1];
+
+    RRDR_OPTIONS used = 0; // to prevent adding duplicates
+    int added = 0;
+    for(int i = 0; api_v1_data_options[i].name ; i++) {
+        if (unlikely((api_v1_data_options[i].value & options) && !(api_v1_data_options[i].value & used))) {
+            const char *name = api_v1_data_options[i].name;
             used |= api_v1_data_options[i].value;
+
+            if(added && write < end)
+                *write++ = ',';
+
+            while(*name && write < end)
+                *write++ = *name++;
+
             added++;
         }
     }
+    *write = *end = '\0';
 }
 
-inline uint32_t web_client_api_request_v1_data_format(char *name) {
+inline DATASOURCE_FORMAT web_client_api_request_v1_data_format(char *name) {
     uint32_t hash = simple_hash(name);
     int i;
 
@@ -584,16 +610,14 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
     char *group_time_str = NULL;
     char *points_str = NULL;
     char *timeout_str = NULL;
-    char *max_anomaly_rates_str = NULL;
     char *context = NULL;
     char *chart_label_key = NULL;
     char *chart_labels_filter = NULL;
     char *group_options = NULL;
-    int tier = 0;
-    int group = RRDR_GROUPING_AVERAGE;
-    int show_dimensions = 0;
-    uint32_t format = DATASOURCE_JSON;
-    uint32_t options = 0x00000000;
+    size_t tier = 0;
+    RRDR_GROUPING group = RRDR_GROUPING_AVERAGE;
+    DATASOURCE_FORMAT format = DATASOURCE_JSON;
+    RRDR_OPTIONS options = 0;
 
     while(url) {
         char *value = mystrsep(&url, "&");
@@ -617,7 +641,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
             buffer_strcat(dimensions, "|");
             buffer_strcat(dimensions, value);
         }
-        else if(!strcmp(name, "show_dimensions")) show_dimensions = 1;
+        else if(!strcmp(name, "show_dimensions")) options |= RRDR_OPTION_ALL_DIMENSIONS;
         else if(!strcmp(name, "after")) after_str = value;
         else if(!strcmp(name, "before")) before_str = value;
         else if(!strcmp(name, "points")) points_str = value;
@@ -670,13 +694,12 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
                     outFileName = tqx_value;
             }
         }
-        else if(!strcmp(name, "max_anomaly_rates")) {
-            max_anomaly_rates_str = value;
-        }
         else if(!strcmp(name, "tier")) {
-            tier = str2i(value);
-            if(tier >= 0 && tier < storage_tiers)
+            tier = str2ul(value);
+            if(tier < storage_tiers)
                 options |= RRDR_OPTION_SELECTED_TIER;
+            else
+                tier = 0;
         }
     }
 
@@ -690,81 +713,17 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
 
     RRDSET *st = NULL;
     ONEWAYALLOC *owa = onewayalloc_create(0);
+    QUERY_TARGET *qt = NULL;
 
-    if((!chart || !*chart) && (!context)) {
-        buffer_sprintf(w->response.data, "No chart id is given at the request.");
+    if(!is_valid_sp(chart) && !is_valid_sp(context)) {
+        buffer_sprintf(w->response.data, "No chart or context is given.");
         goto cleanup;
     }
 
-    struct context_param  *context_param_list = NULL;
-
-    if (context && !chart) {
-        RRDSET *st1;
-
-        uint32_t context_hash = simple_hash(context);
-
-        SIMPLE_PATTERN *chart_label_key_pattern = NULL;
-        if(chart_label_key)
-            chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
-
-        SIMPLE_PATTERN *chart_labels_filter_pattern = NULL;
-        if(chart_labels_filter)
-            chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
-
-        rrdhost_rdlock(host);
-        rrdset_foreach_read(st1, host) {
-            if (st1->hash_context == context_hash && !strcmp(st1->context, context) &&
-                (!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_label_key_pattern, ':')) &&
-                (!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_labels_filter_pattern, ':')))
-                    build_context_param_list(owa, &context_param_list, st1);
-        }
-        rrdhost_unlock(host);
-
-        if (likely(context_param_list && context_param_list->rd))  // Just set the first one
-            st = context_param_list->rd->rrdset;
-        else {
-            if (!chart_label_key && !chart_labels_filter)
-                sql_build_context_param_list(owa, &context_param_list, host, context, NULL);
-        }
-    }
-    else {
+    if(chart && !context) {
+        // check if this is a specific chart
         st = rrdset_find(host, chart);
-        if (!st)
-            st = rrdset_find_byname(host, chart);
-        if (likely(st))
-            st->last_accessed_time = now_realtime_sec();
-        else
-            sql_build_context_param_list(owa, &context_param_list, host, NULL, chart);
-    }
-
-    if (!st) {
-        if (likely(context_param_list && context_param_list->rd && context_param_list->rd->rrdset))
-            st = context_param_list->rd->rrdset;
-        else {
-            free_context_param_list(owa, &context_param_list);
-            context_param_list = NULL;
-        }
-    }
-
-    if (!st && !context_param_list) {
-        if (context && !chart) {
-            if (!chart_label_key) {
-                buffer_strcat(w->response.data, "Context is not found: ");
-                buffer_strcat_htmlescape(w->response.data, context);
-            } else {
-                buffer_strcat(w->response.data, "Context: ");
-                buffer_strcat_htmlescape(w->response.data, context);
-                buffer_strcat(w->response.data, " or chart label key: ");
-                buffer_strcat_htmlescape(w->response.data, chart_label_key);
-                buffer_strcat(w->response.data, " not found");
-            }
-        }
-        else {
-            buffer_strcat(w->response.data, "Chart is not found: ");
-            buffer_strcat_htmlescape(w->response.data, chart);
-        }
-        ret = HTTP_RESP_NOT_FOUND;
-        goto cleanup;
+        if (!st) st = rrdset_find_byname(host, chart);
     }
 
     long long before = (before_str && *before_str)?str2l(before_str):0;
@@ -772,7 +731,35 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
     int       points = (points_str && *points_str)?str2i(points_str):0;
     int       timeout = (timeout_str && *timeout_str)?str2i(timeout_str): 0;
     long      group_time = (group_time_str && *group_time_str)?str2l(group_time_str):0;
-    int       max_anomaly_rates = (max_anomaly_rates_str && *max_anomaly_rates_str) ? str2i(max_anomaly_rates_str) : 0;
+
+    QUERY_TARGET_REQUEST qtr = {
+            .after = after,
+            .before = before,
+            .host = host,
+            .st = st,
+            .hosts = NULL,
+            .contexts = context,
+            .charts = chart,
+            .dimensions = (dimensions)?buffer_tostring(dimensions):NULL,
+            .timeout = timeout,
+            .points = points,
+            .format = format,
+            .options = options,
+            .group_method = group,
+            .group_options = group_options,
+            .resampling_time = group_time,
+            .tier = tier,
+            .chart_label_key = chart_label_key,
+            .charts_labels_filter = chart_labels_filter,
+            .query_source = QUERY_SOURCE_API_DATA,
+    };
+    qt = query_target_create(&qtr);
+
+    if(!qt || !qt->query.used) {
+        buffer_sprintf(w->response.data, "No metrics where matched to query.");
+        ret = HTTP_RESP_NOT_FOUND;
+        goto cleanup;
+    }
 
     if (timeout) {
         struct timeval now;
@@ -782,21 +769,13 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         if (timeout <= 0) {
             buffer_flush(w->response.data);
             buffer_strcat(w->response.data, "Query timeout exceeded");
-            return HTTP_RESP_BACKEND_FETCH_FAILED;
+            ret = HTTP_RESP_BACKEND_FETCH_FAILED;
+            goto cleanup;
         }
     }
 
-    debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%d', format '%u', options '0x%08x'"
-          , w->id
-          , chart
-          , (dimensions)?buffer_tostring(dimensions):""
-          , after
-          , before
-          , points
-          , group
-          , format
-          , options
-    );
+    debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%u', format '%u', options '0x%08x'"
+          , w->id, chart, (dimensions)?buffer_tostring(dimensions):"", after, before , points, group, format, options);
 
     if(outFileName && *outFileName) {
         buffer_sprintf(w->response.header, "Content-Disposition: attachment; filename=\"%s\"\r\n", outFileName);
@@ -827,18 +806,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         buffer_strcat(w->response.data, "(");
     }
 
-    QUERY_PARAMS query_params = {
-        .context_param_list = context_param_list,
-        .timeout = timeout,
-        .max_anomaly_rates = max_anomaly_rates,
-        .show_dimensions = show_dimensions,
-        .chart_label_key = chart_label_key,
-        .wb = w->response.data};
-
-    ret = rrdset2anything_api_v1(owa, st, &query_params, dimensions, format,
-            points, after, before, group, group_options, group_time, options, &last_timestamp_in_data, tier);
-
-    free_context_param_list(owa, &context_param_list);
+    ret = data_query_execute(owa, w->response.data, qt, &last_timestamp_in_data);
 
     if(format == DATASOURCE_DATATABLE_JSONP) {
         if(google_timestamp < last_timestamp_in_data)
@@ -856,6 +824,10 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         buffer_strcat(w->response.data, ");");
 
 cleanup:
+    if(qt && qt->used) {
+        internal_error(true, "QUERY_TARGET: left non-released on query '%s'", qt->id);
+        query_target_release(qt);
+    }
     onewayalloc_destroy(owa);
     buffer_free(dimensions);
     return ret;
@@ -1054,8 +1026,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
 static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST *host, BUFFER *wb) {
     int alarm_normal = 0, alarm_warn = 0, alarm_crit = 0;
     RRDCALC *rc;
-    rrdhost_rdlock(host);
-    for(rc = host->alarms; rc ; rc = rc->next) {
+    foreach_rrdcalc_in_rrdhost_read(host, rc) {
         if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
             continue;
 
@@ -1070,7 +1041,7 @@ static inline void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST
                 alarm_normal++;
         }
     }
-    rrdhost_unlock(host);
+    foreach_rrdcalc_in_rrdhost_done(rc);
     buffer_sprintf(wb, "\t\t\"normal\": %d,\n", alarm_normal);
     buffer_sprintf(wb, "\t\t\"warning\": %d,\n", alarm_warn);
     buffer_sprintf(wb, "\t\t\"critical\": %d\n", alarm_crit);
@@ -1086,7 +1057,7 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {
         if (count > 0)
             buffer_strcat(wb, ",\n");
 
-        buffer_sprintf(wb, "\t\t\"%s\"", host->hostname);
+        buffer_sprintf(wb, "\t\t\"%s\"", rrdhost_hostname(host));
         count++;
     }
 
@@ -1101,7 +1072,7 @@ static inline void web_client_api_request_v1_info_mirrored_hosts(BUFFER *wb) {
         buffer_sprintf(
             wb, "\t\t{ \"guid\": \"%s\", \"hostname\": \"%s\", \"reachable\": %s, \"hops\": %d"
             , host->machine_guid
-            , host->hostname
+            , rrdhost_hostname(host)
             , (host->receiver || host == localhost) ? "true" : "false"
             , host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1
             );
@@ -1140,7 +1111,7 @@ inline void host_labels2json(RRDHOST *host, BUFFER *wb, size_t indentation) {
         indentation--;
     }
 
-    rrdlabels_to_buffer(host->host_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
+    rrdlabels_to_buffer(host->rrdlabels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
     buffer_strcat(wb, "\n");
 }
 
@@ -1148,7 +1119,7 @@ extern int aclk_connected;
 inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
 {
     buffer_strcat(wb, "{\n");
-    buffer_sprintf(wb, "\t\"version\": \"%s\",\n", host->program_version);
+    buffer_sprintf(wb, "\t\"version\": \"%s\",\n", rrdhost_program_version(host));
     buffer_sprintf(wb, "\t\"uid\": \"%s\",\n", host->machine_guid);
 
     web_client_api_request_v1_info_mirrored_hosts(wb);
@@ -1202,6 +1173,10 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
     host_labels2json(host, wb, 2);
     buffer_strcat(wb, "\t},\n");
 
+    buffer_strcat(wb, "\t\"functions\": {\n");
+    host_functions2json(host, wb, 2, "\"", "\"");
+    buffer_strcat(wb, "\t},\n");
+
     buffer_strcat(wb, "\t\"collectors\": [");
     chartcollectors2json(host, wb);
     buffer_strcat(wb, "\n\t],\n");
@@ -1215,14 +1190,8 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
 
 #ifdef ENABLE_ACLK
     buffer_strcat(wb, "\t\"cloud-available\": true,\n");
-    buffer_strcat(wb, "\t\"aclk-ng-available\": true,\n");
-    buffer_strcat(wb, "\t\"aclk-ng-new-cloud-protocol\": true,\n");
-    buffer_strcat(wb, "\t\"aclk-legacy-available\": false,\n");
-    buffer_strcat(wb, "\t\"aclk-implementation\": \"Next Generation\",\n");
 #else
     buffer_strcat(wb, "\t\"cloud-available\": false,\n");
-    buffer_strcat(wb, "\t\"aclk-ng-available\": false,\n");
-    buffer_strcat(wb, "\t\"aclk-legacy-available\": false,\n");
 #endif
     char *agent_id = get_agent_claimid();
     if (agent_id == NULL)
@@ -1234,11 +1203,10 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
 #ifdef ENABLE_ACLK
     if (aclk_connected) {
         buffer_strcat(wb, "\t\"aclk-available\": true,\n");
-        buffer_strcat(wb, "\t\"aclk-available-protocol\": \"New\",\n");
     }
     else
 #endif
-        buffer_strcat(wb, "\t\"aclk-available\": false,\n\t\"aclk-available-protocol\": null,\n");     // Intentionally valid with/without #ifdef above
+        buffer_strcat(wb, "\t\"aclk-available\": false,\n");     // Intentionally valid with/without #ifdef above
 
     buffer_strcat(wb, "\t\"memory-mode\": ");
     analytics_get_data(analytics_data.netdata_config_memory_mode, wb);
@@ -1259,7 +1227,7 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
 #ifdef  ENABLE_COMPRESSION
     if(host->sender){
         buffer_strcat(wb, "\t\"stream-compression\": ");
-        buffer_strcat(wb, (host->sender->rrdpush_compression ? "true" : "false"));
+        buffer_strcat(wb, stream_has_capability(host->sender, STREAM_CAP_COMPRESSION) ? "true" : "false");
         buffer_strcat(wb, ",\n");
     }else{
         buffer_strcat(wb, "\t\"stream-compression\": null,\n");
@@ -1330,7 +1298,7 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
     buffer_strcat(wb, "\t\"ml-info\": ");
     buffer_strcat(wb, ml_info);
 
-    free(ml_info);
+    freez(ml_info);
 #endif
 
     buffer_strcat(wb, "\n}");
@@ -1338,81 +1306,15 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
 }
 
 #if defined(ENABLE_ML)
-int web_client_api_request_v1_anomaly_events(RRDHOST *host, struct web_client *w, char *url) {
-    if (!netdata_ready)
-        return HTTP_RESP_BACKEND_FETCH_FAILED;
-
-    uint32_t after = 0, before = 0;
-
-    while (url) {
-        char *value = mystrsep(&url, "&");
-        if (!value || !*value)
-            continue;
-
-        char *name = mystrsep(&value, "=");
-        if (!name || !*name)
-            continue;
-        if (!value || !*value)
-            continue;
-
-        if (!strcmp(name, "after"))
-            after = (uint32_t) (strtoul(value, NULL, 0) / 1000);
-        else if (!strcmp(name, "before"))
-            before = (uint32_t) (strtoul(value, NULL, 0) / 1000);
-    }
-
-    char *s;
-    if (!before || !after)
-        s = strdupz("{\"error\": \"missing after/before parameters\" }\n");
-    else {
-        s = ml_get_anomaly_events(host, "AD1", 1, after, before);
-        if (!s)
-            s = strdupz("{\"error\": \"json string is empty\" }\n");
-    }
-
-    BUFFER *wb = w->response.data;
-    buffer_flush(wb);
-
-    wb->contenttype = CT_APPLICATION_JSON;
-    buffer_strcat(wb, s);
-    buffer_no_cacheable(wb);
-
-    freez(s);
-
-    return HTTP_RESP_OK;
-}
+int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char *url) {
+    (void) url;
 
-int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_client *w, char *url) {
     if (!netdata_ready)
         return HTTP_RESP_BACKEND_FETCH_FAILED;
 
-    uint32_t after = 0, before = 0;
-
-    while (url) {
-        char *value = mystrsep(&url, "&");
-        if (!value || !*value)
-            continue;
-
-        char *name = mystrsep(&value, "=");
-        if (!name || !*name)
-            continue;
-        if (!value || !*value)
-            continue;
-
-        if (!strcmp(name, "after"))
-            after = (uint32_t) strtoul(value, NULL, 0);
-        else if (!strcmp(name, "before"))
-            before = (uint32_t) strtoul(value, NULL, 0);
-    }
-
-    char *s;
-    if (!before || !after)
-        s = strdupz("{\"error\": \"missing after/before parameters\" }\n");
-    else {
-        s = ml_get_anomaly_event_info(host, "AD1", 1, after, before);
-        if (!s)
-            s = strdupz("{\"error\": \"json string is empty\" }\n");
-    }
+    char *s = ml_get_host_runtime_info(host);
+    if (!s)
+        s = strdupz("{\"error\": \"json string is empty\" }\n");
 
     BUFFER *wb = w->response.data;
     buffer_flush(wb);
@@ -1424,13 +1326,13 @@ int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_clien
     return HTTP_RESP_OK;
 }
 
-int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char *url) {
+int web_client_api_request_v1_ml_models(RRDHOST *host, struct web_client *w, char *url) {
     (void) url;
 
     if (!netdata_ready)
         return HTTP_RESP_BACKEND_FETCH_FAILED;
 
-    char *s = ml_get_host_runtime_info(host);
+    char *s = ml_get_host_models(host);
     if (!s)
         s = strdupz("{\"error\": \"json string is empty\" }\n");
 
@@ -1443,8 +1345,7 @@ int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char
     freez(s);
     return HTTP_RESP_OK;
 }
-
-#endif // defined(ENABLE_ML)
+#endif
 
 inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url) {
     (void)url;
@@ -1485,7 +1386,7 @@ static int web_client_api_request_v1_weights_internal(RRDHOST *host, struct web_
     int options_count = 0;
     RRDR_GROUPING group = RRDR_GROUPING_AVERAGE;
     int timeout = 0;
-    int tier = 0;
+    size_t tier = 0;
     const char *group_options = NULL, *contexts_str = NULL;
 
     while (url) {
@@ -1533,9 +1434,11 @@ static int web_client_api_request_v1_weights_internal(RRDHOST *host, struct web_
             contexts_str = value;
 
         else if(!strcmp(name, "tier")) {
-            tier = str2i(value);
-            if(tier >= 0 && tier < storage_tiers)
+            tier = str2ul(value);
+            if(tier < storage_tiers)
                 options |= RRDR_OPTION_SELECTED_TIER;
+            else
+                tier = 0;
         }
     }
 
@@ -1559,12 +1462,59 @@ int web_client_api_request_v1_weights(RRDHOST *host, struct web_client *w, char
     return web_client_api_request_v1_weights_internal(host, w, url, WEIGHTS_METHOD_ANOMALY_RATE, WEIGHTS_FORMAT_CONTEXTS);
 }
 
+int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char *url) {
+    if (!netdata_ready)
+        return HTTP_RESP_BACKEND_FETCH_FAILED;
+
+    int timeout = 0;
+    const char *function = NULL;
+
+    while (url) {
+        char *value = mystrsep(&url, "&");
+        if (!value || !*value)
+            continue;
+
+        char *name = mystrsep(&value, "=");
+        if (!name || !*name)
+            continue;
+
+        if (!strcmp(name, "function"))
+            function = value;
+
+        else if (!strcmp(name, "timeout"))
+            timeout = (int) strtoul(value, NULL, 0);
+    }
+
+    BUFFER *wb = w->response.data;
+    buffer_flush(wb);
+    wb->contenttype = CT_APPLICATION_JSON;
+    buffer_no_cacheable(wb);
+
+    return rrd_call_function_and_wait(host, wb, timeout, function);
+}
+
+int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, char *url __maybe_unused) {
+    if (!netdata_ready)
+        return HTTP_RESP_BACKEND_FETCH_FAILED;
+
+    BUFFER *wb = w->response.data;
+    buffer_flush(wb);
+    wb->contenttype = CT_APPLICATION_JSON;
+    buffer_no_cacheable(wb);
+
+    buffer_strcat(wb, "{\n");
+    host_functions2json(host, wb, 1, "\"", "\"");
+    buffer_strcat(wb, "}");
+
+    return HTTP_RESP_OK;
+}
+
 #ifndef ENABLE_DBENGINE
 int web_client_api_request_v1_dbengine_stats(RRDHOST *host, struct web_client *w, char *url) {
     return HTTP_RESP_NOT_FOUND;
 }
 #else
-static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, int tier) {
+static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, size_t tier) {
     RRDENG_SIZE_STATS stats = rrdeng_size_statistics(multidb_ctx[tier]);
 
     buffer_sprintf(wb,
@@ -1588,11 +1538,11 @@ static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, int tier) {
                    ",\n\t\t\"metrics_pages\":%zu"
                    ",\n\t\t\"extents_compressed_bytes\":%zu"
                    ",\n\t\t\"pages_uncompressed_bytes\":%zu"
-                   ",\n\t\t\"pages_duration_secs\":%ld"
+                   ",\n\t\t\"pages_duration_secs\":%lld"
                    ",\n\t\t\"single_point_pages\":%zu"
                    ",\n\t\t\"first_t\":%llu"
                    ",\n\t\t\"last_t\":%llu"
-                   ",\n\t\t\"database_retention_secs\":%ld"
+                   ",\n\t\t\"database_retention_secs\":%lld"
                    ",\n\t\t\"average_compression_savings\":%0.2f"
                    ",\n\t\t\"average_point_duration_secs\":%0.2f"
                    ",\n\t\t\"average_metric_retention_secs\":%0.2f"
@@ -1623,11 +1573,11 @@ static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, int tier) {
                    , stats.metrics_pages
                    , stats.extents_compressed_bytes
                    , stats.pages_uncompressed_bytes
-                   , stats.pages_duration_secs
+                   , (long long)stats.pages_duration_secs
                    , stats.single_point_pages
                    , stats.first_t
                    , stats.last_t
-                   , stats.database_retention_secs
+                   , (long long)stats.database_retention_secs
                    , stats.average_compression_savings
                    , stats.average_point_duration_secs
                    , stats.average_metric_retention_secs
@@ -1646,12 +1596,17 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
 
     BUFFER *wb = w->response.data;
     buffer_flush(wb);
+
+    if(!dbengine_enabled) {
+        buffer_strcat(wb, "dbengine is not enabled");
+        return HTTP_RESP_NOT_FOUND;
+    }
+
     wb->contenttype = CT_APPLICATION_JSON;
     buffer_no_cacheable(wb);
-
     buffer_strcat(wb, "{");
-    for(int tier = 0; tier < storage_tiers ;tier++) {
-        buffer_sprintf(wb, "%s\n\t\"tier%d\": {", tier?",":"", tier);
+    for(size_t tier = 0; tier < storage_tiers ;tier++) {
+        buffer_sprintf(wb, "%s\n\t\"tier%zu\": {", tier?",":"", tier);
         web_client_api_v1_dbengine_stats_for_tier(wb, tier);
         buffer_strcat(wb, "\n\t}");
     }
@@ -1661,48 +1616,56 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
 }
 #endif
 
+#ifdef NETDATA_DEV_MODE
+#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_DASHBOARD
+#else
+#define ACL_DEV_OPEN_ACCESS 0
+#endif
+
 static struct api_command {
     const char *command;
     uint32_t hash;
     WEB_CLIENT_ACL acl;
     int (*callback)(RRDHOST *host, struct web_client *w, char *url);
 } api_commands[] = {
-        { "info",            0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_info            },
-        { "data",            0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_data            },
-        { "chart",           0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_chart           },
-        { "charts",          0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_charts          },
-        { "context",         0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_context         },
-        { "contexts",        0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_contexts        },
-        { "archivedcharts",  0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_archivedcharts  },
+        { "info",            0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_info                       },
+        { "data",            0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_data                       },
+        { "chart",           0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_chart                      },
+        { "charts",          0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_charts                     },
+        { "context",         0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_context                    },
+        { "contexts",        0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_contexts                   },
+        { "archivedcharts",  0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_archivedcharts             },
 
         // registry checks the ACL by itself, so we allow everything
-        { "registry",        0, WEB_CLIENT_ACL_NOCHECK,   web_client_api_request_v1_registry        },
+        { "registry",        0, WEB_CLIENT_ACL_NOCHECK,                         web_client_api_request_v1_registry                   },
 
         // badges can be fetched with both dashboard and badge permissions
-        { "badge.svg",       0, WEB_CLIENT_ACL_DASHBOARD|WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge },
+        { "badge.svg",       0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_badge },
 
-        { "alarms",          0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarms          },
-        { "alarms_values",   0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarms_values   },
-        { "alarm_log",       0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarm_log       },
-        { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarm_variables },
-        { "alarm_count",     0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_alarm_count     },
-        { "allmetrics",      0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_allmetrics      },
+        { "alarms",          0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarms                     },
+        { "alarms_values",   0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarms_values             },
+        { "alarm_log",       0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarm_log                 },
+        { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarm_variables           },
+        { "alarm_count",     0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarm_count               },
+        { "allmetrics",      0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_allmetrics                },
 
 #if defined(ENABLE_ML)
-        { "anomaly_events",     0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_events     },
-        { "anomaly_event_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_event_info },
-        { "ml_info",            0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info            },
+      { "ml_info",         0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_ml_info            },
+      { "ml_models",       0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models          },
 #endif
 
-        { "manage/health",       0, WEB_CLIENT_ACL_MGMT,      web_client_api_request_v1_mgmt_health         },
-        { "aclk",                0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_aclk_state          },
-        { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_metric_correlations },
-        { "weights",             0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_weights },
+        { "manage/health",       0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK,      web_client_api_request_v1_mgmt_health           },
+        { "aclk",                0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_aclk_state            },
+        { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_metric_correlations   },
+        { "weights",             0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_weights               },
+
+        { "function",            0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function },
+        { "functions",            0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions },
 
-        { "dbengine_stats",      0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_dbengine_stats },
+        { "dbengine_stats",      0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_dbengine_stats },
 
         // terminator
-        { NULL,              0, WEB_CLIENT_ACL_NONE,      NULL                                      },
+        { NULL,              0, WEB_CLIENT_ACL_NONE,      NULL },
 };
 
 inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url) {
diff --git a/web/api/web_api_v1.h b/web/api/web_api_v1.h
index 544f1e574..e6682c99c 100644
--- a/web/api/web_api_v1.h
+++ b/web/api/web_api_v1.h
@@ -10,30 +10,31 @@
 #include "web/api/queries/weights.h"
 
 #define MAX_CHART_LABELS_FILTER (32)
-extern RRDR_OPTIONS web_client_api_request_v1_data_options(char *o);
-extern void web_client_api_request_v1_data_options_to_string(BUFFER *wb, RRDR_OPTIONS options);
-
-extern uint32_t web_client_api_request_v1_data_format(char *name);
-extern uint32_t web_client_api_request_v1_data_google_format(char *name);
-
-extern int web_client_api_request_v1_alarms(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_alarms_values(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_alarm_log(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_single_chart(RRDHOST *host, struct web_client *w, char *url, void callback(RRDSET *st, BUFFER *buf));
-extern int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url);
-extern int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb);
-extern void host_labels2json(RRDHOST *host, BUFFER *wb, size_t indentation);
-
-extern void web_client_api_v1_init(void);
-extern void web_client_api_v1_management_init(void);
+RRDR_OPTIONS web_client_api_request_v1_data_options(char *o);
+void web_client_api_request_v1_data_options_to_buffer(BUFFER *wb, RRDR_OPTIONS options);
+void web_client_api_request_v1_data_options_to_string(char *buf, size_t size, RRDR_OPTIONS options);
+
+uint32_t web_client_api_request_v1_data_format(char *name);
+uint32_t web_client_api_request_v1_data_google_format(char *name);
+
+int web_client_api_request_v1_alarms(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_alarms_values(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_alarm_log(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_single_chart(RRDHOST *host, struct web_client *w, char *url, void callback(RRDSET *st, BUFFER *buf));
+int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url);
+int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb);
+void host_labels2json(RRDHOST *host, BUFFER *wb, size_t indentation);
+
+void web_client_api_v1_init(void);
+void web_client_api_v1_management_init(void);
 
 extern char *api_secret;