Merging upstream version 2.0.3+dfsg (Closes: #923993, #1042533, #1045145).

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1033 insertions, 0 deletions

diff --git a/src/database/contexts/api_v2_contexts.c b/src/database/contexts/api_v2_contexts.c
new file mode 100644
index 000000000..d8d945afb
--- /dev/null
+++ b/src/database/contexts/api_v2_contexts.c
@@ -0,0 +1,1033 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "api_v2_contexts.h"
+
+#include "aclk/aclk_capas.h"
+
+// ----------------------------------------------------------------------------
+// /api/v2/contexts API
+
+static const char *fts_match_to_string(FTS_MATCH match) {
+    switch(match) {
+        case FTS_MATCHED_HOST:
+            return "HOST";
+
+        case FTS_MATCHED_CONTEXT:
+            return "CONTEXT";
+
+        case FTS_MATCHED_INSTANCE:
+            return "INSTANCE";
+
+        case FTS_MATCHED_DIMENSION:
+            return "DIMENSION";
+
+        case FTS_MATCHED_ALERT:
+            return "ALERT";
+
+        case FTS_MATCHED_ALERT_INFO:
+            return "ALERT_INFO";
+
+        case FTS_MATCHED_LABEL:
+            return "LABEL";
+
+        case FTS_MATCHED_FAMILY:
+            return "FAMILY";
+
+        case FTS_MATCHED_TITLE:
+            return "TITLE";
+
+        case FTS_MATCHED_UNITS:
+            return "UNITS";
+
+        default:
+            return "NONE";
+    }
+}
+
+struct function_v2_entry {
+    size_t size;
+    size_t used;
+    size_t *node_ids;
+    STRING *help;
+    STRING *tags;
+    HTTP_ACCESS access;
+    int priority;
+    uint32_t version;
+};
+
+struct context_v2_entry {
+    size_t count;
+    STRING *id;
+    STRING *family;
+    uint32_t priority;
+    time_t first_time_s;
+    time_t last_time_s;
+    RRD_FLAGS flags;
+    FTS_MATCH match;
+};
+
+static inline bool full_text_search_string(FTS_INDEX *fts, SIMPLE_PATTERN *q, STRING *ptr) {
+    fts->searches++;
+    fts->string_searches++;
+    return simple_pattern_matches_string(q, ptr);
+}
+
+static inline bool full_text_search_char(FTS_INDEX *fts, SIMPLE_PATTERN *q, char *ptr) {
+    fts->searches++;
+    fts->char_searches++;
+    return simple_pattern_matches(q, ptr);
+}
+
+static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc, SIMPLE_PATTERN *q) {
+    if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->id) ||
+                full_text_search_string(&ctl->q.fts, q, rc->family)))
+        return FTS_MATCHED_CONTEXT;
+
+    if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->title)))
+        return FTS_MATCHED_TITLE;
+
+    if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->units)))
+        return FTS_MATCHED_UNITS;
+
+    FTS_MATCH matched = FTS_MATCHED_NONE;
+    RRDINSTANCE *ri;
+    dfe_start_read(rc->rrdinstances, ri) {
+        if(matched) break;
+
+        if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, ri->first_time_s, (ri->flags & RRD_FLAG_COLLECTED) ? ctl->now : ri->last_time_s, 0))
+            continue;
+
+        if(unlikely(full_text_search_string(&ctl->q.fts, q, ri->id)) ||
+           (ri->name != ri->id && full_text_search_string(&ctl->q.fts, q, ri->name))) {
+            matched = FTS_MATCHED_INSTANCE;
+            break;
+        }
+
+        RRDMETRIC *rm;
+        dfe_start_read(ri->rrdmetrics, rm) {
+            if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rm->first_time_s, (rm->flags & RRD_FLAG_COLLECTED) ? ctl->now : rm->last_time_s, 0))
+                continue;
+
+            if(unlikely(full_text_search_string(&ctl->q.fts, q, rm->id)) ||
+               (rm->name != rm->id && full_text_search_string(&ctl->q.fts, q, rm->name))) {
+                matched = FTS_MATCHED_DIMENSION;
+                break;
+            }
+        }
+        dfe_done(rm);
+
+        size_t label_searches = 0;
+        if(unlikely(ri->rrdlabels && rrdlabels_entries(ri->rrdlabels) &&
+                    rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, q, ':', &label_searches) == SP_MATCHED_POSITIVE)) {
+            ctl->q.fts.searches += label_searches;
+            ctl->q.fts.char_searches += label_searches;
+            matched = FTS_MATCHED_LABEL;
+            break;
+        }
+        ctl->q.fts.searches += label_searches;
+        ctl->q.fts.char_searches += label_searches;
+
+        if(ri->rrdset) {
+            RRDSET *st = ri->rrdset;
+            rw_spinlock_read_lock(&st->alerts.spinlock);
+            for (RRDCALC *rcl = st->alerts.base; rcl; rcl = rcl->next) {
+                if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.name))) {
+                    matched = FTS_MATCHED_ALERT;
+                    break;
+                }
+
+                if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.info))) {
+                    matched = FTS_MATCHED_ALERT_INFO;
+                    break;
+                }
+            }
+            rw_spinlock_read_unlock(&st->alerts.spinlock);
+        }
+    }
+    dfe_done(ri);
+    return matched;
+}
+
+static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context __maybe_unused) {
+    struct rrdcontext_to_json_v2_data *ctl = data;
+
+    RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
+
+    if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rc->first_time_s, (rc->flags & RRD_FLAG_COLLECTED) ? ctl->now : rc->last_time_s, 0))
+        return 0; // continue to next context
+
+    FTS_MATCH match = ctl->q.host_match;
+    if((ctl->mode & CONTEXTS_V2_SEARCH) && ctl->q.pattern) {
+        match = rrdcontext_to_json_v2_full_text_search(ctl, rc, ctl->q.pattern);
+
+        if(match == FTS_MATCHED_NONE)
+            return 0; // continue to next context
+    }
+
+    if(ctl->mode & CONTEXTS_V2_ALERTS) {
+        if(!rrdcontext_matches_alert(ctl, rc))
+            return 0; // continue to next context
+    }
+
+    if(ctl->contexts.dict) {
+        struct context_v2_entry t = {
+                .count = 1,
+                .id = rc->id,
+                .family = string_dup(rc->family),
+                .priority = rc->priority,
+                .first_time_s = rc->first_time_s,
+                .last_time_s = rc->last_time_s,
+                .flags = rc->flags,
+                .match = match,
+        };
+
+        dictionary_set(ctl->contexts.dict, string2str(rc->id), &t, sizeof(struct context_v2_entry));
+    }
+
+    return 1;
+}
+
+void buffer_json_agent_status_id(BUFFER *wb, size_t ai, usec_t duration_ut) {
+    buffer_json_member_add_object(wb, "st");
+    {
+        buffer_json_member_add_uint64(wb, "ai", ai);
+        buffer_json_member_add_uint64(wb, "code", 200);
+        buffer_json_member_add_string(wb, "msg", "");
+        if (duration_ut)
+            buffer_json_member_add_double(wb, "ms", (NETDATA_DOUBLE) duration_ut / 1000.0);
+    }
+    buffer_json_object_close(wb);
+}
+
+void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut, bool status) {
+    buffer_json_member_add_string(wb, "mg", host->machine_guid);
+
+    if(!UUIDiszero(host->node_id))
+        buffer_json_member_add_uuid(wb, "nd", host->node_id.uuid);
+    buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
+    buffer_json_member_add_uint64(wb, "ni", ni);
+
+    if(status)
+        buffer_json_agent_status_id(wb, 0, duration_ut);
+}
+
+static void rrdhost_receiver_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) {
+    buffer_json_member_add_object(wb, key);
+    {
+        buffer_json_member_add_uint64(wb, "id", s->ingest.id);
+        buffer_json_member_add_uint64(wb, "hops", s->ingest.hops);
+        buffer_json_member_add_string(wb, "type", rrdhost_ingest_type_to_string(s->ingest.type));
+        buffer_json_member_add_string(wb, "status", rrdhost_ingest_status_to_string(s->ingest.status));
+        buffer_json_member_add_time_t(wb, "since", s->ingest.since);
+        buffer_json_member_add_time_t(wb, "age", s->now - s->ingest.since);
+
+        if(s->ingest.type == RRDHOST_INGEST_TYPE_CHILD) {
+            if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE)
+                buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->ingest.reason));
+
+            if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING) {
+                buffer_json_member_add_object(wb, "replication");
+                {
+                    buffer_json_member_add_boolean(wb, "in_progress", s->ingest.replication.in_progress);
+                    buffer_json_member_add_double(wb, "completion", s->ingest.replication.completion);
+                    buffer_json_member_add_uint64(wb, "instances", s->ingest.replication.instances);
+                }
+                buffer_json_object_close(wb); // replication
+            }
+
+            if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING || s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE) {
+                buffer_json_member_add_object(wb, "source");
+                {
+                    char buf[1024 + 1];
+                    snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.local.ip, s->ingest.peers.local.port, s->ingest.ssl ? ":SSL" : "");
+                    buffer_json_member_add_string(wb, "local", buf);
+
+                    snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.peer.ip, s->ingest.peers.peer.port, s->ingest.ssl ? ":SSL" : "");
+                    buffer_json_member_add_string(wb, "remote", buf);
+
+                    stream_capabilities_to_json_array(wb, s->ingest.capabilities, "capabilities");
+                }
+                buffer_json_object_close(wb); // source
+            }
+        }
+    }
+    buffer_json_object_close(wb); // collection
+}
+
+static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) {
+    if(s->stream.status == RRDHOST_STREAM_STATUS_DISABLED)
+        return;
+
+    buffer_json_member_add_object(wb, key);
+    {
+        buffer_json_member_add_uint64(wb, "id", s->stream.id);
+        buffer_json_member_add_uint64(wb, "hops", s->stream.hops);
+        buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status));
+        buffer_json_member_add_time_t(wb, "since", s->stream.since);
+        buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since);
+
+        if (s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE)
+            buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->stream.reason));
+
+        if (s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) {
+            buffer_json_member_add_object(wb, "replication");
+            {
+                buffer_json_member_add_boolean(wb, "in_progress", s->stream.replication.in_progress);
+                buffer_json_member_add_double(wb, "completion", s->stream.replication.completion);
+                buffer_json_member_add_uint64(wb, "instances", s->stream.replication.instances);
+            }
+            buffer_json_object_close(wb);
+        }
+
+        buffer_json_member_add_object(wb, "destination");
+        {
+            char buf[1024 + 1];
+            snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.local.ip, s->stream.peers.local.port, s->stream.ssl ? ":SSL" : "");
+            buffer_json_member_add_string(wb, "local", buf);
+
+            snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.peer.ip, s->stream.peers.peer.port, s->stream.ssl ? ":SSL" : "");
+            buffer_json_member_add_string(wb, "remote", buf);
+
+            stream_capabilities_to_json_array(wb, s->stream.capabilities, "capabilities");
+
+            buffer_json_member_add_object(wb, "traffic");
+            {
+                buffer_json_member_add_boolean(wb, "compression", s->stream.compression);
+                buffer_json_member_add_uint64(wb, "data", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA]);
+                buffer_json_member_add_uint64(wb, "metadata", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA]);
+                buffer_json_member_add_uint64(wb, "functions", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS]);
+                buffer_json_member_add_uint64(wb, "replication", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION]);
+                buffer_json_member_add_uint64(wb, "dyncfg", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DYNCFG]);
+            }
+            buffer_json_object_close(wb); // traffic
+
+            buffer_json_member_add_array(wb, "candidates");
+            struct rrdpush_destinations *d;
+            for (d = s->host->destinations; d; d = d->next) {
+                buffer_json_add_array_item_object(wb);
+                buffer_json_member_add_uint64(wb, "attempts", d->attempts);
+                {
+
+                    if (d->ssl) {
+                        snprintfz(buf, sizeof(buf) - 1, "%s:SSL", string2str(d->destination));
+                        buffer_json_member_add_string(wb, "destination", buf);
+                    }
+                    else
+                        buffer_json_member_add_string(wb, "destination", string2str(d->destination));
+
+                    buffer_json_member_add_time_t(wb, "since", d->since);
+                    buffer_json_member_add_time_t(wb, "age", s->now - d->since);
+                    buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->reason));
+                    if(d->postpone_reconnection_until > s->now) {
+                        buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until);
+                        buffer_json_member_add_time_t(wb, "next_in", d->postpone_reconnection_until - s->now);
+                    }
+                }
+                buffer_json_object_close(wb); // each candidate
+            }
+            buffer_json_array_close(wb); // candidates
+        }
+        buffer_json_object_close(wb); // destination
+    }
+    buffer_json_object_close(wb); // streaming
+}
+
+void agent_capabilities_to_json(BUFFER *wb, RRDHOST *host, const char *key) {
+    buffer_json_member_add_array(wb, key);
+
+    struct capability *capas = aclk_get_node_instance_capas(host);
+    for(struct capability *capa = capas; capa->name ;capa++) {
+        buffer_json_add_array_item_object(wb);
+        {
+            buffer_json_member_add_string(wb, "name", capa->name);
+            buffer_json_member_add_uint64(wb, "version", capa->version);
+            buffer_json_member_add_boolean(wb, "enabled", capa->enabled);
+        }
+        buffer_json_object_close(wb);
+    }
+    buffer_json_array_close(wb);
+    freez(capas);
+}
+
+static inline void host_dyncfg_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) {
+    buffer_json_member_add_object(wb, key);
+    {
+        buffer_json_member_add_string(wb, "status", rrdhost_dyncfg_status_to_string(s->dyncfg.status));
+    }
+    buffer_json_object_close(wb); // health
+
+}
+
+static inline void rrdhost_health_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) {
+    buffer_json_member_add_object(wb, key);
+    {
+        buffer_json_member_add_string(wb, "status", rrdhost_health_status_to_string(s->health.status));
+        if (s->health.status == RRDHOST_HEALTH_STATUS_RUNNING) {
+            buffer_json_member_add_object(wb, "alerts");
+            {
+                buffer_json_member_add_uint64(wb, "critical", s->health.alerts.critical);
+                buffer_json_member_add_uint64(wb, "warning", s->health.alerts.warning);
+                buffer_json_member_add_uint64(wb, "clear", s->health.alerts.clear);
+                buffer_json_member_add_uint64(wb, "undefined", s->health.alerts.undefined);
+                buffer_json_member_add_uint64(wb, "uninitialized", s->health.alerts.uninitialized);
+            }
+            buffer_json_object_close(wb); // alerts
+        }
+    }
+    buffer_json_object_close(wb); // health
+}
+
+static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdcontext_to_json_v2_data *ctl, size_t node_id) {
+    buffer_json_add_array_item_object(wb); // this node
+    buffer_json_node_add_v2(wb, host, node_id, 0,
+                            (ctl->mode & CONTEXTS_V2_AGENTS) && !(ctl->mode & CONTEXTS_V2_NODE_INSTANCES));
+
+    if(ctl->mode & (CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES)) {
+        RRDHOST_STATUS s;
+        rrdhost_status(host, ctl->now, &s);
+
+        if (ctl->mode & (CONTEXTS_V2_NODES_INFO)) {
+            buffer_json_member_add_string(wb, "v", rrdhost_program_version(host));
+
+            host_labels2json(host, wb, "labels");
+
+            if (host->system_info) {
+                buffer_json_member_add_object(wb, "hw");
+                {
+                    buffer_json_member_add_string_or_empty(wb, "architecture", host->system_info->architecture);
+                    buffer_json_member_add_string_or_empty(wb, "cpu_frequency", host->system_info->host_cpu_freq);
+                    buffer_json_member_add_string_or_empty(wb, "cpus", host->system_info->host_cores);
+                    buffer_json_member_add_string_or_empty(wb, "memory", host->system_info->host_ram_total);
+                    buffer_json_member_add_string_or_empty(wb, "disk_space", host->system_info->host_disk_space);
+                    buffer_json_member_add_string_or_empty(wb, "virtualization", host->system_info->virtualization);
+                    buffer_json_member_add_string_or_empty(wb, "container", host->system_info->container);
+                }
+                buffer_json_object_close(wb);
+
+                buffer_json_member_add_object(wb, "os");
+                {
+                    buffer_json_member_add_string_or_empty(wb, "id", host->system_info->host_os_id);
+                    buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->host_os_name);
+                    buffer_json_member_add_string_or_empty(wb, "v", host->system_info->host_os_version);
+                    buffer_json_member_add_object(wb, "kernel");
+                    buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->kernel_name);
+                    buffer_json_member_add_string_or_empty(wb, "v", host->system_info->kernel_version);
+                    buffer_json_object_close(wb);
+                }
+                buffer_json_object_close(wb);
+            }
+
+            // created      - the node is created but never connected to cloud
+            // unreachable  - not currently connected
+            // stale        - connected but not having live data
+            // reachable    - connected with live data
+            // pruned       - not connected for some time and has been removed
+            buffer_json_member_add_string(wb, "state", rrdhost_is_online(host) ? "reachable" : "stale");
+
+            rrdhost_health_to_json_v2(wb, "health", &s);
+            agent_capabilities_to_json(wb, host, "capabilities");
+            rrdhost_stream_path_to_json(wb, host, STREAM_PATH_JSON_MEMBER, false);
+        }
+
+        if (ctl->mode & (CONTEXTS_V2_NODE_INSTANCES)) {
+            buffer_json_member_add_array(wb, "instances");
+            buffer_json_add_array_item_object(wb); // this instance
+            {
+                buffer_json_agent_status_id(wb, 0, 0);
+
+                buffer_json_member_add_object(wb, "db");
+                {
+                    buffer_json_member_add_string(wb, "status", rrdhost_db_status_to_string(s.db.status));
+                    buffer_json_member_add_string(wb, "liveness", rrdhost_db_liveness_to_string(s.db.liveness));
+                    buffer_json_member_add_string(wb, "mode", rrd_memory_mode_name(s.db.mode));
+                    buffer_json_member_add_time_t(wb, "first_time", s.db.first_time_s);
+                    buffer_json_member_add_time_t(wb, "last_time", s.db.last_time_s);
+                    buffer_json_member_add_uint64(wb, "metrics", s.db.metrics);
+
+                    spinlock_lock(&s.host->accounting.spinlock);
+                    int64_t count = 0;
+
+                    if (s.host->accounting.cache_timestamp &&
+                        ctl->now - s.host->accounting.cache_timestamp < host->rrd_update_every * 1.5)
+                        count = s.host->accounting.currently_collected;
+                    else {
+                        Pvoid_t *Pvalue;
+                        bool first = true;
+                        Word_t dimension_id = 0;
+                        while ((Pvalue = JudyLFirstThenNext(s.host->accounting.JudyL, &dimension_id, &first))) {
+                            RRDDIM *rd = *Pvalue;
+                            if (rd->collector.last_collected_time.tv_sec > ctl->now - (rd->rrdset->update_every * 2))
+                                count++;
+                        }
+                        s.host->accounting.currently_collected = count;
+                        s.host->accounting.cache_timestamp = ctl->now;
+                    }
+                    spinlock_unlock(&s.host->accounting.spinlock);
+
+                    buffer_json_member_add_uint64(wb, "currently_collected_metrics", count);
+                    buffer_json_member_add_uint64(wb, "instances", s.db.instances);
+                    buffer_json_member_add_uint64(wb, "contexts", s.db.contexts);
+                }
+                buffer_json_object_close(wb);
+
+                rrdhost_receiver_to_json(wb, &s, "ingest");
+                rrdhost_sender_to_json(wb, &s, "stream");
+
+                buffer_json_member_add_object(wb, "ml");
+                buffer_json_member_add_string(wb, "status", rrdhost_ml_status_to_string(s.ml.status));
+                buffer_json_member_add_string(wb, "type", rrdhost_ml_type_to_string(s.ml.type));
+                if (s.ml.status == RRDHOST_ML_STATUS_RUNNING) {
+                    buffer_json_member_add_object(wb, "metrics");
+                    {
+                        buffer_json_member_add_uint64(wb, "anomalous", s.ml.metrics.anomalous);
+                        buffer_json_member_add_uint64(wb, "normal", s.ml.metrics.normal);
+                        buffer_json_member_add_uint64(wb, "trained", s.ml.metrics.trained);
+                        buffer_json_member_add_uint64(wb, "pending", s.ml.metrics.pending);
+                        buffer_json_member_add_uint64(wb, "silenced", s.ml.metrics.silenced);
+                    }
+                    buffer_json_object_close(wb); // metrics
+                }
+                buffer_json_object_close(wb); // ml
+
+                rrdhost_health_to_json_v2(wb, "health", &s);
+
+                host_functions2json(host, wb); // functions
+                agent_capabilities_to_json(wb, host, "capabilities");
+
+                host_dyncfg_to_json_v2(wb, "dyncfg", &s);
+            }
+            buffer_json_object_close(wb); // this instance
+            buffer_json_array_close(wb); // instances
+        }
+    }
+    buffer_json_object_close(wb); // this node
+}
+
+static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) {
+    if(!queryable_host || !host->rrdctx.contexts)
+        // the host matches the 'scope_host' but does not match the 'host' patterns
+        // or the host does not have any contexts
+        return 0; // continue to next host
+
+    struct rrdcontext_to_json_v2_data *ctl = data;
+
+    if(ctl->window.enabled && !rrdhost_matches_window(host, ctl->window.after, ctl->window.before, ctl->now))
+        // the host does not have data in the requested window
+        return 0; // continue to next host
+
+    if(ctl->request->timeout_ms && now_monotonic_usec() > ctl->timings.received_ut + ctl->request->timeout_ms * USEC_PER_MS)
+        // timed out
+        return -2; // stop the query
+
+    if(ctl->request->interrupt_callback && ctl->request->interrupt_callback(ctl->request->interrupt_callback_data))
+        // interrupted
+        return -1; // stop the query
+
+    bool host_matched = (ctl->mode & CONTEXTS_V2_NODES);
+    bool do_contexts = (ctl->mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_ALERTS));
+
+    ctl->q.host_match = FTS_MATCHED_NONE;
+    if((ctl->mode & CONTEXTS_V2_SEARCH)) {
+        // check if we match the host itself
+        if(ctl->q.pattern && (
+                full_text_search_string(&ctl->q.fts, ctl->q.pattern, host->hostname) ||
+                full_text_search_char(&ctl->q.fts, ctl->q.pattern, host->machine_guid) ||
+                (ctl->q.pattern && full_text_search_char(&ctl->q.fts, ctl->q.pattern, ctl->q.host_node_id_str)))) {
+            ctl->q.host_match = FTS_MATCHED_HOST;
+            do_contexts = true;
+        }
+    }
+
+    if(do_contexts) {
+        // save it
+        SIMPLE_PATTERN *old_q = ctl->q.pattern;
+
+        if(ctl->q.host_match == FTS_MATCHED_HOST)
+            // do not do pattern matching on contexts - we matched the host itself
+            ctl->q.pattern = NULL;
+
+        ssize_t added = query_scope_foreach_context(
+                host, ctl->request->scope_contexts,
+                ctl->contexts.scope_pattern, ctl->contexts.pattern,
+                rrdcontext_to_json_v2_add_context, queryable_host, ctl);
+
+        // restore it
+        ctl->q.pattern = old_q;
+
+        if(unlikely(added < 0))
+            return -1; // stop the query
+
+        if(added)
+            host_matched = true;
+    }
+
+    if(!host_matched)
+        return 0;
+
+    if(ctl->mode & CONTEXTS_V2_FUNCTIONS) {
+        struct function_v2_entry t = {
+            .used = 1,
+            .size = 1,
+            .node_ids = &ctl->nodes.ni,
+            .help = NULL,
+            .tags = NULL,
+            .access = HTTP_ACCESS_ALL,
+            .priority = RRDFUNCTIONS_PRIORITY_DEFAULT,
+            .version = RRDFUNCTIONS_VERSION_DEFAULT,
+        };
+        host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help, &t.tags, &t.access, &t.priority, &t.version);
+    }
+
+    if(ctl->mode & CONTEXTS_V2_NODES) {
+        struct contexts_v2_node t = {
+            .ni = ctl->nodes.ni++,
+            .host = host,
+        };
+
+        dictionary_set(ctl->nodes.dict, host->machine_guid, &t, sizeof(struct contexts_v2_node));
+    }
+
+    return 1;
+}
+
+static void buffer_json_contexts_v2_mode_to_array(BUFFER *wb, const char *key, CONTEXTS_V2_MODE mode) {
+    buffer_json_member_add_array(wb, key);
+
+    if(mode & CONTEXTS_V2_VERSIONS)
+        buffer_json_add_array_item_string(wb, "versions");
+
+    if(mode & CONTEXTS_V2_AGENTS)
+        buffer_json_add_array_item_string(wb, "agents");
+
+    if(mode & CONTEXTS_V2_AGENTS_INFO)
+        buffer_json_add_array_item_string(wb, "agents-info");
+
+    if(mode & CONTEXTS_V2_NODES)
+        buffer_json_add_array_item_string(wb, "nodes");
+
+    if(mode & CONTEXTS_V2_NODES_INFO)
+        buffer_json_add_array_item_string(wb, "nodes-info");
+
+    if(mode & CONTEXTS_V2_NODE_INSTANCES)
+        buffer_json_add_array_item_string(wb, "nodes-instances");
+
+    if(mode & CONTEXTS_V2_CONTEXTS)
+        buffer_json_add_array_item_string(wb, "contexts");
+
+    if(mode & CONTEXTS_V2_SEARCH)
+        buffer_json_add_array_item_string(wb, "search");
+
+    if(mode & CONTEXTS_V2_ALERTS)
+        buffer_json_add_array_item_string(wb, "alerts");
+
+    if(mode & CONTEXTS_V2_ALERT_TRANSITIONS)
+        buffer_json_add_array_item_string(wb, "alert_transitions");
+
+    buffer_json_array_close(wb);
+}
+
+void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings) {
+    timings->finished_ut = now_monotonic_usec();
+    if(!timings->executed_ut)
+        timings->executed_ut = timings->finished_ut;
+    if(!timings->preprocessed_ut)
+        timings->preprocessed_ut = timings->received_ut;
+    buffer_json_member_add_object(wb, key);
+    buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(timings->preprocessed_ut - timings->received_ut) / USEC_PER_MS);
+    buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(timings->executed_ut - timings->preprocessed_ut) / USEC_PER_MS);
+    buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->executed_ut) / USEC_PER_MS);
+    buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS);
+    buffer_json_member_add_double(wb, "cloud_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS);
+    buffer_json_object_close(wb);
+}
+
+void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings) {
+    if(!timings->finished_ut)
+        timings->finished_ut = now_monotonic_usec();
+
+    buffer_json_member_add_object(wb, key);
+    buffer_json_member_add_double(wb, "routing_ms", 0.0);
+    buffer_json_member_add_double(wb, "node_max_ms", 0.0);
+    buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS);
+    buffer_json_object_close(wb);
+}
+
+static void functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+    struct function_v2_entry *t = value;
+
+    // it is initialized with a static reference - we need to mallocz() the array
+    size_t *v = t->node_ids;
+    t->node_ids = mallocz(sizeof(size_t));
+    *t->node_ids = *v;
+    t->size = 1;
+    t->used = 1;
+}
+
+static bool functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
+    struct function_v2_entry *t = old_value, *n = new_value;
+    size_t *v = n->node_ids;
+
+    if(t->used >= t->size) {
+        t->node_ids = reallocz(t->node_ids, t->size * 2 * sizeof(size_t));
+        t->size *= 2;
+    }
+
+    t->node_ids[t->used++] = *v;
+
+    return true;
+}
+
+static void functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+    struct function_v2_entry *t = value;
+    freez(t->node_ids);
+}
+
+static bool contexts_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
+    struct context_v2_entry *o = old_value;
+    struct context_v2_entry *n = new_value;
+
+    o->count++;
+
+    if(o->family != n->family) {
+        if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED))
+            // keep old
+            ;
+        else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) {
+            // keep new
+            string_freez(o->family);
+            o->family = string_dup(n->family);
+        }
+        else {
+            // merge
+            STRING *old_family = o->family;
+            o->family = string_2way_merge(o->family, n->family);
+            string_freez(old_family);
+        }
+    }
+
+    if(o->priority != n->priority) {
+        if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED))
+            // keep o
+            ;
+        else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED))
+            // keep n
+            o->priority = n->priority;
+        else
+            // keep the min
+            o->priority = MIN(o->priority, n->priority);
+    }
+
+    if(o->first_time_s && n->first_time_s)
+        o->first_time_s = MIN(o->first_time_s, n->first_time_s);
+    else if(!o->first_time_s)
+        o->first_time_s = n->first_time_s;
+
+    if(o->last_time_s && n->last_time_s)
+        o->last_time_s = MAX(o->last_time_s, n->last_time_s);
+    else if(!o->last_time_s)
+        o->last_time_s = n->last_time_s;
+
+    o->flags |= n->flags;
+    o->match = MIN(o->match, n->match);
+
+    string_freez(n->family);
+
+    return true;
+}
+
+static void contexts_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+    struct context_v2_entry *z = value;
+    string_freez(z->family);
+}
+
+int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_MODE mode) {
+    int resp = HTTP_RESP_OK;
+    bool run = true;
+
+    if(mode & CONTEXTS_V2_SEARCH)
+        mode |= CONTEXTS_V2_CONTEXTS;
+
+    if(mode & (CONTEXTS_V2_AGENTS_INFO))
+        mode |= CONTEXTS_V2_AGENTS;
+
+    if(mode & (CONTEXTS_V2_FUNCTIONS | CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES))
+        mode |= CONTEXTS_V2_NODES;
+
+    if(mode & CONTEXTS_V2_ALERTS) {
+        mode |= CONTEXTS_V2_NODES;
+        req->options &= ~CONTEXTS_OPTION_ALERTS_WITH_CONFIGURATIONS;
+
+        if(!(req->options & (CONTEXTS_OPTION_ALERTS_WITH_SUMMARY | CONTEXTS_OPTION_ALERTS_WITH_INSTANCES |
+                              CONTEXTS_OPTION_ALERTS_WITH_VALUES)))
+            req->options |= CONTEXTS_OPTION_ALERTS_WITH_SUMMARY;
+    }
+
+    if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) {
+        mode |= CONTEXTS_V2_NODES;
+        req->options &= ~CONTEXTS_OPTION_ALERTS_WITH_INSTANCES;
+    }
+
+    struct rrdcontext_to_json_v2_data ctl = {
+            .wb = wb,
+            .request = req,
+            .mode = mode,
+            .options = req->options,
+            .versions = { 0 },
+            .nodes.scope_pattern = string_to_simple_pattern(req->scope_nodes),
+            .nodes.pattern = string_to_simple_pattern(req->nodes),
+            .contexts.pattern = string_to_simple_pattern(req->contexts),
+            .contexts.scope_pattern = string_to_simple_pattern(req->scope_contexts),
+            .q.pattern = string_to_simple_pattern_nocase(req->q),
+            .alerts.alert_name_pattern = string_to_simple_pattern(req->alerts.alert),
+            .window = {
+                    .enabled = false,
+                    .relative = false,
+                    .after = req->after,
+                    .before = req->before,
+            },
+            .timings = {
+                    .received_ut = now_monotonic_usec(),
+            }
+    };
+
+    bool debug = ctl.options & CONTEXTS_OPTION_DEBUG;
+
+    if(mode & CONTEXTS_V2_NODES) {
+        ctl.nodes.dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
+                                                    NULL, sizeof(struct contexts_v2_node));
+    }
+
+    if(mode & CONTEXTS_V2_CONTEXTS) {
+        ctl.contexts.dict = dictionary_create_advanced(
+                DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL,
+                sizeof(struct context_v2_entry));
+
+        dictionary_register_conflict_callback(ctl.contexts.dict, contexts_conflict_callback, &ctl);
+        dictionary_register_delete_callback(ctl.contexts.dict, contexts_delete_callback, &ctl);
+    }
+
+    if(mode & CONTEXTS_V2_FUNCTIONS) {
+        ctl.functions.dict = dictionary_create_advanced(
+                DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL,
+                sizeof(struct function_v2_entry));
+
+        dictionary_register_insert_callback(ctl.functions.dict, functions_insert_callback, &ctl);
+        dictionary_register_conflict_callback(ctl.functions.dict, functions_conflict_callback, &ctl);
+        dictionary_register_delete_callback(ctl.functions.dict, functions_delete_callback, &ctl);
+    }
+
+    if(mode & CONTEXTS_V2_ALERTS) {
+        if(!rrdcontexts_v2_init_alert_dictionaries(&ctl, req)) {
+            resp = HTTP_RESP_NOT_FOUND;
+            goto cleanup;
+        }
+    }
+
+    if(req->after || req->before) {
+        ctl.window.relative = rrdr_relative_window_to_absolute_query(
+            &ctl.window.after, &ctl.window.before, &ctl.now, false);
+
+        ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS);
+    }
+    else
+        ctl.now = now_realtime_sec();
+
+    buffer_json_initialize(wb, "\"", "\"", 0, true,
+                           ((req->options & CONTEXTS_OPTION_MINIFY) && !(req->options & CONTEXTS_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
+
+    buffer_json_member_add_uint64(wb, "api", 2);
+
+    if(req->options & CONTEXTS_OPTION_DEBUG) {
+        buffer_json_member_add_object(wb, "request");
+        {
+            buffer_json_contexts_v2_mode_to_array(wb, "mode", mode);
+            contexts_options_to_buffer_json_array(wb, "options", req->options);
+
+            buffer_json_member_add_object(wb, "scope");
+            {
+                buffer_json_member_add_string(wb, "scope_nodes", req->scope_nodes);
+                if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS))
+                    buffer_json_member_add_string(wb, "scope_contexts", req->scope_contexts);
+            }
+            buffer_json_object_close(wb);
+
+            buffer_json_member_add_object(wb, "selectors");
+            {
+                buffer_json_member_add_string(wb, "nodes", req->nodes);
+
+                if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS))
+                    buffer_json_member_add_string(wb, "contexts", req->contexts);
+
+                if(mode & (CONTEXTS_V2_ALERTS | CONTEXTS_V2_ALERT_TRANSITIONS)) {
+                    buffer_json_member_add_object(wb, "alerts");
+
+                    if(mode & CONTEXTS_V2_ALERTS)
+                        contexts_alerts_status_to_buffer_json_array(wb, "status", req->alerts.status);
+
+                    if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) {
+                        buffer_json_member_add_string(wb, "context", req->contexts);
+                        buffer_json_member_add_uint64(wb, "anchor_gi", req->alerts.global_id_anchor);
+                        buffer_json_member_add_uint64(wb, "last", req->alerts.last);
+                    }
+
+                    buffer_json_member_add_string(wb, "alert", req->alerts.alert);
+                    buffer_json_member_add_string(wb, "transition", req->alerts.transition);
+                    buffer_json_object_close(wb); // alerts
+                }
+            }
+            buffer_json_object_close(wb); // selectors
+
+            buffer_json_member_add_object(wb, "filters");
+            {
+                if (mode & CONTEXTS_V2_SEARCH)
+                    buffer_json_member_add_string(wb, "q", req->q);
+
+                buffer_json_member_add_time_t(wb, "after", req->after);
+                buffer_json_member_add_time_t(wb, "before", req->before);
+            }
+            buffer_json_object_close(wb); // filters
+
+            if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) {
+                buffer_json_member_add_object(wb, "facets");
+                {
+                    for (int i = 0; i < ATF_TOTAL_ENTRIES; i++) {
+                        buffer_json_member_add_string(wb, alert_transition_facets[i].query_param, req->alerts.facets[i]);
+                    }
+                }
+                buffer_json_object_close(wb); // facets
+            }
+        }
+        buffer_json_object_close(wb);
+    }
+
+    ssize_t ret = 0;
+    if(run)
+        ret = query_scope_foreach_host(ctl.nodes.scope_pattern, ctl.nodes.pattern,
+                             rrdcontext_to_json_v2_add_host, &ctl,
+                             &ctl.versions, ctl.q.host_node_id_str);
+
+    if(unlikely(ret < 0)) {
+        buffer_flush(wb);
+
+        if(ret == -2) {
+            buffer_strcat(wb, "query timeout");
+            resp = HTTP_RESP_GATEWAY_TIMEOUT;
+        }
+        else {
+            buffer_strcat(wb, "query interrupted");
+            resp = HTTP_RESP_CLIENT_CLOSED_REQUEST;
+        }
+        goto cleanup;
+    }
+
+    ctl.timings.executed_ut = now_monotonic_usec();
+
+    if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) {
+        contexts_v2_alert_transitions_to_json(wb, &ctl, debug);
+    }
+    else {
+        if (mode & CONTEXTS_V2_NODES) {
+            buffer_json_member_add_array(wb, "nodes");
+            struct contexts_v2_node *t;
+            dfe_start_read(ctl.nodes.dict, t) {
+                rrdcontext_to_json_v2_rrdhost(wb, t->host, &ctl, t->ni);
+            }
+            dfe_done(t);
+            buffer_json_array_close(wb);
+        }
+
+        if (mode & CONTEXTS_V2_FUNCTIONS) {
+            buffer_json_member_add_array(wb, "functions");
+            {
+                struct function_v2_entry *t;
+                dfe_start_read(ctl.functions.dict, t) {
+                    buffer_json_add_array_item_object(wb);
+                    {
+                        const char *name = t_dfe.name ? strstr(t_dfe.name, RRDFUNCTIONS_VERSION_SEPARATOR) : NULL;
+                        if(name)
+                            name += sizeof(RRDFUNCTIONS_VERSION_SEPARATOR) - 1;
+                        else
+                            name = t_dfe.name;
+
+                        buffer_json_member_add_string(wb, "name", name);
+                        buffer_json_member_add_string(wb, "help", string2str(t->help));
+                        buffer_json_member_add_array(wb, "ni");
+                        {
+                            for (size_t i = 0; i < t->used; i++)
+                                buffer_json_add_array_item_uint64(wb, t->node_ids[i]);
+                        }
+                        buffer_json_array_close(wb);
+                        buffer_json_member_add_string(wb, "tags", string2str(t->tags));
+                        http_access2buffer_json_array(wb, "access", t->access);
+                        buffer_json_member_add_uint64(wb, "priority", t->priority);
+                        buffer_json_member_add_uint64(wb, "version", t->version);
+                    }
+                    buffer_json_object_close(wb);
+                }
+                dfe_done(t);
+            }
+            buffer_json_array_close(wb);
+        }
+
+        if (mode & CONTEXTS_V2_CONTEXTS) {
+            buffer_json_member_add_object(wb, "contexts");
+            {
+                struct context_v2_entry *z;
+                dfe_start_read(ctl.contexts.dict, z) {
+                    bool collected = z->flags & RRD_FLAG_COLLECTED;
+
+                    buffer_json_member_add_object(wb, string2str(z->id));
+                    {
+                        buffer_json_member_add_string(wb, "family", string2str(z->family));
+                        buffer_json_member_add_uint64(wb, "priority", z->priority);
+                        buffer_json_member_add_time_t(wb, "first_entry", z->first_time_s);
+                        buffer_json_member_add_time_t(wb, "last_entry", collected ? ctl.now : z->last_time_s);
+                        buffer_json_member_add_boolean(wb, "live", collected);
+                        if (mode & CONTEXTS_V2_SEARCH)
+                            buffer_json_member_add_string(wb, "match", fts_match_to_string(z->match));
+                    }
+                    buffer_json_object_close(wb);
+                }
+                dfe_done(z);
+            }
+            buffer_json_object_close(wb); // contexts
+        }
+
+        if (mode & CONTEXTS_V2_ALERTS)
+            contexts_v2_alerts_to_json(wb, &ctl, debug);
+
+        if (mode & CONTEXTS_V2_SEARCH) {
+            buffer_json_member_add_object(wb, "searches");
+            {
+                buffer_json_member_add_uint64(wb, "strings", ctl.q.fts.string_searches);
+                buffer_json_member_add_uint64(wb, "char", ctl.q.fts.char_searches);
+                buffer_json_member_add_uint64(wb, "total", ctl.q.fts.searches);
+            }
+            buffer_json_object_close(wb);
+        }
+
+        if (mode & (CONTEXTS_V2_VERSIONS))
+            version_hashes_api_v2(wb, &ctl.versions);
+
+        if (mode & CONTEXTS_V2_AGENTS)
+            buffer_json_agents_v2(wb, &ctl.timings, ctl.now, mode & (CONTEXTS_V2_AGENTS_INFO), true);
+    }
+
+    buffer_json_cloud_timings(wb, "timings", &ctl.timings);
+
+    buffer_json_finalize(wb);
+
+cleanup:
+    dictionary_destroy(ctl.nodes.dict);
+    dictionary_destroy(ctl.contexts.dict);
+    dictionary_destroy(ctl.functions.dict);
+    rrdcontexts_v2_alerts_cleanup(&ctl);
+    simple_pattern_free(ctl.nodes.scope_pattern);
+    simple_pattern_free(ctl.nodes.pattern);
+    simple_pattern_free(ctl.contexts.pattern);
+    simple_pattern_free(ctl.contexts.scope_pattern);
+    simple_pattern_free(ctl.q.pattern);
+    simple_pattern_free(ctl.alerts.alert_name_pattern);
+
+    return resp;
+}