// SPDX-License-Identifier: GPL-3.0-or-later #include "api_v2_contexts.h" #include "aclk/aclk_capas.h" // ---------------------------------------------------------------------------- // /api/v2/contexts API static const char *fts_match_to_string(FTS_MATCH match) { switch(match) { case FTS_MATCHED_HOST: return "HOST"; case FTS_MATCHED_CONTEXT: return "CONTEXT"; case FTS_MATCHED_INSTANCE: return "INSTANCE"; case FTS_MATCHED_DIMENSION: return "DIMENSION"; case FTS_MATCHED_ALERT: return "ALERT"; case FTS_MATCHED_ALERT_INFO: return "ALERT_INFO"; case FTS_MATCHED_LABEL: return "LABEL"; case FTS_MATCHED_FAMILY: return "FAMILY"; case FTS_MATCHED_TITLE: return "TITLE"; case FTS_MATCHED_UNITS: return "UNITS"; default: return "NONE"; } } struct function_v2_entry { size_t size; size_t used; size_t *node_ids; STRING *help; STRING *tags; HTTP_ACCESS access; int priority; uint32_t version; }; struct context_v2_entry { size_t count; STRING *id; STRING *family; uint32_t priority; time_t first_time_s; time_t last_time_s; RRD_FLAGS flags; FTS_MATCH match; }; static inline bool full_text_search_string(FTS_INDEX *fts, SIMPLE_PATTERN *q, STRING *ptr) { fts->searches++; fts->string_searches++; return simple_pattern_matches_string(q, ptr); } static inline bool full_text_search_char(FTS_INDEX *fts, SIMPLE_PATTERN *q, char *ptr) { fts->searches++; fts->char_searches++; return simple_pattern_matches(q, ptr); } static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc, SIMPLE_PATTERN *q) { if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->id) || full_text_search_string(&ctl->q.fts, q, rc->family))) return FTS_MATCHED_CONTEXT; if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->title))) return FTS_MATCHED_TITLE; if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->units))) return FTS_MATCHED_UNITS; FTS_MATCH matched = FTS_MATCHED_NONE; RRDINSTANCE *ri; dfe_start_read(rc->rrdinstances, ri) { if(matched) break; if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, ri->first_time_s, (ri->flags & RRD_FLAG_COLLECTED) ? ctl->now : ri->last_time_s, 0)) continue; if(unlikely(full_text_search_string(&ctl->q.fts, q, ri->id)) || (ri->name != ri->id && full_text_search_string(&ctl->q.fts, q, ri->name))) { matched = FTS_MATCHED_INSTANCE; break; } RRDMETRIC *rm; dfe_start_read(ri->rrdmetrics, rm) { if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rm->first_time_s, (rm->flags & RRD_FLAG_COLLECTED) ? ctl->now : rm->last_time_s, 0)) continue; if(unlikely(full_text_search_string(&ctl->q.fts, q, rm->id)) || (rm->name != rm->id && full_text_search_string(&ctl->q.fts, q, rm->name))) { matched = FTS_MATCHED_DIMENSION; break; } } dfe_done(rm); size_t label_searches = 0; if(unlikely(ri->rrdlabels && rrdlabels_entries(ri->rrdlabels) && rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, q, ':', &label_searches) == SP_MATCHED_POSITIVE)) { ctl->q.fts.searches += label_searches; ctl->q.fts.char_searches += label_searches; matched = FTS_MATCHED_LABEL; break; } ctl->q.fts.searches += label_searches; ctl->q.fts.char_searches += label_searches; if(ri->rrdset) { RRDSET *st = ri->rrdset; rw_spinlock_read_lock(&st->alerts.spinlock); for (RRDCALC *rcl = st->alerts.base; rcl; rcl = rcl->next) { if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.name))) { matched = FTS_MATCHED_ALERT; break; } if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.info))) { matched = FTS_MATCHED_ALERT_INFO; break; } } rw_spinlock_read_unlock(&st->alerts.spinlock); } } dfe_done(ri); return matched; } static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context __maybe_unused) { struct rrdcontext_to_json_v2_data *ctl = data; RRDCONTEXT *rc = rrdcontext_acquired_value(rca); if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rc->first_time_s, (rc->flags & RRD_FLAG_COLLECTED) ? ctl->now : rc->last_time_s, 0)) return 0; // continue to next context FTS_MATCH match = ctl->q.host_match; if((ctl->mode & CONTEXTS_V2_SEARCH) && ctl->q.pattern) { match = rrdcontext_to_json_v2_full_text_search(ctl, rc, ctl->q.pattern); if(match == FTS_MATCHED_NONE) return 0; // continue to next context } if(ctl->mode & CONTEXTS_V2_ALERTS) { if(!rrdcontext_matches_alert(ctl, rc)) return 0; // continue to next context } if(ctl->contexts.dict) { struct context_v2_entry t = { .count = 1, .id = rc->id, .family = string_dup(rc->family), .priority = rc->priority, .first_time_s = rc->first_time_s, .last_time_s = rc->last_time_s, .flags = rc->flags, .match = match, }; dictionary_set(ctl->contexts.dict, string2str(rc->id), &t, sizeof(struct context_v2_entry)); } return 1; } void buffer_json_agent_status_id(BUFFER *wb, size_t ai, usec_t duration_ut) { buffer_json_member_add_object(wb, "st"); { buffer_json_member_add_uint64(wb, "ai", ai); buffer_json_member_add_uint64(wb, "code", 200); buffer_json_member_add_string(wb, "msg", ""); if (duration_ut) buffer_json_member_add_double(wb, "ms", (NETDATA_DOUBLE) duration_ut / 1000.0); } buffer_json_object_close(wb); } void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut, bool status) { buffer_json_member_add_string(wb, "mg", host->machine_guid); if(!UUIDiszero(host->node_id)) buffer_json_member_add_uuid(wb, "nd", host->node_id.uuid); buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host)); buffer_json_member_add_uint64(wb, "ni", ni); if(status) buffer_json_agent_status_id(wb, 0, duration_ut); } static void rrdhost_receiver_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) { buffer_json_member_add_object(wb, key); { buffer_json_member_add_uint64(wb, "id", s->ingest.id); buffer_json_member_add_uint64(wb, "hops", s->ingest.hops); buffer_json_member_add_string(wb, "type", rrdhost_ingest_type_to_string(s->ingest.type)); buffer_json_member_add_string(wb, "status", rrdhost_ingest_status_to_string(s->ingest.status)); buffer_json_member_add_time_t(wb, "since", s->ingest.since); buffer_json_member_add_time_t(wb, "age", s->now - s->ingest.since); if(s->ingest.type == RRDHOST_INGEST_TYPE_CHILD) { if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE) buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->ingest.reason)); if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING) { buffer_json_member_add_object(wb, "replication"); { buffer_json_member_add_boolean(wb, "in_progress", s->ingest.replication.in_progress); buffer_json_member_add_double(wb, "completion", s->ingest.replication.completion); buffer_json_member_add_uint64(wb, "instances", s->ingest.replication.instances); } buffer_json_object_close(wb); // replication } if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING || s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE) { buffer_json_member_add_object(wb, "source"); { char buf[1024 + 1]; snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.local.ip, s->ingest.peers.local.port, s->ingest.ssl ? ":SSL" : ""); buffer_json_member_add_string(wb, "local", buf); snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.peer.ip, s->ingest.peers.peer.port, s->ingest.ssl ? ":SSL" : ""); buffer_json_member_add_string(wb, "remote", buf); stream_capabilities_to_json_array(wb, s->ingest.capabilities, "capabilities"); } buffer_json_object_close(wb); // source } } } buffer_json_object_close(wb); // collection } static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) { if(s->stream.status == RRDHOST_STREAM_STATUS_DISABLED) return; buffer_json_member_add_object(wb, key); { buffer_json_member_add_uint64(wb, "id", s->stream.id); buffer_json_member_add_uint64(wb, "hops", s->stream.hops); buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status)); buffer_json_member_add_time_t(wb, "since", s->stream.since); buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since); if (s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE) buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->stream.reason)); if (s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) { buffer_json_member_add_object(wb, "replication"); { buffer_json_member_add_boolean(wb, "in_progress", s->stream.replication.in_progress); buffer_json_member_add_double(wb, "completion", s->stream.replication.completion); buffer_json_member_add_uint64(wb, "instances", s->stream.replication.instances); } buffer_json_object_close(wb); } buffer_json_member_add_object(wb, "destination"); { char buf[1024 + 1]; snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.local.ip, s->stream.peers.local.port, s->stream.ssl ? ":SSL" : ""); buffer_json_member_add_string(wb, "local", buf); snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.peer.ip, s->stream.peers.peer.port, s->stream.ssl ? ":SSL" : ""); buffer_json_member_add_string(wb, "remote", buf); stream_capabilities_to_json_array(wb, s->stream.capabilities, "capabilities"); buffer_json_member_add_object(wb, "traffic"); { buffer_json_member_add_boolean(wb, "compression", s->stream.compression); buffer_json_member_add_uint64(wb, "data", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA]); buffer_json_member_add_uint64(wb, "metadata", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA]); buffer_json_member_add_uint64(wb, "functions", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS]); buffer_json_member_add_uint64(wb, "replication", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION]); buffer_json_member_add_uint64(wb, "dyncfg", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DYNCFG]); } buffer_json_object_close(wb); // traffic buffer_json_member_add_array(wb, "candidates"); struct rrdpush_destinations *d; for (d = s->host->destinations; d; d = d->next) { buffer_json_add_array_item_object(wb); buffer_json_member_add_uint64(wb, "attempts", d->attempts); { if (d->ssl) { snprintfz(buf, sizeof(buf) - 1, "%s:SSL", string2str(d->destination)); buffer_json_member_add_string(wb, "destination", buf); } else buffer_json_member_add_string(wb, "destination", string2str(d->destination)); buffer_json_member_add_time_t(wb, "since", d->since); buffer_json_member_add_time_t(wb, "age", s->now - d->since); buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->reason)); if(d->postpone_reconnection_until > s->now) { buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until); buffer_json_member_add_time_t(wb, "next_in", d->postpone_reconnection_until - s->now); } } buffer_json_object_close(wb); // each candidate } buffer_json_array_close(wb); // candidates } buffer_json_object_close(wb); // destination } buffer_json_object_close(wb); // streaming } void agent_capabilities_to_json(BUFFER *wb, RRDHOST *host, const char *key) { buffer_json_member_add_array(wb, key); struct capability *capas = aclk_get_node_instance_capas(host); for(struct capability *capa = capas; capa->name ;capa++) { buffer_json_add_array_item_object(wb); { buffer_json_member_add_string(wb, "name", capa->name); buffer_json_member_add_uint64(wb, "version", capa->version); buffer_json_member_add_boolean(wb, "enabled", capa->enabled); } buffer_json_object_close(wb); } buffer_json_array_close(wb); freez(capas); } static inline void host_dyncfg_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) { buffer_json_member_add_object(wb, key); { buffer_json_member_add_string(wb, "status", rrdhost_dyncfg_status_to_string(s->dyncfg.status)); } buffer_json_object_close(wb); // health } static inline void rrdhost_health_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) { buffer_json_member_add_object(wb, key); { buffer_json_member_add_string(wb, "status", rrdhost_health_status_to_string(s->health.status)); if (s->health.status == RRDHOST_HEALTH_STATUS_RUNNING) { buffer_json_member_add_object(wb, "alerts"); { buffer_json_member_add_uint64(wb, "critical", s->health.alerts.critical); buffer_json_member_add_uint64(wb, "warning", s->health.alerts.warning); buffer_json_member_add_uint64(wb, "clear", s->health.alerts.clear); buffer_json_member_add_uint64(wb, "undefined", s->health.alerts.undefined); buffer_json_member_add_uint64(wb, "uninitialized", s->health.alerts.uninitialized); } buffer_json_object_close(wb); // alerts } } buffer_json_object_close(wb); // health } static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdcontext_to_json_v2_data *ctl, size_t node_id) { buffer_json_add_array_item_object(wb); // this node buffer_json_node_add_v2(wb, host, node_id, 0, (ctl->mode & CONTEXTS_V2_AGENTS) && !(ctl->mode & CONTEXTS_V2_NODE_INSTANCES)); if(ctl->mode & (CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES)) { RRDHOST_STATUS s; rrdhost_status(host, ctl->now, &s); if (ctl->mode & (CONTEXTS_V2_NODES_INFO)) { buffer_json_member_add_string(wb, "v", rrdhost_program_version(host)); host_labels2json(host, wb, "labels"); if (host->system_info) { buffer_json_member_add_object(wb, "hw"); { buffer_json_member_add_string_or_empty(wb, "architecture", host->system_info->architecture); buffer_json_member_add_string_or_empty(wb, "cpu_frequency", host->system_info->host_cpu_freq); buffer_json_member_add_string_or_empty(wb, "cpus", host->system_info->host_cores); buffer_json_member_add_string_or_empty(wb, "memory", host->system_info->host_ram_total); buffer_json_member_add_string_or_empty(wb, "disk_space", host->system_info->host_disk_space); buffer_json_member_add_string_or_empty(wb, "virtualization", host->system_info->virtualization); buffer_json_member_add_string_or_empty(wb, "container", host->system_info->container); } buffer_json_object_close(wb); buffer_json_member_add_object(wb, "os"); { buffer_json_member_add_string_or_empty(wb, "id", host->system_info->host_os_id); buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->host_os_name); buffer_json_member_add_string_or_empty(wb, "v", host->system_info->host_os_version); buffer_json_member_add_object(wb, "kernel"); buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->kernel_name); buffer_json_member_add_string_or_empty(wb, "v", host->system_info->kernel_version); buffer_json_object_close(wb); } buffer_json_object_close(wb); } // created - the node is created but never connected to cloud // unreachable - not currently connected // stale - connected but not having live data // reachable - connected with live data // pruned - not connected for some time and has been removed buffer_json_member_add_string(wb, "state", rrdhost_is_online(host) ? "reachable" : "stale"); rrdhost_health_to_json_v2(wb, "health", &s); agent_capabilities_to_json(wb, host, "capabilities"); rrdhost_stream_path_to_json(wb, host, STREAM_PATH_JSON_MEMBER, false); } if (ctl->mode & (CONTEXTS_V2_NODE_INSTANCES)) { buffer_json_member_add_array(wb, "instances"); buffer_json_add_array_item_object(wb); // this instance { buffer_json_agent_status_id(wb, 0, 0); buffer_json_member_add_object(wb, "db"); { buffer_json_member_add_string(wb, "status", rrdhost_db_status_to_string(s.db.status)); buffer_json_member_add_string(wb, "liveness", rrdhost_db_liveness_to_string(s.db.liveness)); buffer_json_member_add_string(wb, "mode", rrd_memory_mode_name(s.db.mode)); buffer_json_member_add_time_t(wb, "first_time", s.db.first_time_s); buffer_json_member_add_time_t(wb, "last_time", s.db.last_time_s); buffer_json_member_add_uint64(wb, "metrics", s.db.metrics); spinlock_lock(&s.host->accounting.spinlock); int64_t count = 0; if (s.host->accounting.cache_timestamp && ctl->now - s.host->accounting.cache_timestamp < host->rrd_update_every * 1.5) count = s.host->accounting.currently_collected; else { Pvoid_t *Pvalue; bool first = true; Word_t dimension_id = 0; while ((Pvalue = JudyLFirstThenNext(s.host->accounting.JudyL, &dimension_id, &first))) { RRDDIM *rd = *Pvalue; if (rd->collector.last_collected_time.tv_sec > ctl->now - (rd->rrdset->update_every * 2)) count++; } s.host->accounting.currently_collected = count; s.host->accounting.cache_timestamp = ctl->now; } spinlock_unlock(&s.host->accounting.spinlock); buffer_json_member_add_uint64(wb, "currently_collected_metrics", count); buffer_json_member_add_uint64(wb, "instances", s.db.instances); buffer_json_member_add_uint64(wb, "contexts", s.db.contexts); } buffer_json_object_close(wb); rrdhost_receiver_to_json(wb, &s, "ingest"); rrdhost_sender_to_json(wb, &s, "stream"); buffer_json_member_add_object(wb, "ml"); buffer_json_member_add_string(wb, "status", rrdhost_ml_status_to_string(s.ml.status)); buffer_json_member_add_string(wb, "type", rrdhost_ml_type_to_string(s.ml.type)); if (s.ml.status == RRDHOST_ML_STATUS_RUNNING) { buffer_json_member_add_object(wb, "metrics"); { buffer_json_member_add_uint64(wb, "anomalous", s.ml.metrics.anomalous); buffer_json_member_add_uint64(wb, "normal", s.ml.metrics.normal); buffer_json_member_add_uint64(wb, "trained", s.ml.metrics.trained); buffer_json_member_add_uint64(wb, "pending", s.ml.metrics.pending); buffer_json_member_add_uint64(wb, "silenced", s.ml.metrics.silenced); } buffer_json_object_close(wb); // metrics } buffer_json_object_close(wb); // ml rrdhost_health_to_json_v2(wb, "health", &s); host_functions2json(host, wb); // functions agent_capabilities_to_json(wb, host, "capabilities"); host_dyncfg_to_json_v2(wb, "dyncfg", &s); } buffer_json_object_close(wb); // this instance buffer_json_array_close(wb); // instances } } buffer_json_object_close(wb); // this node } static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) { if(!queryable_host || !host->rrdctx.contexts) // the host matches the 'scope_host' but does not match the 'host' patterns // or the host does not have any contexts return 0; // continue to next host struct rrdcontext_to_json_v2_data *ctl = data; if(ctl->window.enabled && !rrdhost_matches_window(host, ctl->window.after, ctl->window.before, ctl->now)) // the host does not have data in the requested window return 0; // continue to next host if(ctl->request->timeout_ms && now_monotonic_usec() > ctl->timings.received_ut + ctl->request->timeout_ms * USEC_PER_MS) // timed out return -2; // stop the query if(ctl->request->interrupt_callback && ctl->request->interrupt_callback(ctl->request->interrupt_callback_data)) // interrupted return -1; // stop the query bool host_matched = (ctl->mode & CONTEXTS_V2_NODES); bool do_contexts = (ctl->mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_ALERTS)); ctl->q.host_match = FTS_MATCHED_NONE; if((ctl->mode & CONTEXTS_V2_SEARCH)) { // check if we match the host itself if(ctl->q.pattern && ( full_text_search_string(&ctl->q.fts, ctl->q.pattern, host->hostname) || full_text_search_char(&ctl->q.fts, ctl->q.pattern, host->machine_guid) || (ctl->q.pattern && full_text_search_char(&ctl->q.fts, ctl->q.pattern, ctl->q.host_node_id_str)))) { ctl->q.host_match = FTS_MATCHED_HOST; do_contexts = true; } } if(do_contexts) { // save it SIMPLE_PATTERN *old_q = ctl->q.pattern; if(ctl->q.host_match == FTS_MATCHED_HOST) // do not do pattern matching on contexts - we matched the host itself ctl->q.pattern = NULL; ssize_t added = query_scope_foreach_context( host, ctl->request->scope_contexts, ctl->contexts.scope_pattern, ctl->contexts.pattern, rrdcontext_to_json_v2_add_context, queryable_host, ctl); // restore it ctl->q.pattern = old_q; if(unlikely(added < 0)) return -1; // stop the query if(added) host_matched = true; } if(!host_matched) return 0; if(ctl->mode & CONTEXTS_V2_FUNCTIONS) { struct function_v2_entry t = { .used = 1, .size = 1, .node_ids = &ctl->nodes.ni, .help = NULL, .tags = NULL, .access = HTTP_ACCESS_ALL, .priority = RRDFUNCTIONS_PRIORITY_DEFAULT, .version = RRDFUNCTIONS_VERSION_DEFAULT, }; host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help, &t.tags, &t.access, &t.priority, &t.version); } if(ctl->mode & CONTEXTS_V2_NODES) { struct contexts_v2_node t = { .ni = ctl->nodes.ni++, .host = host, }; dictionary_set(ctl->nodes.dict, host->machine_guid, &t, sizeof(struct contexts_v2_node)); } return 1; } static void buffer_json_contexts_v2_mode_to_array(BUFFER *wb, const char *key, CONTEXTS_V2_MODE mode) { buffer_json_member_add_array(wb, key); if(mode & CONTEXTS_V2_VERSIONS) buffer_json_add_array_item_string(wb, "versions"); if(mode & CONTEXTS_V2_AGENTS) buffer_json_add_array_item_string(wb, "agents"); if(mode & CONTEXTS_V2_AGENTS_INFO) buffer_json_add_array_item_string(wb, "agents-info"); if(mode & CONTEXTS_V2_NODES) buffer_json_add_array_item_string(wb, "nodes"); if(mode & CONTEXTS_V2_NODES_INFO) buffer_json_add_array_item_string(wb, "nodes-info"); if(mode & CONTEXTS_V2_NODE_INSTANCES) buffer_json_add_array_item_string(wb, "nodes-instances"); if(mode & CONTEXTS_V2_CONTEXTS) buffer_json_add_array_item_string(wb, "contexts"); if(mode & CONTEXTS_V2_SEARCH) buffer_json_add_array_item_string(wb, "search"); if(mode & CONTEXTS_V2_ALERTS) buffer_json_add_array_item_string(wb, "alerts"); if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) buffer_json_add_array_item_string(wb, "alert_transitions"); buffer_json_array_close(wb); } void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings) { timings->finished_ut = now_monotonic_usec(); if(!timings->executed_ut) timings->executed_ut = timings->finished_ut; if(!timings->preprocessed_ut) timings->preprocessed_ut = timings->received_ut; buffer_json_member_add_object(wb, key); buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(timings->preprocessed_ut - timings->received_ut) / USEC_PER_MS); buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(timings->executed_ut - timings->preprocessed_ut) / USEC_PER_MS); buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->executed_ut) / USEC_PER_MS); buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); buffer_json_member_add_double(wb, "cloud_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); buffer_json_object_close(wb); } void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings) { if(!timings->finished_ut) timings->finished_ut = now_monotonic_usec(); buffer_json_member_add_object(wb, key); buffer_json_member_add_double(wb, "routing_ms", 0.0); buffer_json_member_add_double(wb, "node_max_ms", 0.0); buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); buffer_json_object_close(wb); } static void functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct function_v2_entry *t = value; // it is initialized with a static reference - we need to mallocz() the array size_t *v = t->node_ids; t->node_ids = mallocz(sizeof(size_t)); *t->node_ids = *v; t->size = 1; t->used = 1; } static bool functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { struct function_v2_entry *t = old_value, *n = new_value; size_t *v = n->node_ids; if(t->used >= t->size) { t->node_ids = reallocz(t->node_ids, t->size * 2 * sizeof(size_t)); t->size *= 2; } t->node_ids[t->used++] = *v; return true; } static void functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct function_v2_entry *t = value; freez(t->node_ids); } static bool contexts_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { struct context_v2_entry *o = old_value; struct context_v2_entry *n = new_value; o->count++; if(o->family != n->family) { if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED)) // keep old ; else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) { // keep new string_freez(o->family); o->family = string_dup(n->family); } else { // merge STRING *old_family = o->family; o->family = string_2way_merge(o->family, n->family); string_freez(old_family); } } if(o->priority != n->priority) { if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED)) // keep o ; else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) // keep n o->priority = n->priority; else // keep the min o->priority = MIN(o->priority, n->priority); } if(o->first_time_s && n->first_time_s) o->first_time_s = MIN(o->first_time_s, n->first_time_s); else if(!o->first_time_s) o->first_time_s = n->first_time_s; if(o->last_time_s && n->last_time_s) o->last_time_s = MAX(o->last_time_s, n->last_time_s); else if(!o->last_time_s) o->last_time_s = n->last_time_s; o->flags |= n->flags; o->match = MIN(o->match, n->match); string_freez(n->family); return true; } static void contexts_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct context_v2_entry *z = value; string_freez(z->family); } int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_MODE mode) { int resp = HTTP_RESP_OK; bool run = true; if(mode & CONTEXTS_V2_SEARCH) mode |= CONTEXTS_V2_CONTEXTS; if(mode & (CONTEXTS_V2_AGENTS_INFO)) mode |= CONTEXTS_V2_AGENTS; if(mode & (CONTEXTS_V2_FUNCTIONS | CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES)) mode |= CONTEXTS_V2_NODES; if(mode & CONTEXTS_V2_ALERTS) { mode |= CONTEXTS_V2_NODES; req->options &= ~CONTEXTS_OPTION_ALERTS_WITH_CONFIGURATIONS; if(!(req->options & (CONTEXTS_OPTION_ALERTS_WITH_SUMMARY | CONTEXTS_OPTION_ALERTS_WITH_INSTANCES | CONTEXTS_OPTION_ALERTS_WITH_VALUES))) req->options |= CONTEXTS_OPTION_ALERTS_WITH_SUMMARY; } if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { mode |= CONTEXTS_V2_NODES; req->options &= ~CONTEXTS_OPTION_ALERTS_WITH_INSTANCES; } struct rrdcontext_to_json_v2_data ctl = { .wb = wb, .request = req, .mode = mode, .options = req->options, .versions = { 0 }, .nodes.scope_pattern = string_to_simple_pattern(req->scope_nodes), .nodes.pattern = string_to_simple_pattern(req->nodes), .contexts.pattern = string_to_simple_pattern(req->contexts), .contexts.scope_pattern = string_to_simple_pattern(req->scope_contexts), .q.pattern = string_to_simple_pattern_nocase(req->q), .alerts.alert_name_pattern = string_to_simple_pattern(req->alerts.alert), .window = { .enabled = false, .relative = false, .after = req->after, .before = req->before, }, .timings = { .received_ut = now_monotonic_usec(), } }; bool debug = ctl.options & CONTEXTS_OPTION_DEBUG; if(mode & CONTEXTS_V2_NODES) { ctl.nodes.dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct contexts_v2_node)); } if(mode & CONTEXTS_V2_CONTEXTS) { ctl.contexts.dict = dictionary_create_advanced( DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct context_v2_entry)); dictionary_register_conflict_callback(ctl.contexts.dict, contexts_conflict_callback, &ctl); dictionary_register_delete_callback(ctl.contexts.dict, contexts_delete_callback, &ctl); } if(mode & CONTEXTS_V2_FUNCTIONS) { ctl.functions.dict = dictionary_create_advanced( DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct function_v2_entry)); dictionary_register_insert_callback(ctl.functions.dict, functions_insert_callback, &ctl); dictionary_register_conflict_callback(ctl.functions.dict, functions_conflict_callback, &ctl); dictionary_register_delete_callback(ctl.functions.dict, functions_delete_callback, &ctl); } if(mode & CONTEXTS_V2_ALERTS) { if(!rrdcontexts_v2_init_alert_dictionaries(&ctl, req)) { resp = HTTP_RESP_NOT_FOUND; goto cleanup; } } if(req->after || req->before) { ctl.window.relative = rrdr_relative_window_to_absolute_query( &ctl.window.after, &ctl.window.before, &ctl.now, false); ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS); } else ctl.now = now_realtime_sec(); buffer_json_initialize(wb, "\"", "\"", 0, true, ((req->options & CONTEXTS_OPTION_MINIFY) && !(req->options & CONTEXTS_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); buffer_json_member_add_uint64(wb, "api", 2); if(req->options & CONTEXTS_OPTION_DEBUG) { buffer_json_member_add_object(wb, "request"); { buffer_json_contexts_v2_mode_to_array(wb, "mode", mode); contexts_options_to_buffer_json_array(wb, "options", req->options); buffer_json_member_add_object(wb, "scope"); { buffer_json_member_add_string(wb, "scope_nodes", req->scope_nodes); if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS)) buffer_json_member_add_string(wb, "scope_contexts", req->scope_contexts); } buffer_json_object_close(wb); buffer_json_member_add_object(wb, "selectors"); { buffer_json_member_add_string(wb, "nodes", req->nodes); if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS)) buffer_json_member_add_string(wb, "contexts", req->contexts); if(mode & (CONTEXTS_V2_ALERTS | CONTEXTS_V2_ALERT_TRANSITIONS)) { buffer_json_member_add_object(wb, "alerts"); if(mode & CONTEXTS_V2_ALERTS) contexts_alerts_status_to_buffer_json_array(wb, "status", req->alerts.status); if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { buffer_json_member_add_string(wb, "context", req->contexts); buffer_json_member_add_uint64(wb, "anchor_gi", req->alerts.global_id_anchor); buffer_json_member_add_uint64(wb, "last", req->alerts.last); } buffer_json_member_add_string(wb, "alert", req->alerts.alert); buffer_json_member_add_string(wb, "transition", req->alerts.transition); buffer_json_object_close(wb); // alerts } } buffer_json_object_close(wb); // selectors buffer_json_member_add_object(wb, "filters"); { if (mode & CONTEXTS_V2_SEARCH) buffer_json_member_add_string(wb, "q", req->q); buffer_json_member_add_time_t(wb, "after", req->after); buffer_json_member_add_time_t(wb, "before", req->before); } buffer_json_object_close(wb); // filters if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { buffer_json_member_add_object(wb, "facets"); { for (int i = 0; i < ATF_TOTAL_ENTRIES; i++) { buffer_json_member_add_string(wb, alert_transition_facets[i].query_param, req->alerts.facets[i]); } } buffer_json_object_close(wb); // facets } } buffer_json_object_close(wb); } ssize_t ret = 0; if(run) ret = query_scope_foreach_host(ctl.nodes.scope_pattern, ctl.nodes.pattern, rrdcontext_to_json_v2_add_host, &ctl, &ctl.versions, ctl.q.host_node_id_str); if(unlikely(ret < 0)) { buffer_flush(wb); if(ret == -2) { buffer_strcat(wb, "query timeout"); resp = HTTP_RESP_GATEWAY_TIMEOUT; } else { buffer_strcat(wb, "query interrupted"); resp = HTTP_RESP_CLIENT_CLOSED_REQUEST; } goto cleanup; } ctl.timings.executed_ut = now_monotonic_usec(); if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { contexts_v2_alert_transitions_to_json(wb, &ctl, debug); } else { if (mode & CONTEXTS_V2_NODES) { buffer_json_member_add_array(wb, "nodes"); struct contexts_v2_node *t; dfe_start_read(ctl.nodes.dict, t) { rrdcontext_to_json_v2_rrdhost(wb, t->host, &ctl, t->ni); } dfe_done(t); buffer_json_array_close(wb); } if (mode & CONTEXTS_V2_FUNCTIONS) { buffer_json_member_add_array(wb, "functions"); { struct function_v2_entry *t; dfe_start_read(ctl.functions.dict, t) { buffer_json_add_array_item_object(wb); { const char *name = t_dfe.name ? strstr(t_dfe.name, RRDFUNCTIONS_VERSION_SEPARATOR) : NULL; if(name) name += sizeof(RRDFUNCTIONS_VERSION_SEPARATOR) - 1; else name = t_dfe.name; buffer_json_member_add_string(wb, "name", name); buffer_json_member_add_string(wb, "help", string2str(t->help)); buffer_json_member_add_array(wb, "ni"); { for (size_t i = 0; i < t->used; i++) buffer_json_add_array_item_uint64(wb, t->node_ids[i]); } buffer_json_array_close(wb); buffer_json_member_add_string(wb, "tags", string2str(t->tags)); http_access2buffer_json_array(wb, "access", t->access); buffer_json_member_add_uint64(wb, "priority", t->priority); buffer_json_member_add_uint64(wb, "version", t->version); } buffer_json_object_close(wb); } dfe_done(t); } buffer_json_array_close(wb); } if (mode & CONTEXTS_V2_CONTEXTS) { buffer_json_member_add_object(wb, "contexts"); { struct context_v2_entry *z; dfe_start_read(ctl.contexts.dict, z) { bool collected = z->flags & RRD_FLAG_COLLECTED; buffer_json_member_add_object(wb, string2str(z->id)); { buffer_json_member_add_string(wb, "family", string2str(z->family)); buffer_json_member_add_uint64(wb, "priority", z->priority); buffer_json_member_add_time_t(wb, "first_entry", z->first_time_s); buffer_json_member_add_time_t(wb, "last_entry", collected ? ctl.now : z->last_time_s); buffer_json_member_add_boolean(wb, "live", collected); if (mode & CONTEXTS_V2_SEARCH) buffer_json_member_add_string(wb, "match", fts_match_to_string(z->match)); } buffer_json_object_close(wb); } dfe_done(z); } buffer_json_object_close(wb); // contexts } if (mode & CONTEXTS_V2_ALERTS) contexts_v2_alerts_to_json(wb, &ctl, debug); if (mode & CONTEXTS_V2_SEARCH) { buffer_json_member_add_object(wb, "searches"); { buffer_json_member_add_uint64(wb, "strings", ctl.q.fts.string_searches); buffer_json_member_add_uint64(wb, "char", ctl.q.fts.char_searches); buffer_json_member_add_uint64(wb, "total", ctl.q.fts.searches); } buffer_json_object_close(wb); } if (mode & (CONTEXTS_V2_VERSIONS)) version_hashes_api_v2(wb, &ctl.versions); if (mode & CONTEXTS_V2_AGENTS) buffer_json_agents_v2(wb, &ctl.timings, ctl.now, mode & (CONTEXTS_V2_AGENTS_INFO), true); } buffer_json_cloud_timings(wb, "timings", &ctl.timings); buffer_json_finalize(wb); cleanup: dictionary_destroy(ctl.nodes.dict); dictionary_destroy(ctl.contexts.dict); dictionary_destroy(ctl.functions.dict); rrdcontexts_v2_alerts_cleanup(&ctl); simple_pattern_free(ctl.nodes.scope_pattern); simple_pattern_free(ctl.nodes.pattern); simple_pattern_free(ctl.contexts.pattern); simple_pattern_free(ctl.contexts.scope_pattern); simple_pattern_free(ctl.q.pattern); simple_pattern_free(ctl.alerts.alert_name_pattern); return resp; }