diff options
Diffstat (limited to 'src/database/contexts')
| -rw-r--r-- | src/database/contexts/api_v1_contexts.c (renamed from src/database/contexts/api_v1.c) | 7 | ||||
| -rw-r--r-- | src/database/contexts/api_v2.c | 2454 | ||||
| -rw-r--r-- | src/database/contexts/api_v2_contexts.c | 1033 | ||||
| -rw-r--r-- | src/database/contexts/api_v2_contexts.h | 98 | ||||
| -rw-r--r-- | src/database/contexts/api_v2_contexts_agents.c | 162 | ||||
| -rw-r--r-- | src/database/contexts/api_v2_contexts_alert_config.c | 135 | ||||
| -rw-r--r-- | src/database/contexts/api_v2_contexts_alert_transitions.c | 487 | ||||
| -rw-r--r-- | src/database/contexts/api_v2_contexts_alerts.c | 604 | ||||
| -rw-r--r-- | src/database/contexts/api_v2_contexts_alerts.h | 52 | ||||
| -rw-r--r-- | src/database/contexts/instance.c | 18 | ||||
| -rw-r--r-- | src/database/contexts/internal.h | 40 | ||||
| -rw-r--r-- | src/database/contexts/query_scope.c | 4 | ||||
| -rw-r--r-- | src/database/contexts/query_target.c | 14 | ||||
| -rw-r--r-- | src/database/contexts/rrdcontext.c | 27 | ||||
| -rw-r--r-- | src/database/contexts/rrdcontext.h | 4 | ||||
| -rw-r--r-- | src/database/contexts/worker.c | 57 |
16 files changed, 2664 insertions, 2532 deletions
diff --git a/src/database/contexts/api_v1.c b/src/database/contexts/api_v1_contexts.c index 355aaf91a..1a1c83a00 100644 --- a/src/database/contexts/api_v1.c +++ b/src/database/contexts/api_v1_contexts.c @@ -399,8 +399,8 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, char node_uuid[UUID_STR_LEN] = ""; - if(host->node_id) - uuid_unparse(*host->node_id, node_uuid); + if(!UUIDiszero(host->node_id)) + uuid_unparse_lower(host->node_id.uuid, node_uuid); if(after != 0 && before != 0) rrdr_relative_window_to_absolute_query(&after, &before, NULL, false); @@ -409,7 +409,8 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host)); buffer_json_member_add_string(wb, "machine_guid", host->machine_guid); buffer_json_member_add_string(wb, "node_id", node_uuid); - buffer_json_member_add_string(wb, "claim_id", host->aclk_state.claimed_id ? host->aclk_state.claimed_id : ""); + CLAIM_ID claim_id = rrdhost_claim_id_get(host); + buffer_json_member_add_string(wb, "claim_id", claim_id.str); if(options & RRDCONTEXT_OPTION_SHOW_LABELS) { buffer_json_member_add_object(wb, "host_labels"); diff --git a/src/database/contexts/api_v2.c b/src/database/contexts/api_v2.c deleted file mode 100644 index 07cd3ac83..000000000 --- a/src/database/contexts/api_v2.c +++ /dev/null @@ -1,2454 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "internal.h" - -#include "aclk/aclk_capas.h" - -// ---------------------------------------------------------------------------- -// /api/v2/contexts API - -struct alert_transitions_facets alert_transition_facets[] = { - [ATF_STATUS] = { - .id = "f_status", - .name = "Alert Status", - .query_param = "f_status", - .order = 1, - }, - [ATF_TYPE] = { - .id = "f_type", - .name = "Alert Type", - .query_param = "f_type", - .order = 2, - }, - [ATF_ROLE] = { - .id = "f_role", - .name = "Recipient Role", - .query_param = "f_role", - .order = 3, - }, - [ATF_CLASS] = { - .id = "f_class", - .name = "Alert Class", - .query_param = "f_class", - .order = 4, - }, - [ATF_COMPONENT] = { - .id = "f_component", - .name = "Alert Component", - .query_param = "f_component", - .order = 5, - }, - [ATF_NODE] = { - .id = "f_node", - .name = "Alert Node", - .query_param = "f_node", - .order = 6, - }, - [ATF_ALERT_NAME] = { - .id = "f_alert", - .name = "Alert Name", - .query_param = "f_alert", - .order = 7, - }, - [ATF_CHART_NAME] = { - .id = "f_instance", - .name = "Instance Name", - .query_param = "f_instance", - .order = 8, - }, - [ATF_CONTEXT] = { - .id = "f_context", - .name = "Context", - .query_param = "f_context", - .order = 9, - }, - - // terminator - [ATF_TOTAL_ENTRIES] = { - .id = NULL, - .name = NULL, - .query_param = NULL, - .order = 9999, - } -}; - -struct facet_entry { - uint32_t count; -}; - -struct alert_transitions_callback_data { - struct rrdcontext_to_json_v2_data *ctl; - BUFFER *wb; - bool debug; - bool only_one_config; - - struct { - SIMPLE_PATTERN *pattern; - DICTIONARY *dict; - } facets[ATF_TOTAL_ENTRIES]; - - uint32_t max_items_to_return; - uint32_t items_to_return; - - uint32_t items_evaluated; - uint32_t items_matched; - - - struct sql_alert_transition_fixed_size *base; // double linked list - last item is base->prev - struct sql_alert_transition_fixed_size *last_added; // the last item added, not the last of the list - - struct { - size_t first; - size_t skips_before; - size_t skips_after; - size_t backwards; - size_t forwards; - size_t prepend; - size_t append; - size_t shifts; - } operations; - - uint32_t configs_added; -}; - -typedef enum __attribute__ ((__packed__)) { - FTS_MATCHED_NONE = 0, - FTS_MATCHED_HOST, - FTS_MATCHED_CONTEXT, - FTS_MATCHED_INSTANCE, - FTS_MATCHED_DIMENSION, - FTS_MATCHED_LABEL, - FTS_MATCHED_ALERT, - FTS_MATCHED_ALERT_INFO, - FTS_MATCHED_FAMILY, - FTS_MATCHED_TITLE, - FTS_MATCHED_UNITS, -} FTS_MATCH; - -static const char *fts_match_to_string(FTS_MATCH match) { - switch(match) { - case FTS_MATCHED_HOST: - return "HOST"; - - case FTS_MATCHED_CONTEXT: - return "CONTEXT"; - - case FTS_MATCHED_INSTANCE: - return "INSTANCE"; - - case FTS_MATCHED_DIMENSION: - return "DIMENSION"; - - case FTS_MATCHED_ALERT: - return "ALERT"; - - case FTS_MATCHED_ALERT_INFO: - return "ALERT_INFO"; - - case FTS_MATCHED_LABEL: - return "LABEL"; - - case FTS_MATCHED_FAMILY: - return "FAMILY"; - - case FTS_MATCHED_TITLE: - return "TITLE"; - - case FTS_MATCHED_UNITS: - return "UNITS"; - - default: - return "NONE"; - } -} - -struct function_v2_entry { - size_t size; - size_t used; - size_t *node_ids; - STRING *help; - STRING *tags; - HTTP_ACCESS access; - int priority; -}; - -struct context_v2_entry { - size_t count; - STRING *id; - STRING *family; - uint32_t priority; - time_t first_time_s; - time_t last_time_s; - RRD_FLAGS flags; - FTS_MATCH match; -}; - -struct alert_counts { - size_t critical; - size_t warning; - size_t clear; - size_t error; -}; - -struct alert_v2_entry { - RRDCALC *tmp; - - STRING *name; - STRING *summary; - RRDLABELS *recipient; - RRDLABELS *classification; - RRDLABELS *context; - RRDLABELS *component; - RRDLABELS *type; - - size_t ati; - - struct alert_counts counts; - - size_t instances; - DICTIONARY *nodes; - DICTIONARY *configs; -}; - -struct alert_by_x_entry { - struct { - struct alert_counts counts; - size_t silent; - size_t total; - } running; - - struct { - size_t available; - } prototypes; -}; - -typedef struct full_text_search_index { - size_t searches; - size_t string_searches; - size_t char_searches; -} FTS_INDEX; - -static inline bool full_text_search_string(FTS_INDEX *fts, SIMPLE_PATTERN *q, STRING *ptr) { - fts->searches++; - fts->string_searches++; - return simple_pattern_matches_string(q, ptr); -} - -static inline bool full_text_search_char(FTS_INDEX *fts, SIMPLE_PATTERN *q, char *ptr) { - fts->searches++; - fts->char_searches++; - return simple_pattern_matches(q, ptr); -} - -struct contexts_v2_node { - size_t ni; - RRDHOST *host; -}; - -struct rrdcontext_to_json_v2_data { - time_t now; - - BUFFER *wb; - struct api_v2_contexts_request *request; - - CONTEXTS_V2_MODE mode; - CONTEXTS_V2_OPTIONS options; - struct query_versions versions; - - struct { - SIMPLE_PATTERN *scope_pattern; - SIMPLE_PATTERN *pattern; - size_t ni; - DICTIONARY *dict; // the result set - } nodes; - - struct { - SIMPLE_PATTERN *scope_pattern; - SIMPLE_PATTERN *pattern; - size_t ci; - DICTIONARY *dict; // the result set - } contexts; - - struct { - SIMPLE_PATTERN *alert_name_pattern; - time_t alarm_id_filter; - - size_t ati; - - DICTIONARY *summary; - DICTIONARY *alert_instances; - - DICTIONARY *by_type; - DICTIONARY *by_component; - DICTIONARY *by_classification; - DICTIONARY *by_recipient; - DICTIONARY *by_module; - } alerts; - - struct { - FTS_MATCH host_match; - char host_node_id_str[UUID_STR_LEN]; - SIMPLE_PATTERN *pattern; - FTS_INDEX fts; - } q; - - struct { - DICTIONARY *dict; // the result set - } functions; - - struct { - bool enabled; - bool relative; - time_t after; - time_t before; - } window; - - struct query_timings timings; -}; - -static void alert_counts_add(struct alert_counts *t, RRDCALC *rc) { - switch(rc->status) { - case RRDCALC_STATUS_CRITICAL: - t->critical++; - break; - - case RRDCALC_STATUS_WARNING: - t->warning++; - break; - - case RRDCALC_STATUS_CLEAR: - t->clear++; - break; - - case RRDCALC_STATUS_REMOVED: - case RRDCALC_STATUS_UNINITIALIZED: - break; - - case RRDCALC_STATUS_UNDEFINED: - default: - if(!netdata_double_isnumber(rc->value)) - t->error++; - - break; - } -} - -static void alerts_v2_add(struct alert_v2_entry *t, RRDCALC *rc) { - t->instances++; - - alert_counts_add(&t->counts, rc); - - dictionary_set(t->nodes, rc->rrdset->rrdhost->machine_guid, NULL, 0); - - char key[UUID_STR_LEN + 1]; - uuid_unparse_lower(rc->config.hash_id, key); - dictionary_set(t->configs, key, NULL, 0); -} - -static void alerts_by_x_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { - static STRING *silent = NULL; - if(unlikely(!silent)) silent = string_strdupz("silent"); - - struct alert_by_x_entry *b = value; - RRDCALC *rc = data; - if(!rc) { - // prototype - b->prototypes.available++; - } - else { - alert_counts_add(&b->running.counts, rc); - - b->running.total++; - - if (rc->config.recipient == silent) - b->running.silent++; - } -} - -static bool alerts_by_x_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value __maybe_unused, void *data __maybe_unused) { - alerts_by_x_insert_callback(item, old_value, data); - return false; -} - -static void alerts_v2_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { - struct rrdcontext_to_json_v2_data *ctl = data; - struct alert_v2_entry *t = value; - RRDCALC *rc = t->tmp; - t->name = rc->config.name; - t->summary = rc->config.summary; // the original summary - t->context = rrdlabels_create(); - t->recipient = rrdlabels_create(); - t->classification = rrdlabels_create(); - t->component = rrdlabels_create(); - t->type = rrdlabels_create(); - if (string_strlen(rc->rrdset->context)) - rrdlabels_add(t->context, string2str(rc->rrdset->context), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.recipient)) - rrdlabels_add(t->recipient, string2str(rc->config.recipient), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.classification)) - rrdlabels_add(t->classification, string2str(rc->config.classification), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.component)) - rrdlabels_add(t->component, string2str(rc->config.component), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.type)) - rrdlabels_add(t->type, string2str(rc->config.type), "yes", RRDLABEL_SRC_AUTO); - t->ati = ctl->alerts.ati++; - - t->nodes = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_VALUE_LINK_DONT_CLONE|DICT_OPTION_NAME_LINK_DONT_CLONE); - t->configs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_VALUE_LINK_DONT_CLONE|DICT_OPTION_NAME_LINK_DONT_CLONE); - - alerts_v2_add(t, rc); -} - -static bool alerts_v2_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { - struct alert_v2_entry *t = old_value, *n = new_value; - RRDCALC *rc = n->tmp; - if (string_strlen(rc->rrdset->context)) - rrdlabels_add(t->context, string2str(rc->rrdset->context), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.recipient)) - rrdlabels_add(t->recipient, string2str(rc->config.recipient), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.classification)) - rrdlabels_add(t->classification, string2str(rc->config.classification), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.component)) - rrdlabels_add(t->component, string2str(rc->config.component), "yes", RRDLABEL_SRC_AUTO); - if (string_strlen(rc->config.type)) - rrdlabels_add(t->type, string2str(rc->config.type), "yes", RRDLABEL_SRC_AUTO); - alerts_v2_add(t, rc); - return true; -} - -static void alerts_v2_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { - struct alert_v2_entry *t = value; - - rrdlabels_destroy(t->context); - rrdlabels_destroy(t->recipient); - rrdlabels_destroy(t->classification); - rrdlabels_destroy(t->component); - rrdlabels_destroy(t->type); - - dictionary_destroy(t->nodes); - dictionary_destroy(t->configs); -} - -static void alert_instances_v2_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { - struct rrdcontext_to_json_v2_data *ctl = data; - struct sql_alert_instance_v2_entry *t = value; - RRDCALC *rc = t->tmp; - - t->context = rc->rrdset->context; - t->chart_id = rc->rrdset->id; - t->chart_name = rc->rrdset->name; - t->family = rc->rrdset->family; - t->units = rc->config.units; - t->classification = rc->config.classification; - t->type = rc->config.type; - t->recipient = rc->config.recipient; - t->component = rc->config.component; - t->name = rc->config.name; - t->source = rc->config.source; - t->status = rc->status; - t->flags = rc->run_flags; - t->info = rc->config.info; - t->summary = rc->summary; - t->value = rc->value; - t->last_updated = rc->last_updated; - t->last_status_change = rc->last_status_change; - t->last_status_change_value = rc->last_status_change_value; - t->host = rc->rrdset->rrdhost; - t->alarm_id = rc->id; - t->ni = ctl->nodes.ni; - - uuid_copy(t->config_hash_id, rc->config.hash_id); - health_alarm_log_get_global_id_and_transition_id_for_rrdcalc(rc, &t->global_id, &t->last_transition_id); -} - -static bool alert_instances_v2_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value __maybe_unused, void *new_value __maybe_unused, void *data __maybe_unused) { - internal_fatal(true, "This should never happen!"); - return true; -} - -static void alert_instances_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) { - ; -} - -static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc, SIMPLE_PATTERN *q) { - if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->id) || - full_text_search_string(&ctl->q.fts, q, rc->family))) - return FTS_MATCHED_CONTEXT; - - if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->title))) - return FTS_MATCHED_TITLE; - - if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->units))) - return FTS_MATCHED_UNITS; - - FTS_MATCH matched = FTS_MATCHED_NONE; - RRDINSTANCE *ri; - dfe_start_read(rc->rrdinstances, ri) { - if(matched) break; - - if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, ri->first_time_s, (ri->flags & RRD_FLAG_COLLECTED) ? ctl->now : ri->last_time_s, 0)) - continue; - - if(unlikely(full_text_search_string(&ctl->q.fts, q, ri->id)) || - (ri->name != ri->id && full_text_search_string(&ctl->q.fts, q, ri->name))) { - matched = FTS_MATCHED_INSTANCE; - break; - } - - RRDMETRIC *rm; - dfe_start_read(ri->rrdmetrics, rm) { - if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rm->first_time_s, (rm->flags & RRD_FLAG_COLLECTED) ? ctl->now : rm->last_time_s, 0)) - continue; - - if(unlikely(full_text_search_string(&ctl->q.fts, q, rm->id)) || - (rm->name != rm->id && full_text_search_string(&ctl->q.fts, q, rm->name))) { - matched = FTS_MATCHED_DIMENSION; - break; - } - } - dfe_done(rm); - - size_t label_searches = 0; - if(unlikely(ri->rrdlabels && rrdlabels_entries(ri->rrdlabels) && - rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, q, ':', &label_searches) == SP_MATCHED_POSITIVE)) { - ctl->q.fts.searches += label_searches; - ctl->q.fts.char_searches += label_searches; - matched = FTS_MATCHED_LABEL; - break; - } - ctl->q.fts.searches += label_searches; - ctl->q.fts.char_searches += label_searches; - - if(ri->rrdset) { - RRDSET *st = ri->rrdset; - rw_spinlock_read_lock(&st->alerts.spinlock); - for (RRDCALC *rcl = st->alerts.base; rcl; rcl = rcl->next) { - if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.name))) { - matched = FTS_MATCHED_ALERT; - break; - } - - if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.info))) { - matched = FTS_MATCHED_ALERT_INFO; - break; - } - } - rw_spinlock_read_unlock(&st->alerts.spinlock); - } - } - dfe_done(ri); - return matched; -} - -static bool rrdcontext_matches_alert(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc) { - size_t matches = 0; - RRDINSTANCE *ri; - dfe_start_read(rc->rrdinstances, ri) { - if(ri->rrdset) { - RRDSET *st = ri->rrdset; - rw_spinlock_read_lock(&st->alerts.spinlock); - for (RRDCALC *rcl = st->alerts.base; rcl; rcl = rcl->next) { - if(ctl->alerts.alert_name_pattern && !simple_pattern_matches_string(ctl->alerts.alert_name_pattern, rcl->config.name)) - continue; - - if(ctl->alerts.alarm_id_filter && ctl->alerts.alarm_id_filter != rcl->id) - continue; - - size_t m = ctl->request->alerts.status & CONTEXTS_V2_ALERT_STATUSES ? 0 : 1; - - if (!m) { - if ((ctl->request->alerts.status & CONTEXT_V2_ALERT_UNINITIALIZED) && - rcl->status == RRDCALC_STATUS_UNINITIALIZED) - m++; - - if ((ctl->request->alerts.status & CONTEXT_V2_ALERT_UNDEFINED) && - rcl->status == RRDCALC_STATUS_UNDEFINED) - m++; - - if ((ctl->request->alerts.status & CONTEXT_V2_ALERT_CLEAR) && - rcl->status == RRDCALC_STATUS_CLEAR) - m++; - - if ((ctl->request->alerts.status & CONTEXT_V2_ALERT_RAISED) && - rcl->status >= RRDCALC_STATUS_RAISED) - m++; - - if ((ctl->request->alerts.status & CONTEXT_V2_ALERT_WARNING) && - rcl->status == RRDCALC_STATUS_WARNING) - m++; - - if ((ctl->request->alerts.status & CONTEXT_V2_ALERT_CRITICAL) && - rcl->status == RRDCALC_STATUS_CRITICAL) - m++; - - if(!m) - continue; - } - - struct alert_v2_entry t = { - .tmp = rcl, - }; - struct alert_v2_entry *a2e = - dictionary_set(ctl->alerts.summary, string2str(rcl->config.name), - &t, sizeof(struct alert_v2_entry)); - size_t ati = a2e->ati; - matches++; - - dictionary_set_advanced(ctl->alerts.by_type, - string2str(rcl->config.type), - (ssize_t)string_strlen(rcl->config.type), - NULL, - sizeof(struct alert_by_x_entry), - rcl); - - dictionary_set_advanced(ctl->alerts.by_component, - string2str(rcl->config.component), - (ssize_t)string_strlen(rcl->config.component), - NULL, - sizeof(struct alert_by_x_entry), - rcl); - - dictionary_set_advanced(ctl->alerts.by_classification, - string2str(rcl->config.classification), - (ssize_t)string_strlen(rcl->config.classification), - NULL, - sizeof(struct alert_by_x_entry), - rcl); - - dictionary_set_advanced(ctl->alerts.by_recipient, - string2str(rcl->config.recipient), - (ssize_t)string_strlen(rcl->config.recipient), - NULL, - sizeof(struct alert_by_x_entry), - rcl); - - char *module = NULL; - rrdlabels_get_value_strdup_or_null(st->rrdlabels, &module, "_collect_module"); - if(!module || !*module) module = "[unset]"; - - dictionary_set_advanced(ctl->alerts.by_module, - module, - -1, - NULL, - sizeof(struct alert_by_x_entry), - rcl); - - if (ctl->options & (CONTEXT_V2_OPTION_ALERTS_WITH_INSTANCES | CONTEXT_V2_OPTION_ALERTS_WITH_VALUES)) { - char key[20 + 1]; - snprintfz(key, sizeof(key) - 1, "%p", rcl); - - struct sql_alert_instance_v2_entry z = { - .ati = ati, - .tmp = rcl, - }; - dictionary_set(ctl->alerts.alert_instances, key, &z, sizeof(z)); - } - } - rw_spinlock_read_unlock(&st->alerts.spinlock); - } - } - dfe_done(ri); - - return matches != 0; -} - - -static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context __maybe_unused) { - struct rrdcontext_to_json_v2_data *ctl = data; - - RRDCONTEXT *rc = rrdcontext_acquired_value(rca); - - if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rc->first_time_s, (rc->flags & RRD_FLAG_COLLECTED) ? ctl->now : rc->last_time_s, 0)) - return 0; // continue to next context - - FTS_MATCH match = ctl->q.host_match; - if((ctl->mode & CONTEXTS_V2_SEARCH) && ctl->q.pattern) { - match = rrdcontext_to_json_v2_full_text_search(ctl, rc, ctl->q.pattern); - - if(match == FTS_MATCHED_NONE) - return 0; // continue to next context - } - - if(ctl->mode & CONTEXTS_V2_ALERTS) { - if(!rrdcontext_matches_alert(ctl, rc)) - return 0; // continue to next context - } - - if(ctl->contexts.dict) { - struct context_v2_entry t = { - .count = 1, - .id = rc->id, - .family = string_dup(rc->family), - .priority = rc->priority, - .first_time_s = rc->first_time_s, - .last_time_s = rc->last_time_s, - .flags = rc->flags, - .match = match, - }; - - dictionary_set(ctl->contexts.dict, string2str(rc->id), &t, sizeof(struct context_v2_entry)); - } - - return 1; -} - -void buffer_json_agent_status_id(BUFFER *wb, size_t ai, usec_t duration_ut) { - buffer_json_member_add_object(wb, "st"); - { - buffer_json_member_add_uint64(wb, "ai", ai); - buffer_json_member_add_uint64(wb, "code", 200); - buffer_json_member_add_string(wb, "msg", ""); - if (duration_ut) - buffer_json_member_add_double(wb, "ms", (NETDATA_DOUBLE) duration_ut / 1000.0); - } - buffer_json_object_close(wb); -} - -void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut, bool status) { - buffer_json_member_add_string(wb, "mg", host->machine_guid); - - if(host->node_id) - buffer_json_member_add_uuid(wb, "nd", host->node_id); - buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host)); - buffer_json_member_add_uint64(wb, "ni", ni); - - if(status) - buffer_json_agent_status_id(wb, 0, duration_ut); -} - -static void rrdhost_receiver_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) { - buffer_json_member_add_object(wb, key); - { - buffer_json_member_add_uint64(wb, "id", s->ingest.id); - buffer_json_member_add_uint64(wb, "hops", s->ingest.hops); - buffer_json_member_add_string(wb, "type", rrdhost_ingest_type_to_string(s->ingest.type)); - buffer_json_member_add_string(wb, "status", rrdhost_ingest_status_to_string(s->ingest.status)); - buffer_json_member_add_time_t(wb, "since", s->ingest.since); - buffer_json_member_add_time_t(wb, "age", s->now - s->ingest.since); - - if(s->ingest.type == RRDHOST_INGEST_TYPE_CHILD) { - if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE) - buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->ingest.reason)); - - if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING) { - buffer_json_member_add_object(wb, "replication"); - { - buffer_json_member_add_boolean(wb, "in_progress", s->ingest.replication.in_progress); - buffer_json_member_add_double(wb, "completion", s->ingest.replication.completion); - buffer_json_member_add_uint64(wb, "instances", s->ingest.replication.instances); - } - buffer_json_object_close(wb); // replication - } - - if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING || s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE) { - buffer_json_member_add_object(wb, "source"); - { - char buf[1024 + 1]; - snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.local.ip, s->ingest.peers.local.port, s->ingest.ssl ? ":SSL" : ""); - buffer_json_member_add_string(wb, "local", buf); - - snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.peer.ip, s->ingest.peers.peer.port, s->ingest.ssl ? ":SSL" : ""); - buffer_json_member_add_string(wb, "remote", buf); - - stream_capabilities_to_json_array(wb, s->ingest.capabilities, "capabilities"); - } - buffer_json_object_close(wb); // source - } - } - } - buffer_json_object_close(wb); // collection -} - -static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) { - if(s->stream.status == RRDHOST_STREAM_STATUS_DISABLED) - return; - - buffer_json_member_add_object(wb, key); - { - buffer_json_member_add_uint64(wb, "id", s->stream.id); - buffer_json_member_add_uint64(wb, "hops", s->stream.hops); - buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status)); - buffer_json_member_add_time_t(wb, "since", s->stream.since); - buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since); - - if (s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE) - buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->stream.reason)); - - if (s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) { - buffer_json_member_add_object(wb, "replication"); - { - buffer_json_member_add_boolean(wb, "in_progress", s->stream.replication.in_progress); - buffer_json_member_add_double(wb, "completion", s->stream.replication.completion); - buffer_json_member_add_uint64(wb, "instances", s->stream.replication.instances); - } - buffer_json_object_close(wb); - } - - buffer_json_member_add_object(wb, "destination"); - { - char buf[1024 + 1]; - snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.local.ip, s->stream.peers.local.port, s->stream.ssl ? ":SSL" : ""); - buffer_json_member_add_string(wb, "local", buf); - - snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.peer.ip, s->stream.peers.peer.port, s->stream.ssl ? ":SSL" : ""); - buffer_json_member_add_string(wb, "remote", buf); - - stream_capabilities_to_json_array(wb, s->stream.capabilities, "capabilities"); - - buffer_json_member_add_object(wb, "traffic"); - { - buffer_json_member_add_boolean(wb, "compression", s->stream.compression); - buffer_json_member_add_uint64(wb, "data", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA]); - buffer_json_member_add_uint64(wb, "metadata", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA]); - buffer_json_member_add_uint64(wb, "functions", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS]); - buffer_json_member_add_uint64(wb, "replication", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION]); - buffer_json_member_add_uint64(wb, "dyncfg", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DYNCFG]); - } - buffer_json_object_close(wb); // traffic - - buffer_json_member_add_array(wb, "candidates"); - struct rrdpush_destinations *d; - for (d = s->host->destinations; d; d = d->next) { - buffer_json_add_array_item_object(wb); - buffer_json_member_add_uint64(wb, "attempts", d->attempts); - { - - if (d->ssl) { - snprintfz(buf, sizeof(buf) - 1, "%s:SSL", string2str(d->destination)); - buffer_json_member_add_string(wb, "destination", buf); - } - else - buffer_json_member_add_string(wb, "destination", string2str(d->destination)); - - buffer_json_member_add_time_t(wb, "since", d->since); - buffer_json_member_add_time_t(wb, "age", s->now - d->since); - buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->reason)); - if(d->postpone_reconnection_until > s->now) { - buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until); - buffer_json_member_add_time_t(wb, "next_in", d->postpone_reconnection_until - s->now); - } - } - buffer_json_object_close(wb); // each candidate - } - buffer_json_array_close(wb); // candidates - } - buffer_json_object_close(wb); // destination - } - buffer_json_object_close(wb); // streaming -} - -static void agent_capabilities_to_json(BUFFER *wb, RRDHOST *host, const char *key) { - buffer_json_member_add_array(wb, key); - - struct capability *capas = aclk_get_node_instance_capas(host); - for(struct capability *capa = capas; capa->name ;capa++) { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", capa->name); - buffer_json_member_add_uint64(wb, "version", capa->version); - buffer_json_member_add_boolean(wb, "enabled", capa->enabled); - } - buffer_json_object_close(wb); - } - buffer_json_array_close(wb); - freez(capas); -} - -static inline void host_dyncfg_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) { - buffer_json_member_add_object(wb, key); - { - buffer_json_member_add_string(wb, "status", rrdhost_dyncfg_status_to_string(s->dyncfg.status)); - } - buffer_json_object_close(wb); // health - -} - -static inline void rrdhost_health_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) { - buffer_json_member_add_object(wb, key); - { - buffer_json_member_add_string(wb, "status", rrdhost_health_status_to_string(s->health.status)); - if (s->health.status == RRDHOST_HEALTH_STATUS_RUNNING) { - buffer_json_member_add_object(wb, "alerts"); - { - buffer_json_member_add_uint64(wb, "critical", s->health.alerts.critical); - buffer_json_member_add_uint64(wb, "warning", s->health.alerts.warning); - buffer_json_member_add_uint64(wb, "clear", s->health.alerts.clear); - buffer_json_member_add_uint64(wb, "undefined", s->health.alerts.undefined); - buffer_json_member_add_uint64(wb, "uninitialized", s->health.alerts.uninitialized); - } - buffer_json_object_close(wb); // alerts - } - } - buffer_json_object_close(wb); // health -} - -static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdcontext_to_json_v2_data *ctl, size_t node_id) { - buffer_json_add_array_item_object(wb); // this node - buffer_json_node_add_v2(wb, host, node_id, 0, - (ctl->mode & CONTEXTS_V2_AGENTS) && !(ctl->mode & CONTEXTS_V2_NODE_INSTANCES)); - - if(ctl->mode & (CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES)) { - RRDHOST_STATUS s; - rrdhost_status(host, ctl->now, &s); - - if (ctl->mode & (CONTEXTS_V2_NODES_INFO)) { - buffer_json_member_add_string(wb, "v", rrdhost_program_version(host)); - - host_labels2json(host, wb, "labels"); - - if (host->system_info) { - buffer_json_member_add_object(wb, "hw"); - { - buffer_json_member_add_string_or_empty(wb, "architecture", host->system_info->architecture); - buffer_json_member_add_string_or_empty(wb, "cpu_frequency", host->system_info->host_cpu_freq); - buffer_json_member_add_string_or_empty(wb, "cpus", host->system_info->host_cores); - buffer_json_member_add_string_or_empty(wb, "memory", host->system_info->host_ram_total); - buffer_json_member_add_string_or_empty(wb, "disk_space", host->system_info->host_disk_space); - buffer_json_member_add_string_or_empty(wb, "virtualization", host->system_info->virtualization); - buffer_json_member_add_string_or_empty(wb, "container", host->system_info->container); - } - buffer_json_object_close(wb); - - buffer_json_member_add_object(wb, "os"); - { - buffer_json_member_add_string_or_empty(wb, "id", host->system_info->host_os_id); - buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->host_os_name); - buffer_json_member_add_string_or_empty(wb, "v", host->system_info->host_os_version); - buffer_json_member_add_object(wb, "kernel"); - buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->kernel_name); - buffer_json_member_add_string_or_empty(wb, "v", host->system_info->kernel_version); - buffer_json_object_close(wb); - } - buffer_json_object_close(wb); - } - - // created - the node is created but never connected to cloud - // unreachable - not currently connected - // stale - connected but not having live data - // reachable - connected with live data - // pruned - not connected for some time and has been removed - buffer_json_member_add_string(wb, "state", rrdhost_state_cloud_emulation(host) ? "reachable" : "stale"); - - rrdhost_health_to_json_v2(wb, "health", &s); - agent_capabilities_to_json(wb, host, "capabilities"); - } - - if (ctl->mode & (CONTEXTS_V2_NODE_INSTANCES)) { - buffer_json_member_add_array(wb, "instances"); - buffer_json_add_array_item_object(wb); // this instance - { - buffer_json_agent_status_id(wb, 0, 0); - - buffer_json_member_add_object(wb, "db"); - { - buffer_json_member_add_string(wb, "status", rrdhost_db_status_to_string(s.db.status)); - buffer_json_member_add_string(wb, "liveness", rrdhost_db_liveness_to_string(s.db.liveness)); - buffer_json_member_add_string(wb, "mode", rrd_memory_mode_name(s.db.mode)); - buffer_json_member_add_time_t(wb, "first_time", s.db.first_time_s); - buffer_json_member_add_time_t(wb, "last_time", s.db.last_time_s); - buffer_json_member_add_uint64(wb, "metrics", s.db.metrics); - buffer_json_member_add_uint64(wb, "instances", s.db.instances); - buffer_json_member_add_uint64(wb, "contexts", s.db.contexts); - } - buffer_json_object_close(wb); - - rrdhost_receiver_to_json(wb, &s, "ingest"); - rrdhost_sender_to_json(wb, &s, "stream"); - - buffer_json_member_add_object(wb, "ml"); - buffer_json_member_add_string(wb, "status", rrdhost_ml_status_to_string(s.ml.status)); - buffer_json_member_add_string(wb, "type", rrdhost_ml_type_to_string(s.ml.type)); - if (s.ml.status == RRDHOST_ML_STATUS_RUNNING) { - buffer_json_member_add_object(wb, "metrics"); - { - buffer_json_member_add_uint64(wb, "anomalous", s.ml.metrics.anomalous); - buffer_json_member_add_uint64(wb, "normal", s.ml.metrics.normal); - buffer_json_member_add_uint64(wb, "trained", s.ml.metrics.trained); - buffer_json_member_add_uint64(wb, "pending", s.ml.metrics.pending); - buffer_json_member_add_uint64(wb, "silenced", s.ml.metrics.silenced); - } - buffer_json_object_close(wb); // metrics - } - buffer_json_object_close(wb); // ml - - rrdhost_health_to_json_v2(wb, "health", &s); - - host_functions2json(host, wb); // functions - agent_capabilities_to_json(wb, host, "capabilities"); - - host_dyncfg_to_json_v2(wb, "dyncfg", &s); - } - buffer_json_object_close(wb); // this instance - buffer_json_array_close(wb); // instances - } - } - buffer_json_object_close(wb); // this node -} - -static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) { - if(!queryable_host || !host->rrdctx.contexts) - // the host matches the 'scope_host' but does not match the 'host' patterns - // or the host does not have any contexts - return 0; // continue to next host - - struct rrdcontext_to_json_v2_data *ctl = data; - - if(ctl->window.enabled && !rrdhost_matches_window(host, ctl->window.after, ctl->window.before, ctl->now)) - // the host does not have data in the requested window - return 0; // continue to next host - - if(ctl->request->timeout_ms && now_monotonic_usec() > ctl->timings.received_ut + ctl->request->timeout_ms * USEC_PER_MS) - // timed out - return -2; // stop the query - - if(ctl->request->interrupt_callback && ctl->request->interrupt_callback(ctl->request->interrupt_callback_data)) - // interrupted - return -1; // stop the query - - bool host_matched = (ctl->mode & CONTEXTS_V2_NODES); - bool do_contexts = (ctl->mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_ALERTS)); - - ctl->q.host_match = FTS_MATCHED_NONE; - if((ctl->mode & CONTEXTS_V2_SEARCH)) { - // check if we match the host itself - if(ctl->q.pattern && ( - full_text_search_string(&ctl->q.fts, ctl->q.pattern, host->hostname) || - full_text_search_char(&ctl->q.fts, ctl->q.pattern, host->machine_guid) || - (ctl->q.pattern && full_text_search_char(&ctl->q.fts, ctl->q.pattern, ctl->q.host_node_id_str)))) { - ctl->q.host_match = FTS_MATCHED_HOST; - do_contexts = true; - } - } - - if(do_contexts) { - // save it - SIMPLE_PATTERN *old_q = ctl->q.pattern; - - if(ctl->q.host_match == FTS_MATCHED_HOST) - // do not do pattern matching on contexts - we matched the host itself - ctl->q.pattern = NULL; - - ssize_t added = query_scope_foreach_context( - host, ctl->request->scope_contexts, - ctl->contexts.scope_pattern, ctl->contexts.pattern, - rrdcontext_to_json_v2_add_context, queryable_host, ctl); - - // restore it - ctl->q.pattern = old_q; - - if(unlikely(added < 0)) - return -1; // stop the query - - if(added) - host_matched = true; - } - - if(!host_matched) - return 0; - - if(ctl->mode & CONTEXTS_V2_FUNCTIONS) { - struct function_v2_entry t = { - .used = 1, - .size = 1, - .node_ids = &ctl->nodes.ni, - .help = NULL, - .tags = NULL, - .access = HTTP_ACCESS_ALL, - .priority = RRDFUNCTIONS_PRIORITY_DEFAULT, - }; - host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help, &t.tags, &t.access, &t.priority); - } - - if(ctl->mode & CONTEXTS_V2_NODES) { - struct contexts_v2_node t = { - .ni = ctl->nodes.ni++, - .host = host, - }; - - dictionary_set(ctl->nodes.dict, host->machine_guid, &t, sizeof(struct contexts_v2_node)); - } - - return 1; -} - -static void buffer_json_contexts_v2_mode_to_array(BUFFER *wb, const char *key, CONTEXTS_V2_MODE mode) { - buffer_json_member_add_array(wb, key); - - if(mode & CONTEXTS_V2_VERSIONS) - buffer_json_add_array_item_string(wb, "versions"); - - if(mode & CONTEXTS_V2_AGENTS) - buffer_json_add_array_item_string(wb, "agents"); - - if(mode & CONTEXTS_V2_AGENTS_INFO) - buffer_json_add_array_item_string(wb, "agents-info"); - - if(mode & CONTEXTS_V2_NODES) - buffer_json_add_array_item_string(wb, "nodes"); - - if(mode & CONTEXTS_V2_NODES_INFO) - buffer_json_add_array_item_string(wb, "nodes-info"); - - if(mode & CONTEXTS_V2_NODE_INSTANCES) - buffer_json_add_array_item_string(wb, "nodes-instances"); - - if(mode & CONTEXTS_V2_CONTEXTS) - buffer_json_add_array_item_string(wb, "contexts"); - - if(mode & CONTEXTS_V2_SEARCH) - buffer_json_add_array_item_string(wb, "search"); - - if(mode & CONTEXTS_V2_ALERTS) - buffer_json_add_array_item_string(wb, "alerts"); - - if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) - buffer_json_add_array_item_string(wb, "alert_transitions"); - - buffer_json_array_close(wb); -} - -void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings) { - timings->finished_ut = now_monotonic_usec(); - if(!timings->executed_ut) - timings->executed_ut = timings->finished_ut; - if(!timings->preprocessed_ut) - timings->preprocessed_ut = timings->received_ut; - buffer_json_member_add_object(wb, key); - buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(timings->preprocessed_ut - timings->received_ut) / USEC_PER_MS); - buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(timings->executed_ut - timings->preprocessed_ut) / USEC_PER_MS); - buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->executed_ut) / USEC_PER_MS); - buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); - buffer_json_member_add_double(wb, "cloud_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); - buffer_json_object_close(wb); -} - -void build_info_to_json_object(BUFFER *b); - -static void convert_seconds_to_dhms(time_t seconds, char *result, int result_size) { - int days, hours, minutes; - - days = (int) (seconds / (24 * 3600)); - seconds = (int) (seconds % (24 * 3600)); - hours = (int) (seconds / 3600); - seconds %= 3600; - minutes = (int) (seconds / 60); - seconds %= 60; - - // Format the result into the provided string buffer - BUFFER *buf = buffer_create(128, NULL); - if (days) - buffer_sprintf(buf,"%d day%s%s", days, days==1 ? "" : "s", hours || minutes ? ", " : ""); - if (hours) - buffer_sprintf(buf,"%d hour%s%s", hours, hours==1 ? "" : "s", minutes ? ", " : ""); - if (minutes) - buffer_sprintf(buf,"%d minute%s%s", minutes, minutes==1 ? "" : "s", seconds ? ", " : ""); - if (seconds) - buffer_sprintf(buf,"%d second%s", (int) seconds, seconds==1 ? "" : "s"); - strncpyz(result, buffer_tostring(buf), result_size); - buffer_free(buf); -} - -void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now_s, bool info, bool array) { - if(!now_s) - now_s = now_realtime_sec(); - - if(array) { - buffer_json_member_add_array(wb, "agents"); - buffer_json_add_array_item_object(wb); - } - else - buffer_json_member_add_object(wb, "agent"); - - buffer_json_member_add_string(wb, "mg", localhost->machine_guid); - buffer_json_member_add_uuid(wb, "nd", localhost->node_id); - buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost)); - buffer_json_member_add_time_t(wb, "now", now_s); - - if(array) - buffer_json_member_add_uint64(wb, "ai", 0); - - if(info) { - buffer_json_member_add_object(wb, "application"); - build_info_to_json_object(wb); - buffer_json_object_close(wb); // netdata - - buffer_json_cloud_status(wb, now_s); - - buffer_json_member_add_array(wb, "db_size"); - size_t group_seconds = localhost->rrd_update_every; - for (size_t tier = 0; tier < storage_tiers; tier++) { - STORAGE_ENGINE *eng = localhost->db[tier].eng; - if (!eng) continue; - - group_seconds *= storage_tiers_grouping_iterations[tier]; - uint64_t max = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si); - uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si); -#ifdef ENABLE_DBENGINE - if (!max && eng->seb == STORAGE_ENGINE_BACKEND_DBENGINE) { - max = get_directory_free_bytes_space(multidb_ctx[tier]); - max += used; - } -#endif - time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si); - size_t currently_collected_metrics = storage_engine_collected_metrics(eng->seb, localhost->db[tier].si); - - NETDATA_DOUBLE percent; - if (used && max) - percent = (NETDATA_DOUBLE) used * 100.0 / (NETDATA_DOUBLE) max; - else - percent = 0.0; - - buffer_json_add_array_item_object(wb); - buffer_json_member_add_uint64(wb, "tier", tier); - char human_retention[128]; - convert_seconds_to_dhms((time_t) group_seconds, human_retention, sizeof(human_retention) - 1); - buffer_json_member_add_string(wb, "point_every", human_retention); - - buffer_json_member_add_uint64(wb, "metrics", storage_engine_metrics(eng->seb, localhost->db[tier].si)); - buffer_json_member_add_uint64(wb, "samples", storage_engine_samples(eng->seb, localhost->db[tier].si)); - - if(used || max) { - buffer_json_member_add_uint64(wb, "disk_used", used); - buffer_json_member_add_uint64(wb, "disk_max", max); - buffer_json_member_add_double(wb, "disk_percent", percent); - } - - if(first_time_s) { - time_t retention = now_s - first_time_s; - - buffer_json_member_add_time_t(wb, "from", first_time_s); - buffer_json_member_add_time_t(wb, "to", now_s); - buffer_json_member_add_time_t(wb, "retention", retention); - - convert_seconds_to_dhms(retention, human_retention, sizeof(human_retention) - 1); - buffer_json_member_add_string(wb, "retention_human", human_retention); - - if(used || max) { // we have disk space information - time_t time_retention = 0; -#ifdef ENABLE_DBENGINE - time_retention = multidb_ctx[tier]->config.max_retention_s; -#endif - time_t space_retention = (time_t)((NETDATA_DOUBLE)(now_s - first_time_s) * 100.0 / percent); - time_t actual_retention = MIN(space_retention, time_retention ? time_retention : space_retention); - - if (time_retention) { - convert_seconds_to_dhms(time_retention, human_retention, sizeof(human_retention) - 1); - buffer_json_member_add_time_t(wb, "requested_retention", time_retention); - buffer_json_member_add_string(wb, "requested_retention_human", human_retention); - } - - convert_seconds_to_dhms(actual_retention, human_retention, sizeof(human_retention) - 1); - buffer_json_member_add_time_t(wb, "expected_retention", actual_retention); - buffer_json_member_add_string(wb, "expected_retention_human", human_retention); - } - } - - if(currently_collected_metrics) - buffer_json_member_add_uint64(wb, "currently_collected_metrics", currently_collected_metrics); - - buffer_json_object_close(wb); - } - buffer_json_array_close(wb); // db_size - } - - if(timings) - buffer_json_query_timings(wb, "timings", timings); - - buffer_json_object_close(wb); - - if(array) - buffer_json_array_close(wb); -} - -void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings) { - if(!timings->finished_ut) - timings->finished_ut = now_monotonic_usec(); - - buffer_json_member_add_object(wb, key); - buffer_json_member_add_double(wb, "routing_ms", 0.0); - buffer_json_member_add_double(wb, "node_max_ms", 0.0); - buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); - buffer_json_object_close(wb); -} - -static void functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { - struct function_v2_entry *t = value; - - // it is initialized with a static reference - we need to mallocz() the array - size_t *v = t->node_ids; - t->node_ids = mallocz(sizeof(size_t)); - *t->node_ids = *v; - t->size = 1; - t->used = 1; -} - -static bool functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { - struct function_v2_entry *t = old_value, *n = new_value; - size_t *v = n->node_ids; - - if(t->used >= t->size) { - t->node_ids = reallocz(t->node_ids, t->size * 2 * sizeof(size_t)); - t->size *= 2; - } - - t->node_ids[t->used++] = *v; - - return true; -} - -static void functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { - struct function_v2_entry *t = value; - freez(t->node_ids); -} - -static bool contexts_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { - struct context_v2_entry *o = old_value; - struct context_v2_entry *n = new_value; - - o->count++; - - if(o->family != n->family) { - if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED)) - // keep old - ; - else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) { - // keep new - string_freez(o->family); - o->family = string_dup(n->family); - } - else { - // merge - STRING *old_family = o->family; - o->family = string_2way_merge(o->family, n->family); - string_freez(old_family); - } - } - - if(o->priority != n->priority) { - if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED)) - // keep o - ; - else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) - // keep n - o->priority = n->priority; - else - // keep the min - o->priority = MIN(o->priority, n->priority); - } - - if(o->first_time_s && n->first_time_s) - o->first_time_s = MIN(o->first_time_s, n->first_time_s); - else if(!o->first_time_s) - o->first_time_s = n->first_time_s; - - if(o->last_time_s && n->last_time_s) - o->last_time_s = MAX(o->last_time_s, n->last_time_s); - else if(!o->last_time_s) - o->last_time_s = n->last_time_s; - - o->flags |= n->flags; - o->match = MIN(o->match, n->match); - - string_freez(n->family); - - return true; -} - -static void contexts_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { - struct context_v2_entry *z = value; - string_freez(z->family); -} - -static void rrdcontext_v2_set_transition_filter(const char *machine_guid, const char *context, time_t alarm_id, void *data) { - struct rrdcontext_to_json_v2_data *ctl = data; - - if(machine_guid && *machine_guid) { - if(ctl->nodes.scope_pattern) - simple_pattern_free(ctl->nodes.scope_pattern); - - if(ctl->nodes.pattern) - simple_pattern_free(ctl->nodes.pattern); - - ctl->nodes.scope_pattern = string_to_simple_pattern(machine_guid); - ctl->nodes.pattern = NULL; - } - - if(context && *context) { - if(ctl->contexts.scope_pattern) - simple_pattern_free(ctl->contexts.scope_pattern); - - if(ctl->contexts.pattern) - simple_pattern_free(ctl->contexts.pattern); - - ctl->contexts.scope_pattern = string_to_simple_pattern(context); - ctl->contexts.pattern = NULL; - } - - ctl->alerts.alarm_id_filter = alarm_id; -} - -struct alert_instances_callback_data { - BUFFER *wb; - struct rrdcontext_to_json_v2_data *ctl; - bool debug; -}; - -static void contexts_v2_alert_config_to_json_from_sql_alert_config_data(struct sql_alert_config_data *t, void *data) { - struct alert_transitions_callback_data *d = data; - BUFFER *wb = d->wb; - bool debug = d->debug; - d->configs_added++; - - if(d->only_one_config) - buffer_json_add_array_item_object(wb); // alert config - - { - buffer_json_member_add_string(wb, "name", t->name); - buffer_json_member_add_uuid(wb, "config_hash_id", t->config_hash_id); - - buffer_json_member_add_object(wb, "selectors"); - { - bool is_template = t->selectors.on_template && *t->selectors.on_template ? true : false; - buffer_json_member_add_string(wb, "type", is_template ? "template" : "alarm"); - buffer_json_member_add_string(wb, "on", is_template ? t->selectors.on_template : t->selectors.on_key); - - buffer_json_member_add_string(wb, "families", t->selectors.families); - buffer_json_member_add_string(wb, "host_labels", t->selectors.host_labels); - buffer_json_member_add_string(wb, "chart_labels", t->selectors.chart_labels); - } - buffer_json_object_close(wb); // selectors - - buffer_json_member_add_object(wb, "value"); // value - { - // buffer_json_member_add_string(wb, "every", t->value.every); // does not exist in Netdata Cloud - buffer_json_member_add_string(wb, "units", t->value.units); - buffer_json_member_add_uint64(wb, "update_every", t->value.update_every); - - if (t->value.db.after || debug) { - buffer_json_member_add_object(wb, "db"); - { - // buffer_json_member_add_string(wb, "lookup", t->value.db.lookup); // does not exist in Netdata Cloud - - buffer_json_member_add_time_t(wb, "after", t->value.db.after); - buffer_json_member_add_time_t(wb, "before", t->value.db.before); - buffer_json_member_add_string(wb, "time_group_condition", alerts_group_conditions_id2txt(t->value.db.time_group_condition)); - buffer_json_member_add_double(wb, "time_group_value", t->value.db.time_group_value); - buffer_json_member_add_string(wb, "dims_group", alerts_dims_grouping_id2group(t->value.db.dims_group)); - buffer_json_member_add_string(wb, "data_source", alerts_data_source_id2source(t->value.db.data_source)); - buffer_json_member_add_string(wb, "method", t->value.db.method); - buffer_json_member_add_string(wb, "dimensions", t->value.db.dimensions); - rrdr_options_to_buffer_json_array(wb, "options", (RRDR_OPTIONS)t->value.db.options); - } - buffer_json_object_close(wb); // db - } - - if (t->value.calc || debug) - buffer_json_member_add_string(wb, "calc", t->value.calc); - } - buffer_json_object_close(wb); // value - - if (t->status.warn || t->status.crit || debug) { - buffer_json_member_add_object(wb, "status"); // status - { - NETDATA_DOUBLE green = t->status.green ? str2ndd(t->status.green, NULL) : NAN; - NETDATA_DOUBLE red = t->status.red ? str2ndd(t->status.red, NULL) : NAN; - - if (!isnan(green) || debug) - buffer_json_member_add_double(wb, "green", green); - - if (!isnan(red) || debug) - buffer_json_member_add_double(wb, "red", red); - - if (t->status.warn || debug) - buffer_json_member_add_string(wb, "warn", t->status.warn); - - if (t->status.crit || debug) - buffer_json_member_add_string(wb, "crit", t->status.crit); - } - buffer_json_object_close(wb); // status - } - - buffer_json_member_add_object(wb, "notification"); - { - buffer_json_member_add_string(wb, "type", "agent"); - buffer_json_member_add_string(wb, "exec", t->notification.exec ? t->notification.exec : NULL); - buffer_json_member_add_string(wb, "to", t->notification.to_key ? t->notification.to_key : string2str(localhost->health.health_default_recipient)); - buffer_json_member_add_string(wb, "delay", t->notification.delay); - buffer_json_member_add_string(wb, "repeat", t->notification.repeat); - buffer_json_member_add_string(wb, "options", t->notification.options); - } - buffer_json_object_close(wb); // notification - - buffer_json_member_add_string(wb, "class", t->classification); - buffer_json_member_add_string(wb, "component", t->component); - buffer_json_member_add_string(wb, "type", t->type); - buffer_json_member_add_string(wb, "info", t->info); - buffer_json_member_add_string(wb, "summary", t->summary); - // buffer_json_member_add_string(wb, "source", t->source); // moved to alert instance - } - - if(d->only_one_config) - buffer_json_object_close(wb); -} - -int contexts_v2_alert_config_to_json(struct web_client *w, const char *config_hash_id) { - struct alert_transitions_callback_data data = { - .wb = w->response.data, - .debug = false, - .only_one_config = false, - }; - DICTIONARY *configs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - dictionary_set(configs, config_hash_id, NULL, 0); - - buffer_flush(w->response.data); - - buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); - - int added = sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, false); - buffer_json_finalize(w->response.data); - - int ret = HTTP_RESP_OK; - - if(added <= 0) { - buffer_flush(w->response.data); - w->response.data->content_type = CT_TEXT_PLAIN; - if(added < 0) { - buffer_strcat(w->response.data, "Failed to execute SQL query."); - ret = HTTP_RESP_INTERNAL_SERVER_ERROR; - } - else { - buffer_strcat(w->response.data, "Config is not found."); - ret = HTTP_RESP_NOT_FOUND; - } - } - - return ret; -} - -static int contexts_v2_alert_instance_to_json_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { - struct sql_alert_instance_v2_entry *t = value; - struct alert_instances_callback_data *d = data; - struct rrdcontext_to_json_v2_data *ctl = d->ctl; (void)ctl; - bool debug = d->debug; (void)debug; - BUFFER *wb = d->wb; - - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_uint64(wb, "ni", t->ni); - - buffer_json_member_add_string(wb, "nm", string2str(t->name)); - buffer_json_member_add_string(wb, "ch", string2str(t->chart_id)); - buffer_json_member_add_string(wb, "ch_n", string2str(t->chart_name)); - - if(ctl->request->options & CONTEXT_V2_OPTION_ALERTS_WITH_SUMMARY) - buffer_json_member_add_uint64(wb, "ati", t->ati); - - if(ctl->request->options & CONTEXT_V2_OPTION_ALERTS_WITH_INSTANCES) { - buffer_json_member_add_string(wb, "units", string2str(t->units)); - buffer_json_member_add_string(wb, "fami", string2str(t->family)); - buffer_json_member_add_string(wb, "info", string2str(t->info)); - buffer_json_member_add_string(wb, "sum", string2str(t->summary)); - buffer_json_member_add_string(wb, "ctx", string2str(t->context)); - buffer_json_member_add_string(wb, "st", rrdcalc_status2string(t->status)); - buffer_json_member_add_uuid(wb, "tr_i", &t->last_transition_id); - buffer_json_member_add_double(wb, "tr_v", t->last_status_change_value); - buffer_json_member_add_time_t(wb, "tr_t", t->last_status_change); - buffer_json_member_add_uuid(wb, "cfg", &t->config_hash_id); - buffer_json_member_add_string(wb, "src", string2str(t->source)); - - buffer_json_member_add_string(wb, "to", string2str(t->recipient)); - buffer_json_member_add_string(wb, "tp", string2str(t->type)); - buffer_json_member_add_string(wb, "cm", string2str(t->component)); - buffer_json_member_add_string(wb, "cl", string2str(t->classification)); - - // Agent specific fields - buffer_json_member_add_uint64(wb, "gi", t->global_id); - // rrdcalc_flags_to_json_array (wb, "flags", t->flags); - } - - if(ctl->request->options & CONTEXT_V2_OPTION_ALERTS_WITH_VALUES) { - // Netdata Cloud fetched these by querying the agents - buffer_json_member_add_double(wb, "v", t->value); - buffer_json_member_add_time_t(wb, "t", t->last_updated); - } - } - buffer_json_object_close(wb); // alert instance - - return 1; -} - -static void contexts_v2_alerts_by_x_update_prototypes(void *data, STRING *type, STRING *component, STRING *classification, STRING *recipient) { - struct rrdcontext_to_json_v2_data *ctl = data; - - dictionary_set_advanced(ctl->alerts.by_type, string2str(type), (ssize_t)string_strlen(type), NULL, sizeof(struct alert_by_x_entry), NULL); - dictionary_set_advanced(ctl->alerts.by_component, string2str(component), (ssize_t)string_strlen(component), NULL, sizeof(struct alert_by_x_entry), NULL); - dictionary_set_advanced(ctl->alerts.by_classification, string2str(classification), (ssize_t)string_strlen(classification), NULL, sizeof(struct alert_by_x_entry), NULL); - dictionary_set_advanced(ctl->alerts.by_recipient, string2str(recipient), (ssize_t)string_strlen(recipient), NULL, sizeof(struct alert_by_x_entry), NULL); -} - -static void contexts_v2_alerts_by_x_to_json(BUFFER *wb, DICTIONARY *dict, const char *key) { - buffer_json_member_add_array(wb, key); - { - struct alert_by_x_entry *b; - dfe_start_read(dict, b) { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", b_dfe.name); - buffer_json_member_add_uint64(wb, "cr", b->running.counts.critical); - buffer_json_member_add_uint64(wb, "wr", b->running.counts.warning); - buffer_json_member_add_uint64(wb, "cl", b->running.counts.clear); - buffer_json_member_add_uint64(wb, "er", b->running.counts.error); - buffer_json_member_add_uint64(wb, "running", b->running.total); - - buffer_json_member_add_uint64(wb, "running_silent", b->running.silent); - - if(b->prototypes.available) - buffer_json_member_add_uint64(wb, "available", b->prototypes.available); - } - buffer_json_object_close(wb); - } - dfe_done(b); - } - buffer_json_array_close(wb); -} - -static void contexts_v2_alert_instances_to_json(BUFFER *wb, const char *key, struct rrdcontext_to_json_v2_data *ctl, bool debug) { - buffer_json_member_add_array(wb, key); - { - struct alert_instances_callback_data data = { - .wb = wb, - .ctl = ctl, - .debug = debug, - }; - dictionary_walkthrough_rw(ctl->alerts.alert_instances, DICTIONARY_LOCK_READ, - contexts_v2_alert_instance_to_json_callback, &data); - } - buffer_json_array_close(wb); // alerts_instances -} - -static void contexts_v2_alerts_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_data *ctl, bool debug) { - if(ctl->request->options & CONTEXT_V2_OPTION_ALERTS_WITH_SUMMARY) { - buffer_json_member_add_array(wb, "alerts"); - { - struct alert_v2_entry *t; - dfe_start_read(ctl->alerts.summary, t) - { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_uint64(wb, "ati", t->ati); - - buffer_json_member_add_array(wb, "ni"); - void *host_guid; - dfe_start_read(t->nodes, host_guid) { - struct contexts_v2_node *cn = dictionary_get(ctl->nodes.dict,host_guid_dfe.name); - buffer_json_add_array_item_int64(wb, (int64_t) cn->ni); - } - dfe_done(host_guid); - buffer_json_array_close(wb); - - buffer_json_member_add_string(wb, "nm", string2str(t->name)); - buffer_json_member_add_string(wb, "sum", string2str(t->summary)); - - buffer_json_member_add_uint64(wb, "cr", t->counts.critical); - buffer_json_member_add_uint64(wb, "wr", t->counts.warning); - buffer_json_member_add_uint64(wb, "cl", t->counts.clear); - buffer_json_member_add_uint64(wb, "er", t->counts.error); - - buffer_json_member_add_uint64(wb, "in", t->instances); - buffer_json_member_add_uint64(wb, "nd", dictionary_entries(t->nodes)); - buffer_json_member_add_uint64(wb, "cfg", dictionary_entries(t->configs)); - - buffer_json_member_add_array(wb, "ctx"); - rrdlabels_key_to_buffer_array_item(t->context, wb); - buffer_json_array_close(wb); // ctx - - buffer_json_member_add_array(wb, "cls"); - rrdlabels_key_to_buffer_array_item(t->classification, wb); - buffer_json_array_close(wb); // classification - - - buffer_json_member_add_array(wb, "cp"); - rrdlabels_key_to_buffer_array_item(t->component, wb); - buffer_json_array_close(wb); // component - - buffer_json_member_add_array(wb, "ty"); - rrdlabels_key_to_buffer_array_item(t->type, wb); - buffer_json_array_close(wb); // type - - buffer_json_member_add_array(wb, "to"); - rrdlabels_key_to_buffer_array_item(t->recipient, wb); - buffer_json_array_close(wb); // recipient - } - buffer_json_object_close(wb); // alert name - } - dfe_done(t); - } - buffer_json_array_close(wb); // alerts - - health_prototype_metadata_foreach(ctl, contexts_v2_alerts_by_x_update_prototypes); - contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_type, "alerts_by_type"); - contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_component, "alerts_by_component"); - contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_classification, "alerts_by_classification"); - contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_recipient, "alerts_by_recipient"); - contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_module, "alerts_by_module"); - } - - if(ctl->request->options & (CONTEXT_V2_OPTION_ALERTS_WITH_INSTANCES|CONTEXT_V2_OPTION_ALERTS_WITH_VALUES)) { - contexts_v2_alert_instances_to_json(wb, "alert_instances", ctl, debug); - } -} - -#define SQL_TRANSITION_DATA_SMALL_STRING (6 * 8) -#define SQL_TRANSITION_DATA_MEDIUM_STRING (12 * 8) -#define SQL_TRANSITION_DATA_BIG_STRING 512 - -struct sql_alert_transition_fixed_size { - usec_t global_id; - nd_uuid_t transition_id; - nd_uuid_t host_id; - nd_uuid_t config_hash_id; - uint32_t alarm_id; - char alert_name[SQL_TRANSITION_DATA_SMALL_STRING]; - char chart[RRD_ID_LENGTH_MAX]; - char chart_name[RRD_ID_LENGTH_MAX]; - char chart_context[SQL_TRANSITION_DATA_MEDIUM_STRING]; - char family[SQL_TRANSITION_DATA_SMALL_STRING]; - char recipient[SQL_TRANSITION_DATA_MEDIUM_STRING]; - char units[SQL_TRANSITION_DATA_SMALL_STRING]; - char exec[SQL_TRANSITION_DATA_BIG_STRING]; - char info[SQL_TRANSITION_DATA_BIG_STRING]; - char summary[SQL_TRANSITION_DATA_BIG_STRING]; - char classification[SQL_TRANSITION_DATA_SMALL_STRING]; - char type[SQL_TRANSITION_DATA_SMALL_STRING]; - char component[SQL_TRANSITION_DATA_SMALL_STRING]; - time_t when_key; - time_t duration; - time_t non_clear_duration; - uint64_t flags; - time_t delay_up_to_timestamp; - time_t exec_run_timestamp; - int exec_code; - int new_status; - int old_status; - int delay; - time_t last_repeat; - NETDATA_DOUBLE new_value; - NETDATA_DOUBLE old_value; - - char machine_guid[UUID_STR_LEN]; - struct sql_alert_transition_fixed_size *next; - struct sql_alert_transition_fixed_size *prev; -}; - -static struct sql_alert_transition_fixed_size *contexts_v2_alert_transition_dup(struct sql_alert_transition_data *t, const char *machine_guid, struct sql_alert_transition_fixed_size *dst) { - struct sql_alert_transition_fixed_size *n = dst ? dst : mallocz(sizeof(*n)); - - n->global_id = t->global_id; - uuid_copy(n->transition_id, *t->transition_id); - uuid_copy(n->host_id, *t->host_id); - uuid_copy(n->config_hash_id, *t->config_hash_id); - n->alarm_id = t->alarm_id; - strncpyz(n->alert_name, t->alert_name ? t->alert_name : "", sizeof(n->alert_name) - 1); - strncpyz(n->chart, t->chart ? t->chart : "", sizeof(n->chart) - 1); - strncpyz(n->chart_name, t->chart_name ? t->chart_name : n->chart, sizeof(n->chart_name) - 1); - strncpyz(n->chart_context, t->chart_context ? t->chart_context : "", sizeof(n->chart_context) - 1); - strncpyz(n->family, t->family ? t->family : "", sizeof(n->family) - 1); - strncpyz(n->recipient, t->recipient ? t->recipient : "", sizeof(n->recipient) - 1); - strncpyz(n->units, t->units ? t->units : "", sizeof(n->units) - 1); - strncpyz(n->exec, t->exec ? t->exec : "", sizeof(n->exec) - 1); - strncpyz(n->info, t->info ? t->info : "", sizeof(n->info) - 1); - strncpyz(n->summary, t->summary ? t->summary : "", sizeof(n->summary) - 1); - strncpyz(n->classification, t->classification ? t->classification : "", sizeof(n->classification) - 1); - strncpyz(n->type, t->type ? t->type : "", sizeof(n->type) - 1); - strncpyz(n->component, t->component ? t->component : "", sizeof(n->component) - 1); - n->when_key = t->when_key; - n->duration = t->duration; - n->non_clear_duration = t->non_clear_duration; - n->flags = t->flags; - n->delay_up_to_timestamp = t->delay_up_to_timestamp; - n->exec_run_timestamp = t->exec_run_timestamp; - n->exec_code = t->exec_code; - n->new_status = t->new_status; - n->old_status = t->old_status; - n->delay = t->delay; - n->last_repeat = t->last_repeat; - n->new_value = t->new_value; - n->old_value = t->old_value; - - memcpy(n->machine_guid, machine_guid, sizeof(n->machine_guid)); - n->next = n->prev = NULL; - - return n; -} - -static void contexts_v2_alert_transition_free(struct sql_alert_transition_fixed_size *t) { - freez(t); -} - -static inline void contexts_v2_alert_transition_keep(struct alert_transitions_callback_data *d, struct sql_alert_transition_data *t, const char *machine_guid) { - d->items_matched++; - - if(unlikely(t->global_id <= d->ctl->request->alerts.global_id_anchor)) { - // this is in our past, we are not interested - d->operations.skips_before++; - return; - } - - if(unlikely(!d->base)) { - d->last_added = contexts_v2_alert_transition_dup(t, machine_guid, NULL); - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(d->base, d->last_added, prev, next); - d->items_to_return++; - d->operations.first++; - return; - } - - struct sql_alert_transition_fixed_size *last = d->last_added; - while(last->prev != d->base->prev && t->global_id > last->prev->global_id) { - last = last->prev; - d->operations.backwards++; - } - - while(last->next && t->global_id < last->next->global_id) { - last = last->next; - d->operations.forwards++; - } - - if(d->items_to_return >= d->max_items_to_return) { - if(last == d->base->prev && t->global_id < last->global_id) { - d->operations.skips_after++; - return; - } - } - - d->items_to_return++; - - if(t->global_id > last->global_id) { - if(d->items_to_return > d->max_items_to_return) { - d->items_to_return--; - d->operations.shifts++; - d->last_added = d->base->prev; - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(d->base, d->last_added, prev, next); - d->last_added = contexts_v2_alert_transition_dup(t, machine_guid, d->last_added); - } - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(d->base, d->last_added, prev, next); - d->operations.prepend++; - } - else { - d->last_added = contexts_v2_alert_transition_dup(t, machine_guid, NULL); - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(d->base, d->last_added, prev, next); - d->operations.append++; - } - - while(d->items_to_return > d->max_items_to_return) { - // we have to remove something - - struct sql_alert_transition_fixed_size *tmp = d->base->prev; - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(d->base, tmp, prev, next); - d->items_to_return--; - - if(unlikely(d->last_added == tmp)) - d->last_added = d->base; - - contexts_v2_alert_transition_free(tmp); - - d->operations.shifts++; - } -} - -static void contexts_v2_alert_transition_callback(struct sql_alert_transition_data *t, void *data) { - struct alert_transitions_callback_data *d = data; - d->items_evaluated++; - - char machine_guid[UUID_STR_LEN] = ""; - uuid_unparse_lower(*t->host_id, machine_guid); - - const char *facets[ATF_TOTAL_ENTRIES] = { - [ATF_STATUS] = rrdcalc_status2string(t->new_status), - [ATF_CLASS] = t->classification, - [ATF_TYPE] = t->type, - [ATF_COMPONENT] = t->component, - [ATF_ROLE] = t->recipient && *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient), - [ATF_NODE] = machine_guid, - [ATF_ALERT_NAME] = t->alert_name, - [ATF_CHART_NAME] = t->chart_name, - [ATF_CONTEXT] = t->chart_context, - }; - - for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { - if (!facets[i] || !*facets[i]) facets[i] = "unknown"; - - struct facet_entry tmp = { - .count = 0, - }; - dictionary_set(d->facets[i].dict, facets[i], &tmp, sizeof(tmp)); - } - - bool selected[ATF_TOTAL_ENTRIES] = { 0 }; - - uint32_t selected_by = 0; - for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { - selected[i] = !d->facets[i].pattern || simple_pattern_matches(d->facets[i].pattern, facets[i]); - if(selected[i]) - selected_by++; - } - - if(selected_by == ATF_TOTAL_ENTRIES) { - // this item is selected by all facets - // put it in our result (if it fits) - contexts_v2_alert_transition_keep(d, t, machine_guid); - } - - if(selected_by >= ATF_TOTAL_ENTRIES - 1) { - // this item is selected by all, or all except one facet - // in both cases we need to add it to our counters - - for (size_t i = 0; i < ATF_TOTAL_ENTRIES; i++) { - uint32_t counted_by = selected_by; - - if (counted_by != ATF_TOTAL_ENTRIES) { - counted_by = 0; - for (size_t j = 0; j < ATF_TOTAL_ENTRIES; j++) { - if (i == j || selected[j]) - counted_by++; - } - } - - if (counted_by == ATF_TOTAL_ENTRIES) { - // we need to count it on this facet - struct facet_entry *x = dictionary_get(d->facets[i].dict, facets[i]); - internal_fatal(!x, "facet is not found"); - if(x) - x->count++; - } - } - } -} - -static void contexts_v2_alert_transitions_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_data *ctl, bool debug) { - struct alert_transitions_callback_data data = { - .wb = wb, - .ctl = ctl, - .debug = debug, - .only_one_config = true, - .max_items_to_return = ctl->request->alerts.last, - .items_to_return = 0, - .base = NULL, - }; - - for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { - data.facets[i].dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, sizeof(struct facet_entry)); - if(ctl->request->alerts.facets[i]) - data.facets[i].pattern = simple_pattern_create(ctl->request->alerts.facets[i], ",|", SIMPLE_PATTERN_EXACT, false); - } - - sql_alert_transitions( - ctl->nodes.dict, - ctl->window.after, - ctl->window.before, - ctl->request->contexts, - ctl->request->alerts.alert, - ctl->request->alerts.transition, - contexts_v2_alert_transition_callback, - &data, - debug); - - buffer_json_member_add_array(wb, "facets"); - for (size_t i = 0; i < ATF_TOTAL_ENTRIES; i++) { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "id", alert_transition_facets[i].id); - buffer_json_member_add_string(wb, "name", alert_transition_facets[i].name); - buffer_json_member_add_uint64(wb, "order", alert_transition_facets[i].order); - buffer_json_member_add_array(wb, "options"); - { - struct facet_entry *x; - dfe_start_read(data.facets[i].dict, x) { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "id", x_dfe.name); - if (i == ATF_NODE) { - RRDHOST *host = rrdhost_find_by_guid(x_dfe.name); - if (host) - buffer_json_member_add_string(wb, "name", rrdhost_hostname(host)); - else - buffer_json_member_add_string(wb, "name", x_dfe.name); - } else - buffer_json_member_add_string(wb, "name", x_dfe.name); - buffer_json_member_add_uint64(wb, "count", x->count); - } - buffer_json_object_close(wb); - } - dfe_done(x); - } - buffer_json_array_close(wb); // options - } - buffer_json_object_close(wb); // facet - } - buffer_json_array_close(wb); // facets - - buffer_json_member_add_array(wb, "transitions"); - for(struct sql_alert_transition_fixed_size *t = data.base; t ; t = t->next) { - buffer_json_add_array_item_object(wb); - { - RRDHOST *host = rrdhost_find_by_guid(t->machine_guid); - - buffer_json_member_add_uint64(wb, "gi", t->global_id); - buffer_json_member_add_uuid(wb, "transition_id", &t->transition_id); - buffer_json_member_add_uuid(wb, "config_hash_id", &t->config_hash_id); - buffer_json_member_add_string(wb, "machine_guid", t->machine_guid); - - if(host) { - buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host)); - - if(host->node_id) - buffer_json_member_add_uuid(wb, "node_id", host->node_id); - } - - buffer_json_member_add_string(wb, "alert", *t->alert_name ? t->alert_name : NULL); - buffer_json_member_add_string(wb, "instance", *t->chart ? t->chart : NULL); - buffer_json_member_add_string(wb, "instance_n", *t->chart_name ? t->chart_name : NULL); - buffer_json_member_add_string(wb, "context", *t->chart_context ? t->chart_context : NULL); - // buffer_json_member_add_string(wb, "family", *t->family ? t->family : NULL); - buffer_json_member_add_string(wb, "component", *t->component ? t->component : NULL); - buffer_json_member_add_string(wb, "classification", *t->classification ? t->classification : NULL); - buffer_json_member_add_string(wb, "type", *t->type ? t->type : NULL); - - buffer_json_member_add_time_t(wb, "when", t->when_key); - buffer_json_member_add_string(wb, "info", *t->info ? t->info : ""); - buffer_json_member_add_string(wb, "summary", *t->summary ? t->summary : ""); - buffer_json_member_add_string(wb, "units", *t->units ? t->units : NULL); - buffer_json_member_add_object(wb, "new"); - { - buffer_json_member_add_string(wb, "status", rrdcalc_status2string(t->new_status)); - buffer_json_member_add_double(wb, "value", t->new_value); - } - buffer_json_object_close(wb); // new - buffer_json_member_add_object(wb, "old"); - { - buffer_json_member_add_string(wb, "status", rrdcalc_status2string(t->old_status)); - buffer_json_member_add_double(wb, "value", t->old_value); - buffer_json_member_add_time_t(wb, "duration", t->duration); - buffer_json_member_add_time_t(wb, "raised_duration", t->non_clear_duration); - } - buffer_json_object_close(wb); // old - - buffer_json_member_add_object(wb, "notification"); - { - buffer_json_member_add_time_t(wb, "when", t->exec_run_timestamp); - buffer_json_member_add_time_t(wb, "delay", t->delay); - buffer_json_member_add_time_t(wb, "delay_up_to_time", t->delay_up_to_timestamp); - health_entry_flags_to_json_array(wb, "flags", t->flags); - buffer_json_member_add_string(wb, "exec", *t->exec ? t->exec : string2str(localhost->health.health_default_exec)); - buffer_json_member_add_uint64(wb, "exec_code", t->exec_code); - buffer_json_member_add_string(wb, "to", *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient)); - } - buffer_json_object_close(wb); // notification - } - buffer_json_object_close(wb); // a transition - } - buffer_json_array_close(wb); // all transitions - - if(ctl->options & CONTEXT_V2_OPTION_ALERTS_WITH_CONFIGURATIONS) { - DICTIONARY *configs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); - - for(struct sql_alert_transition_fixed_size *t = data.base; t ; t = t->next) { - char guid[UUID_STR_LEN]; - uuid_unparse_lower(t->config_hash_id, guid); - dictionary_set(configs, guid, NULL, 0); - } - - buffer_json_member_add_array(wb, "configurations"); - sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, debug); - buffer_json_array_close(wb); - - dictionary_destroy(configs); - } - - while(data.base) { - struct sql_alert_transition_fixed_size *t = data.base; - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(data.base, t, prev, next); - contexts_v2_alert_transition_free(t); - } - - for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { - dictionary_destroy(data.facets[i].dict); - simple_pattern_free(data.facets[i].pattern); - } - - buffer_json_member_add_object(wb, "items"); - { - // all the items in the window, under the scope_nodes, ignoring the facets (filters) - buffer_json_member_add_uint64(wb, "evaluated", data.items_evaluated); - - // all the items matching the query (if you didn't put anchor_gi and last, these are all the items you would get back) - buffer_json_member_add_uint64(wb, "matched", data.items_matched); - - // the items included in this response - buffer_json_member_add_uint64(wb, "returned", data.items_to_return); - - // same as last=X parameter - buffer_json_member_add_uint64(wb, "max_to_return", data.max_items_to_return); - - // items before the first returned, this should be 0 if anchor_gi is not set - buffer_json_member_add_uint64(wb, "before", data.operations.skips_before); - - // items after the last returned, when this is zero there aren't any items after the current list - buffer_json_member_add_uint64(wb, "after", data.operations.skips_after + data.operations.shifts); - } - buffer_json_object_close(wb); // items - - if(debug) { - buffer_json_member_add_object(wb, "stats"); - { - buffer_json_member_add_uint64(wb, "first", data.operations.first); - buffer_json_member_add_uint64(wb, "prepend", data.operations.prepend); - buffer_json_member_add_uint64(wb, "append", data.operations.append); - buffer_json_member_add_uint64(wb, "backwards", data.operations.backwards); - buffer_json_member_add_uint64(wb, "forwards", data.operations.forwards); - buffer_json_member_add_uint64(wb, "shifts", data.operations.shifts); - buffer_json_member_add_uint64(wb, "skips_before", data.operations.skips_before); - buffer_json_member_add_uint64(wb, "skips_after", data.operations.skips_after); - } - buffer_json_object_close(wb); - } -} - -int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_MODE mode) { - int resp = HTTP_RESP_OK; - bool run = true; - - if(mode & CONTEXTS_V2_SEARCH) - mode |= CONTEXTS_V2_CONTEXTS; - - if(mode & (CONTEXTS_V2_AGENTS_INFO)) - mode |= CONTEXTS_V2_AGENTS; - - if(mode & (CONTEXTS_V2_FUNCTIONS | CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES)) - mode |= CONTEXTS_V2_NODES; - - if(mode & CONTEXTS_V2_ALERTS) { - mode |= CONTEXTS_V2_NODES; - req->options &= ~CONTEXT_V2_OPTION_ALERTS_WITH_CONFIGURATIONS; - - if(!(req->options & (CONTEXT_V2_OPTION_ALERTS_WITH_SUMMARY|CONTEXT_V2_OPTION_ALERTS_WITH_INSTANCES|CONTEXT_V2_OPTION_ALERTS_WITH_VALUES))) - req->options |= CONTEXT_V2_OPTION_ALERTS_WITH_SUMMARY; - } - - if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { - mode |= CONTEXTS_V2_NODES; - req->options &= ~CONTEXT_V2_OPTION_ALERTS_WITH_INSTANCES; - } - - struct rrdcontext_to_json_v2_data ctl = { - .wb = wb, - .request = req, - .mode = mode, - .options = req->options, - .versions = { 0 }, - .nodes.scope_pattern = string_to_simple_pattern(req->scope_nodes), - .nodes.pattern = string_to_simple_pattern(req->nodes), - .contexts.pattern = string_to_simple_pattern(req->contexts), - .contexts.scope_pattern = string_to_simple_pattern(req->scope_contexts), - .q.pattern = string_to_simple_pattern_nocase(req->q), - .alerts.alert_name_pattern = string_to_simple_pattern(req->alerts.alert), - .window = { - .enabled = false, - .relative = false, - .after = req->after, - .before = req->before, - }, - .timings = { - .received_ut = now_monotonic_usec(), - } - }; - - bool debug = ctl.options & CONTEXT_V2_OPTION_DEBUG; - - if(mode & CONTEXTS_V2_NODES) { - ctl.nodes.dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct contexts_v2_node)); - } - - if(mode & CONTEXTS_V2_CONTEXTS) { - ctl.contexts.dict = dictionary_create_advanced( - DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, - sizeof(struct context_v2_entry)); - - dictionary_register_conflict_callback(ctl.contexts.dict, contexts_conflict_callback, &ctl); - dictionary_register_delete_callback(ctl.contexts.dict, contexts_delete_callback, &ctl); - } - - if(mode & CONTEXTS_V2_FUNCTIONS) { - ctl.functions.dict = dictionary_create_advanced( - DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, - sizeof(struct function_v2_entry)); - - dictionary_register_insert_callback(ctl.functions.dict, functions_insert_callback, &ctl); - dictionary_register_conflict_callback(ctl.functions.dict, functions_conflict_callback, &ctl); - dictionary_register_delete_callback(ctl.functions.dict, functions_delete_callback, &ctl); - } - - if(mode & CONTEXTS_V2_ALERTS) { - if(req->alerts.transition) { - ctl.options |= CONTEXT_V2_OPTION_ALERTS_WITH_INSTANCES|CONTEXT_V2_OPTION_ALERTS_WITH_VALUES; - run = sql_find_alert_transition(req->alerts.transition, rrdcontext_v2_set_transition_filter, &ctl); - if(!run) { - resp = HTTP_RESP_NOT_FOUND; - goto cleanup; - } - } - - ctl.alerts.summary = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct alert_v2_entry)); - - dictionary_register_insert_callback(ctl.alerts.summary, alerts_v2_insert_callback, &ctl); - dictionary_register_conflict_callback(ctl.alerts.summary, alerts_v2_conflict_callback, &ctl); - dictionary_register_delete_callback(ctl.alerts.summary, alerts_v2_delete_callback, &ctl); - - ctl.alerts.by_type = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct alert_by_x_entry)); - - dictionary_register_insert_callback(ctl.alerts.by_type, alerts_by_x_insert_callback, NULL); - dictionary_register_conflict_callback(ctl.alerts.by_type, alerts_by_x_conflict_callback, NULL); - - ctl.alerts.by_component = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct alert_by_x_entry)); - - dictionary_register_insert_callback(ctl.alerts.by_component, alerts_by_x_insert_callback, NULL); - dictionary_register_conflict_callback(ctl.alerts.by_component, alerts_by_x_conflict_callback, NULL); - - ctl.alerts.by_classification = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct alert_by_x_entry)); - - dictionary_register_insert_callback(ctl.alerts.by_classification, alerts_by_x_insert_callback, NULL); - dictionary_register_conflict_callback(ctl.alerts.by_classification, alerts_by_x_conflict_callback, NULL); - - ctl.alerts.by_recipient = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct alert_by_x_entry)); - - dictionary_register_insert_callback(ctl.alerts.by_recipient, alerts_by_x_insert_callback, NULL); - dictionary_register_conflict_callback(ctl.alerts.by_recipient, alerts_by_x_conflict_callback, NULL); - - ctl.alerts.by_module = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct alert_by_x_entry)); - - dictionary_register_insert_callback(ctl.alerts.by_module, alerts_by_x_insert_callback, NULL); - dictionary_register_conflict_callback(ctl.alerts.by_module, alerts_by_x_conflict_callback, NULL); - - if(ctl.options & (CONTEXT_V2_OPTION_ALERTS_WITH_INSTANCES | CONTEXT_V2_OPTION_ALERTS_WITH_VALUES)) { - ctl.alerts.alert_instances = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(struct sql_alert_instance_v2_entry)); - - dictionary_register_insert_callback(ctl.alerts.alert_instances, alert_instances_v2_insert_callback, &ctl); - dictionary_register_conflict_callback(ctl.alerts.alert_instances, alert_instances_v2_conflict_callback, &ctl); - dictionary_register_delete_callback(ctl.alerts.alert_instances, alert_instances_delete_callback, &ctl); - } - } - - if(req->after || req->before) { - ctl.window.relative = rrdr_relative_window_to_absolute_query(&ctl.window.after, &ctl.window.before, &ctl.now - , false - ); - ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS); - } - else - ctl.now = now_realtime_sec(); - - buffer_json_initialize(wb, "\"", "\"", 0, true, - ((req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); - - buffer_json_member_add_uint64(wb, "api", 2); - - if(req->options & CONTEXT_V2_OPTION_DEBUG) { - buffer_json_member_add_object(wb, "request"); - { - buffer_json_contexts_v2_mode_to_array(wb, "mode", mode); - web_client_api_request_v2_contexts_options_to_buffer_json_array(wb, "options", req->options); - - buffer_json_member_add_object(wb, "scope"); - { - buffer_json_member_add_string(wb, "scope_nodes", req->scope_nodes); - if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS)) - buffer_json_member_add_string(wb, "scope_contexts", req->scope_contexts); - } - buffer_json_object_close(wb); - - buffer_json_member_add_object(wb, "selectors"); - { - buffer_json_member_add_string(wb, "nodes", req->nodes); - - if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS)) - buffer_json_member_add_string(wb, "contexts", req->contexts); - - if(mode & (CONTEXTS_V2_ALERTS | CONTEXTS_V2_ALERT_TRANSITIONS)) { - buffer_json_member_add_object(wb, "alerts"); - - if(mode & CONTEXTS_V2_ALERTS) - web_client_api_request_v2_contexts_alerts_status_to_buffer_json_array(wb, "status", req->alerts.status); - - if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { - buffer_json_member_add_string(wb, "context", req->contexts); - buffer_json_member_add_uint64(wb, "anchor_gi", req->alerts.global_id_anchor); - buffer_json_member_add_uint64(wb, "last", req->alerts.last); - } - - buffer_json_member_add_string(wb, "alert", req->alerts.alert); - buffer_json_member_add_string(wb, "transition", req->alerts.transition); - buffer_json_object_close(wb); // alerts - } - } - buffer_json_object_close(wb); // selectors - - buffer_json_member_add_object(wb, "filters"); - { - if (mode & CONTEXTS_V2_SEARCH) - buffer_json_member_add_string(wb, "q", req->q); - - buffer_json_member_add_time_t(wb, "after", req->after); - buffer_json_member_add_time_t(wb, "before", req->before); - } - buffer_json_object_close(wb); // filters - - if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { - buffer_json_member_add_object(wb, "facets"); - { - for (int i = 0; i < ATF_TOTAL_ENTRIES; i++) { - buffer_json_member_add_string(wb, alert_transition_facets[i].query_param, req->alerts.facets[i]); - } - } - buffer_json_object_close(wb); // facets - } - } - buffer_json_object_close(wb); - } - - ssize_t ret = 0; - if(run) - ret = query_scope_foreach_host(ctl.nodes.scope_pattern, ctl.nodes.pattern, - rrdcontext_to_json_v2_add_host, &ctl, - &ctl.versions, ctl.q.host_node_id_str); - - if(unlikely(ret < 0)) { - buffer_flush(wb); - - if(ret == -2) { - buffer_strcat(wb, "query timeout"); - resp = HTTP_RESP_GATEWAY_TIMEOUT; - } - else { - buffer_strcat(wb, "query interrupted"); - resp = HTTP_RESP_CLIENT_CLOSED_REQUEST; - } - goto cleanup; - } - - ctl.timings.executed_ut = now_monotonic_usec(); - - if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { - contexts_v2_alert_transitions_to_json(wb, &ctl, debug); - } - else { - if (mode & CONTEXTS_V2_NODES) { - buffer_json_member_add_array(wb, "nodes"); - struct contexts_v2_node *t; - dfe_start_read(ctl.nodes.dict, t) { - rrdcontext_to_json_v2_rrdhost(wb, t->host, &ctl, t->ni); - } - dfe_done(t); - buffer_json_array_close(wb); - } - - if (mode & CONTEXTS_V2_FUNCTIONS) { - buffer_json_member_add_array(wb, "functions"); - { - struct function_v2_entry *t; - dfe_start_read(ctl.functions.dict, t) { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "name", t_dfe.name); - buffer_json_member_add_string(wb, "help", string2str(t->help)); - buffer_json_member_add_array(wb, "ni"); - { - for (size_t i = 0; i < t->used; i++) - buffer_json_add_array_item_uint64(wb, t->node_ids[i]); - } - buffer_json_array_close(wb); - buffer_json_member_add_string(wb, "tags", string2str(t->tags)); - http_access2buffer_json_array(wb, "access", t->access); - buffer_json_member_add_uint64(wb, "priority", t->priority); - } - buffer_json_object_close(wb); - } - dfe_done(t); - } - buffer_json_array_close(wb); - } - - if (mode & CONTEXTS_V2_CONTEXTS) { - buffer_json_member_add_object(wb, "contexts"); - { - struct context_v2_entry *z; - dfe_start_read(ctl.contexts.dict, z) { - bool collected = z->flags & RRD_FLAG_COLLECTED; - - buffer_json_member_add_object(wb, string2str(z->id)); - { - buffer_json_member_add_string(wb, "family", string2str(z->family)); - buffer_json_member_add_uint64(wb, "priority", z->priority); - buffer_json_member_add_time_t(wb, "first_entry", z->first_time_s); - buffer_json_member_add_time_t(wb, "last_entry", collected ? ctl.now : z->last_time_s); - buffer_json_member_add_boolean(wb, "live", collected); - if (mode & CONTEXTS_V2_SEARCH) - buffer_json_member_add_string(wb, "match", fts_match_to_string(z->match)); - } - buffer_json_object_close(wb); - } - dfe_done(z); - } - buffer_json_object_close(wb); // contexts - } - - if (mode & CONTEXTS_V2_ALERTS) - contexts_v2_alerts_to_json(wb, &ctl, debug); - - if (mode & CONTEXTS_V2_SEARCH) { - buffer_json_member_add_object(wb, "searches"); - { - buffer_json_member_add_uint64(wb, "strings", ctl.q.fts.string_searches); - buffer_json_member_add_uint64(wb, "char", ctl.q.fts.char_searches); - buffer_json_member_add_uint64(wb, "total", ctl.q.fts.searches); - } - buffer_json_object_close(wb); - } - - if (mode & (CONTEXTS_V2_VERSIONS)) - version_hashes_api_v2(wb, &ctl.versions); - - if (mode & CONTEXTS_V2_AGENTS) - buffer_json_agents_v2(wb, &ctl.timings, ctl.now, mode & (CONTEXTS_V2_AGENTS_INFO), true); - } - - buffer_json_cloud_timings(wb, "timings", &ctl.timings); - - buffer_json_finalize(wb); - -cleanup: - dictionary_destroy(ctl.nodes.dict); - dictionary_destroy(ctl.contexts.dict); - dictionary_destroy(ctl.functions.dict); - dictionary_destroy(ctl.alerts.summary); - dictionary_destroy(ctl.alerts.alert_instances); - dictionary_destroy(ctl.alerts.by_type); - dictionary_destroy(ctl.alerts.by_component); - dictionary_destroy(ctl.alerts.by_classification); - dictionary_destroy(ctl.alerts.by_recipient); - dictionary_destroy(ctl.alerts.by_module); - simple_pattern_free(ctl.nodes.scope_pattern); - simple_pattern_free(ctl.nodes.pattern); - simple_pattern_free(ctl.contexts.pattern); - simple_pattern_free(ctl.contexts.scope_pattern); - simple_pattern_free(ctl.q.pattern); - simple_pattern_free(ctl.alerts.alert_name_pattern); - - return resp; -} diff --git a/src/database/contexts/api_v2_contexts.c b/src/database/contexts/api_v2_contexts.c new file mode 100644 index 000000000..d8d945afb --- /dev/null +++ b/src/database/contexts/api_v2_contexts.c @@ -0,0 +1,1033 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "api_v2_contexts.h" + +#include "aclk/aclk_capas.h" + +// ---------------------------------------------------------------------------- +// /api/v2/contexts API + +static const char *fts_match_to_string(FTS_MATCH match) { + switch(match) { + case FTS_MATCHED_HOST: + return "HOST"; + + case FTS_MATCHED_CONTEXT: + return "CONTEXT"; + + case FTS_MATCHED_INSTANCE: + return "INSTANCE"; + + case FTS_MATCHED_DIMENSION: + return "DIMENSION"; + + case FTS_MATCHED_ALERT: + return "ALERT"; + + case FTS_MATCHED_ALERT_INFO: + return "ALERT_INFO"; + + case FTS_MATCHED_LABEL: + return "LABEL"; + + case FTS_MATCHED_FAMILY: + return "FAMILY"; + + case FTS_MATCHED_TITLE: + return "TITLE"; + + case FTS_MATCHED_UNITS: + return "UNITS"; + + default: + return "NONE"; + } +} + +struct function_v2_entry { + size_t size; + size_t used; + size_t *node_ids; + STRING *help; + STRING *tags; + HTTP_ACCESS access; + int priority; + uint32_t version; +}; + +struct context_v2_entry { + size_t count; + STRING *id; + STRING *family; + uint32_t priority; + time_t first_time_s; + time_t last_time_s; + RRD_FLAGS flags; + FTS_MATCH match; +}; + +static inline bool full_text_search_string(FTS_INDEX *fts, SIMPLE_PATTERN *q, STRING *ptr) { + fts->searches++; + fts->string_searches++; + return simple_pattern_matches_string(q, ptr); +} + +static inline bool full_text_search_char(FTS_INDEX *fts, SIMPLE_PATTERN *q, char *ptr) { + fts->searches++; + fts->char_searches++; + return simple_pattern_matches(q, ptr); +} + +static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc, SIMPLE_PATTERN *q) { + if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->id) || + full_text_search_string(&ctl->q.fts, q, rc->family))) + return FTS_MATCHED_CONTEXT; + + if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->title))) + return FTS_MATCHED_TITLE; + + if(unlikely(full_text_search_string(&ctl->q.fts, q, rc->units))) + return FTS_MATCHED_UNITS; + + FTS_MATCH matched = FTS_MATCHED_NONE; + RRDINSTANCE *ri; + dfe_start_read(rc->rrdinstances, ri) { + if(matched) break; + + if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, ri->first_time_s, (ri->flags & RRD_FLAG_COLLECTED) ? ctl->now : ri->last_time_s, 0)) + continue; + + if(unlikely(full_text_search_string(&ctl->q.fts, q, ri->id)) || + (ri->name != ri->id && full_text_search_string(&ctl->q.fts, q, ri->name))) { + matched = FTS_MATCHED_INSTANCE; + break; + } + + RRDMETRIC *rm; + dfe_start_read(ri->rrdmetrics, rm) { + if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rm->first_time_s, (rm->flags & RRD_FLAG_COLLECTED) ? ctl->now : rm->last_time_s, 0)) + continue; + + if(unlikely(full_text_search_string(&ctl->q.fts, q, rm->id)) || + (rm->name != rm->id && full_text_search_string(&ctl->q.fts, q, rm->name))) { + matched = FTS_MATCHED_DIMENSION; + break; + } + } + dfe_done(rm); + + size_t label_searches = 0; + if(unlikely(ri->rrdlabels && rrdlabels_entries(ri->rrdlabels) && + rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, q, ':', &label_searches) == SP_MATCHED_POSITIVE)) { + ctl->q.fts.searches += label_searches; + ctl->q.fts.char_searches += label_searches; + matched = FTS_MATCHED_LABEL; + break; + } + ctl->q.fts.searches += label_searches; + ctl->q.fts.char_searches += label_searches; + + if(ri->rrdset) { + RRDSET *st = ri->rrdset; + rw_spinlock_read_lock(&st->alerts.spinlock); + for (RRDCALC *rcl = st->alerts.base; rcl; rcl = rcl->next) { + if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.name))) { + matched = FTS_MATCHED_ALERT; + break; + } + + if(unlikely(full_text_search_string(&ctl->q.fts, q, rcl->config.info))) { + matched = FTS_MATCHED_ALERT_INFO; + break; + } + } + rw_spinlock_read_unlock(&st->alerts.spinlock); + } + } + dfe_done(ri); + return matched; +} + +static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context __maybe_unused) { + struct rrdcontext_to_json_v2_data *ctl = data; + + RRDCONTEXT *rc = rrdcontext_acquired_value(rca); + + if(ctl->window.enabled && !query_matches_retention(ctl->window.after, ctl->window.before, rc->first_time_s, (rc->flags & RRD_FLAG_COLLECTED) ? ctl->now : rc->last_time_s, 0)) + return 0; // continue to next context + + FTS_MATCH match = ctl->q.host_match; + if((ctl->mode & CONTEXTS_V2_SEARCH) && ctl->q.pattern) { + match = rrdcontext_to_json_v2_full_text_search(ctl, rc, ctl->q.pattern); + + if(match == FTS_MATCHED_NONE) + return 0; // continue to next context + } + + if(ctl->mode & CONTEXTS_V2_ALERTS) { + if(!rrdcontext_matches_alert(ctl, rc)) + return 0; // continue to next context + } + + if(ctl->contexts.dict) { + struct context_v2_entry t = { + .count = 1, + .id = rc->id, + .family = string_dup(rc->family), + .priority = rc->priority, + .first_time_s = rc->first_time_s, + .last_time_s = rc->last_time_s, + .flags = rc->flags, + .match = match, + }; + + dictionary_set(ctl->contexts.dict, string2str(rc->id), &t, sizeof(struct context_v2_entry)); + } + + return 1; +} + +void buffer_json_agent_status_id(BUFFER *wb, size_t ai, usec_t duration_ut) { + buffer_json_member_add_object(wb, "st"); + { + buffer_json_member_add_uint64(wb, "ai", ai); + buffer_json_member_add_uint64(wb, "code", 200); + buffer_json_member_add_string(wb, "msg", ""); + if (duration_ut) + buffer_json_member_add_double(wb, "ms", (NETDATA_DOUBLE) duration_ut / 1000.0); + } + buffer_json_object_close(wb); +} + +void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut, bool status) { + buffer_json_member_add_string(wb, "mg", host->machine_guid); + + if(!UUIDiszero(host->node_id)) + buffer_json_member_add_uuid(wb, "nd", host->node_id.uuid); + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host)); + buffer_json_member_add_uint64(wb, "ni", ni); + + if(status) + buffer_json_agent_status_id(wb, 0, duration_ut); +} + +static void rrdhost_receiver_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) { + buffer_json_member_add_object(wb, key); + { + buffer_json_member_add_uint64(wb, "id", s->ingest.id); + buffer_json_member_add_uint64(wb, "hops", s->ingest.hops); + buffer_json_member_add_string(wb, "type", rrdhost_ingest_type_to_string(s->ingest.type)); + buffer_json_member_add_string(wb, "status", rrdhost_ingest_status_to_string(s->ingest.status)); + buffer_json_member_add_time_t(wb, "since", s->ingest.since); + buffer_json_member_add_time_t(wb, "age", s->now - s->ingest.since); + + if(s->ingest.type == RRDHOST_INGEST_TYPE_CHILD) { + if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE) + buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->ingest.reason)); + + if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING) { + buffer_json_member_add_object(wb, "replication"); + { + buffer_json_member_add_boolean(wb, "in_progress", s->ingest.replication.in_progress); + buffer_json_member_add_double(wb, "completion", s->ingest.replication.completion); + buffer_json_member_add_uint64(wb, "instances", s->ingest.replication.instances); + } + buffer_json_object_close(wb); // replication + } + + if(s->ingest.status == RRDHOST_INGEST_STATUS_REPLICATING || s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE) { + buffer_json_member_add_object(wb, "source"); + { + char buf[1024 + 1]; + snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.local.ip, s->ingest.peers.local.port, s->ingest.ssl ? ":SSL" : ""); + buffer_json_member_add_string(wb, "local", buf); + + snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->ingest.peers.peer.ip, s->ingest.peers.peer.port, s->ingest.ssl ? ":SSL" : ""); + buffer_json_member_add_string(wb, "remote", buf); + + stream_capabilities_to_json_array(wb, s->ingest.capabilities, "capabilities"); + } + buffer_json_object_close(wb); // source + } + } + } + buffer_json_object_close(wb); // collection +} + +static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *key) { + if(s->stream.status == RRDHOST_STREAM_STATUS_DISABLED) + return; + + buffer_json_member_add_object(wb, key); + { + buffer_json_member_add_uint64(wb, "id", s->stream.id); + buffer_json_member_add_uint64(wb, "hops", s->stream.hops); + buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status)); + buffer_json_member_add_time_t(wb, "since", s->stream.since); + buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since); + + if (s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE) + buffer_json_member_add_string(wb, "reason", stream_handshake_error_to_string(s->stream.reason)); + + if (s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) { + buffer_json_member_add_object(wb, "replication"); + { + buffer_json_member_add_boolean(wb, "in_progress", s->stream.replication.in_progress); + buffer_json_member_add_double(wb, "completion", s->stream.replication.completion); + buffer_json_member_add_uint64(wb, "instances", s->stream.replication.instances); + } + buffer_json_object_close(wb); + } + + buffer_json_member_add_object(wb, "destination"); + { + char buf[1024 + 1]; + snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.local.ip, s->stream.peers.local.port, s->stream.ssl ? ":SSL" : ""); + buffer_json_member_add_string(wb, "local", buf); + + snprintfz(buf, sizeof(buf) - 1, "[%s]:%d%s", s->stream.peers.peer.ip, s->stream.peers.peer.port, s->stream.ssl ? ":SSL" : ""); + buffer_json_member_add_string(wb, "remote", buf); + + stream_capabilities_to_json_array(wb, s->stream.capabilities, "capabilities"); + + buffer_json_member_add_object(wb, "traffic"); + { + buffer_json_member_add_boolean(wb, "compression", s->stream.compression); + buffer_json_member_add_uint64(wb, "data", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA]); + buffer_json_member_add_uint64(wb, "metadata", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA]); + buffer_json_member_add_uint64(wb, "functions", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS]); + buffer_json_member_add_uint64(wb, "replication", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION]); + buffer_json_member_add_uint64(wb, "dyncfg", s->stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DYNCFG]); + } + buffer_json_object_close(wb); // traffic + + buffer_json_member_add_array(wb, "candidates"); + struct rrdpush_destinations *d; + for (d = s->host->destinations; d; d = d->next) { + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "attempts", d->attempts); + { + + if (d->ssl) { + snprintfz(buf, sizeof(buf) - 1, "%s:SSL", string2str(d->destination)); + buffer_json_member_add_string(wb, "destination", buf); + } + else + buffer_json_member_add_string(wb, "destination", string2str(d->destination)); + + buffer_json_member_add_time_t(wb, "since", d->since); + buffer_json_member_add_time_t(wb, "age", s->now - d->since); + buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->reason)); + if(d->postpone_reconnection_until > s->now) { + buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until); + buffer_json_member_add_time_t(wb, "next_in", d->postpone_reconnection_until - s->now); + } + } + buffer_json_object_close(wb); // each candidate + } + buffer_json_array_close(wb); // candidates + } + buffer_json_object_close(wb); // destination + } + buffer_json_object_close(wb); // streaming +} + +void agent_capabilities_to_json(BUFFER *wb, RRDHOST *host, const char *key) { + buffer_json_member_add_array(wb, key); + + struct capability *capas = aclk_get_node_instance_capas(host); + for(struct capability *capa = capas; capa->name ;capa++) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", capa->name); + buffer_json_member_add_uint64(wb, "version", capa->version); + buffer_json_member_add_boolean(wb, "enabled", capa->enabled); + } + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); + freez(capas); +} + +static inline void host_dyncfg_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) { + buffer_json_member_add_object(wb, key); + { + buffer_json_member_add_string(wb, "status", rrdhost_dyncfg_status_to_string(s->dyncfg.status)); + } + buffer_json_object_close(wb); // health + +} + +static inline void rrdhost_health_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) { + buffer_json_member_add_object(wb, key); + { + buffer_json_member_add_string(wb, "status", rrdhost_health_status_to_string(s->health.status)); + if (s->health.status == RRDHOST_HEALTH_STATUS_RUNNING) { + buffer_json_member_add_object(wb, "alerts"); + { + buffer_json_member_add_uint64(wb, "critical", s->health.alerts.critical); + buffer_json_member_add_uint64(wb, "warning", s->health.alerts.warning); + buffer_json_member_add_uint64(wb, "clear", s->health.alerts.clear); + buffer_json_member_add_uint64(wb, "undefined", s->health.alerts.undefined); + buffer_json_member_add_uint64(wb, "uninitialized", s->health.alerts.uninitialized); + } + buffer_json_object_close(wb); // alerts + } + } + buffer_json_object_close(wb); // health +} + +static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdcontext_to_json_v2_data *ctl, size_t node_id) { + buffer_json_add_array_item_object(wb); // this node + buffer_json_node_add_v2(wb, host, node_id, 0, + (ctl->mode & CONTEXTS_V2_AGENTS) && !(ctl->mode & CONTEXTS_V2_NODE_INSTANCES)); + + if(ctl->mode & (CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES)) { + RRDHOST_STATUS s; + rrdhost_status(host, ctl->now, &s); + + if (ctl->mode & (CONTEXTS_V2_NODES_INFO)) { + buffer_json_member_add_string(wb, "v", rrdhost_program_version(host)); + + host_labels2json(host, wb, "labels"); + + if (host->system_info) { + buffer_json_member_add_object(wb, "hw"); + { + buffer_json_member_add_string_or_empty(wb, "architecture", host->system_info->architecture); + buffer_json_member_add_string_or_empty(wb, "cpu_frequency", host->system_info->host_cpu_freq); + buffer_json_member_add_string_or_empty(wb, "cpus", host->system_info->host_cores); + buffer_json_member_add_string_or_empty(wb, "memory", host->system_info->host_ram_total); + buffer_json_member_add_string_or_empty(wb, "disk_space", host->system_info->host_disk_space); + buffer_json_member_add_string_or_empty(wb, "virtualization", host->system_info->virtualization); + buffer_json_member_add_string_or_empty(wb, "container", host->system_info->container); + } + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "os"); + { + buffer_json_member_add_string_or_empty(wb, "id", host->system_info->host_os_id); + buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->host_os_name); + buffer_json_member_add_string_or_empty(wb, "v", host->system_info->host_os_version); + buffer_json_member_add_object(wb, "kernel"); + buffer_json_member_add_string_or_empty(wb, "nm", host->system_info->kernel_name); + buffer_json_member_add_string_or_empty(wb, "v", host->system_info->kernel_version); + buffer_json_object_close(wb); + } + buffer_json_object_close(wb); + } + + // created - the node is created but never connected to cloud + // unreachable - not currently connected + // stale - connected but not having live data + // reachable - connected with live data + // pruned - not connected for some time and has been removed + buffer_json_member_add_string(wb, "state", rrdhost_is_online(host) ? "reachable" : "stale"); + + rrdhost_health_to_json_v2(wb, "health", &s); + agent_capabilities_to_json(wb, host, "capabilities"); + rrdhost_stream_path_to_json(wb, host, STREAM_PATH_JSON_MEMBER, false); + } + + if (ctl->mode & (CONTEXTS_V2_NODE_INSTANCES)) { + buffer_json_member_add_array(wb, "instances"); + buffer_json_add_array_item_object(wb); // this instance + { + buffer_json_agent_status_id(wb, 0, 0); + + buffer_json_member_add_object(wb, "db"); + { + buffer_json_member_add_string(wb, "status", rrdhost_db_status_to_string(s.db.status)); + buffer_json_member_add_string(wb, "liveness", rrdhost_db_liveness_to_string(s.db.liveness)); + buffer_json_member_add_string(wb, "mode", rrd_memory_mode_name(s.db.mode)); + buffer_json_member_add_time_t(wb, "first_time", s.db.first_time_s); + buffer_json_member_add_time_t(wb, "last_time", s.db.last_time_s); + buffer_json_member_add_uint64(wb, "metrics", s.db.metrics); + + spinlock_lock(&s.host->accounting.spinlock); + int64_t count = 0; + + if (s.host->accounting.cache_timestamp && + ctl->now - s.host->accounting.cache_timestamp < host->rrd_update_every * 1.5) + count = s.host->accounting.currently_collected; + else { + Pvoid_t *Pvalue; + bool first = true; + Word_t dimension_id = 0; + while ((Pvalue = JudyLFirstThenNext(s.host->accounting.JudyL, &dimension_id, &first))) { + RRDDIM *rd = *Pvalue; + if (rd->collector.last_collected_time.tv_sec > ctl->now - (rd->rrdset->update_every * 2)) + count++; + } + s.host->accounting.currently_collected = count; + s.host->accounting.cache_timestamp = ctl->now; + } + spinlock_unlock(&s.host->accounting.spinlock); + + buffer_json_member_add_uint64(wb, "currently_collected_metrics", count); + buffer_json_member_add_uint64(wb, "instances", s.db.instances); + buffer_json_member_add_uint64(wb, "contexts", s.db.contexts); + } + buffer_json_object_close(wb); + + rrdhost_receiver_to_json(wb, &s, "ingest"); + rrdhost_sender_to_json(wb, &s, "stream"); + + buffer_json_member_add_object(wb, "ml"); + buffer_json_member_add_string(wb, "status", rrdhost_ml_status_to_string(s.ml.status)); + buffer_json_member_add_string(wb, "type", rrdhost_ml_type_to_string(s.ml.type)); + if (s.ml.status == RRDHOST_ML_STATUS_RUNNING) { + buffer_json_member_add_object(wb, "metrics"); + { + buffer_json_member_add_uint64(wb, "anomalous", s.ml.metrics.anomalous); + buffer_json_member_add_uint64(wb, "normal", s.ml.metrics.normal); + buffer_json_member_add_uint64(wb, "trained", s.ml.metrics.trained); + buffer_json_member_add_uint64(wb, "pending", s.ml.metrics.pending); + buffer_json_member_add_uint64(wb, "silenced", s.ml.metrics.silenced); + } + buffer_json_object_close(wb); // metrics + } + buffer_json_object_close(wb); // ml + + rrdhost_health_to_json_v2(wb, "health", &s); + + host_functions2json(host, wb); // functions + agent_capabilities_to_json(wb, host, "capabilities"); + + host_dyncfg_to_json_v2(wb, "dyncfg", &s); + } + buffer_json_object_close(wb); // this instance + buffer_json_array_close(wb); // instances + } + } + buffer_json_object_close(wb); // this node +} + +static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) { + if(!queryable_host || !host->rrdctx.contexts) + // the host matches the 'scope_host' but does not match the 'host' patterns + // or the host does not have any contexts + return 0; // continue to next host + + struct rrdcontext_to_json_v2_data *ctl = data; + + if(ctl->window.enabled && !rrdhost_matches_window(host, ctl->window.after, ctl->window.before, ctl->now)) + // the host does not have data in the requested window + return 0; // continue to next host + + if(ctl->request->timeout_ms && now_monotonic_usec() > ctl->timings.received_ut + ctl->request->timeout_ms * USEC_PER_MS) + // timed out + return -2; // stop the query + + if(ctl->request->interrupt_callback && ctl->request->interrupt_callback(ctl->request->interrupt_callback_data)) + // interrupted + return -1; // stop the query + + bool host_matched = (ctl->mode & CONTEXTS_V2_NODES); + bool do_contexts = (ctl->mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_ALERTS)); + + ctl->q.host_match = FTS_MATCHED_NONE; + if((ctl->mode & CONTEXTS_V2_SEARCH)) { + // check if we match the host itself + if(ctl->q.pattern && ( + full_text_search_string(&ctl->q.fts, ctl->q.pattern, host->hostname) || + full_text_search_char(&ctl->q.fts, ctl->q.pattern, host->machine_guid) || + (ctl->q.pattern && full_text_search_char(&ctl->q.fts, ctl->q.pattern, ctl->q.host_node_id_str)))) { + ctl->q.host_match = FTS_MATCHED_HOST; + do_contexts = true; + } + } + + if(do_contexts) { + // save it + SIMPLE_PATTERN *old_q = ctl->q.pattern; + + if(ctl->q.host_match == FTS_MATCHED_HOST) + // do not do pattern matching on contexts - we matched the host itself + ctl->q.pattern = NULL; + + ssize_t added = query_scope_foreach_context( + host, ctl->request->scope_contexts, + ctl->contexts.scope_pattern, ctl->contexts.pattern, + rrdcontext_to_json_v2_add_context, queryable_host, ctl); + + // restore it + ctl->q.pattern = old_q; + + if(unlikely(added < 0)) + return -1; // stop the query + + if(added) + host_matched = true; + } + + if(!host_matched) + return 0; + + if(ctl->mode & CONTEXTS_V2_FUNCTIONS) { + struct function_v2_entry t = { + .used = 1, + .size = 1, + .node_ids = &ctl->nodes.ni, + .help = NULL, + .tags = NULL, + .access = HTTP_ACCESS_ALL, + .priority = RRDFUNCTIONS_PRIORITY_DEFAULT, + .version = RRDFUNCTIONS_VERSION_DEFAULT, + }; + host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help, &t.tags, &t.access, &t.priority, &t.version); + } + + if(ctl->mode & CONTEXTS_V2_NODES) { + struct contexts_v2_node t = { + .ni = ctl->nodes.ni++, + .host = host, + }; + + dictionary_set(ctl->nodes.dict, host->machine_guid, &t, sizeof(struct contexts_v2_node)); + } + + return 1; +} + +static void buffer_json_contexts_v2_mode_to_array(BUFFER *wb, const char *key, CONTEXTS_V2_MODE mode) { + buffer_json_member_add_array(wb, key); + + if(mode & CONTEXTS_V2_VERSIONS) + buffer_json_add_array_item_string(wb, "versions"); + + if(mode & CONTEXTS_V2_AGENTS) + buffer_json_add_array_item_string(wb, "agents"); + + if(mode & CONTEXTS_V2_AGENTS_INFO) + buffer_json_add_array_item_string(wb, "agents-info"); + + if(mode & CONTEXTS_V2_NODES) + buffer_json_add_array_item_string(wb, "nodes"); + + if(mode & CONTEXTS_V2_NODES_INFO) + buffer_json_add_array_item_string(wb, "nodes-info"); + + if(mode & CONTEXTS_V2_NODE_INSTANCES) + buffer_json_add_array_item_string(wb, "nodes-instances"); + + if(mode & CONTEXTS_V2_CONTEXTS) + buffer_json_add_array_item_string(wb, "contexts"); + + if(mode & CONTEXTS_V2_SEARCH) + buffer_json_add_array_item_string(wb, "search"); + + if(mode & CONTEXTS_V2_ALERTS) + buffer_json_add_array_item_string(wb, "alerts"); + + if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) + buffer_json_add_array_item_string(wb, "alert_transitions"); + + buffer_json_array_close(wb); +} + +void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings) { + timings->finished_ut = now_monotonic_usec(); + if(!timings->executed_ut) + timings->executed_ut = timings->finished_ut; + if(!timings->preprocessed_ut) + timings->preprocessed_ut = timings->received_ut; + buffer_json_member_add_object(wb, key); + buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(timings->preprocessed_ut - timings->received_ut) / USEC_PER_MS); + buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(timings->executed_ut - timings->preprocessed_ut) / USEC_PER_MS); + buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->executed_ut) / USEC_PER_MS); + buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); + buffer_json_member_add_double(wb, "cloud_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); + buffer_json_object_close(wb); +} + +void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings) { + if(!timings->finished_ut) + timings->finished_ut = now_monotonic_usec(); + + buffer_json_member_add_object(wb, key); + buffer_json_member_add_double(wb, "routing_ms", 0.0); + buffer_json_member_add_double(wb, "node_max_ms", 0.0); + buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(timings->finished_ut - timings->received_ut) / USEC_PER_MS); + buffer_json_object_close(wb); +} + +static void functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct function_v2_entry *t = value; + + // it is initialized with a static reference - we need to mallocz() the array + size_t *v = t->node_ids; + t->node_ids = mallocz(sizeof(size_t)); + *t->node_ids = *v; + t->size = 1; + t->used = 1; +} + +static bool functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { + struct function_v2_entry *t = old_value, *n = new_value; + size_t *v = n->node_ids; + + if(t->used >= t->size) { + t->node_ids = reallocz(t->node_ids, t->size * 2 * sizeof(size_t)); + t->size *= 2; + } + + t->node_ids[t->used++] = *v; + + return true; +} + +static void functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct function_v2_entry *t = value; + freez(t->node_ids); +} + +static bool contexts_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { + struct context_v2_entry *o = old_value; + struct context_v2_entry *n = new_value; + + o->count++; + + if(o->family != n->family) { + if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED)) + // keep old + ; + else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) { + // keep new + string_freez(o->family); + o->family = string_dup(n->family); + } + else { + // merge + STRING *old_family = o->family; + o->family = string_2way_merge(o->family, n->family); + string_freez(old_family); + } + } + + if(o->priority != n->priority) { + if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED)) + // keep o + ; + else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) + // keep n + o->priority = n->priority; + else + // keep the min + o->priority = MIN(o->priority, n->priority); + } + + if(o->first_time_s && n->first_time_s) + o->first_time_s = MIN(o->first_time_s, n->first_time_s); + else if(!o->first_time_s) + o->first_time_s = n->first_time_s; + + if(o->last_time_s && n->last_time_s) + o->last_time_s = MAX(o->last_time_s, n->last_time_s); + else if(!o->last_time_s) + o->last_time_s = n->last_time_s; + + o->flags |= n->flags; + o->match = MIN(o->match, n->match); + + string_freez(n->family); + + return true; +} + +static void contexts_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct context_v2_entry *z = value; + string_freez(z->family); +} + +int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_MODE mode) { + int resp = HTTP_RESP_OK; + bool run = true; + + if(mode & CONTEXTS_V2_SEARCH) + mode |= CONTEXTS_V2_CONTEXTS; + + if(mode & (CONTEXTS_V2_AGENTS_INFO)) + mode |= CONTEXTS_V2_AGENTS; + + if(mode & (CONTEXTS_V2_FUNCTIONS | CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_NODES_INFO | CONTEXTS_V2_NODE_INSTANCES)) + mode |= CONTEXTS_V2_NODES; + + if(mode & CONTEXTS_V2_ALERTS) { + mode |= CONTEXTS_V2_NODES; + req->options &= ~CONTEXTS_OPTION_ALERTS_WITH_CONFIGURATIONS; + + if(!(req->options & (CONTEXTS_OPTION_ALERTS_WITH_SUMMARY | CONTEXTS_OPTION_ALERTS_WITH_INSTANCES | + CONTEXTS_OPTION_ALERTS_WITH_VALUES))) + req->options |= CONTEXTS_OPTION_ALERTS_WITH_SUMMARY; + } + + if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { + mode |= CONTEXTS_V2_NODES; + req->options &= ~CONTEXTS_OPTION_ALERTS_WITH_INSTANCES; + } + + struct rrdcontext_to_json_v2_data ctl = { + .wb = wb, + .request = req, + .mode = mode, + .options = req->options, + .versions = { 0 }, + .nodes.scope_pattern = string_to_simple_pattern(req->scope_nodes), + .nodes.pattern = string_to_simple_pattern(req->nodes), + .contexts.pattern = string_to_simple_pattern(req->contexts), + .contexts.scope_pattern = string_to_simple_pattern(req->scope_contexts), + .q.pattern = string_to_simple_pattern_nocase(req->q), + .alerts.alert_name_pattern = string_to_simple_pattern(req->alerts.alert), + .window = { + .enabled = false, + .relative = false, + .after = req->after, + .before = req->before, + }, + .timings = { + .received_ut = now_monotonic_usec(), + } + }; + + bool debug = ctl.options & CONTEXTS_OPTION_DEBUG; + + if(mode & CONTEXTS_V2_NODES) { + ctl.nodes.dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(struct contexts_v2_node)); + } + + if(mode & CONTEXTS_V2_CONTEXTS) { + ctl.contexts.dict = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, + sizeof(struct context_v2_entry)); + + dictionary_register_conflict_callback(ctl.contexts.dict, contexts_conflict_callback, &ctl); + dictionary_register_delete_callback(ctl.contexts.dict, contexts_delete_callback, &ctl); + } + + if(mode & CONTEXTS_V2_FUNCTIONS) { + ctl.functions.dict = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, + sizeof(struct function_v2_entry)); + + dictionary_register_insert_callback(ctl.functions.dict, functions_insert_callback, &ctl); + dictionary_register_conflict_callback(ctl.functions.dict, functions_conflict_callback, &ctl); + dictionary_register_delete_callback(ctl.functions.dict, functions_delete_callback, &ctl); + } + + if(mode & CONTEXTS_V2_ALERTS) { + if(!rrdcontexts_v2_init_alert_dictionaries(&ctl, req)) { + resp = HTTP_RESP_NOT_FOUND; + goto cleanup; + } + } + + if(req->after || req->before) { + ctl.window.relative = rrdr_relative_window_to_absolute_query( + &ctl.window.after, &ctl.window.before, &ctl.now, false); + + ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS); + } + else + ctl.now = now_realtime_sec(); + + buffer_json_initialize(wb, "\"", "\"", 0, true, + ((req->options & CONTEXTS_OPTION_MINIFY) && !(req->options & CONTEXTS_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); + + buffer_json_member_add_uint64(wb, "api", 2); + + if(req->options & CONTEXTS_OPTION_DEBUG) { + buffer_json_member_add_object(wb, "request"); + { + buffer_json_contexts_v2_mode_to_array(wb, "mode", mode); + contexts_options_to_buffer_json_array(wb, "options", req->options); + + buffer_json_member_add_object(wb, "scope"); + { + buffer_json_member_add_string(wb, "scope_nodes", req->scope_nodes); + if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS)) + buffer_json_member_add_string(wb, "scope_contexts", req->scope_contexts); + } + buffer_json_object_close(wb); + + buffer_json_member_add_object(wb, "selectors"); + { + buffer_json_member_add_string(wb, "nodes", req->nodes); + + if (mode & (CONTEXTS_V2_CONTEXTS | CONTEXTS_V2_SEARCH | CONTEXTS_V2_ALERTS)) + buffer_json_member_add_string(wb, "contexts", req->contexts); + + if(mode & (CONTEXTS_V2_ALERTS | CONTEXTS_V2_ALERT_TRANSITIONS)) { + buffer_json_member_add_object(wb, "alerts"); + + if(mode & CONTEXTS_V2_ALERTS) + contexts_alerts_status_to_buffer_json_array(wb, "status", req->alerts.status); + + if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { + buffer_json_member_add_string(wb, "context", req->contexts); + buffer_json_member_add_uint64(wb, "anchor_gi", req->alerts.global_id_anchor); + buffer_json_member_add_uint64(wb, "last", req->alerts.last); + } + + buffer_json_member_add_string(wb, "alert", req->alerts.alert); + buffer_json_member_add_string(wb, "transition", req->alerts.transition); + buffer_json_object_close(wb); // alerts + } + } + buffer_json_object_close(wb); // selectors + + buffer_json_member_add_object(wb, "filters"); + { + if (mode & CONTEXTS_V2_SEARCH) + buffer_json_member_add_string(wb, "q", req->q); + + buffer_json_member_add_time_t(wb, "after", req->after); + buffer_json_member_add_time_t(wb, "before", req->before); + } + buffer_json_object_close(wb); // filters + + if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { + buffer_json_member_add_object(wb, "facets"); + { + for (int i = 0; i < ATF_TOTAL_ENTRIES; i++) { + buffer_json_member_add_string(wb, alert_transition_facets[i].query_param, req->alerts.facets[i]); + } + } + buffer_json_object_close(wb); // facets + } + } + buffer_json_object_close(wb); + } + + ssize_t ret = 0; + if(run) + ret = query_scope_foreach_host(ctl.nodes.scope_pattern, ctl.nodes.pattern, + rrdcontext_to_json_v2_add_host, &ctl, + &ctl.versions, ctl.q.host_node_id_str); + + if(unlikely(ret < 0)) { + buffer_flush(wb); + + if(ret == -2) { + buffer_strcat(wb, "query timeout"); + resp = HTTP_RESP_GATEWAY_TIMEOUT; + } + else { + buffer_strcat(wb, "query interrupted"); + resp = HTTP_RESP_CLIENT_CLOSED_REQUEST; + } + goto cleanup; + } + + ctl.timings.executed_ut = now_monotonic_usec(); + + if(mode & CONTEXTS_V2_ALERT_TRANSITIONS) { + contexts_v2_alert_transitions_to_json(wb, &ctl, debug); + } + else { + if (mode & CONTEXTS_V2_NODES) { + buffer_json_member_add_array(wb, "nodes"); + struct contexts_v2_node *t; + dfe_start_read(ctl.nodes.dict, t) { + rrdcontext_to_json_v2_rrdhost(wb, t->host, &ctl, t->ni); + } + dfe_done(t); + buffer_json_array_close(wb); + } + + if (mode & CONTEXTS_V2_FUNCTIONS) { + buffer_json_member_add_array(wb, "functions"); + { + struct function_v2_entry *t; + dfe_start_read(ctl.functions.dict, t) { + buffer_json_add_array_item_object(wb); + { + const char *name = t_dfe.name ? strstr(t_dfe.name, RRDFUNCTIONS_VERSION_SEPARATOR) : NULL; + if(name) + name += sizeof(RRDFUNCTIONS_VERSION_SEPARATOR) - 1; + else + name = t_dfe.name; + + buffer_json_member_add_string(wb, "name", name); + buffer_json_member_add_string(wb, "help", string2str(t->help)); + buffer_json_member_add_array(wb, "ni"); + { + for (size_t i = 0; i < t->used; i++) + buffer_json_add_array_item_uint64(wb, t->node_ids[i]); + } + buffer_json_array_close(wb); + buffer_json_member_add_string(wb, "tags", string2str(t->tags)); + http_access2buffer_json_array(wb, "access", t->access); + buffer_json_member_add_uint64(wb, "priority", t->priority); + buffer_json_member_add_uint64(wb, "version", t->version); + } + buffer_json_object_close(wb); + } + dfe_done(t); + } + buffer_json_array_close(wb); + } + + if (mode & CONTEXTS_V2_CONTEXTS) { + buffer_json_member_add_object(wb, "contexts"); + { + struct context_v2_entry *z; + dfe_start_read(ctl.contexts.dict, z) { + bool collected = z->flags & RRD_FLAG_COLLECTED; + + buffer_json_member_add_object(wb, string2str(z->id)); + { + buffer_json_member_add_string(wb, "family", string2str(z->family)); + buffer_json_member_add_uint64(wb, "priority", z->priority); + buffer_json_member_add_time_t(wb, "first_entry", z->first_time_s); + buffer_json_member_add_time_t(wb, "last_entry", collected ? ctl.now : z->last_time_s); + buffer_json_member_add_boolean(wb, "live", collected); + if (mode & CONTEXTS_V2_SEARCH) + buffer_json_member_add_string(wb, "match", fts_match_to_string(z->match)); + } + buffer_json_object_close(wb); + } + dfe_done(z); + } + buffer_json_object_close(wb); // contexts + } + + if (mode & CONTEXTS_V2_ALERTS) + contexts_v2_alerts_to_json(wb, &ctl, debug); + + if (mode & CONTEXTS_V2_SEARCH) { + buffer_json_member_add_object(wb, "searches"); + { + buffer_json_member_add_uint64(wb, "strings", ctl.q.fts.string_searches); + buffer_json_member_add_uint64(wb, "char", ctl.q.fts.char_searches); + buffer_json_member_add_uint64(wb, "total", ctl.q.fts.searches); + } + buffer_json_object_close(wb); + } + + if (mode & (CONTEXTS_V2_VERSIONS)) + version_hashes_api_v2(wb, &ctl.versions); + + if (mode & CONTEXTS_V2_AGENTS) + buffer_json_agents_v2(wb, &ctl.timings, ctl.now, mode & (CONTEXTS_V2_AGENTS_INFO), true); + } + + buffer_json_cloud_timings(wb, "timings", &ctl.timings); + + buffer_json_finalize(wb); + +cleanup: + dictionary_destroy(ctl.nodes.dict); + dictionary_destroy(ctl.contexts.dict); + dictionary_destroy(ctl.functions.dict); + rrdcontexts_v2_alerts_cleanup(&ctl); + simple_pattern_free(ctl.nodes.scope_pattern); + simple_pattern_free(ctl.nodes.pattern); + simple_pattern_free(ctl.contexts.pattern); + simple_pattern_free(ctl.contexts.scope_pattern); + simple_pattern_free(ctl.q.pattern); + simple_pattern_free(ctl.alerts.alert_name_pattern); + + return resp; +} diff --git a/src/database/contexts/api_v2_contexts.h b/src/database/contexts/api_v2_contexts.h new file mode 100644 index 000000000..3fb5354b9 --- /dev/null +++ b/src/database/contexts/api_v2_contexts.h @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_V2_CONTEXTS_H +#define NETDATA_API_V2_CONTEXTS_H + +#include "internal.h" + +typedef enum __attribute__ ((__packed__)) { + FTS_MATCHED_NONE = 0, + FTS_MATCHED_HOST, + FTS_MATCHED_CONTEXT, + FTS_MATCHED_INSTANCE, + FTS_MATCHED_DIMENSION, + FTS_MATCHED_LABEL, + FTS_MATCHED_ALERT, + FTS_MATCHED_ALERT_INFO, + FTS_MATCHED_FAMILY, + FTS_MATCHED_TITLE, + FTS_MATCHED_UNITS, +} FTS_MATCH; + +typedef struct full_text_search_index { + size_t searches; + size_t string_searches; + size_t char_searches; +} FTS_INDEX; + +struct contexts_v2_node { + size_t ni; + RRDHOST *host; +}; + +struct rrdcontext_to_json_v2_data { + time_t now; + + BUFFER *wb; + struct api_v2_contexts_request *request; + + CONTEXTS_V2_MODE mode; + CONTEXTS_OPTIONS options; + struct query_versions versions; + + struct { + SIMPLE_PATTERN *scope_pattern; + SIMPLE_PATTERN *pattern; + size_t ni; + DICTIONARY *dict; // the result set + } nodes; + + struct { + SIMPLE_PATTERN *scope_pattern; + SIMPLE_PATTERN *pattern; + size_t ci; + DICTIONARY *dict; // the result set + } contexts; + + struct { + SIMPLE_PATTERN *alert_name_pattern; + time_t alarm_id_filter; + + size_t ati; + + DICTIONARY *summary; + DICTIONARY *alert_instances; + + DICTIONARY *by_type; + DICTIONARY *by_component; + DICTIONARY *by_classification; + DICTIONARY *by_recipient; + DICTIONARY *by_module; + } alerts; + + struct { + FTS_MATCH host_match; + char host_node_id_str[UUID_STR_LEN]; + SIMPLE_PATTERN *pattern; + FTS_INDEX fts; + } q; + + struct { + DICTIONARY *dict; // the result set + } functions; + + struct { + bool enabled; + bool relative; + time_t after; + time_t before; + } window; + + struct query_timings timings; +}; + +void agent_capabilities_to_json(BUFFER *wb, RRDHOST *host, const char *key); + +#include "api_v2_contexts_alerts.h" + +#endif //NETDATA_API_V2_CONTEXTS_H diff --git a/src/database/contexts/api_v2_contexts_agents.c b/src/database/contexts/api_v2_contexts_agents.c new file mode 100644 index 000000000..e279405a0 --- /dev/null +++ b/src/database/contexts/api_v2_contexts_agents.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "api_v2_contexts.h" +#include "aclk/aclk_capas.h" + +void build_info_to_json_object(BUFFER *b); + +void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now_s, bool info, bool array) { + if(!now_s) + now_s = now_realtime_sec(); + + if(array) { + buffer_json_member_add_array(wb, "agents"); + buffer_json_add_array_item_object(wb); + } + else + buffer_json_member_add_object(wb, "agent"); + + buffer_json_member_add_string(wb, "mg", localhost->machine_guid); + buffer_json_member_add_uuid(wb, "nd", localhost->node_id.uuid); + buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost)); + buffer_json_member_add_time_t(wb, "now", now_s); + + if(array) + buffer_json_member_add_uint64(wb, "ai", 0); + + if(info) { + buffer_json_member_add_object(wb, "application"); + build_info_to_json_object(wb); + buffer_json_object_close(wb); // application + + buffer_json_cloud_status(wb, now_s); + + size_t currently_collected_metrics = 0; + + buffer_json_member_add_object(wb, "nodes"); + { + size_t receiving = 0, archived = 0, sending = 0, total = 0; + RRDHOST *host; + dfe_start_read(rrdhost_root_index, host) { + total++; + + if(rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED)) + sending++; + + if(host != localhost) { + if (rrdhost_is_online(host)) + receiving++; + else + archived++; + } + currently_collected_metrics += host->accounting.currently_collected; + } + dfe_done(host); + + buffer_json_member_add_uint64(wb, "total", total); + buffer_json_member_add_uint64(wb, "receiving", receiving); + buffer_json_member_add_uint64(wb, "sending", sending); + buffer_json_member_add_uint64(wb, "archived", archived); + } + buffer_json_object_close(wb); // nodes + + agent_capabilities_to_json(wb, localhost, "capabilities"); + + buffer_json_member_add_object(wb, "api"); + { + buffer_json_member_add_uint64(wb, "version", aclk_get_http_api_version()); + buffer_json_member_add_boolean(wb, "bearer_protection", netdata_is_protected_by_bearer); + } + buffer_json_object_close(wb); // api + + buffer_json_member_add_array(wb, "db_size"); + size_t group_seconds = localhost->rrd_update_every; + for (size_t tier = 0; tier < storage_tiers; tier++) { + STORAGE_ENGINE *eng = localhost->db[tier].eng; + if (!eng) continue; + + group_seconds *= storage_tiers_grouping_iterations[tier]; + uint64_t max = storage_engine_disk_space_max(eng->seb, localhost->db[tier].si); + uint64_t used = storage_engine_disk_space_used(eng->seb, localhost->db[tier].si); +#ifdef ENABLE_DBENGINE + if (!max && eng->seb == STORAGE_ENGINE_BACKEND_DBENGINE) { + max = get_directory_free_bytes_space(multidb_ctx[tier]); + max += used; + } +#endif + time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si); +// size_t currently_collected_metrics = storage_engine_collected_metrics(eng->seb, localhost->db[tier].si); + + NETDATA_DOUBLE percent; + if (used && max) + percent = (NETDATA_DOUBLE) used * 100.0 / (NETDATA_DOUBLE) max; + else + percent = 0.0; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_uint64(wb, "tier", tier); + char human_retention[128]; + duration_snprintf_time_t(human_retention, sizeof(human_retention), (stime_t)group_seconds); + buffer_json_member_add_string(wb, "granularity", human_retention); + + buffer_json_member_add_uint64(wb, "metrics", storage_engine_metrics(eng->seb, localhost->db[tier].si)); + buffer_json_member_add_uint64(wb, "samples", storage_engine_samples(eng->seb, localhost->db[tier].si)); + + if(used || max) { + buffer_json_member_add_uint64(wb, "disk_used", used); + buffer_json_member_add_uint64(wb, "disk_max", max); + buffer_json_member_add_double(wb, "disk_percent", percent); + } + + if(first_time_s) { + time_t retention = now_s - first_time_s; + + buffer_json_member_add_time_t(wb, "from", first_time_s); + buffer_json_member_add_time_t(wb, "to", now_s); + buffer_json_member_add_time_t(wb, "retention", retention); + + duration_snprintf_hours(human_retention, sizeof(human_retention), + (int)duration_round_to_resolution(retention, 3600)); + + buffer_json_member_add_string(wb, "retention_human", human_retention); + + if(used || max) { // we have disk space information + time_t time_retention = 0; +#ifdef ENABLE_DBENGINE + time_retention = multidb_ctx[tier]->config.max_retention_s; +#endif + time_t space_retention = (time_t)((NETDATA_DOUBLE)(now_s - first_time_s) * 100.0 / percent); + time_t actual_retention = MIN(space_retention, time_retention ? time_retention : space_retention); + + if (time_retention) { + duration_snprintf_hours(human_retention, sizeof(human_retention), + (int)duration_round_to_resolution(time_retention, 3600)); + + buffer_json_member_add_time_t(wb, "requested_retention", time_retention); + buffer_json_member_add_string(wb, "requested_retention_human", human_retention); + } + + duration_snprintf_hours(human_retention, sizeof(human_retention), + (int)duration_round_to_resolution(actual_retention, 3600)); + + buffer_json_member_add_time_t(wb, "expected_retention", actual_retention); + buffer_json_member_add_string(wb, "expected_retention_human", human_retention); + } + } + + if(currently_collected_metrics) + buffer_json_member_add_uint64(wb, "currently_collected_metrics", currently_collected_metrics); + + buffer_json_object_close(wb); + } + buffer_json_array_close(wb); // db_size + } + + if(timings) + buffer_json_query_timings(wb, "timings", timings); + + buffer_json_object_close(wb); + + if(array) + buffer_json_array_close(wb); +} diff --git a/src/database/contexts/api_v2_contexts_alert_config.c b/src/database/contexts/api_v2_contexts_alert_config.c new file mode 100644 index 000000000..cd3d8fc14 --- /dev/null +++ b/src/database/contexts/api_v2_contexts_alert_config.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "api_v2_contexts_alerts.h" + +void contexts_v2_alert_config_to_json_from_sql_alert_config_data(struct sql_alert_config_data *t, void *data) { + struct alert_transitions_callback_data *d = data; + BUFFER *wb = d->wb; + bool debug = d->debug; + d->configs_added++; + + if(d->only_one_config) + buffer_json_add_array_item_object(wb); // alert config + + { + buffer_json_member_add_string(wb, "name", t->name); + buffer_json_member_add_uuid_ptr(wb, "config_hash_id", t->config_hash_id); + + buffer_json_member_add_object(wb, "selectors"); + { + bool is_template = t->selectors.on_template && *t->selectors.on_template ? true : false; + buffer_json_member_add_string(wb, "type", is_template ? "template" : "alarm"); + buffer_json_member_add_string(wb, "on", is_template ? t->selectors.on_template : t->selectors.on_key); + + buffer_json_member_add_string(wb, "families", t->selectors.families); + buffer_json_member_add_string(wb, "host_labels", t->selectors.host_labels); + buffer_json_member_add_string(wb, "chart_labels", t->selectors.chart_labels); + } + buffer_json_object_close(wb); // selectors + + buffer_json_member_add_object(wb, "value"); // value + { + // buffer_json_member_add_string(wb, "every", t->value.every); // does not exist in Netdata Cloud + buffer_json_member_add_string(wb, "units", t->value.units); + buffer_json_member_add_uint64(wb, "update_every", t->value.update_every); + + if (t->value.db.after || debug) { + buffer_json_member_add_object(wb, "db"); + { + // buffer_json_member_add_string(wb, "lookup", t->value.db.lookup); // does not exist in Netdata Cloud + + buffer_json_member_add_time_t(wb, "after", t->value.db.after); + buffer_json_member_add_time_t(wb, "before", t->value.db.before); + buffer_json_member_add_string(wb, "time_group_condition", alerts_group_conditions_id2txt(t->value.db.time_group_condition)); + buffer_json_member_add_double(wb, "time_group_value", t->value.db.time_group_value); + buffer_json_member_add_string(wb, "dims_group", alerts_dims_grouping_id2group(t->value.db.dims_group)); + buffer_json_member_add_string(wb, "data_source", alerts_data_source_id2source(t->value.db.data_source)); + buffer_json_member_add_string(wb, "method", t->value.db.method); + buffer_json_member_add_string(wb, "dimensions", t->value.db.dimensions); + rrdr_options_to_buffer_json_array(wb, "options", (RRDR_OPTIONS)t->value.db.options); + } + buffer_json_object_close(wb); // db + } + + if (t->value.calc || debug) + buffer_json_member_add_string(wb, "calc", t->value.calc); + } + buffer_json_object_close(wb); // value + + if (t->status.warn || t->status.crit || debug) { + buffer_json_member_add_object(wb, "status"); // status + { + NETDATA_DOUBLE green = t->status.green ? str2ndd(t->status.green, NULL) : NAN; + NETDATA_DOUBLE red = t->status.red ? str2ndd(t->status.red, NULL) : NAN; + + if (!isnan(green) || debug) + buffer_json_member_add_double(wb, "green", green); + + if (!isnan(red) || debug) + buffer_json_member_add_double(wb, "red", red); + + if (t->status.warn || debug) + buffer_json_member_add_string(wb, "warn", t->status.warn); + + if (t->status.crit || debug) + buffer_json_member_add_string(wb, "crit", t->status.crit); + } + buffer_json_object_close(wb); // status + } + + buffer_json_member_add_object(wb, "notification"); + { + buffer_json_member_add_string(wb, "type", "agent"); + buffer_json_member_add_string(wb, "exec", t->notification.exec ? t->notification.exec : NULL); + buffer_json_member_add_string(wb, "to", t->notification.to_key ? t->notification.to_key : string2str(localhost->health.health_default_recipient)); + buffer_json_member_add_string(wb, "delay", t->notification.delay); + buffer_json_member_add_string(wb, "repeat", t->notification.repeat); + buffer_json_member_add_string(wb, "options", t->notification.options); + } + buffer_json_object_close(wb); // notification + + buffer_json_member_add_string(wb, "class", t->classification); + buffer_json_member_add_string(wb, "component", t->component); + buffer_json_member_add_string(wb, "type", t->type); + buffer_json_member_add_string(wb, "info", t->info); + buffer_json_member_add_string(wb, "summary", t->summary); + // buffer_json_member_add_string(wb, "source", t->source); // moved to alert instance + } + + if(d->only_one_config) + buffer_json_object_close(wb); +} + +int contexts_v2_alert_config_to_json(struct web_client *w, const char *config_hash_id) { + struct alert_transitions_callback_data data = { + .wb = w->response.data, + .debug = false, + .only_one_config = false, + }; + DICTIONARY *configs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_set(configs, config_hash_id, NULL, 0); + + buffer_flush(w->response.data); + + buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); + + int added = sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, false); + buffer_json_finalize(w->response.data); + + int ret = HTTP_RESP_OK; + + if(added <= 0) { + buffer_flush(w->response.data); + w->response.data->content_type = CT_TEXT_PLAIN; + if(added < 0) { + buffer_strcat(w->response.data, "Failed to execute SQL query."); + ret = HTTP_RESP_INTERNAL_SERVER_ERROR; + } + else { + buffer_strcat(w->response.data, "Config is not found."); + ret = HTTP_RESP_NOT_FOUND; + } + } + + return ret; +} diff --git a/src/database/contexts/api_v2_contexts_alert_transitions.c b/src/database/contexts/api_v2_contexts_alert_transitions.c new file mode 100644 index 000000000..13061f60f --- /dev/null +++ b/src/database/contexts/api_v2_contexts_alert_transitions.c @@ -0,0 +1,487 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "api_v2_contexts_alerts.h" + +struct alert_transitions_facets alert_transition_facets[] = { + [ATF_STATUS] = { + .id = "f_status", + .name = "Alert Status", + .query_param = "f_status", + .order = 1, + }, + [ATF_TYPE] = { + .id = "f_type", + .name = "Alert Type", + .query_param = "f_type", + .order = 2, + }, + [ATF_ROLE] = { + .id = "f_role", + .name = "Recipient Role", + .query_param = "f_role", + .order = 3, + }, + [ATF_CLASS] = { + .id = "f_class", + .name = "Alert Class", + .query_param = "f_class", + .order = 4, + }, + [ATF_COMPONENT] = { + .id = "f_component", + .name = "Alert Component", + .query_param = "f_component", + .order = 5, + }, + [ATF_NODE] = { + .id = "f_node", + .name = "Alert Node", + .query_param = "f_node", + .order = 6, + }, + [ATF_ALERT_NAME] = { + .id = "f_alert", + .name = "Alert Name", + .query_param = "f_alert", + .order = 7, + }, + [ATF_CHART_NAME] = { + .id = "f_instance", + .name = "Instance Name", + .query_param = "f_instance", + .order = 8, + }, + [ATF_CONTEXT] = { + .id = "f_context", + .name = "Context", + .query_param = "f_context", + .order = 9, + }, + + // terminator + [ATF_TOTAL_ENTRIES] = { + .id = NULL, + .name = NULL, + .query_param = NULL, + .order = 9999, + } +}; + +#define SQL_TRANSITION_DATA_SMALL_STRING (6 * 8) +#define SQL_TRANSITION_DATA_MEDIUM_STRING (12 * 8) +#define SQL_TRANSITION_DATA_BIG_STRING 512 + +struct sql_alert_transition_fixed_size { + usec_t global_id; + nd_uuid_t transition_id; + nd_uuid_t host_id; + nd_uuid_t config_hash_id; + uint32_t alarm_id; + char alert_name[SQL_TRANSITION_DATA_SMALL_STRING]; + char chart[RRD_ID_LENGTH_MAX]; + char chart_name[RRD_ID_LENGTH_MAX]; + char chart_context[SQL_TRANSITION_DATA_MEDIUM_STRING]; + char family[SQL_TRANSITION_DATA_SMALL_STRING]; + char recipient[SQL_TRANSITION_DATA_MEDIUM_STRING]; + char units[SQL_TRANSITION_DATA_SMALL_STRING]; + char exec[SQL_TRANSITION_DATA_BIG_STRING]; + char info[SQL_TRANSITION_DATA_BIG_STRING]; + char summary[SQL_TRANSITION_DATA_BIG_STRING]; + char classification[SQL_TRANSITION_DATA_SMALL_STRING]; + char type[SQL_TRANSITION_DATA_SMALL_STRING]; + char component[SQL_TRANSITION_DATA_SMALL_STRING]; + time_t when_key; + time_t duration; + time_t non_clear_duration; + uint64_t flags; + time_t delay_up_to_timestamp; + time_t exec_run_timestamp; + int exec_code; + int new_status; + int old_status; + int delay; + time_t last_repeat; + NETDATA_DOUBLE new_value; + NETDATA_DOUBLE old_value; + + char machine_guid[UUID_STR_LEN]; + struct sql_alert_transition_fixed_size *next; + struct sql_alert_transition_fixed_size *prev; +}; + +struct facet_entry { + uint32_t count; +}; + +static struct sql_alert_transition_fixed_size *contexts_v2_alert_transition_dup(struct sql_alert_transition_data *t, const char *machine_guid, struct sql_alert_transition_fixed_size *dst) { + struct sql_alert_transition_fixed_size *n = dst ? dst : mallocz(sizeof(*n)); + + n->global_id = t->global_id; + uuid_copy(n->transition_id, *t->transition_id); + uuid_copy(n->host_id, *t->host_id); + uuid_copy(n->config_hash_id, *t->config_hash_id); + n->alarm_id = t->alarm_id; + strncpyz(n->alert_name, t->alert_name ? t->alert_name : "", sizeof(n->alert_name) - 1); + strncpyz(n->chart, t->chart ? t->chart : "", sizeof(n->chart) - 1); + strncpyz(n->chart_name, t->chart_name ? t->chart_name : n->chart, sizeof(n->chart_name) - 1); + strncpyz(n->chart_context, t->chart_context ? t->chart_context : "", sizeof(n->chart_context) - 1); + strncpyz(n->family, t->family ? t->family : "", sizeof(n->family) - 1); + strncpyz(n->recipient, t->recipient ? t->recipient : "", sizeof(n->recipient) - 1); + strncpyz(n->units, t->units ? t->units : "", sizeof(n->units) - 1); + strncpyz(n->exec, t->exec ? t->exec : "", sizeof(n->exec) - 1); + strncpyz(n->info, t->info ? t->info : "", sizeof(n->info) - 1); + strncpyz(n->summary, t->summary ? t->summary : "", sizeof(n->summary) - 1); + strncpyz(n->classification, t->classification ? t->classification : "", sizeof(n->classification) - 1); + strncpyz(n->type, t->type ? t->type : "", sizeof(n->type) - 1); + strncpyz(n->component, t->component ? t->component : "", sizeof(n->component) - 1); + n->when_key = t->when_key; + n->duration = t->duration; + n->non_clear_duration = t->non_clear_duration; + n->flags = t->flags; + n->delay_up_to_timestamp = t->delay_up_to_timestamp; + n->exec_run_timestamp = t->exec_run_timestamp; + n->exec_code = t->exec_code; + n->new_status = t->new_status; + n->old_status = t->old_status; + n->delay = t->delay; + n->last_repeat = t->last_repeat; + n->new_value = t->new_value; + n->old_value = t->old_value; + + memcpy(n->machine_guid, machine_guid, sizeof(n->machine_guid)); + n->next = n->prev = NULL; + + return n; +} + +static void contexts_v2_alert_transition_free(struct sql_alert_transition_fixed_size *t) { + freez(t); +} + +static inline void contexts_v2_alert_transition_keep(struct alert_transitions_callback_data *d, struct sql_alert_transition_data *t, const char *machine_guid) { + d->items_matched++; + + if(unlikely(t->global_id <= d->ctl->request->alerts.global_id_anchor)) { + // this is in our past, we are not interested + d->operations.skips_before++; + return; + } + + if(unlikely(!d->base)) { + d->last_added = contexts_v2_alert_transition_dup(t, machine_guid, NULL); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(d->base, d->last_added, prev, next); + d->items_to_return++; + d->operations.first++; + return; + } + + struct sql_alert_transition_fixed_size *last = d->last_added; + while(last->prev != d->base->prev && t->global_id > last->prev->global_id) { + last = last->prev; + d->operations.backwards++; + } + + while(last->next && t->global_id < last->next->global_id) { + last = last->next; + d->operations.forwards++; + } + + if(d->items_to_return >= d->max_items_to_return) { + if(last == d->base->prev && t->global_id < last->global_id) { + d->operations.skips_after++; + return; + } + } + + d->items_to_return++; + + if(t->global_id > last->global_id) { + if(d->items_to_return > d->max_items_to_return) { + d->items_to_return--; + d->operations.shifts++; + d->last_added = d->base->prev; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(d->base, d->last_added, prev, next); + d->last_added = contexts_v2_alert_transition_dup(t, machine_guid, d->last_added); + } + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(d->base, d->last_added, prev, next); + d->operations.prepend++; + } + else { + d->last_added = contexts_v2_alert_transition_dup(t, machine_guid, NULL); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(d->base, d->last_added, prev, next); + d->operations.append++; + } + + while(d->items_to_return > d->max_items_to_return) { + // we have to remove something + + struct sql_alert_transition_fixed_size *tmp = d->base->prev; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(d->base, tmp, prev, next); + d->items_to_return--; + + if(unlikely(d->last_added == tmp)) + d->last_added = d->base; + + contexts_v2_alert_transition_free(tmp); + + d->operations.shifts++; + } +} + +static void contexts_v2_alert_transition_callback(struct sql_alert_transition_data *t, void *data) { + struct alert_transitions_callback_data *d = data; + d->items_evaluated++; + + char machine_guid[UUID_STR_LEN] = ""; + uuid_unparse_lower(*t->host_id, machine_guid); + + const char *facets[ATF_TOTAL_ENTRIES] = { + [ATF_STATUS] = rrdcalc_status2string(t->new_status), + [ATF_CLASS] = t->classification, + [ATF_TYPE] = t->type, + [ATF_COMPONENT] = t->component, + [ATF_ROLE] = t->recipient && *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient), + [ATF_NODE] = machine_guid, + [ATF_ALERT_NAME] = t->alert_name, + [ATF_CHART_NAME] = t->chart_name, + [ATF_CONTEXT] = t->chart_context, + }; + + for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { + if (!facets[i] || !*facets[i]) facets[i] = "unknown"; + + struct facet_entry tmp = { + .count = 0, + }; + dictionary_set(d->facets[i].dict, facets[i], &tmp, sizeof(tmp)); + } + + bool selected[ATF_TOTAL_ENTRIES] = { 0 }; + + uint32_t selected_by = 0; + for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { + selected[i] = !d->facets[i].pattern || simple_pattern_matches(d->facets[i].pattern, facets[i]); + if(selected[i]) + selected_by++; + } + + if(selected_by == ATF_TOTAL_ENTRIES) { + // this item is selected by all facets + // put it in our result (if it fits) + contexts_v2_alert_transition_keep(d, t, machine_guid); + } + + if(selected_by >= ATF_TOTAL_ENTRIES - 1) { + // this item is selected by all, or all except one facet + // in both cases we need to add it to our counters + + for (size_t i = 0; i < ATF_TOTAL_ENTRIES; i++) { + uint32_t counted_by = selected_by; + + if (counted_by != ATF_TOTAL_ENTRIES) { + counted_by = 0; + for (size_t j = 0; j < ATF_TOTAL_ENTRIES; j++) { + if (i == j || selected[j]) + counted_by++; + } + } + + if (counted_by == ATF_TOTAL_ENTRIES) { + // we need to count it on this facet + struct facet_entry *x = dictionary_get(d->facets[i].dict, facets[i]); + internal_fatal(!x, "facet is not found"); + if(x) + x->count++; + } + } + } +} + +void contexts_v2_alert_transitions_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_data *ctl, bool debug) { + struct alert_transitions_callback_data data = { + .wb = wb, + .ctl = ctl, + .debug = debug, + .only_one_config = true, + .max_items_to_return = ctl->request->alerts.last, + .items_to_return = 0, + .base = NULL, + }; + + for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { + data.facets[i].dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, sizeof(struct facet_entry)); + if(ctl->request->alerts.facets[i]) + data.facets[i].pattern = simple_pattern_create(ctl->request->alerts.facets[i], ",|", SIMPLE_PATTERN_EXACT, false); + } + + sql_alert_transitions( + ctl->nodes.dict, + ctl->window.after, + ctl->window.before, + ctl->request->contexts, + ctl->request->alerts.alert, + ctl->request->alerts.transition, + contexts_v2_alert_transition_callback, + &data, + debug); + + buffer_json_member_add_array(wb, "facets"); + for (size_t i = 0; i < ATF_TOTAL_ENTRIES; i++) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "id", alert_transition_facets[i].id); + buffer_json_member_add_string(wb, "name", alert_transition_facets[i].name); + buffer_json_member_add_uint64(wb, "order", alert_transition_facets[i].order); + buffer_json_member_add_array(wb, "options"); + { + struct facet_entry *x; + dfe_start_read(data.facets[i].dict, x) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "id", x_dfe.name); + if (i == ATF_NODE) { + RRDHOST *host = rrdhost_find_by_guid(x_dfe.name); + if (host) + buffer_json_member_add_string(wb, "name", rrdhost_hostname(host)); + else + buffer_json_member_add_string(wb, "name", x_dfe.name); + } else + buffer_json_member_add_string(wb, "name", x_dfe.name); + buffer_json_member_add_uint64(wb, "count", x->count); + } + buffer_json_object_close(wb); + } + dfe_done(x); + } + buffer_json_array_close(wb); // options + } + buffer_json_object_close(wb); // facet + } + buffer_json_array_close(wb); // facets + + buffer_json_member_add_array(wb, "transitions"); + for(struct sql_alert_transition_fixed_size *t = data.base; t ; t = t->next) { + buffer_json_add_array_item_object(wb); + { + RRDHOST *host = rrdhost_find_by_guid(t->machine_guid); + + buffer_json_member_add_uint64(wb, "gi", t->global_id); + buffer_json_member_add_uuid(wb, "transition_id", t->transition_id); + buffer_json_member_add_uuid(wb, "config_hash_id", t->config_hash_id); + buffer_json_member_add_string(wb, "machine_guid", t->machine_guid); + + if(host) { + buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host)); + + if(!UUIDiszero(host->node_id)) + buffer_json_member_add_uuid(wb, "node_id", host->node_id.uuid); + } + + buffer_json_member_add_string(wb, "alert", *t->alert_name ? t->alert_name : NULL); + buffer_json_member_add_string(wb, "instance", *t->chart ? t->chart : NULL); + buffer_json_member_add_string(wb, "instance_n", *t->chart_name ? t->chart_name : NULL); + buffer_json_member_add_string(wb, "context", *t->chart_context ? t->chart_context : NULL); + // buffer_json_member_add_string(wb, "family", *t->family ? t->family : NULL); + buffer_json_member_add_string(wb, "component", *t->component ? t->component : NULL); + buffer_json_member_add_string(wb, "classification", *t->classification ? t->classification : NULL); + buffer_json_member_add_string(wb, "type", *t->type ? t->type : NULL); + + buffer_json_member_add_time_t(wb, "when", t->when_key); + buffer_json_member_add_string(wb, "info", *t->info ? t->info : ""); + buffer_json_member_add_string(wb, "summary", *t->summary ? t->summary : ""); + buffer_json_member_add_string(wb, "units", *t->units ? t->units : NULL); + buffer_json_member_add_object(wb, "new"); + { + buffer_json_member_add_string(wb, "status", rrdcalc_status2string(t->new_status)); + buffer_json_member_add_double(wb, "value", t->new_value); + } + buffer_json_object_close(wb); // new + buffer_json_member_add_object(wb, "old"); + { + buffer_json_member_add_string(wb, "status", rrdcalc_status2string(t->old_status)); + buffer_json_member_add_double(wb, "value", t->old_value); + buffer_json_member_add_time_t(wb, "duration", t->duration); + buffer_json_member_add_time_t(wb, "raised_duration", t->non_clear_duration); + } + buffer_json_object_close(wb); // old + + buffer_json_member_add_object(wb, "notification"); + { + buffer_json_member_add_time_t(wb, "when", t->exec_run_timestamp); + buffer_json_member_add_time_t(wb, "delay", t->delay); + buffer_json_member_add_time_t(wb, "delay_up_to_time", t->delay_up_to_timestamp); + health_entry_flags_to_json_array(wb, "flags", t->flags); + buffer_json_member_add_string(wb, "exec", *t->exec ? t->exec : string2str(localhost->health.health_default_exec)); + buffer_json_member_add_uint64(wb, "exec_code", t->exec_code); + buffer_json_member_add_string(wb, "to", *t->recipient ? t->recipient : string2str(localhost->health.health_default_recipient)); + } + buffer_json_object_close(wb); // notification + } + buffer_json_object_close(wb); // a transition + } + buffer_json_array_close(wb); // all transitions + + if(ctl->options & CONTEXTS_OPTION_ALERTS_WITH_CONFIGURATIONS) { + DICTIONARY *configs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE); + + for(struct sql_alert_transition_fixed_size *t = data.base; t ; t = t->next) { + char guid[UUID_STR_LEN]; + uuid_unparse_lower(t->config_hash_id, guid); + dictionary_set(configs, guid, NULL, 0); + } + + buffer_json_member_add_array(wb, "configurations"); + sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, debug); + buffer_json_array_close(wb); + + dictionary_destroy(configs); + } + + while(data.base) { + struct sql_alert_transition_fixed_size *t = data.base; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(data.base, t, prev, next); + contexts_v2_alert_transition_free(t); + } + + for(size_t i = 0; i < ATF_TOTAL_ENTRIES ;i++) { + dictionary_destroy(data.facets[i].dict); + simple_pattern_free(data.facets[i].pattern); + } + + buffer_json_member_add_object(wb, "items"); + { + // all the items in the window, under the scope_nodes, ignoring the facets (filters) + buffer_json_member_add_uint64(wb, "evaluated", data.items_evaluated); + + // all the items matching the query (if you didn't put anchor_gi and last, these are all the items you would get back) + buffer_json_member_add_uint64(wb, "matched", data.items_matched); + + // the items included in this response + buffer_json_member_add_uint64(wb, "returned", data.items_to_return); + + // same as last=X parameter + buffer_json_member_add_uint64(wb, "max_to_return", data.max_items_to_return); + + // items before the first returned, this should be 0 if anchor_gi is not set + buffer_json_member_add_uint64(wb, "before", data.operations.skips_before); + + // items after the last returned, when this is zero there aren't any items after the current list + buffer_json_member_add_uint64(wb, "after", data.operations.skips_after + data.operations.shifts); + } + buffer_json_object_close(wb); // items + + if(debug) { + buffer_json_member_add_object(wb, "stats"); + { + buffer_json_member_add_uint64(wb, "first", data.operations.first); + buffer_json_member_add_uint64(wb, "prepend", data.operations.prepend); + buffer_json_member_add_uint64(wb, "append", data.operations.append); + buffer_json_member_add_uint64(wb, "backwards", data.operations.backwards); + buffer_json_member_add_uint64(wb, "forwards", data.operations.forwards); + buffer_json_member_add_uint64(wb, "shifts", data.operations.shifts); + buffer_json_member_add_uint64(wb, "skips_before", data.operations.skips_before); + buffer_json_member_add_uint64(wb, "skips_after", data.operations.skips_after); + } + buffer_json_object_close(wb); + } +} diff --git a/src/database/contexts/api_v2_contexts_alerts.c b/src/database/contexts/api_v2_contexts_alerts.c new file mode 100644 index 000000000..ea7f977bb --- /dev/null +++ b/src/database/contexts/api_v2_contexts_alerts.c @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "api_v2_contexts.h" + +struct alert_counts { + size_t critical; + size_t warning; + size_t clear; + size_t error; +}; + +struct alert_v2_entry { + RRDCALC *tmp; + + STRING *name; + STRING *summary; + RRDLABELS *recipient; + RRDLABELS *classification; + RRDLABELS *context; + RRDLABELS *component; + RRDLABELS *type; + + size_t ati; + + struct alert_counts counts; + + size_t instances; + DICTIONARY *nodes; + DICTIONARY *configs; +}; + +struct alert_by_x_entry { + struct { + struct alert_counts counts; + size_t silent; + size_t total; + } running; + + struct { + size_t available; + } prototypes; +}; + +bool rrdcontext_matches_alert(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc) { + size_t matches = 0; + RRDINSTANCE *ri; + dfe_start_read(rc->rrdinstances, ri) { + if(ri->rrdset) { + RRDSET *st = ri->rrdset; + rw_spinlock_read_lock(&st->alerts.spinlock); + for (RRDCALC *rcl = st->alerts.base; rcl; rcl = rcl->next) { + if(ctl->alerts.alert_name_pattern && !simple_pattern_matches_string(ctl->alerts.alert_name_pattern, rcl->config.name)) + continue; + + if(ctl->alerts.alarm_id_filter && ctl->alerts.alarm_id_filter != rcl->id) + continue; + + size_t m = ctl->request->alerts.status & CONTEXTS_ALERT_STATUSES ? 0 : 1; + + if (!m) { + if ((ctl->request->alerts.status & CONTEXT_ALERT_UNINITIALIZED) && + rcl->status == RRDCALC_STATUS_UNINITIALIZED) + m++; + + if ((ctl->request->alerts.status & CONTEXT_ALERT_UNDEFINED) && + rcl->status == RRDCALC_STATUS_UNDEFINED) + m++; + + if ((ctl->request->alerts.status & CONTEXT_ALERT_CLEAR) && + rcl->status == RRDCALC_STATUS_CLEAR) + m++; + + if ((ctl->request->alerts.status & CONTEXT_ALERT_RAISED) && + rcl->status >= RRDCALC_STATUS_RAISED) + m++; + + if ((ctl->request->alerts.status & CONTEXT_ALERT_WARNING) && + rcl->status == RRDCALC_STATUS_WARNING) + m++; + + if ((ctl->request->alerts.status & CONTEXT_ALERT_CRITICAL) && + rcl->status == RRDCALC_STATUS_CRITICAL) + m++; + + if(!m) + continue; + } + + struct alert_v2_entry t = { + .tmp = rcl, + }; + struct alert_v2_entry *a2e = + dictionary_set(ctl->alerts.summary, string2str(rcl->config.name), + &t, sizeof(struct alert_v2_entry)); + size_t ati = a2e->ati; + matches++; + + dictionary_set_advanced(ctl->alerts.by_type, + string2str(rcl->config.type), + (ssize_t)string_strlen(rcl->config.type), + NULL, + sizeof(struct alert_by_x_entry), + rcl); + + dictionary_set_advanced(ctl->alerts.by_component, + string2str(rcl->config.component), + (ssize_t)string_strlen(rcl->config.component), + NULL, + sizeof(struct alert_by_x_entry), + rcl); + + dictionary_set_advanced(ctl->alerts.by_classification, + string2str(rcl->config.classification), + (ssize_t)string_strlen(rcl->config.classification), + NULL, + sizeof(struct alert_by_x_entry), + rcl); + + dictionary_set_advanced(ctl->alerts.by_recipient, + string2str(rcl->config.recipient), + (ssize_t)string_strlen(rcl->config.recipient), + NULL, + sizeof(struct alert_by_x_entry), + rcl); + + char *module = NULL; + rrdlabels_get_value_strdup_or_null(st->rrdlabels, &module, "_collect_module"); + if(!module || !*module) module = "[unset]"; + + dictionary_set_advanced(ctl->alerts.by_module, + module, + -1, + NULL, + sizeof(struct alert_by_x_entry), + rcl); + + if (ctl->options & (CONTEXTS_OPTION_ALERTS_WITH_INSTANCES | CONTEXTS_OPTION_ALERTS_WITH_VALUES)) { + char key[20 + 1]; + snprintfz(key, sizeof(key) - 1, "%p", rcl); + + struct sql_alert_instance_v2_entry z = { + .ati = ati, + .tmp = rcl, + }; + dictionary_set(ctl->alerts.alert_instances, key, &z, sizeof(z)); + } + } + rw_spinlock_read_unlock(&st->alerts.spinlock); + } + } + dfe_done(ri); + + return matches != 0; +} + +static void alert_counts_add(struct alert_counts *t, RRDCALC *rc) { + switch(rc->status) { + case RRDCALC_STATUS_CRITICAL: + t->critical++; + break; + + case RRDCALC_STATUS_WARNING: + t->warning++; + break; + + case RRDCALC_STATUS_CLEAR: + t->clear++; + break; + + case RRDCALC_STATUS_REMOVED: + case RRDCALC_STATUS_UNINITIALIZED: + break; + + case RRDCALC_STATUS_UNDEFINED: + default: + if(!netdata_double_isnumber(rc->value)) + t->error++; + + break; + } +} + +static void alerts_v2_add(struct alert_v2_entry *t, RRDCALC *rc) { + t->instances++; + + alert_counts_add(&t->counts, rc); + + dictionary_set(t->nodes, rc->rrdset->rrdhost->machine_guid, NULL, 0); + + char key[UUID_STR_LEN + 1]; + uuid_unparse_lower(rc->config.hash_id, key); + dictionary_set(t->configs, key, NULL, 0); +} + +static void alerts_by_x_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + static STRING *silent = NULL; + if(unlikely(!silent)) silent = string_strdupz("silent"); + + struct alert_by_x_entry *b = value; + RRDCALC *rc = data; + if(!rc) { + // prototype + b->prototypes.available++; + } + else { + alert_counts_add(&b->running.counts, rc); + + b->running.total++; + + if (rc->config.recipient == silent) + b->running.silent++; + } +} + +static bool alerts_by_x_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value __maybe_unused, void *data __maybe_unused) { + alerts_by_x_insert_callback(item, old_value, data); + return false; +} + +static void alerts_v2_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + struct rrdcontext_to_json_v2_data *ctl = data; + struct alert_v2_entry *t = value; + RRDCALC *rc = t->tmp; + t->name = rc->config.name; + t->summary = rc->config.summary; // the original summary + t->context = rrdlabels_create(); + t->recipient = rrdlabels_create(); + t->classification = rrdlabels_create(); + t->component = rrdlabels_create(); + t->type = rrdlabels_create(); + if (string_strlen(rc->rrdset->context)) + rrdlabels_add(t->context, string2str(rc->rrdset->context), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.recipient)) + rrdlabels_add(t->recipient, string2str(rc->config.recipient), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.classification)) + rrdlabels_add(t->classification, string2str(rc->config.classification), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.component)) + rrdlabels_add(t->component, string2str(rc->config.component), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.type)) + rrdlabels_add(t->type, string2str(rc->config.type), "yes", RRDLABEL_SRC_AUTO); + t->ati = ctl->alerts.ati++; + + t->nodes = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_VALUE_LINK_DONT_CLONE|DICT_OPTION_NAME_LINK_DONT_CLONE); + t->configs = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_VALUE_LINK_DONT_CLONE|DICT_OPTION_NAME_LINK_DONT_CLONE); + + alerts_v2_add(t, rc); +} + +static bool alerts_v2_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) { + struct alert_v2_entry *t = old_value, *n = new_value; + RRDCALC *rc = n->tmp; + if (string_strlen(rc->rrdset->context)) + rrdlabels_add(t->context, string2str(rc->rrdset->context), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.recipient)) + rrdlabels_add(t->recipient, string2str(rc->config.recipient), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.classification)) + rrdlabels_add(t->classification, string2str(rc->config.classification), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.component)) + rrdlabels_add(t->component, string2str(rc->config.component), "yes", RRDLABEL_SRC_AUTO); + if (string_strlen(rc->config.type)) + rrdlabels_add(t->type, string2str(rc->config.type), "yes", RRDLABEL_SRC_AUTO); + alerts_v2_add(t, rc); + return true; +} + +static void alerts_v2_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct alert_v2_entry *t = value; + + rrdlabels_destroy(t->context); + rrdlabels_destroy(t->recipient); + rrdlabels_destroy(t->classification); + rrdlabels_destroy(t->component); + rrdlabels_destroy(t->type); + + dictionary_destroy(t->nodes); + dictionary_destroy(t->configs); +} + +struct alert_instances_callback_data { + BUFFER *wb; + struct rrdcontext_to_json_v2_data *ctl; + bool debug; +}; + +static int contexts_v2_alert_instance_to_json_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + struct sql_alert_instance_v2_entry *t = value; + struct alert_instances_callback_data *d = data; + struct rrdcontext_to_json_v2_data *ctl = d->ctl; (void)ctl; + bool debug = d->debug; (void)debug; + BUFFER *wb = d->wb; + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_uint64(wb, "ni", t->ni); + + buffer_json_member_add_string(wb, "nm", string2str(t->name)); + buffer_json_member_add_string(wb, "ch", string2str(t->chart_id)); + buffer_json_member_add_string(wb, "ch_n", string2str(t->chart_name)); + + if(ctl->request->options & CONTEXTS_OPTION_ALERTS_WITH_SUMMARY) + buffer_json_member_add_uint64(wb, "ati", t->ati); + + if(ctl->request->options & CONTEXTS_OPTION_ALERTS_WITH_INSTANCES) { + buffer_json_member_add_string(wb, "units", string2str(t->units)); + buffer_json_member_add_string(wb, "fami", string2str(t->family)); + buffer_json_member_add_string(wb, "info", string2str(t->info)); + buffer_json_member_add_string(wb, "sum", string2str(t->summary)); + buffer_json_member_add_string(wb, "ctx", string2str(t->context)); + buffer_json_member_add_string(wb, "st", rrdcalc_status2string(t->status)); + buffer_json_member_add_uuid(wb, "tr_i", t->last_transition_id); + buffer_json_member_add_double(wb, "tr_v", t->last_status_change_value); + buffer_json_member_add_time_t(wb, "tr_t", t->last_status_change); + buffer_json_member_add_uuid(wb, "cfg", t->config_hash_id); + buffer_json_member_add_string(wb, "src", string2str(t->source)); + + buffer_json_member_add_string(wb, "to", string2str(t->recipient)); + buffer_json_member_add_string(wb, "tp", string2str(t->type)); + buffer_json_member_add_string(wb, "cm", string2str(t->component)); + buffer_json_member_add_string(wb, "cl", string2str(t->classification)); + + // Agent specific fields + buffer_json_member_add_uint64(wb, "gi", t->global_id); + // rrdcalc_flags_to_json_array (wb, "flags", t->flags); + } + + if(ctl->request->options & CONTEXTS_OPTION_ALERTS_WITH_VALUES) { + // Netdata Cloud fetched these by querying the agents + buffer_json_member_add_double(wb, "v", t->value); + buffer_json_member_add_time_t(wb, "t", t->last_updated); + } + } + buffer_json_object_close(wb); // alert instance + + return 1; +} + +static void contexts_v2_alerts_by_x_update_prototypes(void *data, STRING *type, STRING *component, STRING *classification, STRING *recipient) { + struct rrdcontext_to_json_v2_data *ctl = data; + + dictionary_set_advanced(ctl->alerts.by_type, string2str(type), (ssize_t)string_strlen(type), NULL, sizeof(struct alert_by_x_entry), NULL); + dictionary_set_advanced(ctl->alerts.by_component, string2str(component), (ssize_t)string_strlen(component), NULL, sizeof(struct alert_by_x_entry), NULL); + dictionary_set_advanced(ctl->alerts.by_classification, string2str(classification), (ssize_t)string_strlen(classification), NULL, sizeof(struct alert_by_x_entry), NULL); + dictionary_set_advanced(ctl->alerts.by_recipient, string2str(recipient), (ssize_t)string_strlen(recipient), NULL, sizeof(struct alert_by_x_entry), NULL); +} + +static void contexts_v2_alerts_by_x_to_json(BUFFER *wb, DICTIONARY *dict, const char *key) { + buffer_json_member_add_array(wb, key); + { + struct alert_by_x_entry *b; + dfe_start_read(dict, b) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "name", b_dfe.name); + buffer_json_member_add_uint64(wb, "cr", b->running.counts.critical); + buffer_json_member_add_uint64(wb, "wr", b->running.counts.warning); + buffer_json_member_add_uint64(wb, "cl", b->running.counts.clear); + buffer_json_member_add_uint64(wb, "er", b->running.counts.error); + buffer_json_member_add_uint64(wb, "running", b->running.total); + + buffer_json_member_add_uint64(wb, "running_silent", b->running.silent); + + if(b->prototypes.available) + buffer_json_member_add_uint64(wb, "available", b->prototypes.available); + } + buffer_json_object_close(wb); + } + dfe_done(b); + } + buffer_json_array_close(wb); +} + +static void contexts_v2_alert_instances_to_json(BUFFER *wb, const char *key, struct rrdcontext_to_json_v2_data *ctl, bool debug) { + buffer_json_member_add_array(wb, key); + { + struct alert_instances_callback_data data = { + .wb = wb, + .ctl = ctl, + .debug = debug, + }; + dictionary_walkthrough_rw(ctl->alerts.alert_instances, DICTIONARY_LOCK_READ, + contexts_v2_alert_instance_to_json_callback, &data); + } + buffer_json_array_close(wb); // alerts_instances +} + +void contexts_v2_alerts_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_data *ctl, bool debug) { + if(ctl->request->options & CONTEXTS_OPTION_ALERTS_WITH_SUMMARY) { + buffer_json_member_add_array(wb, "alerts"); + { + struct alert_v2_entry *t; + dfe_start_read(ctl->alerts.summary, t) + { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_uint64(wb, "ati", t->ati); + + buffer_json_member_add_array(wb, "ni"); + void *host_guid; + dfe_start_read(t->nodes, host_guid) { + struct contexts_v2_node *cn = dictionary_get(ctl->nodes.dict,host_guid_dfe.name); + buffer_json_add_array_item_int64(wb, (int64_t) cn->ni); + } + dfe_done(host_guid); + buffer_json_array_close(wb); + + buffer_json_member_add_string(wb, "nm", string2str(t->name)); + buffer_json_member_add_string(wb, "sum", string2str(t->summary)); + + buffer_json_member_add_uint64(wb, "cr", t->counts.critical); + buffer_json_member_add_uint64(wb, "wr", t->counts.warning); + buffer_json_member_add_uint64(wb, "cl", t->counts.clear); + buffer_json_member_add_uint64(wb, "er", t->counts.error); + + buffer_json_member_add_uint64(wb, "in", t->instances); + buffer_json_member_add_uint64(wb, "nd", dictionary_entries(t->nodes)); + buffer_json_member_add_uint64(wb, "cfg", dictionary_entries(t->configs)); + + buffer_json_member_add_array(wb, "ctx"); + rrdlabels_key_to_buffer_array_item(t->context, wb); + buffer_json_array_close(wb); // ctx + + buffer_json_member_add_array(wb, "cls"); + rrdlabels_key_to_buffer_array_item(t->classification, wb); + buffer_json_array_close(wb); // classification + + + buffer_json_member_add_array(wb, "cp"); + rrdlabels_key_to_buffer_array_item(t->component, wb); + buffer_json_array_close(wb); // component + + buffer_json_member_add_array(wb, "ty"); + rrdlabels_key_to_buffer_array_item(t->type, wb); + buffer_json_array_close(wb); // type + + buffer_json_member_add_array(wb, "to"); + rrdlabels_key_to_buffer_array_item(t->recipient, wb); + buffer_json_array_close(wb); // recipient + } + buffer_json_object_close(wb); // alert name + } + dfe_done(t); + } + buffer_json_array_close(wb); // alerts + + health_prototype_metadata_foreach(ctl, contexts_v2_alerts_by_x_update_prototypes); + contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_type, "alerts_by_type"); + contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_component, "alerts_by_component"); + contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_classification, "alerts_by_classification"); + contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_recipient, "alerts_by_recipient"); + contexts_v2_alerts_by_x_to_json(wb, ctl->alerts.by_module, "alerts_by_module"); + } + + if(ctl->request->options & (CONTEXTS_OPTION_ALERTS_WITH_INSTANCES | CONTEXTS_OPTION_ALERTS_WITH_VALUES)) { + contexts_v2_alert_instances_to_json(wb, "alert_instances", ctl, debug); + } +} + +static void alert_instances_v2_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + struct rrdcontext_to_json_v2_data *ctl = data; + struct sql_alert_instance_v2_entry *t = value; + RRDCALC *rc = t->tmp; + + t->context = rc->rrdset->context; + t->chart_id = rc->rrdset->id; + t->chart_name = rc->rrdset->name; + t->family = rc->rrdset->family; + t->units = rc->config.units; + t->classification = rc->config.classification; + t->type = rc->config.type; + t->recipient = rc->config.recipient; + t->component = rc->config.component; + t->name = rc->config.name; + t->source = rc->config.source; + t->status = rc->status; + t->flags = rc->run_flags; + t->info = rc->config.info; + t->summary = rc->summary; + t->value = rc->value; + t->last_updated = rc->last_updated; + t->last_status_change = rc->last_status_change; + t->last_status_change_value = rc->last_status_change_value; + t->host = rc->rrdset->rrdhost; + t->alarm_id = rc->id; + t->ni = ctl->nodes.ni; + + uuid_copy(t->config_hash_id, rc->config.hash_id); + health_alarm_log_get_global_id_and_transition_id_for_rrdcalc(rc, &t->global_id, &t->last_transition_id); +} + +static bool alert_instances_v2_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value __maybe_unused, void *new_value __maybe_unused, void *data __maybe_unused) { + internal_fatal(true, "This should never happen!"); + return true; +} + +static void alert_instances_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) { + ; +} + +static void rrdcontext_v2_set_transition_filter(const char *machine_guid, const char *context, time_t alarm_id, void *data) { + struct rrdcontext_to_json_v2_data *ctl = data; + + if(machine_guid && *machine_guid) { + if(ctl->nodes.scope_pattern) + simple_pattern_free(ctl->nodes.scope_pattern); + + if(ctl->nodes.pattern) + simple_pattern_free(ctl->nodes.pattern); + + ctl->nodes.scope_pattern = string_to_simple_pattern(machine_guid); + ctl->nodes.pattern = NULL; + } + + if(context && *context) { + if(ctl->contexts.scope_pattern) + simple_pattern_free(ctl->contexts.scope_pattern); + + if(ctl->contexts.pattern) + simple_pattern_free(ctl->contexts.pattern); + + ctl->contexts.scope_pattern = string_to_simple_pattern(context); + ctl->contexts.pattern = NULL; + } + + ctl->alerts.alarm_id_filter = alarm_id; +} + +bool rrdcontexts_v2_init_alert_dictionaries(struct rrdcontext_to_json_v2_data *ctl, struct api_v2_contexts_request *req) { + if(req->alerts.transition) { + ctl->options |= CONTEXTS_OPTION_ALERTS_WITH_INSTANCES | CONTEXTS_OPTION_ALERTS_WITH_VALUES; + if(!sql_find_alert_transition(req->alerts.transition, rrdcontext_v2_set_transition_filter, ctl)) + return false; + } + + ctl->alerts.summary = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, + sizeof(struct alert_v2_entry)); + + dictionary_register_insert_callback(ctl->alerts.summary, alerts_v2_insert_callback, ctl); + dictionary_register_conflict_callback(ctl->alerts.summary, alerts_v2_conflict_callback, ctl); + dictionary_register_delete_callback(ctl->alerts.summary, alerts_v2_delete_callback, ctl); + + ctl->alerts.by_type = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, + sizeof(struct alert_by_x_entry)); + + dictionary_register_insert_callback(ctl->alerts.by_type, alerts_by_x_insert_callback, NULL); + dictionary_register_conflict_callback(ctl->alerts.by_type, alerts_by_x_conflict_callback, NULL); + + ctl->alerts.by_component = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, + sizeof(struct alert_by_x_entry)); + + dictionary_register_insert_callback(ctl->alerts.by_component, alerts_by_x_insert_callback, NULL); + dictionary_register_conflict_callback(ctl->alerts.by_component, alerts_by_x_conflict_callback, NULL); + + ctl->alerts.by_classification = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, + sizeof(struct alert_by_x_entry)); + + dictionary_register_insert_callback(ctl->alerts.by_classification, alerts_by_x_insert_callback, NULL); + dictionary_register_conflict_callback(ctl->alerts.by_classification, alerts_by_x_conflict_callback, NULL); + + ctl->alerts.by_recipient = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, + sizeof(struct alert_by_x_entry)); + + dictionary_register_insert_callback(ctl->alerts.by_recipient, alerts_by_x_insert_callback, NULL); + dictionary_register_conflict_callback(ctl->alerts.by_recipient, alerts_by_x_conflict_callback, NULL); + + ctl->alerts.by_module = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, + sizeof(struct alert_by_x_entry)); + + dictionary_register_insert_callback(ctl->alerts.by_module, alerts_by_x_insert_callback, NULL); + dictionary_register_conflict_callback(ctl->alerts.by_module, alerts_by_x_conflict_callback, NULL); + + if(ctl->options & (CONTEXTS_OPTION_ALERTS_WITH_INSTANCES | CONTEXTS_OPTION_ALERTS_WITH_VALUES)) { + ctl->alerts.alert_instances = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(struct sql_alert_instance_v2_entry)); + + dictionary_register_insert_callback(ctl->alerts.alert_instances, alert_instances_v2_insert_callback, ctl); + dictionary_register_conflict_callback(ctl->alerts.alert_instances, alert_instances_v2_conflict_callback, ctl); + dictionary_register_delete_callback(ctl->alerts.alert_instances, alert_instances_delete_callback, ctl); + } + + return true; +} + +void rrdcontexts_v2_alerts_cleanup(struct rrdcontext_to_json_v2_data *ctl) { + dictionary_destroy(ctl->alerts.summary); + dictionary_destroy(ctl->alerts.alert_instances); + dictionary_destroy(ctl->alerts.by_type); + dictionary_destroy(ctl->alerts.by_component); + dictionary_destroy(ctl->alerts.by_classification); + dictionary_destroy(ctl->alerts.by_recipient); + dictionary_destroy(ctl->alerts.by_module); +} diff --git a/src/database/contexts/api_v2_contexts_alerts.h b/src/database/contexts/api_v2_contexts_alerts.h new file mode 100644 index 000000000..b7be3f4d9 --- /dev/null +++ b/src/database/contexts/api_v2_contexts_alerts.h @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_API_V2_CONTEXTS_ALERTS_H +#define NETDATA_API_V2_CONTEXTS_ALERTS_H + +#include "internal.h" +#include "api_v2_contexts.h" + +struct alert_transitions_callback_data { + struct rrdcontext_to_json_v2_data *ctl; + BUFFER *wb; + bool debug; + bool only_one_config; + + struct { + SIMPLE_PATTERN *pattern; + DICTIONARY *dict; + } facets[ATF_TOTAL_ENTRIES]; + + uint32_t max_items_to_return; + uint32_t items_to_return; + + uint32_t items_evaluated; + uint32_t items_matched; + + + struct sql_alert_transition_fixed_size *base; // double linked list - last item is base->prev + struct sql_alert_transition_fixed_size *last_added; // the last item added, not the last of the list + + struct { + size_t first; + size_t skips_before; + size_t skips_after; + size_t backwards; + size_t forwards; + size_t prepend; + size_t append; + size_t shifts; + } operations; + + uint32_t configs_added; +}; + +void contexts_v2_alerts_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_data *ctl, bool debug); +bool rrdcontext_matches_alert(struct rrdcontext_to_json_v2_data *ctl, RRDCONTEXT *rc); +void contexts_v2_alert_config_to_json_from_sql_alert_config_data(struct sql_alert_config_data *t, void *data); +void contexts_v2_alert_transitions_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_data *ctl, bool debug); + +bool rrdcontexts_v2_init_alert_dictionaries(struct rrdcontext_to_json_v2_data *ctl, struct api_v2_contexts_request *req); +void rrdcontexts_v2_alerts_cleanup(struct rrdcontext_to_json_v2_data *ctl); + +#endif //NETDATA_API_V2_CONTEXTS_ALERTS_H diff --git a/src/database/contexts/instance.c b/src/database/contexts/instance.c index 5d841bc82..ee6a906e7 100644 --- a/src/database/contexts/instance.c +++ b/src/database/contexts/instance.c @@ -37,6 +37,11 @@ inline STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria) { inline RRDLABELS *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) { RRDINSTANCE *ri = rrdinstance_acquired_value(ria); + if (rrd_flag_check(ri, RRD_FLAG_OWN_LABELS | RRD_FLAG_DEMAND_LABELS)) { + rrd_flag_clear(ri, RRD_FLAG_DEMAND_LABELS); + load_instance_labels_on_demand(&ri->uuid, ri); + rrdinstance_trigger_updates(ri, __FUNCTION__ ); + } return ri->rrdlabels; } @@ -101,11 +106,11 @@ static void rrdinstance_insert_callback(const DICTIONARY_ITEM *item __maybe_unus if(ri->rrdset) { ri->rrdlabels = ri->rrdset->rrdlabels; - ri->flags &= ~RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor + ri->flags &= ~(RRD_FLAG_OWN_LABELS| RRD_FLAG_DEMAND_LABELS); // no need of atomics at the constructor } else { ri->rrdlabels = rrdlabels_create(); - ri->flags |= RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor + ri->flags |= (RRD_FLAG_OWN_LABELS | RRD_FLAG_DEMAND_LABELS); // no need of atomics at the constructor } if(ri->rrdset) { @@ -213,12 +218,12 @@ static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_un if(ri->rrdset && rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) { RRDLABELS *old = ri->rrdlabels; ri->rrdlabels = ri->rrdset->rrdlabels; - rrd_flag_clear(ri, RRD_FLAG_OWN_LABELS); + rrd_flag_clear(ri, RRD_FLAG_OWN_LABELS| RRD_FLAG_DEMAND_LABELS); rrdlabels_destroy(old); } else if(!ri->rrdset && !rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) { ri->rrdlabels = rrdlabels_create(); - rrd_flag_set(ri, RRD_FLAG_OWN_LABELS); + rrd_flag_set(ri, RRD_FLAG_OWN_LABELS | RRD_FLAG_DEMAND_LABELS); } } @@ -431,8 +436,9 @@ inline void rrdinstance_rrdset_is_freed(RRDSET *st) { if(!rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) { ri->rrdlabels = rrdlabels_create(); - rrdlabels_copy(ri->rrdlabels, st->rrdlabels); - rrd_flag_set(ri, RRD_FLAG_OWN_LABELS); + // Do not load copy labels, just load on demand + //rrdlabels_copy(ri->rrdlabels, st->rrdlabels); + rrd_flag_set(ri, RRD_FLAG_OWN_LABELS | RRD_FLAG_DEMAND_LABELS); } ri->rrdset = NULL; diff --git a/src/database/contexts/internal.h b/src/database/contexts/internal.h index 270c59649..2c69077e5 100644 --- a/src/database/contexts/internal.h +++ b/src/database/contexts/internal.h @@ -39,25 +39,26 @@ typedef enum __attribute__ ((__packed__)) { RRD_FLAG_UPDATED = (1 << 2), // this object has updates to propagate RRD_FLAG_ARCHIVED = (1 << 3), // this object is not currently being collected RRD_FLAG_OWN_LABELS = (1 << 4), // this instance has its own labels - not linked to an RRDSET - RRD_FLAG_LIVE_RETENTION = (1 << 5), // we have got live retention from the database - RRD_FLAG_QUEUED_FOR_HUB = (1 << 6), // this context is currently queued to be dispatched to hub - RRD_FLAG_QUEUED_FOR_PP = (1 << 7), // this context is currently queued to be post-processed - RRD_FLAG_HIDDEN = (1 << 8), // don't expose this to the hub or the API - - RRD_FLAG_UPDATE_REASON_TRIGGERED = (1 << 9), // the update was triggered by the child object - RRD_FLAG_UPDATE_REASON_LOAD_SQL = (1 << 10), // this object has just been loaded from SQL - RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 11), // this object has just been created - RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT = (1 << 12), // we received an update on this object - RRD_FLAG_UPDATE_REASON_CHANGED_LINKING = (1 << 13), // an instance or a metric switched RRDSET or RRDDIM - RRD_FLAG_UPDATE_REASON_CHANGED_METADATA = (1 << 14), // this context or instance changed uuid, name, units, title, family, chart type, priority, update every, rrd changed flags - RRD_FLAG_UPDATE_REASON_ZERO_RETENTION = (1 << 15), // this object has no retention - RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T = (1 << 16), // this object changed its oldest time in the db - RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T = (1 << 17), // this object change its latest time in the db - RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED = (1 << 18), // this object has stopped being collected - RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED = (1 << 19), // this object has started being collected - RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 20), // this context belongs to a host that just disconnected - RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 21), // this context is not used anymore - RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 22), // this context changed because of a db rotation + RRD_FLAG_DEMAND_LABELS = (1 << 5), // this instance should load labels on demand + RRD_FLAG_LIVE_RETENTION = (1 << 6), // we have got live retention from the database + RRD_FLAG_QUEUED_FOR_HUB = (1 << 7), // this context is currently queued to be dispatched to hub + RRD_FLAG_QUEUED_FOR_PP = (1 << 8), // this context is currently queued to be post-processed + RRD_FLAG_HIDDEN = (1 << 9), // don't expose this to the hub or the API + + RRD_FLAG_UPDATE_REASON_TRIGGERED = (1 << 10), // the update was triggered by the child object + RRD_FLAG_UPDATE_REASON_LOAD_SQL = (1 << 11), // this object has just been loaded from SQL + RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 12), // this object has just been created + RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT = (1 << 13), // we received an update on this object + RRD_FLAG_UPDATE_REASON_CHANGED_LINKING = (1 << 14), // an instance or a metric switched RRDSET or RRDDIM + RRD_FLAG_UPDATE_REASON_CHANGED_METADATA = (1 << 15), // this context or instance changed uuid, name, units, title, family, chart type, priority, update every, rrd changed flags + RRD_FLAG_UPDATE_REASON_ZERO_RETENTION = (1 << 16), // this object has no retention + RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T = (1 << 17), // this object changed its oldest time in the db + RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T = (1 << 18), // this object change its latest time in the db + RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED = (1 << 19), // this object has stopped being collected + RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED = (1 << 20), // this object has started being collected + RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 21), // this context belongs to a host that just disconnected + RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 22), // this context is not used anymore + RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 23), // this context changed because of a db rotation RRD_FLAG_MERGED_COLLECTED_RI_TO_RC = (1 << 29), @@ -354,6 +355,7 @@ static inline void rrdcontext_release(RRDCONTEXT_ACQUIRED *rca) { // ---------------------------------------------------------------------------- // Forward definitions +void load_instance_labels_on_demand(nd_uuid_t *uuid, void *data); void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs); void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs); diff --git a/src/database/contexts/query_scope.c b/src/database/contexts/query_scope.c index f3bcd0b3f..7485ef3e6 100644 --- a/src/database/contexts/query_scope.c +++ b/src/database/contexts/query_scope.c @@ -18,8 +18,8 @@ ssize_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN uint64_t t_hash = 0; dfe_start_read(rrdhost_root_index, host) { - if(host->node_id) - uuid_unparse_lower(*host->node_id, host_node_id_str); + if(!UUIDiszero(host->node_id)) + uuid_unparse_lower(host->node_id.uuid, host_node_id_str); else host_node_id_str[0] = '\0'; diff --git a/src/database/contexts/query_target.c b/src/database/contexts/query_target.c index 29a9c3e59..b25b8e427 100644 --- a/src/database/contexts/query_target.c +++ b/src/database/contexts/query_target.c @@ -897,9 +897,9 @@ static ssize_t query_node_add(void *data, RRDHOST *host, bool queryable_host) { QUERY_TARGET *qt = qtl->qt; QUERY_NODE *qn = query_node_allocate(qt, host); - if(host->node_id) { + if(!UUIDiszero(host->node_id)) { if(!qtl->host_node_id_str[0]) - uuid_unparse_lower(*host->node_id, qn->node_id); + uuid_unparse_lower(host->node_id.uuid, qn->node_id); else memcpy(qn->node_id, qtl->host_node_id_str, sizeof(qn->node_id)); } @@ -958,7 +958,7 @@ static ssize_t query_node_add(void *data, RRDHOST *host, bool queryable_host) { void query_target_generate_name(QUERY_TARGET *qt) { char options_buffer[100 + 1]; - web_client_api_request_v1_data_options_to_string(options_buffer, 100, qt->request.options); + web_client_api_request_data_vX_options_to_string(options_buffer, 100, qt->request.options); char resampling_buffer[20 + 1] = ""; if(qt->request.resampling_time > 1) @@ -1035,8 +1035,8 @@ void query_target_generate_name(QUERY_TARGET *qt) { } QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) { - if(!service_running(ABILITY_DATA_QUERIES)) - return NULL; + //if(!service_running(ABILITY_DATA_QUERIES)) + // return NULL; QUERY_TARGET *qt = query_target_get(); @@ -1120,8 +1120,8 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) { } if(host) { - if(host->node_id) - uuid_unparse_lower(*host->node_id, qtl.host_node_id_str); + if(!UUIDiszero(host->node_id)) + uuid_unparse_lower(host->node_id.uuid, qtl.host_node_id_str); else qtl.host_node_id_str[0] = '\0'; diff --git a/src/database/contexts/rrdcontext.c b/src/database/contexts/rrdcontext.c index f755e1f7e..a98bc98ef 100644 --- a/src/database/contexts/rrdcontext.c +++ b/src/database/contexts/rrdcontext.c @@ -198,21 +198,16 @@ int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char // ---------------------------------------------------------------------------- // ACLK interface -static bool rrdhost_check_our_claim_id(const char *claim_id) { - if(!localhost->aclk_state.claimed_id) return false; - return (strcasecmp(claim_id, localhost->aclk_state.claimed_id) == 0) ? true : false; -} - void rrdcontext_hub_checkpoint_command(void *ptr) { struct ctxs_checkpoint *cmd = ptr; - if(!rrdhost_check_our_claim_id(cmd->claim_id)) { + if(!claim_id_matches(cmd->claim_id)) { + CLAIM_ID claim_id = claim_id_get(); nd_log(NDLS_DAEMON, NDLP_WARNING, "RRDCONTEXT: received checkpoint command for claim_id '%s', node id '%s', " "but this is not our claim id. Ours '%s', received '%s'. Ignoring command.", cmd->claim_id, cmd->node_id, - localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET", - cmd->claim_id); + claim_id.str, cmd->claim_id); return; } @@ -245,11 +240,10 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { "Sending snapshot of all contexts.", cmd->version_hash, rrdhost_hostname(host), our_version_hash); -#ifdef ENABLE_ACLK // prepare the snapshot - char uuid[UUID_STR_LEN]; - uuid_unparse_lower(*host->node_id, uuid); - contexts_snapshot_t bundle = contexts_snapshot_new(cmd->claim_id, uuid, our_version_hash); + char uuid_str[UUID_STR_LEN]; + uuid_unparse_lower(host->node_id.uuid, uuid_str); + contexts_snapshot_t bundle = contexts_snapshot_new(cmd->claim_id, uuid_str, our_version_hash); // do a deep scan on every metric of the host to make sure all our data are updated rrdcontext_recalculate_host_retention(host, RRD_FLAG_NONE, false); @@ -262,7 +256,6 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { // send it aclk_send_contexts_snapshot(bundle); -#endif } nd_log(NDLS_DAEMON, NDLP_DEBUG, @@ -271,7 +264,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS); char node_str[UUID_STR_LEN]; - uuid_unparse_lower(*host->node_id, node_str); + uuid_unparse_lower(host->node_id.uuid, node_str); nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK REQ [%s (%s)]: STREAM CONTEXTS ENABLED", node_str, rrdhost_hostname(host)); @@ -280,13 +273,13 @@ void rrdcontext_hub_checkpoint_command(void *ptr) { void rrdcontext_hub_stop_streaming_command(void *ptr) { struct stop_streaming_ctxs *cmd = ptr; - if(!rrdhost_check_our_claim_id(cmd->claim_id)) { + if(!claim_id_matches(cmd->claim_id)) { + CLAIM_ID claim_id = claim_id_get(); nd_log(NDLS_DAEMON, NDLP_WARNING, "RRDCONTEXT: received stop streaming command for claim_id '%s', node id '%s', " "but this is not our claim id. Ours '%s', received '%s'. Ignoring command.", cmd->claim_id, cmd->node_id, - localhost->aclk_state.claimed_id?localhost->aclk_state.claimed_id:"NOT SET", - cmd->claim_id); + claim_id.str, cmd->claim_id); return; } diff --git a/src/database/contexts/rrdcontext.h b/src/database/contexts/rrdcontext.h index 9fea55d38..0906329bc 100644 --- a/src/database/contexts/rrdcontext.h +++ b/src/database/contexts/rrdcontext.h @@ -623,10 +623,10 @@ struct api_v2_contexts_request { char *contexts; char *q; - CONTEXTS_V2_OPTIONS options; + CONTEXTS_OPTIONS options; struct { - CONTEXTS_V2_ALERT_STATUS status; + CONTEXTS_ALERT_STATUS status; char *alert; char *transition; uint32_t last; diff --git a/src/database/contexts/worker.c b/src/database/contexts/worker.c index 6012c14f5..4ffa92139 100644 --- a/src/database/contexts/worker.c +++ b/src/database/contexts/worker.c @@ -24,6 +24,10 @@ static void rrdinstance_load_clabel(SQL_CLABEL_DATA *sld, void *data) { rrdlabels_add(ri->rrdlabels, sld->label_key, sld->label_value, sld->label_source); } +void load_instance_labels_on_demand(nd_uuid_t *uuid, void *data) { + ctx_get_label_list(uuid, rrdinstance_load_clabel, data); +} + static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) { RRDINSTANCE *ri = data; @@ -73,7 +77,6 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) { RRDINSTANCE *ri = rrdinstance_acquired_value(ria); ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri); - ctx_get_label_list(&ri->uuid, rrdinstance_load_clabel, ri); rrdinstance_trigger_updates(ri, __FUNCTION__ ); rrdinstance_release(ria); rrdcontext_release(rca); @@ -99,8 +102,11 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) { if(host->rrdctx.contexts) return; rrdhost_create_rrdcontexts(host); - ctx_get_context_list(&host->host_uuid, rrdcontext_load_context_callback, host); - ctx_get_chart_list(&host->host_uuid, rrdinstance_load_chart_callback, host); + if (host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return; + + ctx_get_context_list(&host->host_id.uuid, rrdcontext_load_context_callback, host); + ctx_get_chart_list(&host->host_id.uuid, rrdinstance_load_chart_callback, host); RRDCONTEXT *rc; dfe_start_read(host->rrdctx.contexts, rc) { @@ -173,6 +179,8 @@ static void rrdhost_update_cached_retention(RRDHOST *host, time_t first_time_s, spinlock_lock(&host->retention.spinlock); + time_t old_first_time_s = host->retention.first_time_s; + if(global) { host->retention.first_time_s = first_time_s; host->retention.last_time_s = last_time_s; @@ -185,7 +193,12 @@ static void rrdhost_update_cached_retention(RRDHOST *host, time_t first_time_s, host->retention.last_time_s = last_time_s; } + bool stream_path_update_required = old_first_time_s != host->retention.first_time_s; + spinlock_unlock(&host->retention.spinlock); + + if(stream_path_update_required) + stream_path_retention_updated(host); } void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs) { @@ -348,8 +361,11 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) { rc->hub.units = string2str(rc->units); rc->hub.family = string2str(rc->family); + if (rc->rrdhost->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return; + // delete it from SQL - if(ctx_delete_context(&rc->rrdhost->host_uuid, &rc->hub) != 0) + if(ctx_delete_context(&rc->rrdhost->host_id.uuid, &rc->hub) != 0) netdata_log_error("RRDCONTEXT: failed to delete context '%s' version %"PRIu64" from SQL.", rc->hub.id, rc->hub.version); } @@ -818,7 +834,6 @@ void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused rc->hub.last_time_s = rrd_flag_is_collected(rc) ? 0 : rc->last_time_s; rc->hub.deleted = rrd_flag_is_deleted(rc) ? true : false; -#ifdef ENABLE_ACLK struct context_updated message = { .id = rc->hub.id, .version = rc->hub.version, @@ -840,15 +855,19 @@ void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused else contexts_updated_add_ctx_update(bundle, &message); } -#endif // store it to SQL if(rrd_flag_is_deleted(rc)) rrdcontext_delete_from_sql_unsafe(rc); - else if (ctx_store_context(&rc->rrdhost->host_uuid, &rc->hub) != 0) - netdata_log_error("RRDCONTEXT: failed to save context '%s' version %"PRIu64" to SQL.", rc->hub.id, rc->hub.version); + else { + if (rc->rrdhost->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return; + if (ctx_store_context(&rc->rrdhost->host_id.uuid, &rc->hub) != 0) + netdata_log_error( + "RRDCONTEXT: failed to save context '%s' version %" PRIu64 " to SQL.", rc->hub.id, rc->hub.version); + } } static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused) { @@ -956,7 +975,7 @@ static void rrdcontext_dequeue_from_hub_queue(RRDCONTEXT *rc) { static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now_ut) { // check if we have received a streaming command for this host - if(!host->node_id || !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS) || !aclk_connected || !host->rrdctx.hub_queue) + if(UUIDiszero(host->node_id) || !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS) || !aclk_online_for_contexts() || !host->rrdctx.hub_queue) return; // check if there are queued items to send @@ -975,9 +994,9 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now worker_is_busy(WORKER_JOB_QUEUED); usec_t dispatch_ut = rrdcontext_calculate_queued_dispatch_time_ut(rc, now_ut); - char *claim_id = get_agent_claimid(); + CLAIM_ID claim_id = claim_id_get(); - if(unlikely(now_ut >= dispatch_ut) && claim_id) { + if(unlikely(now_ut >= dispatch_ut) && claim_id_is_set(claim_id)) { worker_is_busy(WORKER_JOB_CHECK); rrdcontext_lock(rc); @@ -985,15 +1004,13 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now if(check_if_cloud_version_changed_unsafe(rc, true)) { worker_is_busy(WORKER_JOB_SEND); -#ifdef ENABLE_ACLK if(!bundle) { // prepare the bundle to send the messages - char uuid[UUID_STR_LEN]; - uuid_unparse_lower(*host->node_id, uuid); + char uuid_str[UUID_STR_LEN]; + uuid_unparse_lower(host->node_id.uuid, uuid_str); - bundle = contexts_updated_new(claim_id, uuid, 0, now_ut); + bundle = contexts_updated_new(claim_id.str, uuid_str, 0, now_ut); } -#endif // update the hub data of the context, give a new version, pack the message // and save an update to SQL rrdcontext_message_send_unsafe(rc, false, bundle); @@ -1030,11 +1047,9 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now else rrdcontext_unlock(rc); } - freez(claim_id); } dfe_done(rc); -#ifdef ENABLE_ACLK if(service_running(SERVICE_CONTEXT) && bundle) { // we have a bundle to send messages @@ -1046,7 +1061,6 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now } else if(bundle) contexts_updated_delete(bundle); -#endif } @@ -1085,12 +1099,11 @@ void *rrdcontext_main(void *ptr) { worker_register_job_custom_metric(WORKER_JOB_PP_QUEUE_SIZE, "post processing queue size", "contexts", WORKER_METRIC_ABSOLUTE); heartbeat_t hb; - heartbeat_init(&hb); - usec_t step = RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC; + heartbeat_init(&hb, RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC); while (service_running(SERVICE_CONTEXT)) { worker_is_idle(); - heartbeat_next(&hb, step); + heartbeat_next(&hb); if(unlikely(!service_running(SERVICE_CONTEXT))) break; |