diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2023-08-10 09:18:52 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2023-08-10 09:19:08 +0000 |
commit | a2d7dede737947d7c6afa20a88e1f0c64e0eb96c (patch) | |
tree | fed4aff7dbe0be00cf91de6261d98bc0eb9a2449 /database | |
parent | Releasing debian version 1.41.0-1. (diff) | |
download | netdata-a2d7dede737947d7c6afa20a88e1f0c64e0eb96c.tar.xz netdata-a2d7dede737947d7c6afa20a88e1f0c64e0eb96c.zip |
Merging upstream version 1.42.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'database')
-rw-r--r-- | database/contexts/api_v1.c | 12 | ||||
-rw-r--r-- | database/contexts/api_v2.c | 25 | ||||
-rw-r--r-- | database/contexts/context.c | 125 | ||||
-rw-r--r-- | database/contexts/internal.h | 4 | ||||
-rw-r--r-- | database/contexts/query_target.c | 3 | ||||
-rw-r--r-- | database/contexts/rrdcontext.h | 1 | ||||
-rw-r--r-- | database/contexts/worker.c | 7 | ||||
-rw-r--r-- | database/engine/journalfile.c | 2 | ||||
-rw-r--r-- | database/engine/metric.c | 20 | ||||
-rw-r--r-- | database/engine/metric.h | 2 | ||||
-rwxr-xr-x | database/engine/rrdengineapi.c | 2 | ||||
-rw-r--r-- | database/rrd.h | 4 | ||||
-rw-r--r-- | database/rrdcalc.c | 12 | ||||
-rw-r--r-- | database/rrdfunctions.c | 32 | ||||
-rw-r--r-- | database/sqlite/sqlite_aclk_alert.c | 13 | ||||
-rw-r--r-- | database/sqlite/sqlite_db_migration.c | 17 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.c | 36 | ||||
-rw-r--r-- | database/sqlite/sqlite_health.c | 275 | ||||
-rw-r--r-- | database/sqlite/sqlite_health.h | 3 |
19 files changed, 455 insertions, 140 deletions
diff --git a/database/contexts/api_v1.c b/database/contexts/api_v1.c index b4bcfe4ae..bc7fee496 100644 --- a/database/contexts/api_v1.c +++ b/database/contexts/api_v1.c @@ -148,7 +148,7 @@ static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) { wb_metrics = buffer_create(4096, &netdata_buffers_statistics.buffers_api); - buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, false); + buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT); struct rrdcontext_to_json t_metrics = { .wb = wb_metrics, @@ -268,7 +268,7 @@ static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void || t_parent->chart_dimensions) { wb_instances = buffer_create(4096, &netdata_buffers_statistics.buffers_api); - buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, false); + buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT); struct rrdcontext_to_json t_instances = { .wb = wb_instances, @@ -366,9 +366,9 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R RRDCONTEXT *rc = rrdcontext_acquired_value(rca); if(after != 0 && before != 0) - rrdr_relative_window_to_absolute(&after, &before, NULL); + rrdr_relative_window_to_absolute(&after, &before, NULL, false); - buffer_json_initialize(wb, "\"", "\"", 0, true, false); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); struct rrdcontext_to_json t_contexts = { .wb = wb, .options = options|RRDCONTEXT_OPTION_SKIP_ID, @@ -403,9 +403,9 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, uuid_unparse(*host->node_id, node_uuid); if(after != 0 && before != 0) - rrdr_relative_window_to_absolute(&after, &before, NULL); + rrdr_relative_window_to_absolute(&after, &before, NULL, false); - buffer_json_initialize(wb, "\"", "\"", 0, true, false); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host)); buffer_json_member_add_string(wb, "machine_guid", host->machine_guid); buffer_json_member_add_string(wb, "node_id", node_uuid); diff --git a/database/contexts/api_v2.c b/database/contexts/api_v2.c index ed7f955ad..08739160d 100644 --- a/database/contexts/api_v2.c +++ b/database/contexts/api_v2.c @@ -1104,9 +1104,20 @@ static bool contexts_conflict_callback(const DICTIONARY_ITEM *item __maybe_unuse o->count++; if(o->family != n->family) { - STRING *m = string_2way_merge(o->family, n->family); - string_freez(o->family); - o->family = m; + if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED)) + // keep old + ; + else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) { + // keep new + string_freez(o->family); + o->family = string_dup(n->family); + } + else { + // merge + STRING *old_family = o->family; + o->family = string_2way_merge(o->family, n->family); + string_freez(old_family); + } } if(o->priority != n->priority) { @@ -1287,7 +1298,7 @@ int contexts_v2_alert_config_to_json(struct web_client *w, const char *config_ha buffer_flush(w->response.data); - buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false); + buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); int added = sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, false); buffer_json_finalize(w->response.data); @@ -1923,14 +1934,14 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE } if(req->after || req->before) { - ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now); + ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now, false); ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS); } else ctl.now = now_realtime_sec(); - buffer_json_initialize(wb, "\"", "\"", 0, - true, (req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG)); + buffer_json_initialize(wb, "\"", "\"", 0, true, + ((req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT); buffer_json_member_add_uint64(wb, "api", 2); diff --git a/database/contexts/context.c b/database/contexts/context.c index 47946f1e0..5613c63cf 100644 --- a/database/contexts/context.c +++ b/database/contexts/context.c @@ -94,6 +94,93 @@ static void rrdcontext_delete_callback(const DICTIONARY_ITEM *item __maybe_unuse rrdcontext_freez(rc); } +typedef enum __attribute__((packed)) { + OLDNEW_KEEP_OLD, + OLDNEW_USE_NEW, + OLDNEW_MERGE, +} OLDNEW; + +static inline OLDNEW oldnew_decide(bool archived, bool new_archived) { + if(archived && !new_archived) + return OLDNEW_USE_NEW; + + if(!archived && new_archived) + return OLDNEW_KEEP_OLD; + + return OLDNEW_MERGE; +} + +static inline void string_replace(STRING **stringpp, STRING *new_string) { + STRING *old = *stringpp; + *stringpp = string_dup(new_string); + string_freez(old); +} + +static inline void string_merge(STRING **stringpp, STRING *new_string) { + STRING *old = *stringpp; + *stringpp = string_2way_merge(*stringpp, new_string); + string_freez(old); +} + +static void rrdcontext_merge_with(RRDCONTEXT *rc, bool archived, STRING *title, STRING *family, STRING *units, RRDSET_TYPE chart_type, uint32_t priority) { + OLDNEW oldnew = oldnew_decide(rrd_flag_is_archived(rc), archived); + + switch(oldnew) { + case OLDNEW_KEEP_OLD: + break; + + case OLDNEW_USE_NEW: + if(rc->title != title) { + string_replace(&rc->title, title); + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); + } + if(rc->family != family) { + string_replace(&rc->family, family); + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); + } + break; + + case OLDNEW_MERGE: + if(rc->title != title) { + string_merge(&rc->title, title); + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); + } + if(rc->family != family) { + string_merge(&rc->family, family); + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); + } + break; + } + + switch(oldnew) { + case OLDNEW_KEEP_OLD: + break; + + case OLDNEW_USE_NEW: + case OLDNEW_MERGE: + if(rc->units != units) { + string_replace(&rc->units, units); + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); + } + + if(rc->chart_type != chart_type) { + rc->chart_type = chart_type; + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); + } + + if(rc->priority != priority) { + rc->priority = priority; + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); + } + break; + } +} + +void rrdcontext_update_from_collected_rrdinstance(RRDINSTANCE *ri) { + rrdcontext_merge_with(ri->rc, rrd_flag_is_archived(ri), + ri->title, ri->family, ri->units, ri->chart_type, ri->priority); +} + static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdhost __maybe_unused) { RRDCONTEXT *rc = (RRDCONTEXT *)old_value; RRDCONTEXT *rc_new = (RRDCONTEXT *)new_value; @@ -106,42 +193,8 @@ static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unu rrdcontext_lock(rc); - if(rc->title != rc_new->title) { - STRING *old_title = rc->title; - if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new)) - rc->title = string_dup(rc_new->title); - else - rc->title = string_2way_merge(rc->title, rc_new->title); - string_freez(old_title); - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); - } - - if(rc->units != rc_new->units) { - STRING *old_units = rc->units; - rc->units = string_dup(rc_new->units); - string_freez(old_units); - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); - } - - if(rc->family != rc_new->family) { - STRING *old_family = rc->family; - if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new)) - rc->family = string_dup(rc_new->family); - else - rc->family = string_2way_merge(rc->family, rc_new->family); - string_freez(old_family); - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); - } - - if(rc->chart_type != rc_new->chart_type) { - rc->chart_type = rc_new->chart_type; - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); - } - - if(rc->priority != rc_new->priority) { - rc->priority = rc_new->priority; - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA); - } + rrdcontext_merge_with(rc, rrd_flag_is_archived(rc_new), + rc_new->title, rc_new->family, rc_new->units, rc_new->chart_type, rc_new->priority); rrd_flag_set(rc, rc_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on rc_new diff --git a/database/contexts/internal.h b/database/contexts/internal.h index c5663dd24..04ad0883a 100644 --- a/database/contexts/internal.h +++ b/database/contexts/internal.h @@ -59,6 +59,8 @@ typedef enum __attribute__ ((__packed__)) { RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 21), // this context is not used anymore RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 22), // this context changed because of a db rotation + RRD_FLAG_MERGED_COLLECTED_RI_TO_RC = (1 << 29), + // action to perform on an object RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION = (1 << 30), // this object has to update its retention from the db } RRD_FLAGS; @@ -381,4 +383,6 @@ uint64_t rrdcontext_version_hash_with_callback( void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused); +void rrdcontext_update_from_collected_rrdinstance(RRDINSTANCE *ri); + #endif //NETDATA_RRDCONTEXT_INTERNAL_H diff --git a/database/contexts/query_target.c b/database/contexts/query_target.c index 508977ce7..829640b90 100644 --- a/database/contexts/query_target.c +++ b/database/contexts/query_target.c @@ -1052,7 +1052,8 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) { if(query_target_has_percentage_of_group(qt)) qt->window.options &= ~RRDR_OPTION_PERCENTAGE; - rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now); + qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now, + unittest_running); // prepare our local variables - we need these across all these functions QUERY_TARGET_LOCALS qtl = { diff --git a/database/contexts/rrdcontext.h b/database/contexts/rrdcontext.h index e3a1ab9af..0bcdb68de 100644 --- a/database/contexts/rrdcontext.h +++ b/database/contexts/rrdcontext.h @@ -409,6 +409,7 @@ typedef struct query_target { struct { SPINLOCK spinlock; bool used; // when true, this query is currently being used + bool relative; // when true, this query uses relative timestamps size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries struct query_target *prev; struct query_target *next; diff --git a/database/contexts/worker.c b/database/contexts/worker.c index e6c3ff3df..9d7c18863 100644 --- a/database/contexts/worker.c +++ b/database/contexts/worker.c @@ -611,6 +611,13 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG continue; } + bool ri_collected = rrd_flag_is_collected(ri); + + if(ri_collected && !rrd_flag_check(ri, RRD_FLAG_MERGED_COLLECTED_RI_TO_RC)) { + rrdcontext_update_from_collected_rrdinstance(ri); + rrd_flag_set(ri, RRD_FLAG_MERGED_COLLECTED_RI_TO_RC); + } + if(unlikely(!currently_collected && rrd_flag_is_collected(ri) && ri->first_time_s)) currently_collected = true; diff --git a/database/engine/journalfile.c b/database/engine/journalfile.c index 24d3c1c6d..abb9d2eb9 100644 --- a/database/engine/journalfile.c +++ b/database/engine/journalfile.c @@ -696,12 +696,12 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx, bool update_metric_time = true; if (!metric) { MRG_ENTRY entry = { + .uuid = temp_id, .section = (Word_t)ctx, .first_time_s = vd.start_time_s, .last_time_s = vd.end_time_s, .latest_update_every_s = (uint32_t) vd.update_every_s, }; - uuid_copy(entry.uuid, *temp_id); bool added; metric = mrg_metric_add_and_acquire(main_mrg, entry, &added); diff --git a/database/engine/metric.c b/database/engine/metric.c index 1370f9d7a..0b248c09b 100644 --- a/database/engine/metric.c +++ b/database/engine/metric.c @@ -166,7 +166,7 @@ static inline bool metric_release_and_can_be_deleted(MRG *mrg __maybe_unused, ME } static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) { - size_t partition = uuid_partition(mrg, &entry->uuid); + size_t partition = uuid_partition(mrg, entry->uuid); METRIC *allocation = aral_mallocz(mrg->index[partition].aral); @@ -174,7 +174,7 @@ static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *r size_t mem_before_judyl, mem_after_judyl; - Pvoid_t *sections_judy_pptr = JudyHSIns(&mrg->index[partition].uuid_judy, &entry->uuid, sizeof(uuid_t), PJE0); + Pvoid_t *sections_judy_pptr = JudyHSIns(&mrg->index[partition].uuid_judy, entry->uuid, sizeof(uuid_t), PJE0); if(unlikely(!sections_judy_pptr || sections_judy_pptr == PJERR)) fatal("DBENGINE METRIC: corrupted UUIDs JudyHS array"); @@ -207,8 +207,7 @@ static inline METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *r } METRIC *metric = allocation; - // memcpy(metric->uuid, entry->uuid, sizeof(uuid_t)); - uuid_copy(metric->uuid, entry->uuid); + uuid_copy(metric->uuid, *entry->uuid); metric->section = entry->section; metric->first_time_s = MAX(0, entry->first_time_s); metric->latest_time_s_clean = MAX(0, entry->last_time_s); @@ -690,13 +689,12 @@ inline void mrg_update_metric_retention_and_granularity_by_uuid( METRIC *metric = mrg_metric_get_and_acquire(mrg, uuid, section); if (!metric) { MRG_ENTRY entry = { + .uuid = uuid, .section = section, .first_time_s = first_time_s, .last_time_s = last_time_s, .latest_update_every_s = (uint32_t) update_every_s }; - // memcpy(entry.uuid, *uuid, sizeof(uuid_t)); - uuid_copy(entry.uuid, *uuid); metric = mrg_metric_add_and_acquire(mrg, entry, &added); } @@ -788,13 +786,15 @@ int mrg_unittest(void) { METRIC *m1_t1, *m2_t1, *m3_t1, *m4_t1; bool ret; + uuid_t test_uuid; + uuid_generate(test_uuid); MRG_ENTRY entry = { + .uuid = &test_uuid, .section = 0, .first_time_s = 2, .last_time_s = 3, .latest_update_every_s = 4, }; - uuid_generate(entry.uuid); m1_t0 = mrg_metric_add_and_acquire(mrg, entry, &ret); if(!ret) fatal("DBENGINE METRIC: failed to add metric"); @@ -806,7 +806,7 @@ int mrg_unittest(void) { if(ret) fatal("DBENGINE METRIC: managed to add the same metric twice"); - m3_t0 = mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section); + m3_t0 = mrg_metric_get_and_acquire(mrg, entry.uuid, entry.section); if(m3_t0 != m1_t0) fatal("DBENGINE METRIC: cannot find the metric added"); @@ -830,7 +830,7 @@ int mrg_unittest(void) { if(ret) fatal("DBENGINE METRIC: managed to add the same metric twice in (section 0)"); - m3_t1 = mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section); + m3_t1 = mrg_metric_get_and_acquire(mrg, entry.uuid, entry.section); if(m3_t1 != m1_t1) fatal("DBENGINE METRIC: cannot find the metric added (section %zu)", (size_t)entry.section); @@ -844,7 +844,7 @@ int mrg_unittest(void) { if(!mrg_metric_release_and_delete(mrg, m1_t0)) fatal("DBENGINE METRIC: cannot delete the first metric"); - m4_t1 = mrg_metric_get_and_acquire(mrg, &entry.uuid, entry.section); + m4_t1 = mrg_metric_get_and_acquire(mrg, entry.uuid, entry.section); if(m4_t1 != m1_t1) fatal("DBENGINE METRIC: cannot find the metric added (section %zu), after deleting the first one", (size_t)entry.section); diff --git a/database/engine/metric.h b/database/engine/metric.h index 5cb5b045e..5d5ebd7b1 100644 --- a/database/engine/metric.h +++ b/database/engine/metric.h @@ -9,7 +9,7 @@ typedef struct metric METRIC; typedef struct mrg MRG; typedef struct mrg_entry { - uuid_t uuid; + uuid_t *uuid; Word_t section; time_t first_time_s; time_t last_time_s; diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index 49df5c814..264cbdfeb 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -131,12 +131,12 @@ static METRIC *rrdeng_metric_create(STORAGE_INSTANCE *db_instance, uuid_t *uuid) struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance; MRG_ENTRY entry = { + .uuid = uuid, .section = (Word_t)ctx, .first_time_s = 0, .last_time_s = 0, .latest_update_every_s = 0, }; - uuid_copy(entry.uuid, *uuid); METRIC *metric = mrg_metric_add_and_acquire(main_mrg, entry, NULL); return metric; diff --git a/database/rrd.h b/database/rrd.h index 95da17c82..ea211f64e 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -1045,6 +1045,7 @@ struct alarm_entry { STRING *name; STRING *chart; STRING *chart_context; + STRING *chart_name; STRING *family; STRING *classification; @@ -1086,7 +1087,8 @@ struct alarm_entry { }; #define ae_name(ae) string2str((ae)->name) -#define ae_chart_name(ae) string2str((ae)->chart) +#define ae_chart_id(ae) string2str((ae)->chart) +#define ae_chart_name(ae) string2str((ae)->chart_name) #define ae_chart_context(ae) string2str((ae)->chart_context) #define ae_family(ae) string2str((ae)->family) #define ae_classification(ae) string2str((ae)->classification) diff --git a/database/rrdcalc.c b/database/rrdcalc.c index d85825c50..8e41df8d6 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -82,10 +82,14 @@ uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint3 alarm_id = sql_get_alarm_id(host, chart, name, next_event_id, config_hash_id); if (!alarm_id) { - if (unlikely(!host->health_log.next_alarm_id)) - host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); + //check possible stored config hash as zeroes or null + alarm_id = sql_get_alarm_id_check_zero_hash(host, chart, name, next_event_id, config_hash_id); + if (!alarm_id) { + if (unlikely(!host->health_log.next_alarm_id)) + host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); - alarm_id = host->health_log.next_alarm_id++; + alarm_id = host->health_log.next_alarm_id++; + } } } @@ -292,6 +296,7 @@ static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) { rc->name, rc->rrdset->id, rc->rrdset->context, + rc->rrdset->name, rc->rrdset->family, rc->classification, rc->component, @@ -336,6 +341,7 @@ static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) { rc->name, rc->rrdset->id, rc->rrdset->context, + rc->rrdset->name, rc->rrdset->family, rc->classification, rc->component, diff --git a/database/rrdfunctions.c b/database/rrdfunctions.c index cee1ac89e..d32a4b8c9 100644 --- a/database/rrdfunctions.c +++ b/database/rrdfunctions.c @@ -270,7 +270,7 @@ static inline size_t sanitize_function_text(char *dst, const char *src, size_t d // we keep a dictionary per RRDSET with these functions // the dictionary is created on demand (only when a function is added to an RRDSET) -typedef enum { +typedef enum __attribute__((packed)) { RRD_FUNCTION_LOCAL = (1 << 0), RRD_FUNCTION_GLOBAL = (1 << 1), @@ -279,7 +279,7 @@ typedef enum { struct rrd_collector_function { bool sync; // when true, the function is called synchronously - uint8_t options; // RRD_FUNCTION_OPTIONS + RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS STRING *help; int timeout; // the default timeout of the function @@ -814,7 +814,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_flush(wb); wb->content_type = CT_APPLICATION_JSON; - buffer_json_initialize(wb, "\"", "\"", 0, true, false); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost)); buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); @@ -858,8 +858,8 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha // retention buffer_json_add_array_item_string(wb, rrdhost_hostname(s.host)); // Node - buffer_json_add_array_item_uint64(wb, s.db.first_time_s * 1000); // dbFrom - buffer_json_add_array_item_uint64(wb, s.db.last_time_s * 1000); // dbTo + buffer_json_add_array_item_uint64(wb, s.db.first_time_s * MSEC_PER_SEC); // dbFrom + buffer_json_add_array_item_uint64(wb, s.db.last_time_s * MSEC_PER_SEC); // dbTo if(s.db.first_time_s && s.db.last_time_s && s.db.last_time_s > s.db.first_time_s) buffer_json_add_array_item_uint64(wb, s.db.last_time_s - s.db.first_time_s); // dbDuration @@ -877,7 +877,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha // collection if(s.ingest.since) { - buffer_json_add_array_item_uint64(wb, s.ingest.since * 1000); // InSince + buffer_json_add_array_item_uint64(wb, s.ingest.since * MSEC_PER_SEC); // InSince buffer_json_add_array_item_time_t(wb, s.now - s.ingest.since); // InAge } else { @@ -897,7 +897,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha // streaming if(s.stream.since) { - buffer_json_add_array_item_uint64(wb, s.stream.since * 1000); // OutSince + buffer_json_add_array_item_uint64(wb, s.stream.since * MSEC_PER_SEC); // OutSince buffer_json_add_array_item_time_t(wb, s.now - s.stream.since); // OutAge } else { @@ -990,19 +990,19 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha NULL); buffer_rrdf_table_add_field(wb, field_id++, "dbFrom", "DB Data Retention From", - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME, + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); buffer_rrdf_table_add_field(wb, field_id++, "dbTo", "DB Data Retention To", - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME, + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); buffer_rrdf_table_add_field(wb, field_id++, "dbDuration", "DB Data Retention Duration", - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION, + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1049,13 +1049,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha // --- collection --- buffer_rrdf_table_add_field(wb, field_id++, "InSince", "Last Data Collection Status Change", - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME, + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS, 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); buffer_rrdf_table_add_field(wb, field_id++, "InAge", "Last Data Collection Online Status Change Age", - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION, + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); @@ -1124,13 +1124,13 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha // --- streaming --- buffer_rrdf_table_add_field(wb, field_id++, "OutSince", "Last Streaming Status Change", - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME, + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS, 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); buffer_rrdf_table_add_field(wb, field_id++, "OutAge", "Last Streaming Status Change Age", - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION, + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); @@ -1242,14 +1242,14 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptSince", "Last Outbound Connection Attempt Status Change Time", - RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME, + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS, 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptAge", "Last Outbound Connection Attempt Status Change Age", - RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION, + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); diff --git a/database/sqlite/sqlite_aclk_alert.c b/database/sqlite/sqlite_aclk_alert.c index d57ae043f..f10a866f3 100644 --- a/database/sqlite/sqlite_aclk_alert.c +++ b/database/sqlite/sqlite_aclk_alert.c @@ -7,7 +7,7 @@ #include "../../aclk/aclk_alarm_api.h" #endif -#define SQL_UPDATE_FILTERED_ALERT "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u where filtered_alert_unique_id = %u" +#define SQL_UPDATE_FILTERED_ALERT "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u, date_created = unixepoch() where filtered_alert_unique_id = %u" void update_filtered(ALARM_ENTRY *ae, uint32_t unique_id, char *uuid_str) { char sql[ACLK_SYNC_QUERY_SIZE]; snprintfz(sql, ACLK_SYNC_QUERY_SIZE-1, SQL_UPDATE_FILTERED_ALERT, uuid_str, ae->unique_id, unique_id); @@ -268,7 +268,7 @@ void aclk_push_alert_event(struct aclk_sync_host_config *wc) buffer_sprintf(sql, "select aa.sequence_id, hld.unique_id, hld.alarm_id, hl.config_hash_id, hld.updated_by_id, hld.when_key, " \ " hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, hld.delay_up_to_timestamp, hl.name, " \ " hl.chart, hl.family, hl.exec, hl.recipient, ha.source, hl.units, hld.info, hld.exec_code, hld.new_status, " \ - " hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, hl.chart_context, hld.transition_id, hld.alarm_event_id " \ + " hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, hl.chart_context, hld.transition_id, hld.alarm_event_id, hl.chart_name " \ " from health_log hl, aclk_alert_%s aa, alert_hash ha, health_log_detail hld " \ " where hld.unique_id = aa.alert_unique_id and hl.config_hash_id = ha.hash_id and aa.date_submitted is null " \ " and hl.host_id = @host_id and hl.health_log_id = hld.health_log_id " \ @@ -371,14 +371,12 @@ void aclk_push_alert_event(struct aclk_sync_host_config *wc) alarm_log.value = (NETDATA_DOUBLE) sqlite3_column_double(res, 23); alarm_log.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 24); - alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0; alarm_log.rendered_info = sqlite3_text_strdupz_empty(res, 18); - alarm_log.chart_context = sqlite3_text_strdupz_empty(res, 26); alarm_log.transition_id = sqlite3_uuid_unparse_strdupz(res, 27); - alarm_log.event_id = (time_t) sqlite3_column_int64(res, 28); + alarm_log.chart_name = sqlite3_text_strdupz_empty(res, 29); aclk_send_alarm_log_entry(&alarm_log); @@ -788,7 +786,7 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN char transition_id[UUID_STR_LEN]; uuid_unparse_lower(ae->transition_id, transition_id); - alarm_log->chart = strdupz(ae_chart_name(ae)); + alarm_log->chart = strdupz(ae_chart_id(ae)); alarm_log->name = strdupz(ae_name(ae)); alarm_log->family = strdupz(ae_family(ae)); @@ -827,6 +825,7 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0; alarm_log->rendered_info = strdupz(ae_info(ae)); alarm_log->chart_context = strdupz(ae_chart_context(ae)); + alarm_log->chart_name = strdupz(ae_chart_name(ae)); alarm_log->transition_id = strdupz((char *)transition_id); alarm_log->event_id = (uint64_t) ae->alarm_event_id; @@ -980,7 +979,7 @@ void aclk_push_alert_snapshot_event(char *node_id __maybe_unused) #endif } -#define SQL_DELETE_ALERT_ENTRIES "DELETE FROM aclk_alert_%s WHERE filtered_alert_unique_id + %d < UNIXEPOCH();" +#define SQL_DELETE_ALERT_ENTRIES "DELETE FROM aclk_alert_%s WHERE date_created + %d < UNIXEPOCH();" void sql_aclk_alert_clean_dead_entries(RRDHOST *host) { char uuid_str[UUID_STR_LEN]; diff --git a/database/sqlite/sqlite_db_migration.c b/database/sqlite/sqlite_db_migration.c index 1a6233fce..548b7de07 100644 --- a/database/sqlite/sqlite_db_migration.c +++ b/database/sqlite/sqlite_db_migration.c @@ -83,6 +83,11 @@ const char *database_migrate_v9_v10[] = { NULL }; +const char *database_migrate_v10_v11[] = { + "ALTER TABLE health_log ADD chart_name TEXT;", + NULL +}; + static int do_migration_v1_v2(sqlite3 *database, const char *name) { UNUSED(name); @@ -293,7 +298,6 @@ static int do_migration_v8_v9(sqlite3 *database, const char *name) static int do_migration_v9_v10(sqlite3 *database, const char *name) { - UNUSED(name); netdata_log_info("Running \"%s\" database migration", name); if (table_exists_in_database("alert_hash") && !column_exists_in_table("alert_hash", "chart_labels")) @@ -301,6 +305,16 @@ static int do_migration_v9_v10(sqlite3 *database, const char *name) return 0; } +static int do_migration_v10_v11(sqlite3 *database, const char *name) +{ + netdata_log_info("Running \"%s\" database migration", name); + + if (table_exists_in_database("health_log") && !column_exists_in_table("health_log", "chart_name")) + return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v10_v11[0]); + + return 0; +} + static int do_migration_noop(sqlite3 *database, const char *name) { UNUSED(database); @@ -354,6 +368,7 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = { {.name = "v7 to v8", .func = do_migration_v7_v8}, {.name = "v8 to v9", .func = do_migration_v8_v9}, {.name = "v9 to v10", .func = do_migration_v9_v10}, + {.name = "v10 to v11", .func = do_migration_v10_v11}, // the terminator of this array {.name = NULL, .func = NULL} }; diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index 4200c1590..d976a3c6e 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -3,7 +3,7 @@ #include "sqlite_functions.h" #include "sqlite_db_migration.h" -#define DB_METADATA_VERSION 10 +#define DB_METADATA_VERSION 11 const char *database_config[] = { "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, " @@ -34,8 +34,6 @@ const char *database_config[] = { "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, " "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text);", - "CREATE INDEX IF NOT EXISTS alert_hash_index ON alert_hash (hash_id);", - "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, " "date_created INT, PRIMARY KEY(host_id, system_key));", @@ -47,7 +45,7 @@ const char *database_config[] = { "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, " "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, " - "chart_context text, last_transition_id blob, UNIQUE (host_id, alarm_id)) ;", + "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id)) ;", "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id);", @@ -75,6 +73,7 @@ const char *database_cleanup[] = { "DROP INDEX IF EXISTS ind_d1;", "DROP INDEX IF EXISTS ind_c1;", "DROP INDEX IF EXISTS ind_c2;", + "DROP INDEX IF EXISTS alert_hash_index;", NULL }; @@ -593,15 +592,20 @@ static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id) if (unlikely(!node_id)) { freez(host->node_id); - host->node_id = NULL; + __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED); return; } struct aclk_sync_host_config *wc = host->aclk_sync_host_config; - if (unlikely(!host->node_id)) - host->node_id = mallocz(sizeof(*host->node_id)); - uuid_copy(*(host->node_id), *node_id); + if (unlikely(!host->node_id)) { + uuid_t *t = mallocz(sizeof(*host->node_id)); + uuid_copy(*t, *node_id); + __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED); + } + else { + uuid_copy(*(host->node_id), *node_id); + } if (unlikely(!wc)) sql_create_aclk_table(host, &host->host_uuid, node_id); @@ -617,6 +621,14 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id) RRDHOST *host = NULL; int rc = 2; + char host_guid[GUID_LEN + 1]; + uuid_unparse_lower(*host_id, host_guid); + rrd_wrlock(); + host = rrdhost_find_by_guid(host_guid); + if (likely(host)) + set_host_node_id(host, node_id); + rrd_unlock(); + if (unlikely(!db_meta)) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) error_report("Database has not been initialized"); @@ -646,14 +658,6 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id) error_report("Failed to store node instance information, rc = %d", rc); rc = sqlite3_changes(db_meta); - char host_guid[GUID_LEN + 1]; - uuid_unparse_lower(*host_id, host_guid); - rrd_wrlock(); - host = rrdhost_find_by_guid(host_guid); - if (likely(host)) - set_host_node_id(host, node_id); - rrd_unlock(); - failed: if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) error_report("Failed to finalize the prepared statement when storing node instance information"); diff --git a/database/sqlite/sqlite_health.c b/database/sqlite/sqlite_health.c index 3ecd783dc..9c103f098 100644 --- a/database/sqlite/sqlite_health.c +++ b/database/sqlite/sqlite_health.c @@ -83,9 +83,10 @@ failed: Inserts an entry in the table */ #define SQL_INSERT_HEALTH_LOG "INSERT INTO health_log (host_id, alarm_id, " \ - "config_hash_id, name, chart, family, exec, recipient, units, chart_context, last_transition_id) " \ - "VALUES (?,?,?,?,?,?,?,?,?,?,?) " \ - "ON CONFLICT (host_id, alarm_id) DO UPDATE SET last_transition_id = excluded.last_transition_id RETURNING health_log_id; " + "config_hash_id, name, chart, family, exec, recipient, units, chart_context, last_transition_id, chart_name) " \ + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?) " \ + "ON CONFLICT (host_id, alarm_id) DO UPDATE SET last_transition_id = excluded.last_transition_id, " \ + "chart_name = excluded.chart_name RETURNING health_log_id; " #define SQL_INSERT_HEALTH_LOG_DETAIL "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, " \ "updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, " \ @@ -174,6 +175,12 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) { goto failed; } + rc = sqlite3_bind_string_or_null(res, ae->chart_name, 12); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind chart_name parameter for SQL_INSERT_HEALTH_LOG"); + goto failed; + } + rc = sqlite3_step_monitored(res); if (likely(rc == SQLITE_ROW)) health_log_id = (size_t) sqlite3_column_int64(res, 0); @@ -685,7 +692,10 @@ failed: error_report("HEALTH [N/A]: Failed to finalize the prepared statement for injecting removed event."); } -#define SQL_SELECT_MAX_UNIQUE_ID "SELECT MAX(hld.unique_id) from health_log_detail hld, health_log hl where hl.host_id = @host_id; and hl.health_log_id = hld.health_log_id" +#define SQL_SELECT_MAX_UNIQUE_ID \ + "SELECT MAX(hld.unique_id) FROM health_log_detail hld, health_log hl " \ + "WHERE hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id" + uint32_t sql_get_max_unique_id (RRDHOST *host) { int rc; @@ -766,7 +776,7 @@ void sql_check_removed_alerts_state(RRDHOST *host) "hld.updates_id, hld.when_key, hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, " \ "hld.delay_up_to_timestamp, hl.name, hl.chart, hl.family, hl.exec, hl.recipient, ah.source, hl.units, " \ "hld.info, hld.exec_code, hld.new_status, hld.old_status, hld.delay, hld.new_value, hld.old_value, " \ - "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.global_id " \ + "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.global_id, hl.chart_name " \ "FROM health_log hl, alert_hash ah, health_log_detail hld " \ "WHERE hl.config_hash_id = ah.hash_id and hl.host_id = @host_id and hl.last_transition_id = hld.transition_id;" void sql_health_alarm_log_load(RRDHOST *host) { @@ -945,6 +955,11 @@ void sql_health_alarm_log_load(RRDHOST *host) { if (sqlite3_column_type(res, 32) != SQLITE_NULL) ae->global_id = sqlite3_column_int64(res, 32); + if (sqlite3_column_type(res, 33) != SQLITE_NULL) + ae->chart_name = string_strdupz((char *) sqlite3_column_text(res, 33)); + else + ae->chart_name = NULL; + char value_string[100 + 1]; string_freez(ae->old_value_string); string_freez(ae->new_value_string); @@ -1203,7 +1218,7 @@ bind_fail: if cloud is disabled or openssl is not available (which will prevent cloud connectivity) skip hash calculations */ -#if !defined DISABLE_CLOUD && defined ENABLE_HTTPS +#if defined ENABLE_HTTPS #define DIGEST_ALERT_CONFIG_VAL(v) ((v) ? EVP_DigestUpdate(evpctx, (string2str(v)), string_strlen((v))) : EVP_DigestUpdate(evpctx, "", 1)) #endif int alert_hash_and_store_config( @@ -1211,7 +1226,7 @@ int alert_hash_and_store_config( struct alert_config *cfg, int store_hash) { -#if !defined DISABLE_CLOUD && defined ENABLE_HTTPS +#if defined ENABLE_HTTPS EVP_MD_CTX *evpctx; unsigned char hash_value[EVP_MAX_MD_SIZE]; unsigned int hash_len; @@ -1717,6 +1732,135 @@ uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t * return alarm_id; } +#define SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH "update health_log set config_hash_id = @config_hash_id where host_id = @host_id and alarm_id = @alarm_id and health_log_id = @health_log_id" +void sql_update_alarm_with_config_hash(RRDHOST *host, uint32_t alarm_id, uint64_t health_log_id, uuid_t *config_hash_id) +{ + int rc = 0; + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH, -1, &res, 0); + if (rc != SQLITE_OK) { + error_report("Failed to prepare statement when trying to update an alarm id with a config hash."); + return; + } + + rc = sqlite3_bind_blob(res, 1, config_hash_id, sizeof(*config_hash_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind config_hash_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH."); + sqlite3_finalize(res); + return; + } + + rc = sqlite3_bind_blob(res, 2, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH."); + sqlite3_finalize(res); + return; + } + + rc = sqlite3_bind_int64(res, 3, (sqlite3_int64) alarm_id); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID."); + sqlite3_finalize(res); + return; + } + + rc = sqlite3_bind_int64(res, 4, (sqlite3_int64) health_log_id); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID."); + sqlite3_finalize(res); + return; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) { + error_report("Failed to execute SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH, rc = %d", rc); + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement to update health log detail table with config hash ids, rc = %d", rc); + return; + } +} + +#define SQL_GET_ALARM_ID_CHECK_ZERO_HASH "select alarm_id, health_log_id from health_log where host_id = @host_id and chart = @chart and name = @name and (config_hash_id is null or config_hash_id = zeroblob(16))" +uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id) +{ + int rc = 0; + sqlite3_stmt *res = NULL; + uint32_t alarm_id = 0; + uint64_t health_log_id = 0; + + rc = sqlite3_prepare_v2(db_meta, SQL_GET_ALARM_ID_CHECK_ZERO_HASH, -1, &res, 0); + if (rc != SQLITE_OK) { + error_report("Failed to prepare statement when trying to get an alarm id with zero hash"); + return alarm_id; + } + + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH."); + sqlite3_finalize(res); + return alarm_id; + } + + rc = sqlite3_bind_string_or_null(res, chart, 2); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind char parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH."); + sqlite3_finalize(res); + return alarm_id; + } + + rc = sqlite3_bind_string_or_null(res, name, 3); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind name parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH."); + sqlite3_finalize(res); + return alarm_id; + } + + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + alarm_id = (uint32_t) sqlite3_column_int64(res, 0); + health_log_id = (uint64_t) sqlite3_column_int64(res, 1); + } + + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the statement while getting an alarm id."); + + if (alarm_id) { + sql_update_alarm_with_config_hash(host, alarm_id, health_log_id, config_hash_id); + + rc = sqlite3_prepare_v2(db_meta, SQL_GET_EVENT_ID, -1, &res, 0); + if (rc != SQLITE_OK) { + error_report("Failed to prepare statement when trying to get an event id"); + return alarm_id; + } + + rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) health_log_id); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter for SQL_GET_EVENT_ID."); + sqlite3_finalize(res); + return alarm_id; + } + + rc = sqlite3_bind_int64(res, 2, (sqlite3_int64) alarm_id); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind char parameter for SQL_GET_EVENT_ID."); + sqlite3_finalize(res); + return alarm_id; + } + + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + *next_event_id = (uint32_t) sqlite3_column_int64(res, 0); + } + + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the statement while getting an alarm id."); + } + + return alarm_id; +} + #define SQL_GET_ALARM_ID_FROM_TRANSITION_ID "SELECT hld.alarm_id, hl.host_id, hl.chart_context FROM " \ "health_log_detail hld, health_log hl WHERE hld.transition_id = @transition_id " \ "and hld.health_log_id = hl.health_log_id" @@ -1767,7 +1911,7 @@ fail: #define SQL_POPULATE_TEMP_ALERT_TRANSITION_TABLE "INSERT INTO v_%p (host_id) VALUES (@host_id)" #define SQL_SEARCH_ALERT_TRANSITION_SELECT "SELECT " \ - "h.host_id, h.alarm_id, h.config_hash_id, h.name, h.chart, h.family, h.recipient, h.units, h.exec, " \ + "h.host_id, h.alarm_id, h.config_hash_id, h.name, h.chart, h.chart_name, h.family, h.recipient, h.units, h.exec, " \ "h.chart_context, d.when_key, d.duration, d.non_clear_duration, d.flags, d.delay_up_to_timestamp, " \ "d.info, d.exec_code, d.new_status, d.old_status, d.delay, d.new_value, d.old_value, d.last_repeat, " \ "d.transition_id, d.global_id, ah.class, ah.type, ah.component, d.exec_run_timestamp" @@ -1913,31 +2057,31 @@ run_query:; atd.config_hash_id = (uuid_t *)sqlite3_column_blob(res, 2); atd.alert_name = (const char *) sqlite3_column_text(res, 3); atd.chart = (const char *) sqlite3_column_text(res, 4); - atd.chart_name = (const char *) sqlite3_column_text(res, 4); // FIXME don't copy the id, find the name - atd.family = (const char *) sqlite3_column_text(res, 5); - atd.recipient = (const char *) sqlite3_column_text(res, 6); - atd.units = (const char *) sqlite3_column_text(res, 7); - atd.exec = (const char *) sqlite3_column_text(res, 8); - atd.chart_context = (const char *) sqlite3_column_text(res, 9); - atd.when_key = sqlite3_column_int64(res, 10); - atd.duration = sqlite3_column_int64(res, 11); - atd.non_clear_duration = sqlite3_column_int64(res, 12); - atd.flags = sqlite3_column_int64(res, 13); - atd.delay_up_to_timestamp = sqlite3_column_int64(res, 14); - atd.info = (const char *) sqlite3_column_text(res, 15); - atd.exec_code = sqlite3_column_int(res, 16); - atd.new_status = sqlite3_column_int(res, 17); - atd.old_status = sqlite3_column_int(res, 18); - atd.delay = (int) sqlite3_column_int(res, 19); - atd.new_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 20); - atd.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 21); - atd.last_repeat = sqlite3_column_int64(res, 22); - atd.transition_id = (uuid_t *) sqlite3_column_blob(res, 23); - atd.global_id = sqlite3_column_int64(res, 24); - atd.classification = (const char *) sqlite3_column_text(res, 25); - atd.type = (const char *) sqlite3_column_text(res, 26); - atd.component = (const char *) sqlite3_column_text(res, 27); - atd.exec_run_timestamp = sqlite3_column_int64(res, 28); + atd.chart_name = (const char *) sqlite3_column_text(res, 5); + atd.family = (const char *) sqlite3_column_text(res, 6); + atd.recipient = (const char *) sqlite3_column_text(res, 7); + atd.units = (const char *) sqlite3_column_text(res, 8); + atd.exec = (const char *) sqlite3_column_text(res, 9); + atd.chart_context = (const char *) sqlite3_column_text(res, 10); + atd.when_key = sqlite3_column_int64(res, 11); + atd.duration = sqlite3_column_int64(res, 12); + atd.non_clear_duration = sqlite3_column_int64(res, 13); + atd.flags = sqlite3_column_int64(res, 14); + atd.delay_up_to_timestamp = sqlite3_column_int64(res, 15); + atd.info = (const char *) sqlite3_column_text(res, 16); + atd.exec_code = sqlite3_column_int(res, 17); + atd.new_status = sqlite3_column_int(res, 18); + atd.old_status = sqlite3_column_int(res, 19); + atd.delay = (int) sqlite3_column_int(res, 20); + atd.new_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 21); + atd.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 22); + atd.last_repeat = sqlite3_column_int64(res, 23); + atd.transition_id = (uuid_t *) sqlite3_column_blob(res, 24); + atd.global_id = sqlite3_column_int64(res, 25); + atd.classification = (const char *) sqlite3_column_text(res, 26); + atd.type = (const char *) sqlite3_column_text(res, 27); + atd.component = (const char *) sqlite3_column_text(res, 28); + atd.exec_run_timestamp = sqlite3_column_int64(res, 29); cb(&atd, data); } @@ -2087,3 +2231,68 @@ fail_only_drop: return added; } +#define SQL_FETCH_CHART_NAME "SELECT chart_name FROM health_log where host_id = @host_id LIMIT 1;" +bool is_chart_name_populated(uuid_t *host_uuid) +{ + sqlite3_stmt *res = NULL; + int rc; + + bool status = true; + + rc = sqlite3_prepare_v2(db_meta, SQL_FETCH_CHART_NAME, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to check health_log chart_name"); + return true; + } + + rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id for health_log chart_name check"); + goto fail; + } + + rc = sqlite3_step_monitored(res); + if (likely(rc == SQLITE_ROW)) + status = sqlite3_column_type(res, 0) != SQLITE_NULL; +fail: + + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement for health_log chart_name check"); + + return status; +} + +#define SQL_POPULATE_CHART_NAME " UPDATE health_log SET chart_name = upd.chart_name FROM " \ + "(SELECT c.type || '.' || IFNULL(c.name, c.id) AS chart_name, hl.host_id, hl.health_log_id FROM " \ + "chart c, health_log hl WHERE (c.type || '.' || c.id) = hl.chart AND c.host_id = hl.host_id " \ + "AND hl.host_id = @host_id) AS upd WHERE health_log.host_id = upd.host_id " \ + "AND health_log.health_log_id = upd.health_log_id" + +void chart_name_populate(uuid_t *host_uuid) +{ + sqlite3_stmt *res = NULL; + int rc; + + rc = sqlite3_prepare_v2(db_meta, SQL_POPULATE_CHART_NAME, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to update health_log chart_name"); + return; + } + + rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id for health_log chart_name update"); + goto fail; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to update chart name in health_log, rc = %d", rc); + +fail: + + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement for health_log chart_name update"); +} diff --git a/database/sqlite/sqlite_health.h b/database/sqlite/sqlite_health.h index 55e523d2f..3aebb94b7 100644 --- a/database/sqlite/sqlite_health.h +++ b/database/sqlite/sqlite_health.h @@ -19,6 +19,7 @@ int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_S void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart); int health_migrate_old_health_log_table(char *table); uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id); +uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id); void sql_alert_transitions( DICTIONARY *nodes, time_t after, @@ -37,4 +38,6 @@ int sql_get_alert_configuration( bool debug __maybe_unused); bool sql_find_alert_transition(const char *transition, void (*cb)(const char *machine_guid, const char *context, time_t alert_id, void *data), void *data); +bool is_chart_name_populated(uuid_t *host_uuid); +void chart_name_populate(uuid_t *host_uuid); #endif //NETDATA_SQLITE_HEALTH_H |