Adding upstream version 1.42.0.upstream/1.42.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

7 files changed, 127 insertions, 50 deletions

diff --git a/database/contexts/api_v1.c b/database/contexts/api_v1.c
index b4bcfe4ae..bc7fee496 100644
--- a/database/contexts/api_v1.c
+++ b/database/contexts/api_v1.c
@@ -148,7 +148,7 @@ static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void
     if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) {
 
         wb_metrics = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
-        buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, false);
+        buffer_json_initialize(wb_metrics, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
 
         struct rrdcontext_to_json t_metrics = {
                 .wb = wb_metrics,
@@ -268,7 +268,7 @@ static inline int rrdcontext_to_json_callback(const DICTIONARY_ITEM *item, void
        || t_parent->chart_dimensions) {
 
         wb_instances = buffer_create(4096, &netdata_buffers_statistics.buffers_api);
-        buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, false);
+        buffer_json_initialize(wb_instances, "\"", "\"", wb->json.depth + 2, false, BUFFER_JSON_OPTIONS_DEFAULT);
 
         struct rrdcontext_to_json t_instances = {
                 .wb = wb_instances,
@@ -366,9 +366,9 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
     RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
 
     if(after != 0 && before != 0)
-        rrdr_relative_window_to_absolute(&after, &before, NULL);
+        rrdr_relative_window_to_absolute(&after, &before, NULL, false);
 
-    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     struct rrdcontext_to_json t_contexts = {
             .wb = wb,
             .options = options|RRDCONTEXT_OPTION_SKIP_ID,
@@ -403,9 +403,9 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
         uuid_unparse(*host->node_id, node_uuid);
 
     if(after != 0 && before != 0)
-        rrdr_relative_window_to_absolute(&after, &before, NULL);
+        rrdr_relative_window_to_absolute(&after, &before, NULL, false);
 
-    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
     buffer_json_member_add_string(wb, "machine_guid", host->machine_guid);
     buffer_json_member_add_string(wb, "node_id", node_uuid);
diff --git a/database/contexts/api_v2.c b/database/contexts/api_v2.c
index ed7f955ad..08739160d 100644
--- a/database/contexts/api_v2.c
+++ b/database/contexts/api_v2.c
@@ -1104,9 +1104,20 @@ static bool contexts_conflict_callback(const DICTIONARY_ITEM *item __maybe_unuse
     o->count++;
 
     if(o->family != n->family) {
-        STRING *m = string_2way_merge(o->family, n->family);
-        string_freez(o->family);
-        o->family = m;
+        if((o->flags & RRD_FLAG_COLLECTED) && !(n->flags & RRD_FLAG_COLLECTED))
+            // keep old
+            ;
+        else if(!(o->flags & RRD_FLAG_COLLECTED) && (n->flags & RRD_FLAG_COLLECTED)) {
+            // keep new
+            string_freez(o->family);
+            o->family = string_dup(n->family);
+        }
+        else {
+            // merge
+            STRING *old_family = o->family;
+            o->family = string_2way_merge(o->family, n->family);
+            string_freez(old_family);
+        }
     }
 
     if(o->priority != n->priority) {
@@ -1287,7 +1298,7 @@ int contexts_v2_alert_config_to_json(struct web_client *w, const char *config_ha
 
     buffer_flush(w->response.data);
 
-    buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
+    buffer_json_initialize(w->response.data, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
 
     int added = sql_get_alert_configuration(configs, contexts_v2_alert_config_to_json_from_sql_alert_config_data, &data, false);
     buffer_json_finalize(w->response.data);
@@ -1923,14 +1934,14 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
     }
 
     if(req->after || req->before) {
-        ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now);
+        ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now, false);
         ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS);
     }
     else
         ctl.now = now_realtime_sec();
 
-    buffer_json_initialize(wb, "\"", "\"", 0,
-                           true, (req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG));
+    buffer_json_initialize(wb, "\"", "\"", 0, true,
+                           ((req->options & CONTEXT_V2_OPTION_MINIFY) && !(req->options & CONTEXT_V2_OPTION_DEBUG)) ? BUFFER_JSON_OPTIONS_MINIFY : BUFFER_JSON_OPTIONS_DEFAULT);
 
     buffer_json_member_add_uint64(wb, "api", 2);
 
diff --git a/database/contexts/context.c b/database/contexts/context.c
index 47946f1e0..5613c63cf 100644
--- a/database/contexts/context.c
+++ b/database/contexts/context.c
@@ -94,6 +94,93 @@ static void rrdcontext_delete_callback(const DICTIONARY_ITEM *item __maybe_unuse
     rrdcontext_freez(rc);
 }
 
+typedef enum __attribute__((packed)) {
+    OLDNEW_KEEP_OLD,
+    OLDNEW_USE_NEW,
+    OLDNEW_MERGE,
+} OLDNEW;
+
+static inline OLDNEW oldnew_decide(bool archived, bool new_archived) {
+    if(archived && !new_archived)
+        return OLDNEW_USE_NEW;
+
+    if(!archived && new_archived)
+        return OLDNEW_KEEP_OLD;
+
+    return OLDNEW_MERGE;
+}
+
+static inline void string_replace(STRING **stringpp, STRING *new_string) {
+    STRING *old = *stringpp;
+    *stringpp = string_dup(new_string);
+    string_freez(old);
+}
+
+static inline void string_merge(STRING **stringpp, STRING *new_string) {
+    STRING *old = *stringpp;
+    *stringpp = string_2way_merge(*stringpp, new_string);
+    string_freez(old);
+}
+
+static void rrdcontext_merge_with(RRDCONTEXT *rc, bool archived, STRING *title, STRING *family, STRING *units, RRDSET_TYPE chart_type, uint32_t priority) {
+    OLDNEW oldnew = oldnew_decide(rrd_flag_is_archived(rc), archived);
+
+    switch(oldnew) {
+        case OLDNEW_KEEP_OLD:
+            break;
+
+        case OLDNEW_USE_NEW:
+            if(rc->title != title) {
+                string_replace(&rc->title, title);
+                rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
+            }
+            if(rc->family != family) {
+                string_replace(&rc->family, family);
+                rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
+            }
+            break;
+
+        case OLDNEW_MERGE:
+            if(rc->title != title) {
+                string_merge(&rc->title, title);
+                rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
+            }
+            if(rc->family != family) {
+                string_merge(&rc->family, family);
+                rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
+            }
+            break;
+    }
+
+    switch(oldnew) {
+        case OLDNEW_KEEP_OLD:
+            break;
+
+        case OLDNEW_USE_NEW:
+        case OLDNEW_MERGE:
+            if(rc->units != units) {
+                string_replace(&rc->units, units);
+                rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
+            }
+
+            if(rc->chart_type != chart_type) {
+                rc->chart_type = chart_type;
+                rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
+            }
+
+            if(rc->priority != priority) {
+                rc->priority = priority;
+                rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
+            }
+            break;
+    }
+}
+
+void rrdcontext_update_from_collected_rrdinstance(RRDINSTANCE *ri) {
+    rrdcontext_merge_with(ri->rc, rrd_flag_is_archived(ri),
+                          ri->title, ri->family, ri->units, ri->chart_type, ri->priority);
+}
+
 static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *rrdhost __maybe_unused) {
     RRDCONTEXT *rc = (RRDCONTEXT *)old_value;
     RRDCONTEXT *rc_new = (RRDCONTEXT *)new_value;
@@ -106,42 +193,8 @@ static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unu
 
     rrdcontext_lock(rc);
 
-    if(rc->title != rc_new->title) {
-        STRING *old_title = rc->title;
-        if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new))
-            rc->title = string_dup(rc_new->title);
-        else
-            rc->title = string_2way_merge(rc->title, rc_new->title);
-        string_freez(old_title);
-        rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
-    }
-
-    if(rc->units != rc_new->units) {
-        STRING *old_units = rc->units;
-        rc->units = string_dup(rc_new->units);
-        string_freez(old_units);
-        rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
-    }
-
-    if(rc->family != rc_new->family) {
-        STRING *old_family = rc->family;
-        if (rrd_flag_is_archived(rc) && !rrd_flag_is_archived(rc_new))
-            rc->family = string_dup(rc_new->family);
-        else
-            rc->family = string_2way_merge(rc->family, rc_new->family);
-        string_freez(old_family);
-        rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
-    }
-
-    if(rc->chart_type != rc_new->chart_type) {
-        rc->chart_type = rc_new->chart_type;
-        rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
-    }
-
-    if(rc->priority != rc_new->priority) {
-        rc->priority = rc_new->priority;
-        rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
-    }
+    rrdcontext_merge_with(rc, rrd_flag_is_archived(rc_new),
+                          rc_new->title, rc_new->family, rc_new->units, rc_new->chart_type, rc_new->priority);
 
     rrd_flag_set(rc, rc_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on rc_new
 
diff --git a/database/contexts/internal.h b/database/contexts/internal.h
index c5663dd24..04ad0883a 100644
--- a/database/contexts/internal.h
+++ b/database/contexts/internal.h
@@ -59,6 +59,8 @@ typedef enum __attribute__ ((__packed__)) {
     RRD_FLAG_UPDATE_REASON_UNUSED                  = (1 << 21), // this context is not used anymore
     RRD_FLAG_UPDATE_REASON_DB_ROTATION             = (1 << 22), // this context changed because of a db rotation
 
+    RRD_FLAG_MERGED_COLLECTED_RI_TO_RC             = (1 << 29),
+
     // action to perform on an object
     RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION        = (1 << 30), // this object has to update its retention from the db
 } RRD_FLAGS;
@@ -381,4 +383,6 @@ uint64_t rrdcontext_version_hash_with_callback(
 
 void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused);
 
+void rrdcontext_update_from_collected_rrdinstance(RRDINSTANCE *ri);
+
 #endif //NETDATA_RRDCONTEXT_INTERNAL_H
diff --git a/database/contexts/query_target.c b/database/contexts/query_target.c
index 508977ce7..829640b90 100644
--- a/database/contexts/query_target.c
+++ b/database/contexts/query_target.c
@@ -1052,7 +1052,8 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
     if(query_target_has_percentage_of_group(qt))
         qt->window.options &= ~RRDR_OPTION_PERCENTAGE;
 
-    rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now);
+    qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now,
+                                                             unittest_running);
 
     // prepare our local variables - we need these across all these functions
     QUERY_TARGET_LOCALS qtl = {
diff --git a/database/contexts/rrdcontext.h b/database/contexts/rrdcontext.h
index e3a1ab9af..0bcdb68de 100644
--- a/database/contexts/rrdcontext.h
+++ b/database/contexts/rrdcontext.h
@@ -409,6 +409,7 @@ typedef struct query_target {
     struct {
         SPINLOCK spinlock;
         bool used;                              // when true, this query is currently being used
+        bool relative;                          // when true, this query uses relative timestamps
         size_t queries;                         // how many query we have done so far with this QUERY_TARGET - not related to database queries
         struct query_target *prev;
         struct query_target *next;
diff --git a/database/contexts/worker.c b/database/contexts/worker.c
index e6c3ff3df..9d7c18863 100644
--- a/database/contexts/worker.c
+++ b/database/contexts/worker.c
@@ -611,6 +611,13 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG
                         continue;
                     }
 
+                    bool ri_collected = rrd_flag_is_collected(ri);
+
+                    if(ri_collected && !rrd_flag_check(ri, RRD_FLAG_MERGED_COLLECTED_RI_TO_RC)) {
+                        rrdcontext_update_from_collected_rrdinstance(ri);
+                        rrd_flag_set(ri, RRD_FLAG_MERGED_COLLECTED_RI_TO_RC);
+                    }
+
                     if(unlikely(!currently_collected && rrd_flag_is_collected(ri) && ri->first_time_s))
                         currently_collected = true;