Merging upstream version 1.38.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 80 insertions, 77 deletions

diff --git a/database/rrddim.c b/database/rrddim.c
index 2d909a701..b8059b3c4 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -64,12 +64,15 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
         size_t entries = st->entries;
         if(!entries) entries = 5;
 
-        rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1);
+        rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL);
         if(!rd->db) {
             info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));
             ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;
         }
-        else rd->memsize = entries * sizeof(storage_number);
+        else {
+            rd->memsize = entries * sizeof(storage_number);
+            __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);
+        }
     }
 
     if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) {
@@ -78,42 +81,24 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
 
         rd->db = rrddim_alloc_db(entries);
         rd->memsize = entries * sizeof(storage_number);
+        __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);
     }
 
     rd->rrd_memory_mode = ctr->memory_mode;
 
-    if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid)))) {
+    if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid))))
         uuid_generate(rd->metric_uuid);
-        bool found_in_sql = false; (void)found_in_sql;
-
-//        bool found_in_sql = true;
-//        if(unlikely(sql_find_dimension_uuid(st, rd, &rd->metric_uuid))) {
-//            found_in_sql = false;
-//            uuid_generate(rd->metric_uuid);
-//        }
-
-#ifdef NETDATA_INTERNAL_CHECKS
-        char uuid_str[UUID_STR_LEN];
-        uuid_unparse_lower(rd->metric_uuid, uuid_str);
-        error_report("Dimension UUID for host %s chart [%s] dimension [%s] not found in context. It is now set to %s (%s)",
-                     string2str(host->hostname),
-                     string2str(st->name),
-                     string2str(rd->name),
-                     uuid_str, found_in_sql ? "found in sqlite" : "newly generated");
-#endif
-    }
 
     // initialize the db tiers
     {
         size_t initialized = 0;
         for(size_t tier = 0; tier < storage_tiers ; tier++) {
             STORAGE_ENGINE *eng = host->db[tier].eng;
-            rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier));
-            rd->tiers[tier]->tier_grouping = host->db[tier].tier_grouping;
-            rd->tiers[tier]->collect_ops = &eng->api.collect_ops;
-            rd->tiers[tier]->query_ops = &eng->api.query_ops;
-            rd->tiers[tier]->db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance);
-            storage_point_unset(rd->tiers[tier]->virtual_point);
+            rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping;
+            rd->tiers[tier].collect_ops = &eng->api.collect_ops;
+            rd->tiers[tier].query_ops = &eng->api.query_ops;
+            rd->tiers[tier].db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance);
+            storage_point_unset(rd->tiers[tier].virtual_point);
             initialized++;
 
             // internal_error(true, "TIER GROUPING of chart '%s', dimension '%s' for tier %d is set to %d", rd->rrdset->name, rd->name, tier, rd->tiers[tier]->tier_grouping);
@@ -122,7 +107,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
         if(!initialized)
             error("Failed to initialize all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
 
-        if(!rd->tiers[0])
+        if(!rd->tiers[0].db_metric_handle)
             error("Failed to initialize the first db tier for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
     }
 
@@ -130,8 +115,8 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
     {
         size_t initialized = 0;
         for (size_t tier = 0; tier < storage_tiers; tier++) {
-            if (rd->tiers[tier]) {
-                rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
+            if (rd->tiers[tier].db_metric_handle) {
+                rd->tiers[tier].db_collection_handle = rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
                 initialized++;
             }
         }
@@ -172,7 +157,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
     rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
     rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
 
-    ml_new_dimension(rd);
+    ml_dimension_new(rd);
 
     ctr->react_action = RRDDIM_REACT_NEW;
 
@@ -181,6 +166,25 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
 
 }
 
+bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {
+    size_t tiers_available = 0, tiers_said_no_retention = 0;
+
+    for(size_t tier = 0; tier < storage_tiers ;tier++) {
+        if(!rd->tiers[tier].db_collection_handle)
+            continue;
+
+        tiers_available++;
+
+        if(rd->tiers[tier].collect_ops->finalize(rd->tiers[tier].db_collection_handle))
+            tiers_said_no_retention++;
+
+        rd->tiers[tier].db_collection_handle = NULL;
+    }
+
+    // return true if the dimension has retention in the db
+    return (!tiers_said_no_retention || tiers_available > tiers_said_no_retention);
+}
+
 static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *rrdset) {
     RRDDIM *rd = rrddim;
     RRDSET *st = rrdset;
@@ -191,23 +195,11 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
 
     rrdcontext_removed_rrddim(rd);
 
-    ml_delete_dimension(rd);
+    ml_dimension_delete(rd);
 
     debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
 
-    size_t tiers_available = 0, tiers_said_no_retention = 0;
-    for(size_t tier = 0; tier < storage_tiers ;tier++) {
-        if(rd->tiers[tier] && rd->tiers[tier]->db_collection_handle) {
-            tiers_available++;
-
-            if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle))
-                tiers_said_no_retention++;
-
-            rd->tiers[tier]->db_collection_handle = NULL;
-        }
-    }
-
-    if (tiers_available == tiers_said_no_retention && tiers_said_no_retention && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+    if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
         /* This metric has no data and no references */
         metaqueue_delete_dimension_uuid(&rd->metric_uuid);
     }
@@ -224,16 +216,16 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
     rrddim_memory_file_free(rd);
 
     for(size_t tier = 0; tier < storage_tiers ;tier++) {
-        if(!rd->tiers[tier]) continue;
+        if(!rd->tiers[tier].db_metric_handle) continue;
 
         STORAGE_ENGINE* eng = host->db[tier].eng;
-        eng->api.metric_release(rd->tiers[tier]->db_metric_handle);
-
-        freez(rd->tiers[tier]);
-        rd->tiers[tier] = NULL;
+        eng->api.metric_release(rd->tiers[tier].db_metric_handle);
+        rd->tiers[tier].db_metric_handle = NULL;
     }
 
     if(rd->db) {
+        __atomic_sub_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);
+
         if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
             netdata_munmap(rd->db, rd->memsize);
         else
@@ -259,9 +251,9 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
     rc += rrddim_set_divisor(st, rd, ctr->divisor);
 
     for(size_t tier = 0; tier < storage_tiers ;tier++) {
-        if (rd->tiers[tier] && !rd->tiers[tier]->db_collection_handle)
-            rd->tiers[tier]->db_collection_handle =
-                rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
+        if (!rd->tiers[tier].db_collection_handle)
+            rd->tiers[tier].db_collection_handle =
+                rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
     }
 
     if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
@@ -285,7 +277,6 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
 
     if(ctr->react_action & (RRDDIM_REACT_UPDATED | RRDDIM_REACT_NEW)) {
         rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE);
-        rrdset_flag_set(rd->rrdset, RRDSET_FLAG_METADATA_UPDATE);
         rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
     }
 
@@ -300,7 +291,8 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
 
 void rrddim_index_init(RRDSET *st) {
     if(!st->rrddim_root_index) {
-        st->rrddim_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
+        st->rrddim_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
+                                                           &dictionary_stats_category_rrdset_rrddim, sizeof(RRDDIM));
 
         dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL);
         dictionary_register_conflict_callback(st->rrddim_root_index, rrddim_conflict_callback, NULL);
@@ -420,38 +412,45 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
 
 // ----------------------------------------------------------------------------
 
+time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier) {
+    if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
+        return 0;
+
+    return rd->tiers[tier].query_ops->latest_time_s(rd->tiers[tier].db_metric_handle);
+}
+
 // get the timestamp of the last entry in the round-robin database
-time_t rrddim_last_entry_t(RRDDIM *rd) {
-    time_t latest = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle);
+time_t rrddim_last_entry_s(RRDDIM *rd) {
+    time_t latest_time_s = rrddim_last_entry_s_of_tier(rd, 0);
 
     for(size_t tier = 1; tier < storage_tiers ;tier++) {
-        if(unlikely(!rd->tiers[tier])) continue;
+        if(unlikely(!rd->tiers[tier].db_metric_handle)) continue;
 
-        time_t t = rd->tiers[tier]->query_ops->latest_time(rd->tiers[tier]->db_metric_handle);
-        if(t > latest)
-            latest = t;
+        time_t t = rrddim_last_entry_s_of_tier(rd, tier);
+        if(t > latest_time_s)
+            latest_time_s = t;
     }
 
-    return latest;
+    return latest_time_s;
 }
 
-time_t rrddim_first_entry_t_of_tier(RRDDIM *rd, size_t tier) {
-    if(unlikely(tier > storage_tiers || !rd->tiers[tier]))
+time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier) {
+    if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
         return 0;
 
-    return rd->tiers[tier]->query_ops->oldest_time(rd->tiers[tier]->db_metric_handle);
+    return rd->tiers[tier].query_ops->oldest_time_s(rd->tiers[tier].db_metric_handle);
 }
 
-time_t rrddim_first_entry_t(RRDDIM *rd) {
-    time_t oldest = 0;
+time_t rrddim_first_entry_s(RRDDIM *rd) {
+    time_t oldest_time_s = 0;
 
     for(size_t tier = 0; tier < storage_tiers ;tier++) {
-        time_t t = rrddim_first_entry_t_of_tier(rd, tier);
-        if(t != 0 && (oldest == 0 || t < oldest))
-            oldest = t;
+        time_t t = rrddim_first_entry_s_of_tier(rd, tier);
+        if(t != 0 && (oldest_time_s == 0 || t < oldest_time_s))
+            oldest_time_s = t;
     }
 
-    return oldest;
+    return oldest_time_s;
 }
 
 RRDDIM *rrddim_add_custom(RRDSET *st
@@ -498,8 +497,8 @@ int rrddim_hide(RRDSET *st, const char *id) {
         return 1;
     }
     if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
-        rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
-        metaqueue_dimension_update_flags(rd);
+        rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN | RRDDIM_FLAG_METADATA_UPDATE);
+        rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
     }
 
     rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN);
@@ -518,7 +517,8 @@ int rrddim_unhide(RRDSET *st, const char *id) {
     }
     if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
         rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
-        metaqueue_dimension_update_flags(rd);
+        rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE);
+        rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
     }
 
     rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN);
@@ -650,6 +650,7 @@ void rrddim_memory_file_free(RRDDIM *rd) {
     rrddim_memory_file_update(rd);
 
     struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
+    __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
     freez(rd_on_file->cache_filename);
     netdata_munmap(rd_on_file, rd_on_file->memsize);
 
@@ -686,10 +687,10 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
     char filename[FILENAME_MAX + 1];
     char fullfilename[FILENAME_MAX + 1];
     rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX);
-    snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
+    snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename);
 
-    rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size,
-        ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1);
+    rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(
+        fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL);
 
     if(unlikely(!rd_on_file)) return false;
 
@@ -747,6 +748,8 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
     rd_on_file->rrd_memory_mode = memory_mode;
     rd_on_file->cache_filename = strdupz(fullfilename);
 
+    __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
+
     rd->db = &rd_on_file->values[0];
     rd->rd_on_file = rd_on_file;
     rd->memsize = size;