From aa2fe8ccbfcb117efa207d10229eeeac5d0f97c7 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 6 Feb 2023 17:11:30 +0100 Subject: Adding upstream version 1.38.0. Signed-off-by: Daniel Baumann --- database/rrddim.c | 157 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 77 deletions(-) (limited to 'database/rrddim.c') diff --git a/database/rrddim.c b/database/rrddim.c index 2d909a701..b8059b3c4 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -64,12 +64,15 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v size_t entries = st->entries; if(!entries) entries = 5; - rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1); + rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL); if(!rd->db) { info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd)); ctr->memory_mode = RRD_MEMORY_MODE_ALLOC; } - else rd->memsize = entries * sizeof(storage_number); + else { + rd->memsize = entries * sizeof(storage_number); + __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED); + } } if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) { @@ -78,42 +81,24 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v rd->db = rrddim_alloc_db(entries); rd->memsize = entries * sizeof(storage_number); + __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED); } rd->rrd_memory_mode = ctr->memory_mode; - if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid)))) { + if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid)))) uuid_generate(rd->metric_uuid); - bool found_in_sql = false; (void)found_in_sql; - -// bool found_in_sql = true; -// if(unlikely(sql_find_dimension_uuid(st, rd, &rd->metric_uuid))) { -// found_in_sql = false; -// uuid_generate(rd->metric_uuid); -// } - -#ifdef NETDATA_INTERNAL_CHECKS - char uuid_str[UUID_STR_LEN]; - uuid_unparse_lower(rd->metric_uuid, uuid_str); - error_report("Dimension UUID for host %s chart [%s] dimension [%s] not found in context. It is now set to %s (%s)", - string2str(host->hostname), - string2str(st->name), - string2str(rd->name), - uuid_str, found_in_sql ? "found in sqlite" : "newly generated"); -#endif - } // initialize the db tiers { size_t initialized = 0; for(size_t tier = 0; tier < storage_tiers ; tier++) { STORAGE_ENGINE *eng = host->db[tier].eng; - rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier)); - rd->tiers[tier]->tier_grouping = host->db[tier].tier_grouping; - rd->tiers[tier]->collect_ops = &eng->api.collect_ops; - rd->tiers[tier]->query_ops = &eng->api.query_ops; - rd->tiers[tier]->db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance); - storage_point_unset(rd->tiers[tier]->virtual_point); + rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping; + rd->tiers[tier].collect_ops = &eng->api.collect_ops; + rd->tiers[tier].query_ops = &eng->api.query_ops; + rd->tiers[tier].db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance); + storage_point_unset(rd->tiers[tier].virtual_point); initialized++; // internal_error(true, "TIER GROUPING of chart '%s', dimension '%s' for tier %d is set to %d", rd->rrdset->name, rd->name, tier, rd->tiers[tier]->tier_grouping); @@ -122,7 +107,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v if(!initialized) error("Failed to initialize all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); - if(!rd->tiers[0]) + if(!rd->tiers[0].db_metric_handle) error("Failed to initialize the first db tier for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd)); } @@ -130,8 +115,8 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v { size_t initialized = 0; for (size_t tier = 0; tier < storage_tiers; tier++) { - if (rd->tiers[tier]) { - rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); + if (rd->tiers[tier].db_metric_handle) { + rd->tiers[tier].db_collection_handle = rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); initialized++; } } @@ -172,7 +157,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK); rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED); - ml_new_dimension(rd); + ml_dimension_new(rd); ctr->react_action = RRDDIM_REACT_NEW; @@ -181,6 +166,25 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v } +bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) { + size_t tiers_available = 0, tiers_said_no_retention = 0; + + for(size_t tier = 0; tier < storage_tiers ;tier++) { + if(!rd->tiers[tier].db_collection_handle) + continue; + + tiers_available++; + + if(rd->tiers[tier].collect_ops->finalize(rd->tiers[tier].db_collection_handle)) + tiers_said_no_retention++; + + rd->tiers[tier].db_collection_handle = NULL; + } + + // return true if the dimension has retention in the db + return (!tiers_said_no_retention || tiers_available > tiers_said_no_retention); +} + static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *rrdset) { RRDDIM *rd = rrddim; RRDSET *st = rrdset; @@ -191,23 +195,11 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v rrdcontext_removed_rrddim(rd); - ml_delete_dimension(rd); + ml_dimension_delete(rd); debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd)); - size_t tiers_available = 0, tiers_said_no_retention = 0; - for(size_t tier = 0; tier < storage_tiers ;tier++) { - if(rd->tiers[tier] && rd->tiers[tier]->db_collection_handle) { - tiers_available++; - - if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle)) - tiers_said_no_retention++; - - rd->tiers[tier]->db_collection_handle = NULL; - } - } - - if (tiers_available == tiers_said_no_retention && tiers_said_no_retention && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { /* This metric has no data and no references */ metaqueue_delete_dimension_uuid(&rd->metric_uuid); } @@ -224,16 +216,16 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v rrddim_memory_file_free(rd); for(size_t tier = 0; tier < storage_tiers ;tier++) { - if(!rd->tiers[tier]) continue; + if(!rd->tiers[tier].db_metric_handle) continue; STORAGE_ENGINE* eng = host->db[tier].eng; - eng->api.metric_release(rd->tiers[tier]->db_metric_handle); - - freez(rd->tiers[tier]); - rd->tiers[tier] = NULL; + eng->api.metric_release(rd->tiers[tier].db_metric_handle); + rd->tiers[tier].db_metric_handle = NULL; } if(rd->db) { + __atomic_sub_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED); + if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM) netdata_munmap(rd->db, rd->memsize); else @@ -259,9 +251,9 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, rc += rrddim_set_divisor(st, rd, ctr->divisor); for(size_t tier = 0; tier < storage_tiers ;tier++) { - if (rd->tiers[tier] && !rd->tiers[tier]->db_collection_handle) - rd->tiers[tier]->db_collection_handle = - rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); + if (!rd->tiers[tier].db_collection_handle) + rd->tiers[tier].db_collection_handle = + rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]); } if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { @@ -285,7 +277,6 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo if(ctr->react_action & (RRDDIM_REACT_UPDATED | RRDDIM_REACT_NEW)) { rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE); - rrdset_flag_set(rd->rrdset, RRDSET_FLAG_METADATA_UPDATE); rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE); } @@ -300,7 +291,8 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo void rrddim_index_init(RRDSET *st) { if(!st->rrddim_root_index) { - st->rrddim_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + st->rrddim_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + &dictionary_stats_category_rrdset_rrddim, sizeof(RRDDIM)); dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL); dictionary_register_conflict_callback(st->rrddim_root_index, rrddim_conflict_callback, NULL); @@ -420,38 +412,45 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) // ---------------------------------------------------------------------------- +time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier) { + if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle)) + return 0; + + return rd->tiers[tier].query_ops->latest_time_s(rd->tiers[tier].db_metric_handle); +} + // get the timestamp of the last entry in the round-robin database -time_t rrddim_last_entry_t(RRDDIM *rd) { - time_t latest = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle); +time_t rrddim_last_entry_s(RRDDIM *rd) { + time_t latest_time_s = rrddim_last_entry_s_of_tier(rd, 0); for(size_t tier = 1; tier < storage_tiers ;tier++) { - if(unlikely(!rd->tiers[tier])) continue; + if(unlikely(!rd->tiers[tier].db_metric_handle)) continue; - time_t t = rd->tiers[tier]->query_ops->latest_time(rd->tiers[tier]->db_metric_handle); - if(t > latest) - latest = t; + time_t t = rrddim_last_entry_s_of_tier(rd, tier); + if(t > latest_time_s) + latest_time_s = t; } - return latest; + return latest_time_s; } -time_t rrddim_first_entry_t_of_tier(RRDDIM *rd, size_t tier) { - if(unlikely(tier > storage_tiers || !rd->tiers[tier])) +time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier) { + if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle)) return 0; - return rd->tiers[tier]->query_ops->oldest_time(rd->tiers[tier]->db_metric_handle); + return rd->tiers[tier].query_ops->oldest_time_s(rd->tiers[tier].db_metric_handle); } -time_t rrddim_first_entry_t(RRDDIM *rd) { - time_t oldest = 0; +time_t rrddim_first_entry_s(RRDDIM *rd) { + time_t oldest_time_s = 0; for(size_t tier = 0; tier < storage_tiers ;tier++) { - time_t t = rrddim_first_entry_t_of_tier(rd, tier); - if(t != 0 && (oldest == 0 || t < oldest)) - oldest = t; + time_t t = rrddim_first_entry_s_of_tier(rd, tier); + if(t != 0 && (oldest_time_s == 0 || t < oldest_time_s)) + oldest_time_s = t; } - return oldest; + return oldest_time_s; } RRDDIM *rrddim_add_custom(RRDSET *st @@ -498,8 +497,8 @@ int rrddim_hide(RRDSET *st, const char *id) { return 1; } if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) { - rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN); - metaqueue_dimension_update_flags(rd); + rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN | RRDDIM_FLAG_METADATA_UPDATE); + rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE); } rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN); @@ -518,7 +517,8 @@ int rrddim_unhide(RRDSET *st, const char *id) { } if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) { rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN); - metaqueue_dimension_update_flags(rd); + rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE); + rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE); } rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN); @@ -650,6 +650,7 @@ void rrddim_memory_file_free(RRDDIM *rd) { rrddim_memory_file_update(rd); struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file; + __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED); freez(rd_on_file->cache_filename); netdata_munmap(rd_on_file, rd_on_file->memsize); @@ -686,10 +687,10 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO char filename[FILENAME_MAX + 1]; char fullfilename[FILENAME_MAX + 1]; rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX); - snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename); + snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename); - rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size, - ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1); + rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap( + fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL); if(unlikely(!rd_on_file)) return false; @@ -747,6 +748,8 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO rd_on_file->rrd_memory_mode = memory_mode; rd_on_file->cache_filename = strdupz(fullfilename); + __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED); + rd->db = &rd_on_file->values[0]; rd->rd_on_file = rd_on_file; rd->memsize = size; -- cgit v1.2.3