summaryrefslogtreecommitdiffstats
path: root/database/rrddim.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2023-02-06 16:11:34 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2023-02-06 16:11:34 +0000
commitd079b656b4719739b2247dcd9d46e9bec793095a (patch)
treed2c950c70a776bcf697c963151c5bd959f8a9f03 /database/rrddim.c
parentReleasing debian version 1.37.1-2. (diff)
downloadnetdata-d079b656b4719739b2247dcd9d46e9bec793095a.tar.xz
netdata-d079b656b4719739b2247dcd9d46e9bec793095a.zip
Merging upstream version 1.38.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'database/rrddim.c')
-rw-r--r--database/rrddim.c157
1 files changed, 80 insertions, 77 deletions
diff --git a/database/rrddim.c b/database/rrddim.c
index 2d909a701..b8059b3c4 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -64,12 +64,15 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
size_t entries = st->entries;
if(!entries) entries = 5;
- rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1);
+ rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL);
if(!rd->db) {
info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));
ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;
}
- else rd->memsize = entries * sizeof(storage_number);
+ else {
+ rd->memsize = entries * sizeof(storage_number);
+ __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);
+ }
}
if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) {
@@ -78,42 +81,24 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
rd->db = rrddim_alloc_db(entries);
rd->memsize = entries * sizeof(storage_number);
+ __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);
}
rd->rrd_memory_mode = ctr->memory_mode;
- if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid)))) {
+ if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid))))
uuid_generate(rd->metric_uuid);
- bool found_in_sql = false; (void)found_in_sql;
-
-// bool found_in_sql = true;
-// if(unlikely(sql_find_dimension_uuid(st, rd, &rd->metric_uuid))) {
-// found_in_sql = false;
-// uuid_generate(rd->metric_uuid);
-// }
-
-#ifdef NETDATA_INTERNAL_CHECKS
- char uuid_str[UUID_STR_LEN];
- uuid_unparse_lower(rd->metric_uuid, uuid_str);
- error_report("Dimension UUID for host %s chart [%s] dimension [%s] not found in context. It is now set to %s (%s)",
- string2str(host->hostname),
- string2str(st->name),
- string2str(rd->name),
- uuid_str, found_in_sql ? "found in sqlite" : "newly generated");
-#endif
- }
// initialize the db tiers
{
size_t initialized = 0;
for(size_t tier = 0; tier < storage_tiers ; tier++) {
STORAGE_ENGINE *eng = host->db[tier].eng;
- rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier));
- rd->tiers[tier]->tier_grouping = host->db[tier].tier_grouping;
- rd->tiers[tier]->collect_ops = &eng->api.collect_ops;
- rd->tiers[tier]->query_ops = &eng->api.query_ops;
- rd->tiers[tier]->db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance);
- storage_point_unset(rd->tiers[tier]->virtual_point);
+ rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping;
+ rd->tiers[tier].collect_ops = &eng->api.collect_ops;
+ rd->tiers[tier].query_ops = &eng->api.query_ops;
+ rd->tiers[tier].db_metric_handle = eng->api.metric_get_or_create(rd, host->db[tier].instance);
+ storage_point_unset(rd->tiers[tier].virtual_point);
initialized++;
// internal_error(true, "TIER GROUPING of chart '%s', dimension '%s' for tier %d is set to %d", rd->rrdset->name, rd->name, tier, rd->tiers[tier]->tier_grouping);
@@ -122,7 +107,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
if(!initialized)
error("Failed to initialize all db tiers for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
- if(!rd->tiers[0])
+ if(!rd->tiers[0].db_metric_handle)
error("Failed to initialize the first db tier for chart '%s', dimension '%s", rrdset_name(st), rrddim_name(rd));
}
@@ -130,8 +115,8 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
{
size_t initialized = 0;
for (size_t tier = 0; tier < storage_tiers; tier++) {
- if (rd->tiers[tier]) {
- rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
+ if (rd->tiers[tier].db_metric_handle) {
+ rd->tiers[tier].db_collection_handle = rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
initialized++;
}
}
@@ -172,7 +157,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
- ml_new_dimension(rd);
+ ml_dimension_new(rd);
ctr->react_action = RRDDIM_REACT_NEW;
@@ -181,6 +166,25 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
}
+bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {
+ size_t tiers_available = 0, tiers_said_no_retention = 0;
+
+ for(size_t tier = 0; tier < storage_tiers ;tier++) {
+ if(!rd->tiers[tier].db_collection_handle)
+ continue;
+
+ tiers_available++;
+
+ if(rd->tiers[tier].collect_ops->finalize(rd->tiers[tier].db_collection_handle))
+ tiers_said_no_retention++;
+
+ rd->tiers[tier].db_collection_handle = NULL;
+ }
+
+ // return true if the dimension has retention in the db
+ return (!tiers_said_no_retention || tiers_available > tiers_said_no_retention);
+}
+
static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *rrdset) {
RRDDIM *rd = rrddim;
RRDSET *st = rrdset;
@@ -191,23 +195,11 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
rrdcontext_removed_rrddim(rd);
- ml_delete_dimension(rd);
+ ml_dimension_delete(rd);
debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
- size_t tiers_available = 0, tiers_said_no_retention = 0;
- for(size_t tier = 0; tier < storage_tiers ;tier++) {
- if(rd->tiers[tier] && rd->tiers[tier]->db_collection_handle) {
- tiers_available++;
-
- if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle))
- tiers_said_no_retention++;
-
- rd->tiers[tier]->db_collection_handle = NULL;
- }
- }
-
- if (tiers_available == tiers_said_no_retention && tiers_said_no_retention && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+ if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
/* This metric has no data and no references */
metaqueue_delete_dimension_uuid(&rd->metric_uuid);
}
@@ -224,16 +216,16 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
rrddim_memory_file_free(rd);
for(size_t tier = 0; tier < storage_tiers ;tier++) {
- if(!rd->tiers[tier]) continue;
+ if(!rd->tiers[tier].db_metric_handle) continue;
STORAGE_ENGINE* eng = host->db[tier].eng;
- eng->api.metric_release(rd->tiers[tier]->db_metric_handle);
-
- freez(rd->tiers[tier]);
- rd->tiers[tier] = NULL;
+ eng->api.metric_release(rd->tiers[tier].db_metric_handle);
+ rd->tiers[tier].db_metric_handle = NULL;
}
if(rd->db) {
+ __atomic_sub_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);
+
if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
netdata_munmap(rd->db, rd->memsize);
else
@@ -259,9 +251,9 @@ static bool rrddim_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
rc += rrddim_set_divisor(st, rd, ctr->divisor);
for(size_t tier = 0; tier < storage_tiers ;tier++) {
- if (rd->tiers[tier] && !rd->tiers[tier]->db_collection_handle)
- rd->tiers[tier]->db_collection_handle =
- rd->tiers[tier]->collect_ops->init(rd->tiers[tier]->db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
+ if (!rd->tiers[tier].db_collection_handle)
+ rd->tiers[tier].db_collection_handle =
+ rd->tiers[tier].collect_ops->init(rd->tiers[tier].db_metric_handle, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->storage_metrics_groups[tier]);
}
if(rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
@@ -285,7 +277,6 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
if(ctr->react_action & (RRDDIM_REACT_UPDATED | RRDDIM_REACT_NEW)) {
rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE);
- rrdset_flag_set(rd->rrdset, RRDSET_FLAG_METADATA_UPDATE);
rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
}
@@ -300,7 +291,8 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
void rrddim_index_init(RRDSET *st) {
if(!st->rrddim_root_index) {
- st->rrddim_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
+ st->rrddim_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
+ &dictionary_stats_category_rrdset_rrddim, sizeof(RRDDIM));
dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL);
dictionary_register_conflict_callback(st->rrddim_root_index, rrddim_conflict_callback, NULL);
@@ -420,38 +412,45 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
// ----------------------------------------------------------------------------
+time_t rrddim_last_entry_s_of_tier(RRDDIM *rd, size_t tier) {
+ if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
+ return 0;
+
+ return rd->tiers[tier].query_ops->latest_time_s(rd->tiers[tier].db_metric_handle);
+}
+
// get the timestamp of the last entry in the round-robin database
-time_t rrddim_last_entry_t(RRDDIM *rd) {
- time_t latest = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle);
+time_t rrddim_last_entry_s(RRDDIM *rd) {
+ time_t latest_time_s = rrddim_last_entry_s_of_tier(rd, 0);
for(size_t tier = 1; tier < storage_tiers ;tier++) {
- if(unlikely(!rd->tiers[tier])) continue;
+ if(unlikely(!rd->tiers[tier].db_metric_handle)) continue;
- time_t t = rd->tiers[tier]->query_ops->latest_time(rd->tiers[tier]->db_metric_handle);
- if(t > latest)
- latest = t;
+ time_t t = rrddim_last_entry_s_of_tier(rd, tier);
+ if(t > latest_time_s)
+ latest_time_s = t;
}
- return latest;
+ return latest_time_s;
}
-time_t rrddim_first_entry_t_of_tier(RRDDIM *rd, size_t tier) {
- if(unlikely(tier > storage_tiers || !rd->tiers[tier]))
+time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier) {
+ if(unlikely(tier > storage_tiers || !rd->tiers[tier].db_metric_handle))
return 0;
- return rd->tiers[tier]->query_ops->oldest_time(rd->tiers[tier]->db_metric_handle);
+ return rd->tiers[tier].query_ops->oldest_time_s(rd->tiers[tier].db_metric_handle);
}
-time_t rrddim_first_entry_t(RRDDIM *rd) {
- time_t oldest = 0;
+time_t rrddim_first_entry_s(RRDDIM *rd) {
+ time_t oldest_time_s = 0;
for(size_t tier = 0; tier < storage_tiers ;tier++) {
- time_t t = rrddim_first_entry_t_of_tier(rd, tier);
- if(t != 0 && (oldest == 0 || t < oldest))
- oldest = t;
+ time_t t = rrddim_first_entry_s_of_tier(rd, tier);
+ if(t != 0 && (oldest_time_s == 0 || t < oldest_time_s))
+ oldest_time_s = t;
}
- return oldest;
+ return oldest_time_s;
}
RRDDIM *rrddim_add_custom(RRDSET *st
@@ -498,8 +497,8 @@ int rrddim_hide(RRDSET *st, const char *id) {
return 1;
}
if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
- rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
- metaqueue_dimension_update_flags(rd);
+ rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN | RRDDIM_FLAG_METADATA_UPDATE);
+ rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
}
rrddim_option_set(rd, RRDDIM_OPTION_HIDDEN);
@@ -518,7 +517,8 @@ int rrddim_unhide(RRDSET *st, const char *id) {
}
if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN)) {
rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
- metaqueue_dimension_update_flags(rd);
+ rrddim_flag_set(rd, RRDDIM_FLAG_METADATA_UPDATE);
+ rrdhost_flag_set(rd->rrdset->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
}
rrddim_option_clear(rd, RRDDIM_OPTION_HIDDEN);
@@ -650,6 +650,7 @@ void rrddim_memory_file_free(RRDDIM *rd) {
rrddim_memory_file_update(rd);
struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
+ __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
freez(rd_on_file->cache_filename);
netdata_munmap(rd_on_file, rd_on_file->memsize);
@@ -686,10 +687,10 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
char filename[FILENAME_MAX + 1];
char fullfilename[FILENAME_MAX + 1];
rrdset_strncpyz_name(filename, rrddim_id(rd), FILENAME_MAX);
- snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
+ snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", rrdset_cache_dir(st), filename);
- rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size,
- ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1);
+ rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(
+ fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1, false, NULL);
if(unlikely(!rd_on_file)) return false;
@@ -747,6 +748,8 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO
rd_on_file->rrd_memory_mode = memory_mode;
rd_on_file->cache_filename = strdupz(fullfilename);
+ __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);
+
rd->db = &rd_on_file->values[0];
rd->rd_on_file = rd_on_file;
rd->memsize = size;