summaryrefslogtreecommitdiffstats
path: root/src/database/rrdset.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-05 12:08:03 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-05 12:08:18 +0000
commit5da14042f70711ea5cf66e034699730335462f66 (patch)
tree0f6354ccac934ed87a2d555f45be4c831cf92f4a /src/database/rrdset.c
parentReleasing debian version 1.44.3-2. (diff)
downloadnetdata-5da14042f70711ea5cf66e034699730335462f66.tar.xz
netdata-5da14042f70711ea5cf66e034699730335462f66.zip
Merging upstream version 1.45.3+dfsg.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--src/database/rrdset.c (renamed from database/rrdset.c)367
1 files changed, 28 insertions, 339 deletions
diff --git a/database/rrdset.c b/src/database/rrdset.c
index f4bb48aa7..fc206585d 100644
--- a/database/rrdset.c
+++ b/src/database/rrdset.c
@@ -2,7 +2,6 @@
#define NETDATA_RRD_INTERNALS
#include "rrd.h"
-#include <sched.h>
#include "storage_engine.h"
@@ -270,34 +269,19 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
rw_spinlock_init(&st->alerts.spinlock);
- if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
- if(!rrdset_memory_load_or_create_map_save(st, st->rrd_memory_mode)) {
- netdata_log_info("Failed to use db mode %s for chart '%s', falling back to ram mode.", (st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", rrdset_name(st));
- st->rrd_memory_mode = RRD_MEMORY_MODE_RAM;
- }
- }
-
// initialize the db tiers
{
for(size_t tier = 0; tier < storage_tiers ; tier++) {
STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
if(!eng) continue;
- st->storage_metrics_groups[tier] = storage_engine_metrics_group_get(eng->backend, host->db[tier].instance, &st->chart_uuid);
+ st->smg[tier] = storage_engine_metrics_group_get(eng->seb, host->db[tier].si, &st->chart_uuid);
}
}
rrddim_index_init(st);
- // chart variables - we need this for data collection to work (collector given chart variables) - not only health
- rrdsetvar_index_init(st);
-
- if (host->health.health_enabled) {
- st->rrdfamily = rrdfamily_add_and_acquire(host, rrdset_family(st));
- st->rrdvars = rrdvariables_create();
- rrddimvar_index_init(st);
- }
-
+ st->rrdvars = rrdvariables_create();
st->rrdlabels = rrdlabels_create();
rrdset_update_permanent_labels(st);
@@ -312,6 +296,14 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
}
void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
+ ND_LOG_STACK lgs[] = {
+ ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(st->rrdhost)),
+ ND_LOG_FIELD_TXT(NDF_NIDL_CONTEXT, rrdset_context(st)),
+ ND_LOG_FIELD_TXT(NDF_NIDL_INSTANCE, rrdset_name(st)),
+ ND_LOG_FIELD_END(),
+ };
+ ND_LOG_STACK_PUSH(lgs);
+
RRDHOST *host = st->rrdhost;
rrdset_flag_set(st, RRDSET_FLAG_COLLECTION_FINISHED);
@@ -327,9 +319,9 @@ void rrdset_finalize_collection(RRDSET *st, bool dimensions_too) {
STORAGE_ENGINE *eng = st->rrdhost->db[tier].eng;
if(!eng) continue;
- if(st->storage_metrics_groups[tier]) {
- storage_engine_metrics_group_release(eng->backend, host->db[tier].instance, st->storage_metrics_groups[tier]);
- st->storage_metrics_groups[tier] = NULL;
+ if(st->smg[tier]) {
+ storage_engine_metrics_group_release(eng->seb, host->db[tier].si, st->smg[tier]);
+ st->smg[tier] = NULL;
}
}
@@ -353,40 +345,26 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
// release the collector info
dictionary_destroy(st->functions_view);
- rrdcalc_unlink_all_rrdset_alerts(st);
+ rrdcalc_unlink_and_delete_all_rrdset_alerts(st);
// ------------------------------------------------------------------------
// the order of destruction is important here
- // 1. delete RRDDIMVAR index - this will speed up the destruction of RRDDIMs
- // because each dimension loops to find its own variables in this index.
- // There are no references to the items on this index from the dimensions.
- // To find their own, they have to walk-through the dictionary.
- rrddimvar_index_destroy(st); // destroy the rrddimvar index
-
- // 2. delete RRDSETVAR index
- rrdsetvar_index_destroy(st); // destroy the rrdsetvar index
-
- // 3. delete RRDVAR index after the above, to avoid triggering its garbage collector (they have references on this)
+ // 1. delete RRDVAR index after the above, to avoid triggering its garbage collector (they have references on this)
rrdvariables_destroy(st->rrdvars); // free all variables and destroy the rrdvar dictionary
- // 4. delete RRDFAMILY - this has to be last, because RRDDIMVAR and RRDSETVAR need the reference counter
- rrdfamily_release(host, st->rrdfamily); // release the acquired rrdfamily -- has to be after all variables
-
- // 5. delete RRDDIMs, now their variables are not existing, so this is fast
+ // 2. delete RRDDIMs, now their variables are not existing, so this is fast
rrddim_index_destroy(st); // free all the dimensions and destroy the dimensions index
- // 6. this has to be after the dimensions are freed, but before labels are freed (contexts need the labels)
+ // 3. this has to be after the dimensions are freed, but before labels are freed (contexts need the labels)
rrdcontext_removed_rrdset(st); // let contexts know
- // 7. destroy the chart labels
+ // 4. destroy the chart labels
rrdlabels_destroy(st->rrdlabels); // destroy the labels, after letting the contexts know
- // 8. destroy the ml handle
+ // 5. destroy the ml handle
ml_chart_delete(st);
- rrdset_memory_file_free(st); // remove files of db mode save and map
-
// ------------------------------------------------------------------------
// free it
@@ -403,7 +381,6 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
string_freez(st->module_name);
freez(st->exporting_flags);
- freez(st->db.cache_dir);
}
// the item to be inserted, is already in the dictionary
@@ -470,8 +447,6 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
if(old_family != st->family)
ctr->react_action |= RRDSET_REACT_UPDATED;
string_freez(old_family);
-
- // TODO - we should rename RRDFAMILY variables
}
if(ctr->context && *ctr->context) {
@@ -652,16 +627,10 @@ int rrdset_reset_name(RRDSET *st, const char *name) {
rrdset_index_del_name(host, st);
string_freez(st->name);
st->name = name_string;
- rrdsetvar_rename_all(st);
}
else
st->name = name_string;
- RRDDIM *rd;
- rrddim_foreach_read(rd, st)
- rrddimvar_rename_all(rd);
- rrddim_foreach_done(rd);
-
rrdset_index_add_name(host, st);
rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);
@@ -888,7 +857,7 @@ void rrdset_reset(RRDSET *st) {
if(!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
for(size_t tier = 0; tier < storage_tiers ;tier++)
- storage_engine_store_flush(rd->tiers[tier].db_collection_handle);
+ storage_engine_store_flush(rd->tiers[tier].sch);
}
}
rrddim_foreach_done(rd);
@@ -904,12 +873,9 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
if(entries < 5) entries = 5;
if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
- if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE || mode == RRD_MEMORY_MODE_RAM) {
+ if(mode == RRD_MEMORY_MODE_RAM) {
long header_size = 0;
- if(mode == RRD_MEMORY_MODE_MAP || mode == RRD_MEMORY_MODE_SAVE)
- header_size = (long)rrddim_memory_file_header_size();
-
long page = (long)sysconf(_SC_PAGESIZE);
long size = (long)(header_size + entries * sizeof(storage_number));
if (unlikely(size % page)) {
@@ -946,62 +912,6 @@ void rrdset_free(RRDSET *st) {
rrdset_index_del(st->rrdhost, st);
}
-void rrdset_save(RRDSET *st) {
- rrdset_memory_file_save(st);
-
- RRDDIM *rd;
- rrddim_foreach_read(rd, st)
- rrddim_memory_file_save(rd);
- rrddim_foreach_done(rd);
-}
-
-void rrdset_delete_files(RRDSET *st) {
- RRDDIM *rd;
-
- netdata_log_info("Deleting chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
-
- if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
- const char *cache_filename = rrdset_cache_filename(st);
- if(cache_filename) {
- netdata_log_info("Deleting chart header file '%s'.", cache_filename);
- if (unlikely(unlink(cache_filename) == -1))
- netdata_log_error("Cannot delete chart header file '%s'", cache_filename);
- }
- else
- netdata_log_error("Cannot find the cache filename of chart '%s'", rrdset_id(st));
- }
-
- rrddim_foreach_read(rd, st) {
- const char *cache_filename = rrddim_cache_filename(rd);
- if(!cache_filename) continue;
-
- netdata_log_info("Deleting dimension file '%s'.", cache_filename);
- if(unlikely(unlink(cache_filename) == -1))
- netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
- }
- rrddim_foreach_done(rd);
-
- if(st->db.cache_dir)
- recursively_delete_dir(st->db.cache_dir, "left-over chart");
-}
-
-void rrdset_delete_obsolete_dimensions(RRDSET *st) {
- RRDDIM *rd;
-
- netdata_log_info("Deleting dimensions of chart '%s' ('%s') from disk...", rrdset_id(st), rrdset_name(st));
-
- rrddim_foreach_read(rd, st) {
- if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
- const char *cache_filename = rrddim_cache_filename(rd);
- if(!cache_filename) continue;
- netdata_log_info("Deleting dimension file '%s'.", cache_filename);
- if(unlikely(unlink(cache_filename) == -1))
- netdata_log_error("Cannot delete dimension file '%s'", cache_filename);
- }
- }
- rrddim_foreach_done(rd);
-}
-
// ----------------------------------------------------------------------------
// RRDSET - create a chart
@@ -1267,7 +1177,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
if (likely(!storage_point_is_unset(t->virtual_point))) {
storage_engine_store_metric(
- t->db_collection_handle,
+ t->sch,
t->next_point_end_time_s * USEC_PER_SEC,
t->virtual_point.sum,
t->virtual_point.min,
@@ -1278,7 +1188,7 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG
}
else {
storage_engine_store_metric(
- t->db_collection_handle,
+ t->sch,
t->next_point_end_time_s * USEC_PER_SEC,
NAN,
NAN,
@@ -1357,7 +1267,7 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
#endif // NETDATA_LOG_COLLECTION_ERRORS
// store the metric on tier 0
- storage_engine_store_metric(rd->tiers[0].db_collection_handle, point_end_time_ut,
+ storage_engine_store_metric(rd->tiers[0].sch, point_end_time_ut,
n, 0, 0,
1, 0, flags);
@@ -1377,7 +1287,7 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
};
for(size_t tier = 1; tier < storage_tiers ;tier++) {
- if(unlikely(!rd->tiers[tier].db_metric_handle)) continue;
+ if(unlikely(!rd->tiers[tier].smh)) continue;
struct rrddim_tier *t = &rd->tiers[tier];
@@ -1718,8 +1628,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
// check if we will re-write the entire data set
if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut &&
st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {
- netdata_log_info(
- "'%s': too old data (last updated at %"PRId64".%"PRId64", last collected at %"PRId64".%"PRId64"). "
+ nd_log_daemon(NDLP_DEBUG, "'%s': too old data (last updated at %" PRId64 ".%" PRId64 ", last collected at %" PRId64 ".%" PRId64 "). "
"Resetting it. Will not store the next entry.",
rrdset_id(st),
(int64_t)st->last_updated.tv_sec,
@@ -2101,18 +2010,6 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
// ALL DONE ABOUT THE DATA UPDATE
// --------------------------------------------------------------------
- if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_MAP)) {
- // update the memory mapped files with the latest values
-
- rrdset_memory_file_update(st);
-
- for(dim_id = 0, rda = rda_base; dim_id < rda_slots ; ++dim_id, ++rda) {
- rd = rda->rd;
- if(unlikely(!rd)) continue;
- rrddim_memory_file_update(rd);
- }
- }
-
for(dim_id = 0, rda = rda_base; dim_id < rda_slots ; ++dim_id, ++rda) {
rd = rda->rd;
if(unlikely(!rd)) continue;
@@ -2141,9 +2038,9 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
RRDDIM *rd;
rrddim_foreach_read(rd, st) {
for (size_t tier = 0; tier < storage_tiers; tier++) {
- if (rd->tiers[tier].db_collection_handle)
+ if (rd->tiers[tier].sch)
storage_engine_store_change_collection_frequency(
- rd->tiers[tier].db_collection_handle,
+ rd->tiers[tier].sch,
(int)(st->rrdhost->db[tier].tier_grouping * st->update_every));
}
}
@@ -2151,211 +2048,3 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {
return prev_update_every_s;
}
-
-// ----------------------------------------------------------------------------
-// compatibility layer for RRDSET files v019
-
-#define RRDSET_MAGIC_V019 "NETDATA RRD SET FILE V019"
-#define RRD_ID_LENGTH_MAX_V019 200
-
-struct avl_element_v019 {
- void *avl_link[2];
- signed char avl_balance;
-};
-struct avl_tree_type_v019 {
- void *root;
- int (*compar)(void *a, void *b);
-};
-struct avl_tree_lock_v019 {
- struct avl_tree_type_v019 avl_tree;
- pthread_rwlock_t rwlock;
-};
-struct rrdset_map_save_v019 {
- struct avl_element_v019 avl; // ignored
- struct avl_element_v019 avlname; // ignored
- char id[RRD_ID_LENGTH_MAX_V019 + 1]; // check to reset all - update on load
- void *name; // ignored
- void *unused_ptr; // ignored
- void *type; // ignored
- void *family; // ignored
- void *title; // ignored
- void *units; // ignored
- void *context; // ignored
- uint32_t hash_context; // ignored
- uint32_t chart_type; // ignored
- int update_every; // check to reset all - update on load
- long entries; // check to reset all - update on load
- long current_entry; // NEEDS TO BE UPDATED - FIXED ON LOAD
- uint32_t flags; // ignored
- void *exporting_flags; // ignored
- int gap_when_lost_iterations_above; // ignored
- long priority; // ignored
- uint32_t rrd_memory_mode; // ignored
- void *cache_dir; // ignored
- char cache_filename[FILENAME_MAX+1]; // ignored - update on load
- pthread_rwlock_t rrdset_rwlock; // ignored
- size_t counter; // NEEDS TO BE UPDATED - maintained on load
- size_t counter_done; // ignored
- union { //
- time_t last_accessed_time_s; // ignored
- time_t last_entry_s; // ignored
- }; //
- time_t upstream_resync_time; // ignored
- void *plugin_name; // ignored
- void *module_name; // ignored
- void *chart_uuid; // ignored
- void *state; // ignored
- size_t unused[3]; // ignored
- size_t rrddim_page_alignment; // ignored
- uint32_t hash; // ignored
- uint32_t hash_name; // ignored
- usec_t usec_since_last_update; // NEEDS TO BE UPDATED - maintained on load
- struct timeval last_updated; // NEEDS TO BE UPDATED - check to reset all - fixed on load
- struct timeval last_collected_time; // ignored
- long long collected_total; // ignored
- long long last_collected_total; // ignored
- void *rrdfamily; // ignored
- void *rrdhost; // ignored
- void *next; // ignored
- long double green; // ignored
- long double red; // ignored
- struct avl_tree_lock_v019 rrdvar_root_index; // ignored
- void *variables; // ignored
- void *alarms; // ignored
- unsigned long memsize; // check to reset all - update on load
- char magic[sizeof(RRDSET_MAGIC_V019) + 1]; // check to reset all - update on load
- struct avl_tree_lock_v019 dimensions_index; // ignored
- void *dimensions; // ignored
-};
-
-void rrdset_memory_file_update(RRDSET *st) {
- if(!st->db.st_on_file) return;
- struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
-
- st_on_file->current_entry = st->db.current_entry;
- st_on_file->counter = st->counter;
- st_on_file->usec_since_last_update = st->usec_since_last_update;
- st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;
- st_on_file->last_updated.tv_usec = st->last_updated.tv_usec;
-}
-
-const char *rrdset_cache_filename(RRDSET *st) {
- if(!st->db.st_on_file) return NULL;
- struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
- return st_on_file->cache_filename;
-}
-
-const char *rrdset_cache_dir(RRDSET *st) {
- if(!st->db.cache_dir)
- st->db.cache_dir = rrdhost_cache_dir_for_rrdset_alloc(st->rrdhost, rrdset_id(st));
-
- return st->db.cache_dir;
-}
-
-void rrdset_memory_file_free(RRDSET *st) {
- if(!st->db.st_on_file) return;
-
- // needed for memory mode map, to save the latest state
- rrdset_memory_file_update(st);
-
- struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
- __atomic_sub_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
- netdata_munmap(st_on_file, st_on_file->memsize);
-
- // remove the pointers from the RRDDIM
- st->db.st_on_file = NULL;
-}
-
-void rrdset_memory_file_save(RRDSET *st) {
- if(!st->db.st_on_file) return;
-
- rrdset_memory_file_update(st);
-
- struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;
- if(st_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
-
- memory_file_save(st_on_file->cache_filename, st->db.st_on_file, st_on_file->memsize);
-}
-
-bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mode) {
- if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
- return false;
-
- char fullfilename[FILENAME_MAX + 1];
- snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", rrdset_cache_dir(st));
-
- unsigned long size = sizeof(struct rrdset_map_save_v019);
- struct rrdset_map_save_v019 *st_on_file = (struct rrdset_map_save_v019 *)netdata_mmap(
- fullfilename, size, ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0, false, NULL);
-
- if(!st_on_file) return false;
-
- time_t now_s = now_realtime_sec();
-
- st_on_file->magic[sizeof(RRDSET_MAGIC_V019)] = '\0';
- if(strcmp(st_on_file->magic, RRDSET_MAGIC_V019) != 0) {
- netdata_log_info("Initializing file '%s'.", fullfilename);
- memset(st_on_file, 0, size);
- }
- else if(strncmp(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019) != 0) {
- netdata_log_error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st));
- memset(st_on_file, 0, size);
- }
- else if(st_on_file->memsize != size || st_on_file->entries != st->db.entries) {
- netdata_log_error("File '%s' does not have the desired size. Clearing it.", fullfilename);
- memset(st_on_file, 0, size);
- }
- else if(st_on_file->update_every != st->update_every) {
- netdata_log_error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);
- memset(st_on_file, 0, size);
- }
- else if((now_s - st_on_file->last_updated.tv_sec) > (long)st->update_every * (long)st->db.entries) {
- netdata_log_info("File '%s' is too old. Clearing it.", fullfilename);
- memset(st_on_file, 0, size);
- }
- else if(st_on_file->last_updated.tv_sec > now_s + st->update_every) {
- netdata_log_error("File '%s' refers to the future by %zd secs. Resetting it to now.", fullfilename, (ssize_t)(st_on_file->last_updated.tv_sec - now_s));
- st_on_file->last_updated.tv_sec = now_s;
- }
-
- if(st_on_file->current_entry >= st_on_file->entries)
- st_on_file->current_entry = 0;
-
- // make sure the database is aligned
- bool align_last_updated = false;
- if(st_on_file->last_updated.tv_sec) {
- st_on_file->update_every = st->update_every;
- align_last_updated = true;
- }
-
- // copy the useful values to st
- st->db.current_entry = st_on_file->current_entry;
- st->counter = st_on_file->counter;
- st->usec_since_last_update = st_on_file->usec_since_last_update;
- st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;
- st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;
-
- // link it to st
- st->db.st_on_file = st_on_file;
-
- // clear everything
- memset(st_on_file, 0, size);
-
- // set the values we need
- strncpyz(st_on_file->id, rrdset_id(st), RRD_ID_LENGTH_MAX_V019);
- strcpy(st_on_file->cache_filename, fullfilename);
- strcpy(st_on_file->magic, RRDSET_MAGIC_V019);
- st_on_file->memsize = size;
- st_on_file->entries = st->db.entries;
- st_on_file->update_every = st->update_every;
- st_on_file->rrd_memory_mode = memory_mode;
-
- if(align_last_updated)
- last_updated_time_align(st);
-
- // copy the useful values back to st_on_file
- rrdset_memory_file_update(st);
-
- __atomic_add_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);
- return true;
-}