diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 11:19:16 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-07-24 09:53:24 +0000 |
commit | b5f8ee61a7f7e9bd291dd26b0585d03eb686c941 (patch) | |
tree | d4d31289c39fc00da064a825df13a0b98ce95b10 /src/database/engine/rrdengine.h | |
parent | Adding upstream version 1.44.3. (diff) | |
download | netdata-upstream.tar.xz netdata-upstream.zip |
Adding upstream version 1.46.3.upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r-- | src/database/engine/rrdengine.h (renamed from database/engine/rrdengine.h) | 77 |
1 files changed, 52 insertions, 25 deletions
diff --git a/database/engine/rrdengine.h b/src/database/engine/rrdengine.h index cd3352f12..37ea92b8a 100644 --- a/database/engine/rrdengine.h +++ b/src/database/engine/rrdengine.h @@ -3,9 +3,6 @@ #ifndef NETDATA_RRDENGINE_H #define NETDATA_RRDENGINE_H -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif #include <fcntl.h> #include <lz4.h> #include <Judy.h> @@ -30,11 +27,18 @@ extern unsigned rrdeng_pages_per_extent; struct rrdengine_instance; struct rrdeng_cmd; -#define MAX_PAGES_PER_EXTENT (64) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */ +#define MAX_PAGES_PER_EXTENT (109) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */ +#define DEFAULT_PAGES_PER_EXTENT (64) #define RRDENG_FILE_NUMBER_SCAN_TMPL "%1u-%10u" #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u" +typedef struct dbengine_tier_stats { + RRDSET *st; + RRDDIM *rd_space; + RRDDIM *rd_time; +} DBENGINE_TIER_STATS; + typedef enum __attribute__ ((__packed__)) { // final status for all pages // if a page does not have one of these, it is considered unroutable @@ -142,7 +146,7 @@ struct jv2_extents_info { }; struct jv2_metrics_info { - uuid_t *uuid; + nd_uuid_t *uuid; uint32_t page_list_header; time_t first_time_s; time_t last_time_s; @@ -153,9 +157,9 @@ struct jv2_metrics_info { struct jv2_page_info { time_t start_time_s; time_t end_time_s; - time_t update_every_s; - size_t page_length; + uint32_t update_every_s; uint32_t extent_index; + size_t page_length; void *custom_data; // private @@ -217,7 +221,7 @@ struct rrdeng_query_handle { // internal data time_t now_s; - time_t dt_s; + uint32_t dt_s; unsigned position; unsigned entries; @@ -348,18 +352,25 @@ extern rrdeng_stats_t rrdeng_reserved_file_descriptors; extern rrdeng_stats_t global_pg_cache_over_half_dirty_events; extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of deleted pages */ -struct rrdengine_instance { - struct { - bool legacy; // true when the db is autonomous for a single host - - int tier; // the tier of this ctx - uint8_t page_type; // default page type for this context +typedef struct tier_config_prototype { + int tier; // the tier of this ctx + uint8_t page_type; // default page type for this context + uint64_t max_disk_space; // the max disk space this ctx is allowed to use + time_t max_retention_s; // The max retention in seconds + uint8_t disk_percentage; // percentage of metadata that contribute towards tier space used + uint8_t global_compress_alg; // the wanted compression algorithm + char dbfiles_path[FILENAME_MAX + 1]; - uint64_t max_disk_space; // the max disk space this ctx is allowed to use - uint8_t global_compress_alg; // the wanted compression algorithm + struct { + uint32_t uses; + bool enabled; + bool is_on_disk; + SPINLOCK spinlock; + } _internal; +} TIER_CONFIG_PROTOTYPE; - char dbfiles_path[FILENAME_MAX + 1]; - } config; +struct rrdengine_instance { + TIER_CONFIG_PROTOTYPE config; struct { uv_rwlock_t rwlock; // the linked list of datafiles is protected by this lock @@ -387,6 +398,8 @@ struct rrdengine_instance { unsigned extents_currently_being_flushed; // non-zero until we commit data to disk (both datafile and journal file) time_t first_time_s; + uint64_t metrics; + uint64_t samples; } atomic; struct { @@ -450,7 +463,7 @@ static inline void ctx_last_flush_fileno_set(struct rrdengine_instance *ctx, uns void *dbengine_extent_alloc(size_t size); void dbengine_extent_free(void *extent, size_t size); -bool rrdeng_ctx_exceeded_disk_quota(struct rrdengine_instance *ctx); +bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx); int init_rrd_files(struct rrdengine_instance *ctx); void finalize_rrd_files(struct rrdengine_instance *ctx); bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx); @@ -482,7 +495,7 @@ struct page_descr_with_data *page_descriptor_get(void); typedef struct validated_page_descriptor { time_t start_time_s; time_t end_time_s; - time_t update_every_s; + uint32_t update_every_s; size_t page_length; size_t point_size; size_t entries; @@ -496,19 +509,19 @@ typedef struct validated_page_descriptor { #define page_entries_by_size(page_length_in_bytes, point_size_in_bytes) \ ((page_length_in_bytes) / (point_size_in_bytes)) -VALIDATED_PAGE_DESCRIPTOR validate_page(uuid_t *uuid, +VALIDATED_PAGE_DESCRIPTOR validate_page(nd_uuid_t *uuid, time_t start_time_s, time_t end_time_s, - time_t update_every_s, + uint32_t update_every_s, size_t page_length, uint8_t page_type, size_t entries, time_t now_s, - time_t overwrite_zero_update_every_s, + uint32_t overwrite_zero_update_every_s, bool have_read_error, const char *msg, RRDENG_COLLECT_PAGE_FLAGS flags); -VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_extent_page_descr *descr, time_t now_s, time_t overwrite_zero_update_every_s, bool have_read_error); +VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_extent_page_descr *descr, time_t now_s, uint32_t overwrite_zero_update_every_s, bool have_read_error); void collect_page_flags_to_buffer(BUFFER *wb, RRDENG_COLLECT_PAGE_FLAGS flags); typedef enum { @@ -525,8 +538,22 @@ static inline time_t max_acceptable_collected_time(void) { void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, bool update_retention, bool worker); +// -------------------------------------------------------------------------------------------------------------------- +// the following functions are used to sort UUIDs in the journal files +// DO NOT CHANGE, as this will break backwards compatibility with the data files users have. + +static inline int journal_uuid_memcmp(const nd_uuid_t *uu1, const nd_uuid_t *uu2) { + return memcmp(uu1, uu2, sizeof(nd_uuid_t)); +} + static inline int journal_metric_uuid_compare(const void *key, const void *metric) { - return uuid_memcmp((uuid_t *)key, &(((struct journal_metric_list *) metric)->uuid)); + return journal_uuid_memcmp((const nd_uuid_t *)key, (const nd_uuid_t *)&(((struct journal_metric_list *) metric)->uuid)); } +// -------------------------------------------------------------------------------------------------------------------- +uint64_t get_used_disk_space(struct rrdengine_instance *ctx); +void calculate_tier_disk_space_percentage(void); +void dbengine_retention_statistics(void); +uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx); + #endif /* NETDATA_RRDENGINE_H */ |