Adding upstream version 1.46.3.upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 52 insertions, 25 deletions

diff --git a/database/engine/rrdengine.h b/src/database/engine/rrdengine.h
index cd3352f12..37ea92b8a 100644
--- a/database/engine/rrdengine.h
+++ b/src/database/engine/rrdengine.h
@@ -3,9 +3,6 @@
 #ifndef NETDATA_RRDENGINE_H
 #define NETDATA_RRDENGINE_H
 
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
 #include <fcntl.h>
 #include <lz4.h>
 #include <Judy.h>
@@ -30,11 +27,18 @@ extern unsigned rrdeng_pages_per_extent;
 struct rrdengine_instance;
 struct rrdeng_cmd;
 
-#define MAX_PAGES_PER_EXTENT (64) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */
+#define MAX_PAGES_PER_EXTENT (109) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */
+#define DEFAULT_PAGES_PER_EXTENT (64)
 
 #define RRDENG_FILE_NUMBER_SCAN_TMPL "%1u-%10u"
 #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
 
+typedef struct dbengine_tier_stats  {
+    RRDSET *st;
+    RRDDIM *rd_space;
+    RRDDIM *rd_time;
+} DBENGINE_TIER_STATS;
+
 typedef enum __attribute__ ((__packed__)) {
     // final status for all pages
     // if a page does not have one of these, it is considered unroutable
@@ -142,7 +146,7 @@ struct jv2_extents_info {
 };
 
 struct jv2_metrics_info {
-    uuid_t *uuid;
+    nd_uuid_t *uuid;
     uint32_t page_list_header;
     time_t first_time_s;
     time_t last_time_s;
@@ -153,9 +157,9 @@ struct jv2_metrics_info {
 struct jv2_page_info {
     time_t start_time_s;
     time_t end_time_s;
-    time_t update_every_s;
-    size_t page_length;
+    uint32_t update_every_s;
     uint32_t extent_index;
+    size_t page_length;
     void *custom_data;
 
     // private
@@ -217,7 +221,7 @@ struct rrdeng_query_handle {
 
     // internal data
     time_t now_s;
-    time_t dt_s;
+    uint32_t dt_s;
 
     unsigned position;
     unsigned entries;
@@ -348,18 +352,25 @@ extern rrdeng_stats_t rrdeng_reserved_file_descriptors;
 extern rrdeng_stats_t global_pg_cache_over_half_dirty_events;
 extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of deleted pages */
 
-struct rrdengine_instance {
-    struct {
-        bool legacy;                                // true when the db is autonomous for a single host
-
-        int tier;                                   // the tier of this ctx
-        uint8_t page_type;                          // default page type for this context
+typedef struct tier_config_prototype {
+    int tier;                                   // the tier of this ctx
+    uint8_t page_type;                          // default page type for this context
+    uint64_t max_disk_space;                    // the max disk space this ctx is allowed to use
+    time_t max_retention_s;                     // The max retention in seconds
+    uint8_t disk_percentage;                    // percentage of metadata that contribute towards tier space used
+    uint8_t global_compress_alg;                // the wanted compression algorithm
+    char dbfiles_path[FILENAME_MAX + 1];
 
-        uint64_t max_disk_space;                    // the max disk space this ctx is allowed to use
-        uint8_t global_compress_alg;                // the wanted compression algorithm
+    struct {
+        uint32_t uses;
+        bool enabled;
+        bool is_on_disk;
+        SPINLOCK spinlock;
+    } _internal;
+} TIER_CONFIG_PROTOTYPE;
 
-        char dbfiles_path[FILENAME_MAX + 1];
-    } config;
+struct rrdengine_instance {
+    TIER_CONFIG_PROTOTYPE config;
 
     struct {
         uv_rwlock_t rwlock;                         // the linked list of datafiles is protected by this lock
@@ -387,6 +398,8 @@ struct rrdengine_instance {
         unsigned extents_currently_being_flushed;   // non-zero until we commit data to disk (both datafile and journal file)
 
         time_t first_time_s;
+        uint64_t metrics;
+        uint64_t samples;
     } atomic;
 
     struct {
@@ -450,7 +463,7 @@ static inline void ctx_last_flush_fileno_set(struct rrdengine_instance *ctx, uns
 void *dbengine_extent_alloc(size_t size);
 void dbengine_extent_free(void *extent, size_t size);
 
-bool rrdeng_ctx_exceeded_disk_quota(struct rrdengine_instance *ctx);
+bool rrdeng_ctx_tier_cap_exceeded(struct rrdengine_instance *ctx);
 int init_rrd_files(struct rrdengine_instance *ctx);
 void finalize_rrd_files(struct rrdengine_instance *ctx);
 bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx);
@@ -482,7 +495,7 @@ struct page_descr_with_data *page_descriptor_get(void);
 typedef struct validated_page_descriptor {
     time_t start_time_s;
     time_t end_time_s;
-    time_t update_every_s;
+    uint32_t update_every_s;
     size_t page_length;
     size_t point_size;
     size_t entries;
@@ -496,19 +509,19 @@ typedef struct validated_page_descriptor {
 #define page_entries_by_size(page_length_in_bytes, point_size_in_bytes) \
         ((page_length_in_bytes) / (point_size_in_bytes))
 
-VALIDATED_PAGE_DESCRIPTOR validate_page(uuid_t *uuid,
+VALIDATED_PAGE_DESCRIPTOR validate_page(nd_uuid_t *uuid,
                                         time_t start_time_s,
                                         time_t end_time_s,
-                                        time_t update_every_s,
+                                        uint32_t update_every_s,
                                         size_t page_length,
                                         uint8_t page_type,
                                         size_t entries,
                                         time_t now_s,
-                                        time_t overwrite_zero_update_every_s,
+                                        uint32_t overwrite_zero_update_every_s,
                                         bool have_read_error,
                                         const char *msg,
                                         RRDENG_COLLECT_PAGE_FLAGS flags);
-VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_extent_page_descr *descr, time_t now_s, time_t overwrite_zero_update_every_s, bool have_read_error);
+VALIDATED_PAGE_DESCRIPTOR validate_extent_page_descr(const struct rrdeng_extent_page_descr *descr, time_t now_s, uint32_t overwrite_zero_update_every_s, bool have_read_error);
 void collect_page_flags_to_buffer(BUFFER *wb, RRDENG_COLLECT_PAGE_FLAGS flags);
 
 typedef enum {
@@ -525,8 +538,22 @@ static inline time_t max_acceptable_collected_time(void) {
 
 void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, bool update_retention, bool worker);
 
+// --------------------------------------------------------------------------------------------------------------------
+// the following functions are used to sort UUIDs in the journal files
+// DO NOT CHANGE, as this will break backwards compatibility with the data files users have.
+
+static inline int journal_uuid_memcmp(const nd_uuid_t *uu1, const nd_uuid_t *uu2) {
+    return memcmp(uu1, uu2, sizeof(nd_uuid_t));
+}
+
 static inline int journal_metric_uuid_compare(const void *key, const void *metric) {
-    return uuid_memcmp((uuid_t *)key, &(((struct journal_metric_list *) metric)->uuid));
+    return journal_uuid_memcmp((const nd_uuid_t *)key, (const nd_uuid_t *)&(((struct journal_metric_list *) metric)->uuid));
 }
 
+// --------------------------------------------------------------------------------------------------------------------
+uint64_t get_used_disk_space(struct rrdengine_instance *ctx);
+void calculate_tier_disk_space_percentage(void);
+void dbengine_retention_statistics(void);
+uint64_t get_directory_free_bytes_space(struct rrdengine_instance *ctx);
+
 #endif /* NETDATA_RRDENGINE_H */