Adding upstream version 1.36.0.upstream/1.36.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 269 insertions, 277 deletions

diff --git a/database/rrd.h b/database/rrd.h
index dc32b2a2d..605ff50bc 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -3,6 +3,15 @@
 #ifndef NETDATA_RRD_H
 #define NETDATA_RRD_H 1
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// non-existing structs instead of voids
+// to enable type checking at compile time
+typedef struct storage_instance STORAGE_INSTANCE;
+typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
+
 // forward typedefs
 typedef struct rrdhost RRDHOST;
 typedef struct rrddim RRDDIM;
@@ -19,10 +28,10 @@ typedef void *ml_host_t;
 typedef void *ml_dimension_t;
 
 // forward declarations
-struct rrddim_volatile;
+struct rrddim_tier;
 struct rrdset_volatile;
 struct context_param;
-struct label;
+
 #ifdef ENABLE_DBENGINE
 struct rrdeng_page_descr;
 struct rrdengine_instance;
@@ -31,6 +40,7 @@ struct pg_cache_page_index;
 
 #include "daemon/common.h"
 #include "web/api/queries/query.h"
+#include "web/api/queries/rrdr.h"
 #include "rrdvar.h"
 #include "rrdsetvar.h"
 #include "rrddimvar.h"
@@ -39,6 +49,18 @@ struct pg_cache_page_index;
 #include "streaming/rrdpush.h"
 #include "aclk/aclk_rrdhost_state.h"
 #include "sqlite/sqlite_health.h"
+#include "rrdcontext.h"
+
+extern int storage_tiers;
+extern int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
+
+typedef enum {
+    RRD_BACKFILL_NONE,
+    RRD_BACKFILL_FULL,
+    RRD_BACKFILL_NEW
+} RRD_BACKFILL;
+
+extern RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];
 
 enum {
     CONTEXT_FLAGS_ARCHIVE = 0x01,
@@ -71,9 +93,6 @@ extern time_t rrdset_free_obsolete_time;
 
 #define RRD_ID_LENGTH_MAX 200
 
-#define RRDSET_MAGIC        "NETDATA RRD SET FILE V019"
-#define RRDDIMENSION_MAGIC  "NETDATA RRD DIMENSION FILE V019"
-
 typedef long long total_number;
 #define TOTAL_NUMBER_FORMAT "%lld"
 
@@ -177,66 +196,48 @@ typedef enum rrddim_flags {
 #define rrddim_flag_set(rd, flag)   __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
 #define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
 
-typedef enum label_source {
-    LABEL_SOURCE_AUTO             = 0,
-    LABEL_SOURCE_NETDATA_CONF     = 1,
-    LABEL_SOURCE_DOCKER           = 2,
-    LABEL_SOURCE_ENVIRONMENT      = 3,
-    LABEL_SOURCE_KUBERNETES       = 4
-} LABEL_SOURCE;
-
-#define LABEL_FLAG_UPDATE_STREAM 1
-#define LABEL_FLAG_STOP_STREAM 2
-
-struct label {
-    char *key, *value;
-    uint32_t key_hash;
-    LABEL_SOURCE label_source;
-    struct label *next;
-};
+typedef enum rrdlabel_source {
+    RRDLABEL_SRC_AUTO       = (1 << 0), // set when Netdata found the label by some automation
+    RRDLABEL_SRC_CONFIG     = (1 << 1), // set when the user configured the label
+    RRDLABEL_SRC_K8S        = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
+    RRDLABEL_SRC_ACLK       = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
 
-struct label_index {
-    struct label *head;                     // Label list
-    netdata_rwlock_t labels_rwlock;         // lock for the label list
-    uint32_t labels_flag;                   // Flags for labels
-};
+    // more sources can be added here
+
+    RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
+    RRDLABEL_FLAG_OLD       = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
+    RRDLABEL_FLAG_NEW       = (1 << 31)  // marks for rrdlabels internal use - they are not exposed outside rrdlabels
+} RRDLABEL_SRC;
+
+#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
+
+extern DICTIONARY *rrdlabels_create(void);
+extern void rrdlabels_destroy(DICTIONARY *labels_dict);
+extern void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
+extern void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
+extern void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
+
+extern void rrdlabels_unmark_all(DICTIONARY *labels);
+extern void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
+
+extern int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
+extern int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
+
+extern void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
+extern bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
+extern bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal);
+extern int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
+
+extern void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
+extern void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
 
-typedef enum strip_quotes {
-    DO_NOT_STRIP_QUOTES,
-    STRIP_QUOTES
-} STRIP_QUOTES_OPTION;
-
-typedef enum skip_escaped_characters {
-    DO_NOT_SKIP_ESCAPED_CHARACTERS,
-    SKIP_ESCAPED_CHARACTERS
-} SKIP_ESCAPED_CHARACTERS_OPTION;
-
-char *translate_label_source(LABEL_SOURCE l);
-struct label *create_label(char *key, char *value, LABEL_SOURCE label_source);
-extern struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source);
-extern void update_label_list(struct label **labels, struct label *new_labels);
-extern void replace_label_list(struct label_index *labels, struct label *new_labels);
-extern int is_valid_label_value(char *value);
-extern int is_valid_label_key(char *key);
-extern void free_label_list(struct label *labels);
-extern struct label *label_list_lookup_key(struct label *head, char *key, uint32_t key_hash);
-extern struct label *label_list_lookup_keylist(struct label *head, char *keylist);
-extern int label_list_contains_keylist(struct label *head, char *keylist);
-extern int label_list_contains_key(struct label *head, char *key, uint32_t key_hash);
-extern int label_list_contains(struct label *head, struct label *check);
-extern struct label *merge_label_lists(struct label *lo_pri, struct label *hi_pri);
-extern void strip_last_symbol(
-    char *str,
-    char symbol,
-    SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
-extern char *strip_double_quotes(char *str, SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
 void reload_host_labels(void);
-extern void rrdset_add_label_to_new_list(RRDSET *st, char *key, char *value, LABEL_SOURCE source);
-extern void rrdset_finalize_labels(RRDSET *st);
-extern void rrdset_update_labels(RRDSET *st, struct label *labels);
-extern int rrdset_contains_label_keylist(RRDSET *st, char *key);
-extern int rrdset_matches_label_keys(RRDSET *st, char *key, char *words[], uint32_t *hash_key_list, int *word_count, int size);
-extern struct label *rrdset_lookup_label_key(RRDSET *st, char *key, uint32_t key_hash);
+extern void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
+
+extern int rrdlabels_unittest(void);
+
+// unfortunately this break when defined in exporting_engine.h
+extern bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
 
 // ----------------------------------------------------------------------------
 // RRD DIMENSION - this is a metric
@@ -247,6 +248,8 @@ struct rrddim {
 
     avl_t avl;                                      // the binary index - this has to be first member!
 
+    uuid_t metric_uuid;                             // global UUID for this metric (unique_across hosts)
+
     // ------------------------------------------------------------------------
     // the dimension definition
 
@@ -255,55 +258,54 @@ struct rrddim {
                                                     // this is a pointer to the config structure
                                                     // since the config always has a higher priority
                                                     // (the user overwrites the name of the charts)
-                                                    // DO NOT FREE THIS - IT IS ALLOCATED IN CONFIG
+    uint32_t hash;                                  // a simple hash of the id, to speed up searching / indexing
+                                                    // instead of strcmp() every item in the binary index
+                                                    // we first compare the hashes
+    uint32_t hash_name;                             // a simple hash of the name
+
 
     RRD_ALGORITHM algorithm;                        // the algorithm that is applied to add new collected values
     RRD_MEMORY_MODE rrd_memory_mode;                // the memory mode for this dimension
+    RRDDIM_FLAGS flags;                             // configuration flags for the dimension
+
+    bool updated;                                   // 1 when the dimension has been updated since the last processing
+    bool exposed;                                   // 1 when set what have sent this dimension to the central netdata
 
     collected_number multiplier;                    // the multiplier of the collected values
     collected_number divisor;                       // the divider of the collected values
 
-    uint32_t flags;                                 // configuration flags for the dimension
-
     // ------------------------------------------------------------------------
     // members for temporary data we need for calculations
 
-    uint32_t hash;                                  // a simple hash of the id, to speed up searching / indexing
-                                                    // instead of strcmp() every item in the binary index
-                                                    // we first compare the hashes
-
-    uint32_t hash_name;                             // a simple hash of the name
-
-    char *cache_filename;                           // the filename we load/save from/to this set
-
-    size_t collections_counter;                     // the number of times we added values to this rrdim
-    struct rrddim_volatile *state;                  // volatile state that is not persistently stored
-    size_t unused[8];
-
-    collected_number collected_value_max;           // the absolute maximum of the collected value
-
-    unsigned int updated:1;                         // 1 when the dimension has been updated since the last processing
-    unsigned int exposed:1;                         // 1 when set what have sent this dimension to the central netdata
-
     struct timeval last_collected_time;             // when was this dimension last updated
                                                     // this is actual date time we updated the last_collected_value
                                                     // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
 
-    calculated_number calculated_value;             // the current calculated value, after applying the algorithm - resets to zero after being used
-    calculated_number last_calculated_value;        // the last calculated value processed
+#ifdef ENABLE_ACLK
+    int aclk_live_status;
+#endif
+    ml_dimension_t ml_dimension;
+
+    struct rrddim_tier *tiers[RRD_STORAGE_TIERS];   // our tiers of databases
+
+    size_t collections_counter;                     // the number of times we added values to this rrddim
+    collected_number collected_value_max;           // the absolute maximum of the collected value
 
-    calculated_number last_stored_value;            // the last value as stored in the database (after interpolation)
+    NETDATA_DOUBLE calculated_value;                // the current calculated value, after applying the algorithm - resets to zero after being used
+    NETDATA_DOUBLE last_calculated_value;           // the last calculated value processed
+    NETDATA_DOUBLE last_stored_value;               // the last value as stored in the database (after interpolation)
 
     collected_number collected_value;               // the current value, as collected - resets to 0 after being used
     collected_number last_collected_value;          // the last value that was collected, after being processed
 
     // the *_volume members are used to calculate the accuracy of the rounding done by the
     // storage number - they are printed to debug.log when debug is enabled for a set.
-    calculated_number collected_volume;             // the sum of all collected values so far
-    calculated_number stored_volume;                // the sum of all stored values so far
+    NETDATA_DOUBLE collected_volume;                // the sum of all collected values so far
+    NETDATA_DOUBLE stored_volume;                   // the sum of all stored values so far
 
     struct rrddim *next;                            // linking of dimensions within the same data set
     struct rrdset *rrdset;
+    RRDMETRIC_ACQUIRED *rrdmetric;                  // the rrdmetric of this dimension
 
     // ------------------------------------------------------------------------
     // members for checking the data when loading from disk
@@ -314,18 +316,33 @@ struct rrddim {
 
     int update_every;                               // every how many seconds is this updated
 
-    size_t memsize;                                 // the memory allocated for this dimension
-
-    char magic[sizeof(RRDDIMENSION_MAGIC) + 1];     // a string to be saved, used to identify our data file
+    size_t memsize;                                 // the memory allocated for this dimension (without RRDDIM)
 
     struct rrddimvar *variables;
 
     // ------------------------------------------------------------------------
     // the values stored in this dimension, using our floating point numbers
 
-    storage_number values[];                        // the array of values - THIS HAS TO BE THE LAST MEMBER
+    void *rd_on_file;                               // pointer to the header written on disk
+    storage_number *db;                             // the array of values
 };
 
+// returns the RRDDIM cache filename, or NULL if it does not exist
+extern const char *rrddim_cache_filename(RRDDIM *rd);
+
+// updated the header with the latest RRDDIM value, for memory mode MAP and SAVE
+extern void rrddim_memory_file_update(RRDDIM *rd);
+
+// free the memory file structures for memory mode MAP and SAVE
+extern void rrddim_memory_file_free(RRDDIM *rd);
+
+extern bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode);
+
+// return the v019 header size of RRDDIM files
+extern size_t rrddim_memory_file_header_size(void);
+
+extern void rrddim_memory_file_save(RRDDIM *rd);
+
 // ----------------------------------------------------------------------------
 // engine-specific iterator state for dimension data collection
 typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
@@ -340,30 +357,71 @@ struct rrddim_query_handle {
     RRDDIM *rd;
     time_t start_time;
     time_t end_time;
+    TIER_QUERY_FETCH tier_query_fetch_type;
     STORAGE_QUERY_HANDLE* handle;
 };
 
+typedef struct storage_point {
+    NETDATA_DOUBLE min;     // when count > 1, this is the minimum among them
+    NETDATA_DOUBLE max;     // when count > 1, this is the maximum among them
+    NETDATA_DOUBLE sum;     // the point sum - divided by count gives the average
+
+    // end_time - start_time = point duration
+    time_t start_time;      // the time the point starts
+    time_t end_time;        // the time the point ends
+
+    unsigned count;         // the number of original points aggregated
+    unsigned anomaly_count; // the number of original points found anomalous
+
+    SN_FLAGS flags;         // flags stored with the point
+} STORAGE_POINT;
+
+#define storage_point_unset(x)                     do { \
+    (x).min = (x).max = (x).sum = NAN;                  \
+    (x).count = 0;                                      \
+    (x).anomaly_count = 0;                              \
+    (x).flags = SN_FLAG_NONE;                           \
+    (x).start_time = 0;                                 \
+    (x).end_time = 0;                                   \
+    } while(0)
+
+#define storage_point_empty(x, start_t, end_t)     do { \
+    (x).min = (x).max = (x).sum = NAN;                  \
+    (x).count = 1;                                      \
+    (x).anomaly_count = 0;                              \
+    (x).flags = SN_FLAG_NONE;                           \
+    (x).start_time = start_t;                           \
+    (x).end_time = end_t;                               \
+    } while(0)
+
+#define storage_point_is_unset(x) (!(x).count)
+#define storage_point_is_empty(x) (!netdata_double_isnumber((x).sum))
+
 // ------------------------------------------------------------------------
 // function pointers that handle data collection
 struct rrddim_collect_ops {
     // an initialization function to run before starting collection
-    void (*init)(RRDDIM *rd);
+    STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle);
 
     // run this to store each metric into the database
-    void (*store_metric)(RRDDIM *rd, usec_t point_in_time, storage_number number);
+    void (*store_metric)(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number, NETDATA_DOUBLE min_value,
+                         NETDATA_DOUBLE max_value, uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
+
+    // run this to flush / reset the current data collection sequence
+    void (*flush)(STORAGE_COLLECT_HANDLE *collection_handle);
 
     // an finalization function to run after collection is over
     // returns 1 if it's safe to delete the dimension
-    int (*finalize)(RRDDIM *rd);
+    int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle);
 };
 
 // function pointers that handle database queries
 struct rrddim_query_ops {
     // run this before starting a series of next_metric() database queries
-    void (*init)(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
+    void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
 
     // run this to load each metric number from the database
-    storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time);
+    STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
 
     // run this to test if the series of next_metric() database queries is finished
     int (*is_finished)(struct rrddim_query_handle *handle);
@@ -372,29 +430,31 @@ struct rrddim_query_ops {
     void (*finalize)(struct rrddim_query_handle *handle);
 
     // get the timestamp of the last entry of this metric
-    time_t (*latest_time)(RRDDIM *rd);
+    time_t (*latest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
 
     // get the timestamp of the first entry of this metric
-    time_t (*oldest_time)(RRDDIM *rd);
+    time_t (*oldest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
 };
 
+
 // ----------------------------------------------------------------------------
-// volatile state per RRD dimension
-struct rrddim_volatile {
-#ifdef ENABLE_DBENGINE
-    uuid_t *rrdeng_uuid;                 // database engine metric UUID
-    struct pg_cache_page_index *page_index;
-#endif
-#ifdef ENABLE_ACLK
-    int aclk_live_status;
-#endif
-    uuid_t metric_uuid;                 // global UUID for this metric (unique_across hosts)
-    STORAGE_COLLECT_HANDLE* handle;
+// Storage tier data for every dimension
+
+struct rrddim_tier {
+    int tier_grouping;
+    RRD_MEMORY_MODE mode;                           // the memory mode of this tier
+    RRD_BACKFILL backfill;                          // backfilling configuration
+    STORAGE_METRIC_HANDLE *db_metric_handle;        // the metric handle inside the database
+    STORAGE_COLLECT_HANDLE *db_collection_handle;   // the data collection handle
+    STORAGE_POINT virtual_point;
+    time_t next_point_time;
+    usec_t last_collected_ut;
     struct rrddim_collect_ops collect_ops;
     struct rrddim_query_ops query_ops;
-    ml_dimension_t ml_dimension;
 };
 
+extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now);
+
 // ----------------------------------------------------------------------------
 // volatile state per chart
 struct rrdset_volatile {
@@ -402,8 +462,7 @@ struct rrdset_volatile {
     char *old_units;
     char *old_context;
     uuid_t hash_id;
-    struct label *new_labels;
-    struct label_index labels;
+    DICTIONARY *chart_labels;
     bool is_ar_chart;
 };
 
@@ -469,8 +528,6 @@ struct rrdset {
                                                     // since the config always has a higher priority
                                                     // (the user overwrites the name of the charts)
 
-    void *unused_ptr;                               // Unused field (previously it held the config section of the chart)
-
     char *type;                                     // the type of graph RRD_TYPE_* (a category, for determining graphing options)
     char *family;                                   // grouping sets under the same family
     char *title;                                    // title shown to user
@@ -479,6 +536,9 @@ struct rrdset {
     char *context;                                  // the template of this data set
     uint32_t hash_context;                          // the hash of the chart's context
 
+    RRDINSTANCE_ACQUIRED *rrdinstance;              // the rrdinstance of this chart
+    RRDCONTEXT_ACQUIRED *rrdcontext;                // the rrdcontext this chart belongs to
+
     RRDSET_TYPE chart_type;                         // line, area, stacked
 
     int update_every;                               // every how many seconds is this updated?
@@ -503,7 +563,6 @@ struct rrdset {
     RRD_MEMORY_MODE rrd_memory_mode;                // if set to 1, this is memory mapped
 
     char *cache_dir;                                // the directory to store dimensions
-    char cache_filename[FILENAME_MAX+1];            // the filename to store this set
 
     netdata_rwlock_t rrdset_rwlock;                 // protects dimensions linked list
 
@@ -521,7 +580,6 @@ struct rrdset {
     uuid_t *chart_uuid;                             // Store the global GUID for this chart
                                                     // this object.
     struct rrdset_volatile *state;                  // volatile state that is not persistently stored
-    size_t unused[3];
 
     size_t rrddim_page_alignment;                   // keeps metric pages in alignment when using dbengine
 
@@ -546,8 +604,8 @@ struct rrdset {
     // ------------------------------------------------------------------------
     // local variables
 
-    calculated_number green;                        // green threshold for this chart
-    calculated_number red;                          // red threshold for this chart
+    NETDATA_DOUBLE green;                        // green threshold for this chart
+    NETDATA_DOUBLE red;                          // red threshold for this chart
 
     avl_tree_lock rrdvar_root_index;                // RRDVAR index for this chart
     RRDSETVAR *variables;                           // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs)
@@ -557,15 +615,13 @@ struct rrdset {
     // members for checking the data when loading from disk
 
     unsigned long memsize;                          // how much mem we have allocated for this (without dimensions)
-
-    char magic[sizeof(RRDSET_MAGIC) + 1];           // our magic
+    void *st_on_file;                               // compatibility with V019 RRDSET files
 
     // ------------------------------------------------------------------------
     // the dimensions
 
     avl_tree_lock dimensions_index;                 // the root of the dimensions index
     RRDDIM *dimensions;                             // the actual data for every dimension
-
 };
 
 #define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock))
@@ -583,6 +639,12 @@ struct rrdset {
     for((st) = (host)->rrdset_root, rrdhost_check_wrlock(host); st ; (st) = (st)->next)
 
 
+extern void rrdset_memory_file_save(RRDSET *st);
+extern void rrdset_memory_file_free(RRDSET *st);
+extern void rrdset_memory_file_update(RRDSET *st);
+extern const char *rrdset_cache_filename(RRDSET *st);
+extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode);
+
 // ----------------------------------------------------------------------------
 // RRDHOST flags
 // use this for configuration flags, not for state control
@@ -590,14 +652,17 @@ struct rrdset {
 // and may lead to missing information.
 
 typedef enum rrdhost_flags {
-    RRDHOST_FLAG_ORPHAN                 = 1 << 0, // this host is orphan (not receiving data)
-    RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts
-    RRDHOST_FLAG_DELETE_ORPHAN_HOST     = 1 << 2, // delete the entire host when orphan
-    RRDHOST_FLAG_EXPORTING_SEND           = 1 << 3, // send it to external databases
-    RRDHOST_FLAG_EXPORTING_DONT_SEND      = 1 << 4, // don't send it to external databases
-    RRDHOST_FLAG_ARCHIVED               = 1 << 5, // The host is archived, no collected charts yet
-    RRDHOST_FLAG_MULTIHOST              = 1 << 6, // Host belongs to localhost/megadb
-    RRDHOST_FLAG_PENDING_FOREACH_ALARMS  = 1 << 7, // contains dims with uninitialized foreach alarms
+    RRDHOST_FLAG_ORPHAN                 = (1 << 0), // this host is orphan (not receiving data)
+    RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts
+    RRDHOST_FLAG_DELETE_ORPHAN_HOST     = (1 << 2), // delete the entire host when orphan
+    RRDHOST_FLAG_EXPORTING_SEND         = (1 << 3), // send it to external databases
+    RRDHOST_FLAG_EXPORTING_DONT_SEND    = (1 << 4), // don't send it to external databases
+    RRDHOST_FLAG_ARCHIVED               = (1 << 5), // The host is archived, no collected charts yet
+    RRDHOST_FLAG_MULTIHOST              = (1 << 6), // Host belongs to localhost/megadb
+    RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms
+    RRDHOST_FLAG_STREAM_LABELS_UPDATE   = (1 << 8),
+    RRDHOST_FLAG_STREAM_LABELS_STOP     = (1 << 9),
+    RRDHOST_FLAG_ACLK_STREAM_CONTEXTS   = (1 << 10), // when set, we should send ACLK stream context updates
 } RRDHOST_FLAGS;
 
 #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))
@@ -629,6 +694,7 @@ struct alarm_entry {
 
     char *chart;
     uint32_t hash_chart;
+    char *chart_context;
 
     char *family;
 
@@ -646,8 +712,8 @@ struct alarm_entry {
     char *units;
     char *info;
 
-    calculated_number old_value;
-    calculated_number new_value;
+    NETDATA_DOUBLE old_value;
+    NETDATA_DOUBLE new_value;
 
     char *old_value_string;
     char *new_value_string;
@@ -740,10 +806,6 @@ struct rrdhost {
     const char *tags;                               // tags for this host
     const char *timezone;                           // the timezone of the host
 
-#ifdef ENABLE_ACLK
-    long    deleted_charts_count;
-#endif
-
     const char *abbrev_timezone;                    // the abbriviated timezone of the host
     int32_t utc_offset;                             // the offset in seconds from utc
 
@@ -778,7 +840,7 @@ struct rrdhost {
     netdata_thread_t rrdpush_sender_thread;         // the sender thread
     void *dbsync_worker;
 
-    volatile unsigned int rrdpush_sender_connected; // 1 when the sender is ready to push metrics
+    bool rrdpush_sender_connected;                  // 1 when the sender is ready to push metrics
     int rrdpush_sender_socket;                      // the fd of the socket to the remote host, or -1
 
     volatile unsigned int rrdpush_sender_error_shown; // 1 when we have logged a communication error
@@ -859,7 +921,7 @@ struct rrdhost {
 
     // ------------------------------------------------------------------------
     // Support for host-level labels
-    struct label_index labels;
+    DICTIONARY *host_labels;
 
     // ------------------------------------------------------------------------
     // indexes
@@ -870,9 +932,11 @@ struct rrdhost {
     avl_tree_lock rrdfamily_root_index;             // the host's chart families index
     avl_tree_lock rrdvar_root_index;                // the host's chart variables index
 
-#ifdef ENABLE_DBENGINE
-    struct rrdengine_instance *rrdeng_ctx;          // DB engine instance for this host
-#endif
+    STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS];  // the database instances of the storage tiers
+
+    RRDCONTEXTS *rrdctx_queue;
+    RRDCONTEXTS *rrdctx;
+
     uuid_t  host_uuid;                              // Global GUID for this host
     uuid_t  *node_id;                               // Cloud node_id
 
@@ -916,6 +980,10 @@ extern netdata_rwlock_t rrd_rwlock;
 
 // ----------------------------------------------------------------------------
 
+extern bool is_storage_engine_shared(STORAGE_INSTANCE *engine);
+
+// ----------------------------------------------------------------------------
+
 extern size_t rrd_hosts_available;
 extern time_t rrdhost_free_orphan_time;
 
@@ -944,6 +1012,7 @@ extern RRDHOST *rrdhost_find_or_create(
         , char *rrdpush_api_key
         , char *rrdpush_send_charts_matching
         , struct rrdhost_system_info *system_info
+        , bool is_archived
 );
 
 extern void rrdhost_update(RRDHOST *host
@@ -1027,7 +1096,7 @@ extern void rrdhost_cleanup_all(void);
 
 extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host);
 extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info);
-extern void rrdhost_free(RRDHOST *host);
+extern void rrdhost_free(RRDHOST *host, bool force);
 extern void rrdhost_save_charts(RRDHOST *host);
 extern void rrdhost_delete_charts(RRDHOST *host);
 extern void rrd_cleanup_obsolete_charts();
@@ -1083,28 +1152,49 @@ extern void rrdset_isnot_obsolete(RRDSET *st);
 #define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
 #define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
 
-// get the total duration in seconds of the round robin database
-#define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every ))
-
 // get the timestamp of the last entry in the round robin database
-static inline time_t rrdset_last_entry_t_nolock(RRDSET *st)
-{
-    if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-        RRDDIM *rd;
-        time_t last_entry_t  = 0;
+static inline time_t rrddim_last_entry_t(RRDDIM *rd) {
+    time_t latest = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
 
-        rrddim_foreach_read(rd, st) {
-            last_entry_t = MAX(last_entry_t, rd->state->query_ops.latest_time(rd));
-        }
+    for(int tier = 1; tier < storage_tiers ;tier++) {
+        if(unlikely(!rd->tiers[tier])) continue;
 
-        return last_entry_t;
-    } else {
-        return (time_t)st->last_updated.tv_sec;
+        time_t t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
+        if(t > latest)
+            latest = t;
     }
+
+    return latest;
 }
 
-static inline time_t rrdset_last_entry_t(RRDSET *st)
-{
+static inline time_t rrddim_first_entry_t(RRDDIM *rd) {
+    time_t oldest = 0;
+
+    for(int tier = 0; tier < storage_tiers ;tier++) {
+        if(unlikely(!rd->tiers[tier])) continue;
+
+        time_t t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
+        if(t != 0 && (oldest == 0 || t < oldest))
+            oldest = t;
+    }
+
+    return oldest;
+}
+
+// get the timestamp of the last entry in the round robin database
+static inline time_t rrdset_last_entry_t_nolock(RRDSET *st) {
+    RRDDIM *rd;
+    time_t last_entry_t  = 0;
+
+    rrddim_foreach_read(rd, st) {
+        time_t t = rrddim_last_entry_t(rd);
+        if(t > last_entry_t) last_entry_t = t;
+    }
+
+    return last_entry_t;
+}
+
+static inline time_t rrdset_last_entry_t(RRDSET *st) {
     time_t last_entry_t;
 
     netdata_rwlock_rdlock(&st->rrdset_rwlock);
@@ -1115,24 +1205,18 @@ static inline time_t rrdset_last_entry_t(RRDSET *st)
 }
 
 // get the timestamp of first entry in the round robin database
-static inline time_t rrdset_first_entry_t_nolock(RRDSET *st)
-{
-    if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-        RRDDIM *rd;
-        time_t first_entry_t = LONG_MAX;
-
-        rrddim_foreach_read(rd, st) {
-            first_entry_t =
-                MIN(first_entry_t,
-                    rd->state->query_ops.oldest_time(rd) > st->update_every ?
-                        rd->state->query_ops.oldest_time(rd) - st->update_every : 0);
-        }
-
-        if (unlikely(LONG_MAX == first_entry_t)) return 0;
-        return first_entry_t;
-    } else {
-        return (time_t)(rrdset_last_entry_t_nolock(st) - rrdset_duration(st));
+static inline time_t rrdset_first_entry_t_nolock(RRDSET *st) {
+    RRDDIM *rd;
+    time_t first_entry_t = LONG_MAX;
+
+    rrddim_foreach_read(rd, st) {
+        time_t t = rrddim_first_entry_t(rd);
+        if(t < first_entry_t)
+            first_entry_t = t;
     }
+
+    if (unlikely(LONG_MAX == first_entry_t)) return 0;
+    return first_entry_t;
 }
 
 static inline time_t rrdset_first_entry_t(RRDSET *st)
@@ -1146,107 +1230,8 @@ static inline time_t rrdset_first_entry_t(RRDSET *st)
     return first_entry_t;
 }
 
-// get the timestamp of the last entry in the round robin database
-static inline time_t rrddim_last_entry_t(RRDDIM *rd) {
-    if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-        return rd->state->query_ops.latest_time(rd);
-    return (time_t)rd->rrdset->last_updated.tv_sec;
-}
-
-static inline time_t rrddim_first_entry_t(RRDDIM *rd) {
-    if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-        return rd->state->query_ops.oldest_time(rd);
-    return (time_t)(rd->rrdset->last_updated.tv_sec - rrdset_duration(rd->rrdset));
-}
-
 time_t rrdhost_last_entry_t(RRDHOST *h);
 
-// get the last slot updated in the round robin database
-#define rrdset_last_slot(st) ((size_t)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
-
-// get the first / oldest slot updated in the round robin database
-// #define rrdset_first_slot(st) ((size_t)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) ))
-
-// return the slot that has the oldest value
-
-static inline size_t rrdset_first_slot(RRDSET *st) {
-    if(st->counter >= (size_t)st->entries) {
-        // the database has been rotated at least once
-        // the oldest entry is the one that will be next
-        // overwritten by data collection
-        return (size_t)st->current_entry;
-    }
-
-    // we do not have rotated the db yet
-    // so 0 is the first entry
-    return 0;
-}
-
-// get the slot of the round robin database, for the given timestamp (t)
-// it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database
-// only valid when not using dbengine
-static inline size_t rrdset_time2slot(RRDSET *st, time_t t) {
-    size_t ret = 0;
-    time_t last_entry_t = rrdset_last_entry_t_nolock(st);
-    time_t first_entry_t = rrdset_first_entry_t_nolock(st);
-
-    if(t >= last_entry_t) {
-        // the requested time is after the last entry we have
-        ret = rrdset_last_slot(st);
-    }
-    else {
-        if(t <= first_entry_t) {
-            // the requested time is before the first entry we have
-            ret = rrdset_first_slot(st);
-        }
-        else {
-            if(rrdset_last_slot(st) >= ((last_entry_t - t) / (size_t)(st->update_every)))
-                ret = rrdset_last_slot(st) - ((last_entry_t - t) / (size_t)(st->update_every));
-            else
-                ret = rrdset_last_slot(st) - ((last_entry_t - t) / (size_t)(st->update_every)) + (unsigned long)st->entries;
-        }
-    }
-
-    if(unlikely(ret >= (size_t)st->entries)) {
-        error("INTERNAL ERROR: rrdset_time2slot() on %s returns values outside entries", st->name);
-        ret = (size_t)(st->entries - 1);
-    }
-
-    return ret;
-}
-
-// get the timestamp of a specific slot in the round robin database
-// only valid when not using dbengine
-static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
-    time_t ret;
-    time_t last_entry_t = rrdset_last_entry_t_nolock(st);
-    time_t first_entry_t = rrdset_first_entry_t_nolock(st);
-
-    if(slot >= (size_t)st->entries) {
-        error("INTERNAL ERROR: caller of rrdset_slot2time() gives invalid slot %zu", slot);
-        slot = (size_t)st->entries - 1;
-    }
-
-    if(slot > rrdset_last_slot(st)) {
-        ret = last_entry_t - (size_t)st->update_every * (rrdset_last_slot(st) - slot + (size_t)st->entries);
-    }
-    else {
-        ret = last_entry_t - (size_t)st->update_every;
-    }
-
-    if(unlikely(ret < first_entry_t)) {
-        error("INTERNAL ERROR: rrdset_slot2time() on %s returns time too far in the past", st->name);
-        ret = first_entry_t;
-    }
-
-    if(unlikely(ret > last_entry_t)) {
-        error("INTERNAL ERROR: rrdset_slot2time() on %s returns time into the future", st->name);
-        ret = last_entry_t;
-    }
-
-    return ret;
-}
-
 // ----------------------------------------------------------------------------
 // RRD DIMENSION functions
 
@@ -1281,7 +1266,7 @@ extern void rrddim_isnot_obsolete(RRDSET *st, RRDDIM *rd);
 
 extern collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value);
 extern collected_number rrddim_set(RRDSET *st, const char *id, collected_number value);
-#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+#ifdef ENABLE_ACLK
 extern time_t calc_dimension_liveness(RRDDIM *rd, time_t now);
 #endif
 extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
@@ -1319,8 +1304,7 @@ extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st);
 extern void rrdset_free(RRDSET *st);
 extern void rrdset_reset(RRDSET *st);
 extern void rrdset_save(RRDSET *st);
-#define rrdset_delete(st) rrdset_delete_custom(st, 0)
-extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
+extern void rrdset_delete_files(RRDSET *st);
 extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
 
 extern RRDHOST *rrdhost_create(
@@ -1328,7 +1312,7 @@ extern RRDHOST *rrdhost_create(
     const char *abbrev_timezone, int32_t utc_offset,const char *tags, const char *program_name, const char *program_version,
     int update_every, long entries, RRD_MEMORY_MODE memory_mode, unsigned int health_enabled, unsigned int rrdpush_enabled,
     char *rrdpush_destination, char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info,
-    int is_localhost); //TODO: Remove , int is_archived);
+    int is_localhost, bool is_archived);
 
 #endif /* NETDATA_RRD_INTERNALS */
 
@@ -1337,6 +1321,8 @@ extern void set_host_properties(
     const char *guid, const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
     const char *program_name, const char *program_version);
 
+extern int get_tier_grouping(int tier);
+
 // ----------------------------------------------------------------------------
 // RRD DB engine declarations
 
@@ -1344,9 +1330,15 @@ extern void set_host_properties(
 #include "database/engine/rrdengineapi.h"
 #endif
 #include "sqlite/sqlite_functions.h"
+#include "sqlite/sqlite_context.h"
 #include "sqlite/sqlite_aclk.h"
 #include "sqlite/sqlite_aclk_chart.h"
 #include "sqlite/sqlite_aclk_alert.h"
 #include "sqlite/sqlite_aclk_node.h"
 #include "sqlite/sqlite_health.h"
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* NETDATA_RRD_H */