Adding upstream version 1.36.0.upstream/1.36.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 312 insertions, 162 deletions

diff --git a/database/rrddim.c b/database/rrddim.c
index e488d8b0b..90165a253 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -73,6 +73,7 @@ inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm)
     rd->exposed = 0;
     rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
     rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+    rrdcontext_updated_rrddim_algorithm(rd);
     return 1;
 }
 
@@ -85,6 +86,7 @@ inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip
     rd->exposed = 0;
     rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
     rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+    rrdcontext_updated_rrddim_multiplier(rd);
     return 1;
 }
 
@@ -97,6 +99,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
     rd->exposed = 0;
     rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);
     rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
+    rrdcontext_updated_rrddim_divisor(rd);
     return 1;
 }
 
@@ -139,12 +142,12 @@ void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) {
 //   0 : Dimension is live
 //   last collected time : Dimension is not live
 
-#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+#ifdef ENABLE_ACLK
 time_t calc_dimension_liveness(RRDDIM *rd, time_t now)
 {
     time_t last_updated = rd->last_collected_time.tv_sec;
     int live;
-    if (rd->state->aclk_live_status == 1)
+    if (rd->aclk_live_status == 1)
         live =
             ((now - last_updated) <
              MIN(rrdset_free_obsolete_time, RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER * rd->update_every));
@@ -168,9 +171,16 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
         rc += rrddim_set_algorithm(st, rd, algorithm);
         rc += rrddim_set_multiplier(st, rd, multiplier);
         rc += rrddim_set_divisor(st, rd, divisor);
+
         if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
-            store_active_dimension(&rd->state->metric_uuid);
-            rd->state->collect_ops.init(rd);
+            store_active_dimension(&rd->metric_uuid);
+
+            for(int tier = 0; tier < storage_tiers ;tier++) {
+                if (rd->tiers[tier])
+                    rd->tiers[tier]->db_collection_handle =
+                        rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
+            }
+
             rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
             rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT);
             rrddimvar_create(rd, RRDVAR_TYPE_COLLECTED, NULL, "_raw", &rd->last_collected_value, RRDVAR_OPTION_DEFAULT);
@@ -180,129 +190,34 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
             rrdset_flag_set(st, RRDSET_FLAG_PENDING_FOREACH_ALARMS);
             rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_FOREACH_ALARMS);
         }
+
         if (unlikely(rc)) {
             debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
-            (void)sql_store_dimension(&rd->state->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
+            (void)sql_store_dimension(&rd->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
                                       rd->algorithm);
-#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+#ifdef ENABLE_ACLK
             queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec()));
 #endif
             rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
             rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
         }
         rrdset_unlock(st);
+        rrdcontext_updated_rrddim(rd);
         return rd;
     }
 
     rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
     rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
 
-    char filename[FILENAME_MAX + 1];
-    char fullfilename[FILENAME_MAX + 1];
-
-    unsigned long size = sizeof(RRDDIM) + (st->entries * sizeof(storage_number));
-
-    debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
-
-    rrdset_strncpyz_name(filename, id, FILENAME_MAX);
-    snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
-
-    if(memory_mode == RRD_MEMORY_MODE_SAVE || memory_mode == RRD_MEMORY_MODE_MAP ||
-       memory_mode == RRD_MEMORY_MODE_RAM) {
-        rd = (RRDDIM *)netdata_mmap(
-            (memory_mode == RRD_MEMORY_MODE_RAM) ? NULL : fullfilename,
-            size,
-            ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE),
-            1);
-
-        if(likely(rd)) {
-            // we have a file mapped for rd
-
-            memset(&rd->avl, 0, sizeof(avl_t));
-            rd->id = NULL;
-            rd->name = NULL;
-            rd->cache_filename = NULL;
-            rd->variables = NULL;
-            rd->next = NULL;
-            rd->rrdset = NULL;
-            rd->exposed = 0;
-
-            struct timeval now;
-            now_realtime_timeval(&now);
-
-            if(memory_mode == RRD_MEMORY_MODE_RAM) {
-                memset(rd, 0, size);
-            }
-            else {
-                int reset = 0;
-
-                if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) {
-                    info("Initializing file %s.", fullfilename);
-                    memset(rd, 0, size);
-                    reset = 1;
-                }
-                else if(rd->memsize != size) {
-                    error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd->memsize);
-                    memset(rd, 0, size);
-                    reset = 1;
-                }
-                else if(rd->update_every != st->update_every) {
-                    error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd->update_every);
-                    memset(rd, 0, size);
-                    reset = 1;
-                }
-                else if(dt_usec(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * USEC_PER_SEC)) {
-                    info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd->last_collected_time) / USEC_PER_SEC, rd->entries * rd->update_every);
-                    memset(rd, 0, size);
-                    reset = 1;
-                }
-
-                if(!reset) {
-                    if(rd->algorithm != algorithm) {
-                        info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
-                              fullfilename, algorithm, rrd_algorithm_name(algorithm), rd->algorithm, rrd_algorithm_name(rd->algorithm));
-                    }
-
-                    if(rd->multiplier != multiplier) {
-                        info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, multiplier, rd->multiplier);
-                    }
-
-                    if(rd->divisor != divisor) {
-                        info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, divisor, rd->divisor);
-                    }
-                }
-            }
-
-            // make sure we have the right memory mode
-            // even if we cleared the memory
-            rd->rrd_memory_mode = memory_mode;
-        }
-    }
-
-    if(unlikely(!rd)) {
-        // if we didn't manage to get a mmap'd dimension, just create one
-        rd = callocz(1, size);
-        if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
-        else
-            rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
-    }
-    rd->memsize = size;
-
-    strcpy(rd->magic, RRDDIMENSION_MAGIC);
-
+    rd = callocz(1, sizeof(RRDDIM));
     rd->id = strdupz(id);
     rd->hash = simple_hash(rd->id);
 
-    rd->cache_filename = strdupz(fullfilename);
-
     rd->name = (name && *name)?strdupz(name):strdupz(rd->id);
     rd->hash_name = simple_hash(rd->name);
 
     rd->algorithm = algorithm;
-
     rd->multiplier = multiplier;
-
     rd->divisor = divisor;
     if(!rd->divisor) rd->divisor = 1;
 
@@ -311,40 +226,86 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
 
     if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))
         rd->collections_counter = 1;
-    else
-        rd->collections_counter = 0;
-
-    rd->updated = 0;
-    rd->flags = 0x00000000;
-
-    rd->calculated_value = 0;
-    rd->last_calculated_value = 0;
-    rd->collected_value = 0;
-    rd->last_collected_value = 0;
-    rd->collected_value_max = 0;
-    rd->collected_volume = 0;
-    rd->stored_volume = 0;
-    rd->last_stored_value = 0;
-    rd->last_collected_time.tv_sec = 0;
-    rd->last_collected_time.tv_usec = 0;
+
     rd->rrdset = st;
-    rd->state = callocz(1, sizeof(*rd->state));
+
+    if(memory_mode == RRD_MEMORY_MODE_MAP || memory_mode == RRD_MEMORY_MODE_SAVE) {
+        if(!rrddim_memory_load_or_create_map_save(st, rd, memory_mode)) {
+            info("Failed to use memory mode %s for chart '%s', dimension '%s', falling back to ram", (memory_mode == RRD_MEMORY_MODE_MAP)?"map":"save", st->name, rd->name);
+            memory_mode = RRD_MEMORY_MODE_RAM;
+        }
+    }
+
+    if(memory_mode == RRD_MEMORY_MODE_RAM) {
+        size_t entries = st->entries;
+        if(!entries) entries = 5;
+
+        rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1);
+        if(!rd->db) {
+            info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", st->name, rd->name);
+            memory_mode = RRD_MEMORY_MODE_ALLOC;
+        }
+        else rd->memsize = entries * sizeof(storage_number);
+    }
+
+    if(memory_mode == RRD_MEMORY_MODE_ALLOC || memory_mode == RRD_MEMORY_MODE_NONE) {
+        size_t entries = st->entries;
+        if(entries < 5) entries = 5;
+
+        rd->db = callocz(entries, sizeof(storage_number));
+        rd->memsize = entries * sizeof(storage_number);
+    }
+
+    rd->rrd_memory_mode = memory_mode;
+
 #ifdef ENABLE_ACLK
-    rd->state->aclk_live_status = -1;
+    rd->aclk_live_status = -1;
 #endif
-    (void) find_dimension_uuid(st, rd, &(rd->state->metric_uuid));
+    (void) find_dimension_uuid(st, rd, &(rd->metric_uuid));
+
+    // initialize the db tiers
+    {
+        size_t initialized = 0;
+        RRD_MEMORY_MODE wanted_mode = memory_mode;
+        for(int tier = 0; tier < storage_tiers ; tier++, wanted_mode = RRD_MEMORY_MODE_DBENGINE) {
+            STORAGE_ENGINE *eng = storage_engine_get(wanted_mode);
+            if(!eng) continue;
+
+            rd->tiers[tier] = callocz(1, sizeof(struct rrddim_tier));
+            rd->tiers[tier]->tier_grouping = get_tier_grouping(tier);
+            rd->tiers[tier]->mode = eng->id;
+            rd->tiers[tier]->collect_ops = eng->api.collect_ops;
+            rd->tiers[tier]->query_ops = eng->api.query_ops;
+            rd->tiers[tier]->db_metric_handle = eng->api.init(rd, host->storage_instance[tier]);
+            storage_point_unset(rd->tiers[tier]->virtual_point);
+            initialized++;
+
+            // internal_error(true, "TIER GROUPING of chart '%s', dimension '%s' for tier %d is set to %d", rd->rrdset->name, rd->name, tier, rd->tiers[tier]->tier_grouping);
+        }
 
-    STORAGE_ENGINE* eng = storage_engine_get(memory_mode);
-    rd->state->collect_ops = eng->api.collect_ops;
-    rd->state->query_ops = eng->api.query_ops;
+        if(!initialized)
+            error("Failed to initialize all db tiers for chart '%s', dimension '%s", st->name, rd->name);
 
-#ifdef ENABLE_DBENGINE
-    if(memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-        rrdeng_metric_init(rd);
+        if(!rd->tiers[0])
+            error("Failed to initialize the first db tier for chart '%s', dimension '%s", st->name, rd->name);
     }
-#endif
-    store_active_dimension(&rd->state->metric_uuid);
-    rd->state->collect_ops.init(rd);
+
+    store_active_dimension(&rd->metric_uuid);
+
+    // initialize data collection for all tiers
+    {
+        size_t initialized = 0;
+        for (int tier = 0; tier < storage_tiers; tier++) {
+            if (rd->tiers[tier]) {
+                rd->tiers[tier]->db_collection_handle = rd->tiers[tier]->collect_ops.init(rd->tiers[tier]->db_metric_handle);
+                initialized++;
+            }
+        }
+
+        if(!initialized)
+            error("Failed to initialize data collection for all db tiers for chart '%s', dimension '%s", st->name, rd->name);
+    }
+
     // append this dimension
     if(!st->dimensions)
         st->dimensions = rd;
@@ -387,6 +348,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
     ml_new_dimension(rd);
 
     rrdset_unlock(st);
+    rrdcontext_updated_rrddim(rd);
     return(rd);
 }
 
@@ -395,15 +357,28 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
 
 void rrddim_free(RRDSET *st, RRDDIM *rd)
 {
+    rrdcontext_removed_rrddim(rd);
     ml_delete_dimension(rd);
     
     debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name);
 
     if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
-        uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
-        if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+
+        size_t tiers_available = 0, tiers_said_yes = 0;
+        for(int tier = 0; tier < storage_tiers ;tier++) {
+            if(rd->tiers[tier]) {
+                tiers_available++;
+
+                if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle))
+                    tiers_said_yes++;
+
+                rd->tiers[tier]->db_collection_handle = NULL;
+            }
+        }
+
+        if (tiers_available == tiers_said_yes && tiers_said_yes && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
             /* This metric has no data and no references */
-            delete_dimension_uuid(&rd->state->metric_uuid);
+            delete_dimension_uuid(&rd->metric_uuid);
         }
     }
 
@@ -427,35 +402,35 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
         error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id);
 
     // free(rd->annotations);
-//#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+//#ifdef ENABLE_ACLK
 //    if (!netdata_exit)
 //        aclk_send_dimension_update(rd);
 //#endif
 
-    RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;
-    switch(rrd_memory_mode) {
-        case RRD_MEMORY_MODE_SAVE:
-        case RRD_MEMORY_MODE_MAP:
-        case RRD_MEMORY_MODE_RAM:
-            debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
-            freez((void *)rd->id);
-            freez((void *)rd->name);
-            freez(rd->cache_filename);
-            freez(rd->state);
-            munmap(rd, rd->memsize);
-            break;
-
-        case RRD_MEMORY_MODE_ALLOC:
-        case RRD_MEMORY_MODE_NONE:
-        case RRD_MEMORY_MODE_DBENGINE:
-            debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
-            freez((void *)rd->id);
-            freez((void *)rd->name);
-            freez(rd->cache_filename);
-            freez(rd->state);
-            freez(rd);
-            break;
+    // this will free MEMORY_MODE_SAVE and MEMORY_MODE_MAP structures
+    rrddim_memory_file_free(rd);
+
+    for(int tier = 0; tier < storage_tiers ;tier++) {
+        if(!rd->tiers[tier]) continue;
+
+        STORAGE_ENGINE* eng = storage_engine_get(rd->tiers[tier]->mode);
+        if(eng)
+            eng->api.free(rd->tiers[tier]->db_metric_handle);
+
+        freez(rd->tiers[tier]);
+        rd->tiers[tier] = NULL;
+    }
+
+    if(rd->db) {
+        if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)
+            munmap(rd->db, rd->memsize);
+        else
+            freez(rd->db);
     }
+
+    freez((void *)rd->id);
+    freez((void *)rd->name);
+    freez(rd);
 }
 
 
@@ -473,10 +448,11 @@ int rrddim_hide(RRDSET *st, const char *id) {
         return 1;
     }
     if (!rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
-        (void)sql_set_dimension_option(&rd->state->metric_uuid, "hidden");
+        (void)sql_set_dimension_option(&rd->metric_uuid, "hidden");
 
     rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
     rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
+    rrdcontext_updated_rrddim_flags(rd);
     return 0;
 }
 
@@ -490,10 +466,11 @@ int rrddim_unhide(RRDSET *st, const char *id) {
         return 1;
     }
     if (rrddim_flag_check(rd, RRDDIM_FLAG_META_HIDDEN))
-        (void)sql_set_dimension_option(&rd->state->metric_uuid, NULL);
+        (void)sql_set_dimension_option(&rd->metric_uuid, NULL);
 
     rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
     rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
+    rrdcontext_updated_rrddim_flags(rd);
     return 0;
 }
 
@@ -506,12 +483,14 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
     }
     rrddim_flag_set(rd, RRDDIM_FLAG_OBSOLETE);
     rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
+    rrdcontext_updated_rrddim_flags(rd);
 }
 
 inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
     debug(D_RRD_CALLS, "rrddim_isnot_obsolete() for chart %s, dimension %s", st->name, rd->name);
 
     rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
+    rrdcontext_updated_rrddim_flags(rd);
 }
 
 // ----------------------------------------------------------------------------
@@ -520,6 +499,8 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
 inline collected_number rrddim_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM *rd, collected_number value) {
     debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
 
+    rrdcontext_collected_rrddim(rd);
+
     now_realtime_timeval(&rd->last_collected_time);
     rd->collected_value = value;
     rd->updated = 1;
@@ -529,7 +510,7 @@ inline collected_number rrddim_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM
     collected_number v = (value >= 0) ? value : -value;
     if(unlikely(v > rd->collected_value_max)) rd->collected_value_max = v;
 
-    // fprintf(stderr, "%s.%s %llu " COLLECTED_NUMBER_FORMAT " dt %0.6f" " rate " CALCULATED_NUMBER_FORMAT "\n", st->name, rd->name, st->usec_since_last_update, value, (float)((double)st->usec_since_last_update / (double)1000000), (calculated_number)((value - rd->last_collected_value) * (calculated_number)rd->multiplier / (calculated_number)rd->divisor * 1000000.0 / (calculated_number)st->usec_since_last_update));
+    // fprintf(stderr, "%s.%s %llu " COLLECTED_NUMBER_FORMAT " dt %0.6f" " rate " NETDATA_DOUBLE_FORMAT "\n", st->name, rd->name, st->usec_since_last_update, value, (float)((double)st->usec_since_last_update / (double)1000000), (NETDATA_DOUBLE)((value - rd->last_collected_value) * (NETDATA_DOUBLE)rd->multiplier / (NETDATA_DOUBLE)rd->divisor * 1000000.0 / (NETDATA_DOUBLE)st->usec_since_last_update));
 
     return rd->last_collected_value;
 }
@@ -544,3 +525,172 @@ collected_number rrddim_set(RRDSET *st, const char *id, collected_number value)
 
     return rrddim_set_by_pointer(st, rd, value);
 }
+
+
+// ----------------------------------------------------------------------------
+// compatibility layer for RRDDIM files v019
+
+#define RRDDIMENSION_MAGIC_V019  "NETDATA RRD DIMENSION FILE V019"
+
+struct avl_element_v019 {
+    void *avl_link[2];
+    signed char avl_balance;
+};
+
+struct rrddim_map_save_v019 {
+    struct avl_element_v019 avl;                    // ignored
+    void *id;                                       // ignored
+    void *name;                                     // ignored
+    uint32_t algorithm;                             // print warning on mismatch - update on load
+    uint32_t rrd_memory_mode;                       // ignored
+    long long multiplier;                           // print warning on mismatch - update on load
+    long long divisor;                              // print warning on mismatch - update on load
+    uint32_t flags;                                 // ignored
+    uint32_t hash;                                  // ignored
+    uint32_t hash_name;                             // ignored
+    void *cache_filename;                           // ignored - we use it to keep the filename to save back
+    size_t collections_counter;                     // ignored
+    void *state;                                    // ignored
+    size_t unused[8];                               // ignored
+    long long collected_value_max;                  // ignored
+    unsigned int updated:1;                         // ignored
+    unsigned int exposed:1;                         // ignored
+    struct timeval last_collected_time;             // check to reset all - ignored after load
+    long double calculated_value;                   // ignored
+    long double last_calculated_value;              // ignored
+    long double last_stored_value;                  // ignored
+    long long collected_value;                      // ignored
+    long long last_collected_value;                 // ignored
+    long double collected_volume;                   // ignored
+    long double stored_volume;                      // ignored
+    void *next;                                     // ignored
+    void *rrdset;                                   // ignored
+    long entries;                                   // check to reset all - update on load
+    int update_every;                               // check to reset all - update on load
+    size_t memsize;                                 // check to reset all - update on load
+    char magic[sizeof(RRDDIMENSION_MAGIC_V019) + 1];// check to reset all - update on load
+    void *variables;                                // ignored
+    storage_number values[];                        // the array of values
+};
+
+size_t rrddim_memory_file_header_size(void) {
+    return sizeof(struct rrddim_map_save_v019);
+}
+
+void rrddim_memory_file_update(RRDDIM *rd) {
+    if(!rd->rd_on_file) return;
+    struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
+
+    rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;
+    rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec;
+}
+
+void rrddim_memory_file_free(RRDDIM *rd) {
+    if(!rd->rd_on_file) return;
+
+    // needed for memory mode map, to save the latest state
+    rrddim_memory_file_update(rd);
+
+    struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
+    freez(rd_on_file->cache_filename);
+    munmap(rd_on_file, rd_on_file->memsize);
+
+    // remove the pointers from the RRDDIM
+    rd->rd_on_file = NULL;
+    rd->db = NULL;
+}
+
+const char *rrddim_cache_filename(RRDDIM *rd) {
+    if(!rd->rd_on_file) return NULL;
+    struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
+    return rd_on_file->cache_filename;
+}
+
+void rrddim_memory_file_save(RRDDIM *rd) {
+    if(!rd->rd_on_file) return;
+
+    rrddim_memory_file_update(rd);
+
+    struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
+    if(rd_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;
+
+    memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);
+}
+
+bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode) {
+    if(memory_mode != RRD_MEMORY_MODE_SAVE && memory_mode != RRD_MEMORY_MODE_MAP)
+        return false;
+
+    struct rrddim_map_save_v019 *rd_on_file = NULL;
+
+    unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->entries * sizeof(storage_number));
+
+    char filename[FILENAME_MAX + 1];
+    char fullfilename[FILENAME_MAX + 1];
+    rrdset_strncpyz_name(filename, rd->id, FILENAME_MAX);
+    snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
+
+    rd_on_file = (struct rrddim_map_save_v019 *)netdata_mmap(fullfilename, size,
+        ((memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1);
+
+    if(unlikely(!rd_on_file)) return false;
+
+    struct timeval now;
+    now_realtime_timeval(&now);
+
+    int reset = 0;
+    rd_on_file->magic[sizeof(RRDDIMENSION_MAGIC_V019)] = '\0';
+    if(strcmp(rd_on_file->magic, RRDDIMENSION_MAGIC_V019) != 0) {
+        info("Initializing file %s.", fullfilename);
+        memset(rd_on_file, 0, size);
+        reset = 1;
+    }
+    else if(rd_on_file->memsize != size) {
+        error("File %s does not have the desired size, expected %lu but found %lu. Clearing it.", fullfilename, size, rd_on_file->memsize);
+        memset(rd_on_file, 0, size);
+        reset = 1;
+    }
+    else if(rd_on_file->update_every != st->update_every) {
+        error("File %s does not have the same update frequency, expected %d but found %d. Clearing it.", fullfilename, st->update_every, rd_on_file->update_every);
+        memset(rd_on_file, 0, size);
+        reset = 1;
+    }
+    else if(dt_usec(&now, &rd_on_file->last_collected_time) > (rd_on_file->entries * rd_on_file->update_every * USEC_PER_SEC)) {
+        info("File %s is too old (last collected %llu seconds ago, but the database is %ld seconds). Clearing it.", fullfilename, dt_usec(&now, &rd_on_file->last_collected_time) / USEC_PER_SEC, rd_on_file->entries * rd_on_file->update_every);
+        memset(rd_on_file, 0, size);
+        reset = 1;
+    }
+
+    if(!reset) {
+        if(rd_on_file->algorithm != rd->algorithm)
+            info("File %s does not have the expected algorithm (expected %u '%s', found %u '%s'). Previous values may be wrong.",
+                 fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));
+
+        if(rd_on_file->multiplier != rd->multiplier)
+            info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->multiplier, rd_on_file->multiplier);
+
+        if(rd_on_file->divisor != rd->divisor)
+            info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->divisor, rd_on_file->divisor);
+    }
+
+    // zero the entire header
+    memset(rd_on_file, 0, sizeof(struct rrddim_map_save_v019));
+
+    // set the important fields
+    strcpy(rd_on_file->magic, RRDDIMENSION_MAGIC_V019);
+    rd_on_file->algorithm = rd->algorithm;
+    rd_on_file->multiplier = rd->multiplier;
+    rd_on_file->divisor = rd->divisor;
+    rd_on_file->entries = st->entries;
+    rd_on_file->update_every = rd->update_every;
+    rd_on_file->memsize = size;
+    rd_on_file->rrd_memory_mode = memory_mode;
+    rd_on_file->cache_filename = strdupz(fullfilename);
+
+    rd->db = &rd_on_file->values[0];
+    rd->rd_on_file = rd_on_file;
+    rd->memsize = size;
+    rrddim_memory_file_update(rd);
+
+    return true;
+}