Adding upstream version 1.46.3.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 40 insertions, 19 deletions

diff --git a/database/engine/journalfile.c b/src/database/engine/journalfile.c
index 9005b81ca..4ea988d64 100644
--- a/database/engine/journalfile.c
+++ b/src/database/engine/journalfile.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
+#include "libnetdata/bitmap64.h"
 #include "rrdengine.h"
 
 static void after_extent_write_journalfile_v1_io(uv_fs_t* req)
@@ -572,7 +573,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi
     uv_fs_t req;
     uv_file file;
     int ret, fd;
-    struct rrdeng_jf_sb *superblock;
+    struct rrdeng_jf_sb *superblock = NULL;
     uv_buf_t iov;
     char path[RRDENG_PATH_MAX];
 
@@ -618,7 +619,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi
 static int journalfile_check_superblock(uv_file file)
 {
     int ret;
-    struct rrdeng_jf_sb *superblock;
+    struct rrdeng_jf_sb *superblock = NULL;
     uv_buf_t iov;
     uv_fs_t req;
 
@@ -637,9 +638,12 @@ static int journalfile_check_superblock(uv_file file)
     fatal_assert(req.result >= 0);
     uv_fs_req_cleanup(&req);
 
-    if (strncmp(superblock->magic_number, RRDENG_JF_MAGIC, RRDENG_MAGIC_SZ) ||
-        strncmp(superblock->version, RRDENG_JF_VER, RRDENG_VER_SZ)) {
-        netdata_log_error("DBENGINE: File has invalid superblock.");
+
+    char jf_magic[RRDENG_MAGIC_SZ] = RRDENG_JF_MAGIC;
+    char jf_ver[RRDENG_VER_SZ] = RRDENG_JF_VER;
+    if (strncmp(superblock->magic_number, jf_magic, RRDENG_MAGIC_SZ) != 0 ||
+        strncmp(superblock->version, jf_ver, RRDENG_VER_SZ) != 0) {
+        nd_log(NDLS_DAEMON, NDLP_ERR, "DBENGINE: File has invalid superblock.");
         ret = UV_EINVAL;
     } else {
         ret = 0;
@@ -651,7 +655,7 @@ static int journalfile_check_superblock(uv_file file)
 
 static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile, void *buf, unsigned max_size)
 {
-    static BITMAP256 page_error_map = BITMAP256_INITIALIZER;
+    static bitmap64_t page_error_map = BITMAP64_INITIALIZER;
     unsigned i, count, payload_length, descr_size;
     struct rrdeng_jf_store_data *jf_metric_data;
 
@@ -665,19 +669,20 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
     }
 
     time_t now_s = max_acceptable_collected_time();
+    time_t extent_first_time_s = journalfile->v2.first_time_s ? journalfile->v2.first_time_s : LONG_MAX;
     for (i = 0; i < count ; ++i) {
-        uuid_t *temp_id;
+        nd_uuid_t *temp_id;
         uint8_t page_type = jf_metric_data->descr[i].type;
 
-        if (page_type > PAGE_TYPE_MAX) {
-            if (!bitmap256_get_bit(&page_error_map, page_type)) {
+        if (page_type > RRDENG_PAGE_TYPE_MAX) {
+            if (!bitmap64_get(&page_error_map, page_type)) {
                 netdata_log_error("DBENGINE: unknown page type %d encountered.", page_type);
-                bitmap256_set_bit(&page_error_map, page_type, 1);
+                bitmap64_set(&page_error_map, page_type);
             }
             continue;
         }
 
-        temp_id = (uuid_t *)jf_metric_data->descr[i].uuid;
+        temp_id = (nd_uuid_t *)jf_metric_data->descr[i].uuid;
         METRIC *metric = mrg_metric_get_and_acquire(main_mrg, temp_id, (Word_t) ctx);
 
         struct rrdeng_extent_page_descr *descr = &jf_metric_data->descr[i];
@@ -700,13 +705,19 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
                     .section = (Word_t)ctx,
                     .first_time_s = vd.start_time_s,
                     .last_time_s = vd.end_time_s,
-                    .latest_update_every_s = (uint32_t) vd.update_every_s,
+                    .latest_update_every_s = vd.update_every_s,
             };
 
             bool added;
             metric = mrg_metric_add_and_acquire(main_mrg, entry, &added);
-            if(added)
+            if(added) {
+                __atomic_add_fetch(&ctx->atomic.metrics, 1, __ATOMIC_RELAXED);
                 update_metric_time = false;
+            }
+            if (vd.update_every_s) {
+                uint64_t samples = (vd.end_time_s - vd.start_time_s) / vd.update_every_s;
+                __atomic_add_fetch(&ctx->atomic.samples, samples, __ATOMIC_RELAXED);
+            }
         }
         Word_t metric_id = mrg_metric_id(main_mrg, metric);
 
@@ -718,8 +729,18 @@ static void journalfile_restore_extent_metadata(struct rrdengine_instance *ctx,
                 journalfile->datafile,
                 jf_metric_data->extent_offset, jf_metric_data->extent_size, jf_metric_data->descr[i].page_length);
 
+        extent_first_time_s = MIN(extent_first_time_s, vd.start_time_s);
+
         mrg_metric_release(main_mrg, metric);
     }
+
+    journalfile->v2.first_time_s = extent_first_time_s;
+
+    time_t old = __atomic_load_n(&ctx->atomic.first_time_s, __ATOMIC_RELAXED);;
+    do {
+        if(old <= extent_first_time_s)
+            break;
+    } while(!__atomic_compare_exchange_n(&ctx->atomic.first_time_s, &old, extent_first_time_s, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
 }
 
 /*
@@ -790,7 +811,7 @@ static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx,
     int ret;
     uint64_t pos, pos_i, max_id, id;
     unsigned size_bytes;
-    void *buf;
+    void *buf = NULL;
     uv_buf_t iov;
     uv_fs_t req;
 
@@ -1005,7 +1026,7 @@ void journalfile_v2_populate_retention_to_mrg(struct rrdengine_instance *ctx, st
         time_t end_time_s = header_start_time_s + metric->delta_end_s;
 
         mrg_update_metric_retention_and_granularity_by_uuid(
-                main_mrg, (Word_t)ctx, &metric->uuid, start_time_s, end_time_s, (time_t) metric->update_every_s, now_s);
+                main_mrg, (Word_t)ctx, &metric->uuid, start_time_s, end_time_s, metric->update_every_s, now_s);
 
         metric++;
     }
@@ -1042,7 +1063,7 @@ int journalfile_v2_load(struct rrdengine_instance *ctx, struct rrdengine_journal
         journal_v1_file_size = (uint32_t)statbuf.st_size;
 
     journalfile_v2_generate_path(datafile, path_v2, sizeof(path_v2));
-    fd = open(path_v2, O_RDONLY);
+    fd = open(path_v2, O_RDONLY | O_CLOEXEC);
     if (fd < 0) {
         if (errno == ENOENT)
             return 1;
@@ -1135,7 +1156,7 @@ static int journalfile_metric_compare (const void *item1, const void *item2)
     const struct jv2_metrics_info *metric1 = ((struct journal_metric_list_to_sort *) item1)->metric_info;
     const struct jv2_metrics_info *metric2 = ((struct journal_metric_list_to_sort *) item2)->metric_info;
 
-    return memcmp(metric1->uuid, metric2->uuid, sizeof(uuid_t));
+    return memcmp(metric1->uuid, metric2->uuid, sizeof(nd_uuid_t));
 }
 
 
@@ -1226,7 +1247,7 @@ void *journalfile_v2_write_data_page(struct journal_v2_header *j2_header, void *
     data_page->delta_end_s = (uint32_t) (page_info->end_time_s - (time_t) (j2_header->start_time_ut) / USEC_PER_SEC);
     data_page->extent_index = page_info->extent_index;
 
-    data_page->update_every_s = (uint32_t) page_info->update_every_s;
+    data_page->update_every_s = page_info->update_every_s;
     data_page->page_length = (uint16_t) (ei ? ei->page_length : page_info->page_length);
     data_page->type = 0;
 
@@ -1252,7 +1273,7 @@ static void *journalfile_v2_write_descriptors(struct journal_v2_header *j2_heade
         page_info = *PValue;
         // Write one descriptor and return the next data page location
         data_page = journalfile_v2_write_data_page(j2_header, (void *) data_page, page_info);
-        update_every_s = (uint32_t) page_info->update_every_s;
+        update_every_s = page_info->update_every_s;
         if (NULL == data_page)
             break;
     }