summaryrefslogtreecommitdiffstats
path: root/src/database/engine
diff options
context:
space:
mode:
Diffstat (limited to 'src/database/engine')
-rw-r--r--src/database/engine/cache.c16
-rw-r--r--src/database/engine/cache.h11
-rw-r--r--src/database/engine/datafile.c9
-rw-r--r--src/database/engine/datafile.h7
-rw-r--r--src/database/engine/dbengine-stresstest.c14
-rw-r--r--src/database/engine/dbengine-unittest.c14
-rw-r--r--src/database/engine/journalfile.c1
-rw-r--r--src/database/engine/pdc.c8
-rw-r--r--src/database/engine/rrdengine.c6
-rw-r--r--src/database/engine/rrdengine.h5
10 files changed, 55 insertions, 36 deletions
diff --git a/src/database/engine/cache.c b/src/database/engine/cache.c
index a03df4676..f47674764 100644
--- a/src/database/engine/cache.c
+++ b/src/database/engine/cache.c
@@ -1786,7 +1786,7 @@ PGC *pgc_create(const char *name,
sizeof(PGC_PAGE) + cache->config.additional_bytes_per_page,
0,
16384,
- aral_statistics(pgc_section_pages_aral),
+ aral_get_statistics(pgc_section_pages_aral),
NULL, NULL, false, false);
}
#endif
@@ -1797,7 +1797,7 @@ PGC *pgc_create(const char *name,
}
struct aral_statistics *pgc_aral_statistics(void) {
- return aral_statistics(pgc_section_pages_aral);
+ return aral_get_statistics(pgc_section_pages_aral);
}
size_t pgc_aral_structures(void) {
@@ -2366,7 +2366,7 @@ void *unittest_stress_test_collector(void *ptr) {
time_t start_time_t = pgc_uts.first_time_t + 1;
heartbeat_t hb;
- heartbeat_init(&hb);
+ heartbeat_init(&hb, pgc_uts.time_per_collection_ut);
while(!__atomic_load_n(&pgc_uts.stop, __ATOMIC_RELAXED)) {
// netdata_log_info("COLLECTOR %zu: collecting metrics %zu to %zu, from %ld to %lu", id, metric_start, metric_end, start_time_t, start_time_t + pgc_uts.points_per_page);
@@ -2393,7 +2393,7 @@ void *unittest_stress_test_collector(void *ptr) {
time_t end_time_t = start_time_t + (time_t)pgc_uts.points_per_page;
while(++start_time_t <= end_time_t && !__atomic_load_n(&pgc_uts.stop, __ATOMIC_RELAXED)) {
- heartbeat_next(&hb, pgc_uts.time_per_collection_ut);
+ heartbeat_next(&hb);
for (size_t i = metric_start; i < metric_end; i++) {
if(pgc_uts.metrics[i])
@@ -2480,9 +2480,9 @@ void *unittest_stress_test_queries(void *ptr) {
void *unittest_stress_test_service(void *ptr) {
heartbeat_t hb;
- heartbeat_init(&hb);
+ heartbeat_init(&hb, USEC_PER_SEC);
while(!__atomic_load_n(&pgc_uts.stop, __ATOMIC_RELAXED)) {
- heartbeat_next(&hb, 1 * USEC_PER_SEC);
+ heartbeat_next(&hb);
pgc_flush_pages(pgc_uts.cache, 1000);
pgc_evict_pages(pgc_uts.cache, 0, 0);
@@ -2545,7 +2545,7 @@ void unittest_stress_test(void) {
}
heartbeat_t hb;
- heartbeat_init(&hb);
+ heartbeat_init(&hb, USEC_PER_SEC);
struct {
size_t entries;
@@ -2578,7 +2578,7 @@ void unittest_stress_test(void) {
} stats = {}, old_stats = {};
for(int i = 0; i < 86400 ;i++) {
- heartbeat_next(&hb, 1 * USEC_PER_SEC);
+ heartbeat_next(&hb);
old_stats = stats;
stats.entries = __atomic_load_n(&pgc_uts.cache->stats.entries, __ATOMIC_RELAXED);
diff --git a/src/database/engine/cache.h b/src/database/engine/cache.h
index b6f81bcc2..ef9652028 100644
--- a/src/database/engine/cache.h
+++ b/src/database/engine/cache.h
@@ -248,4 +248,15 @@ struct aral_statistics *pgc_aral_statistics(void);
size_t pgc_aral_structures(void);
size_t pgc_aral_overhead(void);
+static inline size_t indexing_partition(Word_t ptr, Word_t modulo) __attribute__((const));
+static inline size_t indexing_partition(Word_t ptr, Word_t modulo) {
+#ifdef ENV64BIT
+ uint64_t hash = murmur64(ptr);
+ return hash % modulo;
+#else
+ uint32_t hash = murmur32(ptr);
+ return hash % modulo;
+#endif
+}
+
#endif // DBENGINE_CACHE_H
diff --git a/src/database/engine/datafile.c b/src/database/engine/datafile.c
index 35c047722..7bf9487f2 100644
--- a/src/database/engine/datafile.c
+++ b/src/database/engine/datafile.c
@@ -66,7 +66,8 @@ void datafile_release(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS re
spinlock_unlock(&df->users.spinlock);
}
-bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {
+bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdown)
+{
bool can_be_deleted = false;
spinlock_lock(&df->users.spinlock);
@@ -107,7 +108,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {
if(!df->users.time_to_evict) {
// first time we did the above
- df->users.time_to_evict = now_s + 120;
+ df->users.time_to_evict = now_s + (is_shutdown ? DATAFILE_DELETE_TIMEOUT_SHORT : DATAFILE_DELETE_TIMEOUT_LONG);
internal_error(true, "DBENGINE: datafile %u of tier %d is not used by any open cache pages, "
"but it has %u lockers (oc:%u, pd:%u), "
"%zu clean and %zu hot open cache pages "
@@ -572,8 +573,8 @@ void finalize_data_files(struct rrdengine_instance *ctx)
struct rrdengine_journalfile *journalfile = datafile->journalfile;
logged = false;
- size_t iterations = 100;
- while(!datafile_acquire_for_deletion(datafile) && datafile != ctx->datafiles.first->prev && --iterations > 0) {
+ size_t iterations = 10;
+ while(!datafile_acquire_for_deletion(datafile, true) && datafile != ctx->datafiles.first->prev && --iterations > 0) {
if(!logged) {
netdata_log_info("Waiting to acquire data file %u of tier %d to close it...", datafile->fileno, ctx->config.tier);
logged = true;
diff --git a/src/database/engine/datafile.h b/src/database/engine/datafile.h
index 569f1b0a2..843cb8c1e 100644
--- a/src/database/engine/datafile.h
+++ b/src/database/engine/datafile.h
@@ -24,6 +24,11 @@ struct rrdengine_instance;
#define MAX_DATAFILES (65536 * 4) /* Supports up to 64TiB for now */
#define TARGET_DATAFILES (50)
+// When trying to acquire a datafile for deletion and an attempt to evict pages is completed
+// the acquire for deletion will return true after this timeout
+#define DATAFILE_DELETE_TIMEOUT_SHORT (1)
+#define DATAFILE_DELETE_TIMEOUT_LONG (120)
+
typedef enum __attribute__ ((__packed__)) {
DATAFILE_ACQUIRE_OPEN_CACHE = 0,
DATAFILE_ACQUIRE_PAGE_DETAILS,
@@ -72,7 +77,7 @@ struct rrdengine_datafile {
bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason);
void datafile_release(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason);
-bool datafile_acquire_for_deletion(struct rrdengine_datafile *df);
+bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdown);
void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, bool having_lock);
void datafile_list_delete_unsafe(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile);
diff --git a/src/database/engine/dbengine-stresstest.c b/src/database/engine/dbengine-stresstest.c
index 1d978cd52..0447bcf33 100644
--- a/src/database/engine/dbengine-stresstest.c
+++ b/src/database/engine/dbengine-stresstest.c
@@ -22,13 +22,13 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {
default_rrd_history_entries,
RRD_MEMORY_MODE_DBENGINE,
health_plugin_enabled(),
- default_rrdpush_enabled,
- default_rrdpush_destination,
- default_rrdpush_api_key,
- default_rrdpush_send_charts_matching,
- default_rrdpush_enable_replication,
- default_rrdpush_seconds_to_replicate,
- default_rrdpush_replication_step,
+ stream_conf_send_enabled,
+ stream_conf_send_destination,
+ stream_conf_send_api_key,
+ stream_conf_send_charts_matching,
+ stream_conf_replication_enabled,
+ stream_conf_replication_period,
+ stream_conf_replication_step,
NULL,
0
);
diff --git a/src/database/engine/dbengine-unittest.c b/src/database/engine/dbengine-unittest.c
index cfe038df6..755336101 100644
--- a/src/database/engine/dbengine-unittest.c
+++ b/src/database/engine/dbengine-unittest.c
@@ -108,13 +108,13 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {
default_rrd_history_entries,
RRD_MEMORY_MODE_DBENGINE,
health_plugin_enabled(),
- default_rrdpush_enabled,
- default_rrdpush_destination,
- default_rrdpush_api_key,
- default_rrdpush_send_charts_matching,
- default_rrdpush_enable_replication,
- default_rrdpush_seconds_to_replicate,
- default_rrdpush_replication_step,
+ stream_conf_send_enabled,
+ stream_conf_send_destination,
+ stream_conf_send_api_key,
+ stream_conf_send_charts_matching,
+ stream_conf_replication_enabled,
+ stream_conf_replication_period,
+ stream_conf_replication_step,
NULL,
0
);
diff --git a/src/database/engine/journalfile.c b/src/database/engine/journalfile.c
index 4ea988d64..b120ce3c3 100644
--- a/src/database/engine/journalfile.c
+++ b/src/database/engine/journalfile.c
@@ -1,5 +1,4 @@
// SPDX-License-Identifier: GPL-3.0-or-later
-#include "libnetdata/bitmap64.h"
#include "rrdengine.h"
static void after_extent_write_journalfile_v1_io(uv_fs_t* req)
diff --git a/src/database/engine/pdc.c b/src/database/engine/pdc.c
index 28a83e2bc..da5dbd7d8 100644
--- a/src/database/engine/pdc.c
+++ b/src/database/engine/pdc.c
@@ -184,12 +184,12 @@ static struct {
.allocated = 0,
.allocated_bytes = 0,
},
- .max_size = MAX_PAGES_PER_EXTENT * RRDENG_BLOCK_SIZE,
+ .max_size = MAX_EXTENT_UNCOMPRESSED_SIZE
};
void extent_buffer_init(void) {
- size_t max_extent_uncompressed = MAX_PAGES_PER_EXTENT * RRDENG_BLOCK_SIZE;
- size_t max_size = (size_t)LZ4_compressBound(MAX_PAGES_PER_EXTENT * RRDENG_BLOCK_SIZE);
+ size_t max_extent_uncompressed = MAX_EXTENT_UNCOMPRESSED_SIZE;
+ size_t max_size = (size_t)LZ4_compressBound(MAX_EXTENT_UNCOMPRESSED_SIZE);
if(max_size < max_extent_uncompressed)
max_size = max_extent_uncompressed;
@@ -1010,7 +1010,7 @@ static bool epdl_populate_pages_from_extent_data(
uncompressed_payload_length += header->descr[i].page_length;
}
- if(unlikely(uncompressed_payload_length > MAX_PAGES_PER_EXTENT * RRDENG_BLOCK_SIZE))
+ if(unlikely(uncompressed_payload_length > MAX_EXTENT_UNCOMPRESSED_SIZE))
have_read_error = true;
if(likely(!have_read_error)) {
diff --git a/src/database/engine/rrdengine.c b/src/database/engine/rrdengine.c
index a989877fc..78ad873f7 100644
--- a/src/database/engine/rrdengine.c
+++ b/src/database/engine/rrdengine.c
@@ -1218,7 +1218,7 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *
if(worker)
worker_is_busy(UV_EVENT_DBENGINE_DATAFILE_DELETE_WAIT);
- bool datafile_got_for_deletion = datafile_acquire_for_deletion(datafile);
+ bool datafile_got_for_deletion = datafile_acquire_for_deletion(datafile, false);
if (update_retention)
update_metrics_first_time_s(ctx, datafile, datafile->next, worker);
@@ -1227,7 +1227,7 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *
if(worker)
worker_is_busy(UV_EVENT_DBENGINE_DATAFILE_DELETE_WAIT);
- datafile_got_for_deletion = datafile_acquire_for_deletion(datafile);
+ datafile_got_for_deletion = datafile_acquire_for_deletion(datafile, false);
if (!datafile_got_for_deletion) {
netdata_log_info("DBENGINE: waiting for data file '%s/"
@@ -2075,6 +2075,6 @@ void dbengine_event_loop(void* arg) {
}
nd_log(NDLS_DAEMON, NDLP_DEBUG, "Shutting down dbengine thread");
- uv_loop_close(&main->loop);
+ (void) uv_loop_close(&main->loop);
worker_unregister();
}
diff --git a/src/database/engine/rrdengine.h b/src/database/engine/rrdengine.h
index 37ea92b8a..0a22477ac 100644
--- a/src/database/engine/rrdengine.h
+++ b/src/database/engine/rrdengine.h
@@ -28,7 +28,10 @@ struct rrdengine_instance;
struct rrdeng_cmd;
#define MAX_PAGES_PER_EXTENT (109) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */
-#define DEFAULT_PAGES_PER_EXTENT (64)
+#define DEFAULT_PAGES_PER_EXTENT (109)
+
+#define MAX_EXTENT_UNCOMPRESSED_SIZE (MAX_PAGES_PER_EXTENT * (RRDENG_BLOCK_SIZE + RRDENG_GORILLA_32BIT_BUFFER_SIZE))
+
#define RRDENG_FILE_NUMBER_SCAN_TMPL "%1u-%10u"
#define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 08:20:58 +0000