From f99c4526d94d3e04124c5c48ab4a3da6ca53a458 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 31 Mar 2021 14:58:11 +0200 Subject: Adding upstream version 1.30.0. Signed-off-by: Daniel Baumann --- database/engine/pagecache.c | 68 +++++++ database/engine/pagecache.h | 2 + database/engine/rrdengine.c | 53 ++++-- database/engine/rrdengine.h | 20 +- database/engine/rrdengineapi.c | 30 +++ database/engine/rrdengineapi.h | 1 + database/rrd.h | 64 ++++--- database/rrdcalc.c | 14 +- database/rrdcalc.h | 2 +- database/rrddim.c | 32 ++-- database/rrdfamily.c | 6 +- database/rrdhost.c | 40 ++-- database/rrdset.c | 52 ++--- database/rrdvar.c | 6 +- database/rrdvar.h | 2 +- database/sqlite/Makefile.am | 4 + database/sqlite/sqlite_functions.c | 380 ++++++++++++++++++++++++++++++++++--- database/sqlite/sqlite_functions.h | 3 +- 18 files changed, 629 insertions(+), 150 deletions(-) create mode 100644 database/sqlite/Makefile.am (limited to 'database') diff --git a/database/engine/pagecache.c b/database/engine/pagecache.c index a18207100..d7698de01 100644 --- a/database/engine/pagecache.c +++ b/database/engine/pagecache.c @@ -699,6 +699,74 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c } uv_rwlock_rdunlock(&page_index->lock); } + +/** + * Searches for an unallocated page without triggering disk I/O. Attempts to reserve the page and get a reference. + * @param ctx DB context + * @param id lookup by UUID + * @param start_time exact starting time in usec + * @param ret_page_indexp Sets the page index pointer (*ret_page_indexp) for the given UUID. + * @return the page descriptor or NULL on failure. It can fail if: + * 1. The page is already allocated to the page cache. + * 2. It did not succeed to get a reference. + * 3. It did not succeed to reserve a spot in the page cache. + */ +struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id, + usec_t start_time) +{ + struct page_cache *pg_cache = &ctx->pg_cache; + struct rrdeng_page_descr *descr = NULL; + struct page_cache_descr *pg_cache_descr = NULL; + unsigned long flags; + Pvoid_t *PValue; + struct pg_cache_page_index *page_index = NULL; + Word_t Index; + + uv_rwlock_rdlock(&pg_cache->metrics_index.lock); + PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, id, sizeof(uuid_t)); + if (likely(NULL != PValue)) { + page_index = *PValue; + } + uv_rwlock_rdunlock(&pg_cache->metrics_index.lock); + + if ((NULL == PValue) || !pg_cache_try_reserve_pages(ctx, 1)) { + /* Failed to find page or failed to reserve a spot in the cache */ + return NULL; + } + + uv_rwlock_rdlock(&page_index->lock); + Index = (Word_t)(start_time / USEC_PER_SEC); + PValue = JudyLGet(page_index->JudyL_array, Index, PJE0); + if (likely(NULL != PValue)) { + descr = *PValue; + } + if (NULL == PValue || 0 == descr->page_length) { + /* Failed to find non-empty page */ + uv_rwlock_rdunlock(&page_index->lock); + + pg_cache_release_pages(ctx, 1); + return NULL; + } + + rrdeng_page_descr_mutex_lock(ctx, descr); + pg_cache_descr = descr->pg_cache_descr; + flags = pg_cache_descr->flags; + uv_rwlock_rdunlock(&page_index->lock); + + if ((flags & RRD_PAGE_POPULATED) || !pg_cache_try_get_unsafe(descr, 1)) { + /* Failed to get reference or page is already populated */ + rrdeng_page_descr_mutex_unlock(ctx, descr); + + pg_cache_release_pages(ctx, 1); + return NULL; + } + /* success */ + rrdeng_page_descr_mutex_unlock(ctx, descr); + rrd_stat_atomic_add(&ctx->stats.pg_cache_misses, 1); + + return descr; +} + /** * Searches for pages in a time range and triggers disk I/O if necessary and possible. * Does not get a reference. diff --git a/database/engine/pagecache.h b/database/engine/pagecache.h index 31e9739da..d5350ef56 100644 --- a/database/engine/pagecache.h +++ b/database/engine/pagecache.h @@ -172,6 +172,8 @@ extern usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid extern void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index, usec_t point_in_time, pg_cache_page_info_filter_t *filter, struct rrdeng_page_info *page_info); +extern struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id, + usec_t start_time); extern unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time, struct rrdeng_page_info **page_info_arrayp, struct pg_cache_page_index **ret_page_indexp); diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c index 43135ff01..0c4a401cb 100644 --- a/database/engine/rrdengine.c +++ b/database/engine/rrdengine.c @@ -9,6 +9,8 @@ rrdeng_stats_t rrdeng_reserved_file_descriptors = 0; rrdeng_stats_t global_pg_cache_over_half_dirty_events = 0; rrdeng_stats_t global_flushing_pressure_page_deletions = 0; +static unsigned pages_per_extent = MAX_PAGES_PER_EXTENT; + static void sanity_check(void) { /* Magic numbers must fit in the super-blocks */ @@ -305,19 +307,32 @@ after_crc_check: } } - for (i = 0 ; i < xt_io_descr->descr_count; ++i) { - page = mallocz(RRDENG_BLOCK_SIZE); - descr = xt_io_descr->descr_array[i]; - for (j = 0, page_offset = 0; j < count; ++j) { + for (i = 0, page_offset = 0; i < count; page_offset += header->descr[i++].page_length) { + uint8_t is_prefetched_page; + descr = NULL; + for (j = 0 ; j < xt_io_descr->descr_count; ++j) { + struct rrdeng_page_descr *descrj; + + descrj = xt_io_descr->descr_array[j]; /* care, we don't hold the descriptor mutex */ - if (!uuid_compare(*(uuid_t *) header->descr[j].uuid, *descr->id) && - header->descr[j].page_length == descr->page_length && - header->descr[j].start_time == descr->start_time && - header->descr[j].end_time == descr->end_time) { + if (!uuid_compare(*(uuid_t *) header->descr[i].uuid, *descrj->id) && + header->descr[i].page_length == descrj->page_length && + header->descr[i].start_time == descrj->start_time && + header->descr[i].end_time == descrj->end_time) { + descr = descrj; break; } - page_offset += header->descr[j].page_length; } + is_prefetched_page = 0; + if (!descr) { /* This extent page has not been requested. Try populating it for locality (best effort). */ + descr = pg_cache_lookup_unpopulated_and_lock(ctx, (uuid_t *)header->descr[i].uuid, + header->descr[i].start_time); + if (!descr) + continue; /* Failed to reserve a suitable page */ + is_prefetched_page = 1; + } + page = mallocz(RRDENG_BLOCK_SIZE); + /* care, we don't hold the descriptor mutex */ if (have_read_error) { /* Applications should make sure NULL values match 0 as does SN_EMPTY_SLOT */ @@ -334,7 +349,7 @@ after_crc_check: pg_cache_descr->flags &= ~RRD_PAGE_READ_PENDING; rrdeng_page_descr_mutex_unlock(ctx, descr); pg_cache_replaceQ_insert(ctx, descr); - if (xt_io_descr->release_descr) { + if (xt_io_descr->release_descr || is_prefetched_page) { pg_cache_put(ctx, descr); } else { debug(D_RRDENGINE, "%s: Waking up waiters.", __func__); @@ -666,7 +681,7 @@ static int do_flush_pages(struct rrdengine_worker_config* wc, int force, struct PValue = JudyLFirst(pg_cache->committed_page_index.JudyL_array, &Index, PJE0), descr = unlikely(NULL == PValue) ? NULL : *PValue ; - descr != NULL && count != MAX_PAGES_PER_EXTENT ; + descr != NULL && count != pages_per_extent ; PValue = JudyLNext(pg_cache->committed_page_index.JudyL_array, &Index, PJE0), descr = unlikely(NULL == PValue) ? NULL : *PValue) { @@ -1031,6 +1046,21 @@ struct rrdeng_cmd rrdeng_deq_cmd(struct rrdengine_worker_config* wc) return ret; } +static void load_configuration_dynamic(void) +{ + unsigned read_num; + static int printed_error = 0; + + read_num = (unsigned) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine extent pages", + MAX_PAGES_PER_EXTENT); + if (read_num > 0 && read_num <= MAX_PAGES_PER_EXTENT) { + pages_per_extent = read_num; + } else if (!printed_error) { + printed_error = 1; + error("Invalid dbengine extent pages %u given. Defaulting to %u.", read_num, pages_per_extent); + } +} + void async_cb(uv_async_t *handle) { uv_stop(handle->loop); @@ -1084,6 +1114,7 @@ void timer_cb(uv_timer_t* handle) } } } + load_configuration_dynamic(); #ifdef NETDATA_INTERNAL_CHECKS { char buf[4096]; diff --git a/database/engine/rrdengine.h b/database/engine/rrdengine.h index 87af04bff..2d48665f8 100644 --- a/database/engine/rrdengine.h +++ b/database/engine/rrdengine.h @@ -56,16 +56,20 @@ enum rrdeng_opcode { RRDENG_MAX_OPCODE }; +struct rrdeng_read_page { + struct rrdeng_page_descr *page_cache_descr; +}; + +struct rrdeng_read_extent { + struct rrdeng_page_descr *page_cache_descr[MAX_PAGES_PER_EXTENT]; + int page_count; +}; + struct rrdeng_cmd { enum rrdeng_opcode opcode; union { - struct rrdeng_read_page { - struct rrdeng_page_descr *page_cache_descr; - } read_page; - struct rrdeng_read_extent { - struct rrdeng_page_descr *page_cache_descr[MAX_PAGES_PER_EXTENT]; - int page_count; - } read_extent; + struct rrdeng_read_page read_page; + struct rrdeng_read_extent read_extent; struct completion *completion; }; }; @@ -230,4 +234,4 @@ extern void rrdeng_worker(void* arg); extern void rrdeng_enq_cmd(struct rrdengine_worker_config* wc, struct rrdeng_cmd *cmd); extern struct rrdeng_cmd rrdeng_deq_cmd(struct rrdengine_worker_config* wc); -#endif /* NETDATA_RRDENGINE_H */ \ No newline at end of file +#endif /* NETDATA_RRDENGINE_H */ diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c index 7b2ff5b72..cb46e06e3 100755 --- a/database/engine/rrdengineapi.c +++ b/database/engine/rrdengineapi.c @@ -691,6 +691,36 @@ time_t rrdeng_metric_oldest_time(RRDDIM *rd) return page_index->oldest_time / USEC_PER_SEC; } +int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t) +{ + struct page_cache *pg_cache; + struct rrdengine_instance *ctx; + Pvoid_t *PValue; + struct pg_cache_page_index *page_index = NULL; + + ctx = get_rrdeng_ctx_from_host(localhost); + if (unlikely(!ctx)) { + error("Failed to fetch multidb context"); + return 1; + } + pg_cache = &ctx->pg_cache; + + uv_rwlock_rdlock(&pg_cache->metrics_index.lock); + PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, dim_uuid, sizeof(uuid_t)); + if (likely(NULL != PValue)) { + page_index = *PValue; + } + uv_rwlock_rdunlock(&pg_cache->metrics_index.lock); + + if (likely(page_index)) { + *first_entry_t = page_index->oldest_time / USEC_PER_SEC; + *last_entry_t = page_index->latest_time / USEC_PER_SEC; + return 0; + } + + return 1; +} + /* Also gets a reference for the page */ void *rrdeng_create_page(struct rrdengine_instance *ctx, uuid_t *id, struct rrdeng_page_descr **ret_descr) { diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h index 41375b980..00e55e662 100644 --- a/database/engine/rrdengineapi.h +++ b/database/engine/rrdengineapi.h @@ -59,5 +59,6 @@ extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *db extern int rrdeng_exit(struct rrdengine_instance *ctx); extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx); +extern int rrdeng_metric_latest_time_by_uuid(uuid_t *dim_uuid, time_t *first_entry_t, time_t *last_entry_t); #endif /* NETDATA_RRDENGINEAPI_H */ diff --git a/database/rrd.h b/database/rrd.h index b16fcdc16..59d0501bd 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -34,12 +34,24 @@ struct pg_cache_page_index; #include "rrdcalc.h" #include "rrdcalctemplate.h" #include "../streaming/rrdpush.h" + +#ifndef ACLK_NG #include "../aclk/legacy/aclk_rrdhost_state.h" +#else +#include "aclk/aclk.h" +#endif + +enum { + CONTEXT_FLAGS_ARCHIVE = 0x01, + CONTEXT_FLAGS_CHART = 0x02, + CONTEXT_FLAGS_CONTEXT = 0x04 +}; struct context_param { RRDDIM *rd; time_t first_entry_t; time_t last_entry_t; + uint8_t flags; }; #define META_CHART_UPDATED 1 @@ -130,7 +142,7 @@ extern const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); // RRD FAMILY struct rrdfamily { - avl avl; + avl_t avl; const char *family; uint32_t hash_family; @@ -235,7 +247,7 @@ struct rrddim { // ------------------------------------------------------------------------ // binary indexing structures - avl avl; // the binary index - this has to be first member! + avl_t avl; // the binary index - this has to be first member! // ------------------------------------------------------------------------ // the dimension definition @@ -336,6 +348,18 @@ union rrddim_collect_handle { // ---------------------------------------------------------------------------- // iterator state for RRD dimension data queries + +#ifdef ENABLE_DBENGINE +struct rrdeng_query_handle { + struct rrdeng_page_descr *descr; + struct rrdengine_instance *ctx; + struct pg_cache_page_index *page_index; + time_t next_page_time; + time_t now; + unsigned position; +}; +#endif + struct rrddim_query_handle { RRDDIM *rd; time_t start_time; @@ -347,14 +371,7 @@ struct rrddim_query_handle { uint8_t finished; } slotted; // state the legacy code uses #ifdef ENABLE_DBENGINE - struct rrdeng_query_handle { - struct rrdeng_page_descr *descr; - struct rrdengine_instance *ctx; - struct pg_cache_page_index *page_index; - time_t next_page_time; - time_t now; - unsigned position; - } rrdeng; // state the database engine uses + struct rrdeng_query_handle rrdeng; // state the database engine uses #endif }; }; @@ -365,9 +382,9 @@ struct rrddim_query_handle { struct rrddim_volatile { #ifdef ENABLE_DBENGINE uuid_t *rrdeng_uuid; // database engine metric UUID - uuid_t *metric_uuid; // global UUID for this metric (unique_across hosts) struct pg_cache_page_index *page_index; #endif + uuid_t *metric_uuid; // global UUID for this metric (unique_across hosts) union rrddim_collect_handle handle; // ------------------------------------------------------------------------ // function pointers that handle data collection @@ -469,8 +486,8 @@ struct rrdset { // ------------------------------------------------------------------------ // binary indexing structures - avl avl; // the index, with key the id - this has to be first! - avl avlname; // the index, with key the name + avl_t avl; // the index, with key the id - this has to be first! + avl_t avlname; // the index, with key the name // ------------------------------------------------------------------------ // the set configuration @@ -523,7 +540,10 @@ struct rrdset { size_t counter; // the number of times we added values to this database size_t counter_done; // the number of times rrdset_done() has been called - time_t last_accessed_time; // the last time this RRDSET has been accessed + union { + time_t last_accessed_time; // the last time this RRDSET has been accessed + time_t last_entry_t; // the last_entry_t computed for transient RRDSET + }; time_t upstream_resync_time; // the timestamp up to which we should resync clock upstream char *plugin_name; // the name of the plugin that generated this @@ -722,7 +742,7 @@ struct rrdhost_system_info { }; struct rrdhost { - avl avl; // the index of hosts + avl_t avl; // the index of hosts // ------------------------------------------------------------------------ // host information @@ -851,8 +871,8 @@ struct rrdhost { #ifdef ENABLE_DBENGINE struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host - uuid_t host_uuid; // Global GUID for this host #endif + uuid_t host_uuid; // Global GUID for this host #ifdef ENABLE_HTTPS struct netdata_ssl ssl; //Structure used to encrypt the connection @@ -999,13 +1019,13 @@ extern void rrdhost_free_all(void); extern void rrdhost_save_all(void); extern void rrdhost_cleanup_all(void); -extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected); +extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host); extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info); extern void rrdhost_free(RRDHOST *host); extern void rrdhost_save_charts(RRDHOST *host); extern void rrdhost_delete_charts(RRDHOST *host); -extern int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now); +extern int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now); extern void rrdset_update_heterogeneous_flag(RRDSET *st); @@ -1053,7 +1073,7 @@ extern void rrdset_isnot_obsolete(RRDSET *st); // checks if the RRDSET should be offered to viewers #define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE) -#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions) +#define rrdset_is_available_for_exporting_and_alarms(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions) #define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions) // get the total duration in seconds of the round robin database @@ -1281,8 +1301,8 @@ extern int rrdfamily_compare(void *a, void *b); extern RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id); extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc); -#define rrdset_index_add(host, st) (RRDSET *)avl_insert_lock(&((host)->rrdset_root_index), (avl *)(st)) -#define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl *)(st)) +#define rrdset_index_add(host, st) (RRDSET *)avl_insert_lock(&((host)->rrdset_root_index), (avl_t *)(st)) +#define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl_t *)(st)) extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st); extern void rrdset_free(RRDSET *st); @@ -1312,7 +1332,7 @@ extern void set_host_properties( #ifdef ENABLE_DBENGINE #include "database/engine/rrdengineapi.h" -#include "sqlite/sqlite_functions.h" #endif +#include "sqlite/sqlite_functions.h" #endif /* NETDATA_RRD_H */ diff --git a/database/rrdcalc.c b/database/rrdcalc.c index 935ee9c05..bc91da64f 100644 --- a/database/rrdcalc.c +++ b/database/rrdcalc.c @@ -472,7 +472,7 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt, rrdcalc_add_to_host(host, rc); if(!rt->foreachdim) { - RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl *)rc); + RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl_t *)rc); if (rdcmp != rc) { error("Cannot insert the alarm index ID %s",rc->name); } @@ -605,17 +605,17 @@ void rrdcalc_unlink_and_free(RRDHOST *host, RRDCALC *rc) { error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname); } - RRDCALC *rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_health_log, (avl *)rc); + RRDCALC *rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_health_log, (avl_t *)rc); if (rdcmp) { - rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_health_log, (avl *)rc); + rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_health_log, (avl_t *)rc); if (!rdcmp) { error("Cannot remove the health alarm index from health_log"); } } - rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_name, (avl *)rc); + rdcmp = (RRDCALC *) avl_search_lock(&(host)->alarms_idx_name, (avl_t *)rc); if (rdcmp) { - rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_name, (avl *)rc); + rdcmp = (RRDCALC *) avl_remove_lock(&(host)->alarms_idx_name, (avl_t *)rc); if (!rdcmp) { error("Cannot remove the health alarm index from idx_name"); } @@ -727,7 +727,7 @@ void rrdcalc_labels_unlink() { int alarm_isrepeating(RRDHOST *host, uint32_t alarm_id) { RRDCALC findme; findme.id = alarm_id; - RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_health_log, (avl *)&findme); + RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_health_log, (avl_t *)&findme); if (!rc) { return 0; } @@ -761,7 +761,7 @@ RRDCALC *alarm_max_last_repeat(RRDHOST *host, char *alarm_name,uint32_t hash) { RRDCALC findme; findme.name = alarm_name; findme.hash = hash; - RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_name, (avl *)&findme); + RRDCALC *rc = (RRDCALC *)avl_search_lock(&host->alarms_idx_name, (avl_t *)&findme); return rc; } diff --git a/database/rrdcalc.h b/database/rrdcalc.h index cd0d70049..27ff99a89 100644 --- a/database/rrdcalc.h +++ b/database/rrdcalc.h @@ -32,7 +32,7 @@ struct rrdcalc { - avl avl; // the index, with key the id - this has to be first! + avl_t avl; // the index, with key the id - this has to be first! uint32_t id; // the unique id of this alarm uint32_t next_event_id; // the next event id that will be used for this alarm diff --git a/database/rrddim.c b/database/rrddim.c index 6a1408595..b4ea34d2d 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -38,15 +38,15 @@ int rrddim_compare(void* a, void* b) { else return strcmp(((RRDDIM *)a)->id, ((RRDDIM *)b)->id); } -#define rrddim_index_add(st, rd) (RRDDIM *)avl_insert_lock(&((st)->dimensions_index), (avl *)(rd)) -#define rrddim_index_del(st,rd ) (RRDDIM *)avl_remove_lock(&((st)->dimensions_index), (avl *)(rd)) +#define rrddim_index_add(st, rd) (RRDDIM *)avl_insert_lock(&((st)->dimensions_index), (avl_t *)(rd)) +#define rrddim_index_del(st,rd ) (RRDDIM *)avl_remove_lock(&((st)->dimensions_index), (avl_t *)(rd)) static inline RRDDIM *rrddim_index_find(RRDSET *st, const char *id, uint32_t hash) { RRDDIM tmp = { .id = id, .hash = (hash)?hash:simple_hash(id) }; - return (RRDDIM *)avl_search_lock(&(st->dimensions_index), (avl *) &tmp); + return (RRDDIM *)avl_search_lock(&(st->dimensions_index), (avl_t *) &tmp); } @@ -197,7 +197,7 @@ void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) { RRDCALC *child = rrdcalc_create_from_rrdcalc(rrdc, host, usename, rd->name); if (child) { rrdcalc_add_to_host(host, child); - RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl *)child); + RRDCALC *rdcmp = (RRDCALC *) avl_insert_lock(&(host)->alarms_idx_health_log,(avl_t *)child); if (rdcmp != child) { error("Cannot insert the alarm index ID %s",child->name); } @@ -232,9 +232,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte rc += rrddim_set_multiplier(st, rd, multiplier); rc += rrddim_set_divisor(st, rd, divisor); if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { -#ifdef ENABLE_DBENGINE store_active_dimension(rd->state->metric_uuid); -#endif rd->state->collect_ops.init(rd); rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED); rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT); @@ -242,14 +240,11 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT); calc_link_to_rrddim(rd); } - // DBENGINE available and activated? -#ifdef ENABLE_DBENGINE - if (likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && unlikely(rc)) { + if (unlikely(rc)) { debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id); (void)sql_store_dimension(rd->state->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor, rd->algorithm); } -#endif rrdset_unlock(st); return rd; } @@ -277,7 +272,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte if(likely(rd)) { // we have a file mapped for rd - memset(&rd->avl, 0, sizeof(avl)); + memset(&rd->avl, 0, sizeof(avl_t)); rd->id = NULL; rd->name = NULL; rd->cache_filename = NULL; @@ -396,7 +391,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte #ifdef ENABLE_DBENGINE uuid_t *dim_uuid = find_dimension_uuid(st, rd); rrdeng_metric_init(rd, dim_uuid); - store_active_dimension(rd->state->metric_uuid); rd->state->collect_ops.init = rrdeng_store_metric_init; rd->state->collect_ops.store_metric = rrdeng_store_metric_next; rd->state->collect_ops.finalize = rrdeng_store_metric_finalize; @@ -408,6 +402,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time; #endif } else { + rd->state->metric_uuid = find_dimension_uuid(st, rd); + if (unlikely(!rd->state->metric_uuid)) + rd->state->metric_uuid = create_dimension_uuid(rd->rrdset, rd); rd->state->collect_ops.init = rrddim_collect_init; rd->state->collect_ops.store_metric = rrddim_collect_store_metric; rd->state->collect_ops.finalize = rrddim_collect_finalize; @@ -418,6 +415,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte rd->state->query_ops.latest_time = rrddim_query_latest_time; rd->state->query_ops.oldest_time = rrddim_query_oldest_time; } + store_active_dimension(rd->state->metric_uuid); rd->state->collect_ops.init(rd); // append this dimension if(!st->dimensions) @@ -425,7 +423,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte else { RRDDIM *td = st->dimensions; - if(td->algorithm != rd->algorithm || abs(td->multiplier) != abs(rd->multiplier) || abs(td->divisor) != abs(rd->divisor)) { + if(td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor)) { if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { #ifdef NETDATA_INTERNAL_CHECKS info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", @@ -476,10 +474,8 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated) if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) { uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd); if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { -#ifdef ENABLE_DBENGINE /* This metric has no data and no references */ delete_dimension_uuid(rd->state->metric_uuid); -#endif } } @@ -503,6 +499,7 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated) error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id); // free(rd->annotations); + freez(rd->state->metric_uuid); RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode; switch(rrd_memory_mode) { @@ -522,11 +519,6 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated) debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name); freez((void *)rd->id); freez(rd->cache_filename); -#ifdef ENABLE_DBENGINE - if (rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - freez(rd->state->metric_uuid); - } -#endif freez(rd->state); freez(rd); break; diff --git a/database/rrdfamily.c b/database/rrdfamily.c index f75f0adc3..3d91c3788 100644 --- a/database/rrdfamily.c +++ b/database/rrdfamily.c @@ -12,15 +12,15 @@ int rrdfamily_compare(void *a, void *b) { else return strcmp(((RRDFAMILY *)a)->family, ((RRDFAMILY *)b)->family); } -#define rrdfamily_index_add(host, rc) (RRDFAMILY *)avl_insert_lock(&((host)->rrdfamily_root_index), (avl *)(rc)) -#define rrdfamily_index_del(host, rc) (RRDFAMILY *)avl_remove_lock(&((host)->rrdfamily_root_index), (avl *)(rc)) +#define rrdfamily_index_add(host, rc) (RRDFAMILY *)avl_insert_lock(&((host)->rrdfamily_root_index), (avl_t *)(rc)) +#define rrdfamily_index_del(host, rc) (RRDFAMILY *)avl_remove_lock(&((host)->rrdfamily_root_index), (avl_t *)(rc)) static RRDFAMILY *rrdfamily_index_find(RRDHOST *host, const char *id, uint32_t hash) { RRDFAMILY tmp; tmp.family = id; tmp.hash_family = (hash)?hash:simple_hash(tmp.family); - return (RRDFAMILY *)avl_search_lock(&(host->rrdfamily_root_index), (avl *) &tmp); + return (RRDFAMILY *)avl_search_lock(&(host->rrdfamily_root_index), (avl_t *) &tmp); } RRDFAMILY *rrdfamily_create(RRDHOST *host, const char *id) { diff --git a/database/rrdhost.c b/database/rrdhost.c index 45c314602..ae49036a8 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -31,7 +31,7 @@ RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash) { strncpyz(tmp.machine_guid, guid, GUID_LEN); tmp.hash_machine_guid = (hash)?hash:simple_hash(tmp.machine_guid); - return (RRDHOST *)avl_search_lock(&(rrdhost_root_index), (avl *) &tmp); + return (RRDHOST *)avl_search_lock(&(rrdhost_root_index), (avl_t *) &tmp); } RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash) { @@ -53,8 +53,8 @@ RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash) { return NULL; } -#define rrdhost_index_add(rrdhost) (RRDHOST *)avl_insert_lock(&(rrdhost_root_index), (avl *)(rrdhost)) -#define rrdhost_index_del(rrdhost) (RRDHOST *)avl_remove_lock(&(rrdhost_root_index), (avl *)(rrdhost)) +#define rrdhost_index_add(rrdhost) (RRDHOST *)avl_insert_lock(&(rrdhost_root_index), (avl_t *)(rrdhost)) +#define rrdhost_index_del(rrdhost) (RRDHOST *)avl_remove_lock(&(rrdhost_root_index), (avl_t *)(rrdhost)) // ---------------------------------------------------------------------------- @@ -298,15 +298,16 @@ RRDHOST *rrdhost_create(const char *hostname, return NULL; } + if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) { + int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags); + if (unlikely(rc)) + error_report("Failed to store machine GUID to the database"); + } + else + error_report("Host machine GUID %s is not valid", host->machine_guid); + if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { #ifdef ENABLE_DBENGINE - if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) { - int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags); - if (unlikely(rc)) - error_report("Failed to store machine GUID to the database"); - } - else - error_report("Host machine GUID %s is not valid", host->machine_guid); char dbenginepath[FILENAME_MAX + 1]; int ret; @@ -335,6 +336,11 @@ RRDHOST *rrdhost_create(const char *hostname, fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform."); #endif } + else { +#ifdef ENABLE_DBENGINE + host->rrdeng_ctx = &multidb_ctx; +#endif + } // ------------------------------------------------------------------------ // link it and add it to the index @@ -582,8 +588,8 @@ RRDHOST *rrdhost_find_or_create( return host; } -inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) { - if(host != protected +inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now) { + if(host != protected_host && host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN) && host->receiver @@ -594,14 +600,14 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t n return 0; } -void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected) { +void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host) { time_t now = now_realtime_sec(); RRDHOST *host; restart_after_removal: rrdhost_foreach_write(host) { - if(rrdhost_should_be_removed(host, protected, now)) { + if(rrdhost_should_be_removed(host, protected_host, now)) { info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid); if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) @@ -629,11 +635,11 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { if (gap_when_lost_iterations_above < 1) gap_when_lost_iterations_above = 1; -#ifdef ENABLE_DBENGINE if (unlikely(sql_init_database())) { - return 1; + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + return 1; + info("Skipping SQLITE metadata initialization since memory mode is not db engine"); } -#endif health_init(); diff --git a/database/rrdset.c b/database/rrdset.c index d16fe737c..15640d3ed 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -35,7 +35,7 @@ static RRDSET *rrdset_index_find(RRDHOST *host, const char *id, uint32_t hash) { strncpyz(tmp.id, id, RRD_ID_LENGTH_MAX); tmp.hash = (hash)?hash:simple_hash(tmp.id); - return (RRDSET *)avl_search_lock(&(host->rrdset_root_index), (avl *) &tmp); + return (RRDSET *)avl_search_lock(&(host->rrdset_root_index), (avl_t *) &tmp); } // ---------------------------------------------------------------------------- @@ -57,7 +57,7 @@ int rrdset_compare_name(void* a, void* b) { RRDSET *rrdset_index_add_name(RRDHOST *host, RRDSET *st) { void *result; // fprintf(stderr, "ADDING: %s (name: %s)\n", st->id, st->name); - result = avl_insert_lock(&host->rrdset_root_index_name, (avl *) (&st->avlname)); + result = avl_insert_lock(&host->rrdset_root_index_name, (avl_t *) (&st->avlname)); if(result) return rrdset_from_avlname(result); return NULL; } @@ -65,7 +65,7 @@ RRDSET *rrdset_index_add_name(RRDHOST *host, RRDSET *st) { RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st) { void *result; // fprintf(stderr, "DELETING: %s (name: %s)\n", st->id, st->name); - result = (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index_name), (avl *)(&st->avlname)); + result = (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index_name), (avl_t *)(&st->avlname)); if(result) return rrdset_from_avlname(result); return NULL; } @@ -81,7 +81,7 @@ static inline RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name, ui tmp.hash_name = (hash)?hash:simple_hash(tmp.name); // fprintf(stderr, "SEARCHING: %s\n", name); - result = avl_search_lock(&host->rrdset_root_index_name, (avl *) (&(tmp.avlname))); + result = avl_search_lock(&host->rrdset_root_index_name, (avl_t *) (&(tmp.avlname))); if(result) { RRDSET *st = rrdset_from_avlname(result); if(strcmp(st->magic, RRDSET_MAGIC) != 0) @@ -219,11 +219,11 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) { rrdset_flag_clear(st, RRDSET_FLAG_HOMOGENEOUS_CHECK); RRD_ALGORITHM algorithm = st->dimensions->algorithm; - collected_number multiplier = abs(st->dimensions->multiplier); - collected_number divisor = abs(st->dimensions->divisor); + collected_number multiplier = ABS(st->dimensions->multiplier); + collected_number divisor = ABS(st->dimensions->divisor); rrddim_foreach_read(rd, st) { - if(algorithm != rd->algorithm || multiplier != abs(rd->multiplier) || divisor != abs(rd->divisor)) { + if(algorithm != rd->algorithm || multiplier != ABS(rd->multiplier) || divisor != ABS(rd->divisor)) { if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) { #ifdef NETDATA_INTERNAL_CHECKS info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").", @@ -385,6 +385,7 @@ void rrdset_free(RRDSET *st) { freez(st->state->old_context); free_label_list(st->state->labels.head); freez(st->state); + freez(st->chart_uuid); switch(st->rrd_memory_mode) { case RRD_MEMORY_MODE_SAVE: @@ -397,10 +398,6 @@ void rrdset_free(RRDSET *st) { case RRD_MEMORY_MODE_ALLOC: case RRD_MEMORY_MODE_NONE: case RRD_MEMORY_MODE_DBENGINE: -#ifdef ENABLE_DBENGINE - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - freez(st->chart_uuid); -#endif freez(st); break; } @@ -660,15 +657,12 @@ RRDSET *rrdset_create_custom( sched_yield(); } } -#ifdef ENABLE_DBENGINE - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && - (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED))) { + if (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED)) { debug(D_METADATALOG, "CHART [%s] metadata updated", st->id); int rc = update_chart_metadata(st->chart_uuid, st, id, name); if (unlikely(rc)) error_report("Failed to update chart metadata in the database"); } -#endif /* Fall-through during switch from archived to active so that the host lock is taken and health is linked */ if (!changed_from_archived_to_active) return st; @@ -744,8 +738,8 @@ RRDSET *rrdset_create_custom( ); if(st) { - memset(&st->avl, 0, sizeof(avl)); - memset(&st->avlname, 0, sizeof(avl)); + memset(&st->avl, 0, sizeof(avl_t)); + memset(&st->avlname, 0, sizeof(avl_t)); memset(&st->rrdvar_root_index, 0, sizeof(avl_tree_lock)); memset(&st->dimensions_index, 0, sizeof(avl_tree_lock)); memset(&st->rrdset_rwlock, 0, sizeof(netdata_rwlock_t)); @@ -925,15 +919,14 @@ RRDSET *rrdset_create_custom( rrdsetcalc_link_matching(st); rrdcalctemplate_link_matching(st); -#ifdef ENABLE_DBENGINE - if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - st->chart_uuid = find_chart_uuid(host, type, id, name); - if (unlikely(!st->chart_uuid)) - st->chart_uuid = create_chart_uuid(st, id, name); - store_active_chart(st->chart_uuid); - } -#endif + st->chart_uuid = find_chart_uuid(host, type, id, name); + if (unlikely(!st->chart_uuid)) + st->chart_uuid = create_chart_uuid(st, id, name); + else + update_chart_metadata(st->chart_uuid, st, id, name); + + store_active_chart(st->chart_uuid); rrdhost_cleanup_obsolete_charts(host); @@ -1932,6 +1925,15 @@ void rrdset_finalize_labels(RRDSET *st) } else { replace_label_list(labels, new_labels); } + + netdata_rwlock_wrlock(&labels->labels_rwlock); + struct label *lbl = labels->head; + while (lbl) { + sql_store_chart_label(st->chart_uuid, (int)lbl->label_source, lbl->key, lbl->value); + lbl = lbl->next; + } + netdata_rwlock_unlock(&labels->labels_rwlock); + st->state->new_labels = NULL; } diff --git a/database/rrdvar.c b/database/rrdvar.c index 6b824d0d3..25b8ca69e 100644 --- a/database/rrdvar.c +++ b/database/rrdvar.c @@ -27,7 +27,7 @@ int rrdvar_compare(void* a, void* b) { } static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) { - RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl *)(rv)); + RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl_t *)(rv)); if(ret != rv) debug(D_VARIABLES, "Request to insert RRDVAR '%s' into index failed. Already exists.", rv->name); @@ -35,7 +35,7 @@ static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) { } static inline RRDVAR *rrdvar_index_del(avl_tree_lock *tree, RRDVAR *rv) { - RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl *)(rv)); + RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl_t *)(rv)); if(!ret) error("Request to remove RRDVAR '%s' from index failed. Not Found.", rv->name); @@ -47,7 +47,7 @@ static inline RRDVAR *rrdvar_index_find(avl_tree_lock *tree, const char *name, u tmp.name = (char *)name; tmp.hash = (hash)?hash:simple_hash(tmp.name); - return (RRDVAR *)avl_search_lock(tree, (avl *)&tmp); + return (RRDVAR *)avl_search_lock(tree, (avl_t *)&tmp); } inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) { diff --git a/database/rrdvar.h b/database/rrdvar.h index 6d1461b2a..ec6e80a43 100644 --- a/database/rrdvar.h +++ b/database/rrdvar.h @@ -32,7 +32,7 @@ typedef enum rrdvar_options { // 2. at each context (RRDFAMILY.rrdvar_root_index) // 3. at each host (RRDHOST.rrdvar_root_index) struct rrdvar { - avl avl; + avl_t avl; char *name; uint32_t hash; diff --git a/database/sqlite/Makefile.am b/database/sqlite/Makefile.am new file mode 100644 index 000000000..babdcf0df --- /dev/null +++ b/database/sqlite/Makefile.am @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index ab6c59cfa..694b86330 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -17,12 +17,14 @@ const char *database_config[] = { "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int);", "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);", "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);", + "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, " + "label_value text, date_created int, PRIMARY KEY (chart_id, label_key));", "delete from chart_active;", "delete from dimension_active;", - "delete from chart where chart_id not in (select chart_id from dimension);", "delete from host where host_id not in (select host_id from chart);", + "delete from chart_label where chart_id not in (select chart_id from chart);", NULL }; @@ -46,6 +48,31 @@ static int execute_insert(sqlite3_stmt *res) return rc; } +#define MAX_OPEN_STATEMENTS (512) + +static void add_stmt_to_list(sqlite3_stmt *res) +{ + static int idx = 0; + static sqlite3_stmt *statements[MAX_OPEN_STATEMENTS]; + + if (unlikely(!res)) { + while (idx > 0) + sqlite3_finalize(statements[--idx]); + return; + } + + if (unlikely(idx == MAX_OPEN_STATEMENTS)) + return; + statements[idx++] = res; +} + +static int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement) { + int rc = sqlite3_prepare_v2(database, query, -1, statement, 0); + if (likely(rc == SQLITE_OK)) + add_stmt_to_list(*statement); + return rc; +} + /* * Store a chart or dimension UUID in chart_active or dimension_active * The statement that will be prepared determines that @@ -82,7 +109,8 @@ void store_active_chart(uuid_t *chart_uuid) int rc; if (unlikely(!db_meta)) { - error_report("Database has not been initialized"); + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); return; } @@ -109,7 +137,8 @@ void store_active_dimension(uuid_t *dimension_uuid) int rc; if (unlikely(!db_meta)) { - error_report("Database has not been initialized"); + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); return; } @@ -141,7 +170,9 @@ int sql_init_database(void) snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir); rc = sqlite3_open(sqlite_database, &db_meta); if (rc != SQLITE_OK) { - error_report("Failed to initialize database at %s", sqlite_database); + error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc)); + sqlite3_close(db_meta); + db_meta = NULL; return 1; } @@ -172,9 +203,12 @@ void sql_close_database(void) return; info("Closing SQLite database"); - rc = sqlite3_close(db_meta); + + add_stmt_to_list(NULL); + + rc = sqlite3_close_v2(db_meta); if (unlikely(rc != SQLITE_OK)) - error_report("Error %d while closing the SQLite database", rc); + error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc)); return; } @@ -187,7 +221,7 @@ int find_uuid_type(uuid_t *uuid) int uuid_type = 3; if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, FIND_UUID_TYPE, -1, &res, 0); + rc = prepare_statement(db_meta, FIND_UUID_TYPE, &res); if (rc != SQLITE_OK) { error_report("Failed to bind prepare statement to find UUID type in the database"); return 0; @@ -218,8 +252,11 @@ uuid_t *find_dimension_uuid(RRDSET *st, RRDDIM *rd) uuid_t *uuid = NULL; int rc; + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return NULL; + if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_FIND_DIMENSION_UUID, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_FIND_DIMENSION_UUID, &res); if (rc != SQLITE_OK) { error_report("Failed to bind prepare statement to lookup dimension UUID in the database"); return NULL; @@ -299,7 +336,7 @@ void delete_dimension_uuid(uuid_t *dimension_uuid) #endif if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, DELETE_DIMENSION_UUID, -1, &res, 0); + rc = prepare_statement(db_meta, DELETE_DIMENSION_UUID, &res); if (rc != SQLITE_OK) { error_report("Failed to prepare statement to delete a dimension uuid"); return; @@ -331,8 +368,11 @@ uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const c uuid_t *uuid = NULL; int rc; + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return NULL; + if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_FIND_CHART_UUID, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_FIND_CHART_UUID, &res); if (rc != SQLITE_OK) { error_report("Failed to prepare statement to lookup chart UUID in the database"); return NULL; @@ -388,6 +428,9 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const { int rc; + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; + rc = sql_store_chart( chart_uuid, &st->rrdhost->host_uuid, st->type, id, name, st->family, st->context, st->title, st->units, st->plugin_name, st->module_name, st->priority, st->update_every, st->chart_type, st->rrd_memory_mode, st->entries); @@ -427,12 +470,14 @@ int sql_store_host( int rc; if (unlikely(!db_meta)) { + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; error_report("Database has not been initialized"); return 1; } if (unlikely((!res))) { - rc = sqlite3_prepare_v2(db_meta, SQL_STORE_HOST, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_STORE_HOST, &res); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to prepare statement to store host, rc = %d", rc); return 1; @@ -493,16 +538,18 @@ int sql_store_chart( const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every, int chart_type, int memory_mode, long history_entries) { - static __thread sqlite3_stmt *res; + static __thread sqlite3_stmt *res = NULL; int rc, param = 0; if (unlikely(!db_meta)) { + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; error_report("Database has not been initialized"); return 1; } if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_STORE_CHART, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_STORE_CHART, &res); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to prepare statement to store chart, rc = %d", rc); return 1; @@ -530,11 +577,12 @@ int sql_store_chart( goto bind_fail; param++; - if (name) { + if (name && *name) rc = sqlite3_bind_text(res, 5, name, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - } + else + rc = sqlite3_bind_null(res, 5); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; param++; rc = sqlite3_bind_text(res, 6, family, -1, SQLITE_STATIC); @@ -620,12 +668,14 @@ int sql_store_dimension( int rc; if (unlikely(!db_meta)) { + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; error_report("Database has not been initialized"); return 1; } if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_STORE_DIMENSION, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to prepare statement to store dimension, rc = %d", rc); return 1; @@ -733,7 +783,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb) rc = sqlite3_bind_blob(res_chart, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind host parameter to fetch archived charts"); - return; + goto failed; } rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION, -1, &res_dim, 0); @@ -883,25 +933,41 @@ failed: return; } -#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname;" +#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;" +#define SELECT_HOST_BY_UUID "select host_id, registry_hostname, update_every, os, timezone, tags from host where host_id = @host_id ;" RRDHOST *sql_create_host_by_uuid(char *hostname) { int rc; RRDHOST *host = NULL; + uuid_t host_uuid; sqlite3_stmt *res = NULL; - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to fetch host"); - return NULL; + rc = uuid_parse(hostname, host_uuid); + if (!rc) { + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_BY_UUID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host by uuid"); + return NULL; + } + rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to fetch host information"); + goto failed; + } } - - rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind hostname parameter to fetch host information"); - return NULL; + else { + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host by hostname"); + return NULL; + } + rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind hostname parameter to fetch host information"); + goto failed; + } } rc = sqlite3_step(res); @@ -916,13 +982,17 @@ RRDHOST *sql_create_host_by_uuid(char *hostname) host = callocz(1, sizeof(RRDHOST)); set_host_properties(host, sqlite3_column_int(res, 2), RRD_MEMORY_MODE_DBENGINE, hostname, - (char *) sqlite3_column_text(res, 1), (const char *) uuid_str, + (char *) sqlite3_column_text(res, 1), (const char *) uuid_str, (char *) sqlite3_column_text(res, 3), (char *) sqlite3_column_text(res, 5), (char *) sqlite3_column_text(res, 4), NULL, NULL); uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0))); - host->system_info = NULL; + host->system_info = callocz(1, sizeof(*host->system_info));; + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); +#ifdef ENABLE_DBENGINE + host->rrdeng_ctx = &multidb_ctx; +#endif failed: rc = sqlite3_finalize(res); @@ -1020,3 +1090,251 @@ void add_migrated_file(char *path, uint64_t file_size) return; } + +#define SQL_INS_CHART_LABEL "insert or replace into chart_label " \ + "(chart_id, source_type, label_key, label_value, date_created) " \ + "values (@chart, @source, @label, @value, strftime('%s'));" + +void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_INS_CHART_LABEL, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement store chart labels"); + return; + } + + rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind chart_id parameter to store label information"); + goto failed; + } + + rc = sqlite3_bind_int(res, 2, source_type); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind type parameter to store label information"); + goto failed; + } + + rc = sqlite3_bind_text(res, 3, label, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind label parameter to store label information"); + goto failed; + } + + rc = sqlite3_bind_text(res, 4, value, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind value parameter to store label information"); + goto failed; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to store chart label entry, rc = %d", rc); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when storing chart label information"); + + return; +} + +int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id, + uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid) +{ +#ifdef ENABLE_DBENGINE + int rc; + uuid_t legacy_uuid; + uuid_t multihost_legacy_uuid; + time_t dim_first_entry_t, dim_last_entry_t; + + rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t); + if (unlikely(rc)) { + rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid); + rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t); + if (likely(rc)) { + rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid); + rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t); + if (likely(!rc)) + uuid_copy(*rrdeng_uuid, multihost_legacy_uuid); + } + else + uuid_copy(*rrdeng_uuid, legacy_uuid); + } + else + uuid_copy(*rrdeng_uuid, *uuid); + + if (likely(!rc)) { + *first_entry_t = MIN(*first_entry_t, dim_first_entry_t); + *last_entry_t = MAX(*last_entry_t, dim_last_entry_t); + } + return rc; +#else + UNUSED(machine_guid); + UNUSED(chart_id); + UNUSED(dim_id); + UNUSED(uuid); + UNUSED(first_entry_t); + UNUSED(last_entry_t); + UNUSED(rrdeng_uuid); + return 1; +#endif +} + +#ifdef ENABLE_DBENGINE +static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid) +{ + RRDDIM *rd = callocz(1, sizeof(*rd)); + rd->rrdset = st; + rd->last_stored_value = NAN; + rrddim_flag_set(rd, RRDDIM_FLAG_NONE); + rd->state = mallocz(sizeof(*rd->state)); + rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + rd->state->query_ops.init = rrdeng_load_metric_init; + rd->state->query_ops.next_metric = rrdeng_load_metric_next; + rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished; + rd->state->query_ops.finalize = rrdeng_load_metric_finalize; + rd->state->query_ops.latest_time = rrdeng_metric_latest_time; + rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time; + rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t)); + uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid); + rd->state->metric_uuid = rd->state->rrdeng_uuid; + rd->id = strdupz(id); + rd->name = strdupz(name); + return rd; +} +#endif + +#define SELECT_CHART_CONTEXT "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id from chart c, " \ + "dimension d, host h " \ + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.context = @context " \ + "order by c.chart_id asc, c.type||c.id desc;" + +#define SELECT_CHART_SINGLE "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, c.context from chart c, " \ + "dimension d, host h " \ + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \ + "order by c.chart_id asc, c.type||'.'||c.id desc;" + +void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart) +{ +#ifdef ENABLE_DBENGINE + int rc; + + if (unlikely(!param_list) || host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return; + + if (unlikely(!(*param_list))) { + *param_list = mallocz(sizeof(struct context_param)); + (*param_list)->first_entry_t = LONG_MAX; + (*param_list)->last_entry_t = 0; + (*param_list)->rd = NULL; + (*param_list)->flags = CONTEXT_FLAGS_ARCHIVE; + if (chart) + (*param_list)->flags |= CONTEXT_FLAGS_CHART; + else + (*param_list)->flags |= CONTEXT_FLAGS_CONTEXT; + } + + sqlite3_stmt *res = NULL; + + if (context) + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_CONTEXT, -1, &res, 0); + else + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_SINGLE, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host archived charts"); + return; + } + + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter to fetch archived charts"); + goto failed; + } + + if (context) + rc = sqlite3_bind_text(res, 2, context, -1, SQLITE_STATIC); + else + rc = sqlite3_bind_text(res, 2, chart, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter to fetch archived charts"); + goto failed; + } + + RRDSET *st = NULL; + char machine_guid[GUID_LEN + 1]; + uuid_unparse_lower(host->host_uuid, machine_guid); + uuid_t rrdeng_uuid; + uuid_t chart_id; + + while (sqlite3_step(res) == SQLITE_ROW) { + char id[512]; + sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1)); + + if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) { + if (unlikely(st && !st->counter)) { + freez(st->context); + freez((char *) st->name); + freez(st); + } + st = callocz(1, sizeof(*st)); + char n[RRD_ID_LENGTH_MAX + 1]; + + snprintfz( + n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4), + (char *)sqlite3_column_text(res, 3)); + st->name = strdupz(n); + st->update_every = sqlite3_column_int(res, 6); + st->counter = 0; + if (chart) { + st->context = strdupz((char *)sqlite3_column_text(res, 8)); + strncpyz(st->id, chart, RRD_ID_LENGTH_MAX); + } + uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7)); + st->last_entry_t = 0; + st->rrdhost = host; + } + + if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1), + (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t, + &rrdeng_uuid))) + continue; + + st->counter++; + st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t); + + RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid); + rd->next = (*param_list)->rd; + (*param_list)->rd = rd; + } + if (st) { + if (!st->counter) { + freez(st->context); + freez((char *)st->name); + freez(st); + } + else + if (!st->context && context) + st->context = strdupz(context); + } + +failed: + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading archived charts"); +#else + UNUSED(param_list); + UNUSED(host); + UNUSED(context); + UNUSED(chart); +#endif + return; +} diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h index f0b4b775e..d2bee75d2 100644 --- a/database/sqlite/sqlite_functions.h +++ b/database/sqlite/sqlite_functions.h @@ -58,5 +58,6 @@ extern void add_migrated_file(char *path, uint64_t file_size); extern void db_unlock(void); extern void db_lock(void); extern void delete_dimension_uuid(uuid_t *dimension_uuid); - +extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value); +extern void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart); #endif //NETDATA_SQLITE_FUNCTIONS_H -- cgit v1.2.3