diff options
Diffstat (limited to '')
| -rw-r--r-- | database/rrdhost.c | 783 |
1 files changed, 416 insertions, 367 deletions
diff --git a/database/rrdhost.c b/database/rrdhost.c index cb56bf353..7f4bd95ba 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -3,6 +3,26 @@ #define NETDATA_RRD_INTERNALS #include "rrd.h" +int storage_tiers = 1; +int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 }; +RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW }; + +#if RRD_STORAGE_TIERS != 5 +#error RRD_STORAGE_TIERS is not 5 - you need to update the grouping iterations per tier +#endif + +int get_tier_grouping(int tier) { + if(unlikely(tier >= storage_tiers)) tier = storage_tiers - 1; + if(unlikely(tier < 0)) tier = 0; + + int grouping = 1; + // first tier is always 1 iteration of whatever update every the chart has + for(int i = 1; i <= tier ;i++) + grouping *= storage_tiers_grouping_iterations[i]; + + return grouping; +} + RRDHOST *localhost = NULL; size_t rrd_hosts_available = 0; netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER; @@ -10,6 +30,18 @@ netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER; time_t rrdset_free_obsolete_time = 3600; time_t rrdhost_free_orphan_time = 3600; +bool is_storage_engine_shared(STORAGE_INSTANCE *engine) { +#ifdef ENABLE_DBENGINE + for(int tier = 0; tier < storage_tiers ;tier++) { + if (engine == (STORAGE_INSTANCE *)multidb_ctx[tier]) + return true; + } +#endif + + return false; +} + + // ---------------------------------------------------------------------------- // RRDHOST index @@ -155,7 +187,8 @@ RRDHOST *rrdhost_create(const char *hostname, char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info, - int is_localhost + int is_localhost, + bool archived ) { debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid); @@ -175,8 +208,7 @@ RRDHOST *rrdhost_create(const char *hostname, host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries); host->health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled; - host->sender = mallocz(sizeof(*host->sender)); - sender_init(host->sender, host); + sender_init(host); netdata_mutex_init(&host->receiver_lock); host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key) ? 1 : 0; @@ -199,7 +231,7 @@ RRDHOST *rrdhost_create(const char *hostname, #endif netdata_rwlock_init(&host->rrdhost_rwlock); - netdata_rwlock_init(&host->labels.labels_rwlock); + host->host_labels = rrdlabels_create(); netdata_mutex_init(&host->aclk_state_lock); @@ -210,10 +242,10 @@ RRDHOST *rrdhost_create(const char *hostname, avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare); avl_init_lock(&(host->rrdvar_root_index), rrdvar_compare); - if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1)) + if(config_get_boolean(CONFIG_SECTION_DB, "delete obsolete charts files", 1)) rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS); - if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost) + if(config_get_boolean(CONFIG_SECTION_DB, "delete orphan hosts files", 1) && !is_localhost) rrdhost_flag_set(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST); host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never"); @@ -303,18 +335,21 @@ RRDHOST *rrdhost_create(const char *hostname, if(t != host) { error("Host '%s': cannot add host with machine guid '%s' to index. It already exists as host '%s' with machine guid '%s'.", host->hostname, host->machine_guid, t->hostname, t->machine_guid); - rrdhost_free(host); + rrdhost_free(host, 1); return NULL; } if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) { - int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags); - if (unlikely(rc)) - error_report("Failed to store machine GUID to the database"); + int rc; + if (!archived) { + rc = sql_store_host_info(host); + if (unlikely(rc)) + error_report("Failed to store machine GUID to the database"); + } sql_load_node_id(host); if (host->health_enabled) { if (!file_is_migrated(host->health_log_filename)) { - int rc = sql_create_health_log_table(host); + rc = sql_create_health_log_table(host); if (unlikely(rc)) { error_report("Failed to create health log table in the database"); health_alarm_log_load(host); @@ -343,16 +378,33 @@ RRDHOST *rrdhost_create(const char *hostname, if (ret != 0 && errno != EEXIST) error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath); else ret = 0; // succeed - if (is_legacy) // initialize legacy dbengine instance as needed - ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, default_rrdeng_page_cache_mb, - default_rrdeng_disk_quota_mb); // may fail here for legacy dbengine initialization - else - host->rrdeng_ctx = &multidb_ctx; + if (is_legacy) { + // initialize legacy dbengine instance as needed + + ret = rrdeng_init( + host, + (struct rrdengine_instance **)&host->storage_instance[0], + dbenginepath, + default_rrdeng_page_cache_mb, + default_rrdeng_disk_quota_mb, + 0); // may fail here for legacy dbengine initialization + + if(ret == 0) { + // assign the rest of the shared storage instances to it + // to allow them collect its metrics too + for(int tier = 1; tier < storage_tiers ; tier++) + host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier]; + } + } + else { + for(int tier = 0; tier < storage_tiers ; tier++) + host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier]; + } if (ret) { // check legacy or multihost initialization success error( "Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.", host->hostname, host->machine_guid, host->cache_dir); - rrdhost_free(host); + rrdhost_free(host, 1); host = NULL; //rrd_hosts_available++; //TODO: maybe we want this? @@ -365,7 +417,9 @@ RRDHOST *rrdhost_create(const char *hostname, } else { #ifdef ENABLE_DBENGINE - host->rrdeng_ctx = &multidb_ctx; + // the first tier is reserved for the non-dbengine modes + for(int tier = 1; tier < storage_tiers ; tier++) + host->storage_instance[tier] = (STORAGE_INSTANCE *)multidb_ctx[tier]; #endif } @@ -395,8 +449,6 @@ RRDHOST *rrdhost_create(const char *hostname, host->system_info->mc_version = enable_metric_correlations ? metric_correlations_version : 0; } - ml_new_host(host); - info("Host '%s' (at registry as '%s') with guid '%s' initialized" ", os '%s'" ", timezone '%s'" @@ -435,9 +487,15 @@ RRDHOST *rrdhost_create(const char *hostname, , host->health_default_exec , host->health_default_recipient ); + sql_store_host_system_info(&host->host_uuid, system_info); rrd_hosts_available++; + rrdhost_load_rrdcontext_data(host); + if (!archived) + ml_new_host(host); + else + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); return host; } @@ -474,6 +532,7 @@ void rrdhost_update(RRDHOST *host rrdhost_system_info_free(host->system_info); host->system_info = system_info; + sql_store_host_system_info(&host->host_uuid, system_info); rrdhost_init_os(host, os); rrdhost_init_timezone(host, timezone, abbrev_timezone, utc_offset); @@ -517,6 +576,14 @@ void rrdhost_update(RRDHOST *host if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) { rrdhost_flag_clear(host, RRDHOST_FLAG_ARCHIVED); + + host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key) ? 1 : 0; + host->rrdpush_send_destination = (host->rrdpush_send_enabled)?strdupz(rrdpush_destination):NULL; + if (host->rrdpush_send_destination) + host->destinations = destinations_init(host->rrdpush_send_destination); + host->rrdpush_send_api_key = (host->rrdpush_send_enabled)?strdupz(rrdpush_api_key):NULL; + host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL, SIMPLE_PATTERN_EXACT); + if(host->health_enabled) { int r; char filename[FILENAME_MAX + 1]; @@ -552,6 +619,8 @@ void rrdhost_update(RRDHOST *host } } rrd_hosts_available++; + ml_new_host(host); + rrdhost_load_rrdcontext_data(host); info("Host %s is not in archived mode anymore", host->hostname); } @@ -578,6 +647,7 @@ RRDHOST *rrdhost_find_or_create( , char *rrdpush_api_key , char *rrdpush_send_charts_matching , struct rrdhost_system_info *system_info + , bool archived ) { debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid); @@ -587,7 +657,7 @@ RRDHOST *rrdhost_find_or_create( /* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */ error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); - rrdhost_free(host); + rrdhost_free(host, 1); host = NULL; } if(!host) { @@ -612,6 +682,7 @@ RRDHOST *rrdhost_find_or_create( , rrdpush_send_charts_matching , system_info , 0 + , archived ); } else { @@ -651,6 +722,7 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim if(host != protected_host && host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN) + && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) && !host->receiver && host->senders_disconnected_time && host->senders_disconnected_time + rrdhost_free_orphan_time < now) @@ -672,14 +744,14 @@ restart_after_removal: if (rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) #ifdef ENABLE_DBENGINE /* don't delete multi-host DB host files */ - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx) + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0])) #endif ) rrdhost_delete_charts(host); else rrdhost_save_charts(host); - rrdhost_free(host); + rrdhost_free(host, 0); goto restart_after_removal; } } @@ -689,80 +761,177 @@ restart_after_removal: // RRDHOST global / startup initialization int rrd_init(char *hostname, struct rrdhost_system_info *system_info) { - rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", rrdset_free_obsolete_time); - // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short - // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at - // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information. - if (rrdset_free_obsolete_time < 10) { - rrdset_free_obsolete_time = 10; - info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds. A lower delay can potentially cause a segmentation fault."); - } - gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", gap_when_lost_iterations_above); - if (gap_when_lost_iterations_above < 1) - gap_when_lost_iterations_above = 1; - - if (unlikely(sql_init_database(DB_CHECK_NONE, 0))) { + + if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) fatal("Failed to initialize SQLite"); - info("Skipping SQLITE metadata initialization since memory mode is not db engine"); + info("Skipping SQLITE metadata initialization since memory mode is not dbengine"); } - health_init(); + if (unlikely(sql_init_context_database(system_info ? 0 : 1))) { + error_report("Failed to initialize context metadata database"); + } + + if (unlikely(!system_info)) + goto unittest; + +#ifdef ENABLE_DBENGINE + storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers); + if(storage_tiers < 1) { + error("At least 1 storage tier is required. Assuming 1."); + storage_tiers = 1; + config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers); + } + if(storage_tiers > RRD_STORAGE_TIERS) { + error("Up to %d storage tier are supported. Assuming %d.", RRD_STORAGE_TIERS, RRD_STORAGE_TIERS); + storage_tiers = RRD_STORAGE_TIERS; + config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers); + } + + default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT); + if (default_rrdeng_page_fetch_timeout < 1) { + info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout); + default_rrdeng_page_fetch_timeout = 1; + config_set_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", default_rrdeng_page_fetch_timeout); + } + + default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES); + if (default_rrdeng_page_fetch_retries < 1) { + info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries); + default_rrdeng_page_fetch_retries = 1; + config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries); + } + + if(config_get_boolean(CONFIG_SECTION_DB, "dbengine page descriptors in file mapped memory", rrdeng_page_descr_is_mmap()) == CONFIG_BOOLEAN_YES) + rrdeng_page_descr_use_mmap(); + else + rrdeng_page_descr_use_malloc(); + int created_tiers = 0; + char dbenginepath[FILENAME_MAX + 1]; + char dbengineconfig[200 + 1]; + for(int tier = 0; tier < storage_tiers ;tier++) { + if(tier == 0) + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir); + else + snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%d", netdata_configured_cache_dir, tier); + + int ret = mkdir(dbenginepath, 0775); + if (ret != 0 && errno != EEXIST) { + error("DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath); + break; + } + + int page_cache_mb = default_rrdeng_page_cache_mb; + int disk_space_mb = default_multidb_disk_quota_mb; + int grouping_iterations = storage_tiers_grouping_iterations[tier]; + RRD_BACKFILL backfill = storage_tiers_backfill[tier]; + + if(tier > 0) { + snprintfz(dbengineconfig, 200, "dbengine tier %d page cache size MB", tier); + page_cache_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, page_cache_mb); + + snprintfz(dbengineconfig, 200, "dbengine tier %d multihost disk space MB", tier); + disk_space_mb = config_get_number(CONFIG_SECTION_DB, dbengineconfig, disk_space_mb); + + snprintfz(dbengineconfig, 200, "dbengine tier %d update every iterations", tier); + grouping_iterations = config_get_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations); + if(grouping_iterations < 2) { + grouping_iterations = 2; + config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations); + error("DBENGINE on '%s': 'dbegnine tier %d update every iterations' cannot be less than 2. Assuming 2.", hostname, tier); + } + + snprintfz(dbengineconfig, 200, "dbengine tier %d backfill", tier); + const char *bf = config_get(CONFIG_SECTION_DB, dbengineconfig, backfill == RRD_BACKFILL_NEW ? "new" : backfill == RRD_BACKFILL_FULL ? "full" : "none"); + if(strcmp(bf, "new") == 0) backfill = RRD_BACKFILL_NEW; + else if(strcmp(bf, "full") == 0) backfill = RRD_BACKFILL_FULL; + else if(strcmp(bf, "none") == 0) backfill = RRD_BACKFILL_NONE; + else { + error("DBENGINE: unknown backfill value '%s', assuming 'new'", bf); + config_set(CONFIG_SECTION_DB, dbengineconfig, "new"); + backfill = RRD_BACKFILL_NEW; + } + } + + storage_tiers_grouping_iterations[tier] = grouping_iterations; + storage_tiers_backfill[tier] = backfill; + + if(tier > 0 && get_tier_grouping(tier) > 65535) { + storage_tiers_grouping_iterations[tier] = 1; + error("DBENGINE on '%s': dbengine tier %d gives aggregation of more than 65535 points of tier 0. Disabling tiers above %d", hostname, tier, tier); + break; + } + + internal_error(true, "DBENGINE tier %d grouping iterations is set to %d", tier, storage_tiers_grouping_iterations[tier]); + ret = rrdeng_init(NULL, NULL, dbenginepath, page_cache_mb, disk_space_mb, tier); + if(ret != 0) { + error("DBENGINE on '%s': Failed to initialize multi-host database tier %d on path '%s'", + hostname, tier, dbenginepath); + break; + } + else + created_tiers++; + } + + if(created_tiers && created_tiers < storage_tiers) { + error("DBENGINE on '%s': Managed to create %d tiers instead of %d. Continuing with %d available.", + hostname, created_tiers, storage_tiers, created_tiers); + storage_tiers = created_tiers; + } + else if(!created_tiers) + fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir); + +#else + storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1); + if(storage_tiers != 1) { + error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", hostname); + storage_tiers = 1; + config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers); + } +#endif + + health_init(); rrdpush_init(); +unittest: debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname); rrd_wrlock(); localhost = rrdhost_create( - hostname - , registry_get_this_machine_hostname() + hostname + , registry_get_this_machine_hostname() , registry_get_this_machine_guid() , os_type - , netdata_configured_timezone - , netdata_configured_abbrev_timezone - , netdata_configured_utc_offset - , "" - , program_name - , program_version - , default_rrd_update_every - , default_rrd_history_entries - , default_rrd_memory_mode - , default_health_enabled - , default_rrdpush_enabled - , default_rrdpush_destination - , default_rrdpush_api_key - , default_rrdpush_send_charts_matching - , system_info - , 1 + , netdata_configured_timezone + , netdata_configured_abbrev_timezone + , netdata_configured_utc_offset + , "" + , program_name + , program_version + , default_rrd_update_every + , default_rrd_history_entries + , default_rrd_memory_mode + , default_health_enabled + , default_rrdpush_enabled + , default_rrdpush_destination + , default_rrdpush_api_key + , default_rrdpush_send_charts_matching + , system_info + , 1 + , 0 ); if (unlikely(!localhost)) { rrd_unlock(); return 1; } -#ifdef ENABLE_DBENGINE - char dbenginepath[FILENAME_MAX + 1]; - int ret; - snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", localhost->cache_dir); - ret = mkdir(dbenginepath, 0775); - if (ret != 0 && errno != EEXIST) - error("Host '%s': cannot create directory '%s'", localhost->hostname, dbenginepath); - else // Unconditionally create multihost db to support on demand host creation - ret = rrdeng_init(NULL, NULL, dbenginepath, default_rrdeng_page_cache_mb, default_multidb_disk_quota_mb); - if (ret) { - error( - "Host '%s' with machine guid '%s' failed to initialize multi-host DB engine instance at '%s'.", - localhost->hostname, localhost->machine_guid, localhost->cache_dir); - rrdhost_free(localhost); - localhost = NULL; - rrd_unlock(); - fatal("Failed to initialize dbengine"); - } -#endif - sql_aclk_sync_init(); rrd_unlock(); - web_client_api_v1_management_init(); + if (likely(system_info)) { + migrate_localhost(&localhost->host_uuid); + sql_aclk_sync_init(); + web_client_api_v1_management_init(); + } return localhost==NULL; } @@ -844,19 +1013,12 @@ void rrdhost_system_info_free(struct rrdhost_system_info *system_info) { } void destroy_receiver_state(struct receiver_state *rpt); -void rrdhost_free(RRDHOST *host) { - if(!host) return; - - info("Freeing all memory for host '%s'...", host->hostname); - rrd_check_wrlock(); // make sure the RRDs are write locked - - rrdhost_wrlock(host); - ml_delete_host(host); - rrdhost_unlock(host); +void stop_streaming_sender(RRDHOST *host) +{ + if (unlikely(!host->sender)) + return; - // ------------------------------------------------------------------------ - // clean up streaming rrdpush_sender_thread_stop(host); // stop a possibly running thread cbuffer_free(host->sender->buffer); buffer_free(host->sender->build); @@ -866,49 +1028,57 @@ void rrdhost_free(RRDHOST *host) { #endif freez(host->sender); host->sender = NULL; - if (netdata_exit) { - netdata_mutex_lock(&host->receiver_lock); - if (host->receiver) { - if (!host->receiver->exited) - netdata_thread_cancel(host->receiver->thread); - netdata_mutex_unlock(&host->receiver_lock); - struct receiver_state *rpt = host->receiver; - while (host->receiver && !rpt->exited) - sleep_usec(50 * USEC_PER_MS); - // If the receiver detached from the host then its thread will destroy the state - if (host->receiver == rpt) - destroy_receiver_state(host->receiver); - } - else - netdata_mutex_unlock(&host->receiver_lock); - } +} + +void stop_streaming_receiver(RRDHOST *host) +{ + netdata_mutex_lock(&host->receiver_lock); + if (host->receiver) { + if (!host->receiver->exited) + netdata_thread_cancel(host->receiver->thread); + netdata_mutex_unlock(&host->receiver_lock); + struct receiver_state *rpt = host->receiver; + while (host->receiver && !rpt->exited) + sleep_usec(50 * USEC_PER_MS); + // If the receiver detached from the host then its thread will destroy the state + if (host->receiver == rpt) + destroy_receiver_state(host->receiver); + } else + netdata_mutex_unlock(&host->receiver_lock); +} + +void rrdhost_free(RRDHOST *host, bool force) { + if(!host) return; + + if (netdata_exit || force) + info("Freeing all memory for host '%s'...", host->hostname); + + rrd_check_wrlock(); // make sure the RRDs are write locked + rrdhost_wrlock(host); + ml_delete_host(host); + rrdhost_unlock(host); + + // ------------------------------------------------------------------------ + // clean up streaming + stop_streaming_sender(host); + if (netdata_exit || force) + stop_streaming_receiver(host); rrdhost_wrlock(host); // lock this RRDHOST -#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL) - struct aclk_database_worker_config *wc = host->dbsync_worker; - if (wc && !netdata_exit) { - struct aclk_database_cmd cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.opcode = ACLK_DATABASE_ORPHAN_HOST; - struct aclk_completion compl ; - init_aclk_completion(&compl ); - cmd.completion = &compl ; - aclk_database_enq_cmd(wc, &cmd); - wait_for_aclk_completion(&compl ); - destroy_aclk_completion(&compl ); - } -#endif // ------------------------------------------------------------------------ // release its children resources #ifdef ENABLE_DBENGINE - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - if (host->rrdeng_ctx != &multidb_ctx) - rrdeng_prepare_exit(host->rrdeng_ctx); + for(int tier = 0; tier < storage_tiers ;tier++) { + if(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && + host->storage_instance[tier] && + !is_storage_engine_shared(host->storage_instance[tier])) + rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[tier]); } #endif + while(host->rrdset_root) rrdset_free(host->rrdset_root); @@ -940,8 +1110,34 @@ void rrdhost_free(RRDHOST *host) { health_alarm_log_free(host); #ifdef ENABLE_DBENGINE - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx != &multidb_ctx) - rrdeng_exit(host->rrdeng_ctx); + for(int tier = 0; tier < storage_tiers ;tier++) { + if(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && + host->storage_instance[tier] && + !is_storage_engine_shared(host->storage_instance[tier])) + rrdeng_exit((struct rrdengine_instance *)host->storage_instance[tier]); + } +#endif + + if (!netdata_exit && !force) { + info("Setting archive mode for host '%s'...", host->hostname); + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); + rrdhost_unlock(host); + return; + } + +#ifdef ENABLE_ACLK + struct aclk_database_worker_config *wc = host->dbsync_worker; + if (wc && !netdata_exit) { + struct aclk_database_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); + cmd.opcode = ACLK_DATABASE_ORPHAN_HOST; + struct aclk_completion compl ; + init_aclk_completion(&compl ); + cmd.completion = &compl ; + aclk_database_enq_cmd(wc, &cmd); + wait_for_aclk_completion(&compl ); + destroy_aclk_completion(&compl ); + } #endif // ------------------------------------------------------------------------ @@ -950,7 +1146,6 @@ void rrdhost_free(RRDHOST *host) { if(rrdhost_index_del(host) != host) error("RRDHOST '%s' removed from index, deleted the wrong entry.", host->hostname); - // ------------------------------------------------------------------------ // unlink it from the host @@ -967,8 +1162,6 @@ void rrdhost_free(RRDHOST *host) { else error("Request to free RRDHOST '%s': cannot find it", host->hostname); } - - // ------------------------------------------------------------------------ // free it @@ -976,7 +1169,7 @@ void rrdhost_free(RRDHOST *host) { freez(host->aclk_state.claimed_id); freez(host->aclk_state.prev_claimed_id); freez((void *)host->tags); - free_label_list(host->labels.head); + rrdlabels_destroy(host->host_labels); freez((void *)host->os); freez((void *)host->timezone); freez((void *)host->abbrev_timezone); @@ -987,6 +1180,12 @@ void rrdhost_free(RRDHOST *host) { freez(host->varlib_dir); freez(host->rrdpush_send_api_key); freez(host->rrdpush_send_destination); + struct rrdpush_destinations *tmp_destination; + while (host->destinations) { + tmp_destination = host->destinations->next; + freez(host->destinations); + host->destinations = tmp_destination; + } freez(host->health_default_exec); freez(host->health_default_recipient); freez(host->health_log_filename); @@ -994,13 +1193,14 @@ void rrdhost_free(RRDHOST *host) { freez(host->registry_hostname); simple_pattern_free(host->rrdpush_send_charts_matching); rrdhost_unlock(host); - netdata_rwlock_destroy(&host->labels.labels_rwlock); netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock); netdata_rwlock_destroy(&host->rrdhost_rwlock); freez(host->node_id); + rrdhost_destroy_rrdcontexts(host); + freez(host); -#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL) +#ifdef ENABLE_ACLK if (wc) wc->is_orphan = 0; #endif @@ -1010,8 +1210,8 @@ void rrdhost_free(RRDHOST *host) { void rrdhost_free_all(void) { rrd_wrlock(); /* Make sure child-hosts are released before the localhost. */ - while(localhost->next) rrdhost_free(localhost->next); - rrdhost_free(localhost); + while(localhost->next) rrdhost_free(localhost->next, 1); + rrdhost_free(localhost, 1); rrd_unlock(); } @@ -1038,297 +1238,139 @@ void rrdhost_save_charts(RRDHOST *host) { rrdhost_unlock(host); } -static struct label *rrdhost_load_auto_labels(void) -{ - struct label *label_list = NULL; +static void rrdhost_load_auto_labels(void) { + DICTIONARY *labels = localhost->host_labels; if (localhost->system_info->cloud_provider_type) - label_list = - add_label_to_list(label_list, "_cloud_provider_type", localhost->system_info->cloud_provider_type, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_cloud_provider_type", localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO); if (localhost->system_info->cloud_instance_type) - label_list = - add_label_to_list(label_list, "_cloud_instance_type", localhost->system_info->cloud_instance_type, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_cloud_instance_type", localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO); if (localhost->system_info->cloud_instance_region) - label_list = - add_label_to_list(label_list, "_cloud_instance_region", localhost->system_info->cloud_instance_region, LABEL_SOURCE_AUTO); + rrdlabels_add( + labels, "_cloud_instance_region", localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO); if (localhost->system_info->host_os_name) - label_list = - add_label_to_list(label_list, "_os_name", localhost->system_info->host_os_name, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_os_name", localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO); if (localhost->system_info->host_os_version) - label_list = - add_label_to_list(label_list, "_os_version", localhost->system_info->host_os_version, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_os_version", localhost->system_info->host_os_version, RRDLABEL_SRC_AUTO); if (localhost->system_info->kernel_version) - label_list = - add_label_to_list(label_list, "_kernel_version", localhost->system_info->kernel_version, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_kernel_version", localhost->system_info->kernel_version, RRDLABEL_SRC_AUTO); if (localhost->system_info->host_cores) - label_list = - add_label_to_list(label_list, "_system_cores", localhost->system_info->host_cores, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_system_cores", localhost->system_info->host_cores, RRDLABEL_SRC_AUTO); if (localhost->system_info->host_cpu_freq) - label_list = - add_label_to_list(label_list, "_system_cpu_freq", localhost->system_info->host_cpu_freq, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_system_cpu_freq", localhost->system_info->host_cpu_freq, RRDLABEL_SRC_AUTO); if (localhost->system_info->host_ram_total) - label_list = - add_label_to_list(label_list, "_system_ram_total", localhost->system_info->host_ram_total, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_system_ram_total", localhost->system_info->host_ram_total, RRDLABEL_SRC_AUTO); if (localhost->system_info->host_disk_space) - label_list = - add_label_to_list(label_list, "_system_disk_space", localhost->system_info->host_disk_space, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_system_disk_space", localhost->system_info->host_disk_space, RRDLABEL_SRC_AUTO); if (localhost->system_info->architecture) - label_list = - add_label_to_list(label_list, "_architecture", localhost->system_info->architecture, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_architecture", localhost->system_info->architecture, RRDLABEL_SRC_AUTO); if (localhost->system_info->virtualization) - label_list = - add_label_to_list(label_list, "_virtualization", localhost->system_info->virtualization, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_virtualization", localhost->system_info->virtualization, RRDLABEL_SRC_AUTO); if (localhost->system_info->container) - label_list = - add_label_to_list(label_list, "_container", localhost->system_info->container, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_container", localhost->system_info->container, RRDLABEL_SRC_AUTO); if (localhost->system_info->container_detection) - label_list = - add_label_to_list(label_list, "_container_detection", localhost->system_info->container_detection, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_container_detection", localhost->system_info->container_detection, RRDLABEL_SRC_AUTO); if (localhost->system_info->virt_detection) - label_list = - add_label_to_list(label_list, "_virt_detection", localhost->system_info->virt_detection, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_virt_detection", localhost->system_info->virt_detection, RRDLABEL_SRC_AUTO); if (localhost->system_info->is_k8s_node) - label_list = - add_label_to_list(label_list, "_is_k8s_node", localhost->system_info->is_k8s_node, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_is_k8s_node", localhost->system_info->is_k8s_node, RRDLABEL_SRC_AUTO); if (localhost->system_info->install_type) - label_list = - add_label_to_list(label_list, "_install_type", localhost->system_info->install_type, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_install_type", localhost->system_info->install_type, RRDLABEL_SRC_AUTO); if (localhost->system_info->prebuilt_arch) - label_list = - add_label_to_list(label_list, "_prebuilt_arch", localhost->system_info->prebuilt_arch, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_prebuilt_arch", localhost->system_info->prebuilt_arch, RRDLABEL_SRC_AUTO); if (localhost->system_info->prebuilt_dist) - label_list = - add_label_to_list(label_list, "_prebuilt_dist", localhost->system_info->prebuilt_dist, LABEL_SOURCE_AUTO); + rrdlabels_add(labels, "_prebuilt_dist", localhost->system_info->prebuilt_dist, RRDLABEL_SRC_AUTO); - label_list = add_aclk_host_labels(label_list); + add_aclk_host_labels(); - label_list = add_label_to_list( - label_list, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", LABEL_SOURCE_AUTO); + rrdlabels_add( + labels, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", RRDLABEL_SRC_AUTO); if (localhost->rrdpush_send_destination) - label_list = - add_label_to_list(label_list, "_streams_to", localhost->rrdpush_send_destination, LABEL_SOURCE_AUTO); - - return label_list; -} - -static inline int rrdhost_is_valid_label_config_option(char *name, char *value) -{ - return (is_valid_label_key(name) && is_valid_label_value(value) && strcmp(name, "from environment") && - strcmp(name, "from kubernetes pods")); + rrdlabels_add(labels, "_streams_to", localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO); } -static struct label *rrdhost_load_config_labels() -{ +static void rrdhost_load_config_labels(void) { int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL); if(!status) { char *filename = CONFIG_DIR "/" CONFIG_FILENAME; - error("LABEL: Cannot reload the configuration file '%s', using labels in memory", filename); + error("RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", filename); } - struct label *l = NULL; struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL); if(co) { config_section_wrlock(co); struct config_option *cv; for(cv = co->values; cv ; cv = cv->next) { - if(rrdhost_is_valid_label_config_option(cv->name, cv->value)) { - l = add_label_to_list(l, cv->name, cv->value, LABEL_SOURCE_NETDATA_CONF); - cv->flags |= CONFIG_VALUE_USED; - } else { - error("LABELS: It was not possible to create the label '%s' because it contains invalid character(s) or values." - , cv->name); - } + rrdlabels_add(localhost->host_labels, cv->name, cv->value, RRDLABEL_SRC_CONFIG); + cv->flags |= CONFIG_VALUE_USED; } config_section_unlock(co); } - - return l; } -struct label *parse_simple_tags( - struct label *label_list, - const char *tags, - char key_value_separator, - char label_separator, - STRIP_QUOTES_OPTION strip_quotes_from_key, - STRIP_QUOTES_OPTION strip_quotes_from_value, - SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters) -{ - const char *end = tags; - - while (*end) { - const char *start = end; - char key[CONFIG_MAX_VALUE + 1]; - char value[CONFIG_MAX_VALUE + 1]; - - while (*end && *end != key_value_separator) - end++; - strncpyz(key, start, end - start); - - if (*end) - start = ++end; - while (*end && *end != label_separator) - end++; - strncpyz(value, start, end - start); - - label_list = add_label_to_list( - label_list, - strip_quotes_from_key ? strip_double_quotes(trim(key), skip_escaped_characters) : trim(key), - strip_quotes_from_value ? strip_double_quotes(trim(value), skip_escaped_characters) : trim(value), - LABEL_SOURCE_NETDATA_CONF); +static void rrdhost_load_kubernetes_labels(void) { + char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)]; + sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh"); - if (*end) - end++; + if (unlikely(access(label_script, R_OK) != 0)) { + error("Kubernetes pod label fetching script %s not found.",label_script); + return; } - return label_list; -} - -struct label *parse_json_tags(struct label *label_list, const char *tags) -{ - char tags_buf[CONFIG_MAX_VALUE + 1]; - strncpy(tags_buf, tags, CONFIG_MAX_VALUE); - char *str = tags_buf; - - switch (*str) { - case '{': - str++; - strip_last_symbol(str, '}', SKIP_ESCAPED_CHARACTERS); + debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script); - label_list = parse_simple_tags(label_list, str, ':', ',', STRIP_QUOTES, STRIP_QUOTES, SKIP_ESCAPED_CHARACTERS); + pid_t pid; + FILE *fp = mypopen(label_script, &pid); + if(!fp) return; - break; - case '[': - str++; - strip_last_symbol(str, ']', SKIP_ESCAPED_CHARACTERS); + char buffer[1000 + 1]; + while (fgets(buffer, 1000, fp) != NULL) + rrdlabels_add_pair(localhost->host_labels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S); - char *end = str + strlen(str); - size_t i = 0; - - while (str < end) { - char key[CONFIG_MAX_VALUE + 1]; - snprintfz(key, CONFIG_MAX_VALUE, "host_tag%zu", i); - - str = strip_double_quotes(trim(str), SKIP_ESCAPED_CHARACTERS); - - label_list = add_label_to_list(label_list, key, str, LABEL_SOURCE_NETDATA_CONF); - - // skip to the next element in the array - str += strlen(str) + 1; - while (*str && *str != ',') - str++; - str++; - i++; - } - - break; - case '"': - label_list = add_label_to_list( - label_list, "host_tag", strip_double_quotes(str, SKIP_ESCAPED_CHARACTERS), LABEL_SOURCE_NETDATA_CONF); - break; - default: - label_list = add_label_to_list(label_list, "host_tag", str, LABEL_SOURCE_NETDATA_CONF); - break; - } - - return label_list; + // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':' + // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null + int rc = mypclose(fp, pid); + if(rc) error("%s exited abnormally. Failed to get kubernetes labels.", label_script); } -static struct label *rrdhost_load_kubernetes_labels(void) -{ - struct label *l=NULL; - char *label_script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)); - sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh"); - if (unlikely(access(label_script, R_OK) != 0)) { - error("Kubernetes pod label fetching script %s not found.",label_script); - freez(label_script); - } else { - pid_t command_pid; - - debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script); - - FILE *fp = mypopen(label_script, &command_pid); - if(fp) { - int MAX_LINE_SIZE=300; - char buffer[MAX_LINE_SIZE + 1]; - while (fgets(buffer, MAX_LINE_SIZE, fp) != NULL) { - char *name=buffer; - char *value=buffer; - while (*value && *value != ':') value++; - if (*value == ':') { - *value = '\0'; - value++; - } - char *eos=value; - while (*eos && *eos != '\n') eos++; - if (*eos == '\n') *eos = '\0'; - if (strlen(value)>0) { - if (is_valid_label_key(name)){ - l = add_label_to_list(l, name, value, LABEL_SOURCE_KUBERNETES); - } else { - info("Ignoring invalid label name '%s'", name); - } - } else { - error("%s outputted unexpected result: '%s'", label_script, name); - } - }; - // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':' - // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null - int retcode=mypclose(fp, command_pid); - if (retcode) { - error("%s exited abnormally. No kubernetes labels will be added to the host.", label_script); - struct label *ll=l; - while (ll != NULL) { - info("Ignoring Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(ll->label_source), ll->key, ll->value); - ll = ll->next; - freez(l); - l=ll; - } - } - } - freez(label_script); - } - - return l; -} +void reload_host_labels(void) { + if(!localhost->host_labels) + localhost->host_labels = rrdlabels_create(); -void reload_host_labels(void) -{ - struct label *from_auto = rrdhost_load_auto_labels(); - struct label *from_k8s = rrdhost_load_kubernetes_labels(); - struct label *from_config = rrdhost_load_config_labels(); + rrdlabels_unmark_all(localhost->host_labels); - struct label *new_labels = merge_label_lists(from_auto, from_k8s); - new_labels = merge_label_lists(new_labels, from_config); + // priority is important here + rrdhost_load_config_labels(); + rrdhost_load_kubernetes_labels(); + rrdhost_load_auto_labels(); - rrdhost_rdlock(localhost); - replace_label_list(&localhost->labels, new_labels); + rrdlabels_remove_all_unmarked(localhost->host_labels); + sql_store_host_labels(localhost); health_label_log_save(localhost); - rrdhost_unlock(localhost); /* TODO-GAPS - fix this so that it looks properly at the state and version of the sender if(localhost->rrdpush_send_enabled && localhost->rrdpush_sender_buffer){ - localhost->labels.labels_flag |= LABEL_FLAG_UPDATE_STREAM; + localhost->labels.labels_flag |= RRDHOST_FLAG_STREAM_LABELS_UPDATE; rrdpush_send_labels(localhost); } */ @@ -1351,7 +1393,7 @@ void rrdhost_delete_charts(RRDHOST *host) { rrdset_foreach_write(st, host) { rrdset_rdlock(st); - rrdset_delete(st); + rrdset_delete_files(st); rrdset_unlock(st); } @@ -1379,7 +1421,7 @@ void rrdhost_cleanup_charts(RRDHOST *host) { rrdset_rdlock(st); if(rrdhost_delete_obsolete_charts && rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)) - rrdset_delete(st); + rrdset_delete_files(st); else if(rrdhost_delete_obsolete_charts && rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) rrdset_delete_obsolete_dimensions(st); else @@ -1420,7 +1462,7 @@ void rrdhost_cleanup_all(void) { if (host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_ORPHAN_HOST) && !host->receiver #ifdef ENABLE_DBENGINE /* don't delete multi-host DB host files */ - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && host->rrdeng_ctx == &multidb_ctx) + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->storage_instance[0])) #endif ) rrdhost_delete_charts(host); @@ -1476,11 +1518,24 @@ restart_after_removal: if (rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE); + /* only a collector can mark a chart as obsolete, so we must remove the reference */ - uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd); - if (can_delete_metric) { + + size_t tiers_available = 0, tiers_said_yes = 0; + for(int tier = 0; tier < storage_tiers ;tier++) { + if(rd->tiers[tier]) { + tiers_available++; + + if(rd->tiers[tier]->collect_ops.finalize(rd->tiers[tier]->db_collection_handle)) + tiers_said_yes++; + + rd->tiers[tier]->db_collection_handle = NULL; + } + } + + if (tiers_available == tiers_said_yes && tiers_said_yes) { /* This metric has no data and no references */ - delete_dimension_uuid(&rd->state->metric_uuid); + delete_dimension_uuid(&rd->metric_uuid); rrddim_free(st, rd); if (unlikely(!last)) { rd = st->dimensions; @@ -1490,7 +1545,7 @@ restart_after_removal: } continue; } -#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL) +#ifdef ENABLE_ACLK else queue_dimension_to_aclk(rd, rd->last_collected_time.tv_sec); #endif @@ -1514,7 +1569,7 @@ restart_after_removal: rrdset_rdlock(st); if(rrdhost_delete_obsolete_charts) - rrdset_delete(st); + rrdset_delete_files(st); else rrdset_save(st); @@ -1523,7 +1578,7 @@ restart_after_removal: rrdset_free(st); goto restart_after_removal; } -#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL) +#ifdef ENABLE_ACLK else sql_check_chart_liveness(st); #endif @@ -1551,21 +1606,15 @@ void rrd_cleanup_obsolete_charts() { if (host->obsolete_charts_count) { rrdhost_wrlock(host); -#ifdef ENABLE_ACLK - host->deleted_charts_count = 0; -#endif rrdhost_cleanup_obsolete_charts(host); -#ifdef ENABLE_ACLK - if (host->deleted_charts_count) - aclk_update_chart(host, "dummy-chart", 0); -#endif rrdhost_unlock(host); } - if (host != localhost && - host->trigger_chart_obsoletion_check && - host->senders_last_chart_command && - host->senders_last_chart_command + 120 < now_realtime_sec()) { + if ( host != localhost && + host->trigger_chart_obsoletion_check && + ((host->senders_last_chart_command && + host->senders_last_chart_command + host->health_delay_up_to < now_realtime_sec()) + || (host->senders_connect_time + 300 < now_realtime_sec())) ) { rrdhost_rdlock(host); rrdset_check_obsoletion(host); rrdhost_unlock(host); |