summaryrefslogtreecommitdiffstats
path: root/database/rrdhost.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--database/rrdhost.c232
1 files changed, 130 insertions, 102 deletions
diff --git a/database/rrdhost.c b/database/rrdhost.c
index 60b14c13c..88e411de8 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -41,6 +41,22 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
return false;
}
+RRDHOST *find_host_by_node_id(char *node_id) {
+ uuid_t node_uuid;
+ if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
+ return NULL;
+
+ RRDHOST *host, *ret = NULL;
+ dfe_start_read(rrdhost_root_index, host) {
+ if (host->node_id && !(uuid_memcmp(host->node_id, &node_uuid))) {
+ ret = host;
+ break;
+ }
+ }
+ dfe_done(host);
+
+ return ret;
+}
// ----------------------------------------------------------------------------
// RRDHOST indexes management
@@ -62,6 +78,26 @@ static inline void rrdhost_init() {
}
}
+RRDHOST_ACQUIRED *rrdhost_find_and_acquire(const char *machine_guid) {
+ debug(D_RRD_CALLS, "rrdhost_find_and_acquire() host %s", machine_guid);
+
+ return (RRDHOST_ACQUIRED *)dictionary_get_and_acquire_item(rrdhost_root_index, machine_guid);
+}
+
+RRDHOST *rrdhost_acquired_to_rrdhost(RRDHOST_ACQUIRED *rha) {
+ if(unlikely(!rha))
+ return NULL;
+
+ return (RRDHOST *) dictionary_acquired_item_value((const DICTIONARY_ITEM *)rha);
+}
+
+void rrdhost_acquired_release(RRDHOST_ACQUIRED *rha) {
+ if(unlikely(!rha))
+ return;
+
+ dictionary_acquired_item_release(rrdhost_root_index, (const DICTIONARY_ITEM *)rha);
+}
+
// ----------------------------------------------------------------------------
// RRDHOST index by UUID
@@ -104,6 +140,17 @@ inline RRDHOST *rrdhost_find_by_hostname(const char *hostname) {
return dictionary_get(rrdhost_root_index_hostname, hostname);
}
+static inline void rrdhost_index_del_hostname(RRDHOST *host) {
+ if(unlikely(!host->hostname)) return;
+
+ if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_HOSTNAME)) {
+ if(!dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)))
+ error("RRDHOST: %s() failed to delete hostname '%s' from index", __FUNCTION__, rrdhost_hostname(host));
+
+ rrdhost_option_clear(host, RRDHOST_OPTION_INDEXED_HOSTNAME);
+ }
+}
+
static inline RRDHOST *rrdhost_index_add_hostname(RRDHOST *host) {
if(!host->hostname) return host;
@@ -111,24 +158,17 @@ static inline RRDHOST *rrdhost_index_add_hostname(RRDHOST *host) {
if(ret_hostname == host)
rrdhost_option_set(host, RRDHOST_OPTION_INDEXED_HOSTNAME);
else {
- rrdhost_option_clear(host, RRDHOST_OPTION_INDEXED_HOSTNAME);
- error("RRDHOST: %s() host with hostname '%s' is already indexed", __FUNCTION__, rrdhost_hostname(host));
+ //have the same hostname but it's not the same host
+ //keep the new one only if the old one is orphan or archived
+ if (rrdhost_flag_check(ret_hostname, RRDHOST_FLAG_ORPHAN) || rrdhost_flag_check(ret_hostname, RRDHOST_FLAG_ARCHIVED)) {
+ rrdhost_index_del_hostname(ret_hostname);
+ rrdhost_index_add_hostname(host);
+ }
}
return host;
}
-static inline void rrdhost_index_del_hostname(RRDHOST *host) {
- if(unlikely(!host->hostname)) return;
-
- if(rrdhost_option_check(host, RRDHOST_OPTION_INDEXED_HOSTNAME)) {
- if(!dictionary_del(rrdhost_root_index_hostname, rrdhost_hostname(host)))
- error("RRDHOST: %s() failed to delete hostname '%s' from index", __FUNCTION__, rrdhost_hostname(host));
-
- rrdhost_option_clear(host, RRDHOST_OPTION_INDEXED_HOSTNAME);
- }
-}
-
// ----------------------------------------------------------------------------
// RRDHOST - internal helpers
@@ -225,7 +265,8 @@ static void rrdhost_initialize_rrdpush_sender(RRDHOST *host,
rrdpush_destinations_init(host);
host->rrdpush_send_api_key = strdupz(rrdpush_api_key);
- host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL, SIMPLE_PATTERN_EXACT);
+ host->rrdpush_send_charts_matching = simple_pattern_create(rrdpush_send_charts_matching, NULL,
+ SIMPLE_PATTERN_EXACT, true);
rrdhost_option_set(host, RRDHOST_OPTION_SENDER_ENABLED);
}
@@ -329,10 +370,8 @@ int is_legacy = 1;
rrdhost_option_set(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST);
char filename[FILENAME_MAX + 1];
- if(is_localhost) {
+ if(is_localhost)
host->cache_dir = strdupz(netdata_configured_cache_dir);
- host->varlib_dir = strdupz(netdata_configured_varlib_dir);
- }
else {
// this is not localhost - append our GUID to localhost path
if (is_in_multihost) { // don't append to cache dir in multihost
@@ -349,9 +388,6 @@ int is_legacy = 1;
if(r != 0 && errno != EEXIST)
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->cache_dir);
}
-
- snprintfz(filename, FILENAME_MAX, "%s/%s", netdata_configured_varlib_dir, host->machine_guid);
- host->varlib_dir = strdupz(filename);
}
// this is also needed for custom host variables - not only health
@@ -366,7 +402,6 @@ int is_legacy = 1;
rrdfamily_index_init(host);
rrdcalctemplate_index_init(host);
rrdcalc_rrdhost_index_init(host);
- metaqueue_host_update_info(host);
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
#ifdef ENABLE_DBENGINE
@@ -498,7 +533,6 @@ int is_legacy = 1;
" (to '%s' with api key '%s')"
", health %s"
", cache_dir '%s'"
- ", varlib_dir '%s'"
", alarms default handler '%s'"
", alarms default recipient '%s'"
, rrdhost_hostname(host)
@@ -517,20 +551,17 @@ int is_legacy = 1;
, host->rrdpush_send_api_key?host->rrdpush_send_api_key:""
, host->health.health_enabled?"enabled":"disabled"
, host->cache_dir
- , host->varlib_dir
, string2str(host->health.health_default_exec)
, string2str(host->health.health_default_recipient)
);
- if(!archived)
- rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
-
- rrdhost_load_rrdcontext_data(host);
- if (!archived) {
+ if(!archived) {
+ metaqueue_host_update_info(host);
+ rrdhost_load_rrdcontext_data(host);
+// rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
ml_host_new(host);
- ml_start_anomaly_detection_threads(host);
} else
- rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED | RRDHOST_FLAG_ORPHAN);
+ rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD | RRDHOST_FLAG_ARCHIVED | RRDHOST_FLAG_ORPHAN);
return host;
}
@@ -582,6 +613,8 @@ static void rrdhost_update(RRDHOST *host
if(strcmp(rrdhost_hostname(host), hostname) != 0) {
info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", rrdhost_hostname(host), hostname);
rrdhost_init_hostname(host, hostname, true);
+ } else {
+ rrdhost_index_add_hostname(host);
}
if(strcmp(rrdhost_program_name(host), program_name) != 0) {
@@ -643,7 +676,6 @@ static void rrdhost_update(RRDHOST *host
host->rrdpush_replication_step = rrdpush_replication_step;
ml_host_new(host);
- ml_start_anomaly_detection_threads(host);
rrdhost_load_rrdcontext_data(host);
info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
@@ -681,6 +713,10 @@ RRDHOST *rrdhost_find_or_create(
RRDHOST *host = rrdhost_find_by_guid(guid);
if (unlikely(host && host->rrd_memory_mode != mode && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) {
+
+ if (likely(!archived && rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
+ return host;
+
/* If a legacy memory mode instantiates all dbengine state must be discarded to avoid inconsistencies */
error("Archived host '%s' has memory mode '%s', but the wanted one is '%s'. Discarding archived state.",
rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
@@ -720,31 +756,30 @@ RRDHOST *rrdhost_find_or_create(
);
}
else {
-
- rrdhost_update(host
- , hostname
- , registry_hostname
- , guid
- , os
- , timezone
- , abbrev_timezone
- , utc_offset
- , tags
- , program_name
- , program_version
- , update_every
- , history
- , mode
- , health_enabled
- , rrdpush_enabled
- , rrdpush_destination
- , rrdpush_api_key
- , rrdpush_send_charts_matching
- , rrdpush_enable_replication
- , rrdpush_seconds_to_replicate
- , rrdpush_replication_step
- , system_info);
-
+ if (likely(!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
+ rrdhost_update(host
+ , hostname
+ , registry_hostname
+ , guid
+ , os
+ , timezone
+ , abbrev_timezone
+ , utc_offset
+ , tags
+ , program_name
+ , program_version
+ , update_every
+ , history
+ , mode
+ , health_enabled
+ , rrdpush_enabled
+ , rrdpush_destination
+ , rrdpush_api_key
+ , rrdpush_send_charts_matching
+ , rrdpush_enable_replication
+ , rrdpush_seconds_to_replicate
+ , rrdpush_replication_step
+ , system_info);
}
return host;
@@ -811,20 +846,6 @@ void dbengine_init(char *hostname) {
bool parallel_initialization = (storage_tiers <= (size_t)get_netdata_cpus()) ? true : false;
parallel_initialization = config_get_boolean(CONFIG_SECTION_DB, "dbengine parallel initialization", parallel_initialization);
- default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT);
- if (default_rrdeng_page_fetch_timeout < 1) {
- info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout);
- default_rrdeng_page_fetch_timeout = 1;
- config_set_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", default_rrdeng_page_fetch_timeout);
- }
-
- default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES);
- if (default_rrdeng_page_fetch_retries < 1) {
- info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries);
- default_rrdeng_page_fetch_retries = 1;
- config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries);
- }
-
struct dbengine_initialization tiers_init[RRD_STORAGE_TIERS] = {};
size_t created_tiers = 0;
@@ -938,8 +959,10 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unitt
rrdhost_init();
if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
- if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+ set_late_global_environment(system_info);
fatal("Failed to initialize SQLite");
+ }
info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
}
@@ -1071,7 +1094,7 @@ static void rrdhost_streaming_sender_structures_init(RRDHOST *host)
host->sender->host = host;
host->sender->buffer = cbuffer_new(CBUFFER_INITIAL_SIZE, 1024 * 1024, &netdata_buffers_statistics.cbuffers_streaming);
- host->sender->capabilities = STREAM_OUR_CAPABILITIES;
+ host->sender->capabilities = stream_our_capabilities();
host->sender->rrdpush_sender_pipe[PIPE_READ] = -1;
host->sender->rrdpush_sender_pipe[PIPE_WRITE] = -1;
@@ -1160,7 +1183,6 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
rrdcalctemplate_index_destroy(host);
// cleanup ML resources
- ml_stop_anomaly_detection_threads(host);
ml_host_delete(host);
freez(host->exporting_flags);
@@ -1182,21 +1204,6 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
return;
}
-#ifdef ENABLE_ACLK
- struct aclk_database_worker_config *wc = host->dbsync_worker;
- if (wc && !netdata_exit) {
- struct aclk_database_cmd cmd;
- memset(&cmd, 0, sizeof(cmd));
- cmd.opcode = ACLK_DATABASE_ORPHAN_HOST;
- struct aclk_completion compl ;
- init_aclk_completion(&compl );
- cmd.completion = &compl ;
- aclk_database_enq_cmd(wc, &cmd);
- wait_for_aclk_completion(&compl );
- destroy_aclk_completion(&compl );
- }
-#endif
-
// ------------------------------------------------------------------------
// free it
@@ -1212,7 +1219,6 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
string_freez(host->program_version);
rrdhost_system_info_free(host->system_info);
freez(host->cache_dir);
- freez(host->varlib_dir);
freez(host->rrdpush_send_api_key);
freez(host->rrdpush_send_destination);
rrdpush_destinations_free(host);
@@ -1226,16 +1232,14 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
rrdfamily_index_destroy(host);
rrdfunctions_destroy(host);
rrdvariables_destroy(host->rrdvars);
+ if (host == localhost)
+ rrdvariables_destroy(health_rrdvars);
rrdhost_destroy_rrdcontexts(host);
string_freez(host->hostname);
__atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED);
freez(host);
-#ifdef ENABLE_ACLK
- if (wc)
- wc->is_orphan = 0;
-#endif
}
void rrdhost_free_all(void) {
@@ -1253,11 +1257,10 @@ void rrdhost_free_all(void) {
void rrd_finalize_collection_for_all_hosts(void) {
RRDHOST *host;
- rrd_wrlock();
- rrdhost_foreach_read(host) {
+ dfe_start_reentrant(rrdhost_root_index, host) {
rrdhost_finalize_collection(host);
}
- rrd_unlock();
+ dfe_done(host);
}
// ----------------------------------------------------------------------------
@@ -1278,6 +1281,33 @@ void rrdhost_save_charts(RRDHOST *host) {
rrdset_foreach_done(st);
}
+struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels) {
+ struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
+ info->hops = 1;
+
+ rrdlabels_get_value_strdup_or_null(labels, &info->cloud_provider_type, "_cloud_provider_type");
+ rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_type, "_cloud_instance_type");
+ rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_region, "_cloud_instance_region");
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_os_name, "_os_name");
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_os_version, "_os_version");
+ rrdlabels_get_value_strdup_or_null(labels, &info->kernel_version, "_kernel_version");
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_cores, "_system_cores");
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_cpu_freq, "_system_cpu_freq");
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_ram_total, "_system_ram_total");
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_disk_space, "_system_disk_space");
+ rrdlabels_get_value_strdup_or_null(labels, &info->architecture, "_architecture");
+ rrdlabels_get_value_strdup_or_null(labels, &info->virtualization, "_virtualization");
+ rrdlabels_get_value_strdup_or_null(labels, &info->container, "_container");
+ rrdlabels_get_value_strdup_or_null(labels, &info->container_detection, "_container_detection");
+ rrdlabels_get_value_strdup_or_null(labels, &info->virt_detection, "_virt_detection");
+ rrdlabels_get_value_strdup_or_null(labels, &info->is_k8s_node, "_is_k8s_node");
+ rrdlabels_get_value_strdup_or_null(labels, &info->install_type, "_install_type");
+ rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_arch, "_prebuilt_arch");
+ rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_dist, "_prebuilt_dist");
+
+ return info;
+}
+
static void rrdhost_load_auto_labels(void) {
DICTIONARY *labels = localhost->rrdlabels;
@@ -1288,8 +1318,7 @@ static void rrdhost_load_auto_labels(void) {
rrdlabels_add(labels, "_cloud_instance_type", localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO);
if (localhost->system_info->cloud_instance_region)
- rrdlabels_add(
- labels, "_cloud_instance_region", localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
+ rrdlabels_add(labels, "_cloud_instance_region", localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
if (localhost->system_info->host_os_name)
rrdlabels_add(labels, "_os_name", localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO);
@@ -1353,13 +1382,12 @@ void rrdhost_set_is_parent_label(int count) {
DICTIONARY *labels = localhost->rrdlabels;
if (count == 0 || count == 1) {
- rrdlabels_add(
- labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO);
+ rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO);
//queue a node info
#ifdef ENABLE_ACLK
if (netdata_cloud_setting) {
- aclk_queue_node_info(localhost);
+ aclk_queue_node_info(localhost, false);
}
#endif
}
@@ -1430,7 +1458,7 @@ void rrdhost_finalize_collection(RRDHOST *host) {
info("RRD: 'host:%s' stopping data collection...", rrdhost_hostname(host));
RRDSET *st;
- rrdset_foreach_write(st, host)
+ rrdset_foreach_read(st, host)
rrdset_finalize_collection(st, true);
rrdset_foreach_done(st);
}