From 81581f9719bc56f01d5aa08952671d65fda9867a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 8 May 2023 18:27:08 +0200 Subject: Merging upstream version 1.39.0. Signed-off-by: Daniel Baumann --- daemon/main.c | 148 +++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 101 insertions(+), 47 deletions(-) (limited to 'daemon/main.c') diff --git a/daemon/main.c b/daemon/main.c index 7b2076f3f..606de128b 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -13,6 +13,7 @@ int netdata_zero_metrics_enabled; int netdata_anonymous_statistics_enabled; int libuv_worker_threads = MIN_LIBUV_WORKER_THREADS; +bool ieee754_doubles = false; struct netdata_static_thread *static_threads; @@ -147,10 +148,6 @@ static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) { buffer_strcat(wb, "MAINTENANCE "); if(service & SERVICE_COLLECTORS) buffer_strcat(wb, "COLLECTORS "); - if(service & SERVICE_ML_TRAINING) - buffer_strcat(wb, "ML_TRAINING "); - if(service & SERVICE_ML_PREDICTION) - buffer_strcat(wb, "ML_PREDICTION "); if(service & SERVICE_REPLICATION) buffer_strcat(wb, "REPLICATION "); if(service & ABILITY_DATA_QUERIES) @@ -312,6 +309,8 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) { timeout = false; \ } +void web_client_cache_destroy(void); + void netdata_cleanup_and_exit(int ret) { usec_t started_ut = now_monotonic_usec(); usec_t last_ut = started_ut; @@ -339,6 +338,15 @@ void netdata_cleanup_and_exit(int ret) { } #endif + delta_shutdown_time("close webrtc connections"); + + webrtc_close_all_connections(); + + delta_shutdown_time("disable ML detection and training threads"); + + ml_stop_threads(); + ml_fini(); + delta_shutdown_time("disable maintenance, new queries, new web requests, new streaming connections and aclk"); service_signal_exit( @@ -347,14 +355,14 @@ void netdata_cleanup_and_exit(int ret) { | ABILITY_WEB_REQUESTS | ABILITY_STREAMING_CONNECTIONS | SERVICE_ACLK + | SERVICE_ACLKSYNC ); - delta_shutdown_time("stop replication, exporters, ML training, health and web servers threads"); + delta_shutdown_time("stop replication, exporters, health and web servers threads"); timeout = !service_wait_exit( SERVICE_REPLICATION | SERVICE_EXPORTERS - | SERVICE_ML_TRAINING | SERVICE_HEALTH | SERVICE_WEB_SERVER , 3 * USEC_PER_SEC); @@ -366,11 +374,10 @@ void netdata_cleanup_and_exit(int ret) { | SERVICE_STREAMING , 3 * USEC_PER_SEC); - delta_shutdown_time("stop ML prediction and context threads"); + delta_shutdown_time("stop context thread"); timeout = !service_wait_exit( - SERVICE_ML_PREDICTION - | SERVICE_CONTEXT + SERVICE_CONTEXT , 3 * USEC_PER_SEC); delta_shutdown_time("stop maintenance thread"); @@ -379,6 +386,10 @@ void netdata_cleanup_and_exit(int ret) { SERVICE_MAINTENANCE , 3 * USEC_PER_SEC); + delta_shutdown_time("clear web client cache"); + + web_client_cache_destroy(); + delta_shutdown_time("clean rrdhost database"); rrdhost_cleanup_all(); @@ -387,11 +398,6 @@ void netdata_cleanup_and_exit(int ret) { metadata_sync_shutdown_prepare(); -#ifdef ENABLE_ACLK - delta_shutdown_time("signal aclk sync to stop"); - aclk_sync_exit_all(); -#endif - delta_shutdown_time("stop aclk threads"); timeout = !service_wait_exit( @@ -529,38 +535,41 @@ void web_server_config_options(void) web_x_frame_options = NULL; web_allow_connections_from = - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"), - NULL, SIMPLE_PATTERN_EXACT); + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow connections from", "localhost *"), + NULL, SIMPLE_PATTERN_EXACT, true); web_allow_connections_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow connections by dns", "heuristic", web_allow_connections_from); web_allow_dashboard_from = - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"), - NULL, SIMPLE_PATTERN_EXACT); + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow dashboard from", "localhost *"), + NULL, SIMPLE_PATTERN_EXACT, true); web_allow_dashboard_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow dashboard by dns", "heuristic", web_allow_dashboard_from); web_allow_badges_from = - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT); + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow badges from", "*"), NULL, SIMPLE_PATTERN_EXACT, + true); web_allow_badges_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow badges by dns", "heuristic", web_allow_badges_from); web_allow_registry_from = - simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT); + simple_pattern_create(config_get(CONFIG_SECTION_REGISTRY, "allow from", "*"), NULL, SIMPLE_PATTERN_EXACT, + true); web_allow_registry_dns = make_dns_decision(CONFIG_SECTION_REGISTRY, "allow by dns", "heuristic", web_allow_registry_from); web_allow_streaming_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow streaming from", "*"), - NULL, SIMPLE_PATTERN_EXACT); + NULL, SIMPLE_PATTERN_EXACT, true); web_allow_streaming_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow streaming by dns", "heuristic", web_allow_streaming_from); // Note the default is not heuristic, the wildcards could match DNS but the intent is ip-addresses. web_allow_netdataconf_from = simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow netdata.conf from", - "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*" - " 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*" - " 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*" - " 172.31.* UNKNOWN"), NULL, SIMPLE_PATTERN_EXACT); + "localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.*" + " 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.*" + " 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.*" + " 172.31.* UNKNOWN"), NULL, SIMPLE_PATTERN_EXACT, + true); web_allow_netdataconf_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow netdata.conf by dns", "no", web_allow_netdataconf_from); web_allow_mgmt_from = - simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"), - NULL, SIMPLE_PATTERN_EXACT); + simple_pattern_create(config_get(CONFIG_SECTION_WEB, "allow management from", "localhost"), + NULL, SIMPLE_PATTERN_EXACT, true); web_allow_mgmt_dns = make_dns_decision(CONFIG_SECTION_WEB, "allow management by dns","heuristic",web_allow_mgmt_from); @@ -655,9 +664,14 @@ void cancel_main_threads() { int i, found = 0; usec_t max = 5 * USEC_PER_SEC, step = 100000; for (i = 0; static_threads[i].name != NULL ; i++) { - if(static_threads[i].enabled == NETDATA_MAIN_THREAD_RUNNING) { - info("EXIT: Stopping main thread: %s", static_threads[i].name); - netdata_thread_cancel(*static_threads[i].thread); + if (static_threads[i].enabled == NETDATA_MAIN_THREAD_RUNNING) { + if (static_threads[i].thread) { + info("EXIT: Stopping main thread: %s", static_threads[i].name); + netdata_thread_cancel(*static_threads[i].thread); + } else { + info("EXIT: No thread running (marking as EXITED): %s", static_threads[i].name); + static_threads[i].enabled = NETDATA_MAIN_THREAD_EXITED; + } found++; } } @@ -1107,8 +1121,12 @@ static void get_netdata_configured_variables() { // get default Database Engine page cache size in MiB default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb); + default_rrdeng_extent_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine extent cache size MB", default_rrdeng_extent_cache_mb); db_engine_journal_check = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO); + if(default_rrdeng_extent_cache_mb < 0) + default_rrdeng_extent_cache_mb = 0; + if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) { error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB); default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB; @@ -1314,9 +1332,12 @@ void post_conf_load(char **user) prev_msg = msg; \ } +int buffer_unittest(void); int pgc_unittest(void); int mrg_unittest(void); int julytest(void); +int pluginsd_parser_unittest(void); +void replication_initialize(void); int main(int argc, char **argv) { // initialize the system clocks @@ -1437,12 +1458,17 @@ int main(int argc, char **argv) { if(strcmp(optarg, "unittest") == 0) { unittest_running = true; + if (pluginsd_parser_unittest()) + return 1; + if (unit_test_static_threads()) return 1; if (unit_test_buffer()) return 1; if (unit_test_str2ld()) return 1; + if (buffer_unittest()) + return 1; if (unit_test_bitmap256()) return 1; // No call to load the config file on this code-path @@ -1479,15 +1505,6 @@ int main(int argc, char **argv) { else if(strcmp(optarg, "escapetest") == 0) { return command_argument_sanitization_tests(); } -#ifdef ENABLE_DBENGINE - else if(strcmp(optarg, "mctest") == 0) { - unittest_running = true; - return mc_unittest(); - } - else if(strcmp(optarg, "ctxtest") == 0) { - unittest_running = true; - return ctx_unittest(); - } else if(strcmp(optarg, "dicttest") == 0) { unittest_running = true; return dictionary_unittest(10000); @@ -1504,6 +1521,19 @@ int main(int argc, char **argv) { unittest_running = true; return rrdlabels_unittest(); } + else if(strcmp(optarg, "buffertest") == 0) { + unittest_running = true; + return buffer_unittest(); + } +#ifdef ENABLE_DBENGINE + else if(strcmp(optarg, "mctest") == 0) { + unittest_running = true; + return mc_unittest(); + } + else if(strcmp(optarg, "ctxtest") == 0) { + unittest_running = true; + return ctx_unittest(); + } else if(strcmp(optarg, "metatest") == 0) { unittest_running = true; return metadata_unittest(); @@ -1523,6 +1553,14 @@ int main(int argc, char **argv) { else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) { optarg += strlen(createdataset_string); unsigned history_seconds = strtoul(optarg, NULL, 0); + post_conf_load(&user); + get_netdata_configured_variables(); + default_rrd_update_every = 1; + registry_init(); + if(rrd_init("dbengine-dataset", NULL, true)) { + fprintf(stderr, "rrd_init failed for unittest\n"); + return 1; + } generate_dbengine_dataset(history_seconds); return 0; } @@ -1587,12 +1625,16 @@ int main(int argc, char **argv) { size_t len = strlen(needle) + 1; char wildcarded[len]; - SIMPLE_PATTERN *p = simple_pattern_create(haystack, NULL, SIMPLE_PATTERN_EXACT); - int ret = simple_pattern_matches_extract(p, needle, wildcarded, len); + SIMPLE_PATTERN *p = simple_pattern_create(haystack, NULL, SIMPLE_PATTERN_EXACT, true); + SIMPLE_PATTERN_RESULT ret = simple_pattern_matches_extract(p, needle, wildcarded, len); simple_pattern_free(p); - if(ret) { - fprintf(stdout, "RESULT: MATCHED - pattern '%s' matches '%s', wildcarded '%s'\n", haystack, needle, wildcarded); + if(ret == SP_MATCHED_POSITIVE) { + fprintf(stdout, "RESULT: POSITIVE MATCHED - pattern '%s' matches '%s', wildcarded '%s'\n", haystack, needle, wildcarded); + return 0; + } + else if(ret == SP_MATCHED_NEGATIVE) { + fprintf(stdout, "RESULT: NEGATIVE MATCHED - pattern '%s' matches '%s', wildcarded '%s'\n", haystack, needle, wildcarded); return 0; } else { @@ -1801,7 +1843,7 @@ int main(int argc, char **argv) { #endif // set libuv worker threads - libuv_worker_threads = (int)get_netdata_cpus() * 2; + libuv_worker_threads = (int)get_netdata_cpus() * 6; if(libuv_worker_threads < MIN_LIBUV_WORKER_THREADS) libuv_worker_threads = MIN_LIBUV_WORKER_THREADS; @@ -1866,10 +1908,14 @@ int main(int argc, char **argv) { // initialize the log files open_all_log_files(); + ieee754_doubles = is_system_ieee754_double(); + aral_judy_init(); get_system_timezone(); + replication_initialize(); + // -------------------------------------------------------------------- // get the certificate and start security @@ -1988,13 +2034,16 @@ int main(int argc, char **argv) { struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info)); __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED); get_system_info(system_info); + (void) registry_get_this_machine_guid(); system_info->hops = 0; get_install_type(&system_info->install_type, &system_info->prebuilt_arch, &system_info->prebuilt_dist); delta_startup_time("initialize RRD structures"); - if(rrd_init(netdata_configured_hostname, system_info, false)) + if(rrd_init(netdata_configured_hostname, system_info, false)) { + set_late_global_environment(system_info); fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname); + } delta_startup_time("check for incomplete shutdown"); @@ -2036,8 +2085,7 @@ int main(int argc, char **argv) { netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_DB, "enable zero metrics", CONFIG_BOOLEAN_NO); - set_late_global_environment(); - + set_late_global_environment(system_info); for (i = 0; static_threads[i].name != NULL ; i++) { struct netdata_static_thread *st = &static_threads[i]; @@ -2048,6 +2096,7 @@ int main(int argc, char **argv) { } else debug(D_SYSTEM, "Not starting thread %s.", st->name); } + ml_start_threads(); // ------------------------------------------------------------------------ // Initialize netdata agent command serving from cli and signals @@ -2097,6 +2146,11 @@ int main(int argc, char **argv) { } #endif + // ------------------------------------------------------------------------ + // initialize WebRTC + + webrtc_initialize(); + // ------------------------------------------------------------------------ // unblock signals -- cgit v1.2.3