diff options
Diffstat (limited to 'daemon/main.c')
| -rw-r--r-- | daemon/main.c | 202 |
1 files changed, 172 insertions, 30 deletions
diff --git a/daemon/main.c b/daemon/main.c index e10d38b4..ada3c14f 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -47,7 +47,7 @@ void netdata_cleanup_and_exit(int ret) { // stop everything info("EXIT: stopping static threads..."); -#ifdef ENABLE_NEW_CLOUD_PROTOCOL +#ifdef ENABLE_ACLK aclk_sync_exit_all(); #endif cancel_main_threads(); @@ -55,13 +55,16 @@ void netdata_cleanup_and_exit(int ret) { // free the database info("EXIT: freeing database memory..."); #ifdef ENABLE_DBENGINE - rrdeng_prepare_exit(&multidb_ctx); + for(int tier = 0; tier < storage_tiers ; tier++) + rrdeng_prepare_exit(multidb_ctx[tier]); #endif rrdhost_free_all(); #ifdef ENABLE_DBENGINE - rrdeng_exit(&multidb_ctx); + for(int tier = 0; tier < storage_tiers ; tier++) + rrdeng_exit(multidb_ctx[tier]); #endif } + sql_close_context_database(); sql_close_database(); // unlink the pid @@ -349,6 +352,12 @@ int help(int exitcode) { #endif " -W set section option value\n" " set netdata.conf option from the command line.\n\n" + " -W buildinfo Print the version, the configure options,\n" + " a list of optional features, and whether they\n" + " are enabled or not.\n\n" + " -W buildinfojson Print the version, the configure options,\n" + " a list of optional features, and whether they\n" + " are enabled or not, in JSON format.\n\n" " -W simple-pattern pattern string\n" " Check if string matches pattern and exit.\n\n" " -W \"claim -token=TOKEN -rooms=ROOM1,ROOM2\"\n" @@ -393,6 +402,14 @@ static void log_init(void) { snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir); stdaccess_filename = config_get(CONFIG_SECTION_LOGS, "access", filename); +#ifdef ENABLE_ACLK + aclklog_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "conversation log", CONFIG_BOOLEAN_NO); + if (aclklog_enabled) { + snprintfz(filename, FILENAME_MAX, "%s/aclk.log", netdata_configured_log_dir); + aclklog_filename = config_get(CONFIG_SECTION_CLOUD, "conversation log file", filename); + } +#endif + char deffacility[8]; snprintfz(deffacility,7,"%s","daemon"); facility_log = config_get(CONFIG_SECTION_LOGS, "facility", deffacility); @@ -516,6 +533,64 @@ static void backwards_compatible_config() { config_move(CONFIG_SECTION_STATSD, "enabled", CONFIG_SECTION_PLUGINS, "statsd"); + + config_move(CONFIG_SECTION_GLOBAL, "memory mode", + CONFIG_SECTION_DB, "mode"); + + config_move(CONFIG_SECTION_GLOBAL, "history", + CONFIG_SECTION_DB, "retention"); + + config_move(CONFIG_SECTION_GLOBAL, "update every", + CONFIG_SECTION_DB, "update every"); + + config_move(CONFIG_SECTION_GLOBAL, "page cache size", + CONFIG_SECTION_DB, "dbengine page cache size MB"); + + config_move(CONFIG_SECTION_DB, "page cache size", + CONFIG_SECTION_DB, "dbengine page cache size MB"); + + config_move(CONFIG_SECTION_GLOBAL, "page cache uses malloc", + CONFIG_SECTION_DB, "dbengine page cache with malloc"); + + config_move(CONFIG_SECTION_DB, "page cache with malloc", + CONFIG_SECTION_DB, "dbengine page cache with malloc"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine disk space", + CONFIG_SECTION_DB, "dbengine disk space MB"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", + CONFIG_SECTION_DB, "dbengine multihost disk space MB"); + + config_move(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", + CONFIG_SECTION_DB, "memory deduplication (ksm)"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch timeout", + CONFIG_SECTION_DB, "dbengine page fetch timeout secs"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch retries", + CONFIG_SECTION_DB, "dbengine page fetch retries"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine extent pages", + CONFIG_SECTION_DB, "dbengine pages per extent"); + + config_move(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", + CONFIG_SECTION_DB, "cleanup obsolete charts after secs"); + + config_move(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", + CONFIG_SECTION_DB, "gap when lost iterations above"); + + config_move(CONFIG_SECTION_GLOBAL, "cleanup orphan hosts after seconds", + CONFIG_SECTION_DB, "cleanup orphan hosts after secs"); + + config_move(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", + CONFIG_SECTION_DB, "delete obsolete charts files"); + + config_move(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", + CONFIG_SECTION_DB, "delete orphan hosts files"); + + config_move(CONFIG_SECTION_GLOBAL, "enable zero metrics", + CONFIG_SECTION_DB, "enable zero metrics"); + } static void get_netdata_configured_variables() { @@ -533,28 +608,40 @@ static void get_netdata_configured_variables() { debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname); // ------------------------------------------------------------------------ - // get default database size - - default_rrd_history_entries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "history", align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); + // get default database update frequency - long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); - if(h != default_rrd_history_entries) { - config_set_number(CONFIG_SECTION_GLOBAL, "history", h); - default_rrd_history_entries = (int)h; + default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", UPDATE_EVERY); + if(default_rrd_update_every < 1 || default_rrd_update_every > 600) { + error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY); + default_rrd_update_every = UPDATE_EVERY; + config_set_number(CONFIG_SECTION_DB, "update every", default_rrd_update_every); } - if(default_rrd_history_entries < 5 || default_rrd_history_entries > RRD_HISTORY_ENTRIES_MAX) { - error("Invalid history entries %d given. Defaulting to %d.", default_rrd_history_entries, RRD_DEFAULT_HISTORY_ENTRIES); - default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; + // ------------------------------------------------------------------------ + // get default memory mode for the database + + { + const char *mode = config_get(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode)); + default_rrd_memory_mode = rrd_memory_mode_id(mode); + if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) { + error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode)); + config_set(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode)); + } } // ------------------------------------------------------------------------ - // get default database update frequency + // get default database size - default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_GLOBAL, "update every", UPDATE_EVERY); - if(default_rrd_update_every < 1 || default_rrd_update_every > 600) { - error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY_MAX); - default_rrd_update_every = UPDATE_EVERY; + if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) { + default_rrd_history_entries = (int)config_get_number( + CONFIG_SECTION_DB, "retention", + align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); + + long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); + if (h != default_rrd_history_entries) { + config_set_number(CONFIG_SECTION_DB, "retention", h); + default_rrd_history_entries = (int)h; + } } // ------------------------------------------------------------------------ @@ -576,38 +663,38 @@ static void get_netdata_configured_variables() { netdata_configured_primary_plugins_dir = plugin_directories[PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH]; } - // ------------------------------------------------------------------------ - // get default memory mode for the database - - default_rrd_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_GLOBAL, "memory mode", rrd_memory_mode_name(default_rrd_memory_mode))); #ifdef ENABLE_DBENGINE // ------------------------------------------------------------------------ // get default Database Engine page cache size in MiB - default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "page cache size", default_rrdeng_page_cache_mb); + db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "dbengine page cache with malloc", CONFIG_BOOLEAN_NO); + default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb); if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) { error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB); default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB; + config_set_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb); } // ------------------------------------------------------------------------ // get default Database Engine disk space quota in MiB - default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine disk space", default_rrdeng_disk_quota_mb); + default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb); if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB); default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB; + config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb); } - default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", compute_multidb_diskspace()); + default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace()); if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb); default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; + config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb); } #else if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use memory mode ram instead."); - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead."); + default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE; } #endif // ------------------------------------------------------------------------ @@ -619,12 +706,40 @@ static void get_netdata_configured_variables() { // get KSM settings #ifdef MADV_MERGEABLE - enable_ksm = config_get_boolean(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", enable_ksm); + enable_ksm = config_get_boolean(CONFIG_SECTION_DB, "memory deduplication (ksm)", enable_ksm); #endif // -------------------------------------------------------------------- // metric correlations + enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations); + default_metric_correlations_method = weights_string_to_method(config_get( + CONFIG_SECTION_GLOBAL, "metric correlations method", + weights_method_to_string(default_metric_correlations_method))); + + // -------------------------------------------------------------------- + + rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time); + // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short + // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at + // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information. + if (rrdset_free_obsolete_time < 10) { + rrdset_free_obsolete_time = 10; + info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds."); + config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time); + } + + gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above); + if (gap_when_lost_iterations_above < 1) { + gap_when_lost_iterations_above = 1; + config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above); + } + + // -------------------------------------------------------------------- + // rrdcontext + + rrdcontext_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "rrdcontexts", rrdcontext_enabled); + // -------------------------------------------------------------------- // get various system parameters @@ -867,13 +982,16 @@ int main(int argc, char **argv) { return 1; if (unit_test_str2ld()) return 1; - + if (unit_test_bitmap256()) + return 1; // No call to load the config file on this code-path post_conf_load(&user); get_netdata_configured_variables(); + rrdcontext_enabled = CONFIG_BOOLEAN_NO; default_rrd_update_every = 1; default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; default_health_enabled = 0; + storage_tiers = 1; registry_init(); if(rrd_init("unittest", NULL)) { fprintf(stderr, "rrd_init failed for unittest\n"); @@ -886,6 +1004,12 @@ int main(int argc, char **argv) { if(test_dbengine()) return 1; #endif if(test_sqlite()) return 1; + if (dictionary_unittest(10000)) + return 1; + if (rrdlabels_unittest()) + return 1; + if (ctx_unittest()) + return 1; fprintf(stderr, "\n\nALL TESTS PASSED\n\n"); return 0; } @@ -895,9 +1019,18 @@ int main(int argc, char **argv) { } #endif #ifdef ENABLE_DBENGINE + else if(strcmp(optarg, "mctest") == 0) { + return mc_unittest(); + } + else if(strcmp(optarg, "ctxtest") == 0) { + return ctx_unittest(); + } else if(strcmp(optarg, "dicttest") == 0) { return dictionary_unittest(10000); } + else if(strcmp(optarg, "rrdlabelstest") == 0) { + return rrdlabels_unittest(); + } else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) { optarg += strlen(createdataset_string); unsigned history_seconds = strtoul(optarg, NULL, 0); @@ -1214,25 +1347,31 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // get log filenames and settings + log_init(); error_log_limit_unlimited(); + // initialize the log files open_all_log_files(); get_system_timezone(); + // -------------------------------------------------------------------- // get the certificate and start security + #ifdef ENABLE_HTTPS security_init(); #endif // -------------------------------------------------------------------- // This is the safest place to start the SILENCERS structure + set_silencers_filename(); health_initialize_global_silencers(); // -------------------------------------------------------------------- // Initialize ML configuration + ml_init(); // -------------------------------------------------------------------- @@ -1240,9 +1379,11 @@ int main(int argc, char **argv) { // block signals while initializing threads. // this causes the threads to block signals. + signals_block(); // setup the signals we want to use + signals_init(); // setup threads configs @@ -1271,6 +1412,7 @@ int main(int argc, char **argv) { if(web_server_mode != WEB_SERVER_MODE_NONE) api_listen_sockets_setup(); + } #ifdef NETDATA_INTERNAL_CHECKS @@ -1354,7 +1496,7 @@ int main(int argc, char **argv) { web_server_config_options(); - netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_GLOBAL, "enable zero metrics", CONFIG_BOOLEAN_NO); + netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_DB, "enable zero metrics", CONFIG_BOOLEAN_NO); set_late_global_environment(); |