Merging upstream version 2.0.3+dfsg:

- does not include dygraphs anymore (Closes: #923993)
- does not include pako anymore (Closes: #1042533)
- does not include dashboard binaries anymore (Closes: #1045145)

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 215 insertions, 170 deletions

diff --git a/src/daemon/main.c b/src/daemon/main.c
index 17fef8449..03ae7e003 100644
--- a/src/daemon/main.c
+++ b/src/daemon/main.c
@@ -6,6 +6,7 @@
 #include "static_threads.h"
 
 #include "database/engine/page_test.h"
+#include <curl/curl.h>
 
 #ifdef OS_WINDOWS
 #include "win_system-info.h"
@@ -27,18 +28,7 @@ bool ieee754_doubles = false;
 time_t netdata_start_time = 0;
 struct netdata_static_thread *static_threads;
 
-struct config netdata_config = {
-        .first_section = NULL,
-        .last_section = NULL,
-        .mutex = NETDATA_MUTEX_INITIALIZER,
-        .index = {
-                .avl_tree = {
-                        .root = NULL,
-                        .compar = appconfig_section_compare
-                },
-                .rwlock = AVL_LOCK_INITIALIZER
-        }
-};
+struct config netdata_config = APPCONFIG_INITIALIZER;
 
 typedef struct service_thread {
     pid_t tid;
@@ -326,6 +316,7 @@ void web_client_cache_destroy(void);
 void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) {
     netdata_exit = 1;
 
+    usec_t shutdown_start_time = now_monotonic_usec();
     watcher_shutdown_begin();
 
     nd_log_limits_unlimited();
@@ -361,7 +352,7 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
     watcher_step_complete(WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS);
 
     service_signal_exit(SERVICE_MAINTENANCE | ABILITY_DATA_QUERIES | ABILITY_WEB_REQUESTS |
-                        ABILITY_STREAMING_CONNECTIONS | SERVICE_ACLK | SERVICE_ACLKSYNC);
+                        ABILITY_STREAMING_CONNECTIONS | SERVICE_ACLK);
     watcher_step_complete(WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK);
 
     service_wait_exit(SERVICE_MAINTENANCE, 3 * USEC_PER_SEC);
@@ -474,21 +465,22 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
 #endif
     }
 
+    // Don't register a shutdown event if we crashed
+    if (!ret)
+        add_agent_event(EVENT_AGENT_SHUTDOWN_TIME, (int64_t)(now_monotonic_usec() - shutdown_start_time));
     sqlite_close_databases();
     watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_DATABASES);
     sqlite_library_shutdown();
 
 
     // unlink the pid
-    if(pidfile[0]) {
+    if(pidfile && *pidfile) {
         if(unlink(pidfile) != 0)
             netdata_log_error("EXIT: cannot unlink pidfile '%s'.", pidfile);
     }
     watcher_step_complete(WATCHER_STEP_ID_REMOVE_PID_FILE);
 
-#ifdef ENABLE_HTTPS
     netdata_ssl_cleanup();
-#endif
     watcher_step_complete(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);
 
     (void) unlink(agent_incomplete_shutdown_file);
@@ -496,6 +488,7 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
 
     watcher_shutdown_end();
     watcher_thread_stop();
+    curl_global_cleanup();
 
 #ifdef OS_WINDOWS
     return;
@@ -527,12 +520,12 @@ void web_server_threading_selection(void) {
 
 int make_dns_decision(const char *section_name, const char *config_name, const char *default_value, SIMPLE_PATTERN *p)
 {
-    char *value = config_get(section_name,config_name,default_value);
+    const char *value = config_get(section_name,config_name,default_value);
     if(!strcmp("yes",value))
         return 1;
     if(!strcmp("no",value))
         return 0;
-    if(strcmp("heuristic",value))
+    if(strcmp("heuristic",value) != 0)
         netdata_log_error("Invalid configuration option '%s' for '%s'/'%s'. Valid options are 'yes', 'no' and 'heuristic'. Proceeding with 'heuristic'",
               value, section_name, config_name);
 
@@ -542,11 +535,13 @@ int make_dns_decision(const char *section_name, const char *config_name, const c
 void web_server_config_options(void)
 {
     web_client_timeout =
-        (int)config_get_number(CONFIG_SECTION_WEB, "disconnect idle clients after seconds", web_client_timeout);
+        (int)config_get_duration_seconds(CONFIG_SECTION_WEB, "disconnect idle clients after", web_client_timeout);
+
     web_client_first_request_timeout =
-        (int)config_get_number(CONFIG_SECTION_WEB, "timeout for first request", web_client_first_request_timeout);
+        (int)config_get_duration_seconds(CONFIG_SECTION_WEB, "timeout for first request", web_client_first_request_timeout);
+
     web_client_streaming_rate_t =
-        config_get_number(CONFIG_SECTION_WEB, "accept a streaming request every seconds", web_client_streaming_rate_t);
+        config_get_duration_seconds(CONFIG_SECTION_WEB, "accept a streaming request every", web_client_streaming_rate_t);
 
     respect_web_browser_do_not_track_policy =
         config_get_boolean(CONFIG_SECTION_WEB, "respect do not track policy", respect_web_browser_do_not_track_policy);
@@ -595,7 +590,7 @@ void web_server_config_options(void)
 
     web_enable_gzip = config_get_boolean(CONFIG_SECTION_WEB, "enable gzip compression", web_enable_gzip);
 
-    char *s = config_get(CONFIG_SECTION_WEB, "gzip compression strategy", "default");
+    const char *s = config_get(CONFIG_SECTION_WEB, "gzip compression strategy", "default");
     if(!strcmp(s, "default"))
         web_gzip_strategy = Z_DEFAULT_STRATEGY;
     else if(!strcmp(s, "filtered"))
@@ -807,8 +802,6 @@ int help(int exitcode) {
             "                           are enabled or not, in JSON format.\n\n"
             "  -W simple-pattern pattern string\n"
             "                           Check if string matches pattern and exit.\n\n"
-            "  -W \"claim -token=TOKEN -rooms=ROOM1,ROOM2\"\n"
-            "                           Claim the agent to the workspace rooms pointed to by TOKEN and ROOM*.\n\n"
 #ifdef OS_WINDOWS
             "  -W perflibdump [key]\n"
             "                           Dump the Windows Performance Counters Registry in JSON.\n\n"
@@ -825,7 +818,6 @@ int help(int exitcode) {
     return exitcode;
 }
 
-#ifdef ENABLE_HTTPS
 static void security_init(){
     char filename[FILENAME_MAX + 1];
     snprintfz(filename, FILENAME_MAX, "%s/ssl/key.pem",netdata_configured_user_config_dir);
@@ -839,14 +831,13 @@ static void security_init(){
 
     netdata_ssl_initialize_openssl();
 }
-#endif
 
 static void log_init(void) {
     nd_log_set_facility(config_get(CONFIG_SECTION_LOGS, "facility", "daemon"));
 
     time_t period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
     size_t logs = ND_LOG_DEFAULT_THROTTLE_LOGS;
-    period = config_get_number(CONFIG_SECTION_LOGS, "logs flood protection period", period);
+    period = config_get_duration_seconds(CONFIG_SECTION_LOGS, "logs flood protection period", period);
     logs = (unsigned long)config_get_number(CONFIG_SECTION_LOGS, "logs to trigger flood protection", (long long int)logs);
     nd_log_set_flood_protection(logs, period);
 
@@ -856,50 +847,75 @@ static void log_init(void) {
     nd_log_set_priority_level(config_get(CONFIG_SECTION_LOGS, "level", netdata_log_level));
 
     char filename[FILENAME_MAX + 1];
+    char* os_default_method = NULL;
+#if defined(OS_LINUX)
+    os_default_method = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */ ? "journal" : NULL;
+#elif defined(OS_WINDOWS)
+#if defined(HAVE_ETW)
+    os_default_method = "etw";
+#elif defined(HAVE_WEL)
+    os_default_method = "wel";
+#endif
+#endif
+
+#if defined(OS_WINDOWS)
+    // on windows, debug log goes to windows events
+    snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
+#else
     snprintfz(filename, FILENAME_MAX, "%s/debug.log", netdata_configured_log_dir);
+#endif
+
     nd_log_set_user_settings(NDLS_DEBUG, config_get(CONFIG_SECTION_LOGS, "debug", filename));
 
-    bool with_journal = is_stderr_connected_to_journal() /* || nd_log_journal_socket_available() */;
-    if(with_journal)
-        snprintfz(filename, FILENAME_MAX, "journal");
+    if(os_default_method)
+        snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
     else
         snprintfz(filename, FILENAME_MAX, "%s/daemon.log", netdata_configured_log_dir);
     nd_log_set_user_settings(NDLS_DAEMON, config_get(CONFIG_SECTION_LOGS, "daemon", filename));
 
-    if(with_journal)
-        snprintfz(filename, FILENAME_MAX, "journal");
+    if(os_default_method)
+        snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
     else
         snprintfz(filename, FILENAME_MAX, "%s/collector.log", netdata_configured_log_dir);
     nd_log_set_user_settings(NDLS_COLLECTORS, config_get(CONFIG_SECTION_LOGS, "collector", filename));
 
+#if defined(OS_WINDOWS)
+    // on windows, access log goes to windows events
+    snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
+#else
     snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir);
+#endif
     nd_log_set_user_settings(NDLS_ACCESS, config_get(CONFIG_SECTION_LOGS, "access", filename));
 
-    if(with_journal)
-        snprintfz(filename, FILENAME_MAX, "journal");
+    if(os_default_method)
+        snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
     else
         snprintfz(filename, FILENAME_MAX, "%s/health.log", netdata_configured_log_dir);
     nd_log_set_user_settings(NDLS_HEALTH, config_get(CONFIG_SECTION_LOGS, "health", filename));
 
-#ifdef ENABLE_ACLK
     aclklog_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "conversation log", CONFIG_BOOLEAN_NO);
     if (aclklog_enabled) {
+#if defined(OS_WINDOWS)
+        // on windows, aclk log goes to windows events
+        snprintfz(filename, FILENAME_MAX, "%s", os_default_method);
+#else
         snprintfz(filename, FILENAME_MAX, "%s/aclk.log", netdata_configured_log_dir);
+#endif
         nd_log_set_user_settings(NDLS_ACLK, config_get(CONFIG_SECTION_CLOUD, "conversation log file", filename));
     }
-#endif
+
+    aclk_config_get_query_scope();
 }
 
-char *initialize_lock_directory_path(char *prefix)
-{
+static const char *get_varlib_subdir_from_config(const char *prefix, const char *dir) {
     char filename[FILENAME_MAX + 1];
-    snprintfz(filename, FILENAME_MAX, "%s/lock", prefix);
-
-    return config_get(CONFIG_SECTION_DIRECTORIES, "lock", filename);
+    snprintfz(filename, FILENAME_MAX, "%s/%s", prefix, dir);
+    return config_get(CONFIG_SECTION_DIRECTORIES, dir, filename);
 }
 
 static void backwards_compatible_config() {
     // move [global] options to the [web] section
+
     config_move(CONFIG_SECTION_GLOBAL, "http port listen backlog",
                 CONFIG_SECTION_WEB,    "listen backlog");
 
@@ -1003,7 +1019,10 @@ static void backwards_compatible_config() {
                 CONFIG_SECTION_PLUGINS, "statsd");
 
     config_move(CONFIG_SECTION_GLOBAL,  "memory mode",
-                CONFIG_SECTION_DB,      "mode");
+                CONFIG_SECTION_DB,      "db");
+
+    config_move(CONFIG_SECTION_DB,      "mode",
+                CONFIG_SECTION_DB,      "db");
 
     config_move(CONFIG_SECTION_GLOBAL,  "history",
                 CONFIG_SECTION_DB,      "retention");
@@ -1012,7 +1031,13 @@ static void backwards_compatible_config() {
                 CONFIG_SECTION_DB,      "update every");
 
     config_move(CONFIG_SECTION_GLOBAL,  "page cache size",
-                CONFIG_SECTION_DB,      "dbengine page cache size MB");
+                CONFIG_SECTION_DB,      "dbengine page cache size");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine page cache size MB",
+                CONFIG_SECTION_DB,      "dbengine page cache size");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine extent cache size MB",
+                CONFIG_SECTION_DB,      "dbengine extent cache size");
 
     config_move(CONFIG_SECTION_DB,      "page cache size",
                 CONFIG_SECTION_DB,      "dbengine page cache size MB");
@@ -1023,30 +1048,6 @@ static void backwards_compatible_config() {
     config_move(CONFIG_SECTION_DB,      "page cache with malloc",
                 CONFIG_SECTION_DB,      "dbengine page cache with malloc");
 
-    config_move(CONFIG_SECTION_GLOBAL,  "dbengine disk space",
-                CONFIG_SECTION_DB,      "dbengine disk space MB");
-
-    config_move(CONFIG_SECTION_GLOBAL,  "dbengine multihost disk space",
-                CONFIG_SECTION_DB,      "dbengine multihost disk space MB");
-
-    config_move(CONFIG_SECTION_DB,      "dbengine disk space MB",
-                CONFIG_SECTION_DB,      "dbengine multihost disk space MB");
-
-    config_move(CONFIG_SECTION_DB,      "dbengine multihost disk space MB",
-                CONFIG_SECTION_DB,      "dbengine tier 0 disk space MB");
-
-    config_move(CONFIG_SECTION_DB,      "dbengine tier 1 multihost disk space MB",
-                CONFIG_SECTION_DB,      "dbengine tier 1 disk space MB");
-
-    config_move(CONFIG_SECTION_DB,      "dbengine tier 2 multihost disk space MB",
-                CONFIG_SECTION_DB,      "dbengine tier 2 disk space MB");
-
-    config_move(CONFIG_SECTION_DB,      "dbengine tier 3 multihost disk space MB",
-                CONFIG_SECTION_DB,      "dbengine tier 3 disk space MB");
-
-    config_move(CONFIG_SECTION_DB,      "dbengine tier 4 multihost disk space MB",
-                CONFIG_SECTION_DB,      "dbengine tier 4 disk space MB");
-
     config_move(CONFIG_SECTION_GLOBAL,  "memory deduplication (ksm)",
                 CONFIG_SECTION_DB,      "memory deduplication (ksm)");
 
@@ -1060,17 +1061,67 @@ static void backwards_compatible_config() {
                 CONFIG_SECTION_DB,      "dbengine pages per extent");
 
     config_move(CONFIG_SECTION_GLOBAL,  "cleanup obsolete charts after seconds",
-                CONFIG_SECTION_DB,      "cleanup obsolete charts after secs");
+                CONFIG_SECTION_DB,      "cleanup obsolete charts after");
+
+    config_move(CONFIG_SECTION_DB,      "cleanup obsolete charts after secs",
+                CONFIG_SECTION_DB,      "cleanup obsolete charts after");
 
     config_move(CONFIG_SECTION_GLOBAL,  "gap when lost iterations above",
                 CONFIG_SECTION_DB,      "gap when lost iterations above");
 
     config_move(CONFIG_SECTION_GLOBAL,  "cleanup orphan hosts after seconds",
-                CONFIG_SECTION_DB,      "cleanup orphan hosts after secs");
+                CONFIG_SECTION_DB,      "cleanup orphan hosts after");
+
+    config_move(CONFIG_SECTION_DB,      "cleanup orphan hosts after secs",
+                CONFIG_SECTION_DB,      "cleanup orphan hosts after");
+
+    config_move(CONFIG_SECTION_DB,      "cleanup ephemeral hosts after secs",
+                CONFIG_SECTION_DB,      "cleanup ephemeral hosts after");
+
+    config_move(CONFIG_SECTION_DB,      "seconds to replicate",
+                CONFIG_SECTION_DB,      "replication period");
+
+    config_move(CONFIG_SECTION_DB,      "seconds per replication step",
+                CONFIG_SECTION_DB,      "replication step");
 
     config_move(CONFIG_SECTION_GLOBAL,  "enable zero metrics",
                 CONFIG_SECTION_DB,      "enable zero metrics");
 
+    // ----------------------------------------------------------------------------------------------------------------
+
+    config_move(CONFIG_SECTION_GLOBAL,  "dbengine disk space",
+                CONFIG_SECTION_DB,      "dbengine tier 0 retention size");
+
+    config_move(CONFIG_SECTION_GLOBAL,  "dbengine multihost disk space",
+                CONFIG_SECTION_DB,      "dbengine tier 0 retention size");
+
+    config_move(CONFIG_SECTION_DB,      "dbengine disk space MB",
+                CONFIG_SECTION_DB,      "dbengine tier 0 retention size");
+
+    for(size_t tier = 0; tier < RRD_STORAGE_TIERS ;tier++) {
+        char old_config[128], new_config[128];
+
+        snprintfz(old_config, sizeof(old_config), "dbengine tier %zu retention days", tier);
+        snprintfz(new_config, sizeof(new_config), "dbengine tier %zu retention time", tier);
+        config_move(CONFIG_SECTION_DB, old_config,
+                    CONFIG_SECTION_DB, new_config);
+
+        if(tier == 0)
+            snprintfz(old_config, sizeof(old_config), "dbengine multihost disk space MB");
+        else
+            snprintfz(old_config, sizeof(old_config), "dbengine tier %zu multihost disk space MB", tier);
+        snprintfz(new_config, sizeof(new_config), "dbengine tier %zu retention size", tier);
+        config_move(CONFIG_SECTION_DB, old_config,
+                    CONFIG_SECTION_DB, new_config);
+
+        snprintfz(old_config, sizeof(old_config), "dbengine tier %zu disk space MB", tier);
+        snprintfz(new_config, sizeof(new_config), "dbengine tier %zu retention size", tier);
+        config_move(CONFIG_SECTION_DB, old_config,
+                    CONFIG_SECTION_DB, new_config);
+    }
+
+    // ----------------------------------------------------------------------------------------------------------------
+
     config_move(CONFIG_SECTION_LOGS,   "error",
                 CONFIG_SECTION_LOGS,   "daemon");
 
@@ -1082,11 +1133,42 @@ static void backwards_compatible_config() {
 
     config_move(CONFIG_SECTION_LOGS,   "errors flood protection period",
                 CONFIG_SECTION_LOGS,   "logs flood protection period");
+
     config_move(CONFIG_SECTION_HEALTH, "is ephemeral",
                 CONFIG_SECTION_GLOBAL, "is ephemeral node");
 
     config_move(CONFIG_SECTION_HEALTH, "has unstable connection",
                 CONFIG_SECTION_GLOBAL, "has unstable connection");
+
+    config_move(CONFIG_SECTION_HEALTH, "run at least every seconds",
+                CONFIG_SECTION_HEALTH, "run at least every");
+
+    config_move(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds",
+                CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for");
+
+    config_move(CONFIG_SECTION_HEALTH, "health log history",
+                CONFIG_SECTION_HEALTH, "health log retention");
+
+    config_move(CONFIG_SECTION_REGISTRY, "registry expire idle persons days",
+                CONFIG_SECTION_REGISTRY, "registry expire idle persons");
+
+    config_move(CONFIG_SECTION_WEB, "disconnect idle clients after seconds",
+                CONFIG_SECTION_WEB, "disconnect idle clients after");
+
+    config_move(CONFIG_SECTION_WEB, "accept a streaming request every seconds",
+                CONFIG_SECTION_WEB, "accept a streaming request every");
+
+    config_move(CONFIG_SECTION_STATSD, "set charts as obsolete after secs",
+                CONFIG_SECTION_STATSD, "set charts as obsolete after");
+
+    config_move(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds",
+                CONFIG_SECTION_STATSD, "disconnect idle tcp clients after");
+
+    config_move("plugin:idlejitter", "loop time in ms",
+                "plugin:idlejitter", "loop time");
+
+    config_move("plugin:proc:/sys/class/infiniband", "refresh ports state every seconds",
+                "plugin:proc:/sys/class/infiniband", "refresh ports state every");
 }
 
 static int get_hostname(char *buf, size_t buf_size) {
@@ -1119,7 +1201,7 @@ static void get_netdata_configured_variables()
     // get the hostname
 
     netdata_configured_host_prefix = config_get(CONFIG_SECTION_GLOBAL, "host access prefix", "");
-    verify_netdata_host_prefix(true);
+    (void) verify_netdata_host_prefix(true);
 
     char buf[HOSTNAME_MAX + 1];
     if (get_hostname(buf, HOSTNAME_MAX))
@@ -1131,22 +1213,22 @@ static void get_netdata_configured_variables()
     // ------------------------------------------------------------------------
     // get default database update frequency
 
-    default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", UPDATE_EVERY);
+    default_rrd_update_every = (int) config_get_duration_seconds(CONFIG_SECTION_DB, "update every", UPDATE_EVERY);
     if(default_rrd_update_every < 1 || default_rrd_update_every > 600) {
         netdata_log_error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY);
         default_rrd_update_every = UPDATE_EVERY;
-        config_set_number(CONFIG_SECTION_DB, "update every", default_rrd_update_every);
+        config_set_duration_seconds(CONFIG_SECTION_DB, "update every", default_rrd_update_every);
     }
 
     // ------------------------------------------------------------------------
-    // get default memory mode for the database
+    // get the database selection
 
     {
-        const char *mode = config_get(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
+        const char *mode = config_get(CONFIG_SECTION_DB, "db", rrd_memory_mode_name(default_rrd_memory_mode));
         default_rrd_memory_mode = rrd_memory_mode_id(mode);
         if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) {
             netdata_log_error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode));
-            config_set(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode));
+            config_set(CONFIG_SECTION_DB, "db", rrd_memory_mode_name(default_rrd_memory_mode));
         }
     }
 
@@ -1175,7 +1257,8 @@ static void get_netdata_configured_variables()
     netdata_configured_cache_dir        = config_get(CONFIG_SECTION_DIRECTORIES, "cache",        netdata_configured_cache_dir);
     netdata_configured_varlib_dir       = config_get(CONFIG_SECTION_DIRECTORIES, "lib",          netdata_configured_varlib_dir);
 
-    netdata_configured_lock_dir = initialize_lock_directory_path(netdata_configured_varlib_dir);
+    netdata_configured_lock_dir = get_varlib_subdir_from_config(netdata_configured_varlib_dir, "lock");
+    netdata_configured_cloud_dir = get_varlib_subdir_from_config(netdata_configured_varlib_dir, "cloud.d");
 
     {
         pluginsd_initialize_plugin_directories();
@@ -1199,17 +1282,19 @@ static void get_netdata_configured_variables()
     // ------------------------------------------------------------------------
     // get default Database Engine page cache size in MiB
 
-    default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
-    default_rrdeng_extent_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine extent cache size MB", default_rrdeng_extent_cache_mb);
+    default_rrdeng_page_cache_mb = (int) config_get_size_mb(CONFIG_SECTION_DB, "dbengine page cache size", default_rrdeng_page_cache_mb);
+    default_rrdeng_extent_cache_mb = (int) config_get_size_mb(CONFIG_SECTION_DB, "dbengine extent cache size", default_rrdeng_extent_cache_mb);
     db_engine_journal_check = config_get_boolean(CONFIG_SECTION_DB, "dbengine enable journal integrity check", CONFIG_BOOLEAN_NO);
 
-    if(default_rrdeng_extent_cache_mb < 0)
+    if(default_rrdeng_extent_cache_mb < 0) {
         default_rrdeng_extent_cache_mb = 0;
+        config_set_size_mb(CONFIG_SECTION_DB, "dbengine extent cache size", default_rrdeng_extent_cache_mb);
+    }
 
     if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
         netdata_log_error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
         default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
-        config_set_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
+        config_set_size_mb(CONFIG_SECTION_DB, "dbengine page cache size", default_rrdeng_page_cache_mb);
     }
 
     // ------------------------------------------------------------------------
@@ -1242,28 +1327,24 @@ static void get_netdata_configured_variables()
     // get KSM settings
 
 #ifdef MADV_MERGEABLE
-    enable_ksm = config_get_boolean(CONFIG_SECTION_DB, "memory deduplication (ksm)", enable_ksm);
+    enable_ksm = config_get_boolean_ondemand(CONFIG_SECTION_DB, "memory deduplication (ksm)", enable_ksm);
 #endif
 
     // --------------------------------------------------------------------
-    // metric correlations
 
-    enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
-    default_metric_correlations_method = weights_string_to_method(config_get(
-        CONFIG_SECTION_GLOBAL, "metric correlations method",
-        weights_method_to_string(default_metric_correlations_method)));
+    rrdhost_free_ephemeral_time_s =
+        config_get_duration_seconds(CONFIG_SECTION_DB, "cleanup ephemeral hosts after", rrdhost_free_ephemeral_time_s);
 
-    // --------------------------------------------------------------------
+    rrdset_free_obsolete_time_s =
+        config_get_duration_seconds(CONFIG_SECTION_DB, "cleanup obsolete charts after", rrdset_free_obsolete_time_s);
 
-    rrdset_free_obsolete_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time_s);
-    rrdhost_free_ephemeral_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup ephemeral hosts after secs", rrdhost_free_ephemeral_time_s);
     // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
     // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
     // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
     if (rrdset_free_obsolete_time_s < 10) {
         rrdset_free_obsolete_time_s = 10;
-        netdata_log_info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds.");
-        config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time_s);
+        netdata_log_info("The \"cleanup obsolete charts after\" option was set to 10 seconds.");
+        config_set_duration_seconds(CONFIG_SECTION_DB, "cleanup obsolete charts after", rrdset_free_obsolete_time_s);
     }
 
     gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
@@ -1276,14 +1357,13 @@ static void get_netdata_configured_variables()
     // --------------------------------------------------------------------
     // get various system parameters
 
-    os_get_system_HZ();
     os_get_system_cpus_uncached();
     os_get_system_pid_max();
 
 
 }
 
-static void post_conf_load(char **user)
+static void post_conf_load(const char **user)
 {
     // --------------------------------------------------------------------
     // get the user we should run
@@ -1298,7 +1378,7 @@ static void post_conf_load(char **user)
     }
 }
 
-static bool load_netdata_conf(char *filename, char overwrite_used, char **user) {
+static bool load_netdata_conf(char *filename, char overwrite_used, const char **user) {
     errno_clear();
 
     int ret = 0;
@@ -1309,14 +1389,14 @@ static bool load_netdata_conf(char *filename, char overwrite_used, char **user)
             netdata_log_error("CONFIG: cannot load config file '%s'.", filename);
     }
     else {
-        filename = strdupz_path_subpath(netdata_configured_user_config_dir, "netdata.conf");
+        filename = filename_from_path_entry_strdupz(netdata_configured_user_config_dir, "netdata.conf");
 
         ret = config_load(filename, overwrite_used, NULL);
         if(!ret) {
             netdata_log_info("CONFIG: cannot load user config '%s'. Will try the stock version.", filename);
             freez(filename);
 
-            filename = strdupz_path_subpath(netdata_configured_stock_config_dir, "netdata.conf");
+            filename = filename_from_path_entry_strdupz(netdata_configured_stock_config_dir, "netdata.conf");
             ret = config_load(filename, overwrite_used, NULL);
             if(!ret)
                 netdata_log_info("CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename);
@@ -1351,7 +1431,7 @@ int get_system_info(struct rrdhost_system_info *system_info) {
         char line[200 + 1];
         // Removed the double strlens, if the Coverity tainted string warning reappears I'll revert.
         // One time init code, but I'm curious about the warning...
-        while (fgets(line, 200, instance->child_stdout_fp) != NULL) {
+        while (fgets(line, 200, spawn_popen_stdout(instance)) != NULL) {
             char *value=line;
             while (*value && *value != '=') value++;
             if (*value=='=') {
@@ -1366,7 +1446,7 @@ int get_system_info(struct rrdhost_system_info *system_info) {
                 if(unlikely(rrdhost_set_system_info_variable(system_info, line, value))) {
                     netdata_log_error("Unexpected environment variable %s=%s", line, value);
                 } else {
-                    setenv(line, value, 1);
+                    nd_setenv(line, value, 1);
                 }
             }
         }
@@ -1405,12 +1485,13 @@ int unittest_rrdpush_compressions(void);
 int uuid_unittest(void);
 int progress_unittest(void);
 int dyncfg_unittest(void);
+bool netdata_random_session_id_generate(void);
 
 #ifdef OS_WINDOWS
 int windows_perflib_dump(const char *key);
 #endif
 
-int unittest_prepare_rrd(char **user) {
+int unittest_prepare_rrd(const char **user) {
     post_conf_load(user);
     get_netdata_configured_variables();
     default_rrd_update_every = 1;
@@ -1422,13 +1503,12 @@ int unittest_prepare_rrd(char **user) {
         fprintf(stderr, "rrd_init failed for unittest\n");
         return 1;
     }
-    default_rrdpush_enabled = 0;
+    stream_conf_send_enabled = 0;
 
     return 0;
 }
 
 int netdata_main(int argc, char **argv) {
-    clocks_init();
     string_init();
     analytics_init();
 
@@ -1441,7 +1521,7 @@ int netdata_main(int argc, char **argv) {
     int config_loaded = 0;
     bool close_open_fds = true;
     size_t default_stacksize;
-    char *user = NULL;
+    const char *user = NULL;
 
 #ifdef OS_WINDOWS
     int dont_fork = 1;
@@ -1455,6 +1535,8 @@ int netdata_main(int argc, char **argv) {
     // set the name for logging
     program_name = "netdata";
 
+    curl_global_init(CURL_GLOBAL_ALL);
+
     // parse options
     {
         int num_opts = sizeof(option_definitions) / sizeof(struct option_def);
@@ -1483,7 +1565,7 @@ int netdata_main(int argc, char **argv) {
                     }
                     else {
                         netdata_log_debug(D_OPTIONS, "Configuration loaded from %s.", optarg);
-                        load_cloud_conf(1);
+                        cloud_conf_load(1);
                         config_loaded = 1;
                     }
                     break;
@@ -1499,8 +1581,7 @@ int netdata_main(int argc, char **argv) {
                     config_set(CONFIG_SECTION_WEB, "bind to", optarg);
                     break;
                 case 'P':
-                    strncpy(pidfile, optarg, FILENAME_MAX);
-                    pidfile[FILENAME_MAX] = '\0';
+                    pidfile = strdupz(optarg);
                     break;
                 case 'p':
                     config_set(CONFIG_SECTION_GLOBAL, "default port", optarg);
@@ -1522,7 +1603,6 @@ int netdata_main(int argc, char **argv) {
                     {
                         char* stacksize_string = "stacksize=";
                         char* debug_flags_string = "debug_flags=";
-                        char* claim_string = "claim";
 #ifdef ENABLE_DBENGINE
                         char* createdataset_string = "createdataset=";
                         char* stresstest_string = "stresstest=";
@@ -1791,7 +1871,7 @@ int netdata_main(int argc, char **argv) {
                             // so the caller can use -c netdata.conf before or
                             // after this parameter to prevent or allow overwriting
                             // variables at netdata.conf
-                            config_set_default(section, key,  value);
+                            config_set_default_raw_value(section, key, value);
 
                             // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value);
                         }
@@ -1824,7 +1904,7 @@ int netdata_main(int argc, char **argv) {
                             // so the caller can use -c netdata.conf before or
                             // after this parameter to prevent or allow overwriting
                             // variables at netdata.conf
-                            appconfig_set_default(tmp_config, section, key,  value);
+                            appconfig_set_default_raw_value(tmp_config, section, key, value);
 
                             // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value);
                         }
@@ -1870,7 +1950,7 @@ int netdata_main(int argc, char **argv) {
                             if(!config_loaded) {
                                 fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n");
                                 load_netdata_conf(NULL, 0, &user);
-                                load_cloud_conf(1);
+                                cloud_conf_load(1);
                             }
 
                             get_netdata_configured_variables();
@@ -1884,10 +1964,6 @@ int netdata_main(int argc, char **argv) {
                             printf("%s\n", value);
                             return 0;
                         }
-                        else if(strncmp(optarg, claim_string, strlen(claim_string)) == 0) {
-                            /* will trigger a claiming attempt when the agent is initialized */
-                            claiming_pending_arguments = optarg + strlen(claim_string);
-                        }
                         else if(strcmp(optarg, "buildinfo") == 0) {
                             print_build_info();
                             return 0;
@@ -1919,18 +1995,18 @@ int netdata_main(int argc, char **argv) {
     if (close_open_fds == true) {
         // close all open file descriptors, except the standard ones
         // the caller may have left open files (lxc-attach has this issue)
-        os_close_all_non_std_open_fds_except(NULL, 0);
+        os_close_all_non_std_open_fds_except(NULL, 0, 0);
     }
 
     if(!config_loaded) {
         load_netdata_conf(NULL, 0, &user);
-        load_cloud_conf(0);
+        cloud_conf_load(0);
     }
 
     // ------------------------------------------------------------------------
     // initialize netdata
     {
-        char *pmax = config_get(CONFIG_SECTION_GLOBAL, "glibc malloc arena max for plugins", "1");
+        const char *pmax = config_get(CONFIG_SECTION_GLOBAL, "glibc malloc arena max for plugins", "1");
         if(pmax && *pmax)
             setenv("MALLOC_ARENA_MAX", pmax, 1);
 
@@ -1970,7 +2046,8 @@ int netdata_main(int argc, char **argv) {
 
         // prepare configuration environment variables for the plugins
         get_netdata_configured_variables();
-        set_global_environment();
+        set_environment_for_plugins_and_scripts();
+        analytics_reset();
 
         // work while we are cd into config_dir
         // to allow the plugins refer to their config
@@ -1986,8 +2063,8 @@ int netdata_main(int argc, char **argv) {
         // --------------------------------------------------------------------
         // get the debugging flags from the configuration file
 
-        char *flags = config_get(CONFIG_SECTION_LOGS, "debug flags",  "0x0000000000000000");
-        setenv("NETDATA_DEBUG_FLAGS", flags, 1);
+        const char *flags = config_get(CONFIG_SECTION_LOGS, "debug flags",  "0x0000000000000000");
+        nd_setenv("NETDATA_DEBUG_FLAGS", flags, 1);
 
         debug_flags = strtoull(flags, NULL, 0);
         netdata_log_debug(D_OPTIONS, "Debug flags set to '0x%" PRIX64 "'.", debug_flags);
@@ -2013,16 +2090,10 @@ int netdata_main(int argc, char **argv) {
         nd_log_initialize();
         netdata_log_info("Netdata agent version '%s' is starting", NETDATA_VERSION);
 
-        ieee754_doubles = is_system_ieee754_double();
-        if(!ieee754_doubles)
-            globally_disabled_capabilities |= STREAM_CAP_IEEE754;
-
-        aral_judy_init();
+        check_local_streaming_capabilities();
 
         get_system_timezone();
 
-        bearer_tokens_init();
-
         replication_initialize();
 
         rrd_functions_inflight_init();
@@ -2030,9 +2101,7 @@ int netdata_main(int argc, char **argv) {
         // --------------------------------------------------------------------
         // get the certificate and start security
 
-#ifdef ENABLE_HTTPS
         security_init();
-#endif
 
         // --------------------------------------------------------------------
         // This is the safest place to start the SILENCERS structure
@@ -2040,12 +2109,6 @@ int netdata_main(int argc, char **argv) {
         health_set_silencers_filename();
         health_initialize_global_silencers();
 
-//        // --------------------------------------------------------------------
-//        // Initialize ML configuration
-//
-//        delta_startup_time("initialize ML");
-//        ml_init();
-
         // --------------------------------------------------------------------
         // setup process signals
 
@@ -2053,8 +2116,7 @@ int netdata_main(int argc, char **argv) {
         // this causes the threads to block signals.
 
         delta_startup_time("initialize signals");
-        signals_block();
-        signals_init(); // setup the signals we want to use
+        nd_initialize_signals(); // setup the signals we want to use
 
         // --------------------------------------------------------------------
         // check which threads are enabled and initialize them
@@ -2086,7 +2148,7 @@ int netdata_main(int argc, char **argv) {
                 st->init_routine();
 
             if(st->env_name)
-                setenv(st->env_name, st->enabled?"YES":"NO", 1);
+                nd_setenv(st->env_name, st->enabled?"YES":"NO", 1);
 
             if(st->global_variable)
                 *st->global_variable = (st->enabled) ? true : false;
@@ -2097,7 +2159,7 @@ int netdata_main(int argc, char **argv) {
 
         delta_startup_time("initialize web server");
 
-        web_client_api_v1_init();
+        nd_web_api_init();
         web_server_threading_selection();
 
         if(web_server_mode != WEB_SERVER_MODE_NONE) {
@@ -2165,7 +2227,7 @@ int netdata_main(int argc, char **argv) {
         netdata_configured_home_dir = config_get(CONFIG_SECTION_DIRECTORIES, "home", pw->pw_dir);
     }
 
-    setenv("HOME", netdata_configured_home_dir, 1);
+    nd_setenv("HOME", netdata_configured_home_dir, 1);
 
     dyncfg_init(true);
 
@@ -2173,11 +2235,12 @@ int netdata_main(int argc, char **argv) {
 
     delta_startup_time("initialize threads after fork");
 
-    netdata_threads_init_after_fork((size_t)config_get_number(CONFIG_SECTION_GLOBAL, "pthread stack size", (long)default_stacksize));
+    netdata_threads_init_after_fork((size_t)config_get_size_bytes(CONFIG_SECTION_GLOBAL, "pthread stack size", default_stacksize));
 
     // initialize internal registry
     delta_startup_time("initialize registry");
     registry_init();
+    cloud_conf_init_after_registry();
     netdata_random_session_id_generate();
 
     // ------------------------------------------------------------------------
@@ -2203,7 +2266,7 @@ int netdata_main(int argc, char **argv) {
     delta_startup_time("initialize RRD structures");
 
     if(rrd_init(netdata_configured_hostname, system_info, false)) {
-        set_late_global_environment(system_info);
+        set_late_analytics_variables(system_info);
         fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname);
     }
 
@@ -2219,15 +2282,10 @@ int netdata_main(int argc, char **argv) {
     if (fd >= 0)
         close(fd);
 
-
     // ------------------------------------------------------------------------
     // Claim netdata agent to a cloud endpoint
 
     delta_startup_time("collect claiming info");
-
-    if (claiming_pending_arguments)
-         claim_agent(claiming_pending_arguments, false, NULL);
-
     load_claiming_state();
 
     // ------------------------------------------------------------------------
@@ -2242,11 +2300,13 @@ int netdata_main(int argc, char **argv) {
     // ------------------------------------------------------------------------
     // spawn the threads
 
+    bearer_tokens_init();
+
     delta_startup_time("start the static threads");
 
     web_server_config_options();
 
-    set_late_global_environment(system_info);
+    set_late_analytics_variables(system_info);
     for (i = 0; static_threads[i].name != NULL ; i++) {
         struct netdata_static_thread *st = &static_threads[i];
 
@@ -2269,7 +2329,13 @@ int netdata_main(int argc, char **argv) {
     delta_startup_time("ready");
 
     usec_t ready_ut = now_monotonic_usec();
-    netdata_log_info("NETDATA STARTUP: completed in %llu ms. Enjoy real-time performance monitoring!", (ready_ut - started_ut) / USEC_PER_MS);
+    add_agent_event(EVENT_AGENT_START_TIME, (int64_t ) (ready_ut - started_ut));
+    usec_t median_start_time = get_agent_event_time_median(EVENT_AGENT_START_TIME);
+    netdata_log_info(
+        "NETDATA STARTUP: completed in %llu ms (median start up time is %llu ms). Enjoy real-time performance monitoring!",
+        (ready_ut - started_ut) / USEC_PER_MS, median_start_time / USEC_PER_MS);
+
+    cleanup_agent_event_log();
     netdata_ready = true;
 
     analytics_statistic_t start_statistic = { "START", "-",  "-" };
@@ -2295,28 +2361,7 @@ int netdata_main(int argc, char **argv) {
         }
     }
 
-    // ------------------------------------------------------------------------
-    // Report ACLK build failure
-#ifndef ENABLE_ACLK
-    netdata_log_error("This agent doesn't have ACLK.");
-    char filename[FILENAME_MAX + 1];
-    snprintfz(filename, FILENAME_MAX, "%s/.aclk_report_sent", netdata_configured_varlib_dir);
-    if (netdata_anonymous_statistics_enabled > 0 && access(filename, F_OK)) { // -1 -> not initialized
-        analytics_statistic_t statistic = { "ACLK_DISABLED", "-", "-" };
-        analytics_statistic_send(&statistic);
-
-        int fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 444);
-        if (fd == -1)
-            netdata_log_error("Cannot create file '%s'. Please fix this.", filename);
-        else
-            close(fd);
-    }
-#endif
-
     webrtc_initialize();
-
-    signals_unblock();
-
     return 10;
 }
 
@@ -2327,7 +2372,7 @@ int main(int argc, char *argv[])
     if (rc != 10)
         return rc;
 
-    signals_handle();
+    nd_process_signals();
     return 1;
 }
 #endif