diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-08-26 08:15:24 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-08-26 08:15:35 +0000 |
commit | f09848204fa5283d21ea43e262ee41aa578e1808 (patch) | |
tree | c62385d7adf209fa6a798635954d887f718fb3fb /src/daemon | |
parent | Releasing debian version 1.46.3-2. (diff) | |
download | netdata-f09848204fa5283d21ea43e262ee41aa578e1808.tar.xz netdata-f09848204fa5283d21ea43e262ee41aa578e1808.zip |
Merging upstream version 1.47.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/daemon')
-rw-r--r-- | src/daemon/analytics.c | 30 | ||||
-rw-r--r-- | src/daemon/analytics.h | 1 | ||||
-rw-r--r-- | src/daemon/buildinfo.c | 33 | ||||
-rw-r--r-- | src/daemon/commands.c | 97 | ||||
-rw-r--r-- | src/daemon/common.c | 2 | ||||
-rw-r--r-- | src/daemon/common.h | 3 | ||||
-rw-r--r-- | src/daemon/config/README.md | 19 | ||||
-rw-r--r-- | src/daemon/daemon.c | 6 | ||||
-rw-r--r-- | src/daemon/global_statistics.c | 488 | ||||
-rw-r--r-- | src/daemon/main.c | 129 | ||||
-rw-r--r-- | src/daemon/main.h | 1 | ||||
-rw-r--r-- | src/daemon/signals.c | 83 | ||||
-rw-r--r-- | src/daemon/signals.h | 1 | ||||
-rw-r--r-- | src/daemon/static_threads.c | 23 | ||||
-rw-r--r-- | src/daemon/unit_test.c | 17 | ||||
-rw-r--r-- | src/daemon/watcher.c | 3 | ||||
-rw-r--r-- | src/daemon/watcher.h | 1 | ||||
-rw-r--r-- | src/daemon/win_system-info.c | 318 | ||||
-rw-r--r-- | src/daemon/win_system-info.h | 20 | ||||
-rw-r--r-- | src/daemon/winsvc.cc | 252 |
20 files changed, 1006 insertions, 521 deletions
diff --git a/src/daemon/analytics.c b/src/daemon/analytics.c index 33f6f357f..0e5c221c4 100644 --- a/src/daemon/analytics.c +++ b/src/daemon/analytics.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include "analytics.h" #include "common.h" #include "buildinfo.h" @@ -325,18 +326,15 @@ void analytics_alarms_notifications(void) strcat(script, " dump_methods"); - pid_t command_pid; - netdata_log_debug(D_ANALYTICS, "Executing %s", script); BUFFER *b = buffer_create(1000, NULL); int cnt = 0; - FILE *fp_child_input; - FILE *fp_child_output = netdata_popen(script, &command_pid, &fp_child_input); - if (fp_child_output) { + POPEN_INSTANCE *instance = spawn_popen_run(script); + if (instance) { char line[200 + 1]; - while (fgets(line, 200, fp_child_output) != NULL) { + while (fgets(line, 200, instance->child_stdout_fp) != NULL) { char *end = line; while (*end && *end != '\n') end++; @@ -349,7 +347,7 @@ void analytics_alarms_notifications(void) cnt++; } - netdata_pclose(fp_child_input, fp_child_output, command_pid); + spawn_popen_wait(instance); } freez(script); @@ -470,8 +468,6 @@ void analytics_alarms(void) */ void analytics_misc(void) { - spinlock_init(&analytics_data.spinlock); - #ifdef ENABLE_ACLK analytics_set_data(&analytics_data.netdata_host_cloud_available, "true"); analytics_set_data_str(&analytics_data.netdata_host_aclk_implementation, "Next Generation"); @@ -1002,8 +998,6 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) { char *command_to_run = mallocz( sizeof(char) * (strlen(statistic->action) + strlen(action_result) + strlen(action_data) + strlen(as_script) + analytics_data.data_length + (ANALYTICS_NO_OF_ITEMS * 3) + 15)); - pid_t command_pid; - sprintf( command_to_run, "%s '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' ", @@ -1056,12 +1050,11 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) { "%s '%s' '%s' '%s'", as_script, statistic->action, action_result, action_data); - FILE *fp_child_input; - FILE *fp_child_output = netdata_popen(command_to_run, &command_pid, &fp_child_input); - if (fp_child_output) { + POPEN_INSTANCE *instance = spawn_popen_run(command_to_run); + if (instance) { char buffer[4 + 1]; - char *s = fgets(buffer, 4, fp_child_output); - int exit_code = netdata_pclose(fp_child_input, fp_child_output, command_pid); + char *s = fgets(buffer, 4, instance->child_stdout_fp); + int exit_code = spawn_popen_wait(instance); if (exit_code) nd_log(NDLS_DAEMON, NDLP_NOTICE, @@ -1081,3 +1074,8 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) { freez(command_to_run); } + +void analytics_init(void) +{ + spinlock_init(&analytics_data.spinlock); +} diff --git a/src/daemon/analytics.h b/src/daemon/analytics.h index 501eb7b55..747cf6070 100644 --- a/src/daemon/analytics.h +++ b/src/daemon/analytics.h @@ -86,6 +86,7 @@ void analytics_log_dashboard(void); void analytics_gather_mutable_meta_data(void); void analytics_report_oom_score(long long int score); void get_system_timezone(void); +void analytics_init(void); typedef struct { const char *action; diff --git a/src/daemon/buildinfo.c b/src/daemon/buildinfo.c index 4ee5b43de..ace96199a 100644 --- a/src/daemon/buildinfo.c +++ b/src/daemon/buildinfo.c @@ -75,6 +75,7 @@ typedef enum __attribute__((packed)) { BIB_LIB_LIBCAP, BIB_LIB_LIBCRYPTO, BIB_LIB_LIBYAML, + BIB_LIB_LIBMNL, BIB_PLUGIN_APPS, BIB_PLUGIN_LINUX_CGROUPS, BIB_PLUGIN_LINUX_CGROUP_NETWORK, @@ -96,7 +97,6 @@ typedef enum __attribute__((packed)) { BIB_PLUGIN_SLABINFO, BIB_PLUGIN_XEN, BIB_PLUGIN_XEN_VBD_ERROR, - BIB_PLUGIN_LOGS_MANAGEMENT, BIB_EXPORT_AWS_KINESIS, BIB_EXPORT_GCP_PUBSUB, BIB_EXPORT_MONGOC, @@ -698,6 +698,14 @@ static struct { .json = "libyaml", .value = NULL, }, + [BIB_LIB_LIBMNL] = { + .category = BIC_LIBS, + .type = BIT_BOOLEAN, + .analytics = "libmnl", + .print = "libmnl (library for working with netfilter)", + .json = "libmnl", + .value = NULL, + }, [BIB_PLUGIN_APPS] = { .category = BIC_PLUGINS, .type = BIT_BOOLEAN, @@ -866,14 +874,6 @@ static struct { .json = "xen-vbd-error", .value = NULL, }, - [BIB_PLUGIN_LOGS_MANAGEMENT] = { - .category = BIC_PLUGINS, - .type = BIT_BOOLEAN, - .analytics = "Logs Management", - .print = "Logs Management", - .json = "logs-management", - .value = NULL, - }, [BIB_EXPORT_MONGOC] = { .category = BIC_EXPORTERS, .type = BIT_BOOLEAN, @@ -1177,6 +1177,9 @@ __attribute__((constructor)) void initialize_build_info(void) { #ifdef HAVE_LIBYAML build_info_set_status(BIB_LIB_LIBYAML, true); #endif +#ifdef HAVE_LIBMNL + build_info_set_status(BIB_LIB_LIBMNL, true); +#endif #ifdef ENABLE_PLUGIN_APPS build_info_set_status(BIB_PLUGIN_APPS, true); @@ -1217,9 +1220,6 @@ __attribute__((constructor)) void initialize_build_info(void) { #ifdef HAVE_XENSTAT_VBD_ERROR build_info_set_status(BIB_PLUGIN_XEN_VBD_ERROR, true); #endif -#ifdef ENABLE_LOGSMANAGEMENT - build_info_set_status(BIB_PLUGIN_LOGS_MANAGEMENT, true); -#endif build_info_set_status(BIB_EXPORT_PROMETHEUS_EXPORTER, true); build_info_set_status(BIB_EXPORT_GRAPHITE, true); @@ -1278,9 +1278,18 @@ static void populate_system_info(void) { system_info = localhost->system_info; } else { + bool started_spawn_server = false; + if(!netdata_main_spawn_server) { + started_spawn_server = true; + netdata_main_spawn_server_init(NULL, 0, NULL); + } + system_info = callocz(1, sizeof(struct rrdhost_system_info)); get_system_info(system_info); free_system_info = true; + + if(started_spawn_server) + netdata_main_spawn_server_cleanup(); } build_info_set_value_strdupz(BIB_OS_KERNEL_NAME, system_info->kernel_name); diff --git a/src/daemon/commands.c b/src/daemon/commands.c index 70ba11d42..230e8527e 100644 --- a/src/daemon/commands.c +++ b/src/daemon/commands.c @@ -136,7 +136,7 @@ static cmd_status_t cmd_help_execute(char *args, char **message) "dumpconfig\n" " Returns the current netdata.conf on stdout.\n" #ifdef ENABLE_ACLK - "remove-stale-node node_id|machine_guid\n" + "remove-stale-node node_id|machine_guid|hostname|ALL_NODES\n" " Unregisters and removes a node from the cloud.\n" #endif "version\n" @@ -164,7 +164,7 @@ static cmd_status_t cmd_reopen_logs_execute(char *args, char **message) (void)message; nd_log_limits_unlimited(); - nd_log_reopen_log_files(); + nd_log_reopen_log_files(true); nd_log_limits_reset(); return CMD_STATUS_SUCCESS; @@ -306,10 +306,17 @@ static cmd_status_t cmd_ping_execute(char *args, char **message) static cmd_status_t cmd_aclk_state(char *args, char **message) { netdata_log_info("COMMAND: Reopening aclk/cloud state."); +#ifdef ENABLE_ACLK if (strstr(args, "json")) *message = aclk_state_json(); else *message = aclk_state(); +#else + if (strstr(args, "json")) + *message = strdupz("{\"aclk-available\":false}"); + else + *message = strdupz("ACLK Available: No"); +#endif return CMD_STATUS_SUCCESS; } @@ -338,13 +345,48 @@ static cmd_status_t cmd_dumpconfig(char *args, char **message) } #ifdef ENABLE_ACLK + +static int remove_ephemeral_host(BUFFER *wb, RRDHOST *host, bool report_error) +{ + if (host == localhost) { + if (report_error) + buffer_sprintf(wb, "You cannot unregister the parent node (%s)", rrdhost_hostname(host)); + return 0; + } + + if (rrdhost_is_online(host)) { + if (report_error) + buffer_sprintf(wb, "Cannot unregister a live node (%s)", rrdhost_hostname(host)); + return 0; + } + + if (!rrdhost_option_check(host, RRDHOST_OPTION_EPHEMERAL_HOST)) { + rrdhost_option_set(host, RRDHOST_OPTION_EPHEMERAL_HOST); + sql_set_host_label(&host->host_uuid, "_is_ephemeral", "true"); + aclk_host_state_update(host, 0, 0); + unregister_node(host->machine_guid); + freez(host->node_id); + host->node_id = NULL; + buffer_sprintf(wb, "Unregistering node with machine guid %s, hostname = %s", host->machine_guid, rrdhost_hostname(host)); + rrd_wrlock(); + rrdhost_free___while_having_rrd_wrlock(host, true); + rrd_wrunlock(); + return 1; + } + if (report_error) + buffer_sprintf(wb, "Node with machine guid %s, hostname = %s is already unregistered", host->machine_guid, rrdhost_hostname(host)); + return 0; +} + +#define SQL_HOSTNAME_TO_REMOVE "SELECT host_id FROM host WHERE (hostname = @hostname OR @hostname = 'ALL_NODES')" + static cmd_status_t cmd_remove_node(char *args, char **message) { (void)args; BUFFER *wb = buffer_create(1024, NULL); if (strlen(args) == 0) { - buffer_sprintf(wb, "Please specify a machine or node UUID"); + buffer_sprintf(wb, "Please specify a machine or node UUID or hostname"); goto done; } @@ -353,32 +395,43 @@ static cmd_status_t cmd_remove_node(char *args, char **message) if (!host) host = find_host_by_node_id(args); - if (!host) - buffer_sprintf(wb, "Node with machine or node UUID \"%s\" not found", args); - else { + if (!host) { + sqlite3_stmt *res = NULL; - if (host == localhost) { - buffer_sprintf(wb, "You cannot unregister the parent node"); - goto done; - } + bool report_error = strcmp(args, "ALL_NODES"); - if (rrdhost_is_online(host)) { - buffer_sprintf(wb, "Cannot unregister a live node"); + if (!PREPARE_STATEMENT(db_meta, SQL_HOSTNAME_TO_REMOVE, &res)) { + buffer_sprintf(wb, "Failed to prepare database statement to check for stale nodes"); goto done; } - if (!rrdhost_option_check(host, RRDHOST_OPTION_EPHEMERAL_HOST)) { - rrdhost_option_set(host, RRDHOST_OPTION_EPHEMERAL_HOST); - sql_set_host_label(&host->host_uuid, "_is_ephemeral", "true"); - aclk_host_state_update(host, 0, 0); - unregister_node(host->machine_guid); - freez(host->node_id); - host->node_id = NULL; - buffer_sprintf(wb, "Unregistering node with machine guid %s, hostname = %s", host->machine_guid, rrdhost_hostname(host)); + int param = 0; + SQLITE_BIND_FAIL(done0, sqlite3_bind_text(res, ++param, args, -1, SQLITE_STATIC)); + + param = 0; + int cnt = 0; + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + char guid[UUID_STR_LEN]; + uuid_unparse_lower(*(nd_uuid_t *)sqlite3_column_blob(res, 0), guid); + host = rrdhost_find_by_guid(guid); + if (host) { + if (cnt) + buffer_fast_strcat(wb, "\n", 1); + cnt += remove_ephemeral_host(wb, host, report_error); + } } - else - buffer_sprintf(wb, "Node with machine guid %s, hostname = %s is already unregistered", host->machine_guid, rrdhost_hostname(host)); + if (!cnt && buffer_strlen(wb) == 0) { + if (report_error) + buffer_sprintf(wb, "No match for \"%s\"", args); + else + buffer_sprintf(wb, "No stale nodes found"); + } + done0: + REPORT_BIND_FAIL(res, param); + SQLITE_FINALIZE(res); } + else + (void) remove_ephemeral_host(wb, host, true); done: *message = strdupz(buffer_tostring(wb)); diff --git a/src/daemon/common.c b/src/daemon/common.c index a64d53585..6c824eec6 100644 --- a/src/daemon/common.c +++ b/src/daemon/common.c @@ -44,7 +44,7 @@ long get_netdata_cpus(void) { long cores_user_configured = config_get_number(CONFIG_SECTION_GLOBAL, "cpu cores", processors); - errno = 0; + errno_clear(); internal_error(true, "System CPUs: %ld, (" "system: %ld, cgroups cpuset v1: %ld, cgroups cpuset v2: %ld, netdata.conf: %ld" diff --git a/src/daemon/common.h b/src/daemon/common.h index 102ec81e2..1dea19c5b 100644 --- a/src/daemon/common.h +++ b/src/daemon/common.h @@ -84,9 +84,6 @@ // global GUID map functions -// netdata agent spawn server -#include "spawn/spawn.h" - // the netdata daemon #include "daemon.h" #include "main.h" diff --git a/src/daemon/config/README.md b/src/daemon/config/README.md index c59f55620..3c0912fba 100644 --- a/src/daemon/config/README.md +++ b/src/daemon/config/README.md @@ -128,16 +128,21 @@ Please note that your data history will be lost if you have modified `history` p ### [logs] section options +There are additional configuration options for the logs. For more info, see [Netdata Logging](/src/libnetdata/log/README.md). + | setting | default | info | |:----------------------------------:|:-----------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | debug flags | `0x0000000000000000` | Bitmap of debug options to enable. For more information check [Tracing Options](/src/daemon/README.md#debugging). | -| debug | `/var/log/netdata/debug.log` | The filename to save debug information. This file will not be created if debugging is not enabled. You can also set it to `syslog` to send the debug messages to syslog, or `none` to disable this log. For more information check [Tracing Options](/src/daemon/README.md#debugging). | -| error | `/var/log/netdata/error.log` | The filename to save error messages for Netdata daemon and all plugins (`stderr` is sent here for all Netdata programs, including the plugins). You can also set it to `syslog` to send the errors to syslog, or `none` to disable this log. | -| access | `/var/log/netdata/access.log` | The filename to save the log of web clients accessing Netdata charts. You can also set it to `syslog` to send the access log to syslog, or `none` to disable this log. | -| facility | `daemon` | A facility keyword is used to specify the type of system that is logging the message. | -| errors flood protection period | `1200` | Length of period (in sec) during which the number of errors should not exceed the `errors to trigger flood protection`. | -| errors to trigger flood protection | `200` | Number of errors written to the log in `errors flood protection period` sec before flood protection is activated. | -| severity level | `info` | Controls which log messages are logged, with error being the most important. Supported values: `info` and `error`. | +| debug | `/var/log/netdata/debug.log` | The filename to save debug information. This file will not be created if debugging is not enabled. You can also set it to `syslog` to send the debug messages to syslog, or `off` to disable this log. For more information check [Tracing Options](/src/daemon/README.md#debugging). | +| error | `/var/log/netdata/error.log` | The filename to save error messages for Netdata daemon and all plugins (`stderr` is sent here for all Netdata programs, including the plugins). You can also set it to `syslog` to send the errors to syslog, or `off` to disable this log. | +| access | `/var/log/netdata/access.log` | The filename to save the log of web clients accessing Netdata charts. You can also set it to `syslog` to send the access log to syslog, or `off` to disable this log. | +| collector | `journal` | The filename to save the log of Netdata collectors. You can also set it to `syslog` to send the access log to syslog, or `off` to disable this log. Defaults to `Journal` if using systemd. | +| health | `journal` | The filename to save the log of Netdata health collectors. You can also set it to `syslog` to send the access log to syslog, or `off` to disable this log. Defaults to `Journal` if using systemd. | +| daemon | `journal` | The filename to save the log of Netdata daemon. You can also set it to `syslog` to send the access log to syslog, or `off` to disable this log. Defaults to `Journal` if using systemd. | +| facility | `daemon` | A facility keyword is used to specify the type of system that is logging the message. | +| logs flood protection period | `60` | Length of period (in sec) during which the number of errors should not exceed the `errors to trigger flood protection`. | +| logs to trigger flood protection | `1000` | Number of errors written to the log in `errors flood protection period` sec before flood protection is activated. | +| level | `info` | Controls which log messages are logged, with error being the most important. Supported values: `info` and `error`. | ### [environment variables] section options diff --git a/src/daemon/daemon.c b/src/daemon/daemon.c index f77b748a8..2392d4cc1 100644 --- a/src/daemon/daemon.c +++ b/src/daemon/daemon.c @@ -381,14 +381,14 @@ static void sched_setscheduler_set(void) { priority = (int)config_get_number(CONFIG_SECTION_GLOBAL, "process scheduling priority", priority); #ifdef HAVE_SCHED_GET_PRIORITY_MIN - errno = 0; + errno_clear(); if(priority < sched_get_priority_min(policy)) { netdata_log_error("scheduler %s (%d) priority %d is below the minimum %d. Using the minimum.", name, policy, priority, sched_get_priority_min(policy)); priority = sched_get_priority_min(policy); } #endif #ifdef HAVE_SCHED_GET_PRIORITY_MAX - errno = 0; + errno_clear(); if(priority > sched_get_priority_max(policy)) { netdata_log_error("scheduler %s (%d) priority %d is above the maximum %d. Using the maximum.", name, policy, priority, sched_get_priority_max(policy)); priority = sched_get_priority_max(policy); @@ -407,7 +407,7 @@ static void sched_setscheduler_set(void) { .sched_priority = priority }; - errno = 0; + errno_clear(); i = sched_setscheduler(0, policy, ¶m); if(i != 0) { netdata_log_error("Cannot adjust netdata scheduling policy to %s (%d), with priority %d. Falling back to nice.", diff --git a/src/daemon/global_statistics.c b/src/daemon/global_statistics.c index 429f68c0d..17fd53761 100644 --- a/src/daemon/global_statistics.c +++ b/src/daemon/global_statistics.c @@ -5,17 +5,18 @@ #define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01 #define WORKER_JOB_GLOBAL 0 -#define WORKER_JOB_REGISTRY 1 -#define WORKER_JOB_WORKERS 2 -#define WORKER_JOB_DBENGINE 3 -#define WORKER_JOB_HEARTBEAT 4 -#define WORKER_JOB_STRINGS 5 -#define WORKER_JOB_DICTIONARIES 6 -#define WORKER_JOB_MALLOC_TRACE 7 -#define WORKER_JOB_SQLITE3 8 - -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 9 -#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 9 +#define WORKER_JOB_GLOBAL_EXT 1 +#define WORKER_JOB_REGISTRY 2 +#define WORKER_JOB_WORKERS 3 +#define WORKER_JOB_DBENGINE 4 +#define WORKER_JOB_HEARTBEAT 5 +#define WORKER_JOB_STRINGS 6 +#define WORKER_JOB_DICTIONARIES 7 +#define WORKER_JOB_MALLOC_TRACE 8 +#define WORKER_JOB_SQLITE3 9 + +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 10 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 10 #endif bool global_statistics_enabled = true; @@ -249,13 +250,9 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t ((stats).memory.dict + (stats).memory.values + (stats).memory.index) static void global_statistics_charts(void) { - static unsigned long long old_web_requests = 0, - old_web_usec = 0, - old_content_size = 0, - old_compressed_content_size = 0; + static unsigned long long old_web_requests = 0, old_web_usec = 0; - static collected_number compression_ratio = -1, - average_response_time = -1; + static collected_number average_response_time = -1; static time_t netdata_boottime_time = 0; if (!netdata_boottime_time) @@ -265,7 +262,6 @@ static void global_statistics_charts(void) { struct global_statistics gs; struct rusage me; - struct replication_query_statistics replication = replication_get_query_statistics(); global_statistics_copy(&gs, GLOBAL_STATS_RESET_WEB_USEC_MAX); getrusage(RUSAGE_SELF, &me); @@ -301,167 +297,6 @@ static void global_statistics_charts(void) { rrdset_done(st_cpu); } - // ---------------------------------------------------------------- - - { - static RRDSET *st_memory = NULL; - static RRDDIM *rd_database = NULL; - static RRDDIM *rd_collectors = NULL; - static RRDDIM *rd_hosts = NULL; - static RRDDIM *rd_rrd = NULL; - static RRDDIM *rd_contexts = NULL; - static RRDDIM *rd_health = NULL; - static RRDDIM *rd_functions = NULL; - static RRDDIM *rd_labels = NULL; - static RRDDIM *rd_strings = NULL; - static RRDDIM *rd_streaming = NULL; - static RRDDIM *rd_replication = NULL; - static RRDDIM *rd_buffers = NULL; - static RRDDIM *rd_workers = NULL; - static RRDDIM *rd_aral = NULL; - static RRDDIM *rd_judy = NULL; - static RRDDIM *rd_other = NULL; - - if (unlikely(!st_memory)) { - st_memory = rrdset_create_localhost( - "netdata", - "memory", - NULL, - "netdata", - NULL, - "Netdata Memory", - "bytes", - "netdata", - "stats", - 130100, - localhost->rrd_update_every, - RRDSET_TYPE_STACKED); - - rd_database = rrddim_add(st_memory, "db", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_collectors = rrddim_add(st_memory, "collectors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_hosts = rrddim_add(st_memory, "hosts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_rrd = rrddim_add(st_memory, "rrdset rrddim", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_contexts = rrddim_add(st_memory, "contexts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_health = rrddim_add(st_memory, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_functions = rrddim_add(st_memory, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_labels = rrddim_add(st_memory, "labels", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_strings = rrddim_add(st_memory, "strings", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_streaming = rrddim_add(st_memory, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_replication = rrddim_add(st_memory, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers = rrddim_add(st_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_workers = rrddim_add(st_memory, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_aral = rrddim_add(st_memory, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_judy = rrddim_add(st_memory, "judy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - } - - size_t buffers = - netdata_buffers_statistics.query_targets_size + - netdata_buffers_statistics.rrdset_done_rda_size + - netdata_buffers_statistics.buffers_aclk + - netdata_buffers_statistics.buffers_api + - netdata_buffers_statistics.buffers_functions + - netdata_buffers_statistics.buffers_sqlite + - netdata_buffers_statistics.buffers_exporters + - netdata_buffers_statistics.buffers_health + - netdata_buffers_statistics.buffers_streaming + - netdata_buffers_statistics.cbuffers_streaming + - netdata_buffers_statistics.buffers_web + - replication_allocated_buffers() + - aral_by_size_overhead() + - judy_aral_overhead(); - - size_t strings = 0; - string_statistics(NULL, NULL, NULL, NULL, NULL, &strings, NULL, NULL); - - rrddim_set_by_pointer(st_memory, rd_database, (collected_number)dbengine_total_memory + (collected_number)rrddim_db_memory_size); - rrddim_set_by_pointer(st_memory, rd_collectors, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_collectors)); - rrddim_set_by_pointer(st_memory, rd_hosts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhost) + (collected_number)netdata_buffers_statistics.rrdhost_allocations_size); - rrddim_set_by_pointer(st_memory, rd_rrd, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdset_rrddim)); - rrddim_set_by_pointer(st_memory, rd_contexts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdcontext)); - rrddim_set_by_pointer(st_memory, rd_health, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhealth)); - rrddim_set_by_pointer(st_memory, rd_functions, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_functions)); - rrddim_set_by_pointer(st_memory, rd_labels, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdlabels)); - rrddim_set_by_pointer(st_memory, rd_strings, (collected_number)strings); - rrddim_set_by_pointer(st_memory, rd_streaming, (collected_number)netdata_buffers_statistics.rrdhost_senders + (collected_number)netdata_buffers_statistics.rrdhost_receivers); - rrddim_set_by_pointer(st_memory, rd_replication, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + (collected_number)replication_allocated_memory()); - rrddim_set_by_pointer(st_memory, rd_buffers, (collected_number)buffers); - rrddim_set_by_pointer(st_memory, rd_workers, (collected_number) workers_allocated_memory()); - rrddim_set_by_pointer(st_memory, rd_aral, (collected_number) aral_by_size_structures()); - rrddim_set_by_pointer(st_memory, rd_judy, (collected_number) judy_aral_structures()); - rrddim_set_by_pointer(st_memory, rd_other, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other)); - - rrdset_done(st_memory); - } - - { - static RRDSET *st_memory_buffers = NULL; - static RRDDIM *rd_queries = NULL; - static RRDDIM *rd_collectors = NULL; - static RRDDIM *rd_buffers_aclk = NULL; - static RRDDIM *rd_buffers_api = NULL; - static RRDDIM *rd_buffers_functions = NULL; - static RRDDIM *rd_buffers_sqlite = NULL; - static RRDDIM *rd_buffers_exporters = NULL; - static RRDDIM *rd_buffers_health = NULL; - static RRDDIM *rd_buffers_streaming = NULL; - static RRDDIM *rd_cbuffers_streaming = NULL; - static RRDDIM *rd_buffers_replication = NULL; - static RRDDIM *rd_buffers_web = NULL; - static RRDDIM *rd_buffers_aral = NULL; - static RRDDIM *rd_buffers_judy = NULL; - - if (unlikely(!st_memory_buffers)) { - st_memory_buffers = rrdset_create_localhost( - "netdata", - "memory_buffers", - NULL, - "netdata", - NULL, - "Netdata Memory Buffers", - "bytes", - "netdata", - "stats", - 130101, - localhost->rrd_update_every, - RRDSET_TYPE_STACKED); - - rd_queries = rrddim_add(st_memory_buffers, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_collectors = rrddim_add(st_memory_buffers, "collection", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_aclk = rrddim_add(st_memory_buffers, "aclk", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_api = rrddim_add(st_memory_buffers, "api", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_functions = rrddim_add(st_memory_buffers, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_sqlite = rrddim_add(st_memory_buffers, "sqlite", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_exporters = rrddim_add(st_memory_buffers, "exporters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_health = rrddim_add(st_memory_buffers, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_streaming = rrddim_add(st_memory_buffers, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_cbuffers_streaming = rrddim_add(st_memory_buffers, "streaming cbuf", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_replication = rrddim_add(st_memory_buffers, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_web = rrddim_add(st_memory_buffers, "web", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_aral = rrddim_add(st_memory_buffers, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - rd_buffers_judy = rrddim_add(st_memory_buffers, "judy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); - } - - rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory()); - rrddim_set_by_pointer(st_memory_buffers, rd_collectors, (collected_number)netdata_buffers_statistics.rrdset_done_rda_size); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aclk, (collected_number)netdata_buffers_statistics.buffers_aclk); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_api, (collected_number)netdata_buffers_statistics.buffers_api); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_functions, (collected_number)netdata_buffers_statistics.buffers_functions); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_sqlite, (collected_number)netdata_buffers_statistics.buffers_sqlite); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_exporters, (collected_number)netdata_buffers_statistics.buffers_exporters); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_health, (collected_number)netdata_buffers_statistics.buffers_health); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_streaming, (collected_number)netdata_buffers_statistics.buffers_streaming); - rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers()); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aral, (collected_number)aral_by_size_overhead()); - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_judy, (collected_number)judy_aral_overhead()); - - rrdset_done(st_memory_buffers); - } - - // ---------------------------------------------------------------- - { static RRDSET *st_uptime = NULL; static RRDDIM *rd_uptime = NULL; @@ -624,9 +459,174 @@ static void global_statistics_charts(void) { rrddim_set_by_pointer(st_duration, rd_max, ((gs.web_usec_max)?(collected_number)gs.web_usec_max:average_response_time)); rrdset_done(st_duration); + } +} + +static void global_statistics_extended_charts(void) { + static unsigned long long old_content_size = 0, old_compressed_content_size = 0; + static collected_number compression_ratio = -1; + + struct global_statistics gs; + struct replication_query_statistics replication = replication_get_query_statistics(); + global_statistics_copy(&gs, GLOBAL_STATS_RESET_WEB_USEC_MAX); + + { + static RRDSET *st_memory = NULL; + static RRDDIM *rd_database = NULL; + static RRDDIM *rd_collectors = NULL; + static RRDDIM *rd_hosts = NULL; + static RRDDIM *rd_rrd = NULL; + static RRDDIM *rd_contexts = NULL; + static RRDDIM *rd_health = NULL; + static RRDDIM *rd_functions = NULL; + static RRDDIM *rd_labels = NULL; + static RRDDIM *rd_strings = NULL; + static RRDDIM *rd_streaming = NULL; + static RRDDIM *rd_replication = NULL; + static RRDDIM *rd_buffers = NULL; + static RRDDIM *rd_workers = NULL; + static RRDDIM *rd_aral = NULL; + static RRDDIM *rd_judy = NULL; + static RRDDIM *rd_other = NULL; + + if (unlikely(!st_memory)) { + st_memory = rrdset_create_localhost( + "netdata", + "memory", + NULL, + "netdata", + NULL, + "Netdata Memory", + "bytes", + "netdata", + "stats", + 130100, + localhost->rrd_update_every, + RRDSET_TYPE_STACKED); + + rd_database = rrddim_add(st_memory, "db", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_collectors = rrddim_add(st_memory, "collectors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_hosts = rrddim_add(st_memory, "hosts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_rrd = rrddim_add(st_memory, "rrdset rrddim", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_contexts = rrddim_add(st_memory, "contexts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_health = rrddim_add(st_memory, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_functions = rrddim_add(st_memory, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_labels = rrddim_add(st_memory, "labels", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_strings = rrddim_add(st_memory, "strings", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_streaming = rrddim_add(st_memory, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_replication = rrddim_add(st_memory, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers = rrddim_add(st_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_workers = rrddim_add(st_memory, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_aral = rrddim_add(st_memory, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_judy = rrddim_add(st_memory, "judy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + + size_t buffers = + netdata_buffers_statistics.query_targets_size + + netdata_buffers_statistics.rrdset_done_rda_size + + netdata_buffers_statistics.buffers_aclk + + netdata_buffers_statistics.buffers_api + + netdata_buffers_statistics.buffers_functions + + netdata_buffers_statistics.buffers_sqlite + + netdata_buffers_statistics.buffers_exporters + + netdata_buffers_statistics.buffers_health + + netdata_buffers_statistics.buffers_streaming + + netdata_buffers_statistics.cbuffers_streaming + + netdata_buffers_statistics.buffers_web + + replication_allocated_buffers() + + aral_by_size_overhead() + + judy_aral_overhead(); + + size_t strings = 0; + string_statistics(NULL, NULL, NULL, NULL, NULL, &strings, NULL, NULL); + + rrddim_set_by_pointer(st_memory, rd_database, (collected_number)dbengine_total_memory + (collected_number)rrddim_db_memory_size); + rrddim_set_by_pointer(st_memory, rd_collectors, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_collectors)); + rrddim_set_by_pointer(st_memory, rd_hosts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhost) + (collected_number)netdata_buffers_statistics.rrdhost_allocations_size); + rrddim_set_by_pointer(st_memory, rd_rrd, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdset_rrddim)); + rrddim_set_by_pointer(st_memory, rd_contexts, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdcontext)); + rrddim_set_by_pointer(st_memory, rd_health, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhealth)); + rrddim_set_by_pointer(st_memory, rd_functions, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_functions)); + rrddim_set_by_pointer(st_memory, rd_labels, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdlabels)); + rrddim_set_by_pointer(st_memory, rd_strings, (collected_number)strings); + rrddim_set_by_pointer(st_memory, rd_streaming, (collected_number)netdata_buffers_statistics.rrdhost_senders + (collected_number)netdata_buffers_statistics.rrdhost_receivers); + rrddim_set_by_pointer(st_memory, rd_replication, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + (collected_number)replication_allocated_memory()); + rrddim_set_by_pointer(st_memory, rd_buffers, (collected_number)buffers); + rrddim_set_by_pointer(st_memory, rd_workers, (collected_number) workers_allocated_memory()); + rrddim_set_by_pointer(st_memory, rd_aral, (collected_number) aral_by_size_structures()); + rrddim_set_by_pointer(st_memory, rd_judy, (collected_number) judy_aral_structures()); + rrddim_set_by_pointer(st_memory, rd_other, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other)); + + rrdset_done(st_memory); + } + + { + static RRDSET *st_memory_buffers = NULL; + static RRDDIM *rd_queries = NULL; + static RRDDIM *rd_collectors = NULL; + static RRDDIM *rd_buffers_aclk = NULL; + static RRDDIM *rd_buffers_api = NULL; + static RRDDIM *rd_buffers_functions = NULL; + static RRDDIM *rd_buffers_sqlite = NULL; + static RRDDIM *rd_buffers_exporters = NULL; + static RRDDIM *rd_buffers_health = NULL; + static RRDDIM *rd_buffers_streaming = NULL; + static RRDDIM *rd_cbuffers_streaming = NULL; + static RRDDIM *rd_buffers_replication = NULL; + static RRDDIM *rd_buffers_web = NULL; + static RRDDIM *rd_buffers_aral = NULL; + static RRDDIM *rd_buffers_judy = NULL; + + if (unlikely(!st_memory_buffers)) { + st_memory_buffers = rrdset_create_localhost( + "netdata", + "memory_buffers", + NULL, + "netdata", + NULL, + "Netdata Memory Buffers", + "bytes", + "netdata", + "stats", + 130101, + localhost->rrd_update_every, + RRDSET_TYPE_STACKED); + + rd_queries = rrddim_add(st_memory_buffers, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_collectors = rrddim_add(st_memory_buffers, "collection", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_aclk = rrddim_add(st_memory_buffers, "aclk", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_api = rrddim_add(st_memory_buffers, "api", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_functions = rrddim_add(st_memory_buffers, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_sqlite = rrddim_add(st_memory_buffers, "sqlite", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_exporters = rrddim_add(st_memory_buffers, "exporters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_health = rrddim_add(st_memory_buffers, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_streaming = rrddim_add(st_memory_buffers, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_cbuffers_streaming = rrddim_add(st_memory_buffers, "streaming cbuf", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_replication = rrddim_add(st_memory_buffers, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_web = rrddim_add(st_memory_buffers, "web", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_aral = rrddim_add(st_memory_buffers, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_buffers_judy = rrddim_add(st_memory_buffers, "judy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + + rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory()); + rrddim_set_by_pointer(st_memory_buffers, rd_collectors, (collected_number)netdata_buffers_statistics.rrdset_done_rda_size); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aclk, (collected_number)netdata_buffers_statistics.buffers_aclk); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_api, (collected_number)netdata_buffers_statistics.buffers_api); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_functions, (collected_number)netdata_buffers_statistics.buffers_functions); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_sqlite, (collected_number)netdata_buffers_statistics.buffers_sqlite); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_exporters, (collected_number)netdata_buffers_statistics.buffers_exporters); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_health, (collected_number)netdata_buffers_statistics.buffers_health); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_streaming, (collected_number)netdata_buffers_statistics.buffers_streaming); + rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers()); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aral, (collected_number)aral_by_size_overhead()); + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_judy, (collected_number)judy_aral_overhead()); + + rrdset_done(st_memory_buffers); } - // ---------------------------------------------------------------- { static RRDSET *st_compression = NULL; @@ -671,8 +671,6 @@ static void global_statistics_charts(void) { rrdset_done(st_compression); } - // ---------------------------------------------------------------- - { static RRDSET *st_queries = NULL; static RRDDIM *rd_api_data_queries = NULL; @@ -722,8 +720,6 @@ static void global_statistics_charts(void) { rrdset_done(st_queries); } - // ---------------------------------------------------------------- - { static RRDSET *st_points_read = NULL; static RRDDIM *rd_api_data_points_read = NULL; @@ -773,8 +769,6 @@ static void global_statistics_charts(void) { rrdset_done(st_points_read); } - // ---------------------------------------------------------------- - if(gs.api_data_result_points_generated || replication.points_generated) { static RRDSET *st_points_generated = NULL; static RRDDIM *rd_api_data_points_generated = NULL; @@ -818,7 +812,7 @@ static void global_statistics_charts(void) { rrdset_done(st_points_generated); } - // ---------------------------------------------------------------- + ml_update_global_statistics_charts(gs.ml_models_consulted); { static RRDSET *st_points_stored = NULL; @@ -853,10 +847,6 @@ static void global_statistics_charts(void) { rrdset_done(st_points_stored); } - ml_update_global_statistics_charts(gs.ml_models_consulted); - - // ---------------------------------------------------------------- - #ifdef ENABLE_DBENGINE if (tier_page_type[0] == RRDENG_PAGE_TYPE_GORILLA_32BIT) { @@ -921,7 +911,6 @@ static void global_statistics_charts(void) { } #endif } - // ---------------------------------------------------------------------------- // sqlite3 statistics @@ -4203,6 +4192,7 @@ static void worker_utilization_finish(void) { static void global_statistics_register_workers(void) { worker_register("STATS"); worker_register_job_name(WORKER_JOB_GLOBAL, "global"); + worker_register_job_name(WORKER_JOB_GLOBAL_EXT, "global_ext"); worker_register_job_name(WORKER_JOB_REGISTRY, "registry"); worker_register_job_name(WORKER_JOB_DBENGINE, "dbengine"); worker_register_job_name(WORKER_JOB_STRINGS, "strings"); @@ -4239,6 +4229,7 @@ void *global_statistics_main(void *ptr) usec_t step = update_every * USEC_PER_SEC; heartbeat_t hb; heartbeat_init(&hb); + usec_t real_step = USEC_PER_SEC; // keep the randomness at zero // to make sure we are not close to any other thread @@ -4246,36 +4237,15 @@ void *global_statistics_main(void *ptr) while (service_running(SERVICE_COLLECTORS)) { worker_is_idle(); - heartbeat_next(&hb, step); + heartbeat_next(&hb, USEC_PER_SEC); + if (real_step < step) { + real_step += USEC_PER_SEC; + continue; + } + real_step = USEC_PER_SEC; worker_is_busy(WORKER_JOB_GLOBAL); global_statistics_charts(); - - worker_is_busy(WORKER_JOB_REGISTRY); - registry_statistics(); - -#ifdef ENABLE_DBENGINE - if(dbengine_enabled) { - worker_is_busy(WORKER_JOB_DBENGINE); - dbengine2_statistics_charts(); - } -#endif - - worker_is_busy(WORKER_JOB_HEARTBEAT); - update_heartbeat_charts(); - - worker_is_busy(WORKER_JOB_STRINGS); - update_strings_charts(); - -#ifdef DICT_WITH_STATS - worker_is_busy(WORKER_JOB_DICTIONARIES); - dictionary_statistics(); -#endif - -#ifdef NETDATA_TRACE_ALLOCATIONS - worker_is_busy(WORKER_JOB_MALLOC_TRACE); - malloc_trace_statistics(); -#endif } return NULL; @@ -4283,12 +4253,13 @@ void *global_statistics_main(void *ptr) // --------------------------------------------------------------------------------------------------------------------- -// workers thread +// global statistics extended thread -static void global_statistics_workers_cleanup(void *pptr) +static void global_statistics_extended_cleanup(void *pptr) { struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); - if(!static_thread) return; + if (!static_thread) + return; static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; @@ -4300,66 +4271,62 @@ static void global_statistics_workers_cleanup(void *pptr) static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } -void *global_statistics_workers_main(void *ptr) +void *global_statistics_extended_main(void *ptr) { - CLEANUP_FUNCTION_REGISTER(global_statistics_workers_cleanup) cleanup_ptr = ptr; + CLEANUP_FUNCTION_REGISTER(global_statistics_extended_cleanup) cleanup_ptr = ptr; global_statistics_register_workers(); int update_every = - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); + (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); if (update_every < localhost->rrd_update_every) update_every = localhost->rrd_update_every; usec_t step = update_every * USEC_PER_SEC; heartbeat_t hb; heartbeat_init(&hb); + usec_t real_step = USEC_PER_SEC; while (service_running(SERVICE_COLLECTORS)) { worker_is_idle(); - heartbeat_next(&hb, step); - - worker_is_busy(WORKER_JOB_WORKERS); - worker_utilization_charts(); - } - - return NULL; -} - -// --------------------------------------------------------------------------------------------------------------------- -// sqlite3 thread - -static void global_statistics_sqlite3_cleanup(void *pptr) -{ - struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); - if(!static_thread) return; + heartbeat_next(&hb, USEC_PER_SEC); + if (real_step < step) { + real_step += USEC_PER_SEC; + continue; + } + real_step = USEC_PER_SEC; - static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + worker_is_busy(WORKER_JOB_HEARTBEAT); + update_heartbeat_charts(); - worker_unregister(); - netdata_log_info("cleaning up..."); + worker_is_busy(WORKER_JOB_GLOBAL_EXT); + global_statistics_extended_charts(); - static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; -} +#ifdef ENABLE_DBENGINE + if(dbengine_enabled) { + worker_is_busy(WORKER_JOB_DBENGINE); + dbengine2_statistics_charts(); + } +#endif -void *global_statistics_sqlite3_main(void *ptr) -{ - CLEANUP_FUNCTION_REGISTER(global_statistics_sqlite3_cleanup) cleanup_ptr = ptr; + worker_is_busy(WORKER_JOB_REGISTRY); + registry_statistics(); - global_statistics_register_workers(); + worker_is_busy(WORKER_JOB_STRINGS); + update_strings_charts(); - int update_every = - (int)config_get_number(CONFIG_SECTION_GLOBAL_STATISTICS, "update every", localhost->rrd_update_every); - if (update_every < localhost->rrd_update_every) - update_every = localhost->rrd_update_every; +#ifdef DICT_WITH_STATS + worker_is_busy(WORKER_JOB_DICTIONARIES); + dictionary_statistics(); +#endif - usec_t step = update_every * USEC_PER_SEC; - heartbeat_t hb; - heartbeat_init(&hb); +#ifdef NETDATA_TRACE_ALLOCATIONS + worker_is_busy(WORKER_JOB_MALLOC_TRACE); + malloc_trace_statistics(); +#endif - while (service_running(SERVICE_COLLECTORS)) { - worker_is_idle(); - heartbeat_next(&hb, step); + worker_is_busy(WORKER_JOB_WORKERS); + worker_utilization_charts(); worker_is_busy(WORKER_JOB_SQLITE3); sqlite3_statistics_charts(); @@ -4367,4 +4334,3 @@ void *global_statistics_sqlite3_main(void *ptr) return NULL; } - diff --git a/src/daemon/main.c b/src/daemon/main.c index e2db02097..17fef8449 100644 --- a/src/daemon/main.c +++ b/src/daemon/main.c @@ -7,6 +7,10 @@ #include "database/engine/page_test.h" +#ifdef OS_WINDOWS +#include "win_system-info.h" +#endif + #ifdef ENABLE_SENTRY #include "sentry-native/sentry-native.h" #endif @@ -22,7 +26,6 @@ int libuv_worker_threads = MIN_LIBUV_WORKER_THREADS; bool ieee754_doubles = false; time_t netdata_start_time = 0; struct netdata_static_thread *static_threads; -bool i_am_the_spawn_server = false; struct config netdata_config = { .first_section = NULL, @@ -321,8 +324,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) { void web_client_cache_destroy(void); void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) { - if (i_am_the_spawn_server) - exit(ret); + netdata_exit = 1; watcher_shutdown_begin(); @@ -344,6 +346,9 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re (void) rename(agent_crash_file, agent_incomplete_shutdown_file); watcher_step_complete(WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE); + netdata_main_spawn_server_cleanup(); + watcher_step_complete(WATCHER_STEP_ID_DESTROY_MAIN_SPAWN_SERVER); + #ifdef ENABLE_DBENGINE if(dbengine_enabled) { for (size_t tier = 0; tier < storage_tiers; tier++) @@ -488,10 +493,14 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re (void) unlink(agent_incomplete_shutdown_file); watcher_step_complete(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE); - + watcher_shutdown_end(); watcher_thread_stop(); +#ifdef OS_WINDOWS + return; +#endif + #ifdef ENABLE_SENTRY if (ret) abort(); @@ -613,39 +622,6 @@ void web_server_config_options(void) } } - -// killpid kills pid with SIGTERM. -int killpid(pid_t pid) { - int ret; - netdata_log_debug(D_EXIT, "Request to kill pid %d", pid); - - int signal = SIGTERM; -//#ifdef NETDATA_INTERNAL_CHECKS -// if(service_running(SERVICE_COLLECTORS)) -// signal = SIGABRT; -//#endif - - errno = 0; - ret = kill(pid, signal); - if (ret == -1) { - switch(errno) { - case ESRCH: - // We wanted the process to exit so just let the caller handle. - return ret; - - case EPERM: - netdata_log_error("Cannot kill pid %d, but I do not have enough permissions.", pid); - break; - - default: - netdata_log_error("Cannot kill pid %d, but I received an error.", pid); - break; - } - } - - return ret; -} - static void set_nofile_limit(struct rlimit *rl) { // get the num files allowed if(getrlimit(RLIMIT_NOFILE, rl) != 0) { @@ -693,8 +669,6 @@ void cancel_main_threads() { } } - netdata_exit = 1; - while(found && max > 0) { max -= step; netdata_log_info("Waiting %d threads to finish...", found); @@ -1325,7 +1299,7 @@ static void post_conf_load(char **user) } static bool load_netdata_conf(char *filename, char overwrite_used, char **user) { - errno = 0; + errno_clear(); int ret = 0; @@ -1362,6 +1336,7 @@ static inline void coverity_remove_taint(char *s) } int get_system_info(struct rrdhost_system_info *system_info) { +#if !defined(OS_WINDOWS) char *script; script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("system-info.sh") + 2)); sprintf(script, "%s/%s", netdata_configured_primary_plugins_dir, "system-info.sh"); @@ -1371,15 +1346,12 @@ int get_system_info(struct rrdhost_system_info *system_info) { return 1; } - pid_t command_pid; - - FILE *fp_child_input; - FILE *fp_child_output = netdata_popen(script, &command_pid, &fp_child_input); - if(fp_child_output) { + POPEN_INSTANCE *instance = spawn_popen_run(script); + if(instance) { char line[200 + 1]; // Removed the double strlens, if the Coverity tainted string warning reappears I'll revert. // One time init code, but I'm curious about the warning... - while (fgets(line, 200, fp_child_output) != NULL) { + while (fgets(line, 200, instance->child_stdout_fp) != NULL) { char *value=line; while (*value && *value != '=') value++; if (*value=='=') { @@ -1398,9 +1370,12 @@ int get_system_info(struct rrdhost_system_info *system_info) { } } } - netdata_pclose(fp_child_input, fp_child_output, command_pid); + spawn_popen_wait(instance); } freez(script); +#else + netdata_windows_get_system_info(system_info); +#endif return 0; } @@ -1452,35 +1427,34 @@ int unittest_prepare_rrd(char **user) { return 0; } -int main(int argc, char **argv) { - // initialize the system clocks +int netdata_main(int argc, char **argv) { clocks_init(); - netdata_start_time = now_realtime_sec(); + string_init(); + analytics_init(); + netdata_start_time = now_realtime_sec(); usec_t started_ut = now_monotonic_usec(); usec_t last_ut = started_ut; const char *prev_msg = NULL; int i; int config_loaded = 0; - int dont_fork = 0; bool close_open_fds = true; size_t default_stacksize; char *user = NULL; +#ifdef OS_WINDOWS + int dont_fork = 1; +#else + int dont_fork = 0; +#endif + static_threads = static_threads_get(); netdata_ready = false; // set the name for logging program_name = "netdata"; - if (argc > 1 && strcmp(argv[1], SPAWN_SERVER_COMMAND_LINE_ARGUMENT) == 0) { - // don't run netdata, this is the spawn server - i_am_the_spawn_server = true; - spawn_server(); - exit(0); - } - // parse options { int num_opts = sizeof(option_definitions) / sizeof(struct option_def); @@ -1945,7 +1919,7 @@ int main(int argc, char **argv) { if (close_open_fds == true) { // close all open file descriptors, except the standard ones // the caller may have left open files (lxc-attach has this issue) - for_each_open_fd(OPEN_FD_ACTION_CLOSE, OPEN_FD_EXCLUDE_STDIN | OPEN_FD_EXCLUDE_STDOUT | OPEN_FD_EXCLUDE_STDERR); + os_close_all_non_std_open_fds_except(NULL, 0); } if(!config_loaded) { @@ -2175,6 +2149,7 @@ int main(int argc, char **argv) { (void)dont_fork; #endif + netdata_main_spawn_server_init("plugins", argc, (const char **)argv); watcher_thread_start(); // init sentry @@ -2203,19 +2178,7 @@ int main(int argc, char **argv) { // initialize internal registry delta_startup_time("initialize registry"); registry_init(); - - // fork the spawn server - delta_startup_time("fork the spawn server"); - spawn_init(); - - /* - * Libuv uv_spawn() uses SIGCHLD internally: - * https://github.com/libuv/libuv/blob/cc51217a317e96510fbb284721d5e6bc2af31e33/src/unix/process.c#L485 - * and inadvertently replaces the netdata signal handler which was setup during initialization. - * Thusly, we must explicitly restore the signal handler for SIGCHLD. - * Warning: extreme care is needed when mixing and matching POSIX and libuv. - */ - signals_restore_SIGCHLD(); + netdata_random_session_id_generate(); // ------------------------------------------------------------------------ // initialize rrd, registry, health, rrdpush, etc. @@ -2264,6 +2227,7 @@ int main(int argc, char **argv) { if (claiming_pending_arguments) claim_agent(claiming_pending_arguments, false, NULL); + load_claiming_state(); // ------------------------------------------------------------------------ @@ -2349,22 +2313,21 @@ int main(int argc, char **argv) { } #endif - // ------------------------------------------------------------------------ - // initialize WebRTC - webrtc_initialize(); - // ------------------------------------------------------------------------ - // unblock signals - signals_unblock(); - // ------------------------------------------------------------------------ - // Handle signals + return 10; +} - signals_handle(); +#ifndef OS_WINDOWS +int main(int argc, char *argv[]) +{ + int rc = netdata_main(argc, argv); + if (rc != 10) + return rc; - // should never reach this point - // but we need it for rpmlint #2752 + signals_handle(); return 1; } +#endif diff --git a/src/daemon/main.h b/src/daemon/main.h index faf7d5b69..3188623b6 100644 --- a/src/daemon/main.h +++ b/src/daemon/main.h @@ -8,7 +8,6 @@ extern struct config netdata_config; void cancel_main_threads(void); -int killpid(pid_t pid); typedef enum { ABILITY_DATA_QUERIES = (1 << 0), diff --git a/src/daemon/signals.c b/src/daemon/signals.c index c014452b7..4e4d7c4d4 100644 --- a/src/daemon/signals.c +++ b/src/daemon/signals.c @@ -2,6 +2,12 @@ #include "common.h" +/* + * IMPORTANT: Libuv uv_spawn() uses SIGCHLD internally: + * https://github.com/libuv/libuv/blob/cc51217a317e96510fbb284721d5e6bc2af31e33/src/unix/process.c#L485 + * Extreme care is needed when mixing and matching POSIX and libuv. + */ + typedef enum signal_action { NETDATA_SIGNAL_END_OF_LIST, NETDATA_SIGNAL_IGNORE, @@ -9,7 +15,6 @@ typedef enum signal_action { NETDATA_SIGNAL_REOPEN_LOGS, NETDATA_SIGNAL_RELOAD_HEALTH, NETDATA_SIGNAL_FATAL, - NETDATA_SIGNAL_CHILD, } SIGNAL_ACTION; static struct { @@ -25,7 +30,6 @@ static struct { { SIGHUP, "SIGHUP", 0, NETDATA_SIGNAL_REOPEN_LOGS }, { SIGUSR2, "SIGUSR2", 0, NETDATA_SIGNAL_RELOAD_HEALTH }, { SIGBUS, "SIGBUS", 0, NETDATA_SIGNAL_FATAL }, - { SIGCHLD, "SIGCHLD", 0, NETDATA_SIGNAL_CHILD }, // terminator { 0, "NONE", 0, NETDATA_SIGNAL_END_OF_LIST } @@ -93,18 +97,6 @@ void signals_init(void) { } } -void signals_restore_SIGCHLD(void) -{ - struct sigaction sa; - - sa.sa_flags = 0; - sigfillset(&sa.sa_mask); - sa.sa_handler = signal_handler; - - if(sigaction(SIGCHLD, &sa, NULL) == -1) - netdata_log_error("SIGNAL: Failed to change signal handler for: SIGCHLD"); -} - void signals_reset(void) { struct sigaction sa; sigemptyset(&sa.sa_mask); @@ -118,64 +110,6 @@ void signals_reset(void) { } } -// reap_child reaps the child identified by pid. -static void reap_child(pid_t pid) { - siginfo_t i; - - errno = 0; - netdata_log_debug(D_CHILDS, "SIGNAL: reap_child(%d)...", pid); - if (netdata_waitid(P_PID, (id_t)pid, &i, WEXITED|WNOHANG) == -1) { - if (errno != ECHILD) - netdata_log_error("SIGNAL: waitid(%d): failed to wait for child", pid); - else - netdata_log_info("SIGNAL: waitid(%d): failed - it seems the child is already reaped", pid); - return; - } - else if (i.si_pid == 0) { - // Process didn't exit, this shouldn't happen. - netdata_log_error("SIGNAL: waitid(%d): reports pid 0 - child has not exited", pid); - return; - } - - switch (i.si_code) { - case CLD_EXITED: - netdata_log_info("SIGNAL: reap_child(%d) exited with code: %d", pid, i.si_status); - break; - case CLD_KILLED: - netdata_log_info("SIGNAL: reap_child(%d) killed by signal: %d", pid, i.si_status); - break; - case CLD_DUMPED: - netdata_log_info("SIGNAL: reap_child(%d) dumped core by signal: %d", pid, i.si_status); - break; - case CLD_STOPPED: - netdata_log_info("SIGNAL: reap_child(%d) stopped by signal: %d", pid, i.si_status); - break; - case CLD_TRAPPED: - netdata_log_info("SIGNAL: reap_child(%d) trapped by signal: %d", pid, i.si_status); - break; - case CLD_CONTINUED: - netdata_log_info("SIGNAL: reap_child(%d) continued by signal: %d", pid, i.si_status); - break; - default: - netdata_log_info("SIGNAL: reap_child(%d) gave us a SIGCHLD with code %d and status %d.", pid, i.si_code, i.si_status); - break; - } -} - -// reap_children reaps all pending children which are not managed by myp. -static void reap_children() { - siginfo_t i; - - while(1) { - i.si_pid = 0; - if (netdata_waitid(P_ALL, (id_t)0, &i, WEXITED|WNOHANG|WNOWAIT) == -1 || i.si_pid == 0) - // nothing to do - return; - - reap_child(i.si_pid); - } -} - void signals_handle(void) { while(1) { @@ -183,6 +117,7 @@ void signals_handle(void) { // is delivered that either terminates the process or causes the invocation // of a signal-catching function. if(pause() == -1 && errno == EINTR) { + errno_clear(); // loop once, but keep looping while signals are coming in // this is needed because a few operations may take some time @@ -226,10 +161,6 @@ void signals_handle(void) { fatal("SIGNAL: Received %s. netdata now exits.", name); break; - case NETDATA_SIGNAL_CHILD: - reap_children(); - break; - default: netdata_log_info("SIGNAL: Received %s. No signal handler configured. Ignoring it.", name); break; diff --git a/src/daemon/signals.h b/src/daemon/signals.h index 12b1ed198..26dbc6dcd 100644 --- a/src/daemon/signals.h +++ b/src/daemon/signals.h @@ -6,7 +6,6 @@ void signals_init(void); void signals_block(void); void signals_unblock(void); -void signals_restore_SIGCHLD(void); void signals_reset(void); void signals_handle(void) NORETURN; diff --git a/src/daemon/static_threads.c b/src/daemon/static_threads.c index 4199e9306..c6ec79956 100644 --- a/src/daemon/static_threads.c +++ b/src/daemon/static_threads.c @@ -6,8 +6,7 @@ void *aclk_main(void *ptr); void *analytics_main(void *ptr); void *cpuidlejitter_main(void *ptr); void *global_statistics_main(void *ptr); -void *global_statistics_workers_main(void *ptr); -void *global_statistics_sqlite3_main(void *ptr); +void *global_statistics_extended_main(void *ptr); void *health_main(void *ptr); void *pluginsd_main(void *ptr); void *service_main(void *ptr); @@ -51,24 +50,13 @@ const struct netdata_static_thread static_threads_common[] = { .config_name = "netdata monitoring", .env_name = "NETDATA_INTERNALS_MONITORING", .global_variable = &global_statistics_enabled, - .enabled = 0, + .enabled = 1, .thread = NULL, .init_routine = NULL, .start_routine = global_statistics_main }, { - .name = "STATS_WORKERS", - .config_section = CONFIG_SECTION_PLUGINS, - .config_name = "netdata monitoring extended", - .env_name = "NETDATA_INTERNALS_EXTENDED_MONITORING", - .global_variable = &global_statistics_enabled, - .enabled = 0, // this is ignored - check main() for "netdata monitoring extended" - .thread = NULL, - .init_routine = NULL, - .start_routine = global_statistics_workers_main - }, - { - .name = "STATS_SQLITE3", + .name = "STATS_GLOBAL_EXT", .config_section = CONFIG_SECTION_PLUGINS, .config_name = "netdata monitoring extended", .env_name = "NETDATA_INTERNALS_EXTENDED_MONITORING", @@ -76,7 +64,7 @@ const struct netdata_static_thread static_threads_common[] = { .enabled = 0, // this is ignored - check main() for "netdata monitoring extended" .thread = NULL, .init_routine = NULL, - .start_routine = global_statistics_sqlite3_main + .start_routine = global_statistics_extended_main }, { .name = "PLUGINSD", @@ -105,8 +93,6 @@ const struct netdata_static_thread static_threads_common[] = { .init_routine = NULL, .start_routine = statsd_main }, -#ifndef OS_WINDOWS - // this crashes the debugger under windows { .name = "EXPORTING", .config_section = NULL, @@ -116,7 +102,6 @@ const struct netdata_static_thread static_threads_common[] = { .init_routine = NULL, .start_routine = exporting_main }, -#endif { .name = "SNDR[localhost]", .config_section = NULL, diff --git a/src/daemon/unit_test.c b/src/daemon/unit_test.c index e7a743603..0f15f67d7 100644 --- a/src/daemon/unit_test.c +++ b/src/daemon/unit_test.c @@ -1674,22 +1674,7 @@ int test_sqlite(void) { return 1; } - BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, NULL); - char *uuid_str = "0000_000"; - - buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str); - rc = sqlite3_exec_monitored(db_mt, buffer_tostring(sql), 0, 0, NULL); - if (rc != SQLITE_OK) - goto error; - - buffer_free(sql); fprintf(stderr,"SQLite is OK\n"); - rc = sqlite3_close_v2(db_mt); + (void) sqlite3_close_v2(db_mt); return 0; -error: - rc = sqlite3_close_v2(db_mt); - fprintf(stderr,"SQLite statement failed: %s\n", buffer_tostring(sql)); - buffer_free(sql); - fprintf(stderr,"SQLite tests failed\n"); - return 1; } diff --git a/src/daemon/watcher.c b/src/daemon/watcher.c index 1e0090e24..6584073e3 100644 --- a/src/daemon/watcher.c +++ b/src/daemon/watcher.c @@ -65,6 +65,7 @@ void *watcher_main(void *arg) usec_t shutdown_start_time = now_monotonic_usec(); watcher_wait_for_step(WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE); + watcher_wait_for_step(WATCHER_STEP_ID_DESTROY_MAIN_SPAWN_SERVER); watcher_wait_for_step(WATCHER_STEP_ID_DBENGINE_EXIT_MODE); watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS); watcher_wait_for_step(WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK); @@ -105,6 +106,8 @@ void watcher_thread_start() { watcher_steps[WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE].msg = "create shutdown file"; + watcher_steps[WATCHER_STEP_ID_DESTROY_MAIN_SPAWN_SERVER].msg = + "destroy main spawn server"; watcher_steps[WATCHER_STEP_ID_DBENGINE_EXIT_MODE].msg = "dbengine exit mode"; watcher_steps[WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS].msg = diff --git a/src/daemon/watcher.h b/src/daemon/watcher.h index b785ca436..9809e45fb 100644 --- a/src/daemon/watcher.h +++ b/src/daemon/watcher.h @@ -7,6 +7,7 @@ typedef enum { WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE = 0, + WATCHER_STEP_ID_DESTROY_MAIN_SPAWN_SERVER, WATCHER_STEP_ID_DBENGINE_EXIT_MODE, WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS, WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK, diff --git a/src/daemon/win_system-info.c b/src/daemon/win_system-info.c new file mode 100644 index 000000000..2d67862fb --- /dev/null +++ b/src/daemon/win_system-info.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "win_system-info.h" + +#ifdef OS_WINDOWS + +// Hardware +static char *netdata_windows_arch(DWORD value) +{ + switch (value) { + case 9: + return "x86_64"; + case 5: + return "ARM"; + case 12: + return "ARM64"; + case 6: + return "Intel Intaniun-based"; + case 0: + return "x86"; + default: + return NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN; + } +} + +static DWORD netdata_windows_cpu_frequency(HKEY lKey) +{ + DWORD freq = 0; + long ret = netdata_registry_get_dword_from_open_key(&freq, lKey, "~MHz"); + if (ret != ERROR_SUCCESS) + return freq; + + freq *= 1000000; + return freq; +} + +static void netdata_windows_cpu_from_system_info(struct rrdhost_system_info *systemInfo) +{ + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + + char cpuData[256]; + (void)snprintf(cpuData, 255, "%d", sysInfo.dwNumberOfProcessors); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", cpuData); + + char *arch = netdata_windows_arch(sysInfo.wProcessorArchitecture); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_ARCHITECTURE", arch); + + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_SYSTEM_VIRTUALIZATION", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_SYSTEM_VIRT_DETECTION", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_SYSTEM_CONTAINER", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_SYSTEM_CONTAINER_DETECTION", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + +} + +static void netdata_windows_cpu_vendor_model(struct rrdhost_system_info *systemInfo, + HKEY lKey, + char *variable, + char *key) +{ + char cpuData[256]; + long ret = netdata_registry_get_string_from_open_key(cpuData, 255, lKey, key); + (void)rrdhost_set_system_info_variable(systemInfo, + variable, + (ret == ERROR_SUCCESS) ? cpuData : NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN); +} + +static void netdata_windows_cpu_from_registry(struct rrdhost_system_info *systemInfo) +{ + HKEY lKey; + long ret = RegOpenKeyEx(HKEY_LOCAL_MACHINE, + "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", + 0, + KEY_READ, + &lKey); + if (ret != ERROR_SUCCESS) + return; + + ULONGLONG cpuFreq = netdata_windows_cpu_frequency(lKey); + char cpuData[256]; + if (cpuFreq) + (void)snprintf(cpuData, 255, "%lu", (unsigned long)cpuFreq); + + (void)rrdhost_set_system_info_variable(systemInfo, + "NETDATA_SYSTEM_CPU_FREQ", + (!cpuFreq) ? NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN : cpuData); + + netdata_windows_cpu_vendor_model(systemInfo, lKey, "NETDATA_SYSTEM_CPU_VENDOR", "VendorIdentifier"); + netdata_windows_cpu_vendor_model(systemInfo, lKey, "NETDATA_SYSTEM_CPU_MODEL", "ProcessorNameString"); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_CPU_DETECTION", NETDATA_WIN_DETECTION_METHOD); +} + +static void netdata_windows_get_cpu(struct rrdhost_system_info *systemInfo) +{ + netdata_windows_cpu_from_system_info(systemInfo); + + netdata_windows_cpu_from_registry(systemInfo); +} + +static void netdata_windows_get_mem(struct rrdhost_system_info *systemInfo) +{ + ULONGLONG size; + char memSize[256]; + if (!GetPhysicallyInstalledSystemMemory(&size)) + size = 0; + else + (void)snprintf(memSize, 255, "%llu", size); + + (void)rrdhost_set_system_info_variable(systemInfo, + "NETDATA_SYSTEM_TOTAL_RAM", + (!size) ? NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN : memSize); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_RAM_DETECTION", NETDATA_WIN_DETECTION_METHOD); +} + +static ULONGLONG netdata_windows_get_disk_size(char *cVolume) +{ + HANDLE disk = CreateFile(cVolume, GENERIC_READ, FILE_SHARE_VALID_FLAGS, 0, OPEN_EXISTING, 0, 0); + if (!disk) + return 0; + + GET_LENGTH_INFORMATION length; + DWORD ret; + + if (!DeviceIoControl(disk, IOCTL_DISK_GET_LENGTH_INFO, 0, 0, &length, sizeof(length), &ret, 0)) + return 0; + + CloseHandle(disk); + + return length.Length.QuadPart; +} + +static void netdata_windows_get_total_disk_size(struct rrdhost_system_info *systemInfo) +{ + ULONGLONG total = 0; + char cVolume[8]; + snprintf(cVolume, 7, "\\\\.\\C:"); + + DWORD lDrives = GetLogicalDrives(); + if (!lDrives) { + return; + } + + int i; +#define ND_POSSIBLE_VOLUMES 26 + for (i = 0; i < ND_POSSIBLE_VOLUMES; i++) { + if (!(lDrives & 1 << i)) + continue; + + cVolume[4] = 'A' + i; + total += netdata_windows_get_disk_size(cVolume); + } + + char diskSize[256]; + (void)snprintf(diskSize, 255, "%llu", total); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_TOTAL_DISK_SIZE", diskSize); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_DISK_DETECTION", NETDATA_WIN_DETECTION_METHOD); +} + +// Host +static DWORD netdata_windows_get_current_build() +{ + char cBuild[64]; + if (!netdata_registry_get_string( + cBuild, 63, HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", "CurrentBuild")) + return 0; + + errno_clear(); + + DWORD version = strtol(cBuild, NULL, 10); + if (errno == ERANGE) + return 0; + + return version; +} + +static void netdata_windows_discover_os_version(char *os, size_t length, DWORD build) +{ + char versionName[256]; + if (!netdata_registry_get_string(versionName, + 255, + HKEY_LOCAL_MACHINE, + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", + "DisplayVersion")) + { + (void)snprintf(os, length, "Microsoft Windows"); + return; + } + + if (IsWindowsServer()) { + (void)snprintf(os, length, "Microsoft Windows Version %s", versionName); + return; + } + +#define ND_WIN_VER_LENGTH 16 + char version[ND_WIN_VER_LENGTH + 1]; + if (IsWindows10OrGreater()) { + // https://learn.microsoft.com/en-us/windows/release-health/windows11-release-information + (void)snprintf(version, ND_WIN_VER_LENGTH, (build < 22000) ? "10" : "11"); + } else if (IsWindows8Point1OrGreater()) { + (void)snprintf(version, ND_WIN_VER_LENGTH, "8.1"); + } else if (IsWindows8OrGreater()) { + (void)snprintf(version, ND_WIN_VER_LENGTH, "8"); + } else if (IsWindows7SP1OrGreater()) { + (void)snprintf(version, ND_WIN_VER_LENGTH, "7 SP1"); + } else if (IsWindows7OrGreater()) { + (void)snprintf(version, ND_WIN_VER_LENGTH, "7"); + } else if (IsWindowsVistaSP2OrGreater()) { + (void)snprintf(version, ND_WIN_VER_LENGTH, "Vista SP2"); + } else if (IsWindowsVistaSP1OrGreater()) { + (void)snprintf(version, ND_WIN_VER_LENGTH, "Vista SP1"); + } else if (IsWindowsVistaOrGreater()) { + (void)snprintf(version, ND_WIN_VER_LENGTH, "Vista"); + } + // We are not testing older, because it is not supported anymore by Microsoft + + (void)snprintf(os, length, "Microsoft Windows Version %s, Build %d (Name: Windows %s)", versionName, build, version); +} + +static void netdata_windows_os_version(char *out, DWORD length) +{ + if (netdata_registry_get_string(out, + length, + HKEY_LOCAL_MACHINE, + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", + "ProductName")) + return; + + (void)snprintf(out, length, "%s", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN); +} + +static void netdata_windows_os_kernel_version(char *out, DWORD length, DWORD build) +{ + char version[8]; + if (!netdata_registry_get_string(version, + 7, + HKEY_LOCAL_MACHINE, + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion", + "CurrentVersion")) + version[0] = '\0'; + + (void)snprintf(out, length, "%s (build: %u)", version, build); +} + +static void netdata_windows_host(struct rrdhost_system_info *systemInfo) +{ + char osVersion[4096]; + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_HOST_OS_NAME", "Microsoft Windows"); + + DWORD build = netdata_windows_get_current_build(); + + netdata_windows_discover_os_version(osVersion, 4095, build); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_HOST_OS_ID", osVersion); + + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_HOST_OS_ID_LIKE", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN); + + netdata_windows_os_version(osVersion, 4095); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_HOST_OS_VERSION", osVersion); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_HOST_OS_VERSION_ID", osVersion); + + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_HOST_OS_DETECTION", NETDATA_WIN_DETECTION_METHOD); + + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_KERNEL_NAME", "Windows"); + + netdata_windows_os_kernel_version(osVersion, 4095, build); + (void)rrdhost_set_system_info_variable(systemInfo, "NETDATA_SYSTEM_KERNEL_VERSION", osVersion); + + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_HOST_IS_K8S_NODE", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_FALSE); +} + +// Cloud +static void netdata_windows_cloud(struct rrdhost_system_info *systemInfo) +{ + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_INSTANCE_CLOUD_TYPE", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_INSTANCE_CLOUD_INSTANCE_TYPE", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_INSTANCE_CLOUD_INSTANCE_REGION", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN); +} + +// Container +static void netdata_windows_container(struct rrdhost_system_info *systemInfo) +{ + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_CONTAINER_OS_NAME", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_CONTAINER_OS_ID", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_CONTAINER_OS_ID_LIKE", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_CONTAINER_OS_VERSION", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_CONTAINER_OS_VERSION_ID", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_CONTAINER_OS_DETECTION", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE); + (void)rrdhost_set_system_info_variable( + systemInfo, "NETDATA_CONTAINER_IS_OFFICIAL_IMAGE", NETDATA_DEFAULT_SYSTEM_INFO_VALUE_FALSE); +} + +void netdata_windows_get_system_info(struct rrdhost_system_info *systemInfo) +{ + netdata_windows_cloud(systemInfo); + netdata_windows_container(systemInfo); + netdata_windows_host(systemInfo); + netdata_windows_get_cpu(systemInfo); + netdata_windows_get_mem(systemInfo); + netdata_windows_get_total_disk_size(systemInfo); +} +#endif diff --git a/src/daemon/win_system-info.h b/src/daemon/win_system-info.h new file mode 100644 index 000000000..a4f0906f1 --- /dev/null +++ b/src/daemon/win_system-info.h @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef _NETDATA_WIN_SYSTEM_INFO_H_ +#define _NETDATA_WIN_SYSTEM_INFO_H_ + +// the netdata database +#include "database/rrd.h" + +#define NETDATA_DEFAULT_SYSTEM_INFO_VALUE_UNKNOWN "unknown" +#define NETDATA_DEFAULT_SYSTEM_INFO_VALUE_NONE "none" +#define NETDATA_DEFAULT_SYSTEM_INFO_VALUE_FALSE "false" + +#ifdef OS_WINDOWS +#include "windows.h" +#include "versionhelpers.h" + +void netdata_windows_get_system_info(struct rrdhost_system_info *system_info); +#endif + +#endif // _NETDATA_WIN_SYSTEM_INFO_H_ diff --git a/src/daemon/winsvc.cc b/src/daemon/winsvc.cc new file mode 100644 index 000000000..9c5eb49ff --- /dev/null +++ b/src/daemon/winsvc.cc @@ -0,0 +1,252 @@ +extern "C" { + +#include "daemon.h" +#include "libnetdata/libnetdata.h" + +int netdata_main(int argc, char *argv[]); +void signals_handle(void); + +} + +#include <windows.h> + +__attribute__((format(printf, 1, 2))) +static void netdata_service_log(const char *fmt, ...) +{ + char path[FILENAME_MAX + 1]; + snprintfz(path, FILENAME_MAX, "%s/service.log", LOG_DIR); + + FILE *fp = fopen(path, "a"); + if (fp == NULL) { + return; + } + + SYSTEMTIME time; + GetSystemTime(&time); + fprintf(fp, "%d:%d:%d - ", time.wHour, time.wMinute, time.wSecond); + + va_list args; + va_start(args, fmt); + vfprintf(fp, fmt, args); + va_end(args); + + fprintf(fp, "\n"); + + fflush(fp); + fclose(fp); +} + +static SERVICE_STATUS_HANDLE svc_status_handle = nullptr; +static SERVICE_STATUS svc_status = {}; + +static HANDLE svc_stop_event_handle = nullptr; + +static ND_THREAD *cleanup_thread = nullptr; + +static bool ReportSvcStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint, DWORD dwControlsAccepted) +{ + static DWORD dwCheckPoint = 1; + svc_status.dwCurrentState = dwCurrentState; + svc_status.dwWin32ExitCode = dwWin32ExitCode; + svc_status.dwWaitHint = dwWaitHint; + svc_status.dwControlsAccepted = dwControlsAccepted; + + if (dwCurrentState == SERVICE_RUNNING || dwCurrentState == SERVICE_STOPPED) + { + svc_status.dwCheckPoint = 0; + } + else + { + svc_status.dwCheckPoint = dwCheckPoint++; + } + + if (!SetServiceStatus(svc_status_handle, &svc_status)) { + netdata_service_log("@ReportSvcStatus: SetServiceStatusFailed (%d)", GetLastError()); + return false; + } + + return true; +} + +static HANDLE CreateEventHandle(const char *msg) +{ + HANDLE h = CreateEvent(NULL, TRUE, FALSE, NULL); + + if (!h) + { + netdata_service_log(msg); + + if (!ReportSvcStatus(SERVICE_STOPPED, GetLastError(), 1000, 0)) + { + netdata_service_log("Failed to set service status to stopped."); + } + + return NULL; + } + + return h; +} + +static void *call_netdata_cleanup(void *arg) +{ + UNUSED(arg); + + // Wait until we have to stop the service + netdata_service_log("Cleanup thread waiting for stop event..."); + WaitForSingleObject(svc_stop_event_handle, INFINITE); + + // Stop the agent + netdata_service_log("Running netdata cleanup..."); + netdata_cleanup_and_exit(0, NULL, NULL, NULL); + + // Close event handle + netdata_service_log("Closing stop event handle..."); + CloseHandle(svc_stop_event_handle); + + // Set status to stopped + netdata_service_log("Reporting the service as stopped..."); + ReportSvcStatus(SERVICE_STOPPED, 0, 0, 0); + + return nullptr; +} + +static void WINAPI ServiceControlHandler(DWORD controlCode) +{ + switch (controlCode) + { + case SERVICE_CONTROL_STOP: + { + if (svc_status.dwCurrentState != SERVICE_RUNNING) + return; + + // Set service status to stop-pending + netdata_service_log("Setting service status to stop-pending..."); + if (!ReportSvcStatus(SERVICE_STOP_PENDING, 0, 5000, 0)) + return; + + // Create cleanup thread + netdata_service_log("Creating cleanup thread..."); + char tag[NETDATA_THREAD_TAG_MAX + 1]; + snprintfz(tag, NETDATA_THREAD_TAG_MAX, "%s", "CLEANUP"); + cleanup_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_JOINABLE, call_netdata_cleanup, NULL); + + // Signal the stop request + netdata_service_log("Signalling the cleanup thread..."); + SetEvent(svc_stop_event_handle); + break; + } + case SERVICE_CONTROL_INTERROGATE: + { + ReportSvcStatus(svc_status.dwCurrentState, svc_status.dwWin32ExitCode, svc_status.dwWaitHint, svc_status.dwControlsAccepted); + break; + } + default: + break; + } +} + +void WINAPI ServiceMain(DWORD argc, LPSTR* argv) +{ + UNUSED(argc); + UNUSED(argv); + + // Create service status handle + netdata_service_log("Creating service status handle..."); + svc_status_handle = RegisterServiceCtrlHandler("Netdata", ServiceControlHandler); + if (!svc_status_handle) + { + netdata_service_log("@ServiceMain() - RegisterServiceCtrlHandler() failed..."); + return; + } + + // Set status to start-pending + netdata_service_log("Setting service status to start-pending..."); + svc_status.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + svc_status.dwServiceSpecificExitCode = 0; + svc_status.dwCheckPoint = 0; + if (!ReportSvcStatus(SERVICE_START_PENDING, 0, 5000, 0)) + { + netdata_service_log("Failed to set service status to start pending."); + return; + } + + // Create stop service event handle + netdata_service_log("Creating stop service event handle..."); + svc_stop_event_handle = CreateEventHandle("Failed to create stop event handle"); + if (!svc_stop_event_handle) + return; + + // Set status to running + netdata_service_log("Setting service status to running..."); + if (!ReportSvcStatus(SERVICE_RUNNING, 0, 5000, SERVICE_ACCEPT_STOP)) + { + netdata_service_log("Failed to set service status to running."); + return; + } + + // Run the agent + netdata_service_log("Running the agent..."); + netdata_main(argc, argv); + + netdata_service_log("Agent has been started..."); +} + +static bool update_path() { + const char *old_path = getenv("PATH"); + + if (!old_path) { + if (setenv("PATH", "/usr/bin", 1) != 0) { + netdata_service_log("Failed to set PATH to /usr/bin"); + return false; + } + + return true; + } + + size_t new_path_length = strlen(old_path) + strlen("/usr/bin") + 2; + char *new_path = (char *) callocz(new_path_length, sizeof(char)); + snprintfz(new_path, new_path_length, "/usr/bin:%s", old_path); + + if (setenv("PATH", new_path, 1) != 0) { + netdata_service_log("Failed to add /usr/bin to PATH"); + freez(new_path); + return false; + } + + freez(new_path); + return true; +} + +int main(int argc, char *argv[]) +{ + bool tty = isatty(fileno(stdin)) == 1; + + if (!update_path()) { + return 1; + } + + if (tty) + { + int rc = netdata_main(argc, argv); + if (rc != 10) + return rc; + + signals_handle(); + return 1; + } + else + { + SERVICE_TABLE_ENTRY serviceTable[] = { + { strdupz("Netdata"), ServiceMain }, + { nullptr, nullptr } + }; + + if (!StartServiceCtrlDispatcher(serviceTable)) + { + netdata_service_log("@main() - StartServiceCtrlDispatcher() failed..."); + return 1; + } + + return 0; + } +} |