From 483926a283e118590da3f9ecfa75a8a4d62143ce Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 1 Dec 2021 07:15:11 +0100 Subject: Merging upstream version 1.32.0. Signed-off-by: Daniel Baumann --- daemon/README.md | 2 +- daemon/analytics.c | 202 +++++++++++++++++++++++++++++++++----- daemon/analytics.h | 12 ++- daemon/anonymous-statistics.sh.in | 22 ++++- daemon/buildinfo.c | 134 ++++++++++++++++--------- daemon/commands.c | 21 +++- daemon/commands.h | 1 + daemon/common.c | 2 + daemon/common.h | 14 +-- daemon/daemon.c | 5 +- daemon/global_statistics.c | 2 +- daemon/main.c | 94 ++++++++---------- daemon/service.c | 38 +++++++ daemon/service.h | 19 ++++ daemon/system-info.sh | 2 + daemon/unit_test.c | 81 +++++++++------ daemon/unit_test.h | 1 + 17 files changed, 485 insertions(+), 167 deletions(-) create mode 100644 daemon/service.c create mode 100644 daemon/service.h (limited to 'daemon') diff --git a/daemon/README.md b/daemon/README.md index 359b3ea39..1ea865f89 100644 --- a/daemon/README.md +++ b/daemon/README.md @@ -184,7 +184,7 @@ The command line options of the Netdata 1.10.0 version are the following: Check if string matches pattern and exit. -W "claim -token=TOKEN -rooms=ROOM1,ROOM2 url=https://app.netdata.cloud" - Claim the agent to the workspace rooms pointed to by TOKEN and ROOM*. + Connect the agent to the workspace rooms pointed to by TOKEN and ROOM*. Signals netdata handles: diff --git a/daemon/analytics.c b/daemon/analytics.c index 08923a3cb..bb878f708 100644 --- a/daemon/analytics.c +++ b/daemon/analytics.c @@ -4,6 +4,7 @@ struct analytics_data analytics_data; extern void analytics_exporting_connectors (BUFFER *b); +extern void analytics_exporting_connectors_ssl (BUFFER *b); extern void analytics_build_info (BUFFER *b); extern int aclk_connected; @@ -54,6 +55,12 @@ void analytics_log_data(void) debug(D_ANALYTICS, "NETDATA_HOST_ACLK_IMPLEMENTATION : [%s]", analytics_data.netdata_host_aclk_implementation); debug(D_ANALYTICS, "NETDATA_HOST_AGENT_CLAIMED : [%s]", analytics_data.netdata_host_agent_claimed); debug(D_ANALYTICS, "NETDATA_HOST_CLOUD_ENABLED : [%s]", analytics_data.netdata_host_cloud_enabled); + debug(D_ANALYTICS, "NETDATA_CONFIG_HTTPS_AVAILABLE : [%s]", analytics_data.netdata_config_https_available); + debug(D_ANALYTICS, "NETDATA_INSTALL_TYPE : [%s]", analytics_data.netdata_install_type); + debug(D_ANALYTICS, "NETDATA_PREBUILT_DISTRO : [%s]", analytics_data.netdata_prebuilt_distro); + debug(D_ANALYTICS, "NETDATA_CONFIG_IS_PRIVATE_REGISTRY : [%s]", analytics_data.netdata_config_is_private_registry); + debug(D_ANALYTICS, "NETDATA_CONFIG_USE_PRIVATE_REGISTRY: [%s]", analytics_data.netdata_config_use_private_registry); + debug(D_ANALYTICS, "NETDATA_CONFIG_OOM_SCORE : [%s]", analytics_data.netdata_config_oom_score); } /* @@ -93,6 +100,12 @@ void analytics_free_data(void) freez(analytics_data.netdata_host_aclk_implementation); freez(analytics_data.netdata_host_agent_claimed); freez(analytics_data.netdata_host_cloud_enabled); + freez(analytics_data.netdata_config_https_available); + freez(analytics_data.netdata_install_type); + freez(analytics_data.netdata_config_is_private_registry); + freez(analytics_data.netdata_config_use_private_registry); + freez(analytics_data.netdata_config_oom_score); + freez(analytics_data.netdata_prebuilt_distro); } /* @@ -137,7 +150,7 @@ void analytics_get_data(char *name, BUFFER *wb) */ void analytics_log_prometheus(void) { - if (likely(analytics_data.prometheus_hits < ANALYTICS_MAX_PROMETHEUS_HITS)) { + if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.prometheus_hits < ANALYTICS_MAX_PROMETHEUS_HITS)) { analytics_data.prometheus_hits++; char b[7]; snprintfz(b, 6, "%d", analytics_data.prometheus_hits); @@ -150,7 +163,7 @@ void analytics_log_prometheus(void) */ void analytics_log_shell(void) { - if (likely(analytics_data.shell_hits < ANALYTICS_MAX_SHELL_HITS)) { + if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.shell_hits < ANALYTICS_MAX_SHELL_HITS)) { analytics_data.shell_hits++; char b[7]; snprintfz(b, 6, "%d", analytics_data.shell_hits); @@ -163,7 +176,7 @@ void analytics_log_shell(void) */ void analytics_log_json(void) { - if (likely(analytics_data.json_hits < ANALYTICS_MAX_JSON_HITS)) { + if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.json_hits < ANALYTICS_MAX_JSON_HITS)) { analytics_data.json_hits++; char b[7]; snprintfz(b, 6, "%d", analytics_data.json_hits); @@ -176,7 +189,7 @@ void analytics_log_json(void) */ void analytics_log_dashboard(void) { - if (likely(analytics_data.dashboard_hits < ANALYTICS_MAX_DASHBOARD_HITS)) { + if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.dashboard_hits < ANALYTICS_MAX_DASHBOARD_HITS)) { analytics_data.dashboard_hits++; char b[7]; snprintfz(b, 6, "%d", analytics_data.dashboard_hits); @@ -184,6 +197,15 @@ void analytics_log_dashboard(void) } } +/* + * Called when setting the oom score + */ +void analytics_report_oom_score(long long int score){ + char b[7]; + snprintfz(b, 6, "%d", (int)score); + analytics_set_data(&analytics_data.netdata_config_oom_score, b); +} + void analytics_mirrored_hosts(void) { RRDHOST *host; @@ -217,7 +239,7 @@ void analytics_mirrored_hosts(void) void analytics_exporters(void) { //when no exporters are available, an empty string will be sent - //decide if something else is more suitable (but propably not null) + //decide if something else is more suitable (but probably not null) BUFFER *bi = buffer_create(1000); analytics_exporting_connectors(bi); analytics_set_data_str(&analytics_data.netdata_exporting_connectors, (char *)buffer_tostring(bi)); @@ -335,6 +357,67 @@ void analytics_alarms_notifications(void) buffer_free(b); } +char *get_value_from_key(char *buffer, char *key) +{ + char *s = NULL, *t = NULL; + s = t = buffer + strlen(key) + 2; + if (s) { + while (*s == '\'') + s++; + while (*++t != '\0'); + while (--t > s && *t == '\'') + *t = '\0'; + } + return s; +} + +/* + * Checks for the existence of .install_type file and reads it + */ +void analytics_get_install_type(void) +{ + char *install_type_filename; + analytics_set_data_str(&analytics_data.netdata_install_type, ""); + analytics_set_data_str(&analytics_data.netdata_prebuilt_distro, ""); + + int install_type_filename_len = (strlen(netdata_configured_user_config_dir) + strlen(".install-type") + 3); + install_type_filename = mallocz(sizeof(char) * install_type_filename_len); + snprintfz(install_type_filename, install_type_filename_len - 1, "%s/%s", netdata_configured_user_config_dir, ".install-type"); + + FILE *fp = fopen(install_type_filename, "r"); + if (fp) { + char *s, buf[256 + 1]; + size_t len = 0; + + while ((s = fgets_trim_len(buf, 256, fp, &len))) { + if (!strncmp(buf, "INSTALL_TYPE='", 14)) + analytics_set_data_str(&analytics_data.netdata_install_type, (char *)get_value_from_key(buf, "INSTALL_TYPE")); + else if (!strncmp(buf, "PREBUILT_DISTRO='", 17)) + analytics_set_data_str(&analytics_data.netdata_prebuilt_distro, (char *)get_value_from_key(buf, "PREBUILT_DISTRO")); + } + fclose(fp); + } + freez(install_type_filename); +} + +/* + * Pick up if https is actually used + */ +void analytics_https(void) +{ + BUFFER *b = buffer_create(30); +#ifdef ENABLE_HTTPS + analytics_exporting_connectors_ssl(b); + buffer_strcat(b, netdata_client_ctx && localhost->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE && localhost->rrdpush_sender_connected == 1 ? "streaming|" : "|"); + buffer_strcat(b, netdata_srv_ctx ? "web" : ""); +#else + buffer_strcat(b, "||"); +#endif + + analytics_set_data_str(&analytics_data.netdata_config_https_available, (char *)buffer_tostring(b)); + buffer_free(b); +} + void analytics_charts(void) { RRDSET *st; @@ -411,13 +494,13 @@ void analytics_misc(void) { #ifdef ENABLE_ACLK analytics_set_data(&analytics_data.netdata_host_cloud_available, "true"); -#ifdef ACLK_NG - analytics_set_data_str(&analytics_data.netdata_host_aclk_implementation, "Next Generation"); -#else - analytics_set_data_str(&analytics_data.netdata_host_aclk_implementation, "legacy"); -#endif + if (aclk_ng) + analytics_set_data_str(&analytics_data.netdata_host_aclk_implementation, "Next Generation"); + else + analytics_set_data_str(&analytics_data.netdata_host_aclk_implementation, "legacy"); #else analytics_set_data(&analytics_data.netdata_host_cloud_available, "false"); + analytics_set_data_str(&analytics_data.netdata_host_aclk_implementation, ""); #endif #ifdef ENABLE_ACLK @@ -426,17 +509,33 @@ void analytics_misc(void) else #endif analytics_set_data(&analytics_data.netdata_host_aclk_available, "false"); + + analytics_set_data(&analytics_data.netdata_config_exporting_enabled, appconfig_get_boolean(&exporting_config, CONFIG_SECTION_EXPORTING, "enabled", CONFIG_BOOLEAN_NO) ? "true" : "false"); + + analytics_set_data(&analytics_data.netdata_config_is_private_registry, "false"); + analytics_set_data(&analytics_data.netdata_config_use_private_registry, "false"); + + if (strcmp( + config_get(CONFIG_SECTION_REGISTRY, "registry to announce", "https://registry.my-netdata.io"), + "https://registry.my-netdata.io")) + analytics_set_data(&analytics_data.netdata_config_use_private_registry, "true"); + + //do we need both registry to announce and enabled to indicate that this is a private registry ? + if (config_get_boolean(CONFIG_SECTION_REGISTRY, "enabled", CONFIG_BOOLEAN_NO) && + web_server_mode != WEB_SERVER_MODE_NONE) + analytics_set_data(&analytics_data.netdata_config_is_private_registry, "true"); } /* * Get the meta data, called from the thread once after the original delay - * These are values that won't change between agent restarts, and therefore + * These are values that won't change during agent runtime, and therefore * don't try to read them on each META event send */ void analytics_gather_immutable_meta_data(void) { analytics_misc(); analytics_exporters(); + analytics_https(); } /* @@ -490,6 +589,7 @@ void analytics_main_cleanup(void *ptr) static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; debug(D_ANALYTICS, "Cleaning up..."); + analytics_free_data(); static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; } @@ -521,7 +621,7 @@ void *analytics_main(void *ptr) analytics_gather_immutable_meta_data(); analytics_gather_mutable_meta_data(); - send_statistics("META", "-", "-"); + send_statistics("META_START", "-", "-"); analytics_log_data(); sec = 0; @@ -567,7 +667,6 @@ void set_late_global_environment() { analytics_set_data(&analytics_data.netdata_config_stream_enabled, default_rrdpush_enabled ? "true" : "false"); analytics_set_data_str(&analytics_data.netdata_config_memory_mode, (char *)rrd_memory_mode_name(default_rrd_memory_mode)); - analytics_set_data(&analytics_data.netdata_config_exporting_enabled, appconfig_get_boolean(&exporting_config, CONFIG_SECTION_EXPORTING, "enabled", CONFIG_BOOLEAN_NO) ? "true" : "false"); #ifdef DISABLE_CLOUD analytics_set_data(&analytics_data.netdata_host_cloud_enabled, "false"); @@ -607,9 +706,11 @@ void set_late_global_environment() analytics_set_data_str(&analytics_data.netdata_buildinfo, (char *)buffer_tostring(bi)); buffer_free(bi); } + + analytics_get_install_type(); } -static void get_system_timezone(void) +void get_system_timezone(void) { // avoid flood calls to stat(/etc/localtime) // http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux @@ -630,7 +731,7 @@ static void get_system_timezone(void) // use the contents of /etc/timezone if (!timezone && !read_file("/etc/timezone", buffer, FILENAME_MAX)) { timezone = buffer; - info("TIMEZONE: using the contents of /etc/timezone: '%s'", timezone); + info("TIMEZONE: using the contents of /etc/timezone"); } // read the link /etc/localtime @@ -696,6 +797,46 @@ static void get_system_timezone(void) timezone = "unknown"; netdata_configured_timezone = config_get(CONFIG_SECTION_GLOBAL, "timezone", timezone); + + //get the utc offset, and the timezone as returned by strftime + //will be sent to the cloud + //Note: This will need an agent restart to get new offset on time change (dst, etc). + { + time_t t; + struct tm *tmp, tmbuf; + char zone[FILENAME_MAX + 1]; + char sign[2], hh[3], mm[3]; + + t = now_realtime_sec(); + tmp = localtime_r(&t, &tmbuf); + + if (tmp != NULL) { + if (strftime(zone, FILENAME_MAX, "%Z", tmp) == 0) { + netdata_configured_abbrev_timezone = strdupz("UTC"); + } else + netdata_configured_abbrev_timezone = strdupz(zone); + + if (strftime(zone, FILENAME_MAX, "%z", tmp) == 0) { + netdata_configured_utc_offset = 0; + } else { + sign[0] = zone[0] == '-' || zone[0] == '+' ? zone[0] : '0'; + sign[1] = '\0'; + hh[0] = isdigit(zone[1]) ? zone[1] : '0'; + hh[1] = isdigit(zone[2]) ? zone[2] : '0'; + hh[2] = '\0'; + mm[0] = isdigit(zone[3]) ? zone[3] : '0'; + mm[1] = isdigit(zone[4]) ? zone[4] : '0'; + mm[2] = '\0'; + + netdata_configured_utc_offset = (str2i(hh) * 3600) + (str2i(mm) * 60); + netdata_configured_utc_offset = + sign[0] == '-' ? -netdata_configured_utc_offset : netdata_configured_utc_offset; + } + } else { + netdata_configured_abbrev_timezone = strdupz("UTC"); + netdata_configured_utc_offset = 0; + } + } } void set_global_environment() @@ -753,6 +894,12 @@ void set_global_environment() analytics_set_data(&analytics_data.netdata_host_aclk_available, "null"); analytics_set_data(&analytics_data.netdata_host_agent_claimed, "null"); analytics_set_data(&analytics_data.netdata_host_cloud_enabled, "null"); + analytics_set_data(&analytics_data.netdata_config_https_available, "null"); + analytics_set_data(&analytics_data.netdata_install_type, "null"); + analytics_set_data(&analytics_data.netdata_config_is_private_registry, "null"); + analytics_set_data(&analytics_data.netdata_config_use_private_registry, "null"); + analytics_set_data(&analytics_data.netdata_config_oom_score, "null"); + analytics_set_data(&analytics_data.netdata_prebuilt_distro, "null"); analytics_data.prometheus_hits = 0; analytics_data.shell_hits = 0; @@ -770,8 +917,6 @@ void set_global_environment() if (clean) freez(default_port); - get_system_timezone(); - // set the path we need char path[1024 + 1], *p = getenv("PATH"); if (!p) @@ -834,7 +979,7 @@ void send_statistics(const char *action, const char *action_result, const char * sprintf( command_to_run, - "%s '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' ", + "%s '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' ", as_script, action, action_result, @@ -870,16 +1015,27 @@ void send_statistics(const char *action, const char *action_result, const char * analytics_data.netdata_host_aclk_available, analytics_data.netdata_host_aclk_implementation, analytics_data.netdata_host_agent_claimed, - analytics_data.netdata_host_cloud_enabled); + analytics_data.netdata_host_cloud_enabled, + analytics_data.netdata_config_https_available, + analytics_data.netdata_install_type, + analytics_data.netdata_config_is_private_registry, + analytics_data.netdata_config_use_private_registry, + analytics_data.netdata_config_oom_score, + analytics_data.netdata_prebuilt_distro); info("%s '%s' '%s' '%s'", as_script, action, action_result, action_data); FILE *fp = mypopen(command_to_run, &command_pid); if (fp) { - char buffer[100 + 1]; - while (fgets(buffer, 100, fp) != NULL) - ; - mypclose(fp, command_pid); + char buffer[4 + 1]; + char *s = fgets(buffer, 4, fp); + int exit_code = mypclose(fp, command_pid); + if (exit_code) + error("Execution of anonymous statistics script returned %d.", exit_code); + if (s && strncmp(buffer, "200", 3)) + error("Execution of anonymous statistics script returned http code %s.", buffer); + } else { + error("Failed to run anonymous statistics script %s.", as_script); } freez(command_to_run); } diff --git a/daemon/analytics.h b/daemon/analytics.h index e888297df..78ced981e 100644 --- a/daemon/analytics.h +++ b/daemon/analytics.h @@ -3,7 +3,7 @@ #ifndef NETDATA_ANALYTICS_H #define NETDATA_ANALYTICS_H 1 -#include "../daemon/common.h" +#include "daemon/common.h" /* Max number of seconds before the first META analytics is sent */ #define ANALYTICS_INIT_SLEEP_SEC 120 @@ -29,7 +29,7 @@ }, /* Needed to calculate the space needed for parameters */ -#define ANALYTICS_NO_OF_ITEMS 32 +#define ANALYTICS_NO_OF_ITEMS 38 struct analytics_data { char *netdata_config_stream_enabled; @@ -64,6 +64,12 @@ struct analytics_data { char *netdata_host_aclk_implementation; char *netdata_host_agent_claimed; char *netdata_host_cloud_enabled; + char *netdata_config_https_available; + char *netdata_install_type; + char *netdata_config_is_private_registry; + char *netdata_config_use_private_registry; + char *netdata_config_oom_score; + char *netdata_prebuilt_distro; size_t data_length; @@ -84,6 +90,8 @@ extern void analytics_log_json(void); extern void analytics_log_prometheus(void); extern void analytics_log_dashboard(void); extern void analytics_gather_mutable_meta_data(void); +extern void analytics_report_oom_score(long long int score); +extern void get_system_timezone(void); extern struct analytics_data analytics_data; diff --git a/daemon/anonymous-statistics.sh.in b/daemon/anonymous-statistics.sh.in index bd22963d9..f49d19d09 100755 --- a/daemon/anonymous-statistics.sh.in +++ b/daemon/anonymous-statistics.sh.in @@ -22,9 +22,6 @@ if [ -f "@configdir_POST@/.opt-out-from-anonymous-statistics" ] || [ ! "${DO_NOT exit 0 fi -# Shorten version for easier reporting -NETDATA_VERSION=$(echo "${NETDATA_VERSION}" | sed 's/-.*//g' | tr -d 'v') - # ------------------------------------------------------------------------------------------------- # Get the extra variables @@ -60,6 +57,13 @@ NETDATA_HOST_ACLK_AVAILABLE="${32}" NETDATA_HOST_ACLK_IMPLEMENTATION="${33}" NETDATA_HOST_AGENT_CLAIMED="${34}" NETDATA_HOST_CLOUD_ENABLED="${35}" +NETDATA_CONFIG_HTTPS_AVAILABLE="${36}" +NETDATA_INSTALL_TYPE="${37}" +NETDATA_IS_PRIVATE_REGISTRY="${38}" +NETDATA_USE_PRIVATE_REGISTRY="${39}" +NETDATA_CONFIG_OOM_SCORE="${40}" +NETDATA_PREBUILT_DISTRO="${41}" + # define body of request to be sent REQ_BODY="$(cat << EOF @@ -80,6 +84,8 @@ REQ_BODY="$(cat << EOF "netdata_version": "${NETDATA_VERSION}", "netdata_buildinfo": ${NETDATA_BUILDINFO}, "netdata_release_channel": ${NETDATA_CONFIG_RELEASE_CHANNEL}, + "netdata_install_type": ${NETDATA_INSTALL_TYPE}, + "netdata_prebuilt_distro": ${NETDATA_PREBUILT_DISTRO}, "host_os_name": "${NETDATA_HOST_OS_NAME}", "host_os_id": "${NETDATA_HOST_OS_ID}", "host_os_id_like": "${NETDATA_HOST_OS_ID_LIKE}", @@ -100,6 +106,7 @@ REQ_BODY="$(cat << EOF "container_os_version": "${NETDATA_CONTAINER_OS_VERSION}", "container_os_version_id": "${NETDATA_CONTAINER_OS_VERSION_ID}", "container_os_detection": "${NETDATA_CONTAINER_OS_DETECTION}", + "container_is_official_image": ${NETDATA_CONTAINER_IS_OFFICIAL_IMAGE}, "system_cpu_detection": "${NETDATA_SYSTEM_CPU_DETECTION}", "system_cpu_freq": "${NETDATA_SYSTEM_CPU_FREQ}", "system_cpu_logical_cpu_count": "${NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT}", @@ -114,10 +121,14 @@ REQ_BODY="$(cat << EOF "config_page_cache_size": ${NETDATA_CONFIG_PAGE_CACHE_SIZE}, "config_multidb_disk_quota": ${NETDATA_CONFIG_MULTIDB_DISK_QUOTA}, "config_https_enabled": ${NETDATA_CONFIG_HTTPS_ENABLED}, + "config_https_available": ${NETDATA_CONFIG_HTTPS_AVAILABLE}, "config_web_enabled": ${NETDATA_CONFIG_WEB_ENABLED}, "config_exporting_enabled": ${NETDATA_CONFIG_EXPORTING_ENABLED}, "config_is_parent": ${NETDATA_CONFIG_IS_PARENT}, + "config_is_private_registry": ${NETDATA_IS_PRIVATE_REGISTRY}, + "config_private_registry_used": ${NETDATA_USE_PRIVATE_REGISTRY}, "config_hosts_available": ${NETDATA_CONFIG_HOSTS_AVAILABLE}, + "config_oom_score": ${NETDATA_CONFIG_OOM_SCORE}, "alarms_normal": ${NETDATA_ALARMS_NORMAL}, "alarms_warning": ${NETDATA_ALARMS_WARNING}, "alarms_critical": ${NETDATA_ALARMS_CRITICAL}, @@ -148,12 +159,13 @@ EOF # send the anonymous statistics to the Netdata PostHog if [ -n "$(command -v curl 2> /dev/null)" ]; then - curl -X POST --max-time 2 --header "Content-Type: application/json" -d "${REQ_BODY}" https://posthog.netdata.cloud/capture/ > /dev/null 2>&1 + curl --silent -o /dev/null --write-out '%{http_code}' -X POST --max-time 2 --header "Content-Type: application/json" -d "${REQ_BODY}" https://posthog.netdata.cloud/capture/ else wget -q -O - --no-check-certificate \ + --server-response \ --method POST \ --timeout=1 \ --header 'Content-Type: application/json' \ --body-data "${REQ_BODY}" \ - 'https://posthog.netdata.cloud/capture/' > /dev/null 2>&1 + 'https://posthog.netdata.cloud/capture/' 2>&1 | awk '/^ HTTP/{print $2}' fi diff --git a/daemon/buildinfo.c b/daemon/buildinfo.c index ebeaa996d..a15250f48 100644 --- a/daemon/buildinfo.c +++ b/daemon/buildinfo.c @@ -9,13 +9,7 @@ #ifdef ENABLE_ACLK #define FEAT_CLOUD 1 #define FEAT_CLOUD_MSG "" -#ifdef ACLK_NG -#define ACLK_IMPL "Next Generation" -#else -#define ACLK_IMPL "Legacy" -#endif #else -#define ACLK_IMPL "" #ifdef DISABLE_CLOUD #define FEAT_CLOUD 0 #define FEAT_CLOUD_MSG "(by user request)" @@ -25,6 +19,24 @@ #endif #endif +#ifdef ACLK_NG +#define FEAT_ACLK_NG 1 +#else +#define FEAT_ACLK_NG 0 +#endif + +#if defined(ACLK_NG) && defined(ENABLE_NEW_CLOUD_PROTOCOL) +#define NEW_CLOUD_PROTO 1 +#else +#define NEW_CLOUD_PROTO 0 +#endif + +#ifdef ACLK_LEGACY +#define FEAT_ACLK_LEGACY 1 +#else +#define FEAT_ACLK_LEGACY 0 +#endif + #ifdef ENABLE_DBENGINE #define FEAT_DBENGINE 1 #else @@ -43,8 +55,31 @@ #define FEAT_NATIVE_HTTPS 0 #endif +#ifdef ENABLE_ML +#define FEAT_ML 1 +#else +#define FEAT_ML 0 +#endif + // Optional libraries +#ifdef HAVE_PROTOBUF +#if defined(ACLK_NG) || defined(ENABLE_PROMETHEUS_REMOTE_WRITE) +#define FEAT_PROTOBUF 1 +#ifdef BUNDLED_PROTOBUF +#define FEAT_PROTOBUF_BUNDLED " (bundled)" +#else +#define FEAT_PROTOBUF_BUNDLED " (system)" +#endif +#else +#define FEAT_PROTOBUF 0 +#define FEAT_PROTOBUF_BUNDLED "" +#endif +#else +#define FEAT_PROTOBUF 0 +#define FEAT_PROTOBUF_BUNDLED "" +#endif + #ifdef ENABLE_JSONC #define FEAT_JSONC 1 #else @@ -69,29 +104,29 @@ #define FEAT_LIBCAP 0 #endif -#ifndef ACLK_NG -#ifdef ACLK_NO_LIBMOSQ -#define FEAT_MOSQUITTO 0 -#else -#define FEAT_MOSQUITTO 1 -#endif - -#ifdef ACLK_NO_LWS -#define FEAT_LWS 0 -#define FEAT_LWS_MSG "" -#else -#ifdef ENABLE_ACLK -#include -#endif -#ifdef BUNDLED_LWS -#define FEAT_LWS 1 -#define FEAT_LWS_MSG "static" -#else -#define FEAT_LWS 1 -#define FEAT_LWS_MSG "shared-lib" -#endif -#endif -#endif /* ACLK_NG */ +#ifndef ACLK_LEGACY_DISABLED + #ifdef ACLK_NO_LIBMOSQ + #define FEAT_MOSQUITTO 0 + #else + #define FEAT_MOSQUITTO 1 + #endif + + #ifdef ACLK_NO_LWS + #define FEAT_LWS 0 + #define FEAT_LWS_MSG "" + #else + #ifdef ACLK_LEGACY + #include + #endif + #ifdef BUNDLED_LWS + #define FEAT_LWS 1 + #define FEAT_LWS_MSG "static" + #else + #define FEAT_LWS 1 + #define FEAT_LWS_MSG "shared-lib" + #endif + #endif +#endif /* ACLK_LEGACY_DISABLED */ #ifdef NETDATA_WITH_ZLIB #define FEAT_ZLIB 1 @@ -205,22 +240,24 @@ void print_build_info(void) { printf("Configure options: %s\n", CONFIGURE_COMMAND); printf("Features:\n"); - printf(" dbengine: %s\n", FEAT_YES_NO(FEAT_DBENGINE)); - printf(" Native HTTPS: %s\n", FEAT_YES_NO(FEAT_NATIVE_HTTPS)); - printf(" Netdata Cloud: %s %s\n", FEAT_YES_NO(FEAT_CLOUD), FEAT_CLOUD_MSG); -#if FEAT_CLOUD == 1 - printf(" Cloud Implementation: %s\n", ACLK_IMPL); -#endif - printf(" TLS Host Verification: %s\n", FEAT_YES_NO(FEAT_TLS_HOST_VERIFY)); + printf(" dbengine: %s\n", FEAT_YES_NO(FEAT_DBENGINE)); + printf(" Native HTTPS: %s\n", FEAT_YES_NO(FEAT_NATIVE_HTTPS)); + printf(" Netdata Cloud: %s %s\n", FEAT_YES_NO(FEAT_CLOUD), FEAT_CLOUD_MSG); + printf(" ACLK Next Generation: %s\n", FEAT_YES_NO(FEAT_ACLK_NG)); + printf(" ACLK-NG New Cloud Protocol: %s\n", FEAT_YES_NO(NEW_CLOUD_PROTO)); + printf(" ACLK Legacy: %s\n", FEAT_YES_NO(FEAT_ACLK_LEGACY)); + printf(" TLS Host Verification: %s\n", FEAT_YES_NO(FEAT_TLS_HOST_VERIFY)); + printf(" Machine Learning: %s\n", FEAT_YES_NO(FEAT_ML)); printf("Libraries:\n"); + printf(" protobuf: %s%s\n", FEAT_YES_NO(FEAT_PROTOBUF), FEAT_PROTOBUF_BUNDLED); printf(" jemalloc: %s\n", FEAT_YES_NO(FEAT_JEMALLOC)); printf(" JSON-C: %s\n", FEAT_YES_NO(FEAT_JSONC)); printf(" libcap: %s\n", FEAT_YES_NO(FEAT_LIBCAP)); printf(" libcrypto: %s\n", FEAT_YES_NO(FEAT_CRYPTO)); printf(" libm: %s\n", FEAT_YES_NO(FEAT_LIBM)); -#ifndef ACLK_NG -#if defined(ENABLE_ACLK) +#ifndef ACLK_LEGACY_DISABLED +#if defined(ACLK_LEGACY) printf(" LWS: %s %s v%d.%d.%d\n", FEAT_YES_NO(FEAT_LWS), FEAT_LWS_MSG, LWS_LIBRARY_VERSION_MAJOR, LWS_LIBRARY_VERSION_MINOR, LWS_LIBRARY_VERSION_PATCH); #else printf(" LWS: %s %s\n", FEAT_YES_NO(FEAT_LWS), FEAT_LWS_MSG); @@ -266,13 +303,17 @@ void print_build_info_json(void) { #else printf(" \"cloud-disabled\": false,\n"); #endif -#if FEAT_CLOUD == 1 - printf(" \"cloud-implementation\": \"%s\",\n", ACLK_IMPL); -#endif - printf(" \"tls-host-verify\": %s\n", FEAT_JSON_BOOL(FEAT_TLS_HOST_VERIFY)); + printf(" \"aclk-ng\": %s,\n", FEAT_JSON_BOOL(FEAT_ACLK_NG)); + printf(" \"aclk-ng-new-cloud-proto\": %s,\n", FEAT_JSON_BOOL(NEW_CLOUD_PROTO)); + printf(" \"aclk-legacy\": %s,\n", FEAT_JSON_BOOL(FEAT_ACLK_LEGACY)); + + printf(" \"tls-host-verify\": %s,\n", FEAT_JSON_BOOL(FEAT_TLS_HOST_VERIFY)); + printf(" \"machine-learning\": %s\n", FEAT_JSON_BOOL(FEAT_ML)); printf(" },\n"); printf(" \"libs\": {\n"); + printf(" \"protobuf\": %s,\n", FEAT_JSON_BOOL(FEAT_PROTOBUF)); + printf(" \"protobuf-source\": \"%s\",\n", FEAT_PROTOBUF_BUNDLED); printf(" \"jemalloc\": %s,\n", FEAT_JSON_BOOL(FEAT_JEMALLOC)); printf(" \"jsonc\": %s,\n", FEAT_JSON_BOOL(FEAT_JSONC)); printf(" \"libcap\": %s,\n", FEAT_JSON_BOOL(FEAT_LIBCAP)); @@ -320,16 +361,21 @@ void analytics_build_info(BUFFER *b) { if(FEAT_DBENGINE) buffer_strcat (b, "dbengine"); if(FEAT_NATIVE_HTTPS) buffer_strcat (b, "|Native HTTPS"); if(FEAT_CLOUD) buffer_strcat (b, "|Netdata Cloud"); + if(FEAT_ACLK_NG) buffer_strcat (b, "|ACLK Next Generation"); + if(NEW_CLOUD_PROTO) buffer_strcat (b, "|New Cloud Protocol Support"); + if(FEAT_ACLK_LEGACY) buffer_strcat (b, "|ACLK Legacy"); if(FEAT_TLS_HOST_VERIFY) buffer_strcat (b, "|TLS Host Verification"); + if(FEAT_ML) buffer_strcat (b, "|Machine Learning"); + if(FEAT_PROTOBUF) buffer_strcat (b, "|protobuf"); if(FEAT_JEMALLOC) buffer_strcat (b, "|jemalloc"); if(FEAT_JSONC) buffer_strcat (b, "|JSON-C"); if(FEAT_LIBCAP) buffer_strcat (b, "|libcap"); if(FEAT_CRYPTO) buffer_strcat (b, "|libcrypto"); if(FEAT_LIBM) buffer_strcat (b, "|libm"); -#ifndef ACLK_NG -#if defined(ENABLE_ACLK) +#ifndef ACLK_LEGACY_DISABLED +#if defined(ENABLE_ACLK) && defined(ACLK_LEGACY) { char buf[20]; snprintfz(buf, 19, "|LWS v%d.%d.%d", LWS_LIBRARY_VERSION_MAJOR, LWS_LIBRARY_VERSION_MINOR, LWS_LIBRARY_VERSION_PATCH); diff --git a/daemon/commands.c b/daemon/commands.c index eac392e33..18468183f 100644 --- a/daemon/commands.c +++ b/daemon/commands.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "common.h" -#include "../database/engine/rrdenginelib.h" +#include "database/engine/rrdenginelib.h" static uv_thread_t thread; static uv_loop_t* loop; @@ -46,6 +46,7 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message); static cmd_status_t cmd_read_config_execute(char *args, char **message); static cmd_status_t cmd_write_config_execute(char *args, char **message); static cmd_status_t cmd_ping_execute(char *args, char **message); +static cmd_status_t cmd_aclk_state(char *args, char **message); static command_info_t command_info_array[] = { {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY}, // show help menu @@ -58,7 +59,8 @@ static command_info_t command_info_array[] = { {"reload-labels", cmd_reload_labels_execute, CMD_TYPE_ORTHOGONAL}, // reload the labels {"read-config", cmd_read_config_execute, CMD_TYPE_CONCURRENT}, {"write-config", cmd_write_config_execute, CMD_TYPE_ORTHOGONAL}, - {"ping", cmd_ping_execute, CMD_TYPE_ORTHOGONAL} + {"ping", cmd_ping_execute, CMD_TYPE_ORTHOGONAL}, + {"aclk-state", cmd_aclk_state, CMD_TYPE_ORTHOGONAL} }; /* Mutexes for commands of type CMD_TYPE_ORTHOGONAL */ @@ -121,7 +123,9 @@ static cmd_status_t cmd_help_execute(char *args, char **message) "reload-claiming-state\n" " Reload agent claiming state from disk.\n" "ping\n" - " Return with 'pong' if agent is alive.\n", + " Return with 'pong' if agent is alive.\n" + "aclk-state [json]\n" + " Returns current state of ACLK and Cloud connection. (optionally in json)\n", MAX_COMMAND_LENGTH - 1); return CMD_STATUS_SUCCESS; } @@ -310,6 +314,17 @@ static cmd_status_t cmd_ping_execute(char *args, char **message) return CMD_STATUS_SUCCESS; } +static cmd_status_t cmd_aclk_state(char *args, char **message) +{ + info("COMMAND: Reopening aclk/cloud state."); + if (strstr(args, "json")) + *message = aclk_state_json(); + else + *message = aclk_state(); + + return CMD_STATUS_SUCCESS; +} + static void cmd_lock_exclusive(unsigned index) { (void)index; diff --git a/daemon/commands.h b/daemon/commands.h index bd4aabfe1..1253e2dc1 100644 --- a/daemon/commands.h +++ b/daemon/commands.h @@ -24,6 +24,7 @@ typedef enum cmd { CMD_READ_CONFIG, CMD_WRITE_CONFIG, CMD_PING, + CMD_ACLK_STATE, CMD_TOTAL_COMMANDS } cmd_t; diff --git a/daemon/common.c b/daemon/common.c index 45d5fa3fd..85d638631 100644 --- a/daemon/common.c +++ b/daemon/common.c @@ -14,6 +14,8 @@ char *netdata_configured_lock_dir = NULL; char *netdata_configured_home_dir = VARLIB_DIR; char *netdata_configured_host_prefix = NULL; char *netdata_configured_timezone = NULL; +char *netdata_configured_abbrev_timezone = NULL; +int32_t netdata_configured_utc_offset = 0; int netdata_ready; int netdata_cloud_setting; diff --git a/daemon/common.h b/daemon/common.h index 4cb54010c..c892dbdb1 100644 --- a/daemon/common.h +++ b/daemon/common.h @@ -3,7 +3,7 @@ #ifndef NETDATA_COMMON_H #define NETDATA_COMMON_H 1 -#include "../libnetdata/libnetdata.h" +#include "libnetdata/libnetdata.h" // ---------------------------------------------------------------------------- // shortcuts for the default netdata configuration @@ -44,6 +44,9 @@ // health monitoring and alarm notifications #include "health/health.h" +// anomaly detection +#include "ml/ml.h" + // the netdata registry // the registry is actually an API feature #include "registry/registry.h" @@ -66,11 +69,7 @@ #include "claim/claim.h" // netdata agent cloud link -#ifndef ACLK_NG -#include "aclk/legacy/agent_cloud_link.h" -#else -#include "aclk/aclk.h" -#endif +#include "aclk/aclk_api.h" // global GUID map functions @@ -81,6 +80,7 @@ #include "daemon.h" #include "main.h" #include "signals.h" +#include "service.h" #include "commands.h" #include "analytics.h" @@ -97,6 +97,8 @@ extern char *netdata_configured_lock_dir; extern char *netdata_configured_home_dir; extern char *netdata_configured_host_prefix; extern char *netdata_configured_timezone; +extern char *netdata_configured_abbrev_timezone; +extern int32_t netdata_configured_utc_offset; extern int netdata_zero_metrics_enabled; extern int netdata_anonymous_statistics_enabled; diff --git a/daemon/daemon.c b/daemon/daemon.c index 83191109a..68e161a3f 100644 --- a/daemon/daemon.c +++ b/daemon/daemon.c @@ -181,8 +181,10 @@ static void oom_score_adj(void) { return; } - if(old_score != 0) + if (old_score != 0) { wanted_score = old_score; + analytics_report_oom_score(old_score); + } // check the environment char *s = getenv("OOMScoreAdjust"); @@ -234,6 +236,7 @@ static void oom_score_adj(void) { info("Adjusted my Out-Of-Memory (OOM) score from %d to %d.", (int)old_score, (int)final_score); else error("Adjusted my Out-Of-Memory (OOM) score from %d to %d, but it has been set to %d.", (int)old_score, (int)wanted_score, (int)final_score); + analytics_report_oom_score(final_score); } else error("Failed to adjust my Out-Of-Memory (OOM) score to %d. Running with %d. (systemd systems may change it via netdata.service)", (int)wanted_score, (int)old_score); diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index edd261476..a152a00ae 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -157,7 +157,7 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) { uint64_t n = 0; - __atomic_compare_exchange(&global_statistics.web_usec_max, &gs->web_usec_max, &n, 1, __ATOMIC_SEQ_CST, + __atomic_compare_exchange(&global_statistics.web_usec_max, (uint64_t *) &gs->web_usec_max, &n, 1, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } #else diff --git a/daemon/main.c b/daemon/main.c index 61041f540..2ec5c33f9 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -28,7 +28,6 @@ void netdata_cleanup_and_exit(int ret) { info("EXIT: netdata prepares to exit with code %d...", ret); send_statistics("EXIT", ret?"ERROR":"OK","-"); - analytics_free_data(); char agent_crash_file[FILENAME_MAX + 1]; char agent_incomplete_shutdown_file[FILENAME_MAX + 1]; @@ -45,6 +44,9 @@ void netdata_cleanup_and_exit(int ret) { // stop everything info("EXIT: stopping static threads..."); +#ifdef ENABLE_NEW_CLOUD_PROTOCOL + aclk_sync_exit_all(); +#endif cancel_main_threads(); // free the database @@ -104,6 +106,7 @@ struct netdata_static_thread static_threads[] = { NETDATA_PLUGIN_HOOK_PLUGINSD NETDATA_PLUGIN_HOOK_HEALTH NETDATA_PLUGIN_HOOK_ANALYTICS + NETDATA_PLUGIN_HOOK_SERVICE {NULL, NULL, NULL, 0, NULL, NULL, NULL} }; @@ -360,13 +363,16 @@ int help(int exitcode) { " -W stacksize=N Set the stacksize (in bytes).\n\n" " -W debug_flags=N Set runtime tracing to debug.log.\n\n" " -W unittest Run internal unittests and exit.\n\n" + " -W sqlite-check Check metadata database integrity and exit.\n\n" + " -W sqlite-fix Check metadata database integrity, fix if needed and exit.\n\n" + " -W sqlite-compact Reclaim metadata database unused space and exit.\n\n" #ifdef ENABLE_DBENGINE " -W createdataset=N Create a DB engine dataset of N seconds and exit.\n\n" " -W stresstest=A,B,C,D,E,F\n" " Run a DB engine stress test for A seconds,\n" " with B writers and C readers, with a ramp up\n" " time of D seconds for writers, a page cache\n" - " size of E MiB, an optional disk space limit" + " size of E MiB, an optional disk space limit\n" " of F MiB and exit.\n\n" #endif " -W set section option value\n" @@ -388,20 +394,6 @@ int help(int exitcode) { return exitcode; } -// TODO: Remove this function with the nix major release. -void remove_option(int opt_index, int *argc, char **argv) { - int i; - - // remove the options. - do { - *argc = *argc - 1; - for(i = opt_index; i < *argc; i++) { - argv[i] = argv[i+1]; - } - i = opt_index; - } while(argv[i][0] != '-' && opt_index >= *argc); -} - #ifdef ENABLE_HTTPS static void security_init(){ char filename[FILENAME_MAX + 1]; @@ -556,7 +548,6 @@ static void get_netdata_configured_variables() { // get default memory mode for the database default_rrd_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_GLOBAL, "memory mode", rrd_memory_mode_name(default_rrd_memory_mode))); - #ifdef ENABLE_DBENGINE // ------------------------------------------------------------------------ // get default Database Engine page cache size in MiB @@ -581,7 +572,11 @@ static void get_netdata_configured_variables() { error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb); default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; } - +#else + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use memory mode ram instead."); + default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + } #endif // ------------------------------------------------------------------------ @@ -733,34 +728,6 @@ int main(int argc, char **argv) { // set the name for logging program_name = "netdata"; - // parse deprecated options - // TODO: Remove this block with the next major release. - { - i = 1; - while(i < argc) { - if(strcmp(argv[i], "-pidfile") == 0 && (i+1) < argc) { - strncpyz(pidfile, argv[i+1], FILENAME_MAX); - fprintf(stderr, "%s: deprecated option -- %s -- please use -P instead.\n", argv[0], argv[i]); - remove_option(i, &argc, argv); - } - else if(strcmp(argv[i], "-nodaemon") == 0 || strcmp(argv[i], "-nd") == 0) { - dont_fork = 1; - fprintf(stderr, "%s: deprecated option -- %s -- please use -D instead.\n ", argv[0], argv[i]); - remove_option(i, &argc, argv); - } - else if(strcmp(argv[i], "-ch") == 0 && (i+1) < argc) { - config_set(CONFIG_SECTION_GLOBAL, "host access prefix", argv[i+1]); - fprintf(stderr, "%s: deprecated option -- %s -- please use -s instead.\n", argv[0], argv[i]); - remove_option(i, &argc, argv); - } - else if(strcmp(argv[i], "-l") == 0 && (i+1) < argc) { - config_set(CONFIG_SECTION_GLOBAL, "history", argv[i+1]); - fprintf(stderr, "%s: deprecated option -- %s -- This option will be removed with V2.*.\n", argv[0], argv[i]); - remove_option(i, &argc, argv); - } - else i++; - } - } if (argc > 1 && strcmp(argv[1], SPAWN_SERVER_COMMAND_LINE_ARGUMENT) == 0) { // don't run netdata, this is the spawn server spawn_server(); @@ -840,6 +807,20 @@ int main(int argc, char **argv) { char* createdataset_string = "createdataset="; char* stresstest_string = "stresstest="; #endif + if(strcmp(optarg, "sqlite-check") == 0) { + sql_init_database(DB_CHECK_INTEGRITY); + return 0; + } + + if(strcmp(optarg, "sqlite-fix") == 0) { + sql_init_database(DB_CHECK_FIX_DB); + return 0; + } + + if(strcmp(optarg, "sqlite-compact") == 0) { + sql_init_database(DB_CHECK_RECLAIM_SPACE); + return 0; + } if(strcmp(optarg, "unittest") == 0) { if(unit_test_buffer()) return 1; @@ -861,9 +842,15 @@ int main(int argc, char **argv) { #ifdef ENABLE_DBENGINE if(test_dbengine()) return 1; #endif + if(test_sqlite()) return 1; fprintf(stderr, "\n\nALL TESTS PASSED\n\n"); return 0; } +#ifdef ENABLE_ML_TESTS + else if(strcmp(optarg, "mltest") == 0) { + return test_ml(argc, argv); + } +#endif #ifdef ENABLE_DBENGINE else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) { optarg += strlen(createdataset_string); @@ -1167,7 +1154,10 @@ int main(int argc, char **argv) { // get log filenames and settings log_init(); error_log_limit_unlimited(); + // initialize the log files + open_all_log_files(); + get_system_timezone(); // -------------------------------------------------------------------- // get the certificate and start security #ifdef ENABLE_HTTPS @@ -1179,6 +1169,10 @@ int main(int argc, char **argv) { set_silencers_filename(); health_initialize_global_silencers(); + // -------------------------------------------------------------------- + // Initialize ML configuration + ml_init(); + // -------------------------------------------------------------------- // setup process signals @@ -1217,9 +1211,6 @@ int main(int argc, char **argv) { api_listen_sockets_setup(); } - // initialize the log files - open_all_log_files(); - #ifdef NETDATA_INTERNAL_CHECKS if(debug_flags != 0) { struct rlimit rl = { RLIM_INFINITY, RLIM_INFINITY }; @@ -1269,6 +1260,7 @@ int main(int argc, char **argv) { netdata_anonymous_statistics_enabled=-1; struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info)); get_system_info(system_info); + system_info->hops = 0; if(rrd_init(netdata_configured_hostname, system_info)) fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname); @@ -1306,6 +1298,8 @@ int main(int argc, char **argv) { netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_GLOBAL, "enable zero metrics", CONFIG_BOOLEAN_NO); + set_late_global_environment(); + for (i = 0; static_threads[i].name != NULL ; i++) { struct netdata_static_thread *st = &static_threads[i]; @@ -1325,8 +1319,6 @@ int main(int argc, char **argv) { info("netdata initialization completed. Enjoy real-time performance monitoring!"); netdata_ready = 1; - set_late_global_environment(); - send_statistics("START", "-", "-"); if (crash_detected) send_statistics("CRASH", "-", "-"); diff --git a/daemon/service.c b/daemon/service.c new file mode 100644 index 000000000..9cba0694f --- /dev/null +++ b/daemon/service.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "common.h" + +/* Run service jobs every X seconds */ +#define SERVICE_HEARTBEAT 10 + +void service_main_cleanup(void *ptr) +{ + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + + debug(D_SYSTEM, "Cleaning up..."); + + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; +} + +/* + * The service thread. + */ +void *service_main(void *ptr) +{ + netdata_thread_cleanup_push(service_main_cleanup, ptr); + heartbeat_t hb; + heartbeat_init(&hb); + usec_t step = USEC_PER_SEC * SERVICE_HEARTBEAT; + + debug(D_SYSTEM, "Service thread starts"); + + while (!netdata_exit) { + heartbeat_next(&hb, step); + + rrd_cleanup_obsolete_charts(); + } + + netdata_thread_cleanup_pop(1); + return NULL; +} diff --git a/daemon/service.h b/daemon/service.h new file mode 100644 index 000000000..cb03bec2c --- /dev/null +++ b/daemon/service.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SERVICE_H +#define NETDATA_SERVICE_H 1 + +#define NETDATA_PLUGIN_HOOK_SERVICE \ + { \ + .name = "SERVICE", \ + .config_section = NULL, \ + .config_name = NULL, \ + .enabled = 1, \ + .thread = NULL, \ + .init_routine = NULL, \ + .start_routine = service_main \ + }, + +extern void *service_main(void *ptr); + +#endif //NETDATA_SERVICE_H diff --git a/daemon/system-info.sh b/daemon/system-info.sh index 05d8667c2..13c9b90bd 100755 --- a/daemon/system-info.sh +++ b/daemon/system-info.sh @@ -12,6 +12,7 @@ ARCHITECTURE="$(uname -m)" CONTAINER="unknown" CONT_DETECTION="none" +CONTAINER_IS_OFFICIAL_IMAGE="${NETDATA_OFFICIAL_IMAGE:-false}" if [ -z "${VIRTUALIZATION}" ]; then VIRTUALIZATION="unknown" @@ -381,6 +382,7 @@ echo "NETDATA_CONTAINER_OS_ID_LIKE=${CONTAINER_ID_LIKE}" echo "NETDATA_CONTAINER_OS_VERSION=${CONTAINER_VERSION}" echo "NETDATA_CONTAINER_OS_VERSION_ID=${CONTAINER_VERSION_ID}" echo "NETDATA_CONTAINER_OS_DETECTION=${CONTAINER_OS_DETECTION}" +echo "NETDATA_CONTAINER_IS_OFFICIAL_IMAGE=${CONTAINER_IS_OFFICIAL_IMAGE}" echo "NETDATA_HOST_OS_NAME=${HOST_NAME}" echo "NETDATA_HOST_OS_ID=${HOST_ID}" echo "NETDATA_HOST_OS_ID_LIKE=${HOST_ID_LIKE}" diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 81090736e..2dcc88c45 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -97,7 +97,7 @@ static int check_rrdcalc_comparisons(void) { int check_storage_number(calculated_number n, int debug) { char buffer[100]; - uint32_t flags = SN_EXISTS; + uint32_t flags = SN_DEFAULT_FLAGS; storage_number s = pack_storage_number(n, flags); calculated_number d = unpack_storage_number(s); @@ -150,7 +150,7 @@ calculated_number storage_number_min(calculated_number n) { do { last = n; n /= 2.0; - storage_number t = pack_storage_number(n, SN_EXISTS); + storage_number t = pack_storage_number(n, SN_DEFAULT_FLAGS); r = unpack_storage_number(t); } while(r != 0.0 && r != last); @@ -263,7 +263,7 @@ void benchmark_storage_number(int loop, int multiplier) { n *= multiplier; if(n > storage_number_positive_max) n = storage_number_positive_min; - s = pack_storage_number(n, SN_EXISTS); + s = pack_storage_number(n, SN_DEFAULT_FLAGS); d = unpack_storage_number(s); print_calculated_number(buffer, d); } @@ -289,25 +289,12 @@ void benchmark_storage_number(int loop, int multiplier) { } static int check_storage_number_exists() { - uint32_t flags; - - - for(flags = 0; flags < 7 ; flags++) { - if(get_storage_number_flags(flags << 24) != flags << 24) { - fprintf(stderr, "Flag 0x%08x is not checked correctly. It became 0x%08x\n", flags << 24, get_storage_number_flags(flags << 24)); - return 1; - } - } - - flags = SN_EXISTS; + uint32_t flags = SN_DEFAULT_FLAGS; calculated_number n = 0.0; storage_number s = pack_storage_number(n, flags); calculated_number d = unpack_storage_number(s); - if(get_storage_number_flags(s) != flags) { - fprintf(stderr, "Wrong flags. Given %08x, Got %08x!\n", flags, get_storage_number_flags(s)); - return 1; - } + if(n != d) { fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d); return 1; @@ -1192,7 +1179,7 @@ int run_test(struct test *test) unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries; for(c = 0 ; c < max ; c++) { calculated_number v = unpack_storage_number(rd->values[c]); - calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_EXISTS)); + calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS)); int same = (calculated_number_round(v * 10000000.0) == calculated_number_round(n * 10000000.0))?1:0; fprintf(stderr, " %s/%s: checking position %lu (at %lu secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n", test->name, rd->name, c+1, @@ -1475,6 +1462,38 @@ int unit_test(long delay, long shift) return ret; } +int test_sqlite(void) { + sqlite3 *db_meta; + fprintf(stderr, "Testing SQLIte\n"); + + int rc = sqlite3_open(":memory:", &db_meta); + if (rc != SQLITE_OK) { + fprintf(stderr,"Failed to test SQLite: DB init failed\n"); + return 1; + } + + rc = sqlite3_exec(db_meta, "CREATE TABLE IF NOT EXISTS mine (id1, id2);", 0, 0, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr,"Failed to test SQLite: Create table failed\n"); + return 1; + } + + rc = sqlite3_exec(db_meta, "DELETE FROM MINE LIMIT 1;", 0, 0, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr,"Failed to test SQLite: Delete with LIMIT failed\n"); + return 1; + } + + rc = sqlite3_exec(db_meta, "UPDATE MINE SET id1=1 LIMIT 1;", 0, 0, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr,"Failed to test SQLite: Update with LIMIT failed\n"); + return 1; + } + fprintf(stderr,"SQLite is OK\n"); + return 0; +} + + #ifdef ENABLE_DBENGINE static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number value, time_t now) { @@ -1500,6 +1519,8 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) , name , os_type , netdata_configured_timezone + , netdata_configured_abbrev_timezone + , netdata_configured_utc_offset , config_get(CONFIG_SECTION_BACKEND, "host tags", "") , program_name , program_version @@ -1583,7 +1604,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS collected_number next; update_every = REGION_UPDATE_EVERY[current_region]; - time_now = time_start + update_every; + time_now = time_start; // feed it with the test data for (i = 0 ; i < CHARTS ; ++i) { for (j = 0 ; j < DIMS ; ++j) { @@ -1594,7 +1615,7 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS } } for (c = 0; c < REGION_POINTS[current_region] ; ++c) { - time_now += update_every; // time_now = start + (c + 2) * update_every + time_now += update_every; // time_now = start + (c + 1) * update_every for (i = 0 ; i < CHARTS ; ++i) { st[i]->usec_since_last_update = USEC_PER_SEC * update_every; @@ -1626,14 +1647,14 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI // check the result for (c = 0; c < REGION_POINTS[current_region] ; c += QUERY_BATCH) { - time_now = time_start + (c + 2) * update_every; + time_now = time_start + (c + 1) * update_every; for (i = 0 ; i < CHARTS ; ++i) { for (j = 0; j < DIMS; ++j) { rd[i][j]->state->query_ops.init(rd[i][j], &handle, time_now, time_now + QUERY_BATCH * update_every); for (k = 0; k < QUERY_BATCH; ++k) { last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c + k; - expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); + expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); n = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved); value = unpack_storage_number(n); @@ -1671,7 +1692,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] errors = 0; update_every = REGION_UPDATE_EVERY[current_region]; - long points = (time_end - time_start) / update_every - 1; + long points = (time_end - time_start) / update_every; for (i = 0 ; i < CHARTS ; ++i) { RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL); if (!r) { @@ -1690,8 +1711,8 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] value = cn[j]; assert(rd[i][j] == d); - last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c; - expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); + last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c + 1; + expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if(!same) { @@ -1789,7 +1810,7 @@ int test_dbengine(void) current_region = 1; update_every = REGION_UPDATE_EVERY[current_region]; // use the maximum update_every = 3 errors = 0; - long points = (time_end[REGIONS - 1] - time_start[0]) / update_every - 1; // cover all time regions with RRDR + long points = (time_end[REGIONS - 1] - time_start[0]) / update_every; // cover all time regions with RRDR long point_offset = (time_start[current_region] - time_start[0]) / update_every; for (i = 0 ; i < CHARTS ; ++i) { RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL); @@ -1812,8 +1833,8 @@ int test_dbengine(void) calculated_number value = cn[j]; assert(rd[i][j] == d); - collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset; - calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); + collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset + 1; + calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); uint8_t same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if(!same) { @@ -2042,7 +2063,7 @@ static void query_dbengine_chart(void *arg) ++thread_info->queries_nr; for (time_now = time_after ; time_now <= time_before ; time_now += update_every) { generatedv = generate_dbengine_chart_value(i, j, time_now); - expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_EXISTS)); + expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_DEFAULT_FLAGS)); if (unlikely(rd->state->query_ops.is_finished(&handle))) { if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ diff --git a/daemon/unit_test.h b/daemon/unit_test.h index 79d415be0..3a3b64902 100644 --- a/daemon/unit_test.h +++ b/daemon/unit_test.h @@ -8,6 +8,7 @@ extern int unit_test(long delay, long shift); extern int run_all_mockup_tests(void); extern int unit_test_str2ld(void); extern int unit_test_buffer(void); +extern int test_sqlite(void); #ifdef ENABLE_DBENGINE extern int test_dbengine(void); extern void generate_dbengine_dataset(unsigned history_seconds); -- cgit v1.2.3