Merging upstream version 1.40.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

8 files changed, 96 insertions, 88 deletions

diff --git a/daemon/analytics.c b/daemon/analytics.c
index b3c802b86..2689886bd 100644
--- a/daemon/analytics.c
+++ b/daemon/analytics.c
@@ -375,8 +375,12 @@ void analytics_https(void)
     BUFFER *b = buffer_create(30, NULL);
 #ifdef ENABLE_HTTPS
     analytics_exporting_connectors_ssl(b);
-    buffer_strcat(b, netdata_ssl_client_ctx && rrdhost_flag_check(localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) && localhost->sender->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ? "streaming|" : "|");
-    buffer_strcat(b, netdata_ssl_srv_ctx ? "web" : "");
+
+    buffer_strcat(b, netdata_ssl_streaming_sender_ctx &&
+                     rrdhost_flag_check(localhost, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED) &&
+                     SSL_connection(&localhost->sender->ssl) ? "streaming|" : "|");
+
+    buffer_strcat(b, netdata_ssl_web_server_ctx ? "web" : "");
 #else
     buffer_strcat(b, "||");
 #endif
diff --git a/daemon/anonymous-statistics.sh.in b/daemon/anonymous-statistics.sh.in
index 8676ffbe7..32cbc71db 100755
--- a/daemon/anonymous-statistics.sh.in
+++ b/daemon/anonymous-statistics.sh.in
@@ -74,7 +74,6 @@ NETDATA_PREBUILT_DISTRO="${42}"
 # define body of request to be sent
 REQ_BODY="$(cat << EOF
 {
-    "api_key": "mqkwGT0JNFqO-zX2t0mW6Tec9yooaVu7xCBlXtHnt5Y",
     "event": "${ACTION} ${ACTION_RESULT}",
     "properties": {
         "distinct_id": "${NETDATA_REGISTRY_UNIQUE_ID}",
@@ -164,9 +163,9 @@ REQ_BODY="$(cat << EOF
 EOF
 )"
 
-# send the anonymous statistics to the Netdata PostHog
+# send the anonymous statistics to Netdata
 if [ -n "$(command -v curl 2> /dev/null)" ]; then
-  curl --silent -o /dev/null --write-out '%{http_code}' -X POST --max-time 2 --header "Content-Type: application/json" -d "${REQ_BODY}" https://app.posthog.com/capture/
+  curl --silent -o /dev/null --write-out '%{http_code}' -X POST --max-time 2 --header "Content-Type: application/json" -d "${REQ_BODY}" https://us-east1-netdata-analytics-bi.cloudfunctions.net/ingest_agent_events
 else
   wget -q -O - --no-check-certificate \
   --server-response \
@@ -174,5 +173,5 @@ else
   --timeout=1 \
   --header 'Content-Type: application/json' \
   --body-data "${REQ_BODY}" \
-   'https://app.posthog.com/capture/' 2>&1 | awk '/^  HTTP/{print $2}'
+   'https://us-east1-netdata-analytics-bi.cloudfunctions.net/ingest_agent_events' 2>&1 | awk '/^  HTTP/{print $2}'
 fi
diff --git a/daemon/buildinfo.c b/daemon/buildinfo.c
index ef813a961..d277d2b3c 100644
--- a/daemon/buildinfo.c
+++ b/daemon/buildinfo.c
@@ -20,6 +20,12 @@
 #endif
 #endif
 
+#ifdef ENABLE_HTTPD
+#define FEAT_HTTPD 1
+#else
+#define FEAT_HTTPD 0
+#endif
+
 #ifdef ENABLE_DBENGINE
 #define FEAT_DBENGINE 1
 #else
@@ -89,12 +95,6 @@
 #define FEAT_LIBCAP 0
 #endif
 
-#ifdef NETDATA_WITH_ZLIB
-#define FEAT_ZLIB 1
-#else
-#define FEAT_ZLIB 0
-#endif
-
 #ifdef STORAGE_WITH_MATH
 #define FEAT_LIBM 1
 #else
@@ -115,6 +115,12 @@
 #define FEAT_APPS_PLUGIN 0
 #endif
 
+#ifdef ENABLE_DEBUGFS_PLUGIN
+#define FEAT_DEBUGFS_PLUGIN 1
+#else
+#define FEAT_DEBUGFS_PLUGIN 0
+#endif
+
 #ifdef HAVE_FREEIPMI
 #define FEAT_IPMI 1
 #else
@@ -275,6 +281,7 @@ void print_build_info(void) {
     printf("    TLS Host Verification:      %s\n", FEAT_YES_NO(FEAT_TLS_HOST_VERIFY));
     printf("    Machine Learning:           %s\n", FEAT_YES_NO(FEAT_ML));
     printf("    Stream Compression:         %s\n", FEAT_YES_NO(FEAT_STREAM_COMPRESSION));
+    printf("    HTTPD (h2o):                %s\n", FEAT_YES_NO(FEAT_HTTPD));
 
     printf("Libraries:\n");
     printf("    protobuf:                %s%s\n", FEAT_YES_NO(FEAT_PROTOBUF), FEAT_PROTOBUF_BUNDLED);
@@ -284,12 +291,13 @@ void print_build_info(void) {
     printf("    libcrypto:               %s\n", FEAT_YES_NO(FEAT_CRYPTO));
     printf("    libm:                    %s\n", FEAT_YES_NO(FEAT_LIBM));
     printf("    tcalloc:                 %s\n", FEAT_YES_NO(FEAT_TCMALLOC));
-    printf("    zlib:                    %s\n", FEAT_YES_NO(FEAT_ZLIB));
+    printf("    zlib:                    %s\n", FEAT_YES_NO(1));
 
     printf("Plugins:\n");
     printf("    apps:                    %s\n", FEAT_YES_NO(FEAT_APPS_PLUGIN));
     printf("    cgroup Network Tracking: %s\n", FEAT_YES_NO(FEAT_CGROUP_NET));
     printf("    CUPS:                    %s\n", FEAT_YES_NO(FEAT_CUPS));
+    printf("    debugfs:                 %s\n", FEAT_YES_NO(FEAT_DEBUGFS_PLUGIN));
     printf("    EBPF:                    %s\n", FEAT_YES_NO(FEAT_EBPF));
     printf("    IPMI:                    %s\n", FEAT_YES_NO(FEAT_IPMI));
     printf("    NFACCT:                  %s\n", FEAT_YES_NO(FEAT_NFACCT));
@@ -328,7 +336,8 @@ void print_build_info_json(void) {
 
     printf("    \"tls-host-verify\": %s,\n",   FEAT_JSON_BOOL(FEAT_TLS_HOST_VERIFY));
     printf("    \"machine-learning\": %s\n",   FEAT_JSON_BOOL(FEAT_ML));
-    printf("    \"stream-compression\": %s\n",   FEAT_JSON_BOOL(FEAT_STREAM_COMPRESSION));
+    printf("    \"stream-compression\": %s\n", FEAT_JSON_BOOL(FEAT_STREAM_COMPRESSION));
+    printf("    \"httpd-h2o\": %s\n",          FEAT_JSON_BOOL(FEAT_HTTPD));
     printf("  },\n");
 
     printf("  \"libs\": {\n");
@@ -340,13 +349,14 @@ void print_build_info_json(void) {
     printf("    \"libcrypto\": %s,\n",        FEAT_JSON_BOOL(FEAT_CRYPTO));
     printf("    \"libm\": %s,\n",             FEAT_JSON_BOOL(FEAT_LIBM));
     printf("    \"tcmalloc\": %s,\n",         FEAT_JSON_BOOL(FEAT_TCMALLOC));
-    printf("    \"zlib\": %s\n",              FEAT_JSON_BOOL(FEAT_ZLIB));
+    printf("    \"zlib\": %s\n",              FEAT_JSON_BOOL(1));
     printf("  },\n");
 
     printf("  \"plugins\": {\n");
     printf("    \"apps\": %s,\n",             FEAT_JSON_BOOL(FEAT_APPS_PLUGIN));
     printf("    \"cgroup-net\": %s,\n",       FEAT_JSON_BOOL(FEAT_CGROUP_NET));
     printf("    \"cups\": %s,\n",             FEAT_JSON_BOOL(FEAT_CUPS));
+    printf("    \"debugfs\": %s,\n",          FEAT_JSON_BOOL(FEAT_DEBUGFS_PLUGIN));
     printf("    \"ebpf\": %s,\n",             FEAT_JSON_BOOL(FEAT_EBPF));
     printf("    \"ipmi\": %s,\n",             FEAT_JSON_BOOL(FEAT_IPMI));
     printf("    \"nfacct\": %s,\n",           FEAT_JSON_BOOL(FEAT_NFACCT));
@@ -417,13 +427,14 @@ void analytics_build_info(BUFFER *b) {
 #ifdef ENABLE_TCMALLOC
     add_to_bi(b, "tcalloc");
 #endif
-#ifdef NETDATA_WITH_ZLIB
     add_to_bi(b, "zlib");
-#endif
 
 #ifdef ENABLE_APPS_PLUGIN
     add_to_bi(b, "apps");
 #endif
+#ifdef ENABLE_DEBUGFS_PLUGIN
+    add_to_bi(b, "debugfs");
+#endif
 #ifdef HAVE_SETNS
     add_to_bi(b, "cgroup Network Tracking");
 #endif
diff --git a/daemon/common.h b/daemon/common.h
index 66ffd4a74..aeaf01637 100644
--- a/daemon/common.h
+++ b/daemon/common.h
@@ -41,6 +41,11 @@
 // the netdata webserver(s)
 #include "web/server/web_server.h"
 
+// the new h2o based netdata webserver
+#ifdef ENABLE_HTTPD
+#include "httpd/http_server.h"
+#endif
+
 // streaming metrics between netdata servers
 #include "streaming/rrdpush.h"
 
diff --git a/daemon/main.c b/daemon/main.c
index 606de128b..cff6530f3 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -170,6 +170,8 @@ static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) {
         buffer_strcat(wb, "ANALYTICS ");
     if(service & SERVICE_EXPORTERS)
         buffer_strcat(wb, "EXPORTERS ");
+    if(service & SERVICE_HTTPD)
+        buffer_strcat(wb, "HTTPD ");
 }
 
 static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
@@ -365,6 +367,7 @@ void netdata_cleanup_and_exit(int ret) {
             | SERVICE_EXPORTERS
             | SERVICE_HEALTH
             | SERVICE_WEB_SERVER
+            | SERVICE_HTTPD
             , 3 * USEC_PER_SEC);
 
     delta_shutdown_time("stop collectors and streaming threads");
@@ -479,7 +482,7 @@ void netdata_cleanup_and_exit(int ret) {
 
 #ifdef ENABLE_HTTPS
     delta_shutdown_time("free openssl structures");
-    security_clean_openssl();
+    netdata_ssl_cleanup();
 #endif
 
     delta_shutdown_time("remove incomplete shutdown file");
@@ -573,8 +576,6 @@ void web_server_config_options(void)
     web_allow_mgmt_dns         =
         make_dns_decision(CONFIG_SECTION_WEB, "allow management by dns","heuristic",web_allow_mgmt_from);
 
-
-#ifdef NETDATA_WITH_ZLIB
     web_enable_gzip = config_get_boolean(CONFIG_SECTION_WEB, "enable gzip compression", web_enable_gzip);
 
     char *s = config_get(CONFIG_SECTION_WEB, "gzip compression strategy", "default");
@@ -602,7 +603,6 @@ void web_server_config_options(void)
         error("Invalid compression level %d. Valid levels are 1 (fastest) to 9 (best ratio). Proceeding with level 9 (best compression).", web_gzip_level);
         web_gzip_level = 9;
     }
-#endif /* NETDATA_WITH_ZLIB */
 }
 
 
@@ -834,7 +834,7 @@ static void security_init(){
     tls_version    = config_get(CONFIG_SECTION_WEB, "tls version",  "1.3");
     tls_ciphers    = config_get(CONFIG_SECTION_WEB, "tls ciphers",  "none");
 
-    security_openssl_library();
+    netdata_ssl_initialize_openssl();
 }
 #endif
 
@@ -1979,6 +1979,14 @@ int main(int argc, char **argv) {
 
         if(web_server_mode != WEB_SERVER_MODE_NONE)
             api_listen_sockets_setup();
+
+#ifdef ENABLE_HTTPD
+        delta_startup_time("initialize httpd server");
+        for (int i = 0; static_threads[i].name; i++) {
+            if (static_threads[i].start_routine == httpd_main)
+                static_threads[i].enabled = httpd_is_enabled();
+        }
+#endif
     }
 
     delta_startup_time("set resource limits");
diff --git a/daemon/main.h b/daemon/main.h
index 3e32c5ad6..232b7a98a 100644
--- a/daemon/main.h
+++ b/daemon/main.h
@@ -41,7 +41,8 @@ typedef enum {
     SERVICE_CONTEXT               = (1 << 10),
     SERVICE_ANALYTICS             = (1 << 11),
     SERVICE_EXPORTERS             = (1 << 12),
-    SERVICE_ACLKSYNC              = (1 << 13)
+    SERVICE_ACLKSYNC              = (1 << 13),
+    SERVICE_HTTPD                 = (1 << 14)
 } SERVICE_TYPE;
 
 typedef enum {
diff --git a/daemon/signals.c b/daemon/signals.c
index c857a9b57..3699010ce 100644
--- a/daemon/signals.c
+++ b/daemon/signals.c
@@ -2,8 +2,6 @@
 
 #include "common.h"
 
-static int reaper_enabled = 0;
-
 typedef enum signal_action {
     NETDATA_SIGNAL_END_OF_LIST,
     NETDATA_SIGNAL_IGNORE,
@@ -78,16 +76,6 @@ void signals_init(void) {
     struct sigaction sa;
     sa.sa_flags = 0;
 
-    // Enable process tracking / reaper if running as init (pid == 1).
-    // This prevents zombie processes when running in a container.
-    if (getpid() == 1) {
-        info("SIGNAL: Enabling reaper");
-        netdata_popen_tracking_init();
-        reaper_enabled = 1;
-    } else {
-        info("SIGNAL: Not enabling reaper");
-    }
-
     // ignore all signals while we run in a signal handler
     sigfillset(&sa.sa_mask);
 
@@ -97,10 +85,6 @@ void signals_init(void) {
         case NETDATA_SIGNAL_IGNORE:
             sa.sa_handler = SIG_IGN;
             break;
-        case NETDATA_SIGNAL_CHILD:
-            if (reaper_enabled == 0)
-                continue;
-            // FALLTHROUGH
         default:
             sa.sa_handler = signal_handler;
             break;
@@ -115,9 +99,6 @@ void signals_restore_SIGCHLD(void)
 {
     struct sigaction sa;
 
-    if (reaper_enabled == 0)
-        return;
-
     sa.sa_flags = 0;
     sigfillset(&sa.sa_mask);
     sa.sa_handler = signal_handler;
@@ -137,9 +118,6 @@ void signals_reset(void) {
         if(sigaction(signals_waiting[i].signo, &sa, NULL) == -1)
             error("SIGNAL: Failed to reset signal handler for: %s", signals_waiting[i].name);
     }
-
-    if (reaper_enabled == 1)
-        netdata_popen_tracking_cleanup();
 }
 
 // reap_child reaps the child identified by pid.
@@ -147,39 +125,42 @@ static void reap_child(pid_t pid) {
     siginfo_t i;
 
     errno = 0;
-    debug(D_CHILDS, "SIGNAL: Reaping pid: %d...", pid);
-    if (waitid(P_PID, (id_t)pid, &i, WEXITED|WNOHANG) == -1) {
+    debug(D_CHILDS, "SIGNAL: reap_child(%d)...", pid);
+    if (netdata_waitid(P_PID, (id_t)pid, &i, WEXITED|WNOHANG) == -1) {
         if (errno != ECHILD)
-            error("SIGNAL: Failed to wait for: %d", pid);
+            error("SIGNAL: waitid(%d): failed to wait for child", pid);
         else
-            debug(D_CHILDS, "SIGNAL: Already reaped: %d", pid);
+            info("SIGNAL: waitid(%d): failed - it seems the child is already reaped", pid);
         return;
-    } else if (i.si_pid == 0) {
+    }
+    else if (i.si_pid == 0) {
         // Process didn't exit, this shouldn't happen.
+        error("SIGNAL: waitid(%d): reports pid 0 - child has not exited", pid);
         return;
     }
 
     switch (i.si_code) {
-    case CLD_EXITED:
-        debug(D_CHILDS, "SIGNAL: Child %d exited: %d", pid, i.si_status);
-        break;
-    case CLD_KILLED:
-        debug(D_CHILDS, "SIGNAL: Child %d killed by signal: %d", pid, i.si_status);
-        break;
-    case CLD_DUMPED:
-        debug(D_CHILDS, "SIGNAL: Child %d dumped core by signal: %d", pid, i.si_status);
-        break;
-    case CLD_STOPPED:
-        debug(D_CHILDS, "SIGNAL: Child %d stopped by signal: %d", pid, i.si_status);
-        break;
-    case CLD_TRAPPED:
-        debug(D_CHILDS, "SIGNAL: Child %d trapped by signal: %d", pid, i.si_status);
-        break;
-    case CLD_CONTINUED:
-        debug(D_CHILDS, "SIGNAL: Child %d continued by signal: %d", pid, i.si_status);
-        break;
-    default:
-        debug(D_CHILDS, "SIGNAL: Child %d gave us a SIGCHLD with code %d and status %d.", pid, i.si_code, i.si_status);
+        case CLD_EXITED:
+            info("SIGNAL: reap_child(%d) exited with code: %d", pid, i.si_status);
+            break;
+        case CLD_KILLED:
+            info("SIGNAL: reap_child(%d) killed by signal: %d", pid, i.si_status);
+            break;
+        case CLD_DUMPED:
+            info("SIGNAL: reap_child(%d) dumped core by signal: %d", pid, i.si_status);
+            break;
+        case CLD_STOPPED:
+            info("SIGNAL: reap_child(%d) stopped by signal: %d", pid, i.si_status);
+            break;
+        case CLD_TRAPPED:
+            info("SIGNAL: reap_child(%d) trapped by signal: %d", pid, i.si_status);
+            break;
+        case CLD_CONTINUED:
+            info("SIGNAL: reap_child(%d) continued by signal: %d", pid, i.si_status);
+            break;
+        default:
+            info("SIGNAL: reap_child(%d) gave us a SIGCHLD with code %d and status %d.", pid, i.si_code, i.si_status);
+            break;
     }
 }
 
@@ -187,25 +168,13 @@ static void reap_child(pid_t pid) {
 static void reap_children() {
     siginfo_t i;
 
-    while (1 == 1) {
-        // Identify which process caused the signal so we can determine
-        // if we need to reap a re-parented process.
+    while(1) {
         i.si_pid = 0;
-        if (waitid(P_ALL, (id_t)0, &i, WEXITED|WNOHANG|WNOWAIT) == -1) {
-            if (errno != ECHILD) // This shouldn't happen with WNOHANG but does.
-                error("SIGNAL: Failed to wait");
-            return;
-        } else if (i.si_pid == 0) {
-            // No child exited.
+        if (netdata_waitid(P_ALL, (id_t)0, &i, WEXITED|WNOHANG|WNOWAIT) == -1 || i.si_pid == 0)
+            // nothing to do
             return;
-        } else if (netdata_popen_tracking_pid_shoud_be_reaped(i.si_pid) == 0) {
-            // myp managed, sleep for a short time to avoid busy wait while
-            // this is handled by myp.
-            usleep(10000);
-        } else {
-            // Unknown process, likely a re-parented child, reap it.
-            reap_child(i.si_pid);
-        }
+
+        reap_child(i.si_pid);
     }
 }
 
@@ -267,7 +236,6 @@ void signals_handle(void) {
                                 break;
 
                             case NETDATA_SIGNAL_CHILD:
-                                debug(D_CHILDS, "SIGNAL: Received %s. Reaping...", name);
                                 reap_children();
                                 break;
 
diff --git a/daemon/static_threads.c b/daemon/static_threads.c
index d93cfe9d0..fe83945cf 100644
--- a/daemon/static_threads.c
+++ b/daemon/static_threads.c
@@ -142,6 +142,18 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = socket_listen_main_static_threaded
     },
 
+#ifdef ENABLE_HTTPD
+    {
+        .name = "httpd",
+        .config_section = NULL,
+        .config_name = NULL,
+        .enabled = 0,
+        .thread = NULL,
+        .init_routine = NULL,
+        .start_routine = httpd_main
+    },
+#endif
+
 #ifdef ENABLE_ACLK
     {
         .name = "ACLK_MAIN",