Merging upstream version 1.41.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 337 insertions, 49 deletions

diff --git a/claim/README.md b/claim/README.md
index d88167ad6..fed37018c 100644
--- a/claim/README.md
+++ b/claim/README.md
@@ -22,7 +22,7 @@ identity of the Netdata Agent when it connects to the Cloud. While the data does
 from Agents to the browser, we do not store or log it.
 
 You can connect a node during the Netdata Cloud onboarding process, or after you created a Space by clicking on **Connect
-Nodes** in the [Spaces management area](https://github.com/netdata/netdata/blob/master/docs/cloud/spaces.md#manage-spaces).
+Nodes** in the [Spaces management area](https://github.com/netdata/netdata/blob/master/docs/cloud/manage/organize-your-infrastrucutre-invite-your-team.md#netdata-cloud-spaces#manage-spaces).
 
 There are two important notes regarding connecting nodes:
 
diff --git a/claim/claim.c b/claim/claim.c
index 9fe156d21..a30fd0dab 100644
--- a/claim/claim.c
+++ b/claim/claim.c
@@ -42,16 +42,16 @@ char *get_agent_claimid()
 }
 
 #define CLAIMING_COMMAND_LENGTH 16384
-#define CLAIMING_PROXY_LENGTH CLAIMING_COMMAND_LENGTH/4
+#define CLAIMING_PROXY_LENGTH (CLAIMING_COMMAND_LENGTH/4)
 
 extern struct registry registry;
 
 /* rrd_init() and post_conf_load() must have been called before this function */
-void claim_agent(char *claiming_arguments)
+CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, const char **msg)
 {
-    if (!netdata_cloud_setting) {
-        error("Refusing to claim agent -> cloud functionality has been disabled");
-        return;
+    if (!force || !netdata_cloud_enabled) {
+        netdata_log_error("Refusing to claim agent -> cloud functionality has been disabled");
+        return CLAIM_AGENT_CLOUD_DISABLED;
     }
 
 #ifndef DISABLE_CLOUD
@@ -62,8 +62,11 @@ void claim_agent(char *claiming_arguments)
 
     // This is guaranteed to be set early in main via post_conf_load()
     char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);
-    if (cloud_base_url == NULL)
-        fatal("Do not move the cloud base url out of post_conf_load!!");
+    if (cloud_base_url == NULL) {
+        internal_fatal(true, "Do not move the cloud base url out of post_conf_load!!");
+        return CLAIM_AGENT_NO_CLOUD_URL;
+    }
+
     const char *proxy_str;
     ACLK_PROXY_TYPE proxy_type;
     char proxy_flag[CLAIMING_PROXY_LENGTH] = "-noproxy";
@@ -76,44 +79,49 @@ void claim_agent(char *claiming_arguments)
     snprintfz(command_buffer,
               CLAIMING_COMMAND_LENGTH,
               "exec netdata-claim.sh %s -hostname=%s -id=%s -url=%s -noreload %s",
-
               proxy_flag,
               netdata_configured_hostname,
               localhost->machine_guid,
               cloud_base_url,
               claiming_arguments);
 
-    info("Executing agent claiming command 'netdata-claim.sh'");
+    netdata_log_info("Executing agent claiming command 'netdata-claim.sh'");
     fp_child_output = netdata_popen(command_buffer, &command_pid, &fp_child_input);
     if(!fp_child_output) {
-        error("Cannot popen(\"%s\").", command_buffer);
-        return;
+        netdata_log_error("Cannot popen(\"%s\").", command_buffer);
+        return CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT;
     }
-    info("Waiting for claiming command to finish.");
+    netdata_log_info("Waiting for claiming command to finish.");
     while (fgets(command_buffer, CLAIMING_COMMAND_LENGTH, fp_child_output) != NULL) {;}
     exit_code = netdata_pclose(fp_child_input, fp_child_output, command_pid);
-    info("Agent claiming command returned with code %d", exit_code);
+    netdata_log_info("Agent claiming command returned with code %d", exit_code);
     if (0 == exit_code) {
         load_claiming_state();
-        return;
+        return CLAIM_AGENT_OK;
     }
     if (exit_code < 0) {
-        error("Agent claiming command failed to complete its run.");
-        return;
+        netdata_log_error("Agent claiming command failed to complete its run.");
+        return CLAIM_AGENT_CLAIM_SCRIPT_FAILED;
     }
     errno = 0;
     unsigned maximum_known_exit_code = sizeof(claiming_errors) / sizeof(claiming_errors[0]) - 1;
 
     if ((unsigned)exit_code > maximum_known_exit_code) {
-        error("Agent failed to be claimed with an unknown error.");
-        return;
+        netdata_log_error("Agent failed to be claimed with an unknown error.");
+        return CLAIM_AGENT_CLAIM_SCRIPT_RETURNED_INVALID_CODE;
     }
-    error("Agent failed to be claimed with the following error message:");
-    error("\"%s\"", claiming_errors[exit_code]);
+
+    netdata_log_error("Agent failed to be claimed with the following error message:");
+    netdata_log_error("\"%s\"", claiming_errors[exit_code]);
+
+    if(msg) *msg = claiming_errors[exit_code];
+
 #else
     UNUSED(claiming_arguments);
     UNUSED(claiming_errors);
 #endif
+
+    return CLAIM_AGENT_FAILED_WITH_MESSAGE;
 }
 
 #ifdef ENABLE_ACLK
@@ -132,7 +140,7 @@ void load_claiming_state(void)
     // --------------------------------------------------------------------
     // Check if the cloud is enabled
 #if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )
-    netdata_cloud_setting = 0;
+    netdata_cloud_enabled = false;
 #else
     uuid_t uuid;
 
@@ -148,7 +156,7 @@ void load_claiming_state(void)
     }
     if (aclk_connected)
     {
-        info("Agent was already connected to Cloud - forcing reconnection under new credentials");
+        netdata_log_info("Agent was already connected to Cloud - forcing reconnection under new credentials");
         aclk_kill_link = 1;
     }
     aclk_disable_runtime = 0;
@@ -159,7 +167,7 @@ void load_claiming_state(void)
     long bytes_read;
     char *claimed_id = read_by_filename(filename, &bytes_read);
     if(claimed_id && uuid_parse(claimed_id, uuid)) {
-        error("claimed_id \"%s\" doesn't look like valid UUID", claimed_id);
+        netdata_log_error("claimed_id \"%s\" doesn't look like valid UUID", claimed_id);
         freez(claimed_id);
         claimed_id = NULL;
     }
@@ -174,14 +182,14 @@ void load_claiming_state(void)
     rrdhost_aclk_state_unlock(localhost);
 
     if (!claimed_id) {
-        info("Unable to load '%s', setting state to AGENT_UNCLAIMED", filename);
+        netdata_log_info("Unable to load '%s', setting state to AGENT_UNCLAIMED", filename);
         return;
     }
 
     freez(claimed_id);
 
-    info("File '%s' was found. Setting state to AGENT_CLAIMED.", filename);
-    netdata_cloud_setting = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1);
+    netdata_log_info("File '%s' was found. Setting state to AGENT_CLAIMED.", filename);
+    netdata_cloud_enabled = appconfig_get_boolean_ondemand(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", netdata_cloud_enabled);
 #endif
 }
 
@@ -193,6 +201,10 @@ struct config cloud_config = { .first_section = NULL,
 
 void load_cloud_conf(int silent)
 {
+    char *nd_disable_cloud = getenv("NETDATA_DISABLE_CLOUD");
+    if (nd_disable_cloud && !strncmp(nd_disable_cloud, "1", 1))
+        netdata_cloud_enabled = CONFIG_BOOLEAN_NO;
+
     char *filename;
     errno = 0;
 
@@ -201,8 +213,255 @@ void load_cloud_conf(int silent)
     filename = strdupz_path_subpath(netdata_configured_varlib_dir, "cloud.d/cloud.conf");
 
     ret = appconfig_load(&cloud_config, filename, 1, NULL);
-    if(!ret && !silent) {
-        info("CONFIG: cannot load cloud config '%s'. Running with internal defaults.", filename);
-    }
+    if(!ret && !silent)
+        netdata_log_info("CONFIG: cannot load cloud config '%s'. Running with internal defaults.", filename);
+
     freez(filename);
+
+    // --------------------------------------------------------------------
+    // Check if the cloud is enabled
+
+#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )
+    netdata_cloud_enabled = CONFIG_BOOLEAN_NO;
+#else
+    netdata_cloud_enabled = appconfig_get_boolean_ondemand(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", netdata_cloud_enabled);
+#endif
+
+    // This must be set before any point in the code that accesses it. Do not move it from this function.
+    appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL);
+}
+
+static char *netdata_random_session_id_filename = NULL;
+static uuid_t netdata_random_session_id = { 0 };
+
+bool netdata_random_session_id_generate(void) {
+    static char guid[UUID_STR_LEN] = "";
+
+    uuid_generate_random(netdata_random_session_id);
+    uuid_unparse_lower(netdata_random_session_id, guid);
+
+    char filename[FILENAME_MAX + 1];
+    snprintfz(filename, FILENAME_MAX, "%s/netdata_random_session_id", netdata_configured_varlib_dir);
+
+    bool ret = true;
+
+    (void)unlink(filename);
+
+    // save it
+    int fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 640);
+    if(fd == -1) {
+        netdata_log_error("Cannot create random session id file '%s'.", filename);
+        ret = false;
+    }
+    else {
+        if (write(fd, guid, UUID_STR_LEN - 1) != UUID_STR_LEN - 1) {
+            netdata_log_error("Cannot write the random session id file '%s'.", filename);
+            ret = false;
+        } else {
+            ssize_t bytes = write(fd, "\n", 1);
+            UNUSED(bytes);
+        }
+        close(fd);
+    }
+
+    if(ret && (!netdata_random_session_id_filename || strcmp(netdata_random_session_id_filename, filename) != 0)) {
+        freez(netdata_random_session_id_filename);
+        netdata_random_session_id_filename = strdupz(filename);
+    }
+
+    return ret;
+}
+
+const char *netdata_random_session_id_get_filename(void) {
+    if(!netdata_random_session_id_filename)
+        netdata_random_session_id_generate();
+
+    return netdata_random_session_id_filename;
+}
+
+bool netdata_random_session_id_matches(const char *guid) {
+    if(uuid_is_null(netdata_random_session_id))
+        return false;
+
+    uuid_t uuid;
+
+    if(uuid_parse(guid, uuid))
+        return false;
+
+    if(uuid_compare(netdata_random_session_id, uuid) == 0)
+        return true;
+
+    return false;
+}
+
+static bool check_claim_param(const char *s) {
+    if(!s || !*s) return true;
+
+    do {
+        if(isalnum(*s) || *s == '.' || *s == ',' || *s == '-' || *s == ':' || *s == '/' || *s == '_')
+            ;
+        else
+            return false;
+
+    } while(*++s);
+
+    return true;
+}
+
+void claim_reload_all(void) {
+    error_log_limit_unlimited();
+    load_claiming_state();
+    registry_update_cloud_base_url();
+    rrdpush_send_claimed_id(localhost);
+    error_log_limit_reset();
+}
+
+int api_v2_claim(struct web_client *w, char *url) {
+    char *key = NULL;
+    char *token = NULL;
+    char *rooms = NULL;
+    char *base_url = NULL;
+
+    while (url) {
+        char *value = strsep_skip_consecutive_separators(&url, "&");
+        if (!value || !*value) continue;
+
+        char *name = strsep_skip_consecutive_separators(&value, "=");
+        if (!name || !*name) continue;
+        if (!value || !*value) continue;
+
+        if(!strcmp(name, "key"))
+            key = value;
+        else if(!strcmp(name, "token"))
+            token = value;
+        else if(!strcmp(name, "rooms"))
+            rooms = value;
+        else if(!strcmp(name, "url"))
+            base_url = value;
+    }
+
+    BUFFER *wb = w->response.data;
+    buffer_flush(wb);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+
+    time_t now_s = now_realtime_sec();
+    CLOUD_STATUS status = buffer_json_cloud_status(wb, now_s);
+
+    bool can_be_claimed = false;
+    switch(status) {
+        case CLOUD_STATUS_AVAILABLE:
+        case CLOUD_STATUS_DISABLED:
+        case CLOUD_STATUS_OFFLINE:
+            can_be_claimed = true;
+            break;
+
+        case CLOUD_STATUS_UNAVAILABLE:
+        case CLOUD_STATUS_BANNED:
+        case CLOUD_STATUS_ONLINE:
+            can_be_claimed = false;
+            break;
+    }
+
+    buffer_json_member_add_boolean(wb, "can_be_claimed", can_be_claimed);
+
+    if(can_be_claimed && key) {
+        if(!netdata_random_session_id_matches(key)) {
+            buffer_reset(wb);
+            buffer_strcat(wb, "invalid key");
+            netdata_random_session_id_generate(); // generate a new key, to avoid an attack to find it
+            return HTTP_RESP_FORBIDDEN;
+        }
+
+        if(!token || !base_url || !check_claim_param(token) || !check_claim_param(base_url) || (rooms && !check_claim_param(rooms))) {
+            buffer_reset(wb);
+            buffer_strcat(wb, "invalid parameters");
+            netdata_random_session_id_generate(); // generate a new key, to avoid an attack to find it
+            return HTTP_RESP_BAD_REQUEST;
+        }
+
+        netdata_random_session_id_generate(); // generate a new key, to avoid an attack to find it
+
+        netdata_cloud_enabled = CONFIG_BOOLEAN_AUTO;
+        appconfig_set_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", CONFIG_BOOLEAN_AUTO);
+        appconfig_set(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", base_url);
+
+        uuid_t claimed_id;
+        uuid_generate_random(claimed_id);
+        char claimed_id_str[UUID_STR_LEN];
+        uuid_unparse_lower(claimed_id, claimed_id_str);
+
+        BUFFER *t = buffer_create(1024, NULL);
+        if(rooms)
+            buffer_sprintf(t, "-id=%s -token=%s -rooms=%s", claimed_id_str, token, rooms);
+        else
+            buffer_sprintf(t, "-id=%s -token=%s", claimed_id_str, token);
+
+        bool success = false;
+        const char *msg = NULL;
+        CLAIM_AGENT_RESPONSE rc = claim_agent(buffer_tostring(t), true, &msg);
+        switch(rc) {
+            case CLAIM_AGENT_OK:
+                msg = "ok";
+                success = true;
+                can_be_claimed = false;
+                claim_reload_all();
+                {
+                    int ms = 0;
+                    do {
+                        status = cloud_status();
+                        if (status == CLOUD_STATUS_ONLINE)
+                            break;
+
+                        sleep_usec(100 * USEC_PER_MS);
+                        ms += 100;
+                    } while (ms < 5000);
+                }
+                break;
+
+            case CLAIM_AGENT_NO_CLOUD_URL:
+                msg = "No Netdata Cloud URL.";
+                break;
+
+            case CLAIM_AGENT_CLAIM_SCRIPT_FAILED:
+                msg = "Claiming script failed.";
+                break;
+
+            case CLAIM_AGENT_CLOUD_DISABLED:
+                msg = "Netdata Cloud is disabled on this agent.";
+                break;
+
+            case CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT:
+                msg = "Failed to execute claiming script.";
+                break;
+
+            case CLAIM_AGENT_CLAIM_SCRIPT_RETURNED_INVALID_CODE:
+                msg = "Claiming script returned invalid code.";
+                break;
+
+            default:
+            case CLAIM_AGENT_FAILED_WITH_MESSAGE:
+                if(!msg)
+                    msg = "Unknown error";
+                break;
+        }
+
+        // our status may have changed
+        // refresh the status in our output
+        buffer_flush(wb);
+        buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+        now_s = now_realtime_sec();
+        buffer_json_cloud_status(wb, now_s);
+
+        // and this is the status of the claiming command we run
+        buffer_json_member_add_boolean(wb, "success", success);
+        buffer_json_member_add_string(wb, "message", msg);
+    }
+
+    if(can_be_claimed)
+        buffer_json_member_add_string(wb, "key_filename", netdata_random_session_id_get_filename());
+
+    buffer_json_agents_v2(wb, NULL, now_s, false, false);
+    buffer_json_finalize(wb);
+
+    return HTTP_RESP_OK;
 }
diff --git a/claim/claim.h b/claim/claim.h
index fc76037d3..ccab8aaa1 100644
--- a/claim/claim.h
+++ b/claim/claim.h
@@ -8,9 +8,25 @@
 extern char *claiming_pending_arguments;
 extern struct config cloud_config;
 
-void claim_agent(char *claiming_arguments);
+typedef enum __attribute__((packed)) {
+    CLAIM_AGENT_OK,
+    CLAIM_AGENT_CLOUD_DISABLED,
+    CLAIM_AGENT_NO_CLOUD_URL,
+    CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT,
+    CLAIM_AGENT_CLAIM_SCRIPT_FAILED,
+    CLAIM_AGENT_CLAIM_SCRIPT_RETURNED_INVALID_CODE,
+    CLAIM_AGENT_FAILED_WITH_MESSAGE,
+} CLAIM_AGENT_RESPONSE;
+
+CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, const char **msg);
 char *get_agent_claimid(void);
 void load_claiming_state(void);
 void load_cloud_conf(int silent);
+void claim_reload_all(void);
+
+bool netdata_random_session_id_generate(void);
+const char *netdata_random_session_id_get_filename(void);
+bool netdata_random_session_id_matches(const char *guid);
+int api_v2_claim(struct web_client *w, char *url);
 
 #endif //NETDATA_CLAIM_H
diff --git a/claim/netdata-claim.sh.in b/claim/netdata-claim.sh.in
index f87fbc273..43040e316 100755
--- a/claim/netdata-claim.sh.in
+++ b/claim/netdata-claim.sh.in
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # netdata
 # real-time performance and health monitoring, done right!
-# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# (C) 2023 Netdata Inc.
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 # Exit code: 0 - Success
@@ -194,24 +194,37 @@ if [ -r "${CLAIMING_DIR}/rooms" ]; then
         ROOMS="$(cat "${CLAIMING_DIR}/rooms")"
 fi
 
+variable_to_set=
 for arg in "$@"
 do
-        case $arg in
-                -token=*) TOKEN=${arg:7} ;;
-                -url=*) [ -n "${arg:5}" ] && URL_BASE=${arg:5} ;;
-                -id=*) ID=$(echo "${arg:4}" | tr '[:upper:]' '[:lower:]');;
-                -rooms=*) ROOMS=${arg:7} ;;
-                -hostname=*) HOSTNAME=${arg:10} ;;
-                -verbose) VERBOSE=1 ;;
-                -insecure) INSECURE=1 ;;
-                -proxy=*) PROXY=${arg:7} ;;
-                -noproxy) NOPROXY=yes ;;
-                -noreload) RELOAD=0 ;;
-                -user=*) NETDATA_USER=${arg:6} ;;
-                -daemon-not-running) NETDATA_RUNNING=0 ;;
-                *)  echo >&2 "Unknown argument ${arg}"
-                    exit 1 ;;
-        esac
+        if [ -z "$variable_to_set" ]; then
+              case $arg in
+                      --claim-token) variable_to_set="TOKEN" ;;
+                      --claim-rooms) variable_to_set="ROOMS" ;;
+                      --claim-url) variable_to_set="URL_BASE" ;;
+                      -token=*) TOKEN=${arg:7} ;;
+                      -url=*) [ -n "${arg:5}" ] && URL_BASE=${arg:5} ;;
+                      -id=*) ID=$(echo "${arg:4}" | tr '[:upper:]' '[:lower:]');;
+                      -rooms=*) ROOMS=${arg:7} ;;
+                      -hostname=*) HOSTNAME=${arg:10} ;;
+                      -verbose) VERBOSE=1 ;;
+                      -insecure) INSECURE=1 ;;
+                      -proxy=*) PROXY=${arg:7} ;;
+                      -noproxy) NOPROXY=yes ;;
+                      -noreload) RELOAD=0 ;;
+                      -user=*) NETDATA_USER=${arg:6} ;;
+                      -daemon-not-running) NETDATA_RUNNING=0 ;;
+                      *)  echo >&2 "Unknown argument ${arg}"
+                          exit 1 ;;
+              esac
+        else
+              case "$variable_to_set" in
+                      TOKEN) TOKEN="$arg" ;;
+                      ROOMS) ROOMS="$arg" ;;
+                      URL_BASE) URL_BASE="$arg" ;;
+              esac
+              variable_to_set=
+        fi
         shift 1
 done
 
@@ -402,7 +415,7 @@ if [ "${HTTP_STATUS_CODE}" = "204" ] || [ "${ERROR_KEY}" = "ErrAlreadyClaimed" ]
   cloud base url = $URL_BASE
 HERE_DOC
         if [ "$EUID" == "0" ]; then
-            chown -R "${NETDATA_USER}:${NETDATA_USER}" ${CLAIMING_DIR} || (echo >&2 "Claiming failed"; set -e; exit 2)
+            chown -R "${NETDATA_USER}:${NETDATA_USER}" "${CLAIMING_DIR}" || (echo >&2 "Claiming failed"; set -e; exit 2)
         fi
         if [ "${RELOAD}" == "0" ] ; then
             exit $EXIT_CODE