Merging upstream version 1.31.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 481 insertions, 75 deletions

diff --git a/aclk/https_client.c b/aclk/https_client.c
index 1b9546d77..907f512ba 100644
--- a/aclk/https_client.c
+++ b/aclk/https_client.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include "libnetdata/libnetdata.h"
 
 #include "https_client.h"
@@ -10,6 +12,19 @@ enum http_parse_state {
     HTTP_PARSE_CONTENT
 };
 
+static const char *http_req_type_to_str(http_req_type_t req) {
+    switch (req) {
+        case HTTP_REQ_GET:
+            return "GET";
+        case HTTP_REQ_POST:
+            return "POST";
+        case HTTP_REQ_CONNECT:
+            return "CONNECT";
+        default:
+            return "unknown";
+    }
+}
+
 typedef struct {
     enum http_parse_state state;
     int content_length;
@@ -17,6 +32,13 @@ typedef struct {
 } http_parse_ctx;
 
 #define HTTP_PARSE_CTX_INITIALIZER { .state = HTTP_PARSE_INITIAL, .content_length = -1, .http_code = 0 }
+static inline void http_parse_ctx_clear(http_parse_ctx *ctx) {
+    ctx->state = HTTP_PARSE_INITIAL;
+    ctx->content_length = -1;
+    ctx->http_code = 0;
+}
+
+#define POLL_TO_MS 100
 
 #define NEED_MORE_DATA  0
 #define PARSE_SUCCESS   1
@@ -71,8 +93,6 @@ static int parse_http_hdr(rbuf_t buf, http_parse_ctx *parse_ctx)
     rbuf_pop(buf, buf_val, idx_end);
     buf_val[idx_end] = 0;
 
-    rbuf_bump_tail(buf, strlen(HTTP_KEYVAL_SEPARATOR));
-
     for (ptr = buf_key; *ptr; ptr++)
         *ptr = tolower(*ptr);
 
@@ -129,10 +149,10 @@ static int parse_http_response(rbuf_t buf, http_parse_ctx *parse_ctx)
                 rbuf_bump_tail(buf, idx + strlen(HTTP_LINE_TERM));
                 break;
             case HTTP_PARSE_CONTENT:
-                if (parse_ctx->content_length < 0) {
-                    error("content-length missing and http headers ended");
-                    return PARSE_ERROR;
-                }
+                // replies like CONNECT etc. do not have content
+                if (parse_ctx->content_length < 0)
+                    return PARSE_SUCCESS;
+
                 if (rbuf_bytes_available(buf) >= (size_t)parse_ctx->content_length)
                     return PARSE_SUCCESS;
                 return NEED_MORE_DATA;
@@ -140,107 +160,493 @@ static int parse_http_response(rbuf_t buf, http_parse_ctx *parse_ctx)
     } while(1);
 }
 
-int https_request(http_req_type_t method, char *host, int port, char *url, char *b, size_t b_size, char *payload)
+typedef struct https_req_ctx {
+    https_req_t *request;
+
+    int sock;
+    rbuf_t buf_rx;
+
+    struct pollfd poll_fd;
+
+    SSL_CTX *ssl_ctx;
+    SSL *ssl;
+
+    size_t written;
+
+    int self_signed_allowed;
+
+    http_parse_ctx parse_ctx;
+
+    time_t req_start_time;
+} https_req_ctx_t;
+
+static int https_req_check_timedout(https_req_ctx_t *ctx) {
+    if (now_realtime_sec() > ctx->req_start_time + ctx->request->timeout_s) {
+        error("request timed out");
+        return 1;
+    }
+    return 0;
+}
+
+static char *_ssl_err_tos(int err)
 {
-    struct timeval timeout = { .tv_sec = 30, .tv_usec = 0 };
-    char sport[PORT_STR_MAX_BYTES];
-    size_t len = 0;
-    int rc = 1;
+    switch(err){
+        case SSL_ERROR_SSL:
+            return "SSL_ERROR_SSL";
+        case SSL_ERROR_WANT_READ:
+            return "SSL_ERROR_WANT_READ";
+        case SSL_ERROR_WANT_WRITE:
+            return "SSL_ERROR_WANT_WRITE";
+        case SSL_ERROR_NONE:
+            return "SSL_ERROR_NONE";
+        case SSL_ERROR_ZERO_RETURN:
+            return "SSL_ERROR_ZERO_RETURN";
+        case SSL_ERROR_WANT_CONNECT:
+            return "SSL_ERROR_WANT_CONNECT";
+        case SSL_ERROR_WANT_ACCEPT:
+            return "SSL_ERROR_WANT_ACCEPT";
+    }
+    return "Unknown!!!";
+}
+
+static int socket_write_all(https_req_ctx_t *ctx, char *data, size_t data_len) {
+    ctx->written = 0;
+    ctx->poll_fd.events = POLLOUT;
+
+    do {
+        int ret = poll(&ctx->poll_fd, 1, POLL_TO_MS);
+        if (ret < 0) {
+            error("poll error");
+            return 1;
+        }
+        if (ret == 0) {
+            if (https_req_check_timedout(ctx)) {
+                error("Poll timed out");
+                return 2;
+            }
+            continue;
+        }
+
+        ret = write(ctx->sock, &data[ctx->written], data_len - ctx->written);
+        if (ret > 0) {
+            ctx->written += ret;
+        } else if (errno != EAGAIN && errno != EWOULDBLOCK) {
+            error("Error writing to socket");
+            return 3;
+        }
+    } while (ctx->written < data_len);
+
+    return 0;
+}
+
+static int ssl_write_all(https_req_ctx_t *ctx, char *data, size_t data_len) {
+    ctx->written = 0;
+    ctx->poll_fd.events |= POLLOUT;
+
+    do {
+        int ret = poll(&ctx->poll_fd, 1, POLL_TO_MS);
+        if (ret < 0) {
+            error("poll error");
+            return 1;
+        }
+        if (ret == 0) {
+            if (https_req_check_timedout(ctx)) {
+                error("Poll timed out");
+                return 2;
+            }
+            continue;
+        }
+        ctx->poll_fd.events = 0;
+
+        ret = SSL_write(ctx->ssl, &data[ctx->written], data_len - ctx->written);
+        if (ret > 0) {
+            ctx->written += ret;
+        } else {
+            ret = SSL_get_error(ctx->ssl, ret);
+            switch (ret) {
+                case SSL_ERROR_WANT_READ:
+                    ctx->poll_fd.events |= POLLIN;
+                    break;
+                case SSL_ERROR_WANT_WRITE:
+                    ctx->poll_fd.events |= POLLOUT;
+                    break;
+                default:
+                    error("SSL_write Err: %s", _ssl_err_tos(ret));
+                    return 3;
+            }
+        }
+    } while (ctx->written < data_len);
+
+    return 0;
+}
+
+static inline int https_client_write_all(https_req_ctx_t *ctx, char *data, size_t data_len) {
+    if (ctx->ssl_ctx)
+        return ssl_write_all(ctx, data, data_len);
+    return socket_write_all(ctx, data, data_len);
+}
+
+static int read_parse_response(https_req_ctx_t *ctx) {
     int ret;
     char *ptr;
-    http_parse_ctx parse_ctx = HTTP_PARSE_CTX_INITIALIZER;
+    size_t size;
 
-    rbuf_t buffer = rbuf_create(b_size);
-    if (!buffer)
+    ctx->poll_fd.events = POLLIN;
+    do {
+        ret = poll(&ctx->poll_fd, 1, POLL_TO_MS);
+        if (ret < 0) {
+            error("poll error");
+            return 1;
+        }
+        if (ret == 0) {
+            if (https_req_check_timedout(ctx)) {
+                error("Poll timed out");
+                return 2;
+            }
+            continue;
+        }
+        ctx->poll_fd.events = 0;
+
+        ptr = rbuf_get_linear_insert_range(ctx->buf_rx, &size);
+
+        if (ctx->ssl_ctx)
+            ret = SSL_read(ctx->ssl, ptr, size);
+        else
+            ret = read(ctx->sock, ptr, size);
+
+        if (ret > 0) {
+            rbuf_bump_head(ctx->buf_rx, ret);
+        } else {
+            if (ctx->ssl_ctx) {
+                ret = SSL_get_error(ctx->ssl, ret);
+                switch (ret) {
+                    case SSL_ERROR_WANT_READ:
+                        ctx->poll_fd.events |= POLLIN;
+                        break;
+                    case SSL_ERROR_WANT_WRITE:
+                        ctx->poll_fd.events |= POLLOUT;
+                        break;
+                    default:
+                        error("SSL_read Err: %s", _ssl_err_tos(ret));
+                        return 3;
+                }
+            } else {
+                if (errno != EAGAIN && errno != EWOULDBLOCK) {
+                    error("write error");
+                    return 3;
+                }
+                ctx->poll_fd.events |= POLLIN;
+            }
+        }
+    } while (!(ret = parse_http_response(ctx->buf_rx, &ctx->parse_ctx)));
+
+    if (ret != PARSE_SUCCESS) {
+        error("Error parsing HTTP response");
         return 1;
+    }
 
-    snprintf(sport, PORT_STR_MAX_BYTES, "%d", port);
+    return 0;
+}
 
-    if (payload != NULL)
-        len = strlen(payload);
+#define TX_BUFFER_SIZE 8192
+#define RX_BUFFER_SIZE (TX_BUFFER_SIZE*2)
+static int handle_http_request(https_req_ctx_t *ctx) {
+    BUFFER *hdr = buffer_create(TX_BUFFER_SIZE);
+    int rc = 0;
+
+    http_parse_ctx_clear(&ctx->parse_ctx);
+
+    // Prepare data to send
+    switch (ctx->request->request_type) {
+        case HTTP_REQ_CONNECT:
+            buffer_strcat(hdr, "CONNECT ");
+            break;
+        case HTTP_REQ_GET:
+            buffer_strcat(hdr, "GET ");
+            break;
+        case HTTP_REQ_POST:
+            buffer_strcat(hdr, "POST ");
+            break;
+        default:
+            error("Unknown HTTPS request type!");
+            rc = 1;
+            goto err_exit;
+    }
+
+    if (ctx->request->request_type == HTTP_REQ_CONNECT) {
+        buffer_strcat(hdr, ctx->request->host);
+        buffer_sprintf(hdr, ":%d", ctx->request->port);
+    } else {
+        buffer_strcat(hdr, ctx->request->url);
+    }
+
+    buffer_strcat(hdr, " HTTP/1.1\x0D\x0A");
 
-    snprintf(
-        b,
-        b_size,
-        "%s %s HTTP/1.1\r\nHost: %s\r\nAccept: application/json\r\nContent-length: %zu\r\nAccept-Language: en-us\r\n"
-        "User-Agent: Netdata/rocks\r\n\r\n",
-        (method == HTTP_REQ_GET ? "GET" : "POST"), url, host, len);
+    //TODO Headers!
+    if (ctx->request->request_type != HTTP_REQ_CONNECT) {
+        buffer_sprintf(hdr, "Host: %s\x0D\x0A", ctx->request->host);
+    }
+    buffer_strcat(hdr, "User-Agent: Netdata/rocks newhttpclient\x0D\x0A");
 
-    if (payload != NULL)
-        strncat(b, payload, b_size - len);
+    if (ctx->request->request_type == HTTP_REQ_POST && ctx->request->payload && ctx->request->payload_size) {
+        buffer_sprintf(hdr, "Content-Length: %zu\x0D\x0A", ctx->request->payload_size);
+    }
 
-    len = strlen(b);
+    buffer_strcat(hdr, "\x0D\x0A");
 
-    debug(D_ACLK, "Sending HTTPS req (%zu bytes): '%s'", len, b);
-    int sock = connect_to_this_ip46(IPPROTO_TCP, SOCK_STREAM, host, 0, sport, &timeout);
+    // Send the request
+    if (https_client_write_all(ctx, hdr->buffer, hdr->len)) {
+        error("Couldn't write HTTP request header into SSL connection");
+        rc = 2;
+        goto err_exit;
+    }
 
-    if (unlikely(sock == -1)) {
-        error("Handshake failed");
-        goto exit_buf;
+    if (ctx->request->request_type == HTTP_REQ_POST && ctx->request->payload && ctx->request->payload_size) {
+        if (https_client_write_all(ctx, ctx->request->payload, ctx->request->payload_size)) {
+            error("Couldn't write payload into SSL connection");
+            rc = 3;
+            goto err_exit;
+        }
     }
 
-    SSL_CTX *ctx = security_initialize_openssl_client();
-    if (ctx==NULL) {
+    // Read The Response
+    if (read_parse_response(ctx)) {
+        error("Error reading or parsing response from server");
+        rc = 4;
+        goto err_exit;
+    }
+
+err_exit:
+    buffer_free(hdr);
+    return rc;
+}
+
+int https_request(https_req_t *request, https_req_response_t *response) {
+    int rc = 1, ret;
+    char connect_port_str[PORT_STR_MAX_BYTES];
+
+    const char *connect_host = request->proxy_host ? request->proxy_host : request->host;
+    int connect_port = request->proxy_host ? request->proxy_port : request->port;
+    struct timeval timeout = { .tv_sec = request->timeout_s, .tv_usec = 0 };
+
+    https_req_ctx_t *ctx = callocz(1, sizeof(https_req_ctx_t));
+    ctx->req_start_time = now_realtime_sec();
+
+    ctx->buf_rx = rbuf_create(RX_BUFFER_SIZE);
+    if (!ctx->buf_rx) {
+        error("Couldn't allocate buffer for RX data");
+        goto exit_req_ctx;
+    }
+
+    snprintf(connect_port_str, PORT_STR_MAX_BYTES, "%d", connect_port);
+
+    ctx->sock = connect_to_this_ip46(IPPROTO_TCP, SOCK_STREAM, connect_host, 0, connect_port_str, &timeout);
+    if (ctx->sock < 0) {
+        error("Error connecting TCP socket to \"%s\"", connect_host);
+        goto exit_buf_rx;
+    }
+
+    if (fcntl(ctx->sock, F_SETFL, fcntl(ctx->sock, F_GETFL, 0) | O_NONBLOCK) == -1) {
+        error("Error setting O_NONBLOCK to TCP socket.");
+        goto exit_sock;
+    }
+
+    ctx->poll_fd.fd = ctx->sock;
+
+    // Do the CONNECT if proxy is used
+    if (request->proxy_host) {
+        https_req_t req = HTTPS_REQ_T_INITIALIZER;
+        req.request_type = HTTP_REQ_CONNECT;
+        req.timeout_s = request->timeout_s;
+        req.host = request->host;
+        req.port = request->port;
+        req.url = request->url;
+        ctx->request = &req;
+        if (handle_http_request(ctx)) {
+            error("Failed to CONNECT with proxy");
+            goto exit_sock;
+        }
+        if (ctx->parse_ctx.http_code != 200) {
+            error("Proxy didn't return 200 OK (got %d)", ctx->parse_ctx.http_code);
+            goto exit_sock;
+        }
+        info("Proxy accepted CONNECT upgrade");
+    }
+    ctx->request = request;
+
+    ctx->ssl_ctx = security_initialize_openssl_client();
+    if (ctx->ssl_ctx==NULL) {
         error("Cannot allocate SSL context");
         goto exit_sock;
     }
-    // Certificate chain: not updating the stores - do we need private CA roots?
-    // Calls to SSL_CTX_load_verify_locations would go here.
-    SSL *ssl = SSL_new(ctx);
-    if (ssl==NULL) {
+
+    ctx->ssl = SSL_new(ctx->ssl_ctx);
+    if (ctx->ssl==NULL) {
         error("Cannot allocate SSL");
         goto exit_CTX;
     }
-    SSL_set_fd(ssl, sock);
-    ret = SSL_connect(ssl);
-    if (ret != 1) {
-        error("SSL_connect() failed with err=%d", ret);
+
+    SSL_set_fd(ctx->ssl, ctx->sock);
+    ret = SSL_connect(ctx->ssl);
+    if (ret != -1 && ret != 1) {
+        error("SSL could not connect");
         goto exit_SSL;
     }
+    if (ret == -1) {
+        // expected as underlying socket is non blocking!
+        // consult SSL_connect documentation for details
+        int ec = SSL_get_error(ctx->ssl, ret);
+        if (ec != SSL_ERROR_WANT_READ && ec != SSL_ERROR_WANT_WRITE) {
+            error("Failed to start SSL connection");
+            goto exit_SSL;
+        }
+    }
 
-    ret = SSL_write(ssl, b, len);
-    if (ret <= 0)
-    {
-        error("SSL_write() failed with err=%d", ret);
+    // The actual request here
+    if (handle_http_request(ctx)) {
+        error("Couldn't process request");
         goto exit_SSL;
     }
+    response->http_code = ctx->parse_ctx.http_code;
+    if (ctx->parse_ctx.content_length > 0) {
+        response->payload_size = ctx->parse_ctx.content_length;
+        response->payload = mallocz(response->payload_size + 1);
+        ret = rbuf_pop(ctx->buf_rx, response->payload, response->payload_size);
+        if (ret != (int)response->payload_size) {
+            error("Payload size doesn't match remaining data on the buffer!");
+            response->payload_size = ret;
+        }
+        // normally we take payload as it is and copy it
+        // but for convenience in cases where payload is sth. like
+        // json we add terminating zero so that user of the data
+        // doesn't have to convert to C string (0 terminated)
+        // other uses still have correct payload_size and can copy
+        // only exact data without affixed 0x00
+        ((char*)response->payload)[response->payload_size] = 0; // mallocz(response->payload_size + 1);
+    }
+    info("HTTPS \"%s\" request to \"%s\" finished with HTTP code: %d", http_req_type_to_str(ctx->request->request_type), ctx->request->host, response->http_code);
 
-    b[0] = 0;
+    rc = 0;
 
-    do {
-        ptr = rbuf_get_linear_insert_range(buffer, &len);
-        ret = SSL_read(ssl, ptr, len - 1);
-        if (ret)
-            rbuf_bump_head(buffer, ret);
-        if (ret <= 0)
-        {
-            error("No response available - SSL_read()=%d", ret);
-            goto exit_FULL;
+exit_SSL:
+    SSL_free(ctx->ssl);
+exit_CTX:
+    SSL_CTX_free(ctx->ssl_ctx);
+exit_sock:
+    close(ctx->sock);
+exit_buf_rx:
+    rbuf_free(ctx->buf_rx);
+exit_req_ctx:
+    freez(ctx);
+    return rc;
+}
+
+void https_req_response_free(https_req_response_t *res) {
+    freez(res->payload);
+}
+
+void https_req_response_init(https_req_response_t *res) {
+    res->http_code = 0;
+    res->payload = NULL;
+    res->payload_size = 0;
+}
+
+static inline char *min_non_null(char *a, char *b) {
+    if (!a)
+        return b;
+    if (!b)
+        return a;
+    return (a < b ? a : b);
+}
+
+#define URI_PROTO_SEPARATOR "://"
+#define URL_PARSER_LOG_PREFIX "url_parser "
+
+static int parse_host_port(url_t *url) {
+    char *ptr = strrchr(url->host, ':');
+    if (ptr) {
+        size_t port_len = strlen(ptr + 1);
+        if (!port_len) {
+            error(URL_PARSER_LOG_PREFIX ": specified but no port number");
+            return 1;
+        }
+        if (port_len > 5 /* MAX port lenght is 5digit long in decimal */) {
+            error(URL_PARSER_LOG_PREFIX "port # is too long");
+            return 1;
         }
-    } while (!(ret = parse_http_response(buffer, &parse_ctx)));
+        *ptr = 0;
+        if (!strlen(url->host)) {
+            error(URL_PARSER_LOG_PREFIX "host empty after removing port");
+            return 1;
+        }
+        url->port = atoi (ptr + 1);
+    }
+    return 0;
+}
 
-    if (ret != PARSE_SUCCESS) {
-        error("Error parsing HTTP response");
-        goto exit_FULL;
+static inline void port_by_proto(url_t *url) {
+    if (url->port)
+        return;
+    if (!url->proto)
+        return;
+    if (!strcmp(url->proto, "http")) {
+        url->port = 80;
+        return;
+    }
+    if (!strcmp(url->proto, "https")) {
+        url->port = 443;
+        return;
     }
+}
 
-    if (parse_ctx.http_code < 200 || parse_ctx.http_code >= 300) {
-        error("HTTP Response not Success (got %d)", parse_ctx.http_code);
-        goto exit_FULL;
+#define STRDUPZ_2PTR(dest, start, end)                                                                                 \
+    {                                                                                                                  \
+        dest = mallocz(1 + end - start);                                                                               \
+        memcpy(dest, start, end - start);                                                                              \
+        dest[end - start] = 0;                                                                                         \
     }
 
-    len = rbuf_pop(buffer, b, b_size);
-    b[MIN(len, b_size-1)] = 0;
+int url_parse(const char *url, url_t *parsed) {
+    const char *start = url;
+    const char *end = strstr(url, URI_PROTO_SEPARATOR);
 
-    rc = 0;
-exit_FULL:
-exit_SSL:
-    SSL_free(ssl);
-exit_CTX:
-    SSL_CTX_free(ctx);
-exit_sock:
-    close(sock);
-exit_buf:
-    rbuf_free(buffer);
-    return rc;
+    if (end) {
+        if (end == start) {
+            error (URL_PARSER_LOG_PREFIX "found " URI_PROTO_SEPARATOR " without protocol specified");
+            return 1;
+        }
+
+        STRDUPZ_2PTR(parsed->proto, start, end)
+        start = end + strlen(URI_PROTO_SEPARATOR);
+    }
+
+    end = strchr(start, '/');
+    if (!end)
+        end = start + strlen(start);
+    
+    if (start == end) {
+        error(URL_PARSER_LOG_PREFIX "Host empty");
+        return 1;
+    }
+
+    STRDUPZ_2PTR(parsed->host, start, end);
+
+    if (parse_host_port(parsed))
+        return 1;
+
+    if (!*end) {
+        parsed->path = strdupz("/");
+        port_by_proto(parsed);
+        return 0;
+    }
+
+    parsed->path = strdupz(end);
+    port_by_proto(parsed);
+    return 0;
+}
+
+void url_t_destroy(url_t *url) {
+    freez(url->host);
+    freez(url->path);
+    freez(url->proto);
 }