From 6cf8f2d5174a53f582e61d715edbb88d6e3367cc Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 14 Jun 2023 21:20:33 +0200 Subject: Adding upstream version 1.40.0. Signed-off-by: Daniel Baumann --- web/server/static/static-threaded.c | 59 +++++---------- web/server/web_client.c | 142 ++++++++++-------------------------- web/server/web_client.h | 38 +--------- web/server/web_client_cache.c | 10 +-- 4 files changed, 66 insertions(+), 183 deletions(-) (limited to 'web/server') diff --git a/web/server/static/static-threaded.c b/web/server/static/static-threaded.c index 52bb56cd6..4cb3dcd92 100644 --- a/web/server/static/static-threaded.c +++ b/web/server/static/static-threaded.c @@ -211,58 +211,32 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data } #ifdef ENABLE_HTTPS - if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) { - if( sock_delnonblock(w->ifd) < 0 ){ - error("Web server cannot remove the non-blocking flag from socket %d",w->ifd); - } + if ((!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx)) { + sock_delnonblock(w->ifd); //Read the first 7 bytes from the message, but the message //is not removed from the queue, because we are using MSG_PEEK char test[8]; - if ( recv(w->ifd,test, 7,MSG_PEEK) == 7 ) { - test[7] = 0x00; + if ( recv(w->ifd,test, 7, MSG_PEEK) == 7 ) { + test[7] = '\0'; } else { - //Case I do not have success to read 7 bytes, - //this means that the mensage was not completely read, so - //I cannot identify it yet. + // we couldn't read 7 bytes sock_setnonblock(w->ifd); goto cleanup; } - //The next two ifs are not together because I am reusing SSL structure - if (!w->ssl.conn) - { - w->ssl.conn = SSL_new(netdata_ssl_srv_ctx); - if ( w->ssl.conn ) { - SSL_set_accept_state(w->ssl.conn); - } else { - error("Failed to create SSL context on socket fd %d.", w->ifd); - if (test[0] < 0x18){ - WEB_CLIENT_IS_DEAD(w); - sock_setnonblock(w->ifd); - goto cleanup; - } - } + if(test[0] > 0x17) { + // no SSL + netdata_ssl_close(&w->ssl); // free any previous SSL data } - - if (w->ssl.conn) { - if (SSL_set_fd(w->ssl.conn, w->ifd) != 1) { - error("Failed to set the socket to the SSL on socket fd %d.", w->ifd); - //The client is not set dead, because I received a normal HTTP request - //instead a Client Hello(HTTPS). - if ( test[0] < 0x18 ){ - WEB_CLIENT_IS_DEAD(w); - } - } - else{ - w->ssl.flags = security_process_accept(w->ssl.conn, (int)test[0]); - } + else { + // SSL + if(!netdata_ssl_open(&w->ssl, netdata_ssl_web_server_ctx, w->ifd) || !netdata_ssl_accept(&w->ssl)) + WEB_CLIENT_IS_DEAD(w); } sock_setnonblock(w->ifd); - } else{ - w->ssl.flags = NETDATA_SSL_NO_HANDSHAKE; } #endif @@ -525,9 +499,15 @@ void *socket_listen_main_static_threaded(void *ptr) { if(!api_sockets.opened) fatal("LISTENER: no listen sockets available."); + netdata_ssl_validate_certificate = !config_get_boolean(CONFIG_SECTION_WEB, "ssl skip certificate verification", !netdata_ssl_validate_certificate); + + if(!netdata_ssl_validate_certificate_sender) + info("SSL: web server will skip SSL certificates verification."); + #ifdef ENABLE_HTTPS - security_start_ssl(NETDATA_SSL_CONTEXT_SERVER); + netdata_ssl_initialize_ctx(NETDATA_SSL_WEB_SERVER_CTX); #endif + // 6 threads is the optimal value // since 6 are the parallel connections browsers will do // so, if the machine has more CPUs, avoid using resources unnecessarily @@ -541,6 +521,7 @@ void *socket_listen_main_static_threaded(void *ptr) { static_threaded_workers_count = config_get_number(CONFIG_SECTION_WEB, "web server threads", def_thread_count); if (static_threaded_workers_count < 1) static_threaded_workers_count = 1; + #ifdef ENABLE_HTTPS // See https://github.com/netdata/netdata/issues/11081#issuecomment-831998240 for more details if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) { diff --git a/web/server/web_client.c b/web/server/web_client.c index 8bc72e71f..6e3c1225e 100644 --- a/web/server/web_client.c +++ b/web/server/web_client.c @@ -8,9 +8,7 @@ int respect_web_browser_do_not_track_policy = 0; char *web_x_frame_options = NULL; -#ifdef NETDATA_WITH_ZLIB int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY; -#endif /* NETDATA_WITH_ZLIB */ inline int web_client_permission_denied(struct web_client *w) { w->response.data->content_type = CT_TEXT_PLAIN; @@ -36,11 +34,10 @@ static inline int web_client_crock_socket(struct web_client *w __maybe_unused) { return 0; } -static inline void web_client_enable_wait_from_ssl(struct web_client *w, int bytes) { - int ssl_err = SSL_get_error(w->ssl.conn, bytes); - if (ssl_err == SSL_ERROR_WANT_READ) +static inline void web_client_enable_wait_from_ssl(struct web_client *w) { + if (w->ssl.ssl_errno == SSL_ERROR_WANT_READ) web_client_enable_ssl_wait_receive(w); - else if (ssl_err == SSL_ERROR_WANT_WRITE) + else if (w->ssl.ssl_errno == SSL_ERROR_WANT_WRITE) web_client_enable_ssl_wait_send(w); else { web_client_disable_ssl_wait_receive(w); @@ -101,15 +98,6 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) { freez(w->post_payload); w->post_payload = NULL; w->post_payload_size = 0; - -#ifdef ENABLE_HTTPS - if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) { - if (w->ssl.conn) { - SSL_free(w->ssl.conn); - w->ssl.conn = NULL; - } - } -#endif } else { // the web client is to be re-used @@ -123,7 +111,6 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) { buffer_reset(w->response.data); // leave w->post_payload - // leave w->ssl } freez(w->server_host); @@ -142,7 +129,6 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) { w->auth_bearer_token = NULL; // if we had enabled compression, release it -#ifdef NETDATA_WITH_ZLIB if(w->response.zinitialized) { deflateEnd(&w->response.zstream); w->response.zsent = 0; @@ -154,7 +140,6 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) { w->response.zinitialized = false; w->flags &= ~WEB_CLIENT_CHUNKED_TRANSFER; } -#endif // NETDATA_WITH_ZLIB } void web_client_request_done(struct web_client *w) { @@ -168,9 +153,7 @@ void web_client_request_done(struct web_client *w) { size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; size_t sent = size; -#ifdef NETDATA_WITH_ZLIB if(likely(w->response.zoutput)) sent = (size_t)w->response.zstream.total_out; -#endif // -------------------------------------------------------------------- // global statistics @@ -444,9 +427,6 @@ int mysendfile(struct web_client *w, char *filename) { } #endif - - -#ifdef NETDATA_WITH_ZLIB void web_client_enable_deflate(struct web_client *w, int gzip) { if(unlikely(w->response.zinitialized)) { debug(D_DEFLATE, "%llu: Compression has already be initialized for this client.", w->id); @@ -492,7 +472,6 @@ void web_client_enable_deflate(struct web_client *w, int gzip) { debug(D_DEFLATE, "%llu: Initialized compression.", w->id); } -#endif // NETDATA_WITH_ZLIB void buffer_data_options2string(BUFFER *wb, uint32_t options) { int count = 0; @@ -730,16 +709,12 @@ const char *web_response_code_to_string(int code) { static inline char *http_header_parse(struct web_client *w, char *s, int parse_useragent) { static uint32_t hash_origin = 0, hash_connection = 0, hash_donottrack = 0, hash_useragent = 0, hash_authorization = 0, hash_host = 0, hash_forwarded_proto = 0, hash_forwarded_host = 0; -#ifdef NETDATA_WITH_ZLIB static uint32_t hash_accept_encoding = 0; -#endif if(unlikely(!hash_origin)) { hash_origin = simple_uhash("Origin"); hash_connection = simple_uhash("Connection"); -#ifdef NETDATA_WITH_ZLIB hash_accept_encoding = simple_uhash("Accept-Encoding"); -#endif hash_donottrack = simple_uhash("DNT"); hash_useragent = simple_uhash("User-Agent"); hash_authorization = simple_uhash("X-Auth-Token"); @@ -798,7 +773,6 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1)); w->server_host = strdupz(buffer); } -#ifdef NETDATA_WITH_ZLIB else if(hash == hash_accept_encoding && !strcasecmp(s, "Accept-Encoding")) { if(web_enable_gzip) { if(strcasestr(v, "gzip")) @@ -809,13 +783,10 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u // web_client_enable_deflate(w, 0); } } -#endif /* NETDATA_WITH_ZLIB */ -#ifdef ENABLE_HTTPS else if(hash == hash_forwarded_proto && !strcasecmp(s, "X-Forwarded-Proto")) { if(strcasestr(v, "https")) - w->ssl.flags |= NETDATA_SSL_PROXY_HTTPS; + w->flags |= WEB_CLIENT_FLAG_PROXY_HTTPS; } -#endif else if(hash == hash_forwarded_host && !strcasecmp(s, "X-Forwarded-Host")) { char buffer[NI_MAXHOST]; strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1)); @@ -855,7 +826,7 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) { s = &s[7]; #ifdef ENABLE_HTTPS - if (w->ssl.flags && web_client_is_using_ssl_force(w)){ + if (!SSL_connection(&w->ssl) && web_client_is_using_ssl_force(w)) { w->header_parse_tries = 0; w->header_parse_last_size = 0; web_client_disable_wait_receive(w); @@ -996,8 +967,8 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) { *ue = c; #ifdef ENABLE_HTTPS - if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) { - if ((w->ssl.conn) && ((w->ssl.flags & NETDATA_SSL_NO_HANDSHAKE) && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM)) ) { + if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) { + if (!w->ssl.conn && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM)) { w->header_parse_tries = 0; w->header_parse_last_size = 0; web_client_disable_wait_receive(w); @@ -1026,16 +997,15 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf, { ssize_t bytes; #ifdef ENABLE_HTTPS - if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) { - if ( ( w->ssl.conn ) && ( !w->ssl.flags ) ){ - bytes = netdata_ssl_write(w->ssl.conn, buf, len) ; - web_client_enable_wait_from_ssl(w, bytes); - } else { - bytes = send(w->ofd,buf, len , flags); + if ((!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx)) { + if (SSL_connection(&w->ssl)) { + bytes = netdata_ssl_write(&w->ssl, buf, len) ; + web_client_enable_wait_from_ssl(w); } - } else { + else + bytes = send(w->ofd,buf, len , flags); + } else bytes = send(w->ofd,buf, len , flags); - } #else bytes = send(w->ofd, buf, len, flags); #endif @@ -1172,10 +1142,10 @@ static inline void web_client_send_http_header(struct web_client *w) { size_t count = 0; ssize_t bytes; #ifdef ENABLE_HTTPS - if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) { - if ( ( w->ssl.conn ) && ( w->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ) ) { - bytes = netdata_ssl_write(w->ssl.conn, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output)); - web_client_enable_wait_from_ssl(w, bytes); + if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) { + if (SSL_connection(&w->ssl)) { + bytes = netdata_ssl_write(&w->ssl, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output)); + web_client_enable_wait_from_ssl(w); } else { while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) { @@ -1276,11 +1246,11 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch if(!url) { //no delim found debug(D_WEB_CLIENT, "%llu: URL doesn't end with / generating redirect.", w->id); char *protocol, *url_host; + protocol = ( #ifdef ENABLE_HTTPS - protocol = ((w->ssl.conn && !w->ssl.flags) || w->ssl.flags & NETDATA_SSL_PROXY_HTTPS) ? "https" : "http"; -#else - protocol = "http"; + SSL_connection(&w->ssl) || #endif + (w->flags & WEB_CLIENT_FLAG_PROXY_HTTPS)) ? "https" : "http"; url_host = w->forwarded_host; if(!url_host) { @@ -1736,7 +1706,6 @@ ssize_t web_client_send_chunk_finalize(struct web_client *w) return bytes; } -#ifdef NETDATA_WITH_ZLIB ssize_t web_client_send_deflate(struct web_client *w) { ssize_t len = 0, t = 0; @@ -1851,12 +1820,9 @@ ssize_t web_client_send_deflate(struct web_client *w) return(len); } -#endif // NETDATA_WITH_ZLIB ssize_t web_client_send(struct web_client *w) { -#ifdef NETDATA_WITH_ZLIB if(likely(w->response.zoutput)) return web_client_send_deflate(w); -#endif // NETDATA_WITH_ZLIB ssize_t bytes; @@ -1968,11 +1934,12 @@ ssize_t web_client_receive(struct web_client *w) buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE); #ifdef ENABLE_HTTPS - if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) { - if ( ( w->ssl.conn ) && (!w->ssl.flags)) { - bytes = netdata_ssl_read(w->ssl.conn, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1)); - web_client_enable_wait_from_ssl(w, bytes); - }else { + if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) { + if (SSL_connection(&w->ssl)) { + bytes = netdata_ssl_read(&w->ssl, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1)); + web_client_enable_wait_from_ssl(w); + } + else { bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT); } } @@ -2005,26 +1972,6 @@ ssize_t web_client_receive(struct web_client *w) } -int web_client_socket_is_now_used_for_streaming(struct web_client *w) { - // prevent the web_client from closing the streaming socket - - WEB_CLIENT_IS_DEAD(w); - - if(web_server_mode == WEB_SERVER_MODE_STATIC_THREADED) { - web_client_flag_set(w, WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET); - } - else { - if(w->ifd == w->ofd) - w->ifd = w->ofd = -1; - else - w->ifd = -1; - } - - buffer_flush(w->response.data); - - return HTTP_RESP_OK; -} - void web_client_decode_path_and_query_string(struct web_client *w, const char *path_and_query_string) { char buffer[NETDATA_WEB_REQUEST_URL_SIZE + 2]; buffer[0] = '\0'; @@ -2072,25 +2019,6 @@ void web_client_decode_path_and_query_string(struct web_client *w, const char *p } } -#ifdef ENABLE_HTTPS -void web_client_reuse_ssl(struct web_client *w) { - if (netdata_ssl_srv_ctx) { - if (w->ssl.conn) { - SSL_SESSION *session = SSL_get_session(w->ssl.conn); - SSL *old = w->ssl.conn; - w->ssl.conn = SSL_new(netdata_ssl_srv_ctx); - if (session) { -#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_111 - if (SSL_SESSION_is_resumable(session)) -#endif - SSL_set_session(w->ssl.conn, session); - } - SSL_free(old); - } - } -} -#endif - void web_client_zero(struct web_client *w) { // zero everything about it - but keep the buffers @@ -2105,8 +2033,7 @@ void web_client_zero(struct web_client *w) { BUFFER *b6 = w->url_query_string_decoded; #ifdef ENABLE_HTTPS - web_client_reuse_ssl(w); - SSL *ssl = w->ssl.conn; + NETDATA_SSL ssl = w->ssl; #endif size_t use_count = w->use_count; @@ -2120,9 +2047,7 @@ void web_client_zero(struct web_client *w) { w->use_count = use_count; #ifdef ENABLE_HTTPS - w->ssl.conn = ssl; - w->ssl.flags = NETDATA_SSL_START; - debug(D_WEB_CLIENT_ACCESS,"Reusing SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags); + w->ssl = ssl; #endif // restore the pointers of the buffers @@ -2136,6 +2061,11 @@ void web_client_zero(struct web_client *w) { struct web_client *web_client_create(size_t *statistics_memory_accounting) { struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client)); + +#ifdef ENABLE_HTTPS + w->ssl = NETDATA_SSL_UNSET_CONNECTION; +#endif + w->use_count = 1; w->statistics.memory_accounting = statistics_memory_accounting; @@ -2152,6 +2082,10 @@ struct web_client *web_client_create(size_t *statistics_memory_accounting) { } void web_client_free(struct web_client *w) { +#ifdef ENABLE_HTTPS + netdata_ssl_close(&w->ssl); +#endif + web_client_reset_allocations(w, true); __atomic_sub_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED); diff --git a/web/server/web_client.h b/web/server/web_client.h index c61a8b813..4c2b06a70 100644 --- a/web/server/web_client.h +++ b/web/server/web_client.h @@ -5,33 +5,7 @@ #include "libnetdata/libnetdata.h" -#ifdef NETDATA_WITH_ZLIB extern int web_enable_gzip, web_gzip_level, web_gzip_strategy; -#endif /* NETDATA_WITH_ZLIB */ - -// HTTP_CODES 2XX Success -#define HTTP_RESP_OK 200 - -// HTTP_CODES 3XX Redirections -#define HTTP_RESP_MOVED_PERM 301 -#define HTTP_RESP_REDIR_TEMP 307 -#define HTTP_RESP_REDIR_PERM 308 - -// HTTP_CODES 4XX Client Errors -#define HTTP_RESP_BAD_REQUEST 400 -#define HTTP_RESP_UNAUTHORIZED 401 -#define HTTP_RESP_FORBIDDEN 403 -#define HTTP_RESP_NOT_FOUND 404 -#define HTTP_RESP_CONFLICT 409 -#define HTTP_RESP_PRECOND_FAIL 412 -#define HTTP_RESP_CONTENT_TOO_LONG 413 - -// HTTP_CODES 5XX Server Errors -#define HTTP_RESP_INTERNAL_SERVER_ERROR 500 -#define HTTP_RESP_BACKEND_FETCH_FAILED 503 -#define HTTP_RESP_SERVICE_UNAVAILABLE 503 -#define HTTP_RESP_GATEWAY_TIMEOUT 504 -#define HTTP_RESP_BACKEND_RESPONSE_INVALID 591 #define HTTP_REQ_MAX_HEADER_FETCH_TRIES 100 @@ -78,6 +52,8 @@ typedef enum web_client_flags { WEB_CLIENT_FLAG_SSL_WAIT_RECEIVE = 1 << 11, // if set, we are waiting more input data from an ssl conn WEB_CLIENT_FLAG_SSL_WAIT_SEND = 1 << 12, // if set, we have data to send to the client from an ssl conn + + WEB_CLIENT_FLAG_PROXY_HTTPS = 1 << 13, // if set, the client reaches us via an https proxy } WEB_CLIENT_FLAGS; #define web_client_flag_check(w, flag) ((w)->flags & (flag)) @@ -145,13 +121,11 @@ struct response { bool zoutput; // if set to 1, web_client_send() will send compressed data -#ifdef NETDATA_WITH_ZLIB bool zinitialized; z_stream zstream; // zlib stream for sending compressed output to client size_t zsent; // the compressed bytes we have sent to the client size_t zhave; // the compressed bytes that we have received from zlib Bytef zbuffer[NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE]; // temporary buffer for storing compressed output -#endif /* NETDATA_WITH_ZLIB */ }; struct web_client; @@ -196,7 +170,7 @@ struct web_client { size_t pollinfo_filecopy_slot; // POLLINFO slot of the file read #ifdef ENABLE_HTTPS - struct netdata_ssl ssl; + NETDATA_SSL ssl; #endif struct { // A callback to check if the query should be interrupted / stopped @@ -241,16 +215,10 @@ int mysendfile(struct web_client *w, char *filename); void web_client_build_http_header(struct web_client *w); char *strip_control_characters(char *url); -int web_client_socket_is_now_used_for_streaming(struct web_client *w); - void web_client_zero(struct web_client *w); struct web_client *web_client_create(size_t *statistics_memory_accounting); void web_client_free(struct web_client *w); -#ifdef ENABLE_HTTPS -void web_client_reuse_ssl(struct web_client *w); -#endif - #include "web/api/web_api_v1.h" #include "web/api/web_api_v2.h" #include "daemon/common.h" diff --git a/web/server/web_client_cache.c b/web/server/web_client_cache.c index b410ba7f9..394bea32b 100644 --- a/web/server/web_client_cache.c +++ b/web/server/web_client_cache.c @@ -104,11 +104,6 @@ struct web_client *web_client_get_from_cache(void) { // allocate it w = web_client_create(&netdata_buffers_statistics.buffers_web); -#ifdef ENABLE_HTTPS - w->ssl.flags = NETDATA_SSL_START; - debug(D_WEB_CLIENT_ACCESS,"Starting SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags); -#endif - netdata_spinlock_lock(&web_clients_cache.used.spinlock); web_clients_cache.used.allocated++; } @@ -127,6 +122,11 @@ struct web_client *web_client_get_from_cache(void) { } void web_client_release_to_cache(struct web_client *w) { + +#ifdef ENABLE_HTTPS + netdata_ssl_close(&w->ssl); +#endif + // unlink it from the used netdata_spinlock_lock(&web_clients_cache.used.spinlock); DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next); -- cgit v1.2.3