summaryrefslogtreecommitdiffstats
path: root/web/server/web_client.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2023-05-08 16:27:04 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2023-05-08 16:27:04 +0000
commita836a244a3d2bdd4da1ee2641e3e957850668cea (patch)
treecb87c75b3677fab7144f868435243f864048a1e6 /web/server/web_client.c
parentAdding upstream version 1.38.1. (diff)
downloadnetdata-a836a244a3d2bdd4da1ee2641e3e957850668cea.tar.xz
netdata-a836a244a3d2bdd4da1ee2641e3e957850668cea.zip
Adding upstream version 1.39.0.upstream/1.39.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'web/server/web_client.c')
-rw-r--r--web/server/web_client.c815
1 files changed, 537 insertions, 278 deletions
diff --git a/web/server/web_client.c b/web/server/web_client.c
index c14b86f3e..8bc72e71f 100644
--- a/web/server/web_client.c
+++ b/web/server/web_client.c
@@ -13,45 +13,53 @@ int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRAT
#endif /* NETDATA_WITH_ZLIB */
inline int web_client_permission_denied(struct web_client *w) {
- w->response.data->contenttype = CT_TEXT_PLAIN;
+ w->response.data->content_type = CT_TEXT_PLAIN;
buffer_flush(w->response.data);
buffer_strcat(w->response.data, "You are not allowed to access this resource.");
w->response.code = HTTP_RESP_FORBIDDEN;
return HTTP_RESP_FORBIDDEN;
}
-static inline int web_client_crock_socket(struct web_client *w) {
+static inline int web_client_crock_socket(struct web_client *w __maybe_unused) {
#ifdef TCP_CORK
if(likely(web_client_is_corkable(w) && !w->tcp_cork && w->ofd != -1)) {
- w->tcp_cork = 1;
+ w->tcp_cork = true;
if(unlikely(setsockopt(w->ofd, IPPROTO_TCP, TCP_CORK, (char *) &w->tcp_cork, sizeof(int)) != 0)) {
error("%llu: failed to enable TCP_CORK on socket.", w->id);
- w->tcp_cork = 0;
+ w->tcp_cork = false;
return -1;
}
}
-#else
- (void)w;
#endif /* TCP_CORK */
return 0;
}
-static inline int web_client_uncrock_socket(struct web_client *w) {
+static inline void web_client_enable_wait_from_ssl(struct web_client *w, int bytes) {
+ int ssl_err = SSL_get_error(w->ssl.conn, bytes);
+ if (ssl_err == SSL_ERROR_WANT_READ)
+ web_client_enable_ssl_wait_receive(w);
+ else if (ssl_err == SSL_ERROR_WANT_WRITE)
+ web_client_enable_ssl_wait_send(w);
+ else {
+ web_client_disable_ssl_wait_receive(w);
+ web_client_disable_ssl_wait_send(w);
+ }
+}
+
+static inline int web_client_uncrock_socket(struct web_client *w __maybe_unused) {
#ifdef TCP_CORK
if(likely(w->tcp_cork && w->ofd != -1)) {
- w->tcp_cork = 0;
if(unlikely(setsockopt(w->ofd, IPPROTO_TCP, TCP_CORK, (char *) &w->tcp_cork, sizeof(int)) != 0)) {
error("%llu: failed to disable TCP_CORK on socket.", w->id);
- w->tcp_cork = 1;
+ w->tcp_cork = true;
return -1;
}
}
-#else
- (void)w;
#endif /* TCP_CORK */
+ w->tcp_cork = false;
return 0;
}
@@ -67,14 +75,96 @@ char *strip_control_characters(char *url) {
return url;
}
+static void web_client_reset_allocations(struct web_client *w, bool free_all) {
+
+ if(free_all) {
+ // the web client is to be destroyed
+
+ buffer_free(w->url_as_received);
+ w->url_as_received = NULL;
+
+ buffer_free(w->url_path_decoded);
+ w->url_path_decoded = NULL;
+
+ buffer_free(w->url_query_string_decoded);
+ w->url_query_string_decoded = NULL;
+
+ buffer_free(w->response.header_output);
+ w->response.header_output = NULL;
+
+ buffer_free(w->response.header);
+ w->response.header = NULL;
+
+ buffer_free(w->response.data);
+ w->response.data = NULL;
+
+ freez(w->post_payload);
+ w->post_payload = NULL;
+ w->post_payload_size = 0;
+
+#ifdef ENABLE_HTTPS
+ if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) {
+ if (w->ssl.conn) {
+ SSL_free(w->ssl.conn);
+ w->ssl.conn = NULL;
+ }
+ }
+#endif
+ }
+ else {
+ // the web client is to be re-used
+
+ buffer_reset(w->url_as_received);
+ buffer_reset(w->url_path_decoded);
+ buffer_reset(w->url_query_string_decoded);
+
+ buffer_reset(w->response.header_output);
+ buffer_reset(w->response.header);
+ buffer_reset(w->response.data);
+
+ // leave w->post_payload
+ // leave w->ssl
+ }
+
+ freez(w->server_host);
+ w->server_host = NULL;
+
+ freez(w->forwarded_host);
+ w->forwarded_host = NULL;
+
+ freez(w->origin);
+ w->origin = NULL;
+
+ freez(w->user_agent);
+ w->user_agent = NULL;
+
+ freez(w->auth_bearer_token);
+ w->auth_bearer_token = NULL;
+
+ // if we had enabled compression, release it
+#ifdef NETDATA_WITH_ZLIB
+ if(w->response.zinitialized) {
+ deflateEnd(&w->response.zstream);
+ w->response.zsent = 0;
+ w->response.zhave = 0;
+ w->response.zstream.avail_in = 0;
+ w->response.zstream.avail_out = 0;
+ w->response.zstream.total_in = 0;
+ w->response.zstream.total_out = 0;
+ w->response.zinitialized = false;
+ w->flags &= ~WEB_CLIENT_CHUNKED_TRANSFER;
+ }
+#endif // NETDATA_WITH_ZLIB
+}
+
void web_client_request_done(struct web_client *w) {
web_client_uncrock_socket(w);
debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id);
- if(likely(w->last_url[0])) {
+ if(likely(buffer_strlen(w->url_as_received))) {
struct timeval tv;
- now_realtime_timeval(&tv);
+ now_monotonic_high_precision_timeval(&tv);
size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len;
size_t sent = size;
@@ -85,14 +175,14 @@ void web_client_request_done(struct web_client *w) {
// --------------------------------------------------------------------
// global statistics
- global_statistics_web_request_completed(dt_usec(&tv, &w->tv_in),
- w->stats_received_bytes,
- w->stats_sent_bytes,
+ global_statistics_web_request_completed(dt_usec(&tv, &w->timings.tv_in),
+ w->statistics.received_bytes,
+ w->statistics.sent_bytes,
size,
sent);
- w->stats_received_bytes = 0;
- w->stats_sent_bytes = 0;
+ w->statistics.received_bytes = 0;
+ w->statistics.sent_bytes = 0;
// --------------------------------------------------------------------
@@ -111,7 +201,8 @@ void web_client_request_done(struct web_client *w) {
mode = "STREAM";
break;
- case WEB_CLIENT_MODE_NORMAL:
+ case WEB_CLIENT_MODE_POST:
+ case WEB_CLIENT_MODE_GET:
mode = "DATA";
break;
@@ -130,11 +221,11 @@ void web_client_request_done(struct web_client *w) {
, sent
, size
, -((size > 0) ? ((double)(size - sent) / (double) size * 100.0) : 0.0)
- , (double)dt_usec(&w->tv_ready, &w->tv_in) / 1000.0
- , (double)dt_usec(&tv, &w->tv_ready) / 1000.0
- , (double)dt_usec(&tv, &w->tv_in) / 1000.0
+ , (double)dt_usec(&w->timings.tv_ready, &w->timings.tv_in) / 1000.0
+ , (double)dt_usec(&tv, &w->timings.tv_ready) / 1000.0
+ , (double)dt_usec(&tv, &w->timings.tv_in) / 1000.0
, w->response.code
- , strip_control_characters(w->last_url)
+ , strip_control_characters((char *)buffer_tostring(w->url_as_received))
);
}
@@ -152,32 +243,13 @@ void web_client_request_done(struct web_client *w) {
}
}
- w->last_url[0] = '\0';
- w->cookie1[0] = '\0';
- w->cookie2[0] = '\0';
- w->origin[0] = '*';
- w->origin[1] = '\0';
+ web_client_reset_allocations(w, false);
- freez(w->user_agent); w->user_agent = NULL;
- if (w->auth_bearer_token) {
- freez(w->auth_bearer_token);
- w->auth_bearer_token = NULL;
- }
+ w->mode = WEB_CLIENT_MODE_GET;
- w->mode = WEB_CLIENT_MODE_NORMAL;
-
- w->tcp_cork = 0;
web_client_disable_donottrack(w);
web_client_disable_tracking_required(w);
web_client_disable_keepalive(w);
- w->decoded_url[0] = '\0';
-
- buffer_reset(w->response.header_output);
- buffer_reset(w->response.header);
- buffer_reset(w->response.data);
- w->response.rlen = 0;
- w->response.sent = 0;
- w->response.code = 0;
w->header_parse_tries = 0;
w->header_parse_last_size = 0;
@@ -185,23 +257,11 @@ void web_client_request_done(struct web_client *w) {
web_client_enable_wait_receive(w);
web_client_disable_wait_send(w);
- w->response.zoutput = 0;
-
- // if we had enabled compression, release it
-#ifdef NETDATA_WITH_ZLIB
- if(w->response.zinitialized) {
- debug(D_DEFLATE, "%llu: Freeing compression resources.", w->id);
- deflateEnd(&w->response.zstream);
- w->response.zsent = 0;
- w->response.zhave = 0;
- w->response.zstream.avail_in = 0;
- w->response.zstream.avail_out = 0;
- w->response.zstream.total_in = 0;
- w->response.zstream.total_out = 0;
- w->response.zinitialized = 0;
- w->flags &= ~WEB_CLIENT_CHUNKED_TRANSFER;
- }
-#endif // NETDATA_WITH_ZLIB
+ w->response.has_cookies = false;
+ w->response.rlen = 0;
+ w->response.sent = 0;
+ w->response.code = 0;
+ w->response.zoutput = false;
}
static struct {
@@ -273,7 +333,7 @@ static inline uint8_t contenttype_for_filename(const char *filename) {
}
static inline int access_to_file_is_not_permitted(struct web_client *w, const char *filename) {
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data, "Access to file is not permitted: ");
buffer_strcat_htmlescape(w->response.data, filename);
return HTTP_RESP_FORBIDDEN;
@@ -295,7 +355,7 @@ int mysendfile(struct web_client *w, char *filename) {
for(s = filename; *s ;s++) {
if( !isalnum(*s) && *s != '/' && *s != '.' && *s != '-' && *s != '_') {
debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_sprintf(w->response.data, "Filename contains invalid characters: ");
buffer_strcat_htmlescape(w->response.data, filename);
return HTTP_RESP_BAD_REQUEST;
@@ -305,7 +365,7 @@ int mysendfile(struct web_client *w, char *filename) {
// if the filename contains a double dot refuse to serve it
if(strstr(filename, "..") != 0) {
debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data, "Relative filenames are not supported: ");
buffer_strcat_htmlescape(w->response.data, filename);
return HTTP_RESP_BAD_REQUEST;
@@ -321,7 +381,7 @@ int mysendfile(struct web_client *w, char *filename) {
// check if the file exists
if (lstat(webfilename, &statbuf) != 0) {
debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not found.", w->id, webfilename);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data, "File does not exist, or is not accessible: ");
buffer_strcat_htmlescape(w->response.data, webfilename);
return HTTP_RESP_NOT_FOUND;
@@ -347,7 +407,7 @@ int mysendfile(struct web_client *w, char *filename) {
if(errno == EBUSY || errno == EAGAIN) {
error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, webfilename);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_sprintf(w->response.header, "Location: /%s\r\n", filename);
buffer_strcat(w->response.data, "File is currently busy, please try again later: ");
buffer_strcat_htmlescape(w->response.data, webfilename);
@@ -355,7 +415,7 @@ int mysendfile(struct web_client *w, char *filename) {
}
else {
error("%llu: Cannot open file '%s'.", w->id, webfilename);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data, "Cannot open file: ");
buffer_strcat_htmlescape(w->response.data, webfilename);
return HTTP_RESP_NOT_FOUND;
@@ -364,7 +424,7 @@ int mysendfile(struct web_client *w, char *filename) {
sock_setnonblock(w->ifd);
- w->response.data->contenttype = contenttype_for_filename(webfilename);
+ w->response.data->content_type = contenttype_for_filename(webfilename);
debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%"PRId64" bytes, ifd %d, ofd %d).", w->id, webfilename, (int64_t)statbuf.st_size, w->ifd, w->ofd);
w->mode = WEB_CLIENT_MODE_FILECOPY;
@@ -426,8 +486,8 @@ void web_client_enable_deflate(struct web_client *w, int gzip) {
}
w->response.zsent = 0;
- w->response.zoutput = 1;
- w->response.zinitialized = 1;
+ w->response.zoutput = true;
+ w->response.zinitialized = true;
w->flags |= WEB_CLIENT_CHUNKED_TRANSFER;
debug(D_DEFLATE, "%llu: Initialized compression.", w->id);
@@ -527,17 +587,19 @@ static inline int UNUSED_FUNCTION(check_host_and_mgmt_acl_and_call)(RRDHOST *hos
return check_host_and_call(host, w, url, func);
}
-int web_client_api_request(RRDHOST *host, struct web_client *w, char *url)
+int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_fragment)
{
// get the api version
- char *tok = mystrsep(&url, "/");
+ char *tok = strsep_skip_consecutive_separators(&url_path_fragment, "/");
if(tok && *tok) {
debug(D_WEB_CLIENT, "%llu: Searching for API version '%s'.", w->id, tok);
- if(strcmp(tok, "v1") == 0)
- return web_client_api_request_v1(host, w, url);
+ if(strcmp(tok, "v2") == 0)
+ return web_client_api_request_v2(host, w, url_path_fragment);
+ else if(strcmp(tok, "v1") == 0)
+ return web_client_api_request_v1(host, w, url_path_fragment);
else {
buffer_flush(w->response.data);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data, "Unsupported API version: ");
buffer_strcat_htmlescape(w->response.data, tok);
return HTTP_RESP_NOT_FOUND;
@@ -550,8 +612,8 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url)
}
}
-const char *web_content_type_to_string(uint8_t contenttype) {
- switch(contenttype) {
+const char *web_content_type_to_string(HTTP_CONTENT_TYPE content_type) {
+ switch(content_type) {
case CT_TEXT_HTML:
return "text/html; charset=utf-8";
@@ -715,7 +777,7 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
uint32_t hash = simple_uhash(s);
if(hash == hash_origin && !strcasecmp(s, "Origin"))
- strncpyz(w->origin, v, NETDATA_WEB_REQUEST_ORIGIN_HEADER_SIZE);
+ w->origin = strdupz(v);
else if(hash == hash_connection && !strcasecmp(s, "Connection")) {
if(strcasestr(v, "keep-alive"))
@@ -727,11 +789,14 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
}
else if(parse_useragent && hash == hash_useragent && !strcasecmp(s, "User-Agent")) {
w->user_agent = strdupz(v);
- } else if(hash == hash_authorization&& !strcasecmp(s, "X-Auth-Token")) {
+ }
+ else if(hash == hash_authorization&& !strcasecmp(s, "X-Auth-Token")) {
w->auth_bearer_token = strdupz(v);
}
- else if(hash == hash_host && !strcasecmp(s, "Host")){
- strncpyz(w->server_host, v, ((size_t)(ve - v) < sizeof(w->server_host)-1 ? (size_t)(ve - v) : sizeof(w->server_host)-1));
+ else if(hash == hash_host && !strcasecmp(s, "Host")) {
+ char buffer[NI_MAXHOST];
+ strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
+ w->server_host = strdupz(buffer);
}
#ifdef NETDATA_WITH_ZLIB
else if(hash == hash_accept_encoding && !strcasecmp(s, "Accept-Encoding")) {
@@ -751,8 +816,10 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
w->ssl.flags |= NETDATA_SSL_PROXY_HTTPS;
}
#endif
- else if(hash == hash_forwarded_host && !strcasecmp(s, "X-Forwarded-Host")){
- strncpyz(w->forwarded_host, v, ((size_t)(ve - v) < sizeof(w->server_host)-1 ? (size_t)(ve - v) : sizeof(w->server_host)-1));
+ else if(hash == hash_forwarded_host && !strcasecmp(s, "X-Forwarded-Host")) {
+ char buffer[NI_MAXHOST];
+ strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
+ w->forwarded_host = strdupz(buffer);
}
*e = ':';
@@ -774,12 +841,16 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
// is is a valid request?
if(!strncmp(s, "GET ", 4)) {
s = &s[4];
- w->mode = WEB_CLIENT_MODE_NORMAL;
+ w->mode = WEB_CLIENT_MODE_GET;
}
else if(!strncmp(s, "OPTIONS ", 8)) {
s = &s[8];
w->mode = WEB_CLIENT_MODE_OPTIONS;
}
+ else if(!strncmp(s, "POST ", 5)) {
+ s = &s[5];
+ w->mode = WEB_CLIENT_MODE_POST;
+ }
else if(!strncmp(s, "STREAM ", 7)) {
s = &s[7];
@@ -823,63 +894,6 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
}
/**
- * Set Path Query
- *
- * Set the pointers to the path and query string according to the input.
- *
- * @param w is the structure with the client request
- * @param s is the first address of the string.
- * @param ptr is the address of the separator.
- */
-static void web_client_set_path_query(struct web_client *w, const char *s, char *ptr) {
- w->url_path_length = (size_t)(ptr -s);
- w->url_search_path = ptr;
-}
-
-/**
- * Split path query
- *
- * Do the separation between path and query string
- *
- * @param w is the structure with the client request
- * @param s is the string to parse
- */
-void web_client_split_path_query(struct web_client *w, char *s) {
- //I am assuming here that the separator character(?) is not encoded
- char *ptr = strchr(s, '?');
- if(ptr) {
- w->separator = '?';
- web_client_set_path_query(w, s, ptr);
- return;
- }
-
- //Here I test the second possibility, the URL is completely encoded by the user.
- //I am not using the strcasestr, because it is fastest to check %3f and compare
- //the next character.
- //We executed some tests with "encodeURI(uri);" described in https://www.w3schools.com/jsref/jsref_encodeuri.asp
- //on July 1st, 2019, that show us that URLs won't have '?','=' and '&' encoded, but we decided to move in front
- //with the next part, because users can develop their own encoded that won't follow this rule.
- char *moveme = s;
- while (moveme) {
- ptr = strchr(moveme, '%');
- if(ptr) {
- char *test = (ptr+1);
- if (!strncmp(test, "3f", 2) || !strncmp(test, "3F", 2)) {
- w->separator = *ptr;
- web_client_set_path_query(w, s, ptr);
- return;
- }
- ptr++;
- }
-
- moveme = ptr;
- }
-
- w->separator = 0x00;
- w->url_path_length = strlen(s);
-}
-
-/**
* Request validate
*
* @param w is the structure with the client request
@@ -903,14 +917,14 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
if(w->header_parse_last_size < last_pos)
last_pos = 0;
- is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size);
+ is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size, &w->post_payload, &w->post_payload_size);
if(!is_it_valid) {
- if(w->header_parse_tries > 10) {
+ if(w->header_parse_tries > HTTP_REQ_MAX_HEADER_FETCH_TRIES) {
info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
w->header_parse_tries = 0;
w->header_parse_last_size = 0;
web_client_disable_wait_receive(w);
- return HTTP_VALIDATION_NOT_SUPPORTED;
+ return HTTP_VALIDATION_TOO_MANY_READ_RETRIES;
}
return HTTP_VALIDATION_INCOMPLETE;
@@ -919,7 +933,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
is_it_valid = 1;
} else {
last_pos = w->header_parse_last_size;
- is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size);
+ is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size, &w->post_payload, &w->post_payload_size);
}
s = web_client_valid_method(w, s);
@@ -938,10 +952,9 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
w->header_parse_tries = 0;
w->header_parse_last_size = 0;
web_client_disable_wait_receive(w);
- return HTTP_VALIDATION_NOT_SUPPORTED;
+ return HTTP_VALIDATION_EXCESS_REQUEST_DATA;
}
}
-
web_client_enable_wait_receive(w);
return HTTP_VALIDATION_INCOMPLETE;
}
@@ -961,10 +974,6 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
// we have the end of encoded_url - remember it
char *ue = s;
- //Variables used to map the variables in the query string case it is present
- int total_variables;
- char *ptr_variables[WEB_FIELDS_MAX];
-
// make sure we have complete request
// complete requests contain: \r\n\r\n
while(*s) {
@@ -981,50 +990,16 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
if(unlikely(*s == '\r' && s[1] == '\n')) {
// a valid complete HTTP request found
+ char c = *ue;
*ue = '\0';
- //This is to avoid crash in line
- w->url_search_path = NULL;
- if(w->mode != WEB_CLIENT_MODE_NORMAL) {
- if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
- return HTTP_VALIDATION_MALFORMED_URL;
- } else {
- web_client_split_path_query(w, encoded_url);
-
- if (w->url_search_path && w->separator) {
- *w->url_search_path = 0x00;
- }
-
- if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
- return HTTP_VALIDATION_MALFORMED_URL;
-
- if (w->url_search_path && w->separator) {
- *w->url_search_path = w->separator;
-
- char *from = (encoded_url + w->url_path_length);
- total_variables = url_map_query_string(ptr_variables, from);
+ web_client_decode_path_and_query_string(w, encoded_url);
+ *ue = c;
- if (url_parse_query_string(w->decoded_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1, ptr_variables, total_variables)) {
- return HTTP_VALIDATION_MALFORMED_URL;
- }
- } else {
- //make sure there's no leftovers from previous request on the same web client
- w->decoded_query_string[1]='\0';
- }
- }
- *ue = ' ';
-
- // copy the URL - we are going to overwrite parts of it
- // TODO -- ideally we we should avoid copying buffers around
- snprintfz(w->last_url, NETDATA_WEB_REQUEST_URL_SIZE, "%s%s", w->decoded_url, w->decoded_query_string);
#ifdef ENABLE_HTTPS
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
if ((w->ssl.conn) && ((w->ssl.flags & NETDATA_SSL_NO_HANDSHAKE) && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM)) ) {
w->header_parse_tries = 0;
w->header_parse_last_size = 0;
- // The client will be redirected for Netdata and we are preserving the original request.
- *ue = '\0';
- strncpyz(w->last_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE);
- *ue = ' ';
web_client_disable_wait_receive(w);
return HTTP_VALIDATION_REDIRECT;
}
@@ -1038,9 +1013,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
}
// another header line
- s = http_header_parse(w, s,
- (w->mode == WEB_CLIENT_MODE_STREAM) // parse user agent
- );
+ s = http_header_parse(w, s, (w->mode == WEB_CLIENT_MODE_STREAM)); // parse user agent
}
}
@@ -1056,6 +1029,7 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
if ( ( w->ssl.conn ) && ( !w->ssl.flags ) ){
bytes = netdata_ssl_write(w->ssl.conn, buf, len) ;
+ web_client_enable_wait_from_ssl(w, bytes);
} else {
bytes = send(w->ofd,buf, len , flags);
}
@@ -1076,15 +1050,15 @@ void web_client_build_http_header(struct web_client *w) {
// set a proper expiration date, if not already set
if(unlikely(!w->response.data->expires)) {
if(w->response.data->options & WB_CONTENT_NO_CACHEABLE)
- w->response.data->expires = w->tv_ready.tv_sec + localhost->rrd_update_every;
+ w->response.data->expires = w->timings.tv_ready.tv_sec + localhost->rrd_update_every;
else
- w->response.data->expires = w->tv_ready.tv_sec + 86400;
+ w->response.data->expires = w->timings.tv_ready.tv_sec + 86400;
}
// prepare the HTTP response header
debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, w->response.code);
- const char *content_type_string = web_content_type_to_string(w->response.data->contenttype);
+ const char *content_type_string = web_content_type_to_string(w->response.data->content_type);
const char *code_msg = web_response_code_to_string(w->response.code);
// prepare the last modified and expiration dates
@@ -1104,8 +1078,8 @@ void web_client_build_http_header(struct web_client *w) {
"HTTP/1.1 %d %s\r\n"
"Location: https://%s%s\r\n",
w->response.code, code_msg,
- w->server_host,
- w->last_url);
+ w->server_host ? w->server_host : "",
+ buffer_tostring(w->url_as_received));
}else {
buffer_sprintf(w->response.header_output,
"HTTP/1.1 %d %s\r\n"
@@ -1119,7 +1093,7 @@ void web_client_build_http_header(struct web_client *w) {
code_msg,
web_client_has_keepalive(w)?"keep-alive":"close",
VERSION,
- w->origin,
+ w->origin ? w->origin : "*",
content_type_string,
date);
}
@@ -1127,31 +1101,19 @@ void web_client_build_http_header(struct web_client *w) {
if(unlikely(web_x_frame_options))
buffer_sprintf(w->response.header_output, "X-Frame-Options: %s\r\n", web_x_frame_options);
- if(w->cookie1[0] || w->cookie2[0]) {
- if(w->cookie1[0]) {
- buffer_sprintf(w->response.header_output,
- "Set-Cookie: %s\r\n",
- w->cookie1);
- }
-
- if(w->cookie2[0]) {
- buffer_sprintf(w->response.header_output,
- "Set-Cookie: %s\r\n",
- w->cookie2);
- }
-
+ if(w->response.has_cookies) {
if(respect_web_browser_do_not_track_policy)
buffer_sprintf(w->response.header_output,
- "Tk: T;cookies\r\n");
+ "Tk: T;cookies\r\n");
}
else {
if(respect_web_browser_do_not_track_policy) {
if(web_client_has_tracking_required(w))
buffer_sprintf(w->response.header_output,
- "Tk: T;cookies\r\n");
+ "Tk: T;cookies\r\n");
else
buffer_sprintf(w->response.header_output,
- "Tk: N\r\n");
+ "Tk: N\r\n");
}
}
@@ -1211,8 +1173,10 @@ static inline void web_client_send_http_header(struct web_client *w) {
ssize_t bytes;
#ifdef ENABLE_HTTPS
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
- if ( ( w->ssl.conn ) && ( w->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ) )
+ if ( ( w->ssl.conn ) && ( w->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE ) ) {
bytes = netdata_ssl_write(w->ssl.conn, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output));
+ web_client_enable_wait_from_ssl(w, bytes);
+ }
else {
while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
count++;
@@ -1247,7 +1211,7 @@ static inline void web_client_send_http_header(struct web_client *w) {
if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) {
if(bytes > 0)
- w->stats_sent_bytes += bytes;
+ w->statistics.sent_bytes += bytes;
if (bytes < 0) {
@@ -1260,12 +1224,10 @@ static inline void web_client_send_http_header(struct web_client *w) {
}
}
else
- w->stats_sent_bytes += bytes;
+ w->statistics.sent_bytes += bytes;
}
-static inline int web_client_process_url(RRDHOST *host, struct web_client *w, char *url);
-
-static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, char *url) {
+static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, char *url, bool nodeid, int (*func)(RRDHOST *, struct web_client *, char *)) {
static uint32_t hash_localhost = 0;
if(unlikely(!hash_localhost)) {
@@ -1278,51 +1240,132 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
return HTTP_RESP_BAD_REQUEST;
}
- char *tok = mystrsep(&url, "/");
+ char *tok = strsep_skip_consecutive_separators(&url, "/");
if(tok && *tok) {
debug(D_WEB_CLIENT, "%llu: Searching for host with name '%s'.", w->id, tok);
- if(!url) { //no delim found
- debug(D_WEB_CLIENT, "%llu: URL doesn't end with / generating redirect.", w->id);
- char *protocol, *url_host;
+ if(nodeid) {
+ host = find_host_by_node_id(tok);
+ if(!host) {
+ host = rrdhost_find_by_hostname(tok);
+ if (!host)
+ host = rrdhost_find_by_guid(tok);
+ }
+ }
+ else {
+ host = rrdhost_find_by_hostname(tok);
+ if(!host) {
+ host = rrdhost_find_by_guid(tok);
+ if (!host)
+ host = find_host_by_node_id(tok);
+ }
+ }
+
+ if(!host) {
+ // we didn't find it, but it may be a uuid case mismatch for MACHINE_GUID
+ // so, recreate the machine guid in lower-case.
+ uuid_t uuid;
+ char txt[UUID_STR_LEN];
+ if (uuid_parse(tok, uuid) == 0) {
+ uuid_unparse_lower(uuid, txt);
+ host = rrdhost_find_by_guid(txt);
+ }
+ }
+
+ if (host) {
+ if(!url) { //no delim found
+ debug(D_WEB_CLIENT, "%llu: URL doesn't end with / generating redirect.", w->id);
+ char *protocol, *url_host;
#ifdef ENABLE_HTTPS
- protocol = ((w->ssl.conn && !w->ssl.flags) || w->ssl.flags & NETDATA_SSL_PROXY_HTTPS) ? "https" : "http";
+ protocol = ((w->ssl.conn && !w->ssl.flags) || w->ssl.flags & NETDATA_SSL_PROXY_HTTPS) ? "https" : "http";
#else
- protocol = "http";
+ protocol = "http";
#endif
- url_host = (!w->forwarded_host[0])?w->server_host:w->forwarded_host;
- buffer_sprintf(w->response.header, "Location: %s://%s%s/\r\n", protocol, url_host, w->last_url);
- buffer_strcat(w->response.data, "Permanent redirect");
- return HTTP_RESP_REDIR_PERM;
- }
- // copy the URL, we need it to serve files
- w->last_url[0] = '/';
+ url_host = w->forwarded_host;
+ if(!url_host) {
+ url_host = w->server_host;
+ if(!url_host) url_host = "";
+ }
- if(url && *url) strncpyz(&w->last_url[1], url, NETDATA_WEB_REQUEST_URL_SIZE - 1);
- else w->last_url[1] = '\0';
+ buffer_sprintf(w->response.header, "Location: %s://%s/%s/%s/%s",
+ protocol, url_host, nodeid?"node":"host", tok, buffer_tostring(w->url_path_decoded));
- host = rrdhost_find_by_hostname(tok);
- if (!host)
- host = rrdhost_find_by_guid(tok);
- if (host) return web_client_process_url(host, w, url);
+ if(buffer_strlen(w->url_query_string_decoded)) {
+ const char *query_string = buffer_tostring(w->url_query_string_decoded);
+ if(*query_string) {
+ if(*query_string != '?')
+ buffer_fast_strcat(w->response.header, "?", 1);
+ buffer_strcat(w->response.header, query_string);
+ }
+ }
+ buffer_fast_strcat(w->response.header, "\r\n", 2);
+ buffer_strcat(w->response.data, "Permanent redirect");
+ return HTTP_RESP_REDIR_PERM;
+ }
+
+ size_t len = strlen(url) + 2;
+ char buf[len];
+ buf[0] = '/';
+ strcpy(&buf[1], url);
+ buf[len - 1] = '\0';
+
+ buffer_flush(w->url_path_decoded);
+ buffer_strcat(w->url_path_decoded, buf);
+ return func(host, w, buf);
+ }
}
buffer_flush(w->response.data);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data, "This netdata does not maintain a database for host: ");
buffer_strcat_htmlescape(w->response.data, tok?tok:"");
return HTTP_RESP_NOT_FOUND;
}
-static inline int web_client_process_url(RRDHOST *host, struct web_client *w, char *url) {
+int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client *w, char *decoded_url_path) {
+ static uint32_t
+ hash_api = 0,
+ hash_host = 0,
+ hash_node = 0;
+
+ if(unlikely(!hash_api)) {
+ hash_api = simple_hash("api");
+ hash_host = simple_hash("host");
+ hash_node = simple_hash("node");
+ }
+
+ char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
+ if(likely(tok && *tok)) {
+ uint32_t hash = simple_hash(tok);
+
+ if(unlikely(hash == hash_api && strcmp(tok, "api") == 0)) {
+ // current API
+ debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
+ return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
+ }
+ else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) || (hash == hash_node && strcmp(tok, "node") == 0))) {
+ // host switching
+ debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
+ return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_api_request_with_node_selection);
+ }
+ }
+
+ buffer_flush(w->response.data);
+ buffer_strcat(w->response.data, "Unknown API endpoint.");
+ w->response.data->content_type = CT_TEXT_HTML;
+ return HTTP_RESP_NOT_FOUND;
+}
+
+static inline int web_client_process_url(RRDHOST *host, struct web_client *w, char *decoded_url_path) {
if(unlikely(!service_running(ABILITY_WEB_REQUESTS)))
return web_client_permission_denied(w);
static uint32_t
hash_api = 0,
hash_netdata_conf = 0,
- hash_host = 0;
+ hash_host = 0,
+ hash_node = 0;
#ifdef NETDATA_INTERNAL_CHECKS
static uint32_t hash_exit = 0, hash_debug = 0, hash_mirror = 0;
@@ -1332,6 +1375,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
hash_api = simple_hash("api");
hash_netdata_conf = simple_hash("netdata.conf");
hash_host = simple_hash("host");
+ hash_node = simple_hash("node");
#ifdef NETDATA_INTERNAL_CHECKS
hash_exit = simple_hash("exit");
hash_debug = simple_hash("debug");
@@ -1339,25 +1383,29 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
#endif
}
- char *tok = mystrsep(&url, "/?");
+ // keep a copy of the decoded path, in case we need to serve it as a filename
+ char filename[FILENAME_MAX + 1];
+ strncpyz(filename, buffer_tostring(w->url_path_decoded), FILENAME_MAX);
+
+ char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
if(likely(tok && *tok)) {
uint32_t hash = simple_hash(tok);
debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok);
if(unlikely(hash == hash_api && strcmp(tok, "api") == 0)) { // current API
debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
- return check_host_and_call(host, w, url, web_client_api_request);
+ return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
}
- else if(unlikely(hash == hash_host && strcmp(tok, "host") == 0)) { // host switching
+ else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) || (hash == hash_node && strcmp(tok, "node") == 0))) { // host switching
debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
- return web_client_switch_host(host, w, url);
+ return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_process_url);
}
else if(unlikely(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0)) { // netdata.conf
if(unlikely(!web_client_can_access_netdataconf(w)))
return web_client_permission_denied(w);
debug(D_WEB_CLIENT_ACCESS, "%llu: generating netdata.conf ...", w->id);
- w->response.data->contenttype = CT_TEXT_PLAIN;
+ w->response.data->content_type = CT_TEXT_PLAIN;
buffer_flush(w->response.data);
config_generate(w->response.data, 0);
return HTTP_RESP_OK;
@@ -1367,7 +1415,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
if(unlikely(!web_client_can_access_netdataconf(w)))
return web_client_permission_denied(w);
- w->response.data->contenttype = CT_TEXT_PLAIN;
+ w->response.data->content_type = CT_TEXT_PLAIN;
buffer_flush(w->response.data);
if(!netdata_exit)
@@ -1386,7 +1434,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
buffer_flush(w->response.data);
// get the name of the data to show
- tok = mystrsep(&url, "&");
+ tok = strsep_skip_consecutive_separators(&decoded_url_path, "&");
if(tok && *tok) {
debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
@@ -1394,7 +1442,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
RRDSET *st = rrdset_find_byname(host, tok);
if(!st) st = rrdset_find(host, tok);
if(!st) {
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data, "Chart is not found: ");
buffer_strcat_htmlescape(w->response.data, tok);
debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok);
@@ -1408,7 +1456,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
else
rrdset_flag_set(st, RRDSET_FLAG_DEBUG);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_sprintf(w->response.data, "Chart has now debug %s: ", rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?"enabled":"disabled");
buffer_strcat_htmlescape(w->response.data, tok);
debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?"enabled":"disabled");
@@ -1436,18 +1484,14 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
#endif /* NETDATA_INTERNAL_CHECKS */
}
- char filename[FILENAME_MAX+1];
- url = filename;
- strncpyz(filename, w->last_url, FILENAME_MAX);
- tok = mystrsep(&url, "?");
buffer_flush(w->response.data);
- return mysendfile(w, (tok && *tok)?tok:"/");
+ return mysendfile(w, filename);
}
void web_client_process_request(struct web_client *w) {
// start timing us
- now_realtime_timeval(&w->tv_in);
+ web_client_timeout_checkpoint_init(w);
switch(http_request_validate(w)) {
case HTTP_VALIDATION_OK:
@@ -1458,7 +1502,7 @@ void web_client_process_request(struct web_client *w) {
return;
}
- w->response.code = rrdpush_receiver_thread_spawn(w, w->decoded_url);
+ w->response.code = rrdpush_receiver_thread_spawn(w, (char *)buffer_tostring(w->url_query_string_decoded));
return;
case WEB_CLIENT_MODE_OPTIONS:
@@ -1473,14 +1517,15 @@ void web_client_process_request(struct web_client *w) {
break;
}
- w->response.data->contenttype = CT_TEXT_PLAIN;
+ w->response.data->content_type = CT_TEXT_PLAIN;
buffer_flush(w->response.data);
buffer_strcat(w->response.data, "OK");
w->response.code = HTTP_RESP_OK;
break;
case WEB_CLIENT_MODE_FILECOPY:
- case WEB_CLIENT_MODE_NORMAL:
+ case WEB_CLIENT_MODE_POST:
+ case WEB_CLIENT_MODE_GET:
if(unlikely(
!web_client_can_access_dashboard(w) &&
!web_client_can_access_registry(w) &&
@@ -1492,23 +1537,29 @@ void web_client_process_request(struct web_client *w) {
break;
}
- w->response.code = web_client_process_url(localhost, w, w->decoded_url);
+ w->response.code = web_client_process_url(localhost, w, (char *)buffer_tostring(w->url_path_decoded));
break;
}
break;
case HTTP_VALIDATION_INCOMPLETE:
if(w->response.data->len > NETDATA_WEB_REQUEST_MAX_SIZE) {
- strcpy(w->last_url, "too big request");
+ buffer_flush(w->url_as_received);
+ buffer_strcat(w->url_as_received, "too big request");
debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, w->response.data->len);
+ size_t len = w->response.data->len;
buffer_flush(w->response.data);
- buffer_sprintf(w->response.data, "Received request is too big (%zu bytes).\r\n", w->response.data->len);
+ buffer_sprintf(w->response.data, "Received request is too big (received %zu bytes, max is %zu bytes).\r\n", len, (size_t)NETDATA_WEB_REQUEST_MAX_SIZE);
w->response.code = HTTP_RESP_BAD_REQUEST;
}
else {
// wait for more data
+ // set to normal to prevent web_server_rcv_callback
+ // from going into stream mode
+ if (w->mode == WEB_CLIENT_MODE_STREAM)
+ w->mode = WEB_CLIENT_MODE_GET;
return;
}
break;
@@ -1516,7 +1567,7 @@ void web_client_process_request(struct web_client *w) {
case HTTP_VALIDATION_REDIRECT:
{
buffer_flush(w->response.data);
- w->response.data->contenttype = CT_TEXT_HTML;
+ w->response.data->content_type = CT_TEXT_HTML;
buffer_strcat(w->response.data,
"<!DOCTYPE html><!-- SPDX-License-Identifier: GPL-3.0-or-later --><html>"
"<body onload=\"window.location.href ='https://'+ window.location.hostname +"
@@ -1530,29 +1581,43 @@ void web_client_process_request(struct web_client *w) {
}
#endif
case HTTP_VALIDATION_MALFORMED_URL:
- debug(D_WEB_CLIENT_ACCESS, "%llu: URL parsing failed (malformed URL). Cannot understand '%s'.", w->id, w->response.data->buffer);
+ debug(D_WEB_CLIENT_ACCESS, "%llu: Malformed URL '%s'.", w->id, w->response.data->buffer);
+
+ buffer_flush(w->response.data);
+ buffer_strcat(w->response.data, "Malformed URL...\r\n");
+ w->response.code = HTTP_RESP_BAD_REQUEST;
+ break;
+ case HTTP_VALIDATION_EXCESS_REQUEST_DATA:
+ debug(D_WEB_CLIENT_ACCESS, "%llu: Excess data in request '%s'.", w->id, w->response.data->buffer);
+
+ buffer_flush(w->response.data);
+ buffer_strcat(w->response.data, "Excess data in request.\r\n");
+ w->response.code = HTTP_RESP_BAD_REQUEST;
+ break;
+ case HTTP_VALIDATION_TOO_MANY_READ_RETRIES:
+ debug(D_WEB_CLIENT_ACCESS, "%llu: Too many retries to read request '%s'.", w->id, w->response.data->buffer);
buffer_flush(w->response.data);
- buffer_strcat(w->response.data, "URL not valid. I don't understand you...\r\n");
+ buffer_strcat(w->response.data, "Too many retries to read request.\r\n");
w->response.code = HTTP_RESP_BAD_REQUEST;
break;
case HTTP_VALIDATION_NOT_SUPPORTED:
- debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, w->response.data->buffer);
+ debug(D_WEB_CLIENT_ACCESS, "%llu: HTTP method requested is not supported '%s'.", w->id, w->response.data->buffer);
buffer_flush(w->response.data);
- buffer_strcat(w->response.data, "I don't understand you...\r\n");
+ buffer_strcat(w->response.data, "HTTP method requested is not supported...\r\n");
w->response.code = HTTP_RESP_BAD_REQUEST;
break;
}
// keep track of the processing time
- now_realtime_timeval(&w->tv_ready);
+ web_client_timeout_checkpoint_response_ready(w, NULL);
w->response.sent = 0;
// set a proper last modified date
if(unlikely(!w->response.data->date))
- w->response.data->date = w->tv_ready.tv_sec;
+ w->response.data->date = w->timings.tv_ready.tv_sec;
web_client_send_http_header(w);
@@ -1569,7 +1634,8 @@ void web_client_process_request(struct web_client *w) {
debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
break;
- case WEB_CLIENT_MODE_NORMAL:
+ case WEB_CLIENT_MODE_POST:
+ case WEB_CLIENT_MODE_GET:
debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
break;
@@ -1612,7 +1678,7 @@ ssize_t web_client_send_chunk_header(struct web_client *w, size_t len)
bytes = web_client_send_data(w,buf,strlen(buf),0);
if(bytes > 0) {
debug(D_DEFLATE, "%llu: Sent chunk header %zd bytes.", w->id, bytes);
- w->stats_sent_bytes += bytes;
+ w->statistics.sent_bytes += bytes;
}
else if(bytes == 0) {
@@ -1634,7 +1700,7 @@ ssize_t web_client_send_chunk_close(struct web_client *w)
bytes = web_client_send_data(w,"\r\n",2,0);
if(bytes > 0) {
debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
- w->stats_sent_bytes += bytes;
+ w->statistics.sent_bytes += bytes;
}
else if(bytes == 0) {
@@ -1656,7 +1722,7 @@ ssize_t web_client_send_chunk_finalize(struct web_client *w)
bytes = web_client_send_data(w,"\r\n0\r\n\r\n",7,0);
if(bytes > 0) {
debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
- w->stats_sent_bytes += bytes;
+ w->statistics.sent_bytes += bytes;
}
else if(bytes == 0) {
@@ -1734,7 +1800,7 @@ ssize_t web_client_send_deflate(struct web_client *w)
// ask for FINISH if we have all the input
int flush = Z_SYNC_FLUSH;
- if(w->mode == WEB_CLIENT_MODE_NORMAL
+ if((w->mode == WEB_CLIENT_MODE_GET || w->mode == WEB_CLIENT_MODE_POST)
|| (w->mode == WEB_CLIENT_MODE_FILECOPY && !web_client_has_wait_receive(w) && w->response.data->len == w->response.rlen)) {
flush = Z_FINISH;
debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id);
@@ -1768,7 +1834,7 @@ ssize_t web_client_send_deflate(struct web_client *w)
len = web_client_send_data(w,&w->response.zbuffer[w->response.zsent], (size_t) (w->response.zhave - w->response.zsent), MSG_DONTWAIT);
if(len > 0) {
- w->stats_sent_bytes += len;
+ w->statistics.sent_bytes += len;
w->response.zsent += len;
len += t;
debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, len);
@@ -1823,7 +1889,7 @@ ssize_t web_client_send(struct web_client *w) {
bytes = web_client_send_data(w,&w->response.data->buffer[w->response.sent], w->response.data->len - w->response.sent, MSG_DONTWAIT);
if(likely(bytes > 0)) {
- w->stats_sent_bytes += bytes;
+ w->statistics.sent_bytes += bytes;
w->response.sent += bytes;
debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, bytes);
}
@@ -1899,12 +1965,13 @@ ssize_t web_client_receive(struct web_client *w)
ssize_t left = (ssize_t)(w->response.data->size - w->response.data->len);
// do we have any space for more data?
- buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_RECEIVE_SIZE);
+ buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE);
#ifdef ENABLE_HTTPS
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
if ( ( w->ssl.conn ) && (!w->ssl.flags)) {
bytes = netdata_ssl_read(w->ssl.conn, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
+ web_client_enable_wait_from_ssl(w, bytes);
}else {
bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
}
@@ -1917,7 +1984,7 @@ ssize_t web_client_receive(struct web_client *w)
#endif
if(likely(bytes > 0)) {
- w->stats_received_bytes += bytes;
+ w->statistics.received_bytes += bytes;
size_t old = w->response.data->len;
(void)old;
@@ -1957,3 +2024,195 @@ int web_client_socket_is_now_used_for_streaming(struct web_client *w) {
return HTTP_RESP_OK;
}
+
+void web_client_decode_path_and_query_string(struct web_client *w, const char *path_and_query_string) {
+ char buffer[NETDATA_WEB_REQUEST_URL_SIZE + 2];
+ buffer[0] = '\0';
+
+ buffer_flush(w->url_path_decoded);
+ buffer_flush(w->url_query_string_decoded);
+
+ if(buffer_strlen(w->url_as_received) == 0)
+ // do not overwrite this if it is already filled
+ buffer_strcat(w->url_as_received, path_and_query_string);
+
+ if(w->mode == WEB_CLIENT_MODE_STREAM) {
+ // in stream mode, there is no path
+
+ url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
+
+ buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
+ buffer_strcat(w->url_query_string_decoded, buffer);
+ }
+ else {
+ // in non-stream mode, there is a path
+
+ // FIXME - the way this is implemented, query string params never accept the symbol &, not even encoded as %26
+ // To support the symbol & in query string params, we need to turn the url_query_string_decoded into a
+ // dictionary and decode each of the parameters individually.
+ // OR: in url_query_string_decoded use as separator a control character that cannot appear in the URL.
+
+ char *question_mark_start = strchr(path_and_query_string, '?');
+ if (question_mark_start)
+ url_decode_r(buffer, question_mark_start, NETDATA_WEB_REQUEST_URL_SIZE + 1);
+
+ buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
+ buffer_strcat(w->url_query_string_decoded, buffer);
+
+ if (question_mark_start) {
+ char c = *question_mark_start;
+ *question_mark_start = '\0';
+ url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
+ *question_mark_start = c;
+ } else
+ url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
+
+ buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
+ buffer_strcat(w->url_path_decoded, buffer);
+ }
+}
+
+#ifdef ENABLE_HTTPS
+void web_client_reuse_ssl(struct web_client *w) {
+ if (netdata_ssl_srv_ctx) {
+ if (w->ssl.conn) {
+ SSL_SESSION *session = SSL_get_session(w->ssl.conn);
+ SSL *old = w->ssl.conn;
+ w->ssl.conn = SSL_new(netdata_ssl_srv_ctx);
+ if (session) {
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_111
+ if (SSL_SESSION_is_resumable(session))
+#endif
+ SSL_set_session(w->ssl.conn, session);
+ }
+ SSL_free(old);
+ }
+ }
+}
+#endif
+
+void web_client_zero(struct web_client *w) {
+ // zero everything about it - but keep the buffers
+
+ web_client_reset_allocations(w, false);
+
+ // remember the pointers to the buffers
+ BUFFER *b1 = w->response.data;
+ BUFFER *b2 = w->response.header;
+ BUFFER *b3 = w->response.header_output;
+ BUFFER *b4 = w->url_path_decoded;
+ BUFFER *b5 = w->url_as_received;
+ BUFFER *b6 = w->url_query_string_decoded;
+
+#ifdef ENABLE_HTTPS
+ web_client_reuse_ssl(w);
+ SSL *ssl = w->ssl.conn;
+#endif
+
+ size_t use_count = w->use_count;
+ size_t *statistics_memory_accounting = w->statistics.memory_accounting;
+
+ // zero everything
+ memset(w, 0, sizeof(struct web_client));
+
+ w->ifd = w->ofd = -1;
+ w->statistics.memory_accounting = statistics_memory_accounting;
+ w->use_count = use_count;
+
+#ifdef ENABLE_HTTPS
+ w->ssl.conn = ssl;
+ w->ssl.flags = NETDATA_SSL_START;
+ debug(D_WEB_CLIENT_ACCESS,"Reusing SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
+#endif
+
+ // restore the pointers of the buffers
+ w->response.data = b1;
+ w->response.header = b2;
+ w->response.header_output = b3;
+ w->url_path_decoded = b4;
+ w->url_as_received = b5;
+ w->url_query_string_decoded = b6;
+}
+
+struct web_client *web_client_create(size_t *statistics_memory_accounting) {
+ struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
+ w->use_count = 1;
+ w->statistics.memory_accounting = statistics_memory_accounting;
+
+ w->url_as_received = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
+ w->url_path_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
+ w->url_query_string_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
+ w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, w->statistics.memory_accounting);
+ w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
+ w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
+
+ __atomic_add_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
+
+ return w;
+}
+
+void web_client_free(struct web_client *w) {
+ web_client_reset_allocations(w, true);
+
+ __atomic_sub_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
+ freez(w);
+}
+
+inline void web_client_timeout_checkpoint_init(struct web_client *w) {
+ now_monotonic_high_precision_timeval(&w->timings.tv_in);
+}
+
+inline void web_client_timeout_checkpoint_set(struct web_client *w, int timeout_ms) {
+ w->timings.timeout_ut = timeout_ms * USEC_PER_MS;
+
+ if(!w->timings.tv_in.tv_sec)
+ web_client_timeout_checkpoint_init(w);
+
+ if(!w->timings.tv_timeout_last_checkpoint.tv_sec)
+ w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
+}
+
+inline usec_t web_client_timeout_checkpoint(struct web_client *w) {
+ struct timeval now;
+ now_monotonic_high_precision_timeval(&now);
+
+ if (!w->timings.tv_timeout_last_checkpoint.tv_sec)
+ w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
+
+ usec_t since_last_check_ut = dt_usec(&w->timings.tv_timeout_last_checkpoint, &now);
+
+ w->timings.tv_timeout_last_checkpoint = now;
+
+ return since_last_check_ut;
+}
+
+inline usec_t web_client_timeout_checkpoint_response_ready(struct web_client *w, usec_t *usec_since_last_checkpoint) {
+ usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
+ if(usec_since_last_checkpoint)
+ *usec_since_last_checkpoint = since_last_check_ut;
+
+ w->timings.tv_ready = w->timings.tv_timeout_last_checkpoint;
+
+ // return the total time of the query
+ return dt_usec(&w->timings.tv_in, &w->timings.tv_ready);
+}
+
+inline bool web_client_timeout_checkpoint_and_check(struct web_client *w, usec_t *usec_since_last_checkpoint) {
+
+ usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
+ if(usec_since_last_checkpoint)
+ *usec_since_last_checkpoint = since_last_check_ut;
+
+ if(!w->timings.timeout_ut)
+ return false;
+
+ usec_t since_reception_ut = dt_usec(&w->timings.tv_in, &w->timings.tv_timeout_last_checkpoint);
+ if (since_reception_ut >= w->timings.timeout_ut) {
+ buffer_flush(w->response.data);
+ buffer_strcat(w->response.data, "Query timeout exceeded");
+ w->response.code = HTTP_RESP_BACKEND_FETCH_FAILED;
+ return true;
+ }
+
+ return false;
+}