diff options
Diffstat (limited to 'src/web/server')
| -rw-r--r-- | src/web/server/README.md | 203 | ||||
| -rw-r--r-- | src/web/server/h2o/http_server.c | 27 | ||||
| -rw-r--r-- | src/web/server/h2o/rrdpush.c (renamed from src/web/server/h2o/streaming.c) | 2 | ||||
| -rw-r--r-- | src/web/server/h2o/streaming.h | 2 | ||||
| -rw-r--r-- | src/web/server/static/README.md | 10 | ||||
| -rw-r--r-- | src/web/server/static/static-threaded.c | 9 | ||||
| -rw-r--r-- | src/web/server/web_client.c | 163 | ||||
| -rw-r--r-- | src/web/server/web_client.h | 30 | ||||
| -rw-r--r-- | src/web/server/web_client_cache.c | 3 | ||||
| -rw-r--r-- | src/web/server/web_server.c | 4 |
10 files changed, 195 insertions, 258 deletions
diff --git a/src/web/server/README.md b/src/web/server/README.md index e4064087..4052ee2b 100644 --- a/src/web/server/README.md +++ b/src/web/server/README.md @@ -1,6 +1,6 @@ # Web server -The Netdata web server is `static-threaded`, with a fixed, configurable number of threads. +The Netdata web server is `static-threaded`, with a fixed, configurable number of threads. All the threads are concurrently listening for web requests on the same sockets, and the kernel distributes the incoming requests to them. Each thread uses non-blocking I/O so it can serve any number of web requests in parallel. @@ -9,53 +9,48 @@ This web server respects the `keep-alive` HTTP header to serve multiple HTTP req ## Configuration -From within your Netdata config directory (typically `/etc/netdata`), [use `edit-config`](/docs/netdata-agent/configuration/README.md) to -open `netdata.conf`. - -``` -sudo ./edit-config netdata.conf -``` +Edit `netdata.conf` using the [`edit-config` script](/docs/netdata-agent/configuration/README.md) Scroll down to the `[web]` section to find the following settings. ## Settings -| Setting | Default | Description | -|:-------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ssl key` | `/etc/netdata/ssl/key.pem` | Declare the location of an SSL key to [enable HTTPS](#enable-httpstls-support). | -| `ssl certificate` | `/etc/netdata/ssl/cert.pem` | Declare the location of an SSL certificate to [enable HTTPS](#enable-httpstls-support). | -| `tls version` | `1.3` | Choose which TLS version to use. While all versions are allowed (`1` or `1.0`, `1.1`, `1.2` and `1.3`), we recommend `1.3` for the most secure encryption. If left blank, Netdata uses the highest available protocol version on your system. | -| `tls ciphers` | `none` | Choose which TLS cipher to use. Options include `TLS_AES_256_GCM_SHA384`, `TLS_CHACHA20_POLY1305_SHA256`, and `TLS_AES_128_GCM_SHA256`. If left blank, Netdata uses the default cipher list for that protocol provided by your TLS implementation. | -| `ses max window` | `15` | See [single exponential smoothing](/src/web/api/queries/ses/README.md). | -| `des max window` | `15` | See [double exponential smoothing](/src/web/api/queries/des/README.md). | -| `mode` | `static-threaded` | Turns on (`static-threaded` or off (`none`) the static-threaded web server. See the [example](#disable-the-web-server) to turn off the web server and disable the dashboard. | -| `listen backlog` | `4096` | The port backlog. Check `man 2 listen`. | -| `default port` | `19999` | The listen port for the static web server. | -| `web files owner` | `netdata` | The user that owns the web static files. Netdata will refuse to serve a file that is not owned by this user, even if it has read access to that file. If the user given is not found, Netdata will only serve files owned by user given in `run as user`. | -| `web files group` | `netdata` | If this is set, Netdata will check if the file is owned by this group and refuse to serve the file if it's not. | -| `disconnect idle clients after seconds` | `60` | The time in seconds to disconnect web clients after being totally idle. | -| `timeout for first request` | `60` | How long to wait for a client to send a request before closing the socket. Prevents slow request attacks. | -| `accept a streaming request every seconds` | `0` | Can be used to set a limit on how often a parent node will accept streaming requests from child nodes in a [streaming and replication setup](/src/streaming/README.md). | -| `respect do not track policy` | `no` | If set to `yes`, Netdata will respect the user's browser preferences for [Do Not Track](https://www.eff.org/issues/do-not-track) (DNT) and storing cookies. If DNT is _enabled_ in the browser, and this option is set to `yes`, nodes will not connect to any [registry](/src/registry/README.md). For certain browsers, users must disable DNT and change this option to `yes` for full functionality. | -| `x-frame-options response header` | ` ` | Avoid [clickjacking attacks](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options), by ensuring that the content is not embedded into other sites. | -| `allow connections from` | `localhost *` | Declare which IP addresses or full-qualified domain names (FQDNs) are allowed to connect to the web server, including the [dashboard](/docs/dashboards-and-charts/README.md) or [HTTP API](/src/web/api/README.md). This is a global setting with higher priority to any of the ones below. | -| `allow connections by dns` | `heuristic` | See the [access list examples](#access-lists) for details on using `allow` settings. | -| `allow dashboard from` | `localhost *` | | -| `allow dashboard by dns` | `heuristic` | | -| `allow badges from` | `*` | | -| `allow badges by dns` | `heuristic` | | -| `allow streaming from` | `*` | | -| `allow streaming by dns` | `heuristic` | | -| `allow netdata.conf` | `localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.* 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.* 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.* 172.31.* UNKNOWN` | | -| `allow netdata.conf by dns` | `no` | | -| `allow management from` | `localhost` | | -| `allow management by dns` | `heuristic` | | -| `enable gzip compression` | `yes` | When set to `yes`, Netdata web responses will be GZIP compressed, if the web client accepts such responses. | -| `gzip compression strategy` | `default` | Valid settings are `default`, `filtered`, `huffman only`, `rle` and `fixed`. | -| `gzip compression level` | `3` | Valid settings are 1 (fastest) to 9 (best ratio). | -| `web server threads` | ` ` | How many processor threads the web server is allowed. The default is system-specific, the minimum of `6` or the number of CPU cores. | -| `web server max sockets` | ` ` | Available sockets. The default is system-specific, automatically adjusted to 50% of the max number of open files Netdata is allowed to use (via `/etc/security/limits.conf` or systemd), to allow enough file descriptors to be available for data collection. | -| `custom dashboard_info.js` | ` ` | Specifies the location of a custom `dashboard.js` file. See [customizing the standard dashboard](/docs/developer-and-contributor-corner/customize.md#customize-the-standard-dashboard) for details. | +| Setting | Default | Description | +|:-----------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ssl key` | `/etc/netdata/ssl/key.pem` | Declare the location of an SSL key to [enable HTTPS](#enable-httpstls-support). | +| `ssl certificate` | `/etc/netdata/ssl/cert.pem` | Declare the location of an SSL certificate to [enable HTTPS](#enable-httpstls-support). | +| `tls version` | `1.3` | Choose which TLS version to use. While all versions are allowed (`1` or `1.0`, `1.1`, `1.2` and `1.3`), we recommend `1.3` for the most secure encryption. If left blank, Netdata uses the highest available protocol version on your system. | +| `tls ciphers` | `none` | Choose which TLS cipher to use. Options include `TLS_AES_256_GCM_SHA384`, `TLS_CHACHA20_POLY1305_SHA256`, and `TLS_AES_128_GCM_SHA256`. If left blank, Netdata uses the default cipher list for that protocol provided by your TLS implementation. | +| `ses max window` | `15` | See [single exponential smoothing](/src/web/api/queries/ses/README.md). | +| `des max window` | `15` | See [double exponential smoothing](/src/web/api/queries/des/README.md). | +| `mode` | `static-threaded` | Turns on (`static-threaded` or off (`none`) the static-threaded web server. See the [example](#disable-the-web-server) to turn off the web server and disable the dashboard. | +| `listen backlog` | `4096` | The port backlog. Check `man 2 listen`. | +| `default port` | `19999` | The listen port for the static web server. | +| `web files owner` | `netdata` | The user that owns the web static files. Netdata will refuse to serve a file that is not owned by this user, even if it has read access to that file. If the user given is not found, Netdata will only serve files owned by user given in `run as user`. | +| `web files group` | `netdata` | If this is set, Netdata will check if the file is owned by this group and refuse to serve the file if it's not. | +| `disconnect idle clients after` | `1m` | The time in seconds to disconnect web clients after being totally idle. | +| `timeout for first request` | `1m` | How long to wait for a client to send a request before closing the socket. Prevents slow request attacks. | +| `accept a streaming request every` | `off` | Can be used to set a limit on how often a parent node will accept streaming requests from child nodes in a [streaming and replication setup](/src/streaming/README.md). | +| `respect do not track policy` | `no` | If set to `yes`, Netdata will respect the user's browser preferences for [Do Not Track](https://www.eff.org/issues/do-not-track) (DNT) and storing cookies. If DNT is _enabled_ in the browser, and this option is set to `yes`, nodes will not connect to any [registry](/src/registry/README.md). For certain browsers, users must disable DNT and change this option to `yes` for full functionality. | +| `x-frame-options response header` | `` | Avoid [clickjacking attacks](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options), by ensuring that the content is not embedded into other sites. | +| `allow connections from` | `localhost *` | Declare which IP addresses or full-qualified domain names (FQDNs) are allowed to connect to the web server, including the [dashboard](/docs/dashboards-and-charts/README.md) or [HTTP API](/src/web/api/README.md). This is a global setting with higher priority to any of the ones below. | +| `allow connections by dns` | `heuristic` | See the [access list examples](#access-lists) for details on using `allow` settings. | +| `allow dashboard from` | `localhost *` | | +| `allow dashboard by dns` | `heuristic` | | +| `allow badges from` | `*` | | +| `allow badges by dns` | `heuristic` | | +| `allow streaming from` | `*` | | +| `allow streaming by dns` | `heuristic` | | +| `allow netdata.conf` | `localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.* 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.* 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.* 172.31.* UNKNOWN` | | +| `allow netdata.conf by dns` | `no` | | +| `allow management from` | `localhost` | | +| `allow management by dns` | `heuristic` | | +| `enable gzip compression` | `yes` | When set to `yes`, Netdata web responses will be GZIP compressed, if the web client accepts such responses. | +| `gzip compression strategy` | `default` | Valid settings are `default`, `filtered`, `huffman only`, `rle` and `fixed`. | +| `gzip compression level` | `3` | Valid settings are 1 (fastest) to 9 (best ratio). | +| `web server threads` | `` | How many processor threads the web server is allowed. The default is system-specific, the minimum of `6` or the number of CPU cores. | +| `web server max sockets` | `` | Available sockets. The default is system-specific, automatically adjusted to 50% of the max number of open files Netdata is allowed to use (via `/etc/security/limits.conf` or systemd), to allow enough file descriptors to be available for data collection. | +| `custom dashboard_info.js` | `` | Specifies the location of a custom `dashboard.js` file. See [customizing the standard dashboard](/docs/developer-and-contributor-corner/customize.md#customize-the-standard-dashboard) for details. | ## Examples @@ -63,7 +58,7 @@ Scroll down to the `[web]` section to find the following settings. Disable the web server by editing `netdata.conf` and setting: -``` +```text [web] mode = none ``` @@ -72,7 +67,7 @@ Disable the web server by editing `netdata.conf` and setting: Control the number of threads and sockets with the following settings: -``` +```text [web] web server threads = 4 web server max sockets = 512 @@ -84,7 +79,7 @@ Netdata can bind to multiple IPs and ports, offering access to different service The ports to bind are controlled via `[web].bind to`, like this: -``` +```text [web] default port = 19999 bind to = 127.0.0.1=dashboard^SSL=optional 10.1.1.1:19998=management|netdata.conf hostname:19997=badges [::]:19996=streaming^SSL=force localhost:19995=registry *:http=dashboard unix:/run/netdata/netdata.sock @@ -92,25 +87,25 @@ The ports to bind are controlled via `[web].bind to`, like this: Using the above, Netdata will bind to: -- IPv4 127.0.0.1 at port 19999 (port was used from `default port`). Only the UI (dashboard) and the read API will be accessible on this port. Both HTTP and HTTPS requests will be accepted. -- IPv4 10.1.1.1 at port 19998. The management API and `netdata.conf` will be accessible on this port. -- All the IPs `hostname` resolves to (both IPv4 and IPv6 depending on the resolved IPs) at port 19997. Only badges will be accessible on this port. -- All IPv6 IPs at port 19996. Only metric streaming requests from other Netdata agents will be accepted on this port. Only encrypted streams will be allowed (i.e. child nodes also need to be [configured for TLS](/src/streaming/README.md). -- All the IPs `localhost` resolves to (both IPv4 and IPv6 depending the resolved IPs) at port 19996. This port will only accept registry API requests. -- All IPv4 and IPv6 IPs at port `http` as set in `/etc/services`. Only the UI (dashboard) and the read API will be accessible on this port. -- Unix domain socket `/run/netdata/netdata.sock`. All requests are serviceable on this socket. Note that in some OSs like Fedora, every service sees a different `/tmp`, so don't create a Unix socket under `/tmp`. `/run` or `/var/run` is suggested. +- IPv4 127.0.0.1 at port 19999 (port was used from `default port`). Only the UI (dashboard) and the read API will be accessible on this port. Both HTTP and HTTPS requests will be accepted. +- IPv4 10.1.1.1 at port 19998. The management API and `netdata.conf` will be accessible on this port. +- All the IPs `hostname` resolves to (both IPv4 and IPv6 depending on the resolved IPs) at port 19997. Only badges will be accessible on this port. +- All IPv6 IPs at port 19996. Only metric streaming requests from other Netdata agents will be accepted on this port. Only encrypted streams will be allowed (i.e. child nodes also need to be [configured for TLS](/src/streaming/README.md). +- All the IPs `localhost` resolves to (both IPv4 and IPv6 depending the resolved IPs) at port 19996. This port will only accept registry API requests. +- All IPv4 and IPv6 IPs at port `http` as set in `/etc/services`. Only the UI (dashboard) and the read API will be accessible on this port. +- Unix domain socket `/run/netdata/netdata.sock`. All requests are serviceable on this socket. Note that in some OSs like Fedora, every service sees a different `/tmp`, so don't create a Unix socket under `/tmp`. `/run` or `/var/run` is suggested. The option `[web].default port` is used when an entries in `[web].bind to` do not specify a port. -Note that the access permissions specified with the `=request type|request type|...` format are available from version 1.12 onwards. -As shown in the example above, these permissions are optional, with the default being to permit all request types on the specified port. -The request types are strings identical to the `allow X from` directives of the access lists, i.e. `dashboard`, `streaming`, `registry`, `netdata.conf`, `badges` and `management`. -The access lists themselves and the general setting `allow connections from` in the next section are applied regardless of the ports that are configured to provide these services. +Note that the access permissions specified with the `=request type|request type|...` format are available from version 1.12 onwards. +As shown in the example above, these permissions are optional, with the default being to permit all request types on the specified port. +The request types are strings identical to the `allow X from` directives of the access lists, i.e. `dashboard`, `streaming`, `registry`, `netdata.conf`, `badges` and `management`. +The access lists themselves and the general setting `allow connections from` in the next section are applied regardless of the ports that are configured to provide these services. The API requests are serviced as follows: -- `dashboard` gives access to the UI, the read API and badges API calls. -- `badges` gives access only to the badges API calls. -- `management` gives access only to the management API calls. +- `dashboard` gives access to the UI, the read API and badges API calls. +- `badges` gives access only to the badges API calls. +- `management` gives access only to the management API calls. ### Enable HTTPS/TLS support @@ -126,10 +121,10 @@ Inbound unix socket connections are unaffected, regardless of the TLS settings. To enable TLS, provide the path to your certificate and private key in the `[web]` section of `netdata.conf`: -```conf +```text [web] - ssl key = /etc/netdata/ssl/key.pem - ssl certificate = /etc/netdata/ssl/cert.pem + ssl key = /etc/netdata/ssl/key.pem + ssl certificate = /etc/netdata/ssl/cert.pem ``` Both files must be readable by the `netdata` user. If either of these files do not exist or are unreadable, Netdata will fall back to HTTP. For a parent-child connection, only the parent needs these settings. @@ -152,7 +147,7 @@ openssl req -newkey rsa:2048 -nodes -sha512 -x509 -days 365 -keyout key.pem -out Beginning with version `v1.21.0`, specify the TLS version and the ciphers that you want to use: -```conf +```text [web] tls version = 1.3 tls ciphers = TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256 @@ -164,45 +159,45 @@ If you do not specify these options, Netdata will use the highest available prot When the certificates are defined and unless any other options are provided, a Netdata server will: -- Redirect all incoming HTTP web server requests to HTTPS. Applies to the dashboard, the API, `netdata.conf` and badges. -- Allow incoming child connections to use both unencrypted and encrypted communications for streaming. +- Redirect all incoming HTTP web server requests to HTTPS. Applies to the dashboard, the API, `netdata.conf` and badges. +- Allow incoming child connections to use both unencrypted and encrypted communications for streaming. To change this behavior, you need to modify the `bind to` setting in the `[web]` section of `netdata.conf`. At the end of each port definition, append `^SSL=force` or `^SSL=optional`. What happens with these settings differs, depending on whether the port is used for HTTP/S requests, or for streaming. -| SSL setting | HTTP requests|HTTPS requests|Unencrypted Streams|Encrypted Streams| -|:---------:|:-----------:|:------------:|:-----------------:|:----------------| -| none | Redirected to HTTPS|Accepted|Accepted|Accepted| -| `force`| Redirected to HTTPS|Accepted|Denied|Accepted| -| `optional`| Accepted|Accepted|Accepted|Accepted| +| SSL setting | HTTP requests | HTTPS requests | Unencrypted Streams | Encrypted Streams | +|:-----------:|:-------------------:|:--------------:|:-------------------:|:------------------| +| none | Redirected to HTTPS | Accepted | Accepted | Accepted | +| `force` | Redirected to HTTPS | Accepted | Denied | Accepted | +| `optional` | Accepted | Accepted | Accepted | Accepted | Example: -``` +```text [web] bind to = *=dashboard|registry|badges|management|streaming|netdata.conf^SSL=force ``` -For information how to configure the child to use TLS, check [securing the communication](/src/streaming/README.md#securing-streaming-communications) in the streaming documentation. There you will find additional details on the expected behavior for client and server nodes, when their respective TLS options are enabled. +For information how to configure the child to use TLS, check [securing the communication](/src/streaming/README.md#securing-streaming-with-tlsssl) in the streaming documentation. There you will find additional details on the expected behavior for client and server nodes, when their respective TLS options are enabled. When we define the use of SSL in a Netdata agent for different ports, Netdata will apply the behavior specified on each port. For example, using the configuration line below: -``` +```text [web] bind to = *=dashboard|registry|badges|management|streaming|netdata.conf^SSL=force *:20000=netdata.conf^SSL=optional *:20001=dashboard|registry ``` Netdata will: -- Force all HTTP requests to the default port to be redirected to HTTPS (same port). -- Refuse unencrypted streaming connections from child nodes on the default port. -- Allow both HTTP and HTTPS requests to port 20000 for `netdata.conf` -- Force HTTP requests to port 20001 to be redirected to HTTPS (same port). Only allow requests for the dashboard, the read API and the registry on port 20001. +- Force all HTTP requests to the default port to be redirected to HTTPS (same port). +- Refuse unencrypted streaming connections from child nodes on the default port. +- Allow both HTTP and HTTPS requests to port 20000 for `netdata.conf` +- Force HTTP requests to port 20001 to be redirected to HTTPS (same port). Only allow requests for the dashboard, the read API and the registry on port 20001. #### TLS/SSL errors When you start using Netdata with TLS, you may find errors in the Netdata log, which is stored at `/var/log/netdata/error.log` by default. -Most of the time, these errors are due to incompatibilities between your browser's options related to TLS/SSL protocols and Netdata's internal configuration. The most common error is `error:00000006:lib(0):func(0):EVP lib`. +Most of the time, these errors are due to incompatibilities between your browser's options related to TLS/SSL protocols and Netdata's internal configuration. The most common error is `error:00000006:lib(0):func(0):EVP lib`. In the near future, Netdata will allow our users to change the internal configuration to avoid similar errors. Until then, we're recommending only the most common and safe encryption protocols listed above. @@ -210,35 +205,35 @@ In the near future, Netdata will allow our users to change the internal configur Netdata supports access lists in `netdata.conf`: -``` +```text [web] - allow connections from = localhost * - allow dashboard from = localhost * - allow badges from = * - allow streaming from = * - allow netdata.conf from = localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.* 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.* 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.* 172.31.* - allow management from = localhost + allow connections from = localhost * + allow dashboard from = localhost * + allow badges from = * + allow streaming from = * + allow netdata.conf from = localhost fd* 10.* 192.168.* 172.16.* 172.17.* 172.18.* 172.19.* 172.20.* 172.21.* 172.22.* 172.23.* 172.24.* 172.25.* 172.26.* 172.27.* 172.28.* 172.29.* 172.30.* 172.31.* + allow management from = localhost ``` `*` does string matches on the IPs or FQDNs of the clients. -- `allow connections from` matches anyone that connects on the Netdata port(s). +- `allow connections from` matches anyone that connects on the Netdata port(s). So, if someone is not allowed, it will be connected and disconnected immediately, without reading even a single byte from its connection. This is a global setting with higher priority to any of the ones below. -- `allow dashboard from` receives the request and examines if it is a static dashboard file or an API call the +- `allow dashboard from` receives the request and examines if it is a static dashboard file or an API call the dashboards do. -- `allow badges from` checks if the API request is for a badge. Badges are not matched by `allow dashboard from`. +- `allow badges from` checks if the API request is for a badge. Badges are not matched by `allow dashboard from`. -- `allow streaming from` checks if the child willing to stream metrics to this Netdata is allowed. +- `allow streaming from` checks if the child willing to stream metrics to this Netdata is allowed. This can be controlled per API KEY and MACHINE GUID in `stream.conf`. The setting in `netdata.conf` is checked before the ones in `stream.conf`. -- `allow netdata.conf from` checks the IP to allow `http://netdata.host:19999/netdata.conf`. +- `allow netdata.conf from` checks the IP to allow `http://netdata.host:19999/netdata.conf`. The IPs listed are all the private IPv4 addresses, including link local IPv6 addresses. Keep in mind that connections to Netdata API ports are filtered by `allow connections from`. So, IPs allowed by `allow netdata.conf from` should also be allowed by `allow connections from`. -- `allow management from` checks the IPs to allow API management calls. Management via the API is currently supported for [health](/src/web/api/health/README.md#health-management-api) +- `allow management from` checks the IPs to allow API management calls. Management via the API is currently supported for [health](/src/web/api/health/README.md#health-management-api) In order to check the FQDN of the connection without opening the Netdata agent to DNS-spoofing, a reverse-dns record must be setup for the connecting host. At connection time the reverse-dns of the peer IP address is resolved, and @@ -247,13 +242,13 @@ a forward DNS resolution is made to validate the IP address against the name-pat Please note that this process can be expensive on a machine that is serving many connections. Each access list has an associated configuration option to turn off DNS-based patterns completely to avoid incurring this cost at run-time: -``` - allow connections by dns = heuristic - allow dashboard by dns = heuristic - allow badges by dns = heuristic - allow streaming by dns = heuristic - allow netdata.conf by dns = no - allow management by dns = heuristic +```text + allow connections by dns = heuristic + allow dashboard by dns = heuristic + allow badges by dns = heuristic + allow streaming by dns = heuristic + allow netdata.conf by dns = no + allow management by dns = heuristic ``` The three possible values for each of these options are `yes`, `no` and `heuristic`. The `heuristic` option disables @@ -264,8 +259,8 @@ present that may match DNS FQDNs. If you publish your Netdata web server to the internet, you may want to apply some protection against DDoS: -1. Use the `static-threaded` web server (it is the default) -2. Use reasonable `[web].web server max sockets` (the default is) -3. Don't use all your CPU cores for Netdata (lower `[web].web server threads`) -4. Run the `netdata` process with a low process scheduling priority (the default is the lowest) -5. If possible, proxy Netdata via a full featured web server (Nginx, Apache, etc) +1. Use the `static-threaded` web server (it is the default) +2. Use reasonable `[web].web server max sockets` (the default is) +3. Don't use all your CPU cores for Netdata (lower `[web].web server threads`) +4. Run the `netdata` process with a low process scheduling priority (the default is the lowest) +5. If possible, proxy Netdata via a full featured web server (Nginx, Apache, etc) diff --git a/src/web/server/h2o/http_server.c b/src/web/server/h2o/http_server.c index a079c6af..0fc65b35 100644 --- a/src/web/server/h2o/http_server.c +++ b/src/web/server/h2o/http_server.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "daemon/common.h" -#include "streaming/common.h" +#include "streaming/h2o-common.h" #include "http_server.h" #pragma GCC diagnostic push @@ -24,6 +24,7 @@ static h2o_accept_ctx_t accept_ctx; #define NBUF_INITIAL_SIZE_RESP (4096) #define API_V1_PREFIX "/api/v1/" #define API_V2_PREFIX "/api/v2/" +#define API_V3_PREFIX "/api/v3/" #define HOST_SELECT_PREFIX "/host/" #define HTTPD_CONFIG_SECTION "httpd" @@ -182,13 +183,17 @@ static inline int _netdata_uberhandler(h2o_req_t *req, RRDHOST **host) norm_path.len--; } - unsigned int api_version = 2; - size_t api_loc = h2o_strstr(norm_path.base, norm_path.len, H2O_STRLIT(API_V2_PREFIX)); + unsigned int api_version = 3; + size_t api_loc = h2o_strstr(norm_path.base, norm_path.len, H2O_STRLIT(API_V3_PREFIX)); if (api_loc == SIZE_MAX) { - api_version = 1; - api_loc = h2o_strstr(norm_path.base, norm_path.len, H2O_STRLIT(API_V1_PREFIX)); - if (api_loc == SIZE_MAX) - return 1; + api_version = 2; + api_loc = h2o_strstr(norm_path.base, norm_path.len, H2O_STRLIT(API_V2_PREFIX)); + if (api_loc == SIZE_MAX) { + api_version = 1; + api_loc = h2o_strstr(norm_path.base, norm_path.len, H2O_STRLIT(API_V1_PREFIX)); + if (api_loc == SIZE_MAX) + return 1; + } } // API_V1_PREFIX and API_V2_PREFIX are the same length @@ -235,7 +240,9 @@ static inline int _netdata_uberhandler(h2o_req_t *req, RRDHOST **host) } //inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint) { - if (api_version == 2) + if (api_version == 3) + web_client_api_request_v3(*host, &w, path_unescaped); + else if (api_version == 2) web_client_api_request_v2(*host, &w, path_unescaped); else web_client_api_request_v1(*host, &w, path_unescaped); @@ -283,7 +290,7 @@ static int netdata_uberhandler(h2o_handler_t *self, h2o_req_t *req) char host_uuid_str[UUID_STR_LEN]; if (host != NULL) - uuid_unparse_lower(host->host_uuid, host_uuid_str); + uuid_unparse_lower(host->host_id.uuid, host_uuid_str); nd_log(NDLS_ACCESS, NDLP_DEBUG, "HTTPD OK method: " PRINTF_H2O_IOVEC_FMT ", path: " PRINTF_H2O_IOVEC_FMT @@ -314,7 +321,7 @@ static int hdl_netdata_conf(h2o_handler_t *self, h2o_req_t *req) return -1; BUFFER *buf = buffer_create(NBUF_INITIAL_SIZE_RESP, NULL); - config_generate(buf, 0); + netdata_conf_generate(buf, 0); void *managed = h2o_mem_alloc_shared(&req->pool, buf->len, NULL); memcpy(managed, buf->buffer, buf->len); diff --git a/src/web/server/h2o/streaming.c b/src/web/server/h2o/rrdpush.c index fbe3f805..515ec8fd 100644 --- a/src/web/server/h2o/streaming.c +++ b/src/web/server/h2o/rrdpush.c @@ -4,7 +4,7 @@ #include "streaming.h" #include "connlist.h" #include "h2o_utils.h" -#include "streaming/common.h" +#include "streaming/h2o-common.h" static int pending_write_reqs = 0; diff --git a/src/web/server/h2o/streaming.h b/src/web/server/h2o/streaming.h index dfc7b68f..a30f4a8e 100644 --- a/src/web/server/h2o/streaming.h +++ b/src/web/server/h2o/streaming.h @@ -3,8 +3,6 @@ #ifndef HTTPD_STREAMING_H #define HTTPD_STREAMING_H -#include "aclk/mqtt_websockets/c-rbuf/cringbuffer.h" - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-but-set-variable" diff --git a/src/web/server/static/README.md b/src/web/server/static/README.md index e67949e2..d480ae00 100644 --- a/src/web/server/static/README.md +++ b/src/web/server/static/README.md @@ -1,13 +1,3 @@ -<!-- -title: "`static-threaded` web server" -description: "The Netdata Agent's static-threaded web server spawns a fixed number of threads that listen to web requests and uses non-blocking I/O." -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/web/server/static/README.md -sidebar_label: "`static-threaded` web server" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/Web" ---> - # `static-threaded` web server The `static-threaded` web server spawns a fixed number of threads. diff --git a/src/web/server/static/static-threaded.c b/src/web/server/static/static-threaded.c index a4b24c9a..01c9817f 100644 --- a/src/web/server/static/static-threaded.c +++ b/src/web/server/static/static-threaded.c @@ -211,7 +211,6 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data web_client_set_conn_tcp(w); } -#ifdef ENABLE_HTTPS if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) { sock_delnonblock(w->ifd); @@ -239,13 +238,10 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data sock_setnonblock(w->ifd); } -#endif netdata_log_debug(D_WEB_CLIENT, "%llu: ADDED CLIENT FD %d", w->id, pi->fd); -#ifdef ENABLE_HTTPS cleanup: -#endif worker_is_idle(); return w; } @@ -503,14 +499,12 @@ void *socket_listen_main_static_threaded(void *ptr) { if(!api_sockets.opened) fatal("LISTENER: no listen sockets available."); -#ifdef ENABLE_HTTPS netdata_ssl_validate_certificate = !config_get_boolean(CONFIG_SECTION_WEB, "ssl skip certificate verification", !netdata_ssl_validate_certificate); if(!netdata_ssl_validate_certificate_sender) netdata_log_info("SSL: web server will skip SSL certificates verification."); netdata_ssl_initialize_ctx(NETDATA_SSL_WEB_SERVER_CTX); -#endif // 6 threads is the optimal value // since 6 are the parallel connections browsers will do @@ -526,13 +520,11 @@ void *socket_listen_main_static_threaded(void *ptr) { if (static_threaded_workers_count < 1) static_threaded_workers_count = 1; -#ifdef ENABLE_HTTPS // See https://github.com/netdata/netdata/issues/11081#issuecomment-831998240 for more details if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) { static_threaded_workers_count = 1; netdata_log_info("You are running an OpenSSL older than 1.1.0, web server will not enable multithreading."); } -#endif size_t max_sockets = (size_t)config_get_number(CONFIG_SECTION_WEB, "web server max sockets", (long long int)(rlimit_nofile.rlim_cur / 4)); @@ -548,7 +540,6 @@ void *socket_listen_main_static_threaded(void *ptr) { char tag[50 + 1]; snprintfz(tag, sizeof(tag) - 1, "WEB[%d]", i+1); - netdata_log_info("starting worker %d", i+1); static_workers_private_data[i].thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, socket_listen_main_static_threaded_worker, (void *)&static_workers_private_data[i]); diff --git a/src/web/server/web_client.c b/src/web/server/web_client.c index ca1c28e7..a26e9233 100644 --- a/src/web/server/web_client.c +++ b/src/web/server/web_client.c @@ -6,7 +6,7 @@ // it is used by all netdata web servers int respect_web_browser_do_not_track_policy = 0; -char *web_x_frame_options = NULL; +const char *web_x_frame_options = NULL; int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY; @@ -33,6 +33,7 @@ void web_client_set_conn_webrtc(struct web_client *w) { void web_client_reset_permissions(struct web_client *w) { web_client_flags_clear_auth(w); w->access = HTTP_ACCESS_NONE; + w->user_role = HTTP_USER_ROLE_NONE; } void web_client_set_permissions(struct web_client *w, HTTP_ACCESS access, HTTP_USER_ROLE role, WEB_CLIENT_FLAGS auth) { @@ -97,7 +98,6 @@ static inline int web_client_cork_socket(struct web_client *w __maybe_unused) { return 0; } -#ifdef ENABLE_HTTPS static inline void web_client_enable_wait_from_ssl(struct web_client *w) { if (w->ssl.ssl_errno == SSL_ERROR_WANT_READ) web_client_enable_ssl_wait_receive(w); @@ -108,7 +108,6 @@ static inline void web_client_enable_wait_from_ssl(struct web_client *w) { web_client_disable_ssl_wait_send(w); } } -#endif static inline int web_client_uncork_socket(struct web_client *w __maybe_unused) { #ifdef TCP_CORK @@ -211,6 +210,8 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) { } memset(w->transaction, 0, sizeof(w->transaction)); + memset(&w->auth, 0, sizeof(w->auth)); + web_client_reset_permissions(w); web_client_flag_clear(w, WEB_CLIENT_ENCODING_GZIP|WEB_CLIENT_ENCODING_DEFLATE); web_client_reset_path_flags(w); @@ -380,12 +381,14 @@ static inline int dashboard_version(struct web_client *w) { if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION)) return -1; - if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V0)) - return 0; - if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V1)) - return 1; + if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V3)) + return 3; if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V2)) return 2; + if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V1)) + return 1; + if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V0)) + return 0; return -1; } @@ -589,7 +592,9 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f // get the api version char *tok = strsep_skip_consecutive_separators(&url_path_fragment, "/"); if(tok && *tok) { - if(strcmp(tok, "v2") == 0) + if(strcmp(tok, "v3") == 0) + return web_client_api_request_v3(host, w, url_path_fragment); + else if(strcmp(tok, "v2") == 0) return web_client_api_request_v2(host, w, url_path_fragment); else if(strcmp(tok, "v1") == 0) return web_client_api_request_v1(host, w, url_path_fragment); @@ -644,7 +649,6 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) { else if(!strncmp(s, "STREAM ", 7)) { s = &s[7]; -#ifdef ENABLE_HTTPS if (!SSL_connection(&w->ssl) && http_is_using_ssl_force(w)) { w->header_parse_tries = 0; w->header_parse_last_size = 0; @@ -672,7 +676,6 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) { netdata_log_error("The server is configured to always use encrypted connections, please enable the SSL on child with hostname '%s'.",hostname); s = NULL; } -#endif w->mode = HTTP_REQUEST_MODE_STREAM; } @@ -704,11 +707,12 @@ HTTP_VALIDATION http_request_validate(struct web_client *w) { if(last_pos > 4) last_pos -= 4; // allow searching for \r\n\r\n else last_pos = 0; - if(w->header_parse_last_size < last_pos) + if(w->header_parse_last_size <= last_pos) last_pos = 0; - is_it_valid = - url_is_request_complete_and_extract_payload(s, &s[last_pos], w->header_parse_last_size, &w->payload); + is_it_valid = url_is_request_complete_and_extract_payload(s, &s[last_pos], + w->header_parse_last_size, &w->payload); + if(!is_it_valid) { if(w->header_parse_tries > HTTP_REQ_MAX_HEADER_FETCH_TRIES) { netdata_log_info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data)); @@ -736,17 +740,6 @@ HTTP_VALIDATION http_request_validate(struct web_client *w) { return HTTP_VALIDATION_NOT_SUPPORTED; } else if (!is_it_valid) { - //Invalid request, we have more data after the end of message - char *check = strstr((char *)buffer_tostring(w->response.data), "\r\n\r\n"); - if(check) { - check += 4; - if (*check) { - w->header_parse_tries = 0; - w->header_parse_last_size = 0; - web_client_disable_wait_receive(w); - return HTTP_VALIDATION_EXCESS_REQUEST_DATA; - } - } web_client_enable_wait_receive(w); return HTTP_VALIDATION_INCOMPLETE; } @@ -787,7 +780,6 @@ HTTP_VALIDATION http_request_validate(struct web_client *w) { web_client_decode_path_and_query_string(w, encoded_url); *ue = c; -#ifdef ENABLE_HTTPS if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) { if (!w->ssl.conn && (http_is_using_ssl_force(w) || http_is_using_ssl_default(w)) && (w->mode != HTTP_REQUEST_MODE_STREAM)) { w->header_parse_tries = 0; @@ -796,7 +788,6 @@ HTTP_VALIDATION http_request_validate(struct web_client *w) { return HTTP_VALIDATION_REDIRECT; } } -#endif w->header_parse_tries = 0; w->header_parse_last_size = 0; @@ -816,28 +807,28 @@ HTTP_VALIDATION http_request_validate(struct web_client *w) { static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,size_t len, int flags) { - ssize_t bytes; -#ifdef ENABLE_HTTPS - if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) { - if (SSL_connection(&w->ssl)) { - bytes = netdata_ssl_write(&w->ssl, buf, len) ; - web_client_enable_wait_from_ssl(w); - } + do { + errno_clear(); + + ssize_t bytes; + if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) { + if (SSL_connection(&w->ssl)) { + bytes = netdata_ssl_write(&w->ssl, buf, len); + web_client_enable_wait_from_ssl(w); + } else + bytes = send(w->ofd, buf, len, flags); + } else if (web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) + bytes = send(w->ofd, buf, len, flags); else - bytes = send(w->ofd,buf, len , flags); - } - else if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) - bytes = send(w->ofd,buf, len , flags); - else - bytes = -999; -#else - if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) - bytes = send(w->ofd, buf, len, flags); - else - bytes = -999; -#endif + bytes = -999; - return bytes; + if(bytes < 0 && errno == EAGAIN) { + tinysleep(); + continue; + } + + return bytes; + } while(true); } void web_client_build_http_header(struct web_client *w) { @@ -910,8 +901,8 @@ void web_client_build_http_header(struct web_client *w) { if(w->mode == HTTP_REQUEST_MODE_OPTIONS) { buffer_strcat(w->response.header_output, - "Access-Control-Allow-Methods: GET, OPTIONS\r\n" - "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie, pragma, cache-control, x-auth-token\r\n" + "Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n" + "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie, pragma, cache-control, x-auth-token, x-netdata-auth, x-transaction-id\r\n" "Access-Control-Max-Age: 1209600\r\n" // 86400 * 14 ); } @@ -967,7 +958,7 @@ static inline void web_client_send_http_header(struct web_client *w) { size_t count = 0; ssize_t bytes; -#ifdef ENABLE_HTTPS + if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) { if (SSL_connection(&w->ssl)) { bytes = netdata_ssl_write(&w->ssl, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output)); @@ -996,20 +987,6 @@ static inline void web_client_send_http_header(struct web_client *w) { } else bytes = -999; -#else - if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) { - while ((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) { - count++; - - if (count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) { - netdata_log_error("Cannot send HTTP headers to web client."); - break; - } - } - } - else - bytes = -999; -#endif if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) { if(bytes > 0) @@ -1161,7 +1138,8 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch hash_node = 0, hash_v0 = 0, hash_v1 = 0, - hash_v2 = 0; + hash_v2 = 0, + hash_v3 = 0; #ifdef NETDATA_INTERNAL_CHECKS static uint32_t hash_exit = 0, hash_debug = 0, hash_mirror = 0; @@ -1175,6 +1153,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch hash_v0 = simple_hash("v0"); hash_v1 = simple_hash("v1"); hash_v2 = simple_hash("v2"); + hash_v3 = simple_hash("v3"); #ifdef NETDATA_INTERNAL_CHECKS hash_exit = simple_hash("exit"); hash_debug = simple_hash("debug"); @@ -1199,6 +1178,12 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id); return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_process_url); } + else if(unlikely(hash == hash_v3 && strcmp(tok, "v3") == 0)) { + if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION)) + return bad_request_multiple_dashboard_versions(w); + web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V3); + return web_client_process_url(host, w, decoded_url_path); + } else if(unlikely(hash == hash_v2 && strcmp(tok, "v2") == 0)) { if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION)) return bad_request_multiple_dashboard_versions(w); @@ -1224,7 +1209,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: generating netdata.conf ...", w->id); w->response.data->content_type = CT_TEXT_PLAIN; buffer_flush(w->response.data); - config_generate(w->response.data, 0); + netdata_conf_generate(w->response.data, 0); return HTTP_RESP_OK; } #ifdef NETDATA_INTERNAL_CHECKS @@ -1310,11 +1295,7 @@ static bool web_server_log_transport(BUFFER *wb, void *ptr) { if(!w) return false; -#ifdef ENABLE_HTTPS buffer_strcat(wb, SSL_connection(&w->ssl) ? "https" : "http"); -#else - buffer_strcat(wb, "http"); -#endif return true; } @@ -1441,7 +1422,7 @@ void web_client_process_request_from_web_server(struct web_client *w) { buffer_flush(w->url_as_received); buffer_strcat(w->url_as_received, "too big request"); - netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, w->response.data->len); + netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, (size_t)w->response.data->len); size_t len = w->response.data->len; buffer_flush(w->response.data); @@ -1457,7 +1438,7 @@ void web_client_process_request_from_web_server(struct web_client *w) { return; } break; -#ifdef ENABLE_HTTPS + case HTTP_VALIDATION_REDIRECT: { buffer_flush(w->response.data); @@ -1473,7 +1454,7 @@ void web_client_process_request_from_web_server(struct web_client *w) { w->response.code = HTTP_RESP_HTTPS_UPGRADE; break; } -#endif + case HTTP_VALIDATION_MALFORMED_URL: netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Malformed URL '%s'.", w->id, w->response.data->buffer); @@ -1481,13 +1462,6 @@ void web_client_process_request_from_web_server(struct web_client *w) { buffer_strcat(w->response.data, "Malformed URL...\r\n"); w->response.code = HTTP_RESP_BAD_REQUEST; break; - case HTTP_VALIDATION_EXCESS_REQUEST_DATA: - netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Excess data in request '%s'.", w->id, w->response.data->buffer); - - buffer_flush(w->response.data); - buffer_strcat(w->response.data, "Excess data in request.\r\n"); - w->response.code = HTTP_RESP_BAD_REQUEST; - break; case HTTP_VALIDATION_TOO_MANY_READ_RETRIES: netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Too many retries to read request '%s'.", w->id, w->response.data->buffer); @@ -1521,14 +1495,18 @@ void web_client_process_request_from_web_server(struct web_client *w) { break; case HTTP_REQUEST_MODE_OPTIONS: - netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.", w->id, w->response.data->len); + netdata_log_debug(D_WEB_CLIENT, + "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.", + w->id, (size_t)w->response.data->len); break; case HTTP_REQUEST_MODE_POST: case HTTP_REQUEST_MODE_GET: case HTTP_REQUEST_MODE_PUT: case HTTP_REQUEST_MODE_DELETE: - netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%zu bytes) to client.", w->id, w->response.data->len); + netdata_log_debug(D_WEB_CLIENT, + "%llu: Done preparing the response. Sending data (%zu bytes) to client.", + w->id, (size_t)w->response.data->len); break; case HTTP_REQUEST_MODE_FILECOPY: @@ -1635,8 +1613,9 @@ ssize_t web_client_send_deflate(struct web_client *w) // when using compression, // w->response.sent is the amount of bytes passed through compression - netdata_log_debug(D_DEFLATE, "%llu: web_client_send_deflate(): w->response.data->len = %zu, w->response.sent = %zu, w->response.zhave = %zu, w->response.zsent = %zu, w->response.zstream.avail_in = %u, w->response.zstream.avail_out = %u, w->response.zstream.total_in = %lu, w->response.zstream.total_out = %lu.", - w->id, w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out); + netdata_log_debug(D_DEFLATE, + "%llu: web_client_send_deflate(): w->response.data->len = %zu, w->response.sent = %zu, w->response.zhave = %zu, w->response.zsent = %zu, w->response.zstream.avail_in = %u, w->response.zstream.avail_out = %u, w->response.zstream.total_in = %lu, w->response.zstream.total_out = %lu.", + w->id, (size_t)w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out); if(w->response.data->len - w->response.sent == 0 && w->response.zstream.avail_in == 0 && w->response.zhave == w->response.zsent && w->response.zstream.avail_out != 0) { // there is nothing to send @@ -1850,14 +1829,14 @@ ssize_t web_client_receive(struct web_client *w) return web_client_read_file(w); ssize_t bytes; - ssize_t left = (ssize_t)(w->response.data->size - w->response.data->len); // do we have any space for more data? buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE); + ssize_t left = (ssize_t)(w->response.data->size - w->response.data->len); + errno_clear(); -#ifdef ENABLE_HTTPS if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) { if (SSL_connection(&w->ssl)) { bytes = netdata_ssl_read(&w->ssl, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1)); @@ -1872,12 +1851,6 @@ ssize_t web_client_receive(struct web_client *w) } else // other connection methods bytes = -1; -#else - if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) - bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT); - else - bytes = -1; -#endif if(likely(bytes > 0)) { w->statistics.received_bytes += bytes; @@ -1960,9 +1933,7 @@ void web_client_reuse_from_cache(struct web_client *w) { BUFFER *b6 = w->url_query_string_decoded; BUFFER *b7 = w->payload; -#ifdef ENABLE_HTTPS NETDATA_SSL ssl = w->ssl; -#endif size_t use_count = w->use_count; size_t *statistics_memory_accounting = w->statistics.memory_accounting; @@ -1974,9 +1945,7 @@ void web_client_reuse_from_cache(struct web_client *w) { w->statistics.memory_accounting = statistics_memory_accounting; w->use_count = use_count; -#ifdef ENABLE_HTTPS w->ssl = ssl; -#endif // restore the pointers of the buffers w->response.data = b1; @@ -1991,9 +1960,7 @@ void web_client_reuse_from_cache(struct web_client *w) { struct web_client *web_client_create(size_t *statistics_memory_accounting) { struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client)); -#ifdef ENABLE_HTTPS w->ssl = NETDATA_SSL_UNSET_CONNECTION; -#endif w->use_count = 1; w->statistics.memory_accounting = statistics_memory_accounting; @@ -2011,9 +1978,7 @@ struct web_client *web_client_create(size_t *statistics_memory_accounting) { } void web_client_free(struct web_client *w) { -#ifdef ENABLE_HTTPS netdata_ssl_close(&w->ssl); -#endif web_client_reset_allocations(w, true); diff --git a/src/web/server/web_client.h b/src/web/server/web_client.h index 650ddb3e..76bde525 100644 --- a/src/web/server/web_client.h +++ b/src/web/server/web_client.h @@ -12,18 +12,15 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy; #define HTTP_REQ_MAX_HEADER_FETCH_TRIES 100 extern int respect_web_browser_do_not_track_policy; -extern char *web_x_frame_options; +extern const char *web_x_frame_options; typedef enum __attribute__((packed)) { HTTP_VALIDATION_OK, HTTP_VALIDATION_NOT_SUPPORTED, HTTP_VALIDATION_TOO_MANY_READ_RETRIES, - HTTP_VALIDATION_EXCESS_REQUEST_DATA, HTTP_VALIDATION_MALFORMED_URL, HTTP_VALIDATION_INCOMPLETE, -#ifdef ENABLE_HTTPS HTTP_VALIDATION_REDIRECT -#endif } HTTP_VALIDATION; typedef enum __attribute__((packed)) { @@ -58,19 +55,20 @@ typedef enum __attribute__((packed)) { WEB_CLIENT_FLAG_PATH_IS_V0 = (1 << 16), // v0 dashboard found on the path WEB_CLIENT_FLAG_PATH_IS_V1 = (1 << 17), // v1 dashboard found on the path WEB_CLIENT_FLAG_PATH_IS_V2 = (1 << 18), // v2 dashboard found on the path - WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH = (1 << 19), // the path has a trailing hash - WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION = (1 << 20), // the path ends with a filename extension + WEB_CLIENT_FLAG_PATH_IS_V3 = (1 << 19), // v3 dashboard found on the path + WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH = (1 << 20), // the path has a trailing hash + WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION = (1 << 21), // the path ends with a filename extension // authorization - WEB_CLIENT_FLAG_AUTH_CLOUD = (1 << 21), - WEB_CLIENT_FLAG_AUTH_BEARER = (1 << 22), - WEB_CLIENT_FLAG_AUTH_GOD = (1 << 23), + WEB_CLIENT_FLAG_AUTH_CLOUD = (1 << 22), + WEB_CLIENT_FLAG_AUTH_BEARER = (1 << 23), + WEB_CLIENT_FLAG_AUTH_GOD = (1 << 24), // transient settings - WEB_CLIENT_FLAG_PROGRESS_TRACKING = (1 << 24), // flag to avoid redoing progress work + WEB_CLIENT_FLAG_PROGRESS_TRACKING = (1 << 25), // flag to avoid redoing progress work } WEB_CLIENT_FLAGS; -#define WEB_CLIENT_FLAG_PATH_WITH_VERSION (WEB_CLIENT_FLAG_PATH_IS_V0|WEB_CLIENT_FLAG_PATH_IS_V1|WEB_CLIENT_FLAG_PATH_IS_V2) +#define WEB_CLIENT_FLAG_PATH_WITH_VERSION (WEB_CLIENT_FLAG_PATH_IS_V0|WEB_CLIENT_FLAG_PATH_IS_V1|WEB_CLIENT_FLAG_PATH_IS_V2|WEB_CLIENT_FLAG_PATH_IS_V3) #define web_client_reset_path_flags(w) (w)->flags &= ~(WEB_CLIENT_FLAG_PATH_WITH_VERSION|WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH|WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION) #define web_client_flag_check(w, flag) ((w)->flags & (flag)) @@ -112,9 +110,9 @@ typedef enum __attribute__((packed)) { #define web_client_check_conn_tcp(w) web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_TCP) #define web_client_check_conn_cloud(w) web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_CLOUD) #define web_client_check_conn_webrtc(w) web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_WEBRTC) - -#define WEB_CLIENT_FLAG_ALL_AUTHS (WEB_CLIENT_FLAG_AUTH_CLOUD | WEB_CLIENT_FLAG_AUTH_BEARER) #define web_client_flags_clear_conn(w) web_client_flag_clear(w, WEB_CLIENT_FLAG_CONN_TCP | WEB_CLIENT_FLAG_CONN_UNIX | WEB_CLIENT_FLAG_CONN_CLOUD | WEB_CLIENT_FLAG_CONN_WEBRTC) + +#define WEB_CLIENT_FLAG_ALL_AUTHS (WEB_CLIENT_FLAG_AUTH_CLOUD | WEB_CLIENT_FLAG_AUTH_BEARER | WEB_CLIENT_FLAG_AUTH_GOD) #define web_client_flags_check_auth(w) web_client_flag_check(w, WEB_CLIENT_FLAG_ALL_AUTHS) #define web_client_flags_clear_auth(w) web_client_flag_clear(w, WEB_CLIENT_FLAG_ALL_AUTHS) @@ -136,7 +134,7 @@ void web_client_set_conn_webrtc(struct web_client *w); #define NETDATA_WEB_REQUEST_MAX_SIZE 65536 #define NETDATA_WEB_DECODED_URL_INITIAL_SIZE 512 -#define CLOUD_USER_NAME_LENGTH 64 +#define CLOUD_CLIENT_NAME_LENGTH 64 struct response { BUFFER *header; // our response header @@ -202,14 +200,12 @@ struct web_client { size_t pollinfo_slot; // POLLINFO slot of the web client size_t pollinfo_filecopy_slot; // POLLINFO slot of the file read -#ifdef ENABLE_HTTPS NETDATA_SSL ssl; -#endif struct { nd_uuid_t bearer_token; nd_uuid_t cloud_account_id; - char client_name[CLOUD_USER_NAME_LENGTH]; + char client_name[CLOUD_CLIENT_NAME_LENGTH]; } auth; struct { // A callback to check if the query should be interrupted / stopped diff --git a/src/web/server/web_client_cache.c b/src/web/server/web_client_cache.c index 654577e8..ebc42889 100644 --- a/src/web/server/web_client_cache.c +++ b/src/web/server/web_client_cache.c @@ -119,15 +119,14 @@ struct web_client *web_client_get_from_cache(void) { w->mode = HTTP_REQUEST_MODE_GET; web_client_reset_permissions(w); memset(w->transaction, 0, sizeof(w->transaction)); + memset(&w->auth, 0, sizeof(w->auth)); return w; } void web_client_release_to_cache(struct web_client *w) { -#ifdef ENABLE_HTTPS netdata_ssl_close(&w->ssl); -#endif // unlink it from the used spinlock_lock(&web_clients_cache.used.spinlock); diff --git a/src/web/server/web_server.c b/src/web/server/web_server.c index 3497af13..f7d6ceca 100644 --- a/src/web/server/web_server.c +++ b/src/web/server/web_server.c @@ -134,11 +134,7 @@ void web_client_update_acl_matches(struct web_client *w) { void web_server_log_connection(struct web_client *w, const char *msg) { ND_LOG_STACK lgs[] = { ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id), -#ifdef ENABLE_HTTPS ND_LOG_FIELD_TXT(NDF_SRC_TRANSPORT, SSL_connection(&w->ssl) ? "https" : "http"), -#else - ND_LOG_FIELD_TXT(NDF_SRC_TRANSPORT, "http"), -#endif ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip), ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port), ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host), |