diff options
Diffstat (limited to '')
| -rw-r--r-- | backends/Makefile.am | 20 | ||||
| -rw-r--r-- | backends/README.md | 203 | ||||
| -rw-r--r-- | backends/WALKTHROUGH.md | 294 | ||||
| -rw-r--r-- | backends/backends.c | 662 | ||||
| -rw-r--r-- | backends/backends.h | 52 | ||||
| -rw-r--r-- | backends/graphite/Makefile.am | 4 | ||||
| -rw-r--r-- | backends/graphite/graphite.c | 90 | ||||
| -rw-r--r-- | backends/graphite/graphite.h | 35 | ||||
| -rw-r--r-- | backends/json/Makefile.am | 4 | ||||
| -rw-r--r-- | backends/json/json.c | 152 | ||||
| -rw-r--r-- | backends/json/json.h | 34 | ||||
| -rwxr-xr-x | backends/nc-backend.sh | 158 | ||||
| -rw-r--r-- | backends/opentsdb/Makefile.am | 4 | ||||
| -rw-r--r-- | backends/opentsdb/opentsdb.c | 90 | ||||
| -rw-r--r-- | backends/opentsdb/opentsdb.h | 35 | ||||
| -rw-r--r-- | backends/prometheus/Makefile.am | 8 | ||||
| -rw-r--r-- | backends/prometheus/README.md | 382 | ||||
| -rw-r--r-- | backends/prometheus/backend_prometheus.c | 562 | ||||
| -rw-r--r-- | backends/prometheus/backend_prometheus.h | 20 |
19 files changed, 2809 insertions, 0 deletions
diff --git a/backends/Makefile.am b/backends/Makefile.am new file mode 100644 index 0000000..b8daefc --- /dev/null +++ b/backends/Makefile.am @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +SUBDIRS = \ + graphite \ + json \ + opentsdb \ + prometheus \ + $(NULL) + +dist_noinst_DATA = \ + README.md \ + WALKTHROUGH.md \ + $(NULL) + +dist_noinst_SCRIPTS = \ + nc-backend.sh \ + $(NULL) diff --git a/backends/README.md b/backends/README.md new file mode 100644 index 0000000..22dc775 --- /dev/null +++ b/backends/README.md @@ -0,0 +1,203 @@ +# Metrics long term archiving + +netdata supports backends for archiving the metrics, or providing long term dashboards, +using Grafana or other tools, like this: + + + +Since netdata collects thousands of metrics per server per second, which would easily congest any backend +server when several netdata servers are sending data to it, netdata allows sending metrics at a lower +frequency, by resampling them. + +So, although netdata collects metrics every second, it can send to the backend servers averages or sums every +X seconds (though, it can send them per second if you need it to). + +## features + +1. Supported backends + + - **graphite** (`plaintext interface`, used by **Graphite**, **InfluxDB**, **KairosDB**, + **Blueflood**, **ElasticSearch** via logstash tcp input and the graphite codec, etc) + + metrics are sent to the backend server as `prefix.hostname.chart.dimension`. `prefix` is + configured below, `hostname` is the hostname of the machine (can also be configured). + + - **opentsdb** (`telnet interface`, used by **OpenTSDB**, **InfluxDB**, **KairosDB**, etc) + + metrics are sent to opentsdb as `prefix.chart.dimension` with tag `host=hostname`. + + - **json** document DBs + + metrics are sent to a document db, `JSON` formatted. + + - **prometheus** is described at [prometheus page](prometheus/) since it pulls data from netdata. + +2. Only one backend may be active at a time. + +3. Netdata can filter metrics (at the chart level), to send only a subset of the collected metrics. + +4. Netdata supports three modes of operation for all backends: + + - `as-collected` sends to backends the metrics as they are collected, in the units they are collected. + So, counters are sent as counters and gauges are sent as gauges, much like all data collectors do. + For example, to calculate CPU utilization in this format, you need to know how to convert kernel ticks to percentage. + + - `average` sends to backends normalized metrics from the netdata database. + In this mode, all metrics are sent as gauges, in the units netdata uses. This abstracts data collection + and simplifies visualization, but you will not be able to copy and paste queries from other sources to convert units. + For example, CPU utilization percentage is calculated by netdata, so netdata will convert ticks to percentage and + send the average percentage to the backend. + + - `sum` or `volume`: the sum of the interpolated values shown on the netdata graphs is sent to the backend. + So, if netdata is configured to send data to the backend every 10 seconds, the sum of the 10 values shown on the + netdata charts will be used. + +Time-series databases suggest to collect the raw values (`as-collected`). If you plan to invest on building your monitoring around a time-series database and you already know (or you will invest in learning) how to convert units and normalize the metrics in Grafana or other visualization tools, we suggest to use `as-collected`. + +If, on the other hand, you just need long term archiving of netdata metrics and you plan to mainly work with netdata, we suggest to use `average`. It decouples visualization from data collection, so it will generally be a lot simpler. Furthermore, if you use `average`, the charts shown in the back-end will match exactly what you see in Netdata, which is not necessarily true for the other modes of operation. + +5. This code is smart enough, not to slow down netdata, independently of the speed of the backend server. + +## configuration + +In `/etc/netdata/netdata.conf` you should have something like this (if not download the latest version +of `netdata.conf` from your netdata): + +``` +[backend] + enabled = yes | no + type = graphite | opentsdb | json + host tags = list of TAG=VALUE + destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used + data source = average | sum | as collected + prefix = netdata + hostname = my-name + update every = 10 + buffer on failures = 10 + timeout ms = 20000 + send charts matching = * + send hosts matching = localhost * + send names instead of ids = yes +``` + +- `enabled = yes | no`, enables or disables sending data to a backend + +- `type = graphite | opentsdb | json`, selects the backend type + +- `destination = host1 host2 host3 ...`, accepts **a space separated list** of hostnames, + IPs (IPv4 and IPv6) and ports to connect to. + Netdata will use the **first available** to send the metrics. + + The format of each item in this list, is: `[PROTOCOL:]IP[:PORT]`. + + `PROTOCOL` can be `udp` or `tcp`. `tcp` is the default and only supported by the current backends. + + `IP` can be `XX.XX.XX.XX` (IPv4), or `[XX:XX...XX:XX]` (IPv6). + For IPv6 you can to enclose the IP in `[]` to separate it from the port. + + `PORT` can be a number of a service name. If omitted, the default port for the backend will be used + (graphite = 2003, opentsdb = 4242). + + Example IPv4: + +``` + destination = 10.11.14.2:4242 10.11.14.3:4242 10.11.14.4:4242 +``` + + Example IPv6 and IPv4 together: + +``` + destination = [ffff:...:0001]:2003 10.11.12.1:2003 +``` + + When multiple servers are defined, netdata will try the next one when the first one fails. This allows + you to load-balance different servers: give your backend servers in different order on each netdata. + + netdata also ships [`nc-backend.sh`](nc-backend.sh), + a script that can be used as a fallback backend to save the metrics to disk and push them to the + time-series database when it becomes available again. It can also be used to monitor / trace / debug + the metrics netdata generates. + +- `data source = as collected`, or `data source = average`, or `data source = sum`, selects the kind of + data that will be sent to the backend. + +- `hostname = my-name`, is the hostname to be used for sending data to the backend server. By default + this is `[global].hostname`. + +- `prefix = netdata`, is the prefix to add to all metrics. + +- `update every = 10`, is the number of seconds between sending data to the backend. netdata will add + some randomness to this number, to prevent stressing the backend server when many netdata servers send + data to the same backend. This randomness does not affect the quality of the data, only the time they + are sent. + +- `buffer on failures = 10`, is the number of iterations (each iteration is `[backend].update every` seconds) + to buffer data, when the backend is not available. If the backend fails to receive the data after that + many failures, data loss on the backend is expected (netdata will also log it). + +- `timeout ms = 20000`, is the timeout in milliseconds to wait for the backend server to process the data. + By default this is `2 * update_every * 1000`. + +- `send hosts matching = localhost *` includes one or more space separated patterns, using ` * ` as wildcard + (any number of times within each pattern). The patterns are checked against the hostname (the localhost + is always checked as `localhost`), allowing us to filter which hosts will be sent to the backend when + this netdata is a central netdata aggregating multiple hosts. A pattern starting with ` ! ` gives a + negative match. So to match all hosts named `*db*` except hosts containing `*slave*`, use + `!*slave* *db*` (so, the order is important: the first pattern matching the hostname will be used - positive + or negative). + +- `send charts matching = *` includes one or more space separated patterns, using ` * ` as wildcard (any + number of times within each pattern). The patterns are checked against both chart id and chart name. + A pattern starting with ` ! ` gives a negative match. So to match all charts named `apps.*` + except charts ending in `*reads`, use `!*reads apps.*` (so, the order is important: the first pattern + matching the chart id or the chart name will be used - positive or negative). + +- `send names instead of ids = yes | no` controls the metric names netdata should send to backend. + netdata supports names and IDs for charts and dimensions. Usually IDs are unique identifiers as read + by the system and names are human friendly labels (also unique). Most charts and metrics have the same + ID and name, but in several cases they are different: disks with device-mapper, interrupts, QoS classes, + statsd synthetic charts, etc. + +- `host tags = list of TAG=VALUE` defines tags that should be appended on all metrics for the given host. + These are currently only sent to opentsdb and prometheus. Please use the appropriate format for each + time-series db. For example opentsdb likes them like `TAG1=VALUE1 TAG2=VALUE2`, but prometheus like + `tag1="value1",tag2="value2"`. Host tags are mirrored with database replication (streaming of metrics + between netdata servers). + +## monitoring operation + +netdata provides 5 charts: + +1. **Buffered metrics**, the number of metrics netdata added to the buffer for dispatching them to the + backend server. + +2. **Buffered data size**, the amount of data (in KB) netdata added the buffer. + +3. ~~**Backend latency**, the time the backend server needed to process the data netdata sent. + If there was a re-connection involved, this includes the connection time.~~ + (this chart has been removed, because it only measures the time netdata needs to give the data + to the O/S - since the backend servers do not ack the reception, netdata does not have any means + to measure this properly). + +4. **Backend operations**, the number of operations performed by netdata. + +5. **Backend thread CPU usage**, the CPU resources consumed by the netdata thread, that is responsible + for sending the metrics to the backend server. + + + +## alarms + +The latest version of the alarms configuration for monitoring the backend is [here](../health/health.d/backend.conf) + +netdata adds 4 alarms: + +1. `backend_last_buffering`, number of seconds since the last successful buffering of backend data +2. `backend_metrics_sent`, percentage of metrics sent to the backend server +3. `backend_metrics_lost`, number of metrics lost due to repeating failures to contact the backend server +4. ~~`backend_slow`, the percentage of time between iterations needed by the backend time to process the data sent by netdata~~ (this was misleading and has been removed). + + + + +[]() diff --git a/backends/WALKTHROUGH.md b/backends/WALKTHROUGH.md new file mode 100644 index 0000000..0c330ee --- /dev/null +++ b/backends/WALKTHROUGH.md @@ -0,0 +1,294 @@ +# Netdata, Prometheus, Grafana stack + +## Intro +In this article I will walk you through the basics of getting Netdata, +Prometheus and Grafana all working together and monitoring your application +servers. This article will be using docker on your local workstation. We will be +working with docker in an ad-hoc way, launching containers that run ‘/bin/bash’ +and attaching a TTY to them. I use docker here in a purely academic fashion and +do not condone running Netdata in a container. I pick this method so individuals +without cloud accounts or access to VMs can try this out and for it’s speed of +deployment. + +## Why Netdata, Prometheus, and Grafana +Some time ago I was introduced to Netdata by a coworker. We were attempting to +troubleshoot python code which seemed to be bottlenecked. I was instantly +impressed by the amount of metrics Netdata exposes to you. I quickly added +Netdata to my set of go-to tools when troubleshooting systems performance. + +Some time ago, even later, I was introduced to Prometheus. Prometheus is a +monitoring application which flips the normal architecture around and polls +rest endpoints for its metrics. This architectural change greatly simplifies +and decreases the time necessary to begin monitoring your applications. +Compared to current monitoring solutions the time spent on designing the +infrastructure is greatly reduced. Running a single Prometheus server per +application becomes feasible with the help of Grafana. + +Grafana has been the go to graphing tool for… some time now. It’s awesome, +anyone that has used it knows it’s awesome. We can point Grafana at Prometheus +and use Prometheus as a data source. This allows a pretty simple overall +monitoring architecture: Install Netdata on your application servers, point +Prometheus at Netdata, and then point Grafana at Prometheus. + +I’m omitting an important ingredient in this stack in order to keep this tutorial +simple and that is service discovery. My personal preference is to use Consul. +Prometheus can plug into consul and automatically begin to scrape new hosts that +register a Netdata client with Consul. + +At the end of this tutorial you will understand how each technology fits +together to create a modern monitoring stack. This stack will offer you +visibility into your application and systems performance. + +## Getting Started - Netdata +To begin let’s create our container which we will install Netdata on. We need +to run a container, forward the necessary port that netdata listens on, and +attach a tty so we can interact with the bash shell on the container. But +before we do this we want name resolution between the two containers to work. +In order to accomplish this we will create a user-defined network and attach +both containers to this network. The first command we should run is: + +``` +docker network create --driver bridge netdata-tutorial +``` + +With this user-defined network created we can now launch our container we will +install Netdata on and point it to this network. + +``` +docker run -it --name netdata --hostname netdata --network=netdata-tutorial -p 19999:19999 centos:latest '/bin/bash' +``` + +This command creates an interactive tty session (-it), gives the container both +a name in relation to the docker daemon and a hostname (this is so you know what +container is which when working in the shells and docker maps hostname +resolution to this container), forwards the local port 19999 to the container’s +port 19999 (-p 19999:19999), sets the command to run (/bin/bash) and then +chooses the base container images (centos:latest). After running this you should +be sitting inside the shell of the container. + +After we have entered the shell we can install Netdata. This process could not +be easier. If you take a look at [this link](../packaging/installer/#installation), the Netdata devs give us +several one-liners to install netdata. I have not had any issues with these one +liners and their bootstrapping scripts so far (If you guys run into anything do +share). Run the following command in your container. + +``` +bash <(curl -Ss https://my-netdata.io/kickstart.sh) --dont-wait +``` + +After the install completes you should be able to hit the Netdata dashboard at +http://localhost:19999/ (replace localhost if you’re doing this on a VM or have +the docker container hosted on a machine not on your local system). If this is +your first time using Netdata I suggest you take a look around. The amount of +time I’ve spent digging through /proc and calculating my own metrics has been +greatly reduced by this tool. Take it all in. + +Next I want to draw your attention to a particular endpoint. Navigate to +http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes In your +browser. This is the endpoint which publishes all the metrics in a format which +Prometheus understands. Let’s take a look at one of these metrics. +`netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} +0.0831255 1501271696000` This metric is representing several things which I will +go in more details in the section on prometheus. For now understand that this +metric: `netdata_system_cpu_percentage_average` has several labels: [chart, +family, dimension]. This corresponds with the first cpu chart you see on the +Netdata dashboard. + + + +This CHART is called ‘system.cpu’, The FAMILY is cpu, and the DIMENSION we are +observing is “system”. You can begin to draw links between the charts in netdata +to the prometheus metrics format in this manner. + +## Prometheus +We will be installing prometheus in a container for purpose of demonstration. +While prometheus does have an official container I would like to walk through +the install process and setup on a fresh container. This will allow anyone +reading to migrate this tutorial to a VM or Server of any sort. + +Let’s start another container in the same fashion as we did the Netdata +container. `docker run -it --name prometheus --hostname prometheus +--network=netdata-tutorial -p 9090:9090 centos:latest '/bin/bash'` This should +drop you into a shell once again. Once there quickly install your favorite +editor as we will be editing files later in this tutorial. `yum install vim -y` + +Prometheus provides a tarball of their latest stable versions here: +https://prometheus.io/download/. Let’s download the latest version and install +into your container. + +``` +curl -L 'https://github.com/prometheus/prometheus/releases/download/v1.7.1/prometheus-1.7.1.linux-amd64.tar.gz' -o /tmp/prometheus.tar.gz + +mkdir /opt/prometheus + +tar -xf /tmp/prometheus.tar.gz -C /opt/prometheus/ --strip-components 1 +``` + +This should get prometheus installed into the container. Let’s test that we can run +prometheus and connect to it’s web interface. This will look similar to what +follows: + +``` +[root@prometheus prometheus]# /opt/prometheus/prometheus +INFO[0000] Starting prometheus (version=1.7.1, branch=master, revision=3afb3fffa3a29c3de865e1172fb740442e9d0133) + source="main.go:88" +INFO[0000] Build context (go=go1.8.3, user=root@0aa1b7fc430d, date=20170612-11:44:05) source="main.go:89" +INFO[0000] Host details (Linux 4.9.36-moby #1 SMP Wed Jul 12 15:29:07 UTC 2017 x86_64 prometheus (none)) source="main.go:90" +INFO[0000] Loading configuration file prometheus.yml source="main.go:252" +INFO[0000] Loading series map and head chunks... source="storage.go:428" +INFO[0000] 0 series loaded. source="storage.go:439" +INFO[0000] Starting target manager... source="targetmanager.go:63" +INFO[0000] Listening on :9090 source="web.go:259" +``` + +Now attempt to go to http://localhost:9090/. You should be presented with the +prometheus homepage. This is a good point to talk about Prometheus’s data model +which can be viewed here: https://prometheus.io/docs/concepts/data_model/ As +explained we have two key elements in Prometheus metrics. We have the ‘metric’ +and its ‘labels’. Labels allow for granularity between metrics. Let’s use our +previous example to further explain. + +``` +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0831255 1501271696000 +``` + +Here our metric is +‘netdata_system_cpu_percentage_average’ and our labels are ‘chart’, ‘family’, +and ‘dimension. The last two values constitute the actual metric value for the +metric type (gauge, counter, etc…). We can begin graphing system metrics with +this information, but first we need to hook up Prometheus to poll Netdata stats. + +Let’s move our attention to Prometheus’s configuration. Prometheus gets it +config from the file located (in our example) at +`/opt/prometheus/prometheus.yml`. I won’t spend an extensive amount of time +going over the configuration values documented here: +https://prometheus.io/docs/operating/configuration/. We will be adding a new +“job” under the “scrape_configs”. Let’s make the “scrape_configs” section look +like this (we can use the dns name Netdata due to the custom user-defined +network we created in docker beforehand). + +```yml +scrape_configs: + # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'netdata' + + metrics_path: /api/v1/allmetrics + params: + format: [ prometheus ] + + static_configs: + - targets: ['netdata:19999'] +``` + +Let’s start prometheus once again by running `/opt/prometheus/prometheus`. If we +now navigate to prometheus at ‘http://localhost:9090/targets’ we should see our +target being successfully scraped. If we now go back to the Prometheus’s +homepage and begin to type ‘netdata_’ Prometheus should auto complete metrics +it is now scraping. + + + +Let’s now start exploring how we can graph some metrics. Back in our NetData +container lets get the CPU spinning with a pointless busy loop. On the shell do +the following: + +``` +[root@netdata /]# while true; do echo "HOT HOT HOT CPU"; done +``` + +Our NetData cpu graph should be showing some activity. Let’s represent this in +Prometheus. In order to do this let’s keep our metrics page open for reference: +http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes We are +setting out to graph the data in the CPU chart so let’s search for “system.cpu” +in the metrics page above. We come across a section of metrics with the first +comments `# COMMENT homogeneus chart "system.cpu", context "system.cpu", family +"cpu", units "percentage"` Followed by the metrics. This is a good start now let +us drill down to the specific metric we would like to graph. + +``` +# COMMENT +netdata_system_cpu_percentage_average: dimension "system", value is percentage, gauge, dt 1501275951 to 1501275951 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0000000 1501275951000 +``` + +Here we learn that the metric name we care about is +‘netdata_system_cpu_percentage_average’ so throw this into Prometheus and see +what we get. We should see something similar to this (I shut off my busy loop) + + + +This is a good step toward what we want. Also make note that Prometheus will tag +on an ‘instance’ label for us which corresponds to our statically defined job in +the configuration file. This allows us to tailor our queries to specific +instances. Now we need to isolate the dimension we want in our query. To do this +let us refine the query slightly. Let’s query the dimension also. Place this +into our query text box. +`netdata_system_cpu_percentage_average{dimension="system"}` We now wind up with +the following graph. + + + +Awesome, this is exactly what we wanted. If you haven’t caught on yet we can +emulate entire charts from NetData by using the `chart` dimension. If you’d like +you can combine the ‘chart’ and ‘instance’ dimension to create per-instance +charts. Let’s give this a try: +`netdata_system_cpu_percentage_average{chart="system.cpu", instance="netdata:19999"}` + +This is the basics of using Prometheus to query NetData. I’d advise everyone at +this point to read [this page](../backends/prometheus/#using-netdata-with-prometheus). +The key point here is that NetData can export metrics from its internal DB or +can send metrics “as-collected” by specifying the ‘source=as-collected’ url +parameter like so. +http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes&types=yes&source=as-collected +If you choose to use this method you will need to use Prometheus's set of +functions here: https://prometheus.io/docs/querying/functions/ to obtain useful +metrics as you are now dealing with raw counters from the system. For example +you will have to use the `irate()` function over a counter to get that metric’s +rate per second. If your graphing needs are met by using the metrics returned by +NetData’s internal database (not specifying any source= url parameter) then use +that. If you find limitations then consider re-writing your queries using the +raw data and using Prometheus functions to get the desired chart. + +## Grafana +Finally we make it to grafana. This is the easiest part in my opinion. This time +we will actually run the official grafana docker container as all configuration +we need to do is done via the GUI. Let’s run the following command: + +``` +docker run -i -p 3000:3000 --network=netdata-tutorial grafana/grafana +``` + +This will get grafana running at ‘http://localhost:3000/’ Let’s go there and +login using the credentials Admin:Admin. + +The first thing we want to do is click ‘Add data source’. Let’s make it look +like the following screenshot + + + +With this completed let’s graph! Create a new Dashboard by clicking on the top +left Grafana Icon and create a new graph in that dashboard. Fill in the query +like we did above and save. + + + +## Conclusion + +There you have it, a complete systems monitoring stack which is very easy to +deploy. From here I would begin to understand how Prometheus and a service +discovery mechanism such as Consul can play together nicely. My current prod +deployments automatically register Netdata services into Consul and Prometheus +automatically begins to scrape them. Once achieved you do not have to think +about the monitoring system until Prometheus cannot keep up with your scale. +Once this happens there are options presented in the Prometheus documentation +for solving this. Hope this was helpful, happy monitoring. + +[]() diff --git a/backends/backends.c b/backends/backends.c new file mode 100644 index 0000000..da818c5 --- /dev/null +++ b/backends/backends.c @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#define BACKENDS_INTERNALS +#include "backends.h" + +// ---------------------------------------------------------------------------- +// How backends work in netdata: +// +// 1. There is an independent thread that runs at the required interval +// (for example, once every 10 seconds) +// +// 2. Every time it wakes, it calls the backend formatting functions to build +// a buffer of data. This is a very fast, memory only operation. +// +// 3. If the buffer already includes data, the new data are appended. +// If the buffer becomes too big, because the data cannot be sent, a +// log is written and the buffer is discarded. +// +// 4. Then it tries to send all the data. It blocks until all the data are sent +// or the socket returns an error. +// If the time required for this is above the interval, it starts skipping +// intervals, but the calculated values include the entire database, without +// gaps (it remembers the timestamps and continues from where it stopped). +// +// 5. repeats the above forever. +// + +const char *global_backend_prefix = "netdata"; +int global_backend_update_every = 10; +BACKEND_OPTIONS global_backend_options = BACKEND_SOURCE_DATA_AVERAGE | BACKEND_OPTION_SEND_NAMES; + +// ---------------------------------------------------------------------------- +// helper functions for backends + +size_t backend_name_copy(char *d, const char *s, size_t usable) { + size_t n; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + char c = *s; + + if(c != '.' && !isalnum(c)) *d = '_'; + else *d = c; + } + *d = '\0'; + + return n; +} + +// calculate the SUM or AVERAGE of a dimension, for any timeframe +// may return NAN if the database does not have any value in the give timeframe + +calculated_number backend_calculate_value_from_stored_data( + RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap + , time_t *first_timestamp // the first point of the database used in this response + , time_t *last_timestamp // the timestamp that should be reported to backend +) { + RRDHOST *host = st->rrdhost; + (void)host; + + // find the edges of the rrd database for this chart + time_t first_t = rrdset_first_entry_t(st); + time_t last_t = rrdset_last_entry_t(st); + time_t update_every = st->update_every; + + // step back a little, to make sure we have complete data collection + // for all metrics + after -= update_every * 2; + before -= update_every * 2; + + // align the time-frame + after = after - (after % update_every); + before = before - (before % update_every); + + // for before, loose another iteration + // the latest point will be reported the next time + before -= update_every; + + if(unlikely(after > before)) + // this can happen when update_every > before - after + after = before; + + if(unlikely(after < first_t)) + after = first_t; + + if(unlikely(before > last_t)) + before = last_t; + + if(unlikely(before < first_t || after > last_t)) { + // the chart has not been updated in the wanted timeframe + debug(D_BACKEND, "BACKEND: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu", + host->hostname, st->id, rd->id, + (unsigned long)after, (unsigned long)before, + (unsigned long)first_t, (unsigned long)last_t + ); + return NAN; + } + + *first_timestamp = after; + *last_timestamp = before; + + size_t counter = 0; + calculated_number sum = 0; + + long start_at_slot = rrdset_time2slot(st, before), + stop_at_slot = rrdset_time2slot(st, after), + slot, stop_now = 0; + + for(slot = start_at_slot; !stop_now ; slot--) { + + if(unlikely(slot < 0)) slot = st->entries - 1; + if(unlikely(slot == stop_at_slot)) stop_now = 1; + + storage_number n = rd->values[slot]; + + if(unlikely(!does_storage_number_exist(n))) { + // not collected + continue; + } + + calculated_number value = unpack_storage_number(n); + sum += value; + + counter++; + } + + if(unlikely(!counter)) { + debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu", + host->hostname, st->id, rd->id, + (unsigned long)after, (unsigned long)before + ); + return NAN; + } + + if(unlikely(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)) + return sum; + + return sum / (calculated_number)counter; +} + + +// discard a response received by a backend +// after logging a simple of it to error.log + +int discard_response(BUFFER *b, const char *backend) { + char sample[1024]; + const char *s = buffer_tostring(b); + char *d = sample, *e = &sample[sizeof(sample) - 1]; + + for(; *s && d < e ;s++) { + char c = *s; + if(unlikely(!isprint(c))) c = ' '; + *d++ = c; + } + *d = '\0'; + + info("BACKEND: received %zu bytes from %s backend. Ignoring them. Sample: '%s'", buffer_strlen(b), backend, sample); + buffer_flush(b); + return 0; +} + + +// ---------------------------------------------------------------------------- +// the backend thread + +static SIMPLE_PATTERN *charts_pattern = NULL; +static SIMPLE_PATTERN *hosts_pattern = NULL; + +inline int backends_can_send_rrdset(BACKEND_OPTIONS backend_options, RRDSET *st) { + RRDHOST *host = st->rrdhost; + (void)host; + + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE))) + return 0; + + if(unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) { + // we have not checked this chart + if(simple_pattern_matches(charts_pattern, st->id) || simple_pattern_matches(charts_pattern, st->name)) + rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND); + else { + rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE); + debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, host->hostname); + return 0; + } + } + + if(unlikely(!rrdset_is_available_for_backends(st))) { + debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.", st->id, host->hostname); + return 0; + } + + if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED))) { + debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode)); + return 0; + } + + return 1; +} + +inline BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options) { + if(!strcmp(source, "raw") || !strcmp(source, "as collected") || !strcmp(source, "as-collected") || !strcmp(source, "as_collected") || !strcmp(source, "ascollected")) { + backend_options |= BACKEND_SOURCE_DATA_AS_COLLECTED; + backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AS_COLLECTED); + } + else if(!strcmp(source, "average")) { + backend_options |= BACKEND_SOURCE_DATA_AVERAGE; + backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AVERAGE); + } + else if(!strcmp(source, "sum") || !strcmp(source, "volume")) { + backend_options |= BACKEND_SOURCE_DATA_SUM; + backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_SUM); + } + else { + error("BACKEND: invalid data source method '%s'.", source); + } + + return backend_options; +} + +static void backends_main_cleanup(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + + info("cleaning up..."); + + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; +} + +void *backends_main(void *ptr) { + netdata_thread_cleanup_push(backends_main_cleanup, ptr); + + int default_port = 0; + int sock = -1; + BUFFER *b = buffer_create(1), *response = buffer_create(1); + int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS) = NULL; + int (*backend_response_checker)(BUFFER *) = NULL; + + // ------------------------------------------------------------------------ + // collect configuration options + + struct timeval timeout = { + .tv_sec = 0, + .tv_usec = 0 + }; + int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0); + const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average"); + const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite"); + const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost"); + global_backend_prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata"); + const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname); + global_backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", global_backend_update_every); + int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10); + long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", global_backend_update_every * 2 * 1000); + + if(config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", (global_backend_options & BACKEND_OPTION_SEND_NAMES))) + global_backend_options |= BACKEND_OPTION_SEND_NAMES; + else + global_backend_options &= ~BACKEND_OPTION_SEND_NAMES; + + charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT); + hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT); + + + // ------------------------------------------------------------------------ + // validate configuration options + // and prepare for sending data to our backend + + global_backend_options = backend_parse_data_source(source, global_backend_options); + + if(timeoutms < 1) { + error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, global_backend_update_every * 2 * 1000); + timeoutms = global_backend_update_every * 2 * 1000; + } + timeout.tv_sec = (timeoutms * 1000) / 1000000; + timeout.tv_usec = (timeoutms * 1000) % 1000000; + + if(!enabled || global_backend_update_every < 1) + goto cleanup; + + // ------------------------------------------------------------------------ + // select the backend type + + if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) { + + default_port = 2003; + backend_response_checker = process_graphite_response; + + if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) + backend_request_formatter = format_dimension_collected_graphite_plaintext; + else + backend_request_formatter = format_dimension_stored_graphite_plaintext; + + } + else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) { + + default_port = 4242; + backend_response_checker = process_opentsdb_response; + + if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) + backend_request_formatter = format_dimension_collected_opentsdb_telnet; + else + backend_request_formatter = format_dimension_stored_opentsdb_telnet; + + } + else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) { + + default_port = 5448; + backend_response_checker = process_json_response; + + if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) + backend_request_formatter = format_dimension_collected_json_plaintext; + else + backend_request_formatter = format_dimension_stored_json_plaintext; + + } + else { + error("BACKEND: Unknown backend type '%s'", type); + goto cleanup; + } + + if(backend_request_formatter == NULL || backend_response_checker == NULL) { + error("BACKEND: backend is misconfigured - disabling it."); + goto cleanup; + } + + + // ------------------------------------------------------------------------ + // prepare the charts for monitoring the backend operation + + struct rusage thread; + + collected_number + chart_buffered_metrics = 0, + chart_lost_metrics = 0, + chart_sent_metrics = 0, + chart_buffered_bytes = 0, + chart_received_bytes = 0, + chart_sent_bytes = 0, + chart_receptions = 0, + chart_transmission_successes = 0, + chart_transmission_failures = 0, + chart_data_lost_events = 0, + chart_lost_bytes = 0, + chart_backend_reconnects = 0; + // chart_backend_latency = 0; + + RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, global_backend_update_every, RRDSET_TYPE_LINE); + rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KiB", "backends", NULL, 130610, global_backend_update_every, RRDSET_TYPE_AREA); + rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE); + + RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", "backends", NULL, 130630, global_backend_update_every, RRDSET_TYPE_LINE); + rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_ops, "failure", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(chart_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + + /* + * this is misleading - we can only measure the time we need to send data + * this time is not related to the time required for the data to travel to + * the backend database and the time that server needed to process them + * + * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html + * + RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, global_backend_update_every, RRDSET_TYPE_AREA); + rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + */ + + RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", "backends", NULL, 130630, global_backend_update_every, RRDSET_TYPE_STACKED); + rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); + + + // ------------------------------------------------------------------------ + // prepare the backend main loop + + info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, global_backend_update_every, hostname, global_backend_prefix); + + usec_t step_ut = global_backend_update_every * USEC_PER_SEC; + time_t after = now_realtime_sec(); + int failures = 0; + heartbeat_t hb; + heartbeat_init(&hb); + + while(!netdata_exit) { + + // ------------------------------------------------------------------------ + // Wait for the next iteration point. + + heartbeat_next(&hb, step_ut); + time_t before = now_realtime_sec(); + debug(D_BACKEND, "BACKEND: preparing buffer for timeframe %lu to %lu", (unsigned long)after, (unsigned long)before); + + // ------------------------------------------------------------------------ + // add to the buffer the data we need to send to the backend + + netdata_thread_disable_cancelability(); + + size_t count_hosts = 0; + size_t count_charts_total = 0; + size_t count_dims_total = 0; + + rrd_rdlock(); + RRDHOST *host; + rrdhost_foreach_read(host) { + if(unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_BACKEND_SEND|RRDHOST_FLAG_BACKEND_DONT_SEND))) { + char *name = (host == localhost)?"localhost":host->hostname; + if (!hosts_pattern || simple_pattern_matches(hosts_pattern, name)) { + rrdhost_flag_set(host, RRDHOST_FLAG_BACKEND_SEND); + info("enabled backend for host '%s'", name); + } + else { + rrdhost_flag_set(host, RRDHOST_FLAG_BACKEND_DONT_SEND); + info("disabled backend for host '%s'", name); + } + } + + if(unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_BACKEND_SEND))) + continue; + + rrdhost_rdlock(host); + + count_hosts++; + size_t count_charts = 0; + size_t count_dims = 0; + size_t count_dims_skipped = 0; + + const char *__hostname = (host == localhost)?hostname:host->hostname; + + RRDSET *st; + rrdset_foreach_read(st, host) { + if(likely(backends_can_send_rrdset(global_backend_options, st))) { + rrdset_rdlock(st); + + count_charts++; + + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if (likely(rd->last_collected_time.tv_sec >= after)) { + chart_buffered_metrics += backend_request_formatter(b, global_backend_prefix, host, __hostname, st, rd, after, before, global_backend_options); + count_dims++; + } + else { + debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before); + count_dims_skipped++; + } + } + + rrdset_unlock(st); + } + } + + debug(D_BACKEND, "BACKEND: sending host '%s', metrics of %zu dimensions, of %zu charts. Skipped %zu dimensions.", __hostname, count_dims, count_charts, count_dims_skipped); + count_charts_total += count_charts; + count_dims_total += count_dims; + + rrdhost_unlock(host); + } + rrd_unlock(); + + netdata_thread_enable_cancelability(); + + debug(D_BACKEND, "BACKEND: buffer has %zu bytes, added metrics for %zu dimensions, of %zu charts, from %zu hosts", buffer_strlen(b), count_dims_total, count_charts_total, count_hosts); + + // ------------------------------------------------------------------------ + + chart_buffered_bytes = (collected_number)buffer_strlen(b); + + // reset the monitoring chart counters + chart_received_bytes = + chart_sent_bytes = + chart_sent_metrics = + chart_lost_metrics = + chart_transmission_successes = + chart_transmission_failures = + chart_data_lost_events = + chart_lost_bytes = + chart_backend_reconnects = 0; + // chart_backend_latency = 0; + + if(unlikely(netdata_exit)) break; + + //fprintf(stderr, "\nBACKEND BEGIN:\n%s\nBACKEND END\n", buffer_tostring(b)); + //fprintf(stderr, "after = %lu, before = %lu\n", after, before); + + // prepare for the next iteration + // to add incrementally data to buffer + after = before; + + // ------------------------------------------------------------------------ + // if we are connected, receive a response, without blocking + + if(likely(sock != -1)) { + errno = 0; + + // loop through to collect all data + while(sock != -1 && errno != EWOULDBLOCK) { + buffer_need_bytes(response, 4096); + + ssize_t r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT); + if(likely(r > 0)) { + // we received some data + response->len += r; + chart_received_bytes += r; + chart_receptions++; + } + else if(r == 0) { + error("BACKEND: '%s' closed the socket", destination); + close(sock); + sock = -1; + } + else { + // failed to receive data + if(errno != EAGAIN && errno != EWOULDBLOCK) { + error("BACKEND: cannot receive data from backend '%s'.", destination); + } + } + } + + // if we received data, process them + if(buffer_strlen(response)) + backend_response_checker(response); + } + + // ------------------------------------------------------------------------ + // if we are not connected, connect to a backend server + + if(unlikely(sock == -1)) { + // usec_t start_ut = now_monotonic_usec(); + size_t reconnects = 0; + + sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0); + + chart_backend_reconnects += reconnects; + // chart_backend_latency += now_monotonic_usec() - start_ut; + } + + if(unlikely(netdata_exit)) break; + + // ------------------------------------------------------------------------ + // if we are connected, send our buffer to the backend server + + if(likely(sock != -1)) { + size_t len = buffer_strlen(b); + // usec_t start_ut = now_monotonic_usec(); + int flags = 0; +#ifdef MSG_NOSIGNAL + flags += MSG_NOSIGNAL; +#endif + + ssize_t written = send(sock, buffer_tostring(b), len, flags); + // chart_backend_latency += now_monotonic_usec() - start_ut; + if(written != -1 && (size_t)written == len) { + // we sent the data successfully + chart_transmission_successes++; + chart_sent_bytes += written; + chart_sent_metrics = chart_buffered_metrics; + + // reset the failures count + failures = 0; + + // empty the buffer + buffer_flush(b); + } + else { + // oops! we couldn't send (all or some of the) data + error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written); + chart_transmission_failures++; + + if(written != -1) + chart_sent_bytes += written; + + // increment the counter we check for data loss + failures++; + + // close the socket - we will re-open it next time + close(sock); + sock = -1; + } + } + else { + error("BACKEND: failed to update database backend '%s'", destination); + chart_transmission_failures++; + + // increment the counter we check for data loss + failures++; + } + + if(failures > buffer_on_failures) { + // too bad! we are going to lose data + chart_lost_bytes += buffer_strlen(b); + error("BACKEND: reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination); + buffer_flush(b); + failures = 0; + chart_data_lost_events++; + chart_lost_metrics = chart_buffered_metrics; + } + + if(unlikely(netdata_exit)) break; + + // ------------------------------------------------------------------------ + // update the monitoring charts + + if(likely(chart_ops->counter_done)) rrdset_next(chart_ops); + rrddim_set(chart_ops, "read", chart_receptions); + rrddim_set(chart_ops, "write", chart_transmission_successes); + rrddim_set(chart_ops, "discard", chart_data_lost_events); + rrddim_set(chart_ops, "failure", chart_transmission_failures); + rrddim_set(chart_ops, "reconnect", chart_backend_reconnects); + rrdset_done(chart_ops); + + if(likely(chart_metrics->counter_done)) rrdset_next(chart_metrics); + rrddim_set(chart_metrics, "buffered", chart_buffered_metrics); + rrddim_set(chart_metrics, "lost", chart_lost_metrics); + rrddim_set(chart_metrics, "sent", chart_sent_metrics); + rrdset_done(chart_metrics); + + if(likely(chart_bytes->counter_done)) rrdset_next(chart_bytes); + rrddim_set(chart_bytes, "buffered", chart_buffered_bytes); + rrddim_set(chart_bytes, "lost", chart_lost_bytes); + rrddim_set(chart_bytes, "sent", chart_sent_bytes); + rrddim_set(chart_bytes, "received", chart_received_bytes); + rrdset_done(chart_bytes); + + /* + if(likely(chart_latency->counter_done)) rrdset_next(chart_latency); + rrddim_set(chart_latency, "latency", chart_backend_latency); + rrdset_done(chart_latency); + */ + + getrusage(RUSAGE_THREAD, &thread); + if(likely(chart_rusage->counter_done)) rrdset_next(chart_rusage); + rrddim_set(chart_rusage, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set(chart_rusage, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrdset_done(chart_rusage); + + if(likely(buffer_strlen(b) == 0)) + chart_buffered_metrics = 0; + + if(unlikely(netdata_exit)) break; + } + +cleanup: + if(sock != -1) + close(sock); + + buffer_free(b); + buffer_free(response); + + netdata_thread_cleanup_pop(1); + return NULL; +} diff --git a/backends/backends.h b/backends/backends.h new file mode 100644 index 0000000..468e4fe --- /dev/null +++ b/backends/backends.h @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_BACKENDS_H +#define NETDATA_BACKENDS_H 1 + +#include "daemon/common.h" + +typedef enum backend_options { + BACKEND_OPTION_NONE = 0, + + BACKEND_SOURCE_DATA_AS_COLLECTED = (1 << 0), + BACKEND_SOURCE_DATA_AVERAGE = (1 << 1), + BACKEND_SOURCE_DATA_SUM = (1 << 2), + + BACKEND_OPTION_SEND_NAMES = (1 << 16) +} BACKEND_OPTIONS; + +#define BACKEND_OPTIONS_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM) +#define BACKEND_OPTIONS_DATA_SOURCE(backend_options) (backend_options & BACKEND_OPTIONS_SOURCE_BITS) + +extern int global_backend_update_every; +extern BACKEND_OPTIONS global_backend_options; +extern const char *global_backend_prefix; + +extern void *backends_main(void *ptr); + +extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options); + +#ifdef BACKENDS_INTERNALS + +extern int backends_can_send_rrdset(BACKEND_OPTIONS backend_options, RRDSET *st); +extern calculated_number backend_calculate_value_from_stored_data( + RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap + , time_t *first_timestamp // the timestamp of the first point used in this response + , time_t *last_timestamp // the timestamp that should be reported to backend +); + +extern size_t backend_name_copy(char *d, const char *s, size_t usable); +extern int discard_response(BUFFER *b, const char *backend); + +#endif // BACKENDS_INTERNALS + +#include "backends/prometheus/backend_prometheus.h" +#include "backends/graphite/graphite.h" +#include "backends/json/json.h" +#include "backends/opentsdb/opentsdb.h" + +#endif /* NETDATA_BACKENDS_H */ diff --git a/backends/graphite/Makefile.am b/backends/graphite/Makefile.am new file mode 100644 index 0000000..babdcf0 --- /dev/null +++ b/backends/graphite/Makefile.am @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in diff --git a/backends/graphite/graphite.c b/backends/graphite/graphite.c new file mode 100644 index 0000000..8057038 --- /dev/null +++ b/backends/graphite/graphite.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#define BACKENDS_INTERNALS +#include "graphite.h" + +// ---------------------------------------------------------------------------- +// graphite backend + +int format_dimension_collected_graphite_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +) { + (void)host; + (void)after; + (void)before; + + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); + + buffer_sprintf( + b + , "%s.%s.%s.%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n" + , prefix + , hostname + , chart_name + , dimension_name + , (host->tags)?";":"" + , (host->tags)?host->tags:"" + , rd->last_collected_value + , (unsigned long long)rd->last_collected_time.tv_sec + ); + + return 1; +} + +int format_dimension_stored_graphite_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +) { + (void)host; + + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t); + + if(!isnan(value)) { + + buffer_sprintf( + b + , "%s.%s.%s.%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n" + , prefix + , hostname + , chart_name + , dimension_name + , (host->tags)?";":"" + , (host->tags)?host->tags:"" + , value + , (unsigned long long) last_t + ); + + return 1; + } + return 0; +} + +int process_graphite_response(BUFFER *b) { + return discard_response(b, "graphite"); +} + + diff --git a/backends/graphite/graphite.h b/backends/graphite/graphite.h new file mode 100644 index 0000000..b7b0930 --- /dev/null +++ b/backends/graphite/graphite.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + + +#ifndef NETDATA_BACKEND_GRAPHITE_H +#define NETDATA_BACKEND_GRAPHITE_H + +#include "backends/backends.h" + +extern int format_dimension_collected_graphite_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +); + +extern int format_dimension_stored_graphite_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +); + +extern int process_graphite_response(BUFFER *b); + +#endif //NETDATA_BACKEND_GRAPHITE_H diff --git a/backends/json/Makefile.am b/backends/json/Makefile.am new file mode 100644 index 0000000..babdcf0 --- /dev/null +++ b/backends/json/Makefile.am @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in diff --git a/backends/json/json.c b/backends/json/json.c new file mode 100644 index 0000000..a53c0f1 --- /dev/null +++ b/backends/json/json.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#define BACKENDS_INTERNALS +#include "json.h" + +// ---------------------------------------------------------------------------- +// json backend + +int format_dimension_collected_json_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +) { + (void)host; + (void)after; + (void)before; + (void)backend_options; + + const char *tags_pre = "", *tags_post = "", *tags = host->tags; + if(!tags) tags = ""; + + if(*tags) { + if(*tags == '{' || *tags == '[' || *tags == '"') { + tags_pre = "\"host_tags\":"; + tags_post = ","; + } + else { + tags_pre = "\"host_tags\":\""; + tags_post = "\","; + } + } + + buffer_sprintf(b, "{" + "\"prefix\":\"%s\"," + "\"hostname\":\"%s\"," + "%s%s%s" + + "\"chart_id\":\"%s\"," + "\"chart_name\":\"%s\"," + "\"chart_family\":\"%s\"," + "\"chart_context\": \"%s\"," + "\"chart_type\":\"%s\"," + "\"units\": \"%s\"," + + "\"id\":\"%s\"," + "\"name\":\"%s\"," + "\"value\":" COLLECTED_NUMBER_FORMAT "," + + "\"timestamp\": %llu}\n", + prefix, + hostname, + tags_pre, tags, tags_post, + + st->id, + st->name, + st->family, + st->context, + st->type, + st->units, + + rd->id, + rd->name, + rd->last_collected_value, + + (unsigned long long) rd->last_collected_time.tv_sec + ); + + return 1; +} + +int format_dimension_stored_json_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +) { + (void)host; + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t); + + if(!isnan(value)) { + const char *tags_pre = "", *tags_post = "", *tags = host->tags; + if(!tags) tags = ""; + + if(*tags) { + if(*tags == '{' || *tags == '[' || *tags == '"') { + tags_pre = "\"host_tags\":"; + tags_post = ","; + } + else { + tags_pre = "\"host_tags\":\""; + tags_post = "\","; + } + } + + buffer_sprintf(b, "{" + "\"prefix\":\"%s\"," + "\"hostname\":\"%s\"," + "%s%s%s" + + "\"chart_id\":\"%s\"," + "\"chart_name\":\"%s\"," + "\"chart_family\":\"%s\"," + "\"chart_context\": \"%s\"," + "\"chart_type\":\"%s\"," + "\"units\": \"%s\"," + + "\"id\":\"%s\"," + "\"name\":\"%s\"," + "\"value\":" CALCULATED_NUMBER_FORMAT "," + + "\"timestamp\": %llu}\n", + prefix, + hostname, + tags_pre, tags, tags_post, + + st->id, + st->name, + st->family, + st->context, + st->type, + st->units, + + rd->id, + rd->name, + value, + + (unsigned long long) last_t + ); + + return 1; + } + return 0; +} + +int process_json_response(BUFFER *b) { + return discard_response(b, "json"); +} + + diff --git a/backends/json/json.h b/backends/json/json.h new file mode 100644 index 0000000..1101565 --- /dev/null +++ b/backends/json/json.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_BACKEND_JSON_H +#define NETDATA_BACKEND_JSON_H + +#include "backends/backends.h" + +extern int format_dimension_collected_json_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +); + +extern int format_dimension_stored_json_plaintext( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +); + +extern int process_json_response(BUFFER *b); + +#endif //NETDATA_BACKEND_JSON_H diff --git a/backends/nc-backend.sh b/backends/nc-backend.sh new file mode 100755 index 0000000..7280f86 --- /dev/null +++ b/backends/nc-backend.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash + +# SPDX-License-Identifier: GPL-3.0-or-later + +# This is a simple backend database proxy, written in BASH, using the nc command. +# Run the script without any parameters for help. + +MODE="${1}" +MY_PORT="${2}" +BACKEND_HOST="${3}" +BACKEND_PORT="${4}" +FILE="${NETDATA_NC_BACKEND_DIR-/tmp}/netdata-nc-backend-${MY_PORT}" + +log() { + logger --stderr --id=$$ --tag "netdata-nc-backend" "${*}" +} + +mync() { + local ret + + log "Running: nc ${*}" + nc "${@}" + ret=$? + + log "nc stopped with return code ${ret}." + + return ${ret} +} + +listen_save_replay_forever() { + local file="${1}" port="${2}" real_backend_host="${3}" real_backend_port="${4}" ret delay=1 started ended + + while true + do + log "Starting nc to listen on port ${port} and save metrics to ${file}" + + started=$(date +%s) + mync -l -p "${port}" | tee -a -p --output-error=exit "${file}" + ended=$(date +%s) + + if [ -s "${file}" ] + then + if [ ! -z "${real_backend_host}" ] && [ ! -z "${real_backend_port}" ] + then + log "Attempting to send the metrics to the real backend at ${real_backend_host}:${real_backend_port}" + + mync "${real_backend_host}" "${real_backend_port}" <"${file}" + ret=$? + + if [ ${ret} -eq 0 ] + then + log "Successfuly sent the metrics to ${real_backend_host}:${real_backend_port}" + mv "${file}" "${file}.old" + touch "${file}" + else + log "Failed to send the metrics to ${real_backend_host}:${real_backend_port} (nc returned ${ret}) - appending more data to ${file}" + fi + else + log "No backend configured - appending more data to ${file}" + fi + fi + + # prevent a CPU hungry infinite loop + # if nc cannot listen to port + if [ $((ended - started)) -lt 5 ] + then + log "nc has been stopped too fast." + delay=30 + else + delay=1 + fi + + log "Waiting ${delay} seconds before listening again for data." + sleep ${delay} + done +} + +if [ "${MODE}" = "start" ] + then + + # start the listener, in exclusive mode + # only one can use the same file/port at a time + { + flock -n 9 + # shellcheck disable=SC2181 + if [ $? -ne 0 ] + then + log "Cannot get exclusive lock on file ${FILE}.lock - Am I running multiple times?" + exit 2 + fi + + # save our PID to the lock file + echo "$$" >"${FILE}.lock" + + listen_save_replay_forever "${FILE}" "${MY_PORT}" "${BACKEND_HOST}" "${BACKEND_PORT}" + ret=$? + + log "listener exited." + exit ${ret} + + } 9>>"${FILE}.lock" + + # we can only get here if ${FILE}.lock cannot be created + log "Cannot create file ${FILE}." + exit 3 + +elif [ "${MODE}" = "stop" ] + then + + { + flock -n 9 + # shellcheck disable=SC2181 + if [ $? -ne 0 ] + then + pid=$(<"${FILE}".lock) + log "Killing process ${pid}..." + kill -TERM "-${pid}" + exit 0 + fi + + log "File ${FILE}.lock has been locked by me but it shouldn't. Is a collector running?" + exit 4 + + } 9<"${FILE}.lock" + + log "File ${FILE}.lock does not exist. Is a collector running?" + exit 5 + +else + + cat <<EOF +Usage: + + "${0}" start|stop PORT [BACKEND_HOST BACKEND_PORT] + + PORT The port this script will listen + (configure netdata to use this as a second backend) + + BACKEND_HOST The real backend host + BACKEND_PORT The real backend port + + This script can act as fallback backend for netdata. + It will receive metrics from netdata, save them to + ${FILE} + and once netdata reconnects to the real-backend, this script + will push all metrics collected to the real-backend too and + wait for a failure to happen again. + + Only one netdata can connect to this script at a time. + If you need fallback for multiple netdata, run this script + multiple times with different ports. + + You can run me in the background with this: + + screen -d -m "${0}" start PORT [BACKEND_HOST BACKEND_PORT] +EOF + exit 1 +fi diff --git a/backends/opentsdb/Makefile.am b/backends/opentsdb/Makefile.am new file mode 100644 index 0000000..babdcf0 --- /dev/null +++ b/backends/opentsdb/Makefile.am @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in diff --git a/backends/opentsdb/opentsdb.c b/backends/opentsdb/opentsdb.c new file mode 100644 index 0000000..6e3a31a --- /dev/null +++ b/backends/opentsdb/opentsdb.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#define BACKENDS_INTERNALS +#include "opentsdb.h" + +// ---------------------------------------------------------------------------- +// opentsdb backend + +int format_dimension_collected_opentsdb_telnet( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +) { + (void)host; + (void)after; + (void)before; + + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); + + buffer_sprintf( + b + , "put %s.%s.%s %llu " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n" + , prefix + , chart_name + , dimension_name + , (unsigned long long)rd->last_collected_time.tv_sec + , rd->last_collected_value + , hostname + , (host->tags)?" ":"" + , (host->tags)?host->tags:"" + ); + + return 1; +} + +int format_dimension_stored_opentsdb_telnet( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +) { + (void)host; + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t); + + char chart_name[RRD_ID_LENGTH_MAX + 1]; + char dimension_name[RRD_ID_LENGTH_MAX + 1]; + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX); + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX); + + if(!isnan(value)) { + + buffer_sprintf( + b + , "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n" + , prefix + , chart_name + , dimension_name + , (unsigned long long) last_t + , value + , hostname + , (host->tags)?" ":"" + , (host->tags)?host->tags:"" + ); + + return 1; + } + return 0; +} + +int process_opentsdb_response(BUFFER *b) { + return discard_response(b, "opentsdb"); +} + + diff --git a/backends/opentsdb/opentsdb.h b/backends/opentsdb/opentsdb.h new file mode 100644 index 0000000..fc83b39 --- /dev/null +++ b/backends/opentsdb/opentsdb.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_BACKEND_OPENTSDB_H +#define NETDATA_BACKEND_OPENTSDB_H + +#include "backends/backends.h" + +extern int format_dimension_collected_opentsdb_telnet( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +); + +extern int format_dimension_stored_opentsdb_telnet( + BUFFER *b // the buffer to write data to + , const char *prefix // the prefix to use + , RRDHOST *host // the host this chart comes from + , const char *hostname // the hostname (to override host->hostname) + , RRDSET *st // the chart + , RRDDIM *rd // the dimension + , time_t after // the start timestamp + , time_t before // the end timestamp + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap +); + +extern int process_opentsdb_response(BUFFER *b); + + +#endif //NETDATA_BACKEND_OPENTSDB_H diff --git a/backends/prometheus/Makefile.am b/backends/prometheus/Makefile.am new file mode 100644 index 0000000..19554be --- /dev/null +++ b/backends/prometheus/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/backends/prometheus/README.md b/backends/prometheus/README.md new file mode 100644 index 0000000..8cf83cb --- /dev/null +++ b/backends/prometheus/README.md @@ -0,0 +1,382 @@ +# Using netdata with Prometheus + +> IMPORTANT: the format netdata sends metrics to prometheus has changed since netdata v1.7. The new prometheus backend for netdata supports a lot more features and is aligned to the development of the rest of the netdata backends. + +Prometheus is a distributed monitoring system which offers a very simple setup along with a robust data model. Recently netdata added support for Prometheus. I'm going to quickly show you how to install both netdata and prometheus on the same server. We can then use grafana pointed at Prometheus to obtain long term metrics netdata offers. I'm assuming we are starting at a fresh ubuntu shell (whether you'd like to follow along in a VM or a cloud instance is up to you). + + +## Installing netdata and prometheus + +### Installing netdata + +There are number of ways to install netdata according to [Installation](../../packaging/installer/#installation) +The suggested way of installing the latest netdata and keep it upgrade automatically. Using one line installation: + +``` +bash <(curl -Ss https://my-netdata.io/kickstart.sh) +``` + +At this point we should have netdata listening on port 19999. Attempt to take your browser here: + +``` +http://your.netdata.ip:19999 +``` + +*(replace `your.netdata.ip` with the IP or hostname of the server running netdata)* + +### Installing Prometheus + +In order to install prometheus we are going to introduce our own systemd startup script along with an example of prometheus.yaml configuration. Prometheus needs to be pointed to your server at a specific target url for it to scrape netdata's api. Prometheus is always a pull model meaning netdata is the passive client within this architecture. Prometheus always initiates the connection with netdata. + +#### Download Prometheus + +```sh +wget -O /tmp/prometheus-2.3.2.linux-amd64.tar.gz https://github.com/prometheus/prometheus/releases/download/v2.3.2/prometheus-2.3.2.linux-amd64.tar.gz +``` + +#### Create prometheus system user + +```sh +sudo useradd -r prometheus +``` + +#### Create prometheus directory + +```sh +sudo mkdir /opt/prometheus +sudo chown prometheus:prometheus /opt/prometheus +``` + +#### Untar prometheus directory + +```sh +sudo tar -xvf /tmp/prometheus-2.3.2.linux-amd64.tar.gz -C /opt/prometheus --strip=1 +``` + +#### Install prometheus.yml + +We will use the following `prometheus.yml` file. Save it at `/opt/prometheus/prometheus.yml`. + +Make sure to replace `your.netdata.ip` with the IP or hostname of the host running netdata. + +```yaml +# my global config +global: + scrape_interval: 5s # Set the scrape interval to every 5 seconds. Default is every 1 minute. + evaluation_interval: 5s # Evaluate rules every 5 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first.rules" + # - "second.rules" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['0.0.0.0:9090'] + + - job_name: 'netdata-scrape' + + metrics_path: '/api/v1/allmetrics' + params: + # format: prometheus | prometheus_all_hosts + # You can use `prometheus_all_hosts` if you want Prometheus to set the `instance` to your hostname instead of IP + format: [prometheus] + # + # sources: as-collected | raw | average | sum | volume + # default is: average + #source: [as-collected] + # + # server name for this prometheus - the default is the client IP + # for netdata to uniquely identify it + #server: ['prometheus1'] + honor_labels: true + + static_configs: + - targets: ['{your.netdata.ip}:19999'] +``` + +#### Install nodes.yml + +The following is completely optional, it will enable Prometheus to generate alerts from some NetData sources. Tweak the values to your own needs. We will use the following `nodes.yml` file below. Save it at `/opt/prometheus/nodes.yml`, and add a *- "nodes.yml"* entry under the *rule_files:* section in the example prometheus.yml file above. +``` +groups: +- name: nodes + + rules: + - alert: node_high_cpu_usage_70 + expr: avg(rate(netdata_cpu_cpu_percentage_average{dimension="idle"}[1m])) by (job) > 70 + for: 1m + annotations: + description: '{{ $labels.job }} on ''{{ $labels.job }}'' CPU usage is at {{ humanize $value }}%.' + summary: CPU alert for container node '{{ $labels.job }}' + + - alert: node_high_memory_usage_70 + expr: 100 / sum(netdata_system_ram_MB_average) by (job) + * sum(netdata_system_ram_MB_average{dimension=~"free|cached"}) by (job) < 30 + for: 1m + annotations: + description: '{{ $labels.job }} memory usage is {{ humanize $value}}%.' + summary: Memory alert for container node '{{ $labels.job }}' + + - alert: node_low_root_filesystem_space_20 + expr: 100 / sum(netdata_disk_space_GB_average{family="/"}) by (job) + * sum(netdata_disk_space_GB_average{family="/",dimension=~"avail|cached"}) by (job) < 20 + for: 1m + annotations: + description: '{{ $labels.job }} root filesystem space is {{ humanize $value}}%.' + summary: Root filesystem alert for container node '{{ $labels.job }}' + + - alert: node_root_filesystem_fill_rate_6h + expr: predict_linear(netdata_disk_space_GB_average{family="/",dimension=~"avail|cached"}[1h], 6 * 3600) < 0 + for: 1h + labels: + severity: critical + annotations: + description: Container node {{ $labels.job }} root filesystem is going to fill up in 6h. + summary: Disk fill alert for Swarm node '{{ $labels.job }}' +``` + +#### Install prometheus.service + +Save this service file as `/etc/systemd/system/prometheus.service`: + +``` +[Unit] +Description=Prometheus Server +AssertPathExists=/opt/prometheus + +[Service] +Type=simple +WorkingDirectory=/opt/prometheus +User=prometheus +Group=prometheus +ExecStart=/opt/prometheus/prometheus --config.file=/opt/prometheus/prometheus.yml --log.level=info +ExecReload=/bin/kill -SIGHUP $MAINPID +ExecStop=/bin/kill -SIGINT $MAINPID + +[Install] +WantedBy=multi-user.target +``` +##### Start Prometheus + +``` +sudo systemctl start prometheus +sudo systemctl enable prometheus +``` + +Prometheus should now start and listen on port 9090. Attempt to head there with your browser. + +If everything is working correctly when you fetch `http://your.prometheus.ip:9090` you will see a 'Status' tab. Click this and click on 'targets' We should see the netdata host as a scraped target. + +--- + +## Netdata support for prometheus + +> IMPORTANT: the format netdata sends metrics to prometheus has changed since netdata v1.6. The new format allows easier queries for metrics and supports both `as collected` and normalized metrics. + +Before explaining the changes, we have to understand the key differences between netdata and prometheus. + +### understanding netdata metrics + +##### charts + +Each chart in netdata has several properties (common to all its metrics): + +- `chart_id` - uniquely identifies a chart. + +- `chart_name` - a more human friendly name for `chart_id`, also unique. + +- `context` - this is the template of the chart. All disk I/O charts have the same context, all mysql requests charts have the same context, etc. This is used for alarm templates to match all the charts they should be attached to. + +- `family` groups a set of charts together. It is used as the submenu of the dashboard. + +- `units` is the units for all the metrics attached to the chart. + +##### dimensions + +Then each netdata chart contains metrics called `dimensions`. All the dimensions of a chart have the same units of measurement, and are contextually in the same category (ie. the metrics for disk bandwidth are `read` and `write` and they are both in the same chart). + +### netdata data source + +Netdata can send metrics to prometheus from 3 data sources: + +- `as collected` or `raw` - this data source sends the metrics to prometheus as they are collected. No conversion is done by netdata. The latest value for each metric is just given to prometheus. This is the most preferred method by prometheus, but it is also the harder to work with. To work with this data source, you will need to understand how to get meaningful values out of them. + + The format of the metrics is: `CONTEXT{chart="CHART",family="FAMILY",dimension="DIMENSION"}`. + + If the metric is a counter (`incremental` in netdata lingo), `_total` is appended the context. + + Unlike prometheus, netdata allows each dimension of a chart to have a different algorithm and conversion constants (`multiplier` and `divisor`). In this case, that the dimensions of a charts are heterogeneous, netdata will use this format: `CONTEXT_DIMENSION{chart="CHART",family="FAMILY"}` + +- `average` - this data source uses the netdata database to send the metrics to prometheus as they are presented on the netdata dashboard. So, all the metrics are sent as gauges, at the units they are presented in the netdata dashboard charts. This is the easiest to work with. + + The format of the metrics is: `CONTEXT_UNITS_average{chart="CHART",family="FAMILY",dimension="DIMENSION"}`. + + When this source is used, netdata keeps track of the last access time for each prometheus server fetching the metrics. This last access time is used at the subsequent queries of the same prometheus server to identify the time-frame the `average` will be calculated. So, no matter how frequently prometheus scrapes netdata, it will get all the database data. To identify each prometheus server, netdata uses by default the IP of the client fetching the metrics. If there are multiple prometheus servers fetching data from the same netdata, using the same IP, each prometheus server can append `server=NAME` to the URL. Netdata will use this `NAME` to uniquely identify the prometheus server. + +- `sum` or `volume`, is like `average` but instead of averaging the values, it sums them. + + The format of the metrics is: `CONTEXT_UNITS_sum{chart="CHART",family="FAMILY",dimension="DIMENSION"}`. + All the other operations are the same with `average`. + +Keep in mind that early versions of netdata were sending the metrics as: `CHART_DIMENSION{}`. + +### Querying Metrics + +Fetch with your web browser this URL: + +`http://your.netdata.ip:19999/api/v1/allmetrics?format=prometheus&help=yes` + +*(replace `your.netdata.ip` with the ip or hostname of your netdata server)* + +netdata will respond with all the metrics it sends to prometheus. + +If you search that page for `"system.cpu"` you will find all the metrics netdata is exporting to prometheus for this chart. `system.cpu` is the chart name on the netdata dashboard (on the netdata dashboard all charts have a text heading such as : `Total CPU utilization (system.cpu)`. What we are interested here in the chart name: `system.cpu`). + +Searching for `"system.cpu"` reveals: + +```sh +# COMMENT homogeneus chart "system.cpu", context "system.cpu", family "cpu", units "percentage" +# COMMENT netdata_system_cpu_percentage_average: dimension "guest_nice", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="guest_nice"} 0.0000000 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "guest", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="guest"} 1.7837326 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "steal", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="steal"} 0.0000000 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "softirq", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="softirq"} 0.5275442 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "irq", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="irq"} 0.2260836 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "user", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="user"} 2.3362762 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "system", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 1.7961062 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "nice", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="nice"} 0.0000000 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "iowait", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="iowait"} 0.9671802 1500066662000 +# COMMENT netdata_system_cpu_percentage_average: dimension "idle", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive +netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="idle"} 92.3630770 1500066662000 +``` +*(netdata response for `system.cpu` with source=`average`)* + +In `average` or `sum` data sources, all values are normalized and are reported to prometheus as gauges. Now, use the 'expression' text form in prometheus. Begin to type the metrics we are looking for: `netdata_system_cpu`. You should see that the text form begins to auto-fill as prometheus knows about this metric. + +If the data source was `as collected`, the response would be: + +```sh +# COMMENT homogeneus chart "system.cpu", context "system.cpu", family "cpu", units "percentage" +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "guest_nice", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="guest_nice"} 0 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "guest", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="guest"} 63945 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "steal", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="steal"} 0 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "softirq", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="softirq"} 8295 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "irq", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="irq"} 4079 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "user", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="user"} 116488 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "system", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="system"} 35084 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "nice", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="nice"} 505 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "iowait", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="iowait"} 23314 1500066716438 +# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "idle", value * 1 / 1 delta gives percentage (counter) +netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="idle"} 918470 1500066716438 +``` + +*(netdata response for `system.cpu` with source=`as-collected`)* + +For more information check prometheus documentation. + +### Streaming data from upstream hosts + +The `format=prometheus` parameter only exports the host's netdata metrics. If you are using the master/slave functionality of netdata this ignores any upstream hosts - so you should consider using the below in your **prometheus.yml**: + +``` + metrics_path: '/api/v1/allmetrics' + params: + format: [prometheus_all_hosts] + honor_labels: true +``` + +This will report all upstream host data, and `honor_labels` will make Prometheus take note of the instance names provided. + +### Timestamps + +To pass the metrics through prometheus pushgateway, netdata supports the option `×tamps=no` to send the metrics without timestamps. + +## Netdata host variables + +netdata collects various system configuration metrics, like the max number of TCP sockets supported, the max number of files allowed system-wide, various IPC sizes, etc. These metrics are not exposed to prometheus by default. + +To expose them, append `variables=yes` to the netdata URL. + +### TYPE and HELP + +To save bandwidth, and because prometheus does not use them anyway, `# TYPE` and `# HELP` lines are suppressed. If wanted they can be re-enabled via `types=yes` and `help=yes`, e.g. `/api/v1/allmetrics?format=prometheus&types=yes&help=yes` + +### Names and IDs + +netdata supports names and IDs for charts and dimensions. Usually IDs are unique identifiers as read by the system and names are human friendly labels (also unique). + +Most charts and metrics have the same ID and name, but in several cases they are different: disks with device-mapper, interrupts, QoS classes, statsd synthetic charts, etc. + +The default is controlled in `netdata.conf`: + +``` +[backend] + send names instead of ids = yes | no +``` + +You can overwrite it from prometheus, by appending to the URL: + +* `&names=no` to get IDs (the old behaviour) +* `&names=yes` to get names + +### Filtering metrics sent to prometheus + +netdata can filter the metrics it sends to prometheus with this setting: + +``` +[backend] + send charts matching = * +``` + +This settings accepts a space separated list of patterns to match the **charts** to be sent to prometheus. Each pattern can use ` * ` as wildcard, any number of times (e.g `*a*b*c*` is valid). Patterns starting with ` ! ` give a negative match (e.g `!*.bad users.* groups.*` will send all the users and groups except `bad` user and `bad` group). The order is important: the first match (positive or negative) left to right, is used. + +### Changing the prefix of netdata metrics + +netdata sends all metrics prefixed with `netdata_`. You can change this in `netdata.conf`, like this: + +``` +[backend] + prefix = netdata +``` + +It can also be changed from the URL, by appending `&prefix=netdata`. + +### Accuracy of `average` and `sum` data sources + +When the data source is set to `average` or `sum`, netdata remembers the last access of each client accessing prometheus metrics and uses this last access time to respond with the `average` or `sum` of all the entries in the database since that. This means that prometheus servers are not losing data when they access netdata with data source = `average` or `sum`. + +To uniquely identify each prometheus server, netdata uses the IP of the client accessing the metrics. If however the IP is not good enough for identifying a single prometheus server (e.g. when prometheus servers are accessing netdata through a web proxy, or when multiple prometheus servers are NATed to a single IP), each prometheus may append `&server=NAME` to the URL. This `NAME` is used by netdata to uniquely identify each prometheus server and keep track of its last access time. + +[]() diff --git a/backends/prometheus/backend_prometheus.c b/backends/prometheus/backend_prometheus.c new file mode 100644 index 0000000..6b0d7ca --- /dev/null +++ b/backends/prometheus/backend_prometheus.c @@ -0,0 +1,562 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#define BACKENDS_INTERNALS +#include "backend_prometheus.h" + +// ---------------------------------------------------------------------------- +// PROMETHEUS +// /api/v1/allmetrics?format=prometheus and /api/v1/allmetrics?format=prometheus_all_hosts + +static struct prometheus_server { + const char *server; + uint32_t hash; + RRDHOST *host; + time_t last_access; + struct prometheus_server *next; +} *prometheus_server_root = NULL; + +static inline time_t prometheus_server_last_access(const char *server, RRDHOST *host, time_t now) { + static netdata_mutex_t prometheus_server_root_mutex = NETDATA_MUTEX_INITIALIZER; + + uint32_t hash = simple_hash(server); + + netdata_mutex_lock(&prometheus_server_root_mutex); + + struct prometheus_server *ps; + for(ps = prometheus_server_root; ps ;ps = ps->next) { + if (host == ps->host && hash == ps->hash && !strcmp(server, ps->server)) { + time_t last = ps->last_access; + ps->last_access = now; + netdata_mutex_unlock(&prometheus_server_root_mutex); + return last; + } + } + + ps = callocz(1, sizeof(struct prometheus_server)); + ps->server = strdupz(server); + ps->hash = hash; + ps->host = host; + ps->last_access = now; + ps->next = prometheus_server_root; + prometheus_server_root = ps; + + netdata_mutex_unlock(&prometheus_server_root_mutex); + return 0; +} + +static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) { + size_t n; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(!isalnum(c)) *d = '_'; + else *d = c; + } + *d = '\0'; + + return n; +} + +static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable) { + size_t n; + + // make sure we can escape one character without overflowing the buffer + usable--; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(unlikely(c == '"' || c == '\\' || c == '\n')) { + *d++ = '\\'; + n++; + } + *d = c; + } + *d = '\0'; + + return n; +} + +static inline char *prometheus_units_copy(char *d, const char *s, size_t usable) { + const char *sorig = s; + char *ret = d; + size_t n; + + *d++ = '_'; + for(n = 1; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(!isalnum(c)) *d = '_'; + else *d = c; + } + + if(n == 2 && sorig[0] == '%') { + n = 0; + d = ret; + s = "_percent"; + for( ; *s && n < usable ; n++) *d++ = *s++; + } + else if(n > 3 && sorig[n-3] == '/' && sorig[n-2] == 's') { + n = n - 2; + d -= 2; + s = "_persec"; + for( ; *s && n < usable ; n++) *d++ = *s++; + } + + *d = '\0'; + + return ret; +} + + +#define PROMETHEUS_ELEMENT_MAX 256 +#define PROMETHEUS_LABELS_MAX 1024 +#define PROMETHEUS_VARIABLE_MAX 256 + +struct host_variables_callback_options { + RRDHOST *host; + BUFFER *wb; + BACKEND_OPTIONS backend_options; + PROMETHEUS_OUTPUT_OPTIONS output_options; + const char *prefix; + const char *labels; + time_t now; + int host_header_printed; + char name[PROMETHEUS_VARIABLE_MAX+1]; +}; + +static int print_host_variables(RRDVAR *rv, void *data) { + struct host_variables_callback_options *opts = data; + + if(rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_CUSTOM_CHART_VAR)) { + if(!opts->host_header_printed) { + opts->host_header_printed = 1; + + if(opts->output_options & PROMETHEUS_OUTPUT_HELP) { + buffer_sprintf(opts->wb, "\n# COMMENT global host and chart variables\n"); + } + } + + calculated_number value = rrdvar2number(rv); + if(isnan(value) || isinf(value)) { + if(opts->output_options & PROMETHEUS_OUTPUT_HELP) + buffer_sprintf(opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value))?"NAN":"INF"); + + return 0; + } + + char *label_pre = ""; + char *label_post = ""; + if(opts->labels && *opts->labels) { + label_pre = "{"; + label_post = "}"; + } + + prometheus_name_copy(opts->name, rv->name, sizeof(opts->name)); + + if(opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) + buffer_sprintf(opts->wb + , "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n" + , opts->prefix + , opts->name + , label_pre + , opts->labels + , label_post + , value + , ((rv->last_updated) ? rv->last_updated : opts->now) * 1000ULL + ); + else + buffer_sprintf(opts->wb, "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n" + , opts->prefix + , opts->name + , label_pre + , opts->labels + , label_post + , value + ); + + return 1; + } + + return 0; +} + +static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, BACKEND_OPTIONS backend_options, time_t after, time_t before, int allhosts, PROMETHEUS_OUTPUT_OPTIONS output_options) { + rrdhost_rdlock(host); + + char hostname[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX); + + char labels[PROMETHEUS_LABELS_MAX + 1] = ""; + if(allhosts) { + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) + buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS); + else + buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version); + + if(host->tags && *(host->tags)) { + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) { + buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS); + + // deprecated, exists only for compatibility with older queries + buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS); + } + else { + buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, host->tags); + + // deprecated, exists only for compatibility with older queries + buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, host->tags); + } + + } + + snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname); + } + else { + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) + buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS); + else + buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version); + + if(host->tags && *(host->tags)) { + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) { + buffer_sprintf(wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS); + + // deprecated, exists only for compatibility with older queries + buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS); + } + else { + buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", host->tags); + + // deprecated, exists only for compatibility with older queries + buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", host->tags); + } + } + } + + // send custom variables set for the host + if(output_options & PROMETHEUS_OUTPUT_VARIABLES){ + struct host_variables_callback_options opts = { + .host = host, + .wb = wb, + .labels = (labels[0] == ',')?&labels[1]:labels, + .backend_options = backend_options, + .output_options = output_options, + .prefix = prefix, + .now = now_realtime_sec(), + .host_header_printed = 0 + }; + foreach_host_variable_callback(host, print_host_variables, &opts); + } + + // for each chart + RRDSET *st; + rrdset_foreach_read(st, host) { + char chart[PROMETHEUS_ELEMENT_MAX + 1]; + char context[PROMETHEUS_ELEMENT_MAX + 1]; + char family[PROMETHEUS_ELEMENT_MAX + 1]; + char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; + + prometheus_label_copy(chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + + if(likely(backends_can_send_rrdset(backend_options, st))) { + rrdset_rdlock(st); + + int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED); + int homogeneus = 1; + if(as_collected) { + if(rrdset_flag_check(st, RRDSET_FLAG_HOMEGENEOUS_CHECK)) + rrdset_update_heterogeneous_flag(st); + + if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) + homogeneus = 0; + } + else { + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE) + prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX); + } + + if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) + buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n" + , (homogeneus)?"homogeneus":"heterogeneous" + , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id + , st->context + , st->family + , st->units + ); + + // for each dimension + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if(rd->collections_counter) { + char dimension[PROMETHEUS_ELEMENT_MAX + 1]; + char *suffix = ""; + + if (as_collected) { + // we need as-collected / raw data + + if(unlikely(rd->last_collected_time.tv_sec < after)) + continue; + + const char *t = "gauge", *h = "gives"; + if(rd->algorithm == RRD_ALGORITHM_INCREMENTAL || + rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) { + t = "counter"; + h = "delta gives"; + suffix = "_total"; + } + + if(homogeneus) { + // all the dimensions of the chart, has the same algorithm, multiplier and divisor + // we add all dimensions as labels + + prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) + buffer_sprintf(wb + , "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n" + , prefix + , context + , suffix + , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id + , st->context + , st->family + , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id + , rd->multiplier + , rd->divisor + , h + , st->units + , t + ); + + if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n" + , prefix + , context + , suffix + , t + ); + + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) + buffer_sprintf(wb + , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , suffix + , chart + , family + , dimension + , labels + , rd->last_collected_value + , timeval_msec(&rd->last_collected_time) + ); + else + buffer_sprintf(wb + , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT "\n" + , prefix + , context + , suffix + , chart + , family + , dimension + , labels + , rd->last_collected_value + ); + } + else { + // the dimensions of the chart, do not have the same algorithm, multiplier or divisor + // we create a metric per dimension + + prometheus_name_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) + buffer_sprintf(wb + , "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n" + , prefix + , context + , dimension + , suffix + , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id + , st->context + , st->family + , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id + , rd->multiplier + , rd->divisor + , h + , st->units + , t + ); + + if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s_%s%s %s\n" + , prefix + , context + , dimension + , suffix + , t + ); + + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) + buffer_sprintf(wb + , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , dimension + , suffix + , chart + , family + , labels + , rd->last_collected_value + , timeval_msec(&rd->last_collected_time) + ); + else + buffer_sprintf(wb + , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT "\n" + , prefix + , context + , dimension + , suffix + , chart + , family + , labels + , rd->last_collected_value + ); + } + } + else { + // we need average or sum of the data + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t); + + if(!isnan(value) && !isinf(value)) { + + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE) + suffix = "_average"; + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM) + suffix = "_sum"; + + prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) + buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n" + , prefix + , context + , units + , suffix + , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id + , st->units + , (unsigned long long)first_t + , (unsigned long long)last_t + ); + + if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n" + , prefix + , context + , units + , suffix + ); + + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) + buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , units + , suffix + , chart + , family + , dimension + , labels + , value + , last_t * MSEC_PER_SEC + ); + else + buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT "\n" + , prefix + , context + , units + , suffix + , chart + , family + , dimension + , labels + , value + ); + } + } + } + } + + rrdset_unlock(st); + } + } + + rrdhost_unlock(host); +} + +static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_OPTIONS backend_options, const char *server, time_t now, PROMETHEUS_OUTPUT_OPTIONS output_options) { + if(!server || !*server) server = "default"; + + time_t after = prometheus_server_last_access(server, host, now); + + int first_seen = 0; + if(!after) { + after = now - global_backend_update_every; + first_seen = 1; + } + + if(after > now) { + // oops! this should never happen + after = now - global_backend_update_every; + } + + if(output_options & PROMETHEUS_OUTPUT_HELP) { + char *mode; + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) + mode = "as collected"; + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE) + mode = "average"; + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM) + mode = "sum"; + else + mode = "unknown"; + + buffer_sprintf(wb, "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s, time range %lu to %lu\n\n" + , host->hostname + , (first_seen)?"FIRST SEEN ":"" + , server + , mode + , (unsigned long)((first_seen)?0:(now - after)) + , (first_seen)?"never":"seconds ago" + , (unsigned long)after, (unsigned long)now + ); + } + + return after; +} + +void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options) { + time_t before = now_realtime_sec(); + + // we start at the point we had stopped before + time_t after = prometheus_preparation(host, wb, backend_options, server, before, output_options); + + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, backend_options, after, before, 0, output_options); +} + +void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options) { + time_t before = now_realtime_sec(); + + // we start at the point we had stopped before + time_t after = prometheus_preparation(host, wb, backend_options, server, before, output_options); + + rrd_rdlock(); + rrdhost_foreach_read(host) { + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, backend_options, after, before, 1, output_options); + } + rrd_unlock(); +} diff --git a/backends/prometheus/backend_prometheus.h b/backends/prometheus/backend_prometheus.h new file mode 100644 index 0000000..dc4ec75 --- /dev/null +++ b/backends/prometheus/backend_prometheus.h @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_BACKEND_PROMETHEUS_H +#define NETDATA_BACKEND_PROMETHEUS_H 1 + +#include "backends/backends.h" + +typedef enum prometheus_output_flags { + PROMETHEUS_OUTPUT_NONE = 0, + PROMETHEUS_OUTPUT_HELP = (1 << 0), + PROMETHEUS_OUTPUT_TYPES = (1 << 1), + PROMETHEUS_OUTPUT_NAMES = (1 << 2), + PROMETHEUS_OUTPUT_TIMESTAMPS = (1 << 3), + PROMETHEUS_OUTPUT_VARIABLES = (1 << 4) +} PROMETHEUS_OUTPUT_OPTIONS; + +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options); +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options); + +#endif //NETDATA_BACKEND_PROMETHEUS_H |