diff options
Diffstat (limited to 'backends/prometheus/remote_write')
-rw-r--r-- | backends/prometheus/remote_write/Makefile.am | 14 | ||||
-rw-r--r-- | backends/prometheus/remote_write/README.md | 41 | ||||
-rw-r--r-- | backends/prometheus/remote_write/remote_write.cc | 120 | ||||
-rw-r--r-- | backends/prometheus/remote_write/remote_write.h | 30 | ||||
-rw-r--r-- | backends/prometheus/remote_write/remote_write.proto | 29 |
5 files changed, 234 insertions, 0 deletions
diff --git a/backends/prometheus/remote_write/Makefile.am b/backends/prometheus/remote_write/Makefile.am new file mode 100644 index 0000000..d049ef4 --- /dev/null +++ b/backends/prometheus/remote_write/Makefile.am @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +CLEANFILES = \ + remote_write.pb.cc \ + remote_write.pb.h \ + $(NULL) + +dist_noinst_DATA = \ + remote_write.proto \ + README.md \ + $(NULL) diff --git a/backends/prometheus/remote_write/README.md b/backends/prometheus/remote_write/README.md new file mode 100644 index 0000000..b83575e --- /dev/null +++ b/backends/prometheus/remote_write/README.md @@ -0,0 +1,41 @@ +<!-- +title: "Prometheus remote write backend" +custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/prometheus/remote_write/README.md +--> + +# Prometheus remote write backend + +## Prerequisites + +To use the prometheus remote write API with [storage +providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage) +[protobuf](https://developers.google.com/protocol-buffers/) and [snappy](https://github.com/google/snappy) libraries +should be installed first. Next, Netdata should be re-installed from the source. The installer will detect that the +required libraries and utilities are now available. + +## Configuration + +An additional option in the backend configuration section is available for the remote write backend: + +```conf +[backend] + remote write URL path = /receive +``` + +The default value is `/receive`. `remote write URL path` is used to set an endpoint path for the remote write protocol. +For example, if your endpoint is `http://example.domain:example_port/storage/read` you should set + +```conf +[backend] + destination = example.domain:example_port + remote write URL path = /storage/read +``` + +`buffered` and `lost` dimensions in the Netdata Backend Data Size operation monitoring chart estimate uncompressed +buffer size on failures. + +## Notes + +The remote write backend does not support `buffer on failures` + +[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fbackends%2Fprometheus%2Fremote_write%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>) diff --git a/backends/prometheus/remote_write/remote_write.cc b/backends/prometheus/remote_write/remote_write.cc new file mode 100644 index 0000000..9448595 --- /dev/null +++ b/backends/prometheus/remote_write/remote_write.cc @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <snappy.h> +#include "remote_write.pb.h" +#include "remote_write.h" + +using namespace prometheus; + +static google::protobuf::Arena arena; +static WriteRequest *write_request; + +void backends_init_write_request() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + write_request = google::protobuf::Arena::CreateMessage<WriteRequest>(&arena); +} + +void backends_clear_write_request() { + write_request->clear_timeseries(); +} + +void backends_add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp) { + TimeSeries *timeseries; + Sample *sample; + Label *label; + + timeseries = write_request->add_timeseries(); + + label = timeseries->add_labels(); + label->set_name("__name__"); + label->set_value(name); + + label = timeseries->add_labels(); + label->set_name("instance"); + label->set_value(instance); + + if(application) { + label = timeseries->add_labels(); + label->set_name("application"); + label->set_value(application); + } + + if(version) { + label = timeseries->add_labels(); + label->set_name("version"); + label->set_value(version); + } + + sample = timeseries->add_samples(); + sample->set_value(1); + sample->set_timestamp(timestamp); +} + +// adds tag to the last created timeseries +void backends_add_tag(char *tag, char *value) { + TimeSeries *timeseries; + Label *label; + + timeseries = write_request->mutable_timeseries(write_request->timeseries_size() - 1); + + label = timeseries->add_labels(); + label->set_name(tag); + label->set_value(value); +} + +void backends_add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp) { + TimeSeries *timeseries; + Sample *sample; + Label *label; + + timeseries = write_request->add_timeseries(); + + label = timeseries->add_labels(); + label->set_name("__name__"); + label->set_value(name); + + label = timeseries->add_labels(); + label->set_name("chart"); + label->set_value(chart); + + label = timeseries->add_labels(); + label->set_name("family"); + label->set_value(family); + + if(dimension) { + label = timeseries->add_labels(); + label->set_name("dimension"); + label->set_value(dimension); + } + + label = timeseries->add_labels(); + label->set_name("instance"); + label->set_value(instance); + + sample = timeseries->add_samples(); + sample->set_value(value); + sample->set_timestamp(timestamp); +} + +size_t backends_get_write_request_size(){ +#if GOOGLE_PROTOBUF_VERSION < 3001000 + size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSize()); +#else + size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSizeLong()); +#endif + + return (size < INT_MAX)?size:0; +} + +int backends_pack_write_request(char *buffer, size_t *size) { + std::string uncompressed_write_request; + if(write_request->SerializeToString(&uncompressed_write_request) == false) return 1; + + snappy::RawCompress(uncompressed_write_request.data(), uncompressed_write_request.size(), buffer, size); + + return 0; +} + +void backends_protocol_buffers_shutdown() { + google::protobuf::ShutdownProtobufLibrary(); +} diff --git a/backends/prometheus/remote_write/remote_write.h b/backends/prometheus/remote_write/remote_write.h new file mode 100644 index 0000000..1307d72 --- /dev/null +++ b/backends/prometheus/remote_write/remote_write.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H +#define NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H + +#ifdef __cplusplus +extern "C" { +#endif + +void backends_init_write_request(); + +void backends_clear_write_request(); + +void backends_add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp); + +void backends_add_tag(char *tag, char *value); + +void backends_add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp); + +size_t backends_get_write_request_size(); + +int backends_pack_write_request(char *buffer, size_t *size); + +void backends_protocol_buffers_shutdown(); + +#ifdef __cplusplus +} +#endif + +#endif //NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H diff --git a/backends/prometheus/remote_write/remote_write.proto b/backends/prometheus/remote_write/remote_write.proto new file mode 100644 index 0000000..dfde254 --- /dev/null +++ b/backends/prometheus/remote_write/remote_write.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; +package prometheus; + +option cc_enable_arenas = true; + +import "google/protobuf/descriptor.proto"; + +message WriteRequest { + repeated TimeSeries timeseries = 1 [(nullable) = false]; +} + +message TimeSeries { + repeated Label labels = 1 [(nullable) = false]; + repeated Sample samples = 2 [(nullable) = false]; +} + +message Label { + string name = 1; + string value = 2; +} + +message Sample { + double value = 1; + int64 timestamp = 2; +} + +extend google.protobuf.FieldOptions { + bool nullable = 65001; +} |