diff options
Diffstat (limited to 'backends/prometheus')
| -rw-r--r-- | backends/prometheus/Makefile.am | 4 | ||||
| -rw-r--r-- | backends/prometheus/backend_prometheus.c | 198 | ||||
| -rw-r--r-- | backends/prometheus/backend_prometheus.h | 15 | ||||
| -rw-r--r-- | backends/prometheus/remote_write/Makefile.am | 14 | ||||
| -rw-r--r-- | backends/prometheus/remote_write/README.md | 30 | ||||
| -rw-r--r-- | backends/prometheus/remote_write/remote_write.cc | 117 | ||||
| -rw-r--r-- | backends/prometheus/remote_write/remote_write.h | 30 | ||||
| -rw-r--r-- | backends/prometheus/remote_write/remote_write.proto | 29 |
8 files changed, 436 insertions, 1 deletions
diff --git a/backends/prometheus/Makefile.am b/backends/prometheus/Makefile.am index 19554bed..e5f74851 100644 --- a/backends/prometheus/Makefile.am +++ b/backends/prometheus/Makefile.am @@ -3,6 +3,10 @@ AUTOMAKE_OPTIONS = subdir-objects MAINTAINERCLEANFILES = $(srcdir)/Makefile.in +SUBDIRS = \ + remote_write \ + $(NULL) + dist_noinst_DATA = \ README.md \ $(NULL) diff --git a/backends/prometheus/backend_prometheus.c b/backends/prometheus/backend_prometheus.c index 3641b07c..67342ea7 100644 --- a/backends/prometheus/backend_prometheus.c +++ b/backends/prometheus/backend_prometheus.c @@ -153,6 +153,8 @@ static inline char *prometheus_units_copy(char *d, const char *s, size_t usable, #define PROMETHEUS_LABELS_MAX 1024 #define PROMETHEUS_VARIABLE_MAX 256 +#define PROMETHEUS_LABELS_MAX_NUMBER 128 + struct host_variables_callback_options { RRDHOST *host; BUFFER *wb; @@ -307,7 +309,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED); int homogeneous = 1; if(as_collected) { - if(rrdset_flag_check(st, RRDSET_FLAG_HOMEGENEOUS_CHECK)) + if(rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK)) rrdset_update_heterogeneous_flag(st); if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) @@ -537,6 +539,177 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER rrdhost_unlock(host); } +#if ENABLE_PROMETHEUS_REMOTE_WRITE +inline static void remote_write_split_words(char *str, char **words, int max_words) { + char *s = str; + int i = 0; + + while(*s && i < max_words - 1) { + while(*s && isspace(*s)) s++; // skip spaces to the begining of a tag name + + if(*s) + words[i] = s; + + while(*s && !isspace(*s) && *s != '=') s++; // find the end of the tag name + + if(*s != '=') { + words[i] = NULL; + break; + } + *s = '\0'; + s++; + i++; + + while(*s && isspace(*s)) s++; // skip spaces to the begining of a tag value + + if(*s && *s == '"') s++; // strip an opening quote + if(*s) + words[i] = s; + + while(*s && !isspace(*s) && *s != ',') s++; // find the end of the tag value + + if(*s && *s != ',') { + words[i] = NULL; + break; + } + if(s != words[i] && *(s - 1) == '"') *(s - 1) = '\0'; // strip a closing quote + if(*s != '\0') { + *s = '\0'; + s++; + i++; + } + } +} + +void rrd_stats_remote_write_allmetrics_prometheus( + RRDHOST *host + , const char *__hostname + , const char *prefix + , BACKEND_OPTIONS backend_options + , time_t after + , time_t before + , size_t *count_charts + , size_t *count_dims + , size_t *count_dims_skipped +) { + char hostname[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_label_copy(hostname, __hostname, PROMETHEUS_ELEMENT_MAX); + + add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS); + + if(host->tags && *(host->tags)) { + char tags[PROMETHEUS_LABELS_MAX + 1]; + strncpy(tags, host->tags, PROMETHEUS_LABELS_MAX); + char *words[PROMETHEUS_LABELS_MAX_NUMBER] = {NULL}; + int i; + + remote_write_split_words(tags, words, PROMETHEUS_LABELS_MAX_NUMBER); + + add_host_info("netdata_host_tags_info", hostname, NULL, NULL, now_realtime_usec() / USEC_PER_MS); + + for(i = 0; words[i] != NULL && words[i + 1] != NULL && (i + 1) < PROMETHEUS_LABELS_MAX_NUMBER; i += 2) { + add_tag(words[i], words[i + 1]); + } + } + + // for each chart + RRDSET *st; + rrdset_foreach_read(st, host) { + char chart[PROMETHEUS_ELEMENT_MAX + 1]; + char context[PROMETHEUS_ELEMENT_MAX + 1]; + char family[PROMETHEUS_ELEMENT_MAX + 1]; + char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; + + prometheus_label_copy(chart, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + + if(likely(backends_can_send_rrdset(backend_options, st))) { + rrdset_rdlock(st); + + (*count_charts)++; + + int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED); + int homogeneous = 1; + if(as_collected) { + if(rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK)) + rrdset_update_heterogeneous_flag(st); + + if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) + homogeneous = 0; + } + else { + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE) + prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0); + } + + // for each dimension + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { + char name[PROMETHEUS_LABELS_MAX + 1]; + char dimension[PROMETHEUS_ELEMENT_MAX + 1]; + char *suffix = ""; + + if (as_collected) { + // we need as-collected / raw data + + if(unlikely(rd->last_collected_time.tv_sec < after)) { + debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before); + (*count_dims_skipped)++; + continue; + } + + if(homogeneous) { + // all the dimensions of the chart, has the same algorithm, multiplier and divisor + // we add all dimensions as labels + + prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", prefix, context, suffix); + + add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)); + (*count_dims)++; + } + else { + // the dimensions of the chart, do not have the same algorithm, multiplier or divisor + // we create a metric per dimension + + prometheus_name_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", prefix, context, dimension, suffix); + + add_metric(name, chart, family, NULL, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)); + (*count_dims)++; + } + } + else { + // we need average or sum of the data + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t); + + if(!isnan(value) && !isinf(value)) { + + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE) + suffix = "_average"; + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM) + suffix = "_sum"; + + prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", prefix, context, units, suffix); + + add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)); + (*count_dims)++; + } + } + } + } + + rrdset_unlock(st); + } + } +} +#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */ + static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_OPTIONS backend_options, const char *server, time_t now, PROMETHEUS_OUTPUT_OPTIONS output_options) { if(!server || !*server) server = "default"; @@ -599,3 +772,26 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFF } rrd_unlock(); } + +#if ENABLE_PROMETHEUS_REMOTE_WRITE +int process_prometheus_remote_write_response(BUFFER *b) { + if(unlikely(!b)) return 1; + + const char *s = buffer_tostring(b); + int len = buffer_strlen(b); + + // do nothing with HTTP response 200 + + while(!isspace(*s) && len) { + s++; + len--; + } + s++; + len--; + + if(likely(len > 4 && !strncmp(s, "200 ", 4))) + return 0; + else + return discard_response(b, "prometheus remote write"); +} +#endif diff --git a/backends/prometheus/backend_prometheus.h b/backends/prometheus/backend_prometheus.h index 72b65a22..d58d2400 100644 --- a/backends/prometheus/backend_prometheus.h +++ b/backends/prometheus/backend_prometheus.h @@ -19,4 +19,19 @@ typedef enum prometheus_output_flags { extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options); extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options); +#if ENABLE_PROMETHEUS_REMOTE_WRITE +extern void rrd_stats_remote_write_allmetrics_prometheus( + RRDHOST *host + , const char *__hostname + , const char *prefix + , BACKEND_OPTIONS backend_options + , time_t after + , time_t before + , size_t *count_charts + , size_t *count_dims + , size_t *count_dims_skipped +); +extern int process_prometheus_remote_write_response(BUFFER *b); +#endif + #endif //NETDATA_BACKEND_PROMETHEUS_H diff --git a/backends/prometheus/remote_write/Makefile.am b/backends/prometheus/remote_write/Makefile.am new file mode 100644 index 00000000..5f8f9d4c --- /dev/null +++ b/backends/prometheus/remote_write/Makefile.am @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +CLEANFILES = \ + remote_write.pb.cc \ + remote_write.pb.h \ + $(NULL) + +dist_noinst_DATA = \ + remote_write.proto \ + README.md \ + $(NULL) diff --git a/backends/prometheus/remote_write/README.md b/backends/prometheus/remote_write/README.md new file mode 100644 index 00000000..73cb1daf --- /dev/null +++ b/backends/prometheus/remote_write/README.md @@ -0,0 +1,30 @@ +# Prometheus remote write backend + +## Prerequisites + +To use the prometheus remote write API with [storage providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage) [protobuf](https://developers.google.com/protocol-buffers/) and [snappy](https://github.com/google/snappy) libraries should be installed first. Next, netdata should be re-installed from the source. The installer will detect that the required libraries and utilities are now available. + +## Configuration + +An additional option in the backend configuration section is available for the remote write backend: + +``` +[backend] + remote write URL path = /receive +``` + +The default value is `/receive`. `remote write URL path` is used to set an endpoint path for the remote write protocol. For example, if your endpoint is `http://example.domain:example_port/storage/read` you should set + +``` +[backend] + destination = example.domain:example_port + remote write URL path = /storage/read +``` + +`buffered` and `lost` dimensions in the Netdata Backend Data Size operation monitoring chart estimate uncompressed buffer size on failures. + +## Notes + +The remote write backend does not support `buffer on failures` + +[]() diff --git a/backends/prometheus/remote_write/remote_write.cc b/backends/prometheus/remote_write/remote_write.cc new file mode 100644 index 00000000..91d4305b --- /dev/null +++ b/backends/prometheus/remote_write/remote_write.cc @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <snappy.h> +#include "remote_write.pb.h" +#include "remote_write.h" + +using namespace prometheus; + + +google::protobuf::Arena arena; +WriteRequest *write_request; + +void init_write_request() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + write_request = google::protobuf::Arena::CreateMessage<WriteRequest>(&arena); +} + +void clear_write_request() { + write_request->clear_timeseries(); +} + +void add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp) { + TimeSeries *timeseries; + Sample *sample; + Label *label; + + timeseries = write_request->add_timeseries(); + + label = timeseries->add_labels(); + label->set_name("__name__"); + label->set_value(name); + + label = timeseries->add_labels(); + label->set_name("instance"); + label->set_value(instance); + + if(application) { + label = timeseries->add_labels(); + label->set_name("application"); + label->set_value(application); + } + + if(version) { + label = timeseries->add_labels(); + label->set_name("version"); + label->set_value(version); + } + + sample = timeseries->add_samples(); + sample->set_value(1); + sample->set_timestamp(timestamp); +} + +// adds tag to the last created timeseries +void add_tag(char *tag, char *value) { + TimeSeries *timeseries; + Label *label; + + timeseries = write_request->mutable_timeseries(write_request->timeseries_size() - 1); + + label = timeseries->add_labels(); + label->set_name(tag); + label->set_value(value); +} + +void add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp) { + TimeSeries *timeseries; + Sample *sample; + Label *label; + + timeseries = write_request->add_timeseries(); + + label = timeseries->add_labels(); + label->set_name("__name__"); + label->set_value(name); + + label = timeseries->add_labels(); + label->set_name("chart"); + label->set_value(chart); + + label = timeseries->add_labels(); + label->set_name("family"); + label->set_value(family); + + if(dimension) { + label = timeseries->add_labels(); + label->set_name("dimension"); + label->set_value(dimension); + } + + label = timeseries->add_labels(); + label->set_name("instance"); + label->set_value(instance); + + sample = timeseries->add_samples(); + sample->set_value(value); + sample->set_timestamp(timestamp); +} + +size_t get_write_request_size(){ + size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSize()); + + return (size < INT_MAX)?size:0; +} + +int pack_write_request(char *buffer, size_t *size) { + std::string uncompressed_write_request; + if(write_request->SerializeToString(&uncompressed_write_request) == false) return 1; + + snappy::RawCompress(uncompressed_write_request.data(), uncompressed_write_request.size(), buffer, size); + + return 0; +} + +void protocol_buffers_shutdown() { + google::protobuf::ShutdownProtobufLibrary(); +} diff --git a/backends/prometheus/remote_write/remote_write.h b/backends/prometheus/remote_write/remote_write.h new file mode 100644 index 00000000..edcc477b --- /dev/null +++ b/backends/prometheus/remote_write/remote_write.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H +#define NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H + +#ifdef __cplusplus +extern "C" { +#endif + +void init_write_request(); + +void clear_write_request(); + +void add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp); + +void add_tag(char *tag, char *value); + +void add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp); + +size_t get_write_request_size(); + +int pack_write_request(char *buffer, size_t *size); + +void protocol_buffers_shutdown(); + +#ifdef __cplusplus +} +#endif + +#endif //NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H diff --git a/backends/prometheus/remote_write/remote_write.proto b/backends/prometheus/remote_write/remote_write.proto new file mode 100644 index 00000000..dfde254e --- /dev/null +++ b/backends/prometheus/remote_write/remote_write.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; +package prometheus; + +option cc_enable_arenas = true; + +import "google/protobuf/descriptor.proto"; + +message WriteRequest { + repeated TimeSeries timeseries = 1 [(nullable) = false]; +} + +message TimeSeries { + repeated Label labels = 1 [(nullable) = false]; + repeated Sample samples = 2 [(nullable) = false]; +} + +message Label { + string name = 1; + string value = 2; +} + +message Sample { + double value = 1; + int64 timestamp = 2; +} + +extend google.protobuf.FieldOptions { + bool nullable = 65001; +} |