From a133c9c3b637b1dbe7b5b053f7e2572c1950cead Mon Sep 17 00:00:00 2001 From: Lennart Weller Date: Thu, 27 Jul 2017 11:55:47 +0200 Subject: New upstream version 1.7.0+dfsg --- src/backend_prometheus.c | 397 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 src/backend_prometheus.c (limited to 'src/backend_prometheus.c') diff --git a/src/backend_prometheus.c b/src/backend_prometheus.c new file mode 100644 index 000000000..88ec2c65b --- /dev/null +++ b/src/backend_prometheus.c @@ -0,0 +1,397 @@ +#include "common.h" + +// ---------------------------------------------------------------------------- +// PROMETHEUS +// /api/v1/allmetrics?format=prometheus + +static struct prometheus_server { + const char *server; + uint32_t hash; + time_t last_access; + struct prometheus_server *next; +} *prometheus_server_root = NULL; + +static inline time_t prometheus_server_last_access(const char *server, time_t now) { + uint32_t hash = simple_hash(server); + + struct prometheus_server *ps; + for(ps = prometheus_server_root; ps ;ps = ps->next) { + if (hash == ps->hash && !strcmp(server, ps->server)) { + time_t last = ps->last_access; + ps->last_access = now; + return last; + } + } + + ps = callocz(1, sizeof(struct prometheus_server)); + ps->server = strdupz(server); + ps->hash = hash; + ps->last_access = now; + ps->next = prometheus_server_root; + prometheus_server_root = ps; + + return 0; +} + +static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) { + size_t n; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(!isalnum(c)) *d = '_'; + else *d = c; + } + *d = '\0'; + + return n; +} + +static inline size_t prometheus_label_copy(char *d, const char *s, size_t usable) { + size_t n; + + // make sure we can escape one character without overflowing the buffer + usable--; + + for(n = 0; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(unlikely(c == '"' || c == '\\' || c == '\n')) { + *d++ = '\\'; + n++; + } + *d = c; + } + *d = '\0'; + + return n; +} + +static inline char *prometheus_units_copy(char *d, const char *s, size_t usable) { + const char *sorig = s; + char *ret = d; + size_t n; + + *d++ = '_'; + for(n = 1; *s && n < usable ; d++, s++, n++) { + register char c = *s; + + if(!isalnum(c)) *d = '_'; + else *d = c; + } + + if(n == 2 && sorig[0] == '%') { + n = 0; + d = ret; + s = "_percent"; + for( ; *s && n < usable ; n++) *d++ = *s++; + } + else if(n > 3 && sorig[n-3] == '/' && sorig[n-2] == 's') { + n = n - 2; + d -= 2; + s = "_persec"; + for( ; *s && n < usable ; n++) *d++ = *s++; + } + + *d = '\0'; + + return ret; +} + + +#define PROMETHEUS_ELEMENT_MAX 256 +#define PROMETHEUS_LABELS_MAX 1024 + +static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, uint32_t options, time_t after, time_t before, int allhosts, int help, int types, int names) { + rrdhost_rdlock(host); + + char hostname[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX); + + char labels[PROMETHEUS_LABELS_MAX + 1] = ""; + if(allhosts) { + if(host->tags && *(host->tags)) + buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS); + + snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname); + } + else { + if(host->tags && *(host->tags)) + buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS); + } + + // for each chart + RRDSET *st; + rrdset_foreach_read(st, host) { + char chart[PROMETHEUS_ELEMENT_MAX + 1]; + char context[PROMETHEUS_ELEMENT_MAX + 1]; + char family[PROMETHEUS_ELEMENT_MAX + 1]; + char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; + + prometheus_label_copy(chart, (names && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + + if(likely(backends_can_send_rrdset(options, st))) { + rrdset_rdlock(st); + + int as_collected = ((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED); + int homogeneus = 1; + if(as_collected) { + if(rrdset_flag_check(st, RRDSET_FLAG_HOMEGENEOUS_CHECK)) + rrdset_update_heterogeneous_flag(st); + + if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) + homogeneus = 0; + } + else { + if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE) + prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX); + } + + if(unlikely(help)) + buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n" + , (homogeneus)?"homogeneus":"heterogeneous" + , (names && st->name) ? st->name : st->id + , st->context + , st->family + , st->units + ); + + // for each dimension + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if(rd->collections_counter) { + char dimension[PROMETHEUS_ELEMENT_MAX + 1]; + char *suffix = ""; + + if (as_collected) { + // we need as-collected / raw data + + const char *t = "gauge", *h = "gives"; + if(rd->algorithm == RRD_ALGORITHM_INCREMENTAL || + rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) { + t = "counter"; + h = "delta gives"; + suffix = "_total"; + } + + if(homogeneus) { + // all the dimensions of the chart, has the same algorithm, multiplier and divisor + // we add all dimensions as labels + + prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if(unlikely(help)) + buffer_sprintf(wb + , "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n" + , prefix + , context + , suffix + , (names && st->name) ? st->name : st->id + , st->context + , st->family + , (names && rd->name) ? rd->name : rd->id + , rd->multiplier + , rd->divisor + , h + , st->units + , t + ); + + if(unlikely(types)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n" + , prefix + , context + , suffix + , t + ); + + buffer_sprintf(wb + , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , suffix + , chart + , family + , dimension + , labels + , rd->last_collected_value + , timeval_msec(&rd->last_collected_time) + ); + } + else { + // the dimensions of the chart, do not have the same algorithm, multiplier or divisor + // we create a metric per dimension + + prometheus_name_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if(unlikely(help)) + buffer_sprintf(wb + , "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n" + , prefix + , context + , dimension + , suffix + , (names && st->name) ? st->name : st->id + , st->context + , st->family + , (names && rd->name) ? rd->name : rd->id + , rd->multiplier + , rd->divisor + , h + , st->units + , t + ); + + if(unlikely(types)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s_%s%s %s\n" + , prefix + , context + , dimension + , suffix + , t + ); + + buffer_sprintf(wb + , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , dimension + , suffix + , chart + , family + , labels + , rd->last_collected_value + , timeval_msec(&rd->last_collected_time) + ); + } + } + else { + // we need average or sum of the data + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t); + + if(!isnan(value) && !isinf(value)) { + + if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE) + suffix = "_average"; + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM) + suffix = "_sum"; + + prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + + if (unlikely(help)) + buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n" + , prefix + , context + , units + , suffix + , (names && rd->name) ? rd->name : rd->id + , st->units + , (unsigned long long)first_t + , (unsigned long long)last_t + ); + + if (unlikely(types)) + buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n" + , prefix + , context + , units + , suffix + ); + + buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n" + , prefix + , context + , units + , suffix + , chart + , family + , dimension + , labels + , value + , last_t * MSEC_PER_SEC + ); + } + } + } + } + + rrdset_unlock(st); + } + } + + rrdhost_unlock(host); +} + +static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, uint32_t options, const char *server, time_t now, int help) { + if(!server || !*server) server = "default"; + + time_t after = prometheus_server_last_access(server, now); + + int first_seen = 0; + if(!after) { + after = now - backend_update_every; + first_seen = 1; + } + + if(after > now) { + // oops! this should never happen + after = now - backend_update_every; + } + + if(help) { + int show_range = 1; + char *mode; + if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) { + mode = "as collected"; + show_range = 0; + } + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE) + mode = "average"; + else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM) + mode = "sum"; + else + mode = "unknown"; + + buffer_sprintf(wb, "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s" + , host->hostname + , (first_seen)?"FIRST SEEN ":"" + , server + , mode + , (unsigned long)((first_seen)?0:(now - after)) + , (first_seen)?"never":"seconds ago" + ); + + if(show_range) + buffer_sprintf(wb, ", time range %lu to %lu", (unsigned long)after, (unsigned long)now); + + buffer_strcat(wb, "\n\n"); + } + + return after; +} + +void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names) { + time_t before = now_realtime_sec(); + + // we start at the point we had stopped before + time_t after = prometheus_preparation(host, wb, options, server, before, help); + + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 0, help, types, names); +} + +void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names) { + time_t before = now_realtime_sec(); + + // we start at the point we had stopped before + time_t after = prometheus_preparation(host, wb, options, server, before, help); + + rrd_rdlock(); + rrdhost_foreach_read(host) { + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 1, help, types, names); + } + rrd_unlock(); +} -- cgit v1.2.3