From 3c315f0fff93aa072472abc10815963ac0035268 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 12 Aug 2022 09:26:11 +0200 Subject: Adding upstream version 1.36.0. Signed-off-by: Daniel Baumann --- exporting/aws_kinesis/aws_kinesis.c | 1 + exporting/check_filters.c | 8 ++ exporting/clean_connectors.c | 2 +- exporting/exporting.conf | 1 + exporting/exporting_engine.h | 19 ++- exporting/graphite/graphite.c | 34 ++--- exporting/graphite/graphite.h | 2 +- exporting/init_connectors.c | 8 +- exporting/json/json.c | 40 ++---- exporting/mongodb/mongodb.c | 1 + exporting/opentsdb/opentsdb.c | 79 ++++------- exporting/opentsdb/opentsdb.h | 2 +- exporting/process_data.c | 54 +++++--- exporting/prometheus/prometheus.c | 153 ++++++++------------- exporting/prometheus/remote_write/remote_write.c | 88 +++++++++--- exporting/prometheus/remote_write/remote_write.h | 7 + .../remote_write/remote_write_request.cc | 32 +++++ .../prometheus/remote_write/remote_write_request.h | 3 + exporting/pubsub/pubsub.c | 1 + exporting/read_config.c | 5 + exporting/tests/exporting_doubles.c | 14 +- exporting/tests/exporting_fixtures.c | 39 ++---- exporting/tests/netdata_doubles.c | 30 ++-- exporting/tests/test_exporting_engine.c | 38 ++--- exporting/tests/test_exporting_engine.h | 20 ++- 25 files changed, 370 insertions(+), 311 deletions(-) (limited to 'exporting') diff --git a/exporting/aws_kinesis/aws_kinesis.c b/exporting/aws_kinesis/aws_kinesis.c index fe4181e3a..1d89cc79a 100644 --- a/exporting/aws_kinesis/aws_kinesis.c +++ b/exporting/aws_kinesis/aws_kinesis.c @@ -45,6 +45,7 @@ int init_aws_kinesis_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_json_plaintext; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = NULL; diff --git a/exporting/check_filters.c b/exporting/check_filters.c index f7eba22db..726fd02a1 100644 --- a/exporting/check_filters.c +++ b/exporting/check_filters.c @@ -2,6 +2,14 @@ #include "exporting_engine.h" + +bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { + (void)name; + (void)value; + struct instance *instance = (struct instance *)data; + return should_send_label(instance, ls); +} + /** * Check if the connector instance should export the host metrics * diff --git a/exporting/clean_connectors.c b/exporting/clean_connectors.c index 4af1219a6..e93563741 100644 --- a/exporting/clean_connectors.c +++ b/exporting/clean_connectors.c @@ -33,7 +33,7 @@ static void clean_instance_config(struct instance_config *config) void clean_instance(struct instance *instance) { clean_instance_config(&instance->config); - buffer_free(instance->labels); + buffer_free(instance->labels_buffer); uv_cond_destroy(&instance->cond_var); // uv_mutex_destroy(&instance->mutex); diff --git a/exporting/exporting.conf b/exporting/exporting.conf index 314e1541e..f2ec56a52 100644 --- a/exporting/exporting.conf +++ b/exporting/exporting.conf @@ -45,6 +45,7 @@ # send names instead of ids = yes # send charts matching = * # send hosts matching = localhost * + # send variables = no # [kinesis:my_kinesis_instance] # enabled = no diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h index 20f260c15..2141caa41 100644 --- a/exporting/exporting_engine.h +++ b/exporting/exporting_engine.h @@ -4,7 +4,6 @@ #define NETDATA_EXPORTING_ENGINE_H 1 #include "daemon/common.h" - #include #define exporter_get(section, name, value) expconfig_get(&exporting_config, section, name, value) @@ -27,7 +26,8 @@ typedef enum exporting_options { EXPORTING_OPTION_SEND_AUTOMATIC_LABELS = (1 << 4), EXPORTING_OPTION_USE_TLS = (1 << 5), - EXPORTING_OPTION_SEND_NAMES = (1 << 16) + EXPORTING_OPTION_SEND_NAMES = (1 << 16), + EXPORTING_OPTION_SEND_VARIABLES = (1 << 17) } EXPORTING_OPTIONS; #define EXPORTING_OPTIONS_SOURCE_BITS \ @@ -40,11 +40,13 @@ extern const char *global_exporting_prefix; #define sending_labels_configured(instance) \ (instance->config.options & (EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS)) -#define should_send_label(instance, label) \ +#define should_send_label(instance, label_source) \ ((instance->config.options & EXPORTING_OPTION_SEND_CONFIGURED_LABELS && \ - label->label_source == LABEL_SOURCE_NETDATA_CONF) || \ + label_source & RRDLABEL_SRC_CONFIG) || \ (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS && \ - label->label_source != LABEL_SOURCE_NETDATA_CONF)) + label_source & RRDLABEL_SRC_AUTO)) + +#define should_send_variables(instance) (instance->config.options & EXPORTING_OPTION_SEND_VARIABLES) typedef enum exporting_connector_types { EXPORTING_CONNECTOR_TYPE_UNKNOWN, // Invalid type @@ -205,7 +207,7 @@ struct instance { int skip_host; int skip_chart; - BUFFER *labels; + BUFFER *labels_buffer; time_t after; time_t before; @@ -220,6 +222,7 @@ struct instance { int (*start_chart_formatting)(struct instance *instance, RRDSET *st); int (*metric_formatting)(struct instance *instance, RRDDIM *rd); int (*end_chart_formatting)(struct instance *instance, RRDSET *st); + int (*variables_formatting)(struct instance *instance, RRDHOST *host); int (*end_host_formatting)(struct instance *instance, RRDHOST *host); int (*end_batch_formatting)(struct instance *instance); @@ -270,7 +273,8 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st); extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options); -calculated_number exporting_calculate_value_from_stored_data( +NETDATA_DOUBLE +exporting_calculate_value_from_stored_data( struct instance *instance, RRDDIM *rd, time_t *last_timestamp); @@ -280,6 +284,7 @@ void start_host_formatting(struct engine *engine, RRDHOST *host); void start_chart_formatting(struct engine *engine, RRDSET *st); void metric_formatting(struct engine *engine, RRDDIM *rd); void end_chart_formatting(struct engine *engine, RRDSET *st); +void variables_formatting(struct engine *engine, RRDHOST *host); void end_host_formatting(struct engine *engine, RRDHOST *host); void end_batch_formatting(struct engine *engine); int flush_host_labels(struct instance *instance, RRDHOST *host); diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c index 84d4febf1..8ca094b3b 100644 --- a/exporting/graphite/graphite.c +++ b/exporting/graphite/graphite.c @@ -37,6 +37,7 @@ int init_graphite_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_graphite_plaintext; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = simple_connector_end_batch; @@ -71,7 +72,7 @@ int init_graphite_instance(struct instance *instance) * @param len the maximum number of characters copied. */ -void sanitize_graphite_label_value(char *dst, char *src, size_t len) +void sanitize_graphite_label_value(char *dst, const char *src, size_t len) { while (*src != '\0' && len) { if (isspace(*src) || *src == ';' || *src == '~') @@ -91,29 +92,18 @@ void sanitize_graphite_label_value(char *dst, char *src, size_t len) * @param host a data collecting host. * @return Always returns 0. */ + int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host) { - if (!instance->labels) - instance->labels = buffer_create(1024); + if (!instance->labels_buffer) + instance->labels_buffer = buffer_create(1024); if (unlikely(!sending_labels_configured(instance))) return 0; - rrdhost_check_rdlock(host); - netdata_rwlock_rdlock(&host->labels.labels_rwlock); - for (struct label *label = host->labels.head; label; label = label->next) { - if (!should_send_label(instance, label)) - continue; - - char value[CONFIG_MAX_VALUE + 1]; - sanitize_graphite_label_value(value, label->value, CONFIG_MAX_VALUE); - - if (*value) { - buffer_strcat(instance->labels, ";"); - buffer_sprintf(instance->labels, "%s=%s", label->key, value); - } - } - netdata_rwlock_unlock(&host->labels.labels_rwlock); + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, ";", "=", "", "", + exporting_labels_filter_callback, instance, + NULL, sanitize_graphite_label_value); return 0; } @@ -151,7 +141,7 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD dimension_name, (host->tags) ? ";" : "", (host->tags) ? host->tags : "", - (instance->labels) ? buffer_tostring(instance->labels) : "", + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "", rd->last_collected_value, (unsigned long long)rd->last_collected_time.tv_sec); @@ -183,21 +173,21 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM RRD_ID_LENGTH_MAX); time_t last_t; - calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); + NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); if(isnan(value)) return 0; buffer_sprintf( instance->buffer, - "%s.%s.%s.%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n", + "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n", instance->config.prefix, (host == localhost) ? instance->config.hostname : host->hostname, chart_name, dimension_name, (host->tags) ? ";" : "", (host->tags) ? host->tags : "", - (instance->labels) ? buffer_tostring(instance->labels) : "", + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "", value, (unsigned long long)last_t); diff --git a/exporting/graphite/graphite.h b/exporting/graphite/graphite.h index 993c12e57..79f87e46e 100644 --- a/exporting/graphite/graphite.h +++ b/exporting/graphite/graphite.h @@ -7,7 +7,7 @@ int init_graphite_instance(struct instance *instance); -void sanitize_graphite_label_value(char *dst, char *src, size_t len); +void sanitize_graphite_label_value(char *dst, const char *src, size_t len); int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host); int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd); diff --git a/exporting/init_connectors.c b/exporting/init_connectors.c index 69ea0685c..bfb6525ea 100644 --- a/exporting/init_connectors.c +++ b/exporting/init_connectors.c @@ -117,7 +117,7 @@ static size_t base64_encode(unsigned char *input, size_t input_size, char *outpu return 0; } size_t count = 0; - while (input_size > 3) { + while (input_size >= 3) { value = ((input[0] << 16) + (input[1] << 8) + input[2]) & 0xffffff; output[0] = lookup[value >> 18]; output[1] = lookup[(value >> 12) & 0x3f]; @@ -138,6 +138,7 @@ static size_t base64_encode(unsigned char *input, size_t input_size, char *outpu output[3] = '='; //error("Base-64 encode (%06x) -> %c %c %c %c\n", (value>>2)&0xffff, output[0], output[1], output[2], output[3]); count += 4; + output[4] = '\0'; break; case 1: value = input[0] << 4; @@ -147,10 +148,13 @@ static size_t base64_encode(unsigned char *input, size_t input_size, char *outpu output[3] = '='; //error("Base-64 encode (%06x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]); count += 4; + output[4] = '\0'; break; case 0: + output[0] = '\0'; break; } + return count; } @@ -199,7 +203,7 @@ void simple_connector_init(struct instance *instance) char *encoded_credentials = callocz(1, encoded_size); base64_encode((unsigned char*)buffer_tostring(auth_string), buffer_strlen(auth_string), encoded_credentials, encoded_size); - + buffer_flush(auth_string); buffer_sprintf(auth_string, "Authorization: Basic %s\n", encoded_credentials); diff --git a/exporting/json/json.c b/exporting/json/json.c index 50278c5b8..45a8c9d9f 100644 --- a/exporting/json/json.c +++ b/exporting/json/json.c @@ -29,6 +29,7 @@ int init_json_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_json_plaintext; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = simple_connector_end_batch; @@ -87,6 +88,7 @@ int init_json_http_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_json_plaintext; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = close_batch_json_http; @@ -113,34 +115,20 @@ int init_json_http_instance(struct instance *instance) * @param host a data collecting host. * @return Always returns 0. */ + int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host) { - if (!instance->labels) - instance->labels = buffer_create(1024); + if (!instance->labels_buffer) + instance->labels_buffer = buffer_create(1024); if (unlikely(!sending_labels_configured(instance))) return 0; - buffer_strcat(instance->labels, "\"labels\":{"); - - int count = 0; - rrdhost_check_rdlock(host); - netdata_rwlock_rdlock(&host->labels.labels_rwlock); - for (struct label *label = host->labels.head; label; label = label->next) { - if (!should_send_label(instance, label)) - continue; - - char value[CONFIG_MAX_VALUE * 2 + 1]; - sanitize_json_string(value, label->value, CONFIG_MAX_VALUE); - if (count > 0) - buffer_strcat(instance->labels, ","); - buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value); - - count++; - } - netdata_rwlock_unlock(&host->labels.labels_rwlock); - - buffer_strcat(instance->labels, "},"); + buffer_strcat(instance->labels_buffer, "\"labels\":{"); + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",", + exporting_labels_filter_callback, instance, + NULL, sanitize_json_string); + buffer_strcat(instance->labels_buffer, "},"); return 0; } @@ -203,7 +191,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM tags_pre, tags, tags_post, - instance->labels ? buffer_tostring(instance->labels) : "", + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "", st->id, st->name, @@ -238,7 +226,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd RRDHOST *host = st->rrdhost; time_t last_t; - calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); + NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); if(isnan(value)) return 0; @@ -279,7 +267,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd "\"id\":\"%s\"," "\"name\":\"%s\"," - "\"value\":" CALCULATED_NUMBER_FORMAT "," + "\"value\":" NETDATA_DOUBLE_FORMAT "," "\"timestamp\": %llu}", @@ -288,7 +276,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd tags_pre, tags, tags_post, - instance->labels ? buffer_tostring(instance->labels) : "", + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "", st->id, st->name, diff --git a/exporting/mongodb/mongodb.c b/exporting/mongodb/mongodb.c index aab1770d2..850d07fb3 100644 --- a/exporting/mongodb/mongodb.c +++ b/exporting/mongodb/mongodb.c @@ -99,6 +99,7 @@ int init_mongodb_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_json_plaintext; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = format_batch_mongodb; diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c index 7ed88fd6d..282de2e6b 100644 --- a/exporting/opentsdb/opentsdb.c +++ b/exporting/opentsdb/opentsdb.c @@ -38,6 +38,7 @@ int init_opentsdb_telnet_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_opentsdb_telnet; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = simple_connector_end_batch; @@ -94,6 +95,7 @@ int init_opentsdb_http_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_opentsdb_http; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = close_batch_json_http; @@ -124,13 +126,14 @@ int init_opentsdb_http_instance(struct instance *instance) * @param len the maximum number of characters copied. */ -void sanitize_opentsdb_label_value(char *dst, char *src, size_t len) +void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len) { while (*src != '\0' && len) { - if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src)) + if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src)) *dst++ = *src; else *dst++ = '_'; + src++; len--; } @@ -144,28 +147,18 @@ void sanitize_opentsdb_label_value(char *dst, char *src, size_t len) * @param host a data collecting host. * @return Always returns 0. */ -int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) -{ - if (!instance->labels) - instance->labels = buffer_create(1024); + +int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) { + if(!instance->labels_buffer) + instance->labels_buffer = buffer_create(1024); if (unlikely(!sending_labels_configured(instance))) return 0; - rrdhost_check_rdlock(host); - netdata_rwlock_rdlock(&host->labels.labels_rwlock); - for (struct label *label = host->labels.head; label; label = label->next) { - if (!should_send_label(instance, label)) - continue; - - char value[CONFIG_MAX_VALUE + 1]; - sanitize_opentsdb_label_value(value, label->value, CONFIG_MAX_VALUE); - - if (*value) - buffer_sprintf(instance->labels, " %s=%s", label->key, value); - } - netdata_rwlock_unlock(&host->labels.labels_rwlock); - + buffer_strcat(instance->labels_buffer, " "); + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", " ", + exporting_labels_filter_callback, instance, + NULL, sanitize_opentsdb_label_value); return 0; } @@ -204,7 +197,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM (host == localhost) ? instance->config.hostname : host->hostname, (host->tags) ? " " : "", (host->tags) ? host->tags : "", - (instance->labels) ? buffer_tostring(instance->labels) : ""); + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : ""); return 0; } @@ -234,14 +227,14 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r RRD_ID_LENGTH_MAX); time_t last_t; - calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); + NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); if(isnan(value)) return 0; buffer_sprintf( instance->buffer, - "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s%s\n", + "put %s.%s.%s %llu " NETDATA_DOUBLE_FORMAT " host=%s%s%s%s\n", instance->config.prefix, chart_name, dimension_name, @@ -250,7 +243,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r (host == localhost) ? instance->config.hostname : host->hostname, (host->tags) ? " " : "", (host->tags) ? host->tags : "", - (instance->labels) ? buffer_tostring(instance->labels) : ""); + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : ""); return 0; } @@ -287,33 +280,17 @@ void opentsdb_http_prepare_header(struct instance *instance) * @param host a data collecting host. * @return Always returns 0. */ -int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) -{ - if (!instance->labels) - instance->labels = buffer_create(1024); + +int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) { + if (!instance->labels_buffer) + instance->labels_buffer = buffer_create(1024); if (unlikely(!sending_labels_configured(instance))) return 0; - rrdhost_check_rdlock(host); - netdata_rwlock_rdlock(&host->labels.labels_rwlock); - for (struct label *label = host->labels.head; label; label = label->next) { - if (!should_send_label(instance, label)) - continue; - - char escaped_value[CONFIG_MAX_VALUE * 2 + 1]; - sanitize_json_string(escaped_value, label->value, CONFIG_MAX_VALUE); - - char value[CONFIG_MAX_VALUE + 1]; - sanitize_opentsdb_label_value(value, escaped_value, CONFIG_MAX_VALUE); - - if (*value) { - buffer_strcat(instance->labels, ","); - buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value); - } - } - netdata_rwlock_unlock(&host->labels.labels_rwlock); - + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, ",", ":", "\"", "", + exporting_labels_filter_callback, instance, + NULL, sanitize_opentsdb_label_value); return 0; } @@ -362,7 +339,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM * (host == localhost) ? instance->config.hostname : host->hostname, (host->tags) ? " " : "", (host->tags) ? host->tags : "", - instance->labels ? buffer_tostring(instance->labels) : ""); + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : ""); return 0; } @@ -392,7 +369,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd) RRD_ID_LENGTH_MAX); time_t last_t; - calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); + NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); if(isnan(value)) return 0; @@ -405,7 +382,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd) "{" "\"metric\":\"%s.%s.%s\"," "\"timestamp\":%llu," - "\"value\":"CALCULATED_NUMBER_FORMAT"," + "\"value\":" NETDATA_DOUBLE_FORMAT "," "\"tags\":{" "\"host\":\"%s%s%s\"%s" "}" @@ -418,7 +395,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd) (host == localhost) ? instance->config.hostname : host->hostname, (host->tags) ? " " : "", (host->tags) ? host->tags : "", - instance->labels ? buffer_tostring(instance->labels) : ""); + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : ""); return 0; } diff --git a/exporting/opentsdb/opentsdb.h b/exporting/opentsdb/opentsdb.h index d53a5054f..b544ba8c1 100644 --- a/exporting/opentsdb/opentsdb.h +++ b/exporting/opentsdb/opentsdb.h @@ -8,7 +8,7 @@ int init_opentsdb_telnet_instance(struct instance *instance); int init_opentsdb_http_instance(struct instance *instance); -void sanitize_opentsdb_label_value(char *dst, char *src, size_t len); +void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len); int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host); int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host); diff --git a/exporting/process_data.c b/exporting/process_data.c index c77b7ad4a..d5138b787 100644 --- a/exporting/process_data.c +++ b/exporting/process_data.c @@ -64,7 +64,7 @@ int mark_scheduled_instances(struct engine *engine) * @param last_timestamp the timestamp that should be reported to the exporting connector instance. * @return Returns the value, calculated over the given period. */ -calculated_number exporting_calculate_value_from_stored_data( +NETDATA_DOUBLE exporting_calculate_value_from_stored_data( struct instance *instance, RRDDIM *rd, time_t *last_timestamp) @@ -77,11 +77,10 @@ calculated_number exporting_calculate_value_from_stored_data( time_t before = instance->before; // find the edges of the rrd database for this chart - time_t first_t = rd->state->query_ops.oldest_time(rd); - time_t last_t = rd->state->query_ops.latest_time(rd); + time_t first_t = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle); + time_t last_t = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle); time_t update_every = st->update_every; struct rrddim_query_handle handle; - storage_number n; // step back a little, to make sure we have complete data collection // for all metrics @@ -124,23 +123,21 @@ calculated_number exporting_calculate_value_from_stored_data( *last_timestamp = before; size_t counter = 0; - calculated_number sum = 0; + NETDATA_DOUBLE sum = 0; - for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) { - time_t curr_t; - n = rd->state->query_ops.next_metric(&handle, &curr_t); + for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before, TIER_QUERY_FETCH_SUM); !rd->tiers[0]->query_ops.is_finished(&handle);) { + STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle); - if (unlikely(!does_storage_number_exist(n))) { + if (unlikely(storage_point_is_empty(sp))) { // not collected continue; } - calculated_number value = unpack_storage_number(n); - sum += value; - - counter++; + sum += sp.sum; + counter += sp.count; } - rd->state->query_ops.finalize(&handle); + rd->tiers[0]->query_ops.finalize(&handle); + if (unlikely(!counter)) { debug( D_EXPORTING, @@ -156,7 +153,7 @@ calculated_number exporting_calculate_value_from_stored_data( if (unlikely(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM)) return sum; - return sum / (calculated_number)counter; + return sum / (NETDATA_DOUBLE)counter; } /** @@ -261,6 +258,27 @@ void end_chart_formatting(struct engine *engine, RRDSET *st) } } +/** + * Format variables for every connector instance's buffer + * + * @param engine an engine data structure. + * @param host a data collecting host. + */ +void variables_formatting(struct engine *engine, RRDHOST *host) +{ + for (struct instance *instance = engine->instance_root; instance; instance = instance->next) { + if (instance->scheduled && !instance->skip_host && should_send_variables(instance)) { + if (instance->variables_formatting && instance->variables_formatting(instance, host) != 0){ + error("EXPORTING: cannot format variables for %s", instance->config.name); + disable_instance(instance); + continue; + } + // sum all variables as one metrics + instance->stats.buffered_metrics++; + } + } +} + /** * End host formatting for every connector instance's buffer * @@ -337,7 +355,7 @@ void prepare_buffers(struct engine *engine) end_chart_formatting(engine, st); rrdset_unlock(st); } - + variables_formatting(engine, host); end_host_formatting(engine, host); rrdhost_unlock(host); } @@ -358,8 +376,8 @@ int flush_host_labels(struct instance *instance, RRDHOST *host) { (void)host; - if (instance->labels) - buffer_flush(instance->labels); + if (instance->labels_buffer) + buffer_flush(instance->labels_buffer); return 0; } diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c index c7f3f1d38..7d632164f 100644 --- a/exporting/prometheus/prometheus.c +++ b/exporting/prometheus/prometheus.c @@ -290,35 +290,44 @@ inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int sh * @param instance an instance data structure. * @param host a data collecting host. */ -void format_host_labels_prometheus(struct instance *instance, RRDHOST *host) -{ - if (unlikely(!sending_labels_configured(instance))) - return; - if (!instance->labels) - instance->labels = buffer_create(1024); +struct format_prometheus_label_callback { + struct instance *instance; + size_t count; +}; - int count = 0; - rrdhost_check_rdlock(host); - netdata_rwlock_rdlock(&host->labels.labels_rwlock); - for (struct label *label = host->labels.head; label; label = label->next) { - if (!should_send_label(instance, label)) - continue; +static int format_prometheus_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { + struct format_prometheus_label_callback *d = (struct format_prometheus_label_callback *)data; - char key[PROMETHEUS_ELEMENT_MAX + 1]; - char value[PROMETHEUS_ELEMENT_MAX + 1]; + if (!should_send_label(d->instance, ls)) return 0; - prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX); - prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX); + char k[PROMETHEUS_ELEMENT_MAX + 1]; + char v[PROMETHEUS_ELEMENT_MAX + 1]; - if (*key && *value) { - if (count > 0) - buffer_strcat(instance->labels, ","); - buffer_sprintf(instance->labels, "%s=\"%s\"", key, value); - count++; - } + prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX); + + if (*k && *v) { + if (d->count > 0) buffer_strcat(d->instance->labels_buffer, ","); + buffer_sprintf(d->instance->labels_buffer, "%s=\"%s\"", k, v); + d->count++; } - netdata_rwlock_unlock(&host->labels.labels_rwlock); + return 1; +} + +void format_host_labels_prometheus(struct instance *instance, RRDHOST *host) +{ + if (unlikely(!sending_labels_configured(instance))) + return; + + if (!instance->labels_buffer) + instance->labels_buffer = buffer_create(1024); + + struct format_prometheus_label_callback tmp = { + .instance = instance, + .count = 0 + }; + rrdlabels_walkthrough_read(host->host_labels, format_prometheus_label_callback, &tmp); } struct host_variables_callback_options { @@ -353,7 +362,7 @@ static int print_host_variables(RRDVAR *rv, void *data) } } - calculated_number value = rrdvar2number(rv); + NETDATA_DOUBLE value = rrdvar2number(rv); if (isnan(value) || isinf(value)) { if (opts->output_options & PROMETHEUS_OUTPUT_HELP) buffer_sprintf( @@ -374,7 +383,7 @@ static int print_host_variables(RRDVAR *rv, void *data) if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) buffer_sprintf( opts->wb, - "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n", + "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n", opts->prefix, opts->name, label_pre, @@ -385,7 +394,7 @@ static int print_host_variables(RRDVAR *rv, void *data) else buffer_sprintf( opts->wb, - "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n", + "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT "\n", opts->prefix, opts->name, label_pre, @@ -474,9 +483,9 @@ static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters if (prometheus_collector) buffer_sprintf( wb, - CALCULATED_NUMBER_FORMAT, - (calculated_number)p->rd->last_collected_value * (calculated_number)p->rd->multiplier / - (calculated_number)p->rd->divisor); + NETDATA_DOUBLE_FORMAT, + (NETDATA_DOUBLE)p->rd->last_collected_value * (NETDATA_DOUBLE)p->rd->multiplier / + (NETDATA_DOUBLE)p->rd->divisor); else buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT, p->rd->last_collected_value); @@ -516,77 +525,29 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( format_host_labels_prometheus(instance, host); + buffer_sprintf( + wb, + "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"", + hostname, + host->program_name, + host->program_version); + + if (instance->labels_buffer && *buffer_tostring(instance->labels_buffer)) { + buffer_sprintf(wb, ",%s", buffer_tostring(instance->labels_buffer)); + } + if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) - buffer_sprintf( - wb, - "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", - hostname, - host->program_name, - host->program_version, - now_realtime_usec() / USEC_PER_MS); + buffer_sprintf(wb, "} 1 %llu\n", now_realtime_usec() / USEC_PER_MS); else - buffer_sprintf( - wb, - "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", - hostname, - host->program_name, - host->program_version); + buffer_sprintf(wb, "} 1\n"); char labels[PROMETHEUS_LABELS_MAX + 1] = ""; if (allhosts) { - if (instance->labels && buffer_tostring(instance->labels)) { - if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) { - buffer_sprintf( - wb, - "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n", - hostname, - buffer_tostring(instance->labels), - now_realtime_usec() / USEC_PER_MS); - - // deprecated, exists only for compatibility with older queries - buffer_sprintf( - wb, - "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", - hostname, - buffer_tostring(instance->labels), - now_realtime_usec() / USEC_PER_MS); - } else { - buffer_sprintf( - wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels)); - - // deprecated, exists only for compatibility with older queries - buffer_sprintf( - wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels)); - } - } - snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname); - } else { - if (instance->labels && buffer_tostring(instance->labels)) { - if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) { - buffer_sprintf( - wb, - "netdata_host_tags_info{%s} 1 %llu\n", - buffer_tostring(instance->labels), - now_realtime_usec() / USEC_PER_MS); - - // deprecated, exists only for compatibility with older queries - buffer_sprintf( - wb, - "netdata_host_tags{%s} 1 %llu\n", - buffer_tostring(instance->labels), - now_realtime_usec() / USEC_PER_MS); - } else { - buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels)); - - // deprecated, exists only for compatibility with older queries - buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels)); - } - } - } + } - if (instance->labels) - buffer_flush(instance->labels); + if (instance->labels_buffer) + buffer_flush(instance->labels_buffer); // send custom variables set for the host if (output_options & PROMETHEUS_OUTPUT_VARIABLES) { @@ -723,7 +684,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( time_t first_time = instance->after; time_t last_time = instance->before; - calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_time); + NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_time); if (!isnan(value) && !isinf(value)) { if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE) @@ -755,7 +716,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) buffer_sprintf( wb, - "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT + "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT " %llu\n", prefix, context, @@ -770,7 +731,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( else buffer_sprintf( wb, - "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT + "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT "\n", prefix, context, diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c index 59a488e1b..03feb2c08 100644 --- a/exporting/prometheus/remote_write/remote_write.c +++ b/exporting/prometheus/remote_write/remote_write.c @@ -97,6 +97,7 @@ int init_prometheus_remote_write_instance(struct instance *instance) instance->start_chart_formatting = format_chart_prometheus_remote_write; instance->metric_formatting = format_dimension_prometheus_remote_write; instance->end_chart_formatting = NULL; + instance->variables_formatting = format_variables_prometheus_remote_write; instance->end_host_formatting = NULL; instance->end_batch_formatting = format_batch_prometheus_remote_write; @@ -134,6 +135,25 @@ int init_prometheus_remote_write_instance(struct instance *instance) return 0; } +struct format_remote_write_label_callback { + struct instance *instance; + void *write_request; +}; + +static int format_remote_write_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { + struct format_remote_write_label_callback *d = (struct format_remote_write_label_callback *)data; + + if (!should_send_label(d->instance, ls)) return 0; + + char k[PROMETHEUS_ELEMENT_MAX + 1]; + char v[PROMETHEUS_ELEMENT_MAX + 1]; + + prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX); + add_label(d->write_request, k, v); + return 1; +} + /** * Format host data for Prometheus Remote Write connector * @@ -157,23 +177,13 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host add_host_info( connector_specific_data->write_request, "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS); - + if (unlikely(sending_labels_configured(instance))) { - rrdhost_check_rdlock(host); - netdata_rwlock_rdlock(&host->labels.labels_rwlock); - for (struct label *label = host->labels.head; label; label = label->next) { - if (!should_send_label(instance, label)) - continue; - - char key[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX); - - char value[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX); - - add_label(connector_specific_data->write_request, key, value); - } - netdata_rwlock_unlock(&host->labels.labels_rwlock); + struct format_remote_write_label_callback tmp = { + .write_request = connector_specific_data->write_request, + .instance = instance + }; + rrdlabels_walkthrough_read(host->host_labels, format_remote_write_label_callback, &tmp); } return 0; @@ -284,7 +294,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * // we need average or sum of the data time_t last_t = instance->before; - calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); + NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t); if (!isnan(value) && !isinf(value)) { if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE) @@ -311,6 +321,49 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * return 0; } +int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) { + struct prometheus_remote_write_variables_callback_options *opts = data; + + if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) { + RRDHOST *host = opts->host; + struct instance *instance = opts->instance; + struct simple_connector_data *simple_connector_data = + (struct simple_connector_data *)instance->connector_specific_data; + struct prometheus_remote_write_specific_data *connector_specific_data = + (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data; + + char name[PROMETHEUS_LABELS_MAX + 1]; + char *suffix = ""; + + prometheus_name_copy(context, rv->name, PROMETHEUS_ELEMENT_MAX); + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix); + + NETDATA_DOUBLE value = rrdvar2number(rv); + add_variable(connector_specific_data->write_request, name, + (host == localhost) ? instance->config.hostname : host->hostname, value, opts->now / USEC_PER_MS); + } + + return 0; +} + +/** + * Format a variable for Prometheus Remote Write connector + * + * @param rv a variable. + * @param instance an instance data structure. + * @return Always returns 0. + */ +int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host) +{ + struct prometheus_remote_write_variables_callback_options opt = { + .host = host, + .instance = instance, + .now = now_realtime_usec(), + }; + + return foreach_host_variable_callback(host, format_variable_prometheus_remote_write_callback, &opt); +} + /** * Format a batch for Prometheus Remote Write connector * @@ -339,7 +392,6 @@ int format_batch_prometheus_remote_write(struct instance *instance) return 1; } buffer->len = data_size; - instance->stats.buffered_bytes = (collected_number)buffer_strlen(buffer); simple_connector_end_batch(instance); diff --git a/exporting/prometheus/remote_write/remote_write.h b/exporting/prometheus/remote_write/remote_write.h index d738f5126..4740772d0 100644 --- a/exporting/prometheus/remote_write/remote_write.h +++ b/exporting/prometheus/remote_write/remote_write.h @@ -11,12 +11,19 @@ struct prometheus_remote_write_specific_data { void *write_request; }; +struct prometheus_remote_write_variables_callback_options { + RRDHOST *host; + time_t now; + struct instance *instance; +}; + int init_prometheus_remote_write_instance(struct instance *instance); extern void clean_prometheus_remote_write(struct instance *instance); int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host); int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st); int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd); +int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host); int format_batch_prometheus_remote_write(struct instance *instance); void prometheus_remote_write_prepare_header(struct instance *instance); diff --git a/exporting/prometheus/remote_write/remote_write_request.cc b/exporting/prometheus/remote_write/remote_write_request.cc index cfd61271e..ecfa11fa8 100644 --- a/exporting/prometheus/remote_write/remote_write_request.cc +++ b/exporting/prometheus/remote_write/remote_write_request.cc @@ -137,6 +137,38 @@ void add_metric( sample->set_timestamp(timestamp); } +/** + * Adds a metric to a write request + * + * @param write_request_p the write request + * @param name the name of the metric + * @param instance the name of the host, the metric belongs to + * @param value the value of the metric + * @param timestamp the timestamp for the metric in milliseconds + */ +void add_variable( + void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp) +{ + WriteRequest *write_request = (WriteRequest *)write_request_p; + TimeSeries *timeseries; + Sample *sample; + Label *label; + + timeseries = write_request->add_timeseries(); + + label = timeseries->add_labels(); + label->set_name("__name__"); + label->set_value(name); + + label = timeseries->add_labels(); + label->set_name("instance"); + label->set_value(instance); + + sample = timeseries->add_samples(); + sample->set_value(value); + sample->set_timestamp(timestamp); +} + /** * Gets the size of a write request * diff --git a/exporting/prometheus/remote_write/remote_write_request.h b/exporting/prometheus/remote_write/remote_write_request.h index 5f242b941..b25370133 100644 --- a/exporting/prometheus/remote_write/remote_write_request.h +++ b/exporting/prometheus/remote_write/remote_write_request.h @@ -20,6 +20,9 @@ void add_metric( const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp); +void add_variable( + void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp); + size_t get_write_request_size(void *write_request_p); int pack_and_clear_write_request(void *write_request_p, char *buffer, size_t *size); diff --git a/exporting/pubsub/pubsub.c b/exporting/pubsub/pubsub.c index 5a5afbdc2..b218338f1 100644 --- a/exporting/pubsub/pubsub.c +++ b/exporting/pubsub/pubsub.c @@ -23,6 +23,7 @@ int init_pubsub_instance(struct instance *instance) instance->metric_formatting = format_dimension_stored_json_plaintext; instance->end_chart_formatting = NULL; + instance->variables_formatting = NULL; instance->end_host_formatting = flush_host_labels; instance->end_batch_formatting = NULL; diff --git a/exporting/read_config.c b/exporting/read_config.c index b834e867d..1cba16826 100644 --- a/exporting/read_config.c +++ b/exporting/read_config.c @@ -399,6 +399,11 @@ struct engine *read_exporting_config() else tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES; + if (exporter_get_boolean(instance_name, "send variables", CONFIG_BOOLEAN_YES)) + tmp_instance->config.options |= EXPORTING_OPTION_SEND_VARIABLES; + else + tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_VARIABLES; + if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) { struct prometheus_remote_write_specific_config *connector_specific_config = callocz(1, sizeof(struct prometheus_remote_write_specific_config)); diff --git a/exporting/tests/exporting_doubles.c b/exporting/tests/exporting_doubles.c index b8c9f3756..75ab7ba43 100644 --- a/exporting/tests/exporting_doubles.c +++ b/exporting/tests/exporting_doubles.c @@ -52,11 +52,11 @@ int __wrap_mark_scheduled_instances(struct engine *engine) return mock_type(int); } -calculated_number __real_exporting_calculate_value_from_stored_data( +NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data( struct instance *instance, RRDDIM *rd, time_t *last_timestamp); -calculated_number __wrap_exporting_calculate_value_from_stored_data( +NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data( struct instance *instance, RRDDIM *rd, time_t *last_timestamp) @@ -67,7 +67,7 @@ calculated_number __wrap_exporting_calculate_value_from_stored_data( *last_timestamp = 15052; function_called(); - return mock_type(calculated_number); + return mock_type(NETDATA_DOUBLE); } int __real_prepare_buffers(struct engine *engine); @@ -156,6 +156,14 @@ int __mock_end_chart_formatting(struct instance *instance, RRDSET *st) return mock_type(int); } +int __mock_variables_formatting(struct instance *instance, RRDHOST *host) +{ + function_called(); + check_expected_ptr(instance); + check_expected_ptr(host); + return mock_type(int); +} + int __mock_end_host_formatting(struct instance *instance, RRDHOST *host) { function_called(); diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c index 501fc405c..aae1c53fb 100644 --- a/exporting/tests/exporting_fixtures.c +++ b/exporting/tests/exporting_fixtures.c @@ -41,17 +41,9 @@ int setup_rrdhost() localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2"); - struct label *label = calloc(1, sizeof(struct label)); - label->key = strdupz("key1"); - label->value = strdupz("value1"); - label->label_source = LABEL_SOURCE_NETDATA_CONF; - localhost->labels.head = label; - - label = calloc(1, sizeof(struct label)); - label->key = strdupz("key2"); - label->value = strdupz("value2"); - label->label_source = LABEL_SOURCE_AUTO; - localhost->labels.head->next = label; + localhost->host_labels = rrdlabels_create(); + rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG); + rrdlabels_add(localhost->host_labels, "key2", "value2", RRDLABEL_SRC_CONFIG); localhost->rrdset_root = calloc(1, sizeof(RRDSET)); RRDSET *st = localhost->rrdset_root; @@ -71,13 +63,13 @@ int setup_rrdhost() rd->collections_counter++; rd->next = NULL; - rd->state = calloc(1, sizeof(*rd->state)); - rd->state->query_ops.oldest_time = __mock_rrddim_query_oldest_time; - rd->state->query_ops.latest_time = __mock_rrddim_query_latest_time; - rd->state->query_ops.init = __mock_rrddim_query_init; - rd->state->query_ops.is_finished = __mock_rrddim_query_is_finished; - rd->state->query_ops.next_metric = __mock_rrddim_query_next_metric; - rd->state->query_ops.finalize = __mock_rrddim_query_finalize; + rd->tiers[0] = calloc(1, sizeof(struct rrddim_tier)); + rd->tiers[0]->query_ops.oldest_time = __mock_rrddim_query_oldest_time; + rd->tiers[0]->query_ops.latest_time = __mock_rrddim_query_latest_time; + rd->tiers[0]->query_ops.init = __mock_rrddim_query_init; + rd->tiers[0]->query_ops.is_finished = __mock_rrddim_query_is_finished; + rd->tiers[0]->query_ops.next_metric = __mock_rrddim_query_next_metric; + rd->tiers[0]->query_ops.finalize = __mock_rrddim_query_finalize; return 0; } @@ -87,19 +79,14 @@ int teardown_rrdhost() RRDDIM *rd = localhost->rrdset_root->dimensions; free((void *)rd->name); free((void *)rd->id); - free(rd->state); + free(rd->tiers[0]); free(rd); RRDSET *st = localhost->rrdset_root; free((void *)st->name); free(st); - free(localhost->labels.head->next->key); - free(localhost->labels.head->next->value); - free(localhost->labels.head->next); - free(localhost->labels.head->key); - free(localhost->labels.head->value); - free(localhost->labels.head); + rrdlabels_destroy(localhost->host_labels); free((void *)localhost->tags); free(localhost); @@ -124,7 +111,7 @@ int teardown_initialized_engine(void **state) struct engine *engine = *state; teardown_rrdhost(); - buffer_free(engine->instance_root->labels); + buffer_free(engine->instance_root->labels_buffer); buffer_free(engine->instance_root->buffer); teardown_configured_engine(state); diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c index a9a184336..ee36e887a 100644 --- a/exporting/tests/netdata_doubles.c +++ b/exporting/tests/netdata_doubles.c @@ -177,7 +177,7 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) return RRD_MEMORY_MODE_NONE_NAME; } -calculated_number rrdvar2number(RRDVAR *rv) +NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) { (void)rv; return 0; @@ -196,26 +196,27 @@ void rrdset_update_heterogeneous_flag(RRDSET *st) (void)st; } -time_t __mock_rrddim_query_oldest_time(RRDDIM *rd) +time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle) { - (void)rd; + (void)db_metric_handle; function_called(); return mock_type(time_t); } -time_t __mock_rrddim_query_latest_time(RRDDIM *rd) +time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle) { - (void)rd; + (void)db_metric_handle; function_called(); return mock_type(time_t); } -void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time) +void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type) { - (void)rd; + (void)db_metric_handle; (void)handle; + (void)tier_query_fetch_type; function_called(); check_expected(start_time); @@ -230,13 +231,14 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle) return mock_type(int); } -storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time) +STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle) { (void)handle; - (void)current_time; function_called(); - return mock_type(storage_number); + + STORAGE_POINT sp = {}; + return sp; } void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle) @@ -245,3 +247,11 @@ void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle) function_called(); } + +void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) +{ + (void)chart_uuid; + (void)source_type; + (void)label; + (void)value; +} diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c index 6bb7d2efd..56a28059f 100644 --- a/exporting/tests/test_exporting_engine.c +++ b/exporting/tests/test_exporting_engine.c @@ -307,19 +307,17 @@ static void test_exporting_calculate_value_from_stored_data(void **state) expect_function_call(__mock_rrddim_query_is_finished); will_return(__mock_rrddim_query_is_finished, 0); expect_function_call(__mock_rrddim_query_next_metric); - will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_DEFAULT_FLAGS)); expect_function_call(__mock_rrddim_query_is_finished); will_return(__mock_rrddim_query_is_finished, 0); expect_function_call(__mock_rrddim_query_next_metric); - will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_DEFAULT_FLAGS)); expect_function_call(__mock_rrddim_query_is_finished); will_return(__mock_rrddim_query_is_finished, 1); expect_function_call(__mock_rrddim_query_finalize); - assert_int_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, ×tamp), 36); + assert_float_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, ×tamp), 36, 0.1); } static void test_prepare_buffers(void **state) @@ -381,7 +379,7 @@ static void test_prepare_buffers(void **state) expect_value(__mock_end_batch_formatting, instance, instance); will_return(__mock_end_batch_formatting, 0); - assert_int_equal(__real_prepare_buffers(engine), 0); + __real_prepare_buffers(engine); assert_int_equal(instance->stats.buffered_metrics, 1); @@ -393,7 +391,7 @@ static void test_prepare_buffers(void **state) instance->end_chart_formatting = NULL; instance->end_host_formatting = NULL; instance->end_batch_formatting = NULL; - assert_int_equal(__real_prepare_buffers(engine), 0); + __real_prepare_buffers(engine); assert_int_equal(instance->scheduled, 0); assert_int_equal(instance->after, 2); @@ -705,7 +703,7 @@ static void test_format_host_labels_json_plaintext(void **state) instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; assert_int_equal(format_host_labels_json_plaintext(instance, localhost), 0); - assert_string_equal(buffer_tostring(instance->labels), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},"); + assert_string_equal(buffer_tostring(instance->labels_buffer), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},"); } static void test_format_host_labels_graphite_plaintext(void **state) @@ -717,7 +715,7 @@ static void test_format_host_labels_graphite_plaintext(void **state) instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; assert_int_equal(format_host_labels_graphite_plaintext(instance, localhost), 0); - assert_string_equal(buffer_tostring(instance->labels), ";key1=value1;key2=value2"); + assert_string_equal(buffer_tostring(instance->labels_buffer), ";key1=value1;key2=value2"); } static void test_format_host_labels_opentsdb_telnet(void **state) @@ -729,7 +727,7 @@ static void test_format_host_labels_opentsdb_telnet(void **state) instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; assert_int_equal(format_host_labels_opentsdb_telnet(instance, localhost), 0); - assert_string_equal(buffer_tostring(instance->labels), " key1=value1 key2=value2"); + assert_string_equal(buffer_tostring(instance->labels_buffer), " key1=value1 key2=value2"); } static void test_format_host_labels_opentsdb_http(void **state) @@ -741,7 +739,7 @@ static void test_format_host_labels_opentsdb_http(void **state) instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; assert_int_equal(format_host_labels_opentsdb_http(instance, localhost), 0); - assert_string_equal(buffer_tostring(instance->labels), ",\"key1\":\"value1\",\"key2\":\"value2\""); + assert_string_equal(buffer_tostring(instance->labels_buffer), ",\"key1\":\"value1\",\"key2\":\"value2\""); } static void test_flush_host_labels(void **state) @@ -749,12 +747,12 @@ static void test_flush_host_labels(void **state) struct engine *engine = *state; struct instance *instance = engine->instance_root; - instance->labels = buffer_create(12); - buffer_strcat(instance->labels, "check string"); - assert_int_equal(buffer_strlen(instance->labels), 12); + instance->labels_buffer = buffer_create(12); + buffer_strcat(instance->labels_buffer, "check string"); + assert_int_equal(buffer_strlen(instance->labels_buffer), 12); assert_int_equal(flush_host_labels(instance, localhost), 0); - assert_int_equal(buffer_strlen(instance->labels), 0); + assert_int_equal(buffer_strlen(instance->labels_buffer), 0); } static void test_create_main_rusage_chart(void **state) @@ -1048,7 +1046,7 @@ static void test_format_host_labels_prometheus(void **state) instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; format_host_labels_prometheus(instance, localhost); - assert_string_equal(buffer_tostring(instance->labels), "key1=\"value1\",key2=\"value2\""); + assert_string_equal(buffer_tostring(instance->labels_buffer), "key1=\"value1\",key2=\"value2\""); } static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) @@ -1071,9 +1069,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n" - "netdata_host_tags_info{key1=\"value1\",key2=\"value2\"} 1\n" - "netdata_host_tags{key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\"} 690565856.0000000\n"); buffer_flush(buffer); @@ -1089,9 +1085,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n" - "netdata_host_tags_info{key1=\"value1\",key2=\"value2\"} 1\n" - "netdata_host_tags{key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" "# TYPE test_prefix_test_context gauge\n" "test_prefix_test_context{chart=\"chart_name\",family=\"test_family\",dimension=\"dimension_name\"} 690565856.0000000\n"); @@ -1107,9 +1101,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n" - "netdata_host_tags_info{instance=\"test_hostname\",key1=\"value1\",key2=\"value2\"} 1\n" - "netdata_host_tags{instance=\"test_hostname\",key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\",instance=\"test_hostname\"} 690565856.0000000\n"); free(localhost->rrdset_root->context); diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h index 800be1b99..ae0b7df9a 100644 --- a/exporting/tests/test_exporting_engine.h +++ b/exporting/tests/test_exporting_engine.h @@ -30,7 +30,14 @@ #include #include #include + +#ifndef UNIT_TESTING +#include +#else +#undef UNIT_TESTING #include +#define UNIT_TESTING +#endif #define MAX_LOG_LINE 1024 extern char log_line[]; @@ -50,11 +57,11 @@ int __wrap_connect_to_one_of( void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line); void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line); void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line); -time_t __mock_rrddim_query_oldest_time(RRDDIM *rd); -time_t __mock_rrddim_query_latest_time(RRDDIM *rd); -void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time); +time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle); +time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle); +void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type); int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle); -storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time); +STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle); void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle); // ----------------------------------------------------------------------- @@ -81,11 +88,11 @@ int __wrap_init_connectors(struct engine *engine); int __real_mark_scheduled_instances(struct engine *engine); int __wrap_mark_scheduled_instances(struct engine *engine); -calculated_number __real_exporting_calculate_value_from_stored_data( +NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data( struct instance *instance, RRDDIM *rd, time_t *last_timestamp); -calculated_number __wrap_exporting_calculate_value_from_stored_data( +NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data( struct instance *instance, RRDDIM *rd, time_t *last_timestamp); @@ -113,6 +120,7 @@ int __mock_start_host_formatting(struct instance *instance, RRDHOST *host); int __mock_start_chart_formatting(struct instance *instance, RRDSET *st); int __mock_metric_formatting(struct instance *instance, RRDDIM *rd); int __mock_end_chart_formatting(struct instance *instance, RRDSET *st); +int __mock_variables_formatting(struct instance *instance, RRDHOST *host); int __mock_end_host_formatting(struct instance *instance, RRDHOST *host); int __mock_end_batch_formatting(struct instance *instance); -- cgit v1.2.3