From 841395dd16f470e3c051a0a4fff5b91efc983c30 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 19 May 2021 14:33:27 +0200 Subject: Adding upstream version 1.31.0. Signed-off-by: Daniel Baumann --- exporting/aws_kinesis/aws_kinesis_put_record.cc | 4 +- exporting/check_filters.c | 8 ++-- exporting/exporting_engine.c | 60 +++++++++++++++++++++++- exporting/exporting_engine.h | 6 +++ exporting/graphite/graphite.c | 4 +- exporting/opentsdb/opentsdb.c | 4 +- exporting/prometheus/prometheus.c | 15 +++--- exporting/prometheus/remote_write/remote_write.c | 4 +- exporting/pubsub/pubsub_publish.cc | 6 +-- exporting/read_config.c | 8 +++- exporting/send_data.c | 7 ++- exporting/tests/exporting_fixtures.c | 2 + exporting/tests/netdata_doubles.c | 2 +- exporting/tests/test_exporting_engine.c | 15 +++--- 14 files changed, 113 insertions(+), 32 deletions(-) (limited to 'exporting') diff --git a/exporting/aws_kinesis/aws_kinesis_put_record.cc b/exporting/aws_kinesis/aws_kinesis_put_record.cc index b20ec1373..62c6b0301 100644 --- a/exporting/aws_kinesis/aws_kinesis_put_record.cc +++ b/exporting/aws_kinesis/aws_kinesis_put_record.cc @@ -110,11 +110,11 @@ void kinesis_put_record( } /** - * Get results from service responces + * Get results from service responses * * @param request_outcomes_p request outcome information. * @param error_message report error message to a caller. - * @param sent_bytes report to a caller how many bytes was successfuly sent. + * @param sent_bytes report to a caller how many bytes was successfully sent. * @param lost_bytes report to a caller how many bytes was lost during transmission. * @return Returns 0 if all data was sent successfully, 1 when data was lost on transmission */ diff --git a/exporting/check_filters.c b/exporting/check_filters.c index 8d70c6f68..64ced7238 100644 --- a/exporting/check_filters.c +++ b/exporting/check_filters.c @@ -50,15 +50,15 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st) RRDSET_FLAGS *flags = &st->exporting_flags[instance->index]; - if(unlikely(*flags & RRDSET_FLAG_BACKEND_IGNORE)) + if(unlikely(*flags & RRDSET_FLAG_EXPORTING_IGNORE)) return 0; - if(unlikely(!(*flags & RRDSET_FLAG_BACKEND_SEND))) { + if(unlikely(!(*flags & RRDSET_FLAG_EXPORTING_SEND))) { // we have not checked this chart if(simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name)) - *flags |= RRDSET_FLAG_BACKEND_SEND; + *flags |= RRDSET_FLAG_EXPORTING_SEND; else { - *flags |= RRDSET_FLAG_BACKEND_IGNORE; + *flags |= RRDSET_FLAG_EXPORTING_IGNORE; debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, host->hostname); return 0; } diff --git a/exporting/exporting_engine.c b/exporting/exporting_engine.c index 6a1320cd1..70aceea8c 100644 --- a/exporting/exporting_engine.c +++ b/exporting/exporting_engine.c @@ -4,12 +4,70 @@ static struct engine *engine = NULL; +void analytics_exporting_connectors(BUFFER *b) +{ + if (!engine) + return; + + uint8_t count = 0; + + for (struct instance *instance = engine->instance_root; instance; instance = instance->next) { + if (count) + buffer_strcat(b, "|"); + + switch (instance->config.type) { + case EXPORTING_CONNECTOR_TYPE_GRAPHITE: + buffer_strcat(b, "Graphite"); + break; + case EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP: + buffer_strcat(b, "GraphiteHTTP"); + break; + case EXPORTING_CONNECTOR_TYPE_JSON: + buffer_strcat(b, "JSON"); + break; + case EXPORTING_CONNECTOR_TYPE_JSON_HTTP: + buffer_strcat(b, "JSONHTTP"); + break; + case EXPORTING_CONNECTOR_TYPE_OPENTSDB: + buffer_strcat(b, "OpenTSDB"); + break; + case EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP: + buffer_strcat(b, "OpenTSDBHTTP"); + break; + case EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE: +#if ENABLE_PROMETHEUS_REMOTE_WRITE + buffer_strcat(b, "PrometheusRemoteWrite"); +#endif + break; + case EXPORTING_CONNECTOR_TYPE_KINESIS: +#if HAVE_KINESIS + buffer_strcat(b, "Kinesis"); +#endif + break; + case EXPORTING_CONNECTOR_TYPE_PUBSUB: +#if ENABLE_EXPORTING_PUBSUB + buffer_strcat(b, "Pubsub"); +#endif + break; + case EXPORTING_CONNECTOR_TYPE_MONGODB: +#if HAVE_MONGOC + buffer_strcat(b, "MongoDB"); +#endif + break; + default: + buffer_strcat(b, "Unknown"); + } + + count++; + } +} + /** * Exporting Clean Engine * * Clean all variables allocated inside engine structure * - * @param en a pointer to the strcuture that will be cleaned. + * @param en a pointer to the structure that will be cleaned. */ static void exporting_clean_engine() { diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h index 1d9feb7dd..1ad6e6856 100644 --- a/exporting/exporting_engine.h +++ b/exporting/exporting_engine.h @@ -77,6 +77,8 @@ struct instance_config { SIMPLE_PATTERN *charts_pattern; SIMPLE_PATTERN *hosts_pattern; + int initialized; + void *connector_specific_config; }; @@ -96,9 +98,13 @@ struct simple_connector_buffer { struct simple_connector_buffer *next; }; +#define CONNECTED_TO_MAX 1024 + struct simple_connector_data { void *connector_specific_data; + char connected_to[CONNECTED_TO_MAX]; + size_t total_buffered_metrics; BUFFER *header; diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c index 9c09631f1..722db0fff 100644 --- a/exporting/graphite/graphite.c +++ b/exporting/graphite/graphite.c @@ -64,7 +64,7 @@ int init_graphite_instance(struct instance *instance) } /** - * Copy a label value and substitute underscores in place of charachters which can't be used in Graphite output + * Copy a label value and substitute underscores in place of characters which can't be used in Graphite output * * @param dst a destination string. * @param src a source string. @@ -205,7 +205,7 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM } /** - * Ppepare HTTP header + * Prepare HTTP header * * @param instance an instance data structure. * @return Returns 0 on success, 1 on failure. diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c index d7b843dff..1310c150e 100644 --- a/exporting/opentsdb/opentsdb.c +++ b/exporting/opentsdb/opentsdb.c @@ -117,7 +117,7 @@ int init_opentsdb_http_instance(struct instance *instance) } /** - * Copy a label value and substitute underscores in place of charachters which can't be used in OpenTSDB output + * Copy a label value and substitute underscores in place of characters which can't be used in OpenTSDB output * * @param dst a destination string. * @param src a source string. @@ -256,7 +256,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r } /** - * Ppepare HTTP header + * Prepare HTTP header * * @param instance an instance data structure. * @return Returns 0 on success, 1 on failure. diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c index c10d94b90..6759313c3 100644 --- a/exporting/prometheus/prometheus.c +++ b/exporting/prometheus/prometheus.c @@ -18,16 +18,16 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st) { RRDHOST *host = st->rrdhost; - if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE))) + if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_EXPORTING_IGNORE))) return 0; - if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) { + if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_EXPORTING_SEND))) { // we have not checked this chart if (simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name)) - rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND); + rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_SEND); else { - rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE); + rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_IGNORE); debug( D_BACKEND, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", @@ -794,6 +794,9 @@ static inline time_t prometheus_preparation( time_t now, PROMETHEUS_OUTPUT_OPTIONS output_options) { +#ifndef UNIT_TESTING + analytics_log_prometheus(); +#endif if (!server || !*server) server = "default"; @@ -855,7 +858,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host( EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options) { - if (unlikely(!prometheus_exporter_instance)) + if (unlikely(!prometheus_exporter_instance || !prometheus_exporter_instance->config.initialized)) return; prometheus_exporter_instance->before = now_realtime_sec(); @@ -892,7 +895,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts( EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options) { - if (unlikely(!prometheus_exporter_instance)) + if (unlikely(!prometheus_exporter_instance || !prometheus_exporter_instance->config.initialized)) return; prometheus_exporter_instance->before = now_realtime_sec(); diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c index 30bd05ad7..986ad9f0e 100644 --- a/exporting/prometheus/remote_write/remote_write.c +++ b/exporting/prometheus/remote_write/remote_write.c @@ -31,12 +31,12 @@ void prometheus_remote_write_prepare_header(struct instance *instance) "Content-Length: %zu\r\n" "\r\n", connector_specific_config->remote_write_path, - instance->config.destination, + simple_connector_data->connected_to, buffer_strlen(simple_connector_data->last_buffer->buffer)); } /** - * Process a responce received after Prometheus remote write connector had sent data + * Process a response received after Prometheus remote write connector had sent data * * @param buffer a response from a remote service. * @param instance an instance data structure. diff --git a/exporting/pubsub/pubsub_publish.cc b/exporting/pubsub/pubsub_publish.cc index dc237cf22..6122dddba 100644 --- a/exporting/pubsub/pubsub_publish.cc +++ b/exporting/pubsub/pubsub_publish.cc @@ -178,12 +178,12 @@ int pubsub_publish(void *pubsub_specific_data_p, char *error_message, size_t buf } /** - * Get results from service responces + * Get results from service responses * * @param pubsub_specific_data_p a pointer to a structure with instance-wide data. * @param error_message report error message to a caller. - * @param sent_metrics report to a caller how many metrics was successfuly sent. - * @param sent_bytes report to a caller how many bytes was successfuly sent. + * @param sent_metrics report to a caller how many metrics was successfully sent. + * @param sent_bytes report to a caller how many bytes was successfully sent. * @param lost_metrics report to a caller how many metrics was lost during transmission. * @param lost_bytes report to a caller how many bytes was lost during transmission. * @return Returns 0 if all data was sent successfully, 1 when data was lost on transmission. diff --git a/exporting/read_config.c b/exporting/read_config.c index 995ba578f..ea50fa0f6 100644 --- a/exporting/read_config.c +++ b/exporting/read_config.c @@ -267,12 +267,16 @@ struct engine *read_exporting_config() else prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; - prometheus_exporter_instance->config.charts_pattern = - simple_pattern_create(prometheus_config_get("send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT); + prometheus_exporter_instance->config.charts_pattern = simple_pattern_create( + prometheus_config_get("send charts matching", global_backend_send_charts_matching), + NULL, + SIMPLE_PATTERN_EXACT); prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create( prometheus_config_get("send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT); prometheus_exporter_instance->config.prefix = prometheus_config_get("prefix", global_backend_prefix); + + prometheus_exporter_instance->config.initialized = 1; } // TODO: change BACKEND to EXPORTING diff --git a/exporting/send_data.c b/exporting/send_data.c index 1e932e98f..0f5e41929 100644 --- a/exporting/send_data.c +++ b/exporting/send_data.c @@ -314,7 +314,12 @@ void simple_connector_worker(void *instance_p) size_t reconnects = 0; sock = connect_to_one_of( - instance->config.destination, connector_specific_config->default_port, &timeout, &reconnects, NULL, 0); + instance->config.destination, + connector_specific_config->default_port, + &timeout, + &reconnects, + connector_specific_data->connected_to, + CONNECTED_TO_MAX); #ifdef ENABLE_HTTPS if (exporting_tls_is_enabled(instance->config.type, options) && sock != -1) { if (netdata_exporting_ctx) { diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c index 00bb0ed0f..b5b0ce816 100644 --- a/exporting/tests/exporting_fixtures.c +++ b/exporting/tests/exporting_fixtures.c @@ -146,6 +146,8 @@ int setup_prometheus(void **state) prometheus_exporter_instance->config.charts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT); prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create("*", NULL, SIMPLE_PATTERN_EXACT); + prometheus_exporter_instance->config.initialized = 1; + return 0; } diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c index f4da7769f..a9a184336 100644 --- a/exporting/tests/netdata_doubles.c +++ b/exporting/tests/netdata_doubles.c @@ -2,7 +2,7 @@ #include "test_exporting_engine.h" -// Use memomy allocation functions guarded by CMocka in strdupz +// Use memory allocation functions guarded by CMocka in strdupz const char *__wrap_strdupz(const char *s) { char *duplicate = malloc(sizeof(char) * (strlen(s) + 1)); diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c index 774d1a265..73fd3ca66 100644 --- a/exporting/tests/test_exporting_engine.c +++ b/exporting/tests/test_exporting_engine.c @@ -17,6 +17,7 @@ char log_line[MAX_LOG_LINE + 1]; BACKEND_OPTIONS global_backend_options = 0; const char *global_backend_source = "average"; const char *global_backend_prefix = "netdata"; +const char *global_backend_send_charts_matching = "*"; void init_connectors_in_tests(struct engine *engine) { @@ -268,7 +269,7 @@ static void test_rrdset_is_exportable(void **state) assert_int_equal(__real_rrdset_is_exportable(instance, st), 1); assert_ptr_not_equal(st->exporting_flags, NULL); - assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_SEND); + assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_EXPORTING_SEND); } static void test_false_rrdset_is_exportable(void **state) @@ -285,7 +286,7 @@ static void test_false_rrdset_is_exportable(void **state) assert_int_equal(__real_rrdset_is_exportable(instance, st), 0); assert_ptr_not_equal(st->exporting_flags, NULL); - assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_BACKEND_IGNORE); + assert_int_equal(st->exporting_flags[0], RRDSET_FLAG_EXPORTING_IGNORE); } static void test_exporting_calculate_value_from_stored_data(void **state) @@ -617,6 +618,7 @@ static void test_simple_connector_worker(void **state) simple_connector_data->buffer = buffer_create(0); simple_connector_data->last_buffer->header = buffer_create(0); simple_connector_data->last_buffer->buffer = buffer_create(0); + strcpy(simple_connector_data->connected_to, "localhost"); buffer_sprintf(simple_connector_data->last_buffer->header, "test header"); buffer_sprintf(simple_connector_data->last_buffer->buffer, "test buffer"); @@ -625,8 +627,8 @@ static void test_simple_connector_worker(void **state) expect_string(__wrap_connect_to_one_of, destination, "localhost"); expect_value(__wrap_connect_to_one_of, default_port, 2003); expect_not_value(__wrap_connect_to_one_of, reconnects_counter, 0); - expect_value(__wrap_connect_to_one_of, connected_to, 0); - expect_value(__wrap_connect_to_one_of, connected_to_size, 0); + expect_string(__wrap_connect_to_one_of, connected_to, "localhost"); + expect_value(__wrap_connect_to_one_of, connected_to_size, CONNECTED_TO_MAX); will_return(__wrap_connect_to_one_of, 2); expect_function_call(__wrap_send); @@ -993,9 +995,9 @@ static void test_can_send_rrdset(void **state) assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 1); - rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE); + rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE); assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root), 0); - rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_BACKEND_IGNORE); + rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE); // TODO: test with a denying simple pattern @@ -1169,6 +1171,7 @@ static void test_prometheus_remote_write_prepare_header(void **state) simple_connector_data->last_buffer = callocz(1, sizeof(struct simple_connector_buffer)); simple_connector_data->last_buffer->header = buffer_create(0); simple_connector_data->last_buffer->buffer = buffer_create(0); + strcpy(simple_connector_data->connected_to, "localhost"); buffer_sprintf(simple_connector_data->last_buffer->buffer, "test buffer"); -- cgit v1.2.3