diff options
Diffstat (limited to 'exporting')
| -rw-r--r-- | exporting/check_filters.c | 14 | ||||
| -rw-r--r-- | exporting/exporting_engine.c | 2 | ||||
| -rw-r--r-- | exporting/exporting_engine.h | 4 | ||||
| -rw-r--r-- | exporting/graphite/graphite.c | 20 | ||||
| -rw-r--r-- | exporting/json/json.c | 46 | ||||
| -rw-r--r-- | exporting/opentsdb/opentsdb.c | 38 | ||||
| -rw-r--r-- | exporting/process_data.c | 39 | ||||
| -rw-r--r-- | exporting/prometheus/README.md | 2 | ||||
| -rw-r--r-- | exporting/prometheus/prometheus.c | 122 | ||||
| -rw-r--r-- | exporting/prometheus/prometheus.h | 6 | ||||
| -rw-r--r-- | exporting/prometheus/remote_write/remote_write.c | 47 | ||||
| -rw-r--r-- | exporting/prometheus/remote_write/remote_write.h | 2 | ||||
| -rw-r--r-- | exporting/send_data.c | 6 | ||||
| -rw-r--r-- | exporting/send_internal_metrics.c | 34 | ||||
| -rw-r--r-- | exporting/tests/exporting_fixtures.c | 95 | ||||
| -rw-r--r-- | exporting/tests/netdata_doubles.c | 40 | ||||
| -rw-r--r-- | exporting/tests/test_exporting_engine.c | 217 | ||||
| -rw-r--r-- | exporting/tests/test_exporting_engine.h | 3 |
18 files changed, 424 insertions, 313 deletions
diff --git a/exporting/check_filters.c b/exporting/check_filters.c index 726fd02a1..009a010b3 100644 --- a/exporting/check_filters.c +++ b/exporting/check_filters.c @@ -25,7 +25,7 @@ int rrdhost_is_exportable(struct instance *instance, RRDHOST *host) RRDHOST_FLAGS *flags = &host->exporting_flags[instance->index]; if (unlikely((*flags & (RRDHOST_FLAG_EXPORTING_SEND | RRDHOST_FLAG_EXPORTING_DONT_SEND)) == 0)) { - char *host_name = (host == localhost) ? "localhost" : host->hostname; + const char *host_name = (host == localhost) ? "localhost" : rrdhost_hostname(host); if (!instance->config.hosts_pattern || simple_pattern_matches(instance->config.hosts_pattern, host_name)) { *flags |= RRDHOST_FLAG_EXPORTING_SEND; @@ -55,10 +55,6 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st) RRDHOST *host = st->rrdhost; #endif - // Do not export anomaly rates charts. - if (st->state && st->state->is_ar_chart) - return 0; - if (st->exporting_flags == NULL) st->exporting_flags = callocz(instance->engine->instance_num, sizeof(size_t)); @@ -69,22 +65,22 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st) if(unlikely(!(*flags & RRDSET_FLAG_EXPORTING_SEND))) { // we have not checked this chart - if(simple_pattern_matches(instance->config.charts_pattern, st->id) || simple_pattern_matches(instance->config.charts_pattern, st->name)) + if(simple_pattern_matches(instance->config.charts_pattern, rrdset_id(st)) || simple_pattern_matches(instance->config.charts_pattern, rrdset_name(st))) *flags |= RRDSET_FLAG_EXPORTING_SEND; else { *flags |= RRDSET_FLAG_EXPORTING_IGNORE; - debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", st->id, host->hostname); + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", rrdset_id(st), rrdhost_hostname(host)); return 0; } } if(unlikely(!rrdset_is_available_for_exporting_and_alarms(st))) { - debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", st->id, host->hostname); + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", rrdset_id(st), rrdhost_hostname(host)); return 0; } if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) { - debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode)); + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode)); return 0; } diff --git a/exporting/exporting_engine.c b/exporting/exporting_engine.c index faace86d9..fd16d982b 100644 --- a/exporting/exporting_engine.c +++ b/exporting/exporting_engine.c @@ -7,7 +7,7 @@ static struct engine *engine = NULL; void analytics_exporting_connectors_ssl(BUFFER *b) { #ifdef ENABLE_HTTPS - if (netdata_exporting_ctx) { + if (netdata_ssl_exporting_ctx) { for (struct instance *instance = engine->instance_root; instance; instance = instance->next) { struct simple_connector_data *connector_specific_data = instance->connector_specific_data; if (connector_specific_data->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) { diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h index 2141caa41..5f961c303 100644 --- a/exporting/exporting_engine.h +++ b/exporting/exporting_engine.h @@ -271,7 +271,7 @@ size_t exporting_name_copy(char *dst, const char *src, size_t max_len); int rrdhost_is_exportable(struct instance *instance, RRDHOST *host); int rrdset_is_exportable(struct instance *instance, RRDSET *st); -extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options); +EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options); NETDATA_DOUBLE exporting_calculate_value_from_stored_data( @@ -300,7 +300,7 @@ void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_ void send_main_rusage(RRDSET *st_rusage, RRDDIM *rd_user, RRDDIM *rd_system); void send_internal_metrics(struct instance *instance); -extern void clean_instance(struct instance *ptr); +void clean_instance(struct instance *ptr); void simple_connector_cleanup(struct instance *instance); static inline void disable_instance(struct instance *instance) diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c index 8ca094b3b..0b33f6428 100644 --- a/exporting/graphite/graphite.c +++ b/exporting/graphite/graphite.c @@ -101,7 +101,7 @@ int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *ho if (unlikely(!sending_labels_configured(instance))) return 0; - rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, ";", "=", "", "", + rrdlabels_to_buffer(host->rrdlabels, instance->labels_buffer, ";", "=", "", "", exporting_labels_filter_callback, instance, NULL, sanitize_graphite_label_value); @@ -123,24 +123,24 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); buffer_sprintf( instance->buffer, "%s.%s.%s.%s%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), chart_name, dimension_name, (host->tags) ? ";" : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "", rd->last_collected_value, (unsigned long long)rd->last_collected_time.tv_sec); @@ -163,13 +163,13 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); time_t last_t; @@ -182,11 +182,11 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM instance->buffer, "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), chart_name, dimension_name, (host->tags) ? ";" : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "", value, (unsigned long long)last_t); @@ -214,7 +214,7 @@ void graphite_http_prepare_header(struct instance *instance) "\r\n", instance->config.destination, simple_connector_data->auth_string ? simple_connector_data->auth_string : "", - buffer_strlen(simple_connector_data->last_buffer->buffer)); + (unsigned long int) buffer_strlen(simple_connector_data->last_buffer->buffer)); return; } diff --git a/exporting/json/json.c b/exporting/json/json.c index 45a8c9d9f..dd53f6f0a 100644 --- a/exporting/json/json.c +++ b/exporting/json/json.c @@ -125,7 +125,7 @@ int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host) return 0; buffer_strcat(instance->labels_buffer, "\"labels\":{"); - rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",", + rrdlabels_to_buffer(host->rrdlabels, instance->labels_buffer, "", ":", "\"", ",", exporting_labels_filter_callback, instance, NULL, sanitize_json_string); buffer_strcat(instance->labels_buffer, "},"); @@ -145,7 +145,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM RRDSET *st = rd->rrdset; RRDHOST *host = st->rrdhost; - const char *tags_pre = "", *tags_post = "", *tags = host->tags; + const char *tags_pre = "", *tags_post = "", *tags = rrdhost_tags(host); if (!tags) tags = ""; @@ -187,21 +187,20 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM "\"timestamp\":%llu}", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), tags_pre, tags, tags_post, instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "", - st->id, - st->name, - st->family, - st->context, - st->type, - st->units, - - rd->id, - rd->name, + rrdset_id(st), + rrdset_name(st), + rrdset_family(st), + rrdset_context(st), + rrdset_parts_type(st), + rrdset_units(st), + rrddim_id(rd), + rrddim_name(rd), rd->last_collected_value, (unsigned long long)rd->last_collected_time.tv_sec); @@ -231,7 +230,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd if(isnan(value)) return 0; - const char *tags_pre = "", *tags_post = "", *tags = host->tags; + const char *tags_pre = "", *tags_post = "", *tags = rrdhost_tags(host); if (!tags) tags = ""; @@ -272,21 +271,20 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd "\"timestamp\": %llu}", instance->config.prefix, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), tags_pre, tags, tags_post, instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "", - st->id, - st->name, - st->family, - st->context, - st->type, - st->units, - - rd->id, - rd->name, + rrdset_id(st), + rrdset_name(st), + rrdset_family(st), + rrdset_context(st), + rrdset_parts_type(st), + rrdset_units(st), + rrddim_id(rd), + rrddim_name(rd), value, (unsigned long long)last_t); @@ -346,7 +344,7 @@ void json_http_prepare_header(struct instance *instance) "\r\n", instance->config.destination, simple_connector_data->auth_string ? simple_connector_data->auth_string : "", - buffer_strlen(simple_connector_data->last_buffer->buffer)); + (unsigned long int) buffer_strlen(simple_connector_data->last_buffer->buffer)); return; } diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c index 282de2e6b..a974c1264 100644 --- a/exporting/opentsdb/opentsdb.c +++ b/exporting/opentsdb/opentsdb.c @@ -156,7 +156,7 @@ int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) return 0; buffer_strcat(instance->labels_buffer, " "); - rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", " ", + rrdlabels_to_buffer(host->rrdlabels, instance->labels_buffer, "", "=", "", " ", exporting_labels_filter_callback, instance, NULL, sanitize_opentsdb_label_value); return 0; @@ -177,13 +177,13 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); buffer_sprintf( @@ -194,9 +194,9 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM dimension_name, (unsigned long long)rd->last_collected_time.tv_sec, rd->last_collected_value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : ""); return 0; @@ -217,13 +217,13 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); time_t last_t; @@ -240,9 +240,9 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r dimension_name, (unsigned long long)last_t, value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : ""); return 0; @@ -268,7 +268,7 @@ void opentsdb_http_prepare_header(struct instance *instance) "\r\n", instance->config.destination, simple_connector_data->auth_string ? simple_connector_data->auth_string : "", - buffer_strlen(simple_connector_data->last_buffer->buffer)); + (unsigned long int) buffer_strlen(simple_connector_data->last_buffer->buffer)); return; } @@ -288,7 +288,7 @@ int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) { if (unlikely(!sending_labels_configured(instance))) return 0; - rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, ",", ":", "\"", "", + rrdlabels_to_buffer(host->rrdlabels, instance->labels_buffer, ",", ":", "\"", "", exporting_labels_filter_callback, instance, NULL, sanitize_opentsdb_label_value); return 0; @@ -309,13 +309,13 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM * char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); if (buffer_strlen((BUFFER *)instance->buffer) > 2) @@ -336,9 +336,9 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM * dimension_name, (unsigned long long)rd->last_collected_time.tv_sec, rd->last_collected_value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : ""); return 0; @@ -359,13 +359,13 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd) char chart_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( chart_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), RRD_ID_LENGTH_MAX); char dimension_name[RRD_ID_LENGTH_MAX + 1]; exporting_name_copy( dimension_name, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), RRD_ID_LENGTH_MAX); time_t last_t; @@ -392,9 +392,9 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd) dimension_name, (unsigned long long)last_t, value, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (host->tags) ? " " : "", - (host->tags) ? host->tags : "", + (host->tags) ? rrdhost_tags(host) : "", instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : ""); return 0; diff --git a/exporting/process_data.c b/exporting/process_data.c index d5138b787..fbcda0d9b 100644 --- a/exporting/process_data.c +++ b/exporting/process_data.c @@ -77,10 +77,10 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data( time_t before = instance->before; // find the edges of the rrd database for this chart - time_t first_t = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle); - time_t last_t = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle); + time_t first_t = rd->tiers[0]->query_ops->oldest_time(rd->tiers[0]->db_metric_handle); + time_t last_t = rd->tiers[0]->query_ops->latest_time(rd->tiers[0]->db_metric_handle); time_t update_every = st->update_every; - struct rrddim_query_handle handle; + struct storage_engine_query_handle handle; // step back a little, to make sure we have complete data collection // for all metrics @@ -110,9 +110,9 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data( debug( D_EXPORTING, "EXPORTING: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu", - host->hostname, - st->id, - rd->id, + rrdhost_hostname(host), + rrdset_id(st), + rrddim_id(rd), (unsigned long)after, (unsigned long)before, (unsigned long)first_t, @@ -122,11 +122,13 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data( *last_timestamp = before; + size_t points_read = 0; size_t counter = 0; NETDATA_DOUBLE sum = 0; - for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before, TIER_QUERY_FETCH_SUM); !rd->tiers[0]->query_ops.is_finished(&handle);) { - STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle); + for (rd->tiers[0]->query_ops->init(rd->tiers[0]->db_metric_handle, &handle, after, before); !rd->tiers[0]->query_ops->is_finished(&handle);) { + STORAGE_POINT sp = rd->tiers[0]->query_ops->next_metric(&handle); + points_read++; if (unlikely(storage_point_is_empty(sp))) { // not collected @@ -136,15 +138,16 @@ NETDATA_DOUBLE exporting_calculate_value_from_stored_data( sum += sp.sum; counter += sp.count; } - rd->tiers[0]->query_ops.finalize(&handle); + rd->tiers[0]->query_ops->finalize(&handle); + global_statistics_exporters_query_completed(points_read); if (unlikely(!counter)) { debug( D_EXPORTING, "EXPORTING: %s.%s.%s: no values stored in database for range %lu to %lu", - host->hostname, - st->id, - rd->id, + rrdhost_hostname(host), + rrdset_id(st), + rrddim_id(rd), (unsigned long)after, (unsigned long)before); return NAN; @@ -338,26 +341,22 @@ void prepare_buffers(struct engine *engine) rrd_rdlock(); RRDHOST *host; - rrdhost_foreach_read(host) - { - rrdhost_rdlock(host); + rrdhost_foreach_read(host) { start_host_formatting(engine, host); RRDSET *st; - rrdset_foreach_read(st, host) - { - rrdset_rdlock(st); + rrdset_foreach_read(st, host) { start_chart_formatting(engine, st); RRDDIM *rd; rrddim_foreach_read(rd, st) metric_formatting(engine, rd); + rrddim_foreach_done(rd); end_chart_formatting(engine, st); - rrdset_unlock(st); } + rrdset_foreach_done(st); variables_formatting(engine, host); end_host_formatting(engine, host); - rrdhost_unlock(host); } rrd_unlock(); netdata_thread_enable_cancelability(); diff --git a/exporting/prometheus/README.md b/exporting/prometheus/README.md index 5c15ca580..ae94867fa 100644 --- a/exporting/prometheus/README.md +++ b/exporting/prometheus/README.md @@ -4,7 +4,7 @@ description: "Export Netdata metrics to Prometheus for archiving and further ana custom_edit_url: https://github.com/netdata/netdata/edit/master/exporting/prometheus/README.md sidebar_label: "Using Netdata with Prometheus" --> -import { OneLineInstallWget, OneLineInstallCurl } from '../../../src/components/OneLineInstall/' +import { OneLineInstallWget, OneLineInstallCurl } from '@site/src/components/OneLineInstall/' # Using Netdata with Prometheus diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c index 7d632164f..294d8ec2c 100644 --- a/exporting/prometheus/prometheus.c +++ b/exporting/prometheus/prometheus.c @@ -9,9 +9,9 @@ static int is_matches_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN *filter) { if (instance->config.options & EXPORTING_OPTION_SEND_NAMES) { - return simple_pattern_matches(filter, st->name); + return simple_pattern_matches(filter, rrdset_name(st)); } - return simple_pattern_matches(filter, st->id); + return simple_pattern_matches(filter, rrdset_id(st)); } /** @@ -28,10 +28,6 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN RRDHOST *host = st->rrdhost; #endif - // Do not send anomaly rates charts. - if (st->state && st->state->is_ar_chart) - return 0; - if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_EXPORTING_IGNORE))) return 0; @@ -48,8 +44,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN debug( D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", - st->id, - host->hostname); + rrdset_id(st), + rrdhost_hostname(host)); return 0; } } @@ -58,8 +54,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN debug( D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", - st->id, - host->hostname); + rrdset_id(st), + rrdhost_hostname(host)); return 0; } @@ -69,8 +65,8 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN debug( D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.", - st->id, - host->hostname, + rrdset_id(st), + rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode)); return 0; } @@ -327,7 +323,7 @@ void format_host_labels_prometheus(struct instance *instance, RRDHOST *host) .instance = instance, .count = 0 }; - rrdlabels_walkthrough_read(host->host_labels, format_prometheus_label_callback, &tmp); + rrdlabels_walkthrough_read(host->rrdlabels, format_prometheus_label_callback, &tmp); } struct host_variables_callback_options { @@ -349,11 +345,12 @@ struct host_variables_callback_options { * @param data callback options. * @return Returns 1 if the chart can be sent, 0 otherwise. */ -static int print_host_variables(RRDVAR *rv, void *data) -{ +static int print_host_variables_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rv_ptr __maybe_unused, void *data) { + const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item; + struct host_variables_callback_options *opts = data; - if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) { + if (rrdvar_flags(rv) & (RRDVAR_FLAG_CUSTOM_HOST_VAR | RRDVAR_FLAG_CUSTOM_CHART_VAR)) { if (!opts->host_header_printed) { opts->host_header_printed = 1; @@ -366,7 +363,7 @@ static int print_host_variables(RRDVAR *rv, void *data) if (isnan(value) || isinf(value)) { if (opts->output_options & PROMETHEUS_OUTPUT_HELP) buffer_sprintf( - opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value)) ? "NAN" : "INF"); + opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rrdvar_name(rv), (isnan(value)) ? "NAN" : "INF"); return 0; } @@ -378,7 +375,7 @@ static int print_host_variables(RRDVAR *rv, void *data) label_post = "}"; } - prometheus_name_copy(opts->name, rv->name, sizeof(opts->name)); + prometheus_name_copy(opts->name, rrdvar_name(rv), sizeof(opts->name)); if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) buffer_sprintf( @@ -445,17 +442,17 @@ static void generate_as_collected_prom_help(BUFFER *wb, struct gen_parameters *p wb, "%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * ", p->suffix, - (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->st->name) ? p->st->name : p->st->id, - p->st->context, - p->st->family, - (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->rd->name) ? p->rd->name : p->rd->id); + (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->st->name) ? rrdset_name(p->st) : rrdset_id(p->st), + rrdset_context(p->st), + rrdset_family(p->st), + (p->output_options & PROMETHEUS_OUTPUT_NAMES && p->rd->name) ? rrddim_name(p->rd) : rrddim_id(p->rd)); if (prometheus_collector) buffer_sprintf(wb, "1 / 1"); else buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT, p->rd->multiplier, p->rd->divisor); - buffer_sprintf(wb, " %s %s (%s)\n", p->relation, p->st->units, p->type); + buffer_sprintf(wb, " %s %s (%s)\n", p->relation, rrdset_units(p->st), p->type); } /** @@ -518,10 +515,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( PROMETHEUS_OUTPUT_OPTIONS output_options) { SIMPLE_PATTERN *filter = simple_pattern_create(filter_string, NULL, SIMPLE_PATTERN_EXACT); - rrdhost_rdlock(host); char hostname[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(hostname, rrdhost_hostname(host), PROMETHEUS_ELEMENT_MAX); format_host_labels_prometheus(instance, host); @@ -529,8 +525,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"", hostname, - host->program_name, - host->program_version); + rrdhost_program_name(host), + rrdhost_program_version(host)); if (instance->labels_buffer && *buffer_tostring(instance->labels_buffer)) { buffer_sprintf(wb, ",%s", buffer_tostring(instance->labels_buffer)); @@ -551,34 +547,34 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( // send custom variables set for the host if (output_options & PROMETHEUS_OUTPUT_VARIABLES) { - struct host_variables_callback_options opts = { .host = host, - .wb = wb, - .labels = (labels[0] == ',') ? &labels[1] : labels, - .exporting_options = exporting_options, - .output_options = output_options, - .prefix = prefix, - .now = now_realtime_sec(), - .host_header_printed = 0 }; - foreach_host_variable_callback(host, print_host_variables, &opts); + + struct host_variables_callback_options opts = { + .host = host, + .wb = wb, + .labels = (labels[0] == ',') ? &labels[1] : labels, + .exporting_options = exporting_options, + .output_options = output_options, + .prefix = prefix, + .now = now_realtime_sec(), + .host_header_printed = 0 + }; + + rrdvar_walkthrough_read(host->rrdvars, print_host_variables_callback, &opts); } // for each chart RRDSET *st; - rrdset_foreach_read(st, host) - { + rrdset_foreach_read(st, host) { if (likely(can_send_rrdset(instance, st, filter))) { - rrdset_rdlock(st); - char chart[PROMETHEUS_ELEMENT_MAX + 1]; char context[PROMETHEUS_ELEMENT_MAX + 1]; char family[PROMETHEUS_ELEMENT_MAX + 1]; char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; - prometheus_label_copy( - chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX); - prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, rrdset_family(st), PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, rrdset_context(st), PROMETHEUS_ELEMENT_MAX); int as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED); int homogeneous = 1; @@ -590,13 +586,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( if (rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) homogeneous = 0; - if (st->module_name && !strcmp(st->module_name, "prometheus")) + if (!strcmp(rrdset_module_name(st), "prometheus")) prometheus_collector = 1; } else { if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE && !(output_options & PROMETHEUS_OUTPUT_HIDEUNITS)) prometheus_units_copy( - units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS); + units, rrdset_units(st), PROMETHEUS_ELEMENT_MAX, output_options & PROMETHEUS_OUTPUT_OLDUNITS); } if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -604,15 +600,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n", (homogeneous) ? "homogeneous" : "heterogeneous", - (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, - st->context, - st->family, - st->units); + (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), + rrdset_context(st), + rrdset_family(st), + rrdset_units(st)); // for each dimension RRDDIM *rd; - rrddim_foreach_read(rd, st) - { + rrddim_foreach_read(rd, st) { if (rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { char dimension[PROMETHEUS_ELEMENT_MAX + 1]; char *suffix = ""; @@ -651,7 +646,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( prometheus_label_copy( dimension, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -661,13 +656,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( buffer_sprintf(wb, "# TYPE %s_%s%s %s\n", prefix, context, suffix, p.type); generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector); - } else { + } + else { // the dimensions of the chart, do not have the same algorithm, multiplier or divisor // we create a metric per dimension prometheus_name_copy( dimension, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -679,7 +675,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( generate_as_collected_prom_metric(wb, &p, homogeneous, prometheus_collector); } - } else { + } + else { // we need average or sum of the data time_t first_time = instance->after; @@ -694,7 +691,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( prometheus_label_copy( dimension, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP)) @@ -705,8 +702,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( context, units, suffix, - (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, - st->units, + (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), + rrdset_units(st), (unsigned long long)first_time, (unsigned long long)last_time); @@ -746,12 +743,11 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus( } } } - - rrdset_unlock(st); + rrddim_foreach_done(rd); } } + rrdset_foreach_done(st); - rrdhost_unlock(host); simple_pattern_free(filter); } @@ -809,7 +805,7 @@ static inline time_t prometheus_preparation( buffer_sprintf( wb, "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s, time range %lu to %lu\n\n", - host->hostname, + rrdhost_hostname(host), (first_seen) ? "FIRST SEEN " : "", server, mode, diff --git a/exporting/prometheus/prometheus.h b/exporting/prometheus/prometheus.h index 4b8860ded..e80b682ae 100644 --- a/exporting/prometheus/prometheus.h +++ b/exporting/prometheus/prometheus.h @@ -22,10 +22,10 @@ typedef enum prometheus_output_flags { PROMETHEUS_OUTPUT_HIDEUNITS = (1 << 6) } PROMETHEUS_OUTPUT_OPTIONS; -extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host( +void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host( RRDHOST *host, const char *filter_string, BUFFER *wb, const char *server, const char *prefix, EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options); -extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts( +void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts( RRDHOST *host, const char *filter_string, BUFFER *wb, const char *server, const char *prefix, EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options); @@ -36,6 +36,6 @@ char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldun void format_host_labels_prometheus(struct instance *instance, RRDHOST *host); -extern void prometheus_clean_server_root(); +void prometheus_clean_server_root(); #endif //NETDATA_EXPORTING_PROMETHEUS_H diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c index 03feb2c08..2e2fa3c12 100644 --- a/exporting/prometheus/remote_write/remote_write.c +++ b/exporting/prometheus/remote_write/remote_write.c @@ -171,19 +171,19 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host char hostname[PROMETHEUS_ELEMENT_MAX + 1]; prometheus_label_copy( hostname, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), PROMETHEUS_ELEMENT_MAX); add_host_info( connector_specific_data->write_request, - "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS); + "netdata_info", hostname, rrdhost_program_name(host), rrdhost_program_version(host), now_realtime_usec() / USEC_PER_MS); if (unlikely(sending_labels_configured(instance))) { struct format_remote_write_label_callback tmp = { .write_request = connector_specific_data->write_request, .instance = instance }; - rrdlabels_walkthrough_read(host->host_labels, format_remote_write_label_callback, &tmp); + rrdlabels_walkthrough_read(host->rrdlabels, format_remote_write_label_callback, &tmp); } return 0; @@ -200,10 +200,10 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st) { prometheus_label_copy( chart, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? st->name : st->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && st->name) ? rrdset_name(st) : rrdset_id(st), PROMETHEUS_ELEMENT_MAX); - prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); - prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, rrdset_family(st), PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, rrdset_context(st), PROMETHEUS_ELEMENT_MAX); as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED); homogeneous = 1; @@ -215,7 +215,7 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st) homogeneous = 0; } else { if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE) - prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0); + prometheus_units_copy(units, rrdset_units(st), PROMETHEUS_ELEMENT_MAX, 0); } return 0; @@ -249,28 +249,33 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * D_EXPORTING, "EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', " "its last data collection (%lu) is not within our timeframe (%lu to %lu)", - rd->id, rd->rrdset->id, - (host == localhost) ? instance->config.hostname : host->hostname, + rrddim_id(rd), rrdset_id(rd->rrdset), + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)instance->after, (unsigned long)instance->before); return 0; } + if (rd->algorithm == RRD_ALGORITHM_INCREMENTAL || rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) { + if (strcmp(rrdset_module_name(rd->rrdset), "prometheus")) + suffix = "_total"; + } + if (homogeneous) { // all the dimensions of the chart, has the same algorithm, multiplier and divisor // we add all dimensions as labels prometheus_label_copy( dimension, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix); add_metric( connector_specific_data->write_request, name, chart, family, dimension, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), rd->last_collected_value, timeval_msec(&rd->last_collected_time)); } else { // the dimensions of the chart, do not have the same algorithm, multiplier or divisor @@ -278,7 +283,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * prometheus_name_copy( dimension, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); snprintf( name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", instance->config.prefix, context, dimension, @@ -287,7 +292,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * add_metric( connector_specific_data->write_request, name, chart, family, NULL, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), rd->last_collected_value, timeval_msec(&rd->last_collected_time)); } } else { @@ -304,7 +309,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * prometheus_label_copy( dimension, - (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, + (instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rrddim_name(rd) : rrddim_id(rd), PROMETHEUS_ELEMENT_MAX); snprintf( name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", instance->config.prefix, context, units, suffix); @@ -312,7 +317,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * add_metric( connector_specific_data->write_request, name, chart, family, dimension, - (host == localhost) ? instance->config.hostname : host->hostname, + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, last_t * MSEC_PER_SEC); } } @@ -321,10 +326,12 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM * return 0; } -int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) { +static int format_variable_prometheus_remote_write_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rv_ptr __maybe_unused, void *data) { + const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item; + struct prometheus_remote_write_variables_callback_options *opts = data; - if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) { + if (rrdvar_flags(rv) & (RRDVAR_FLAG_CUSTOM_HOST_VAR | RRDVAR_FLAG_CUSTOM_CHART_VAR)) { RRDHOST *host = opts->host; struct instance *instance = opts->instance; struct simple_connector_data *simple_connector_data = @@ -335,12 +342,12 @@ int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) { char name[PROMETHEUS_LABELS_MAX + 1]; char *suffix = ""; - prometheus_name_copy(context, rv->name, PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, rrdvar_name(rv), PROMETHEUS_ELEMENT_MAX); snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix); NETDATA_DOUBLE value = rrdvar2number(rv); add_variable(connector_specific_data->write_request, name, - (host == localhost) ? instance->config.hostname : host->hostname, value, opts->now / USEC_PER_MS); + (host == localhost) ? instance->config.hostname : rrdhost_hostname(host), value, opts->now / USEC_PER_MS); } return 0; @@ -361,7 +368,7 @@ int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST .now = now_realtime_usec(), }; - return foreach_host_variable_callback(host, format_variable_prometheus_remote_write_callback, &opt); + return rrdvar_walkthrough_read(host->rrdvars, format_variable_prometheus_remote_write_callback, &opt); } /** diff --git a/exporting/prometheus/remote_write/remote_write.h b/exporting/prometheus/remote_write/remote_write.h index 4740772d0..d4e86494b 100644 --- a/exporting/prometheus/remote_write/remote_write.h +++ b/exporting/prometheus/remote_write/remote_write.h @@ -18,7 +18,7 @@ struct prometheus_remote_write_variables_callback_options { }; int init_prometheus_remote_write_instance(struct instance *instance); -extern void clean_prometheus_remote_write(struct instance *instance); +void clean_prometheus_remote_write(struct instance *instance); int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host); int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st); diff --git a/exporting/send_data.c b/exporting/send_data.c index ed649b640..1d20f3b74 100644 --- a/exporting/send_data.c +++ b/exporting/send_data.c @@ -313,7 +313,7 @@ void simple_connector_worker(void *instance_p) if (unlikely(sock == -1)) { size_t reconnects = 0; - sock = connect_to_one_of( + sock = connect_to_one_of_urls( instance->config.destination, connector_specific_config->default_port, &timeout, @@ -322,12 +322,12 @@ void simple_connector_worker(void *instance_p) CONNECTED_TO_MAX); #ifdef ENABLE_HTTPS if (exporting_tls_is_enabled(instance->config.type, options) && sock != -1) { - if (netdata_exporting_ctx) { + if (netdata_ssl_exporting_ctx) { if (sock_delnonblock(sock) < 0) error("Exporting cannot remove the non-blocking flag from socket %d", sock); if (connector_specific_data->conn == NULL) { - connector_specific_data->conn = SSL_new(netdata_exporting_ctx); + connector_specific_data->conn = SSL_new(netdata_ssl_exporting_ctx); if (connector_specific_data->conn == NULL) { error("Failed to allocate SSL structure to socket %d.", sock); connector_specific_data->flags = NETDATA_SSL_NO_HANDSHAKE; diff --git a/exporting/send_internal_metrics.c b/exporting/send_internal_metrics.c index defb8d047..515cda3b2 100644 --- a/exporting/send_internal_metrics.c +++ b/exporting/send_internal_metrics.c @@ -11,6 +11,9 @@ */ void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_system) { + if (!global_statistics_enabled) + return; + if (*st_rusage && *rd_user && *rd_system) return; @@ -31,12 +34,12 @@ void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_ */ void send_main_rusage(RRDSET *st_rusage, RRDDIM *rd_user, RRDDIM *rd_system) { + if (!global_statistics_enabled) + return; + struct rusage thread; getrusage(RUSAGE_THREAD, &thread); - if (likely(st_rusage->counter_done)) - rrdset_next(st_rusage); - rrddim_set_by_pointer(st_rusage, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); rrddim_set_by_pointer(st_rusage, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); @@ -52,6 +55,9 @@ void send_main_rusage(RRDSET *st_rusage, RRDDIM *rd_user, RRDDIM *rd_system) */ void send_internal_metrics(struct instance *instance) { + if (!global_statistics_enabled) + return; + struct stats *stats = &instance->stats; // ------------------------------------------------------------------------ @@ -123,50 +129,28 @@ void send_internal_metrics(struct instance *instance) // ------------------------------------------------------------------------ // update the monitoring charts - if (likely(stats->st_metrics->counter_done)) - rrdset_next(stats->st_metrics); - rrddim_set_by_pointer(stats->st_metrics, stats->rd_buffered_metrics, stats->buffered_metrics); rrddim_set_by_pointer(stats->st_metrics, stats->rd_lost_metrics, stats->lost_metrics); rrddim_set_by_pointer(stats->st_metrics, stats->rd_sent_metrics, stats->sent_metrics); - rrdset_done(stats->st_metrics); - // ------------------------------------------------------------------------ - - if (likely(stats->st_bytes->counter_done)) - rrdset_next(stats->st_bytes); - rrddim_set_by_pointer(stats->st_bytes, stats->rd_buffered_bytes, stats->buffered_bytes); rrddim_set_by_pointer(stats->st_bytes, stats->rd_lost_bytes, stats->lost_bytes); rrddim_set_by_pointer(stats->st_bytes, stats->rd_sent_bytes, stats->sent_bytes); rrddim_set_by_pointer(stats->st_bytes, stats->rd_received_bytes, stats->received_bytes); - rrdset_done(stats->st_bytes); - // ------------------------------------------------------------------------ - - if (likely(stats->st_ops->counter_done)) - rrdset_next(stats->st_ops); - rrddim_set_by_pointer(stats->st_ops, stats->rd_transmission_successes, stats->transmission_successes); rrddim_set_by_pointer(stats->st_ops, stats->rd_data_lost_events, stats->data_lost_events); rrddim_set_by_pointer(stats->st_ops, stats->rd_reconnects, stats->reconnects); rrddim_set_by_pointer(stats->st_ops, stats->rd_transmission_failures, stats->transmission_failures); rrddim_set_by_pointer(stats->st_ops, stats->rd_receptions, stats->receptions); - rrdset_done(stats->st_ops); - // ------------------------------------------------------------------------ - struct rusage thread; getrusage(RUSAGE_THREAD, &thread); - if (likely(stats->st_rusage->counter_done)) - rrdset_next(stats->st_rusage); - rrddim_set_by_pointer(stats->st_rusage, stats->rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); rrddim_set_by_pointer(stats->st_rusage, stats->rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); - rrdset_done(stats->st_rusage); } diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c index aae1c53fb..c9fc9458c 100644 --- a/exporting/tests/exporting_fixtures.c +++ b/exporting/tests/exporting_fixtures.c @@ -33,31 +33,13 @@ int teardown_configured_engine(void **state) return 0; } -int setup_rrdhost() -{ - localhost = calloc(1, sizeof(RRDHOST)); - - localhost->rrd_update_every = 1; - - localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2"); - - localhost->host_labels = rrdlabels_create(); - rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG); - rrdlabels_add(localhost->host_labels, "key2", "value2", RRDLABEL_SRC_CONFIG); +static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrddim, void *st) { + RRDDIM *rd = rrddim; - localhost->rrdset_root = calloc(1, sizeof(RRDSET)); - RRDSET *st = localhost->rrdset_root; - st->rrdhost = localhost; - strcpy(st->id, "chart_id"); - st->name = strdupz("chart_name"); - st->rrd_memory_mode |= RRD_MEMORY_MODE_SAVE; - st->update_every = 1; + rd->id = string_strdupz("dimension_id"); + rd->name = string_strdupz("dimension_name"); - localhost->rrdset_root->dimensions = calloc(1, sizeof(RRDDIM)); - RRDDIM *rd = localhost->rrdset_root->dimensions; - rd->rrdset = st; - rd->id = strdupz("dimension_id"); - rd->name = strdupz("dimension_name"); + rd->rrdset = (RRDSET *)st; rd->last_collected_value = 123000321; rd->last_collected_time.tv_sec = 15051; rd->collections_counter++; @@ -70,25 +52,74 @@ int setup_rrdhost() rd->tiers[0]->query_ops.is_finished = __mock_rrddim_query_is_finished; rd->tiers[0]->query_ops.next_metric = __mock_rrddim_query_next_metric; rd->tiers[0]->query_ops.finalize = __mock_rrddim_query_finalize; +} + +static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdset, void *constructor_data __maybe_unused) { + RRDHOST *host = localhost; + RRDSET *st = rrdset; + + // const char *chart_full_id = dictionary_acquired_item_name(item); + + st->id = string_strdupz("chart_id"); + st->name = string_strdupz("chart_name"); + + st->update_every = 1; + st->rrd_memory_mode = RRD_MEMORY_MODE_SAVE; + + st->rrdhost = host; + + st->rrddim_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL); +} + +int setup_rrdhost() +{ + localhost = calloc(1, sizeof(RRDHOST)); + + localhost->rrd_update_every = 1; + + localhost->tags = string_strdupz("TAG1=VALUE1 TAG2=VALUE2"); + + localhost->rrdlabels = rrdlabels_create(); + rrdlabels_add(localhost->rrdlabels, "key1", "value1", RRDLABEL_SRC_CONFIG); + rrdlabels_add(localhost->rrdlabels, "key2", "value2", RRDLABEL_SRC_CONFIG); + + localhost->rrdset_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_insert_callback(localhost->rrdset_root_index, rrdset_insert_callback, NULL); + RRDSET *st = dictionary_set_advanced(localhost->rrdset_root_index, "chart_id", -1, NULL, sizeof(RRDSET), NULL); + + st->rrddim_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL); + st->dimensions = dictionary_set_advanced(st->rrddim_root_index, "dimension_id", -1, NULL, sizeof(RRDDIM), st); return 0; } int teardown_rrdhost() { - RRDDIM *rd = localhost->rrdset_root->dimensions; - free((void *)rd->name); - free((void *)rd->id); + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); + + string_freez(rd->id); + string_freez(rd->name); free(rd->tiers[0]); - free(rd); - RRDSET *st = localhost->rrdset_root; - free((void *)st->name); - free(st); + string_freez(st->id); + string_freez(st->name); + dictionary_destroy(st->rrddim_root_index); - rrdlabels_destroy(localhost->host_labels); + rrdlabels_destroy(localhost->rrdlabels); - free((void *)localhost->tags); + string_freez(localhost->tags); + dictionary_destroy(localhost->rrdset_root_index); free(localhost); return 0; diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c index ee36e887a..7e5017a5f 100644 --- a/exporting/tests/netdata_doubles.c +++ b/exporting/tests/netdata_doubles.c @@ -177,20 +177,6 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) return RRD_MEMORY_MODE_NONE_NAME; } -NETDATA_DOUBLE rrdvar2number(RRDVAR *rv) -{ - (void)rv; - return 0; -} - -int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data) -{ - (void)host; - (void)callback; - (void)data; - return 0; -} - void rrdset_update_heterogeneous_flag(RRDSET *st) { (void)st; @@ -212,11 +198,10 @@ time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle) return mock_type(time_t); } -void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type) +void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time) { (void)db_metric_handle; (void)handle; - (void)tier_query_fetch_type; function_called(); check_expected(start_time); @@ -248,10 +233,23 @@ void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle) function_called(); } -void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) +void rrdcalc_update_rrdlabels(RRDSET *st) +{ + (void)st; +} + +void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva) +{ + (void)host; + (void)rva; +} + +void db_execute(const char *cmd) { - (void)chart_uuid; - (void)source_type; - (void)label; - (void)value; + (void)cmd; +} + +DICTIONARY *rrdfamily_rrdvars_dict(const RRDFAMILY_ACQUIRED *rfa) { + (void)rfa; + return NULL; } diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c index 56a28059f..6ea6b1e5c 100644 --- a/exporting/tests/test_exporting_engine.c +++ b/exporting/tests/test_exporting_engine.c @@ -11,6 +11,7 @@ struct config netdata_config; char *netdata_configured_user_config_dir = "."; char *netdata_configured_stock_config_dir = "."; char *netdata_configured_hostname = "test_global_host"; +bool global_statistics_enabled = true; char log_line[MAX_LOG_LINE + 1]; @@ -257,7 +258,10 @@ static void test_rrdset_is_exportable(void **state) { struct engine *engine = *state; struct instance *instance = engine->instance_root; - RRDSET *st = localhost->rrdset_root; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); assert_ptr_equal(st->exporting_flags, NULL); @@ -271,7 +275,10 @@ static void test_false_rrdset_is_exportable(void **state) { struct engine *engine = *state; struct instance *instance = engine->instance_root; - RRDSET *st = localhost->rrdset_root; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); simple_pattern_free(instance->config.charts_pattern); instance->config.charts_pattern = simple_pattern_create("!*", NULL, SIMPLE_PATTERN_EXACT); @@ -288,7 +295,17 @@ static void test_exporting_calculate_value_from_stored_data(void **state) { struct engine *engine = *state; struct instance *instance = engine->instance_root; - RRDDIM *rd = localhost->rrdset_root->dimensions; + + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); + time_t timestamp; instance->after = 3; @@ -348,7 +365,11 @@ static void test_prepare_buffers(void **state) expect_value(__mock_start_host_formatting, host, localhost); will_return(__mock_start_host_formatting, 0); - RRDSET *st = localhost->rrdset_root; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + expect_function_call(__wrap_rrdset_is_exportable); expect_value(__wrap_rrdset_is_exportable, instance, instance); expect_value(__wrap_rrdset_is_exportable, st, st); @@ -359,7 +380,10 @@ static void test_prepare_buffers(void **state) expect_value(__mock_start_chart_formatting, st, st); will_return(__mock_start_chart_formatting, 0); - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); expect_function_call(__mock_metric_formatting); expect_value(__mock_metric_formatting, instance, instance); expect_value(__mock_metric_formatting, rd, rd); @@ -412,7 +436,15 @@ static void test_format_dimension_collected_graphite_plaintext(void **state) { struct engine *engine = *state; - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_collected_graphite_plaintext(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), @@ -426,7 +458,15 @@ static void test_format_dimension_stored_graphite_plaintext(void **state) expect_function_call(__wrap_exporting_calculate_value_from_stored_data); will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_stored_graphite_plaintext(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), @@ -437,13 +477,21 @@ static void test_format_dimension_collected_json_plaintext(void **state) { struct engine *engine = *state; - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_collected_json_plaintext(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), "{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\"," - "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\"," - "\"chart_context\":\"(null)\",\"chart_type\":\"(null)\",\"units\":\"(null)\",\"id\":\"dimension_id\"," + "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"\"," + "\"chart_context\":\"\",\"chart_type\":\"\",\"units\":\"\",\"id\":\"dimension_id\"," "\"name\":\"dimension_name\",\"value\":123000321,\"timestamp\":15051}\n"); } @@ -454,13 +502,21 @@ static void test_format_dimension_stored_json_plaintext(void **state) expect_function_call(__wrap_exporting_calculate_value_from_stored_data); will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_stored_json_plaintext(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), "{\"prefix\":\"netdata\",\"hostname\":\"test-host\",\"host_tags\":\"TAG1=VALUE1 TAG2=VALUE2\"," - "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"(null)\"," \ - "\"chart_context\": \"(null)\",\"chart_type\":\"(null)\",\"units\": \"(null)\",\"id\":\"dimension_id\"," + "\"chart_id\":\"chart_id\",\"chart_name\":\"chart_name\",\"chart_family\":\"\"," \ + "\"chart_context\": \"\",\"chart_type\":\"\",\"units\": \"\",\"id\":\"dimension_id\"," "\"name\":\"dimension_name\",\"value\":690565856.0000000,\"timestamp\": 15052}\n"); } @@ -468,7 +524,15 @@ static void test_format_dimension_collected_opentsdb_telnet(void **state) { struct engine *engine = *state; - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_collected_opentsdb_telnet(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), @@ -482,7 +546,15 @@ static void test_format_dimension_stored_opentsdb_telnet(void **state) expect_function_call(__wrap_exporting_calculate_value_from_stored_data); will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_stored_opentsdb_telnet(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), @@ -493,7 +565,15 @@ static void test_format_dimension_collected_opentsdb_http(void **state) { struct engine *engine = *state; - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_collected_opentsdb_http(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), @@ -510,7 +590,15 @@ static void test_format_dimension_stored_opentsdb_http(void **state) expect_function_call(__wrap_exporting_calculate_value_from_stored_data); will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); assert_int_equal(format_dimension_stored_opentsdb_http(engine->instance_root, rd), 0); assert_string_equal( buffer_tostring(engine->instance_root->buffer), @@ -616,27 +704,14 @@ static void test_simple_connector_worker(void **state) buffer_sprintf(simple_connector_data->last_buffer->header, "test header"); buffer_sprintf(simple_connector_data->last_buffer->buffer, "test buffer"); - expect_function_call(__wrap_connect_to_one_of); - expect_string(__wrap_connect_to_one_of, destination, "localhost"); - expect_value(__wrap_connect_to_one_of, default_port, 2003); - expect_not_value(__wrap_connect_to_one_of, reconnects_counter, 0); - expect_string(__wrap_connect_to_one_of, connected_to, "localhost"); - expect_value(__wrap_connect_to_one_of, connected_to_size, CONNECTED_TO_MAX); - will_return(__wrap_connect_to_one_of, 2); + expect_function_call(__wrap_now_realtime_sec); + will_return(__wrap_now_realtime_sec, 2); - expect_function_call(__wrap_send); - expect_value(__wrap_send, sockfd, 2); - expect_not_value(__wrap_send, buf, buffer_tostring(simple_connector_data->last_buffer->buffer)); - expect_string(__wrap_send, buf, "test header"); - expect_value(__wrap_send, len, 11); - expect_value(__wrap_send, flags, MSG_NOSIGNAL); + expect_function_call(__wrap_now_realtime_sec); + will_return(__wrap_now_realtime_sec, 2); - expect_function_call(__wrap_send); - expect_value(__wrap_send, sockfd, 2); - expect_value(__wrap_send, buf, buffer_tostring(simple_connector_data->last_buffer->buffer)); - expect_string(__wrap_send, buf, "test buffer"); - expect_value(__wrap_send, len, 11); - expect_value(__wrap_send, flags, MSG_NOSIGNAL); + expect_function_call(__wrap_now_realtime_sec); + will_return(__wrap_now_realtime_sec, 2); expect_function_call(__wrap_send_internal_metrics); expect_value(__wrap_send_internal_metrics, instance, instance); @@ -986,21 +1061,26 @@ static void test_can_send_rrdset(void **state) { (void)*state; - assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root, NULL), 1); + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 1); - rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE); - assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root, NULL), 0); - rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_EXPORTING_IGNORE); + rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_IGNORE); + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 0); + rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE); // TODO: test with a denying simple pattern - rrdset_flag_set(localhost->rrdset_root, RRDSET_FLAG_OBSOLETE); - assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root, NULL), 0); - rrdset_flag_clear(localhost->rrdset_root, RRDSET_FLAG_OBSOLETE); + rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE); + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 0); + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); - localhost->rrdset_root->rrd_memory_mode = RRD_MEMORY_MODE_NONE; + st->rrd_memory_mode = RRD_MEMORY_MODE_NONE; prometheus_exporter_instance->config.options |= EXPORTING_SOURCE_DATA_AVERAGE; - assert_int_equal(can_send_rrdset(prometheus_exporter_instance, localhost->rrdset_root, NULL), 0); + assert_int_equal(can_send_rrdset(prometheus_exporter_instance, st, NULL), 0); } static void test_prometheus_name_copy(void **state) @@ -1055,9 +1135,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) BUFFER *buffer = buffer_create(0); - localhost->hostname = strdupz("test_hostname"); - localhost->rrdset_root->family = strdupz("test_family"); - localhost->rrdset_root->context = strdupz("test_context"); + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + localhost->hostname = string_strdupz("test_hostname"); + st->family = string_strdupz("test_family"); + st->context = string_strdupz("test_context"); expect_function_call(__wrap_now_realtime_sec); will_return(__wrap_now_realtime_sec, 2); @@ -1069,7 +1154,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n" "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\"} 690565856.0000000\n"); buffer_flush(buffer); @@ -1085,7 +1170,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n" "# TYPE test_prefix_test_context gauge\n" "test_prefix_test_context{chart=\"chart_name\",family=\"test_family\",dimension=\"dimension_name\"} 690565856.0000000\n"); @@ -1101,11 +1186,11 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) assert_string_equal( buffer_tostring(buffer), - "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n" + "netdata_info{instance=\"test_hostname\",application=\"\",version=\"\",key1=\"value1\",key2=\"value2\"} 1\n" "test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\",instance=\"test_hostname\"} 690565856.0000000\n"); - free(localhost->rrdset_root->context); - free(localhost->rrdset_root->family); + free(st->context); + free(st->family); free(localhost->hostname); buffer_free(buffer); } @@ -1207,8 +1292,8 @@ static void test_format_host_prometheus_remote_write(void **state) simple_connector_data->connector_specific_data = (void *)connector_specific_data; connector_specific_data->write_request = (void *)0xff; - localhost->program_name = strdupz("test_program"); - localhost->program_version = strdupz("test_version"); + localhost->program_name = string_strdupz("test_program"); + localhost->program_version = string_strdupz("test_version"); expect_function_call(__wrap_add_host_info); expect_value(__wrap_add_host_info, write_request_p, 0xff); @@ -1249,7 +1334,15 @@ static void test_format_dimension_prometheus_remote_write(void **state) simple_connector_data->connector_specific_data = (void *)connector_specific_data; connector_specific_data->write_request = (void *)0xff; - RRDDIM *rd = localhost->rrdset_root->dimensions; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st); + break; + rrddim_foreach_done(rd); expect_function_call(__wrap_exporting_calculate_value_from_stored_data); will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); @@ -1428,7 +1521,11 @@ static void test_aws_kinesis_connector_worker(void **state) expect_value(__wrap_rrdhost_is_exportable, host, localhost); will_return(__wrap_rrdhost_is_exportable, 1); - RRDSET *st = localhost->rrdset_root; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + expect_function_call(__wrap_rrdset_is_exportable); expect_value(__wrap_rrdset_is_exportable, instance, instance); expect_value(__wrap_rrdset_is_exportable, st, st); @@ -1563,7 +1660,11 @@ static void test_pubsub_connector_worker(void **state) expect_value(__wrap_rrdhost_is_exportable, host, localhost); will_return(__wrap_rrdhost_is_exportable, 1); - RRDSET *st = localhost->rrdset_root; + RRDSET *st; + rrdset_foreach_read(st, localhost); + break; + rrdset_foreach_done(st); + expect_function_call(__wrap_rrdset_is_exportable); expect_value(__wrap_rrdset_is_exportable, instance, instance); expect_value(__wrap_rrdset_is_exportable, st, st); diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h index ae0b7df9a..a9180a518 100644 --- a/exporting/tests/test_exporting_engine.h +++ b/exporting/tests/test_exporting_engine.h @@ -4,6 +4,7 @@ #define TEST_EXPORTING_ENGINE_H 1 #include "libnetdata/libnetdata.h" +#include "database/rrdvar.h" #include "exporting/exporting_engine.h" #include "exporting/graphite/graphite.h" @@ -59,7 +60,7 @@ void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, c void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line); time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle); time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle); -void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type); +void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time); int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle); STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle); void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle); |