summaryrefslogtreecommitdiffstats
path: root/exporting
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2022-08-12 07:26:17 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2022-08-12 07:26:17 +0000
commit7877a98bd9c00db5e81dd2f8c734cba2bab20be7 (patch)
treed18b767250f7c7ced9b8abe2ece784ac1fe24d3e /exporting
parentReleasing debian version 1.35.1-2. (diff)
downloadnetdata-7877a98bd9c00db5e81dd2f8c734cba2bab20be7.tar.xz
netdata-7877a98bd9c00db5e81dd2f8c734cba2bab20be7.zip
Merging upstream version 1.36.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'exporting')
-rw-r--r--exporting/aws_kinesis/aws_kinesis.c1
-rw-r--r--exporting/check_filters.c8
-rw-r--r--exporting/clean_connectors.c2
-rw-r--r--exporting/exporting.conf1
-rw-r--r--exporting/exporting_engine.h19
-rw-r--r--exporting/graphite/graphite.c34
-rw-r--r--exporting/graphite/graphite.h2
-rw-r--r--exporting/init_connectors.c8
-rw-r--r--exporting/json/json.c40
-rw-r--r--exporting/mongodb/mongodb.c1
-rw-r--r--exporting/opentsdb/opentsdb.c79
-rw-r--r--exporting/opentsdb/opentsdb.h2
-rw-r--r--exporting/process_data.c54
-rw-r--r--exporting/prometheus/prometheus.c153
-rw-r--r--exporting/prometheus/remote_write/remote_write.c88
-rw-r--r--exporting/prometheus/remote_write/remote_write.h7
-rw-r--r--exporting/prometheus/remote_write/remote_write_request.cc32
-rw-r--r--exporting/prometheus/remote_write/remote_write_request.h3
-rw-r--r--exporting/pubsub/pubsub.c1
-rw-r--r--exporting/read_config.c5
-rw-r--r--exporting/tests/exporting_doubles.c14
-rw-r--r--exporting/tests/exporting_fixtures.c39
-rw-r--r--exporting/tests/netdata_doubles.c30
-rw-r--r--exporting/tests/test_exporting_engine.c38
-rw-r--r--exporting/tests/test_exporting_engine.h20
25 files changed, 370 insertions, 311 deletions
diff --git a/exporting/aws_kinesis/aws_kinesis.c b/exporting/aws_kinesis/aws_kinesis.c
index fe4181e3a..1d89cc79a 100644
--- a/exporting/aws_kinesis/aws_kinesis.c
+++ b/exporting/aws_kinesis/aws_kinesis.c
@@ -45,6 +45,7 @@ int init_aws_kinesis_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_json_plaintext;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = NULL;
diff --git a/exporting/check_filters.c b/exporting/check_filters.c
index f7eba22db..726fd02a1 100644
--- a/exporting/check_filters.c
+++ b/exporting/check_filters.c
@@ -2,6 +2,14 @@
#include "exporting_engine.h"
+
+bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+ (void)name;
+ (void)value;
+ struct instance *instance = (struct instance *)data;
+ return should_send_label(instance, ls);
+}
+
/**
* Check if the connector instance should export the host metrics
*
diff --git a/exporting/clean_connectors.c b/exporting/clean_connectors.c
index 4af1219a6..e93563741 100644
--- a/exporting/clean_connectors.c
+++ b/exporting/clean_connectors.c
@@ -33,7 +33,7 @@ static void clean_instance_config(struct instance_config *config)
void clean_instance(struct instance *instance)
{
clean_instance_config(&instance->config);
- buffer_free(instance->labels);
+ buffer_free(instance->labels_buffer);
uv_cond_destroy(&instance->cond_var);
// uv_mutex_destroy(&instance->mutex);
diff --git a/exporting/exporting.conf b/exporting/exporting.conf
index 314e1541e..f2ec56a52 100644
--- a/exporting/exporting.conf
+++ b/exporting/exporting.conf
@@ -45,6 +45,7 @@
# send names instead of ids = yes
# send charts matching = *
# send hosts matching = localhost *
+ # send variables = no
# [kinesis:my_kinesis_instance]
# enabled = no
diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h
index 20f260c15..2141caa41 100644
--- a/exporting/exporting_engine.h
+++ b/exporting/exporting_engine.h
@@ -4,7 +4,6 @@
#define NETDATA_EXPORTING_ENGINE_H 1
#include "daemon/common.h"
-
#include <uv.h>
#define exporter_get(section, name, value) expconfig_get(&exporting_config, section, name, value)
@@ -27,7 +26,8 @@ typedef enum exporting_options {
EXPORTING_OPTION_SEND_AUTOMATIC_LABELS = (1 << 4),
EXPORTING_OPTION_USE_TLS = (1 << 5),
- EXPORTING_OPTION_SEND_NAMES = (1 << 16)
+ EXPORTING_OPTION_SEND_NAMES = (1 << 16),
+ EXPORTING_OPTION_SEND_VARIABLES = (1 << 17)
} EXPORTING_OPTIONS;
#define EXPORTING_OPTIONS_SOURCE_BITS \
@@ -40,11 +40,13 @@ extern const char *global_exporting_prefix;
#define sending_labels_configured(instance) \
(instance->config.options & (EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS))
-#define should_send_label(instance, label) \
+#define should_send_label(instance, label_source) \
((instance->config.options & EXPORTING_OPTION_SEND_CONFIGURED_LABELS && \
- label->label_source == LABEL_SOURCE_NETDATA_CONF) || \
+ label_source & RRDLABEL_SRC_CONFIG) || \
(instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS && \
- label->label_source != LABEL_SOURCE_NETDATA_CONF))
+ label_source & RRDLABEL_SRC_AUTO))
+
+#define should_send_variables(instance) (instance->config.options & EXPORTING_OPTION_SEND_VARIABLES)
typedef enum exporting_connector_types {
EXPORTING_CONNECTOR_TYPE_UNKNOWN, // Invalid type
@@ -205,7 +207,7 @@ struct instance {
int skip_host;
int skip_chart;
- BUFFER *labels;
+ BUFFER *labels_buffer;
time_t after;
time_t before;
@@ -220,6 +222,7 @@ struct instance {
int (*start_chart_formatting)(struct instance *instance, RRDSET *st);
int (*metric_formatting)(struct instance *instance, RRDDIM *rd);
int (*end_chart_formatting)(struct instance *instance, RRDSET *st);
+ int (*variables_formatting)(struct instance *instance, RRDHOST *host);
int (*end_host_formatting)(struct instance *instance, RRDHOST *host);
int (*end_batch_formatting)(struct instance *instance);
@@ -270,7 +273,8 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st);
extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options);
-calculated_number exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE
+exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);
@@ -280,6 +284,7 @@ void start_host_formatting(struct engine *engine, RRDHOST *host);
void start_chart_formatting(struct engine *engine, RRDSET *st);
void metric_formatting(struct engine *engine, RRDDIM *rd);
void end_chart_formatting(struct engine *engine, RRDSET *st);
+void variables_formatting(struct engine *engine, RRDHOST *host);
void end_host_formatting(struct engine *engine, RRDHOST *host);
void end_batch_formatting(struct engine *engine);
int flush_host_labels(struct instance *instance, RRDHOST *host);
diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c
index 84d4febf1..8ca094b3b 100644
--- a/exporting/graphite/graphite.c
+++ b/exporting/graphite/graphite.c
@@ -37,6 +37,7 @@ int init_graphite_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_graphite_plaintext;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = simple_connector_end_batch;
@@ -71,7 +72,7 @@ int init_graphite_instance(struct instance *instance)
* @param len the maximum number of characters copied.
*/
-void sanitize_graphite_label_value(char *dst, char *src, size_t len)
+void sanitize_graphite_label_value(char *dst, const char *src, size_t len)
{
while (*src != '\0' && len) {
if (isspace(*src) || *src == ';' || *src == '~')
@@ -91,29 +92,18 @@ void sanitize_graphite_label_value(char *dst, char *src, size_t len)
* @param host a data collecting host.
* @return Always returns 0.
*/
+
int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host)
{
- if (!instance->labels)
- instance->labels = buffer_create(1024);
+ if (!instance->labels_buffer)
+ instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
- rrdhost_check_rdlock(host);
- netdata_rwlock_rdlock(&host->labels.labels_rwlock);
- for (struct label *label = host->labels.head; label; label = label->next) {
- if (!should_send_label(instance, label))
- continue;
-
- char value[CONFIG_MAX_VALUE + 1];
- sanitize_graphite_label_value(value, label->value, CONFIG_MAX_VALUE);
-
- if (*value) {
- buffer_strcat(instance->labels, ";");
- buffer_sprintf(instance->labels, "%s=%s", label->key, value);
- }
- }
- netdata_rwlock_unlock(&host->labels.labels_rwlock);
+ rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, ";", "=", "", "",
+ exporting_labels_filter_callback, instance,
+ NULL, sanitize_graphite_label_value);
return 0;
}
@@ -151,7 +141,7 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
dimension_name,
(host->tags) ? ";" : "",
(host->tags) ? host->tags : "",
- (instance->labels) ? buffer_tostring(instance->labels) : "",
+ (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
rd->last_collected_value,
(unsigned long long)rd->last_collected_time.tv_sec);
@@ -183,21 +173,21 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
RRD_ID_LENGTH_MAX);
time_t last_t;
- calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+ NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
buffer_sprintf(
instance->buffer,
- "%s.%s.%s.%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
+ "%s.%s.%s.%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
instance->config.prefix,
(host == localhost) ? instance->config.hostname : host->hostname,
chart_name,
dimension_name,
(host->tags) ? ";" : "",
(host->tags) ? host->tags : "",
- (instance->labels) ? buffer_tostring(instance->labels) : "",
+ (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
value,
(unsigned long long)last_t);
diff --git a/exporting/graphite/graphite.h b/exporting/graphite/graphite.h
index 993c12e57..79f87e46e 100644
--- a/exporting/graphite/graphite.h
+++ b/exporting/graphite/graphite.h
@@ -7,7 +7,7 @@
int init_graphite_instance(struct instance *instance);
-void sanitize_graphite_label_value(char *dst, char *src, size_t len);
+void sanitize_graphite_label_value(char *dst, const char *src, size_t len);
int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host);
int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd);
diff --git a/exporting/init_connectors.c b/exporting/init_connectors.c
index 69ea0685c..bfb6525ea 100644
--- a/exporting/init_connectors.c
+++ b/exporting/init_connectors.c
@@ -117,7 +117,7 @@ static size_t base64_encode(unsigned char *input, size_t input_size, char *outpu
return 0;
}
size_t count = 0;
- while (input_size > 3) {
+ while (input_size >= 3) {
value = ((input[0] << 16) + (input[1] << 8) + input[2]) & 0xffffff;
output[0] = lookup[value >> 18];
output[1] = lookup[(value >> 12) & 0x3f];
@@ -138,6 +138,7 @@ static size_t base64_encode(unsigned char *input, size_t input_size, char *outpu
output[3] = '=';
//error("Base-64 encode (%06x) -> %c %c %c %c\n", (value>>2)&0xffff, output[0], output[1], output[2], output[3]);
count += 4;
+ output[4] = '\0';
break;
case 1:
value = input[0] << 4;
@@ -147,10 +148,13 @@ static size_t base64_encode(unsigned char *input, size_t input_size, char *outpu
output[3] = '=';
//error("Base-64 encode (%06x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]);
count += 4;
+ output[4] = '\0';
break;
case 0:
+ output[0] = '\0';
break;
}
+
return count;
}
@@ -199,7 +203,7 @@ void simple_connector_init(struct instance *instance)
char *encoded_credentials = callocz(1, encoded_size);
base64_encode((unsigned char*)buffer_tostring(auth_string), buffer_strlen(auth_string), encoded_credentials, encoded_size);
-
+
buffer_flush(auth_string);
buffer_sprintf(auth_string, "Authorization: Basic %s\n", encoded_credentials);
diff --git a/exporting/json/json.c b/exporting/json/json.c
index 50278c5b8..45a8c9d9f 100644
--- a/exporting/json/json.c
+++ b/exporting/json/json.c
@@ -29,6 +29,7 @@ int init_json_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_json_plaintext;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = simple_connector_end_batch;
@@ -87,6 +88,7 @@ int init_json_http_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_json_plaintext;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = close_batch_json_http;
@@ -113,34 +115,20 @@ int init_json_http_instance(struct instance *instance)
* @param host a data collecting host.
* @return Always returns 0.
*/
+
int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host)
{
- if (!instance->labels)
- instance->labels = buffer_create(1024);
+ if (!instance->labels_buffer)
+ instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
- buffer_strcat(instance->labels, "\"labels\":{");
-
- int count = 0;
- rrdhost_check_rdlock(host);
- netdata_rwlock_rdlock(&host->labels.labels_rwlock);
- for (struct label *label = host->labels.head; label; label = label->next) {
- if (!should_send_label(instance, label))
- continue;
-
- char value[CONFIG_MAX_VALUE * 2 + 1];
- sanitize_json_string(value, label->value, CONFIG_MAX_VALUE);
- if (count > 0)
- buffer_strcat(instance->labels, ",");
- buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
-
- count++;
- }
- netdata_rwlock_unlock(&host->labels.labels_rwlock);
-
- buffer_strcat(instance->labels, "},");
+ buffer_strcat(instance->labels_buffer, "\"labels\":{");
+ rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",",
+ exporting_labels_filter_callback, instance,
+ NULL, sanitize_json_string);
+ buffer_strcat(instance->labels_buffer, "},");
return 0;
}
@@ -203,7 +191,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
tags_pre,
tags,
tags_post,
- instance->labels ? buffer_tostring(instance->labels) : "",
+ instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
st->id,
st->name,
@@ -238,7 +226,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
RRDHOST *host = st->rrdhost;
time_t last_t;
- calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+ NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
@@ -279,7 +267,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
"\"id\":\"%s\","
"\"name\":\"%s\","
- "\"value\":" CALCULATED_NUMBER_FORMAT ","
+ "\"value\":" NETDATA_DOUBLE_FORMAT ","
"\"timestamp\": %llu}",
@@ -288,7 +276,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
tags_pre,
tags,
tags_post,
- instance->labels ? buffer_tostring(instance->labels) : "",
+ instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
st->id,
st->name,
diff --git a/exporting/mongodb/mongodb.c b/exporting/mongodb/mongodb.c
index aab1770d2..850d07fb3 100644
--- a/exporting/mongodb/mongodb.c
+++ b/exporting/mongodb/mongodb.c
@@ -99,6 +99,7 @@ int init_mongodb_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_json_plaintext;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = format_batch_mongodb;
diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c
index 7ed88fd6d..282de2e6b 100644
--- a/exporting/opentsdb/opentsdb.c
+++ b/exporting/opentsdb/opentsdb.c
@@ -38,6 +38,7 @@ int init_opentsdb_telnet_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_opentsdb_telnet;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = simple_connector_end_batch;
@@ -94,6 +95,7 @@ int init_opentsdb_http_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_opentsdb_http;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = close_batch_json_http;
@@ -124,13 +126,14 @@ int init_opentsdb_http_instance(struct instance *instance)
* @param len the maximum number of characters copied.
*/
-void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
+void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len)
{
while (*src != '\0' && len) {
- if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
+ if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
*dst++ = *src;
else
*dst++ = '_';
+
src++;
len--;
}
@@ -144,28 +147,18 @@ void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
* @param host a data collecting host.
* @return Always returns 0.
*/
-int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host)
-{
- if (!instance->labels)
- instance->labels = buffer_create(1024);
+
+int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) {
+ if(!instance->labels_buffer)
+ instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
- rrdhost_check_rdlock(host);
- netdata_rwlock_rdlock(&host->labels.labels_rwlock);
- for (struct label *label = host->labels.head; label; label = label->next) {
- if (!should_send_label(instance, label))
- continue;
-
- char value[CONFIG_MAX_VALUE + 1];
- sanitize_opentsdb_label_value(value, label->value, CONFIG_MAX_VALUE);
-
- if (*value)
- buffer_sprintf(instance->labels, " %s=%s", label->key, value);
- }
- netdata_rwlock_unlock(&host->labels.labels_rwlock);
-
+ buffer_strcat(instance->labels_buffer, " ");
+ rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", " ",
+ exporting_labels_filter_callback, instance,
+ NULL, sanitize_opentsdb_label_value);
return 0;
}
@@ -204,7 +197,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
- (instance->labels) ? buffer_tostring(instance->labels) : "");
+ (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}
@@ -234,14 +227,14 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
RRD_ID_LENGTH_MAX);
time_t last_t;
- calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+ NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
buffer_sprintf(
instance->buffer,
- "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s%s\n",
+ "put %s.%s.%s %llu " NETDATA_DOUBLE_FORMAT " host=%s%s%s%s\n",
instance->config.prefix,
chart_name,
dimension_name,
@@ -250,7 +243,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
- (instance->labels) ? buffer_tostring(instance->labels) : "");
+ (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}
@@ -287,33 +280,17 @@ void opentsdb_http_prepare_header(struct instance *instance)
* @param host a data collecting host.
* @return Always returns 0.
*/
-int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host)
-{
- if (!instance->labels)
- instance->labels = buffer_create(1024);
+
+int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) {
+ if (!instance->labels_buffer)
+ instance->labels_buffer = buffer_create(1024);
if (unlikely(!sending_labels_configured(instance)))
return 0;
- rrdhost_check_rdlock(host);
- netdata_rwlock_rdlock(&host->labels.labels_rwlock);
- for (struct label *label = host->labels.head; label; label = label->next) {
- if (!should_send_label(instance, label))
- continue;
-
- char escaped_value[CONFIG_MAX_VALUE * 2 + 1];
- sanitize_json_string(escaped_value, label->value, CONFIG_MAX_VALUE);
-
- char value[CONFIG_MAX_VALUE + 1];
- sanitize_opentsdb_label_value(value, escaped_value, CONFIG_MAX_VALUE);
-
- if (*value) {
- buffer_strcat(instance->labels, ",");
- buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
- }
- }
- netdata_rwlock_unlock(&host->labels.labels_rwlock);
-
+ rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, ",", ":", "\"", "",
+ exporting_labels_filter_callback, instance,
+ NULL, sanitize_opentsdb_label_value);
return 0;
}
@@ -362,7 +339,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
- instance->labels ? buffer_tostring(instance->labels) : "");
+ instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}
@@ -392,7 +369,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
RRD_ID_LENGTH_MAX);
time_t last_t;
- calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+ NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if(isnan(value))
return 0;
@@ -405,7 +382,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
"{"
"\"metric\":\"%s.%s.%s\","
"\"timestamp\":%llu,"
- "\"value\":"CALCULATED_NUMBER_FORMAT","
+ "\"value\":" NETDATA_DOUBLE_FORMAT ","
"\"tags\":{"
"\"host\":\"%s%s%s\"%s"
"}"
@@ -418,7 +395,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
(host == localhost) ? instance->config.hostname : host->hostname,
(host->tags) ? " " : "",
(host->tags) ? host->tags : "",
- instance->labels ? buffer_tostring(instance->labels) : "");
+ instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
return 0;
}
diff --git a/exporting/opentsdb/opentsdb.h b/exporting/opentsdb/opentsdb.h
index d53a5054f..b544ba8c1 100644
--- a/exporting/opentsdb/opentsdb.h
+++ b/exporting/opentsdb/opentsdb.h
@@ -8,7 +8,7 @@
int init_opentsdb_telnet_instance(struct instance *instance);
int init_opentsdb_http_instance(struct instance *instance);
-void sanitize_opentsdb_label_value(char *dst, char *src, size_t len);
+void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len);
int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host);
int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host);
diff --git a/exporting/process_data.c b/exporting/process_data.c
index c77b7ad4a..d5138b787 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -64,7 +64,7 @@ int mark_scheduled_instances(struct engine *engine)
* @param last_timestamp the timestamp that should be reported to the exporting connector instance.
* @return Returns the value, calculated over the given period.
*/
-calculated_number exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp)
@@ -77,11 +77,10 @@ calculated_number exporting_calculate_value_from_stored_data(
time_t before = instance->before;
// find the edges of the rrd database for this chart
- time_t first_t = rd->state->query_ops.oldest_time(rd);
- time_t last_t = rd->state->query_ops.latest_time(rd);
+ time_t first_t = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle);
+ time_t last_t = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
time_t update_every = st->update_every;
struct rrddim_query_handle handle;
- storage_number n;
// step back a little, to make sure we have complete data collection
// for all metrics
@@ -124,23 +123,21 @@ calculated_number exporting_calculate_value_from_stored_data(
*last_timestamp = before;
size_t counter = 0;
- calculated_number sum = 0;
+ NETDATA_DOUBLE sum = 0;
- for (rd->state->query_ops.init(rd, &handle, after, before); !rd->state->query_ops.is_finished(&handle);) {
- time_t curr_t;
- n = rd->state->query_ops.next_metric(&handle, &curr_t);
+ for (rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, after, before, TIER_QUERY_FETCH_SUM); !rd->tiers[0]->query_ops.is_finished(&handle);) {
+ STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
- if (unlikely(!does_storage_number_exist(n))) {
+ if (unlikely(storage_point_is_empty(sp))) {
// not collected
continue;
}
- calculated_number value = unpack_storage_number(n);
- sum += value;
-
- counter++;
+ sum += sp.sum;
+ counter += sp.count;
}
- rd->state->query_ops.finalize(&handle);
+ rd->tiers[0]->query_ops.finalize(&handle);
+
if (unlikely(!counter)) {
debug(
D_EXPORTING,
@@ -156,7 +153,7 @@ calculated_number exporting_calculate_value_from_stored_data(
if (unlikely(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_SUM))
return sum;
- return sum / (calculated_number)counter;
+ return sum / (NETDATA_DOUBLE)counter;
}
/**
@@ -262,6 +259,27 @@ void end_chart_formatting(struct engine *engine, RRDSET *st)
}
/**
+ * Format variables for every connector instance's buffer
+ *
+ * @param engine an engine data structure.
+ * @param host a data collecting host.
+ */
+void variables_formatting(struct engine *engine, RRDHOST *host)
+{
+ for (struct instance *instance = engine->instance_root; instance; instance = instance->next) {
+ if (instance->scheduled && !instance->skip_host && should_send_variables(instance)) {
+ if (instance->variables_formatting && instance->variables_formatting(instance, host) != 0){
+ error("EXPORTING: cannot format variables for %s", instance->config.name);
+ disable_instance(instance);
+ continue;
+ }
+ // sum all variables as one metrics
+ instance->stats.buffered_metrics++;
+ }
+ }
+}
+
+/**
* End host formatting for every connector instance's buffer
*
* @param engine an engine data structure.
@@ -337,7 +355,7 @@ void prepare_buffers(struct engine *engine)
end_chart_formatting(engine, st);
rrdset_unlock(st);
}
-
+ variables_formatting(engine, host);
end_host_formatting(engine, host);
rrdhost_unlock(host);
}
@@ -358,8 +376,8 @@ int flush_host_labels(struct instance *instance, RRDHOST *host)
{
(void)host;
- if (instance->labels)
- buffer_flush(instance->labels);
+ if (instance->labels_buffer)
+ buffer_flush(instance->labels_buffer);
return 0;
}
diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c
index c7f3f1d38..7d632164f 100644
--- a/exporting/prometheus/prometheus.c
+++ b/exporting/prometheus/prometheus.c
@@ -290,35 +290,44 @@ inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int sh
* @param instance an instance data structure.
* @param host a data collecting host.
*/
-void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
-{
- if (unlikely(!sending_labels_configured(instance)))
- return;
- if (!instance->labels)
- instance->labels = buffer_create(1024);
+struct format_prometheus_label_callback {
+ struct instance *instance;
+ size_t count;
+};
- int count = 0;
- rrdhost_check_rdlock(host);
- netdata_rwlock_rdlock(&host->labels.labels_rwlock);
- for (struct label *label = host->labels.head; label; label = label->next) {
- if (!should_send_label(instance, label))
- continue;
+static int format_prometheus_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+ struct format_prometheus_label_callback *d = (struct format_prometheus_label_callback *)data;
- char key[PROMETHEUS_ELEMENT_MAX + 1];
- char value[PROMETHEUS_ELEMENT_MAX + 1];
+ if (!should_send_label(d->instance, ls)) return 0;
- prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
- prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
+ char k[PROMETHEUS_ELEMENT_MAX + 1];
+ char v[PROMETHEUS_ELEMENT_MAX + 1];
- if (*key && *value) {
- if (count > 0)
- buffer_strcat(instance->labels, ",");
- buffer_sprintf(instance->labels, "%s=\"%s\"", key, value);
- count++;
- }
+ prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
+ prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
+
+ if (*k && *v) {
+ if (d->count > 0) buffer_strcat(d->instance->labels_buffer, ",");
+ buffer_sprintf(d->instance->labels_buffer, "%s=\"%s\"", k, v);
+ d->count++;
}
- netdata_rwlock_unlock(&host->labels.labels_rwlock);
+ return 1;
+}
+
+void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
+{
+ if (unlikely(!sending_labels_configured(instance)))
+ return;
+
+ if (!instance->labels_buffer)
+ instance->labels_buffer = buffer_create(1024);
+
+ struct format_prometheus_label_callback tmp = {
+ .instance = instance,
+ .count = 0
+ };
+ rrdlabels_walkthrough_read(host->host_labels, format_prometheus_label_callback, &tmp);
}
struct host_variables_callback_options {
@@ -353,7 +362,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
}
}
- calculated_number value = rrdvar2number(rv);
+ NETDATA_DOUBLE value = rrdvar2number(rv);
if (isnan(value) || isinf(value)) {
if (opts->output_options & PROMETHEUS_OUTPUT_HELP)
buffer_sprintf(
@@ -374,7 +383,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
if (opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
buffer_sprintf(
opts->wb,
- "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n",
+ "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT " %llu\n",
opts->prefix,
opts->name,
label_pre,
@@ -385,7 +394,7 @@ static int print_host_variables(RRDVAR *rv, void *data)
else
buffer_sprintf(
opts->wb,
- "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n",
+ "%s_%s%s%s%s " NETDATA_DOUBLE_FORMAT "\n",
opts->prefix,
opts->name,
label_pre,
@@ -474,9 +483,9 @@ static void generate_as_collected_prom_metric(BUFFER *wb, struct gen_parameters
if (prometheus_collector)
buffer_sprintf(
wb,
- CALCULATED_NUMBER_FORMAT,
- (calculated_number)p->rd->last_collected_value * (calculated_number)p->rd->multiplier /
- (calculated_number)p->rd->divisor);
+ NETDATA_DOUBLE_FORMAT,
+ (NETDATA_DOUBLE)p->rd->last_collected_value * (NETDATA_DOUBLE)p->rd->multiplier /
+ (NETDATA_DOUBLE)p->rd->divisor);
else
buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT, p->rd->last_collected_value);
@@ -516,77 +525,29 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
format_host_labels_prometheus(instance, host);
+ buffer_sprintf(
+ wb,
+ "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"",
+ hostname,
+ host->program_name,
+ host->program_version);
+
+ if (instance->labels_buffer && *buffer_tostring(instance->labels_buffer)) {
+ buffer_sprintf(wb, ",%s", buffer_tostring(instance->labels_buffer));
+ }
+
if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
- buffer_sprintf(
- wb,
- "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
- hostname,
- host->program_name,
- host->program_version,
- now_realtime_usec() / USEC_PER_MS);
+ buffer_sprintf(wb, "} 1 %llu\n", now_realtime_usec() / USEC_PER_MS);
else
- buffer_sprintf(
- wb,
- "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
- hostname,
- host->program_name,
- host->program_version);
+ buffer_sprintf(wb, "} 1\n");
char labels[PROMETHEUS_LABELS_MAX + 1] = "";
if (allhosts) {
- if (instance->labels && buffer_tostring(instance->labels)) {
- if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
- buffer_sprintf(
- wb,
- "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n",
- hostname,
- buffer_tostring(instance->labels),
- now_realtime_usec() / USEC_PER_MS);
-
- // deprecated, exists only for compatibility with older queries
- buffer_sprintf(
- wb,
- "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n",
- hostname,
- buffer_tostring(instance->labels),
- now_realtime_usec() / USEC_PER_MS);
- } else {
- buffer_sprintf(
- wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
-
- // deprecated, exists only for compatibility with older queries
- buffer_sprintf(
- wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
- }
- }
-
snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
- } else {
- if (instance->labels && buffer_tostring(instance->labels)) {
- if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
- buffer_sprintf(
- wb,
- "netdata_host_tags_info{%s} 1 %llu\n",
- buffer_tostring(instance->labels),
- now_realtime_usec() / USEC_PER_MS);
-
- // deprecated, exists only for compatibility with older queries
- buffer_sprintf(
- wb,
- "netdata_host_tags{%s} 1 %llu\n",
- buffer_tostring(instance->labels),
- now_realtime_usec() / USEC_PER_MS);
- } else {
- buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels));
-
- // deprecated, exists only for compatibility with older queries
- buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels));
- }
- }
- }
+ }
- if (instance->labels)
- buffer_flush(instance->labels);
+ if (instance->labels_buffer)
+ buffer_flush(instance->labels_buffer);
// send custom variables set for the host
if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {
@@ -723,7 +684,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
time_t first_time = instance->after;
time_t last_time = instance->before;
- calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
+ NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
if (!isnan(value) && !isinf(value)) {
if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
@@ -755,7 +716,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
buffer_sprintf(
wb,
- "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
+ "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
" %llu\n",
prefix,
context,
@@ -770,7 +731,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
else
buffer_sprintf(
wb,
- "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT
+ "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " NETDATA_DOUBLE_FORMAT
"\n",
prefix,
context,
diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c
index 59a488e1b..03feb2c08 100644
--- a/exporting/prometheus/remote_write/remote_write.c
+++ b/exporting/prometheus/remote_write/remote_write.c
@@ -97,6 +97,7 @@ int init_prometheus_remote_write_instance(struct instance *instance)
instance->start_chart_formatting = format_chart_prometheus_remote_write;
instance->metric_formatting = format_dimension_prometheus_remote_write;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = format_variables_prometheus_remote_write;
instance->end_host_formatting = NULL;
instance->end_batch_formatting = format_batch_prometheus_remote_write;
@@ -134,6 +135,25 @@ int init_prometheus_remote_write_instance(struct instance *instance)
return 0;
}
+struct format_remote_write_label_callback {
+ struct instance *instance;
+ void *write_request;
+};
+
+static int format_remote_write_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+ struct format_remote_write_label_callback *d = (struct format_remote_write_label_callback *)data;
+
+ if (!should_send_label(d->instance, ls)) return 0;
+
+ char k[PROMETHEUS_ELEMENT_MAX + 1];
+ char v[PROMETHEUS_ELEMENT_MAX + 1];
+
+ prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
+ prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
+ add_label(d->write_request, k, v);
+ return 1;
+}
+
/**
* Format host data for Prometheus Remote Write connector
*
@@ -157,23 +177,13 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
add_host_info(
connector_specific_data->write_request,
"netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
-
+
if (unlikely(sending_labels_configured(instance))) {
- rrdhost_check_rdlock(host);
- netdata_rwlock_rdlock(&host->labels.labels_rwlock);
- for (struct label *label = host->labels.head; label; label = label->next) {
- if (!should_send_label(instance, label))
- continue;
-
- char key[PROMETHEUS_ELEMENT_MAX + 1];
- prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
-
- char value[PROMETHEUS_ELEMENT_MAX + 1];
- prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
-
- add_label(connector_specific_data->write_request, key, value);
- }
- netdata_rwlock_unlock(&host->labels.labels_rwlock);
+ struct format_remote_write_label_callback tmp = {
+ .write_request = connector_specific_data->write_request,
+ .instance = instance
+ };
+ rrdlabels_walkthrough_read(host->host_labels, format_remote_write_label_callback, &tmp);
}
return 0;
@@ -284,7 +294,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
// we need average or sum of the data
time_t last_t = instance->before;
- calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+ NETDATA_DOUBLE value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
if (!isnan(value) && !isinf(value)) {
if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AVERAGE)
@@ -311,6 +321,49 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
return 0;
}
+int format_variable_prometheus_remote_write_callback(RRDVAR *rv, void *data) {
+ struct prometheus_remote_write_variables_callback_options *opts = data;
+
+ if (rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR | RRDVAR_OPTION_CUSTOM_CHART_VAR)) {
+ RRDHOST *host = opts->host;
+ struct instance *instance = opts->instance;
+ struct simple_connector_data *simple_connector_data =
+ (struct simple_connector_data *)instance->connector_specific_data;
+ struct prometheus_remote_write_specific_data *connector_specific_data =
+ (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data;
+
+ char name[PROMETHEUS_LABELS_MAX + 1];
+ char *suffix = "";
+
+ prometheus_name_copy(context, rv->name, PROMETHEUS_ELEMENT_MAX);
+ snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", instance->config.prefix, context, suffix);
+
+ NETDATA_DOUBLE value = rrdvar2number(rv);
+ add_variable(connector_specific_data->write_request, name,
+ (host == localhost) ? instance->config.hostname : host->hostname, value, opts->now / USEC_PER_MS);
+ }
+
+ return 0;
+}
+
+/**
+ * Format a variable for Prometheus Remote Write connector
+ *
+ * @param rv a variable.
+ * @param instance an instance data structure.
+ * @return Always returns 0.
+ */
+int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host)
+{
+ struct prometheus_remote_write_variables_callback_options opt = {
+ .host = host,
+ .instance = instance,
+ .now = now_realtime_usec(),
+ };
+
+ return foreach_host_variable_callback(host, format_variable_prometheus_remote_write_callback, &opt);
+}
+
/**
* Format a batch for Prometheus Remote Write connector
*
@@ -339,7 +392,6 @@ int format_batch_prometheus_remote_write(struct instance *instance)
return 1;
}
buffer->len = data_size;
- instance->stats.buffered_bytes = (collected_number)buffer_strlen(buffer);
simple_connector_end_batch(instance);
diff --git a/exporting/prometheus/remote_write/remote_write.h b/exporting/prometheus/remote_write/remote_write.h
index d738f5126..4740772d0 100644
--- a/exporting/prometheus/remote_write/remote_write.h
+++ b/exporting/prometheus/remote_write/remote_write.h
@@ -11,12 +11,19 @@ struct prometheus_remote_write_specific_data {
void *write_request;
};
+struct prometheus_remote_write_variables_callback_options {
+ RRDHOST *host;
+ time_t now;
+ struct instance *instance;
+};
+
int init_prometheus_remote_write_instance(struct instance *instance);
extern void clean_prometheus_remote_write(struct instance *instance);
int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host);
int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st);
int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd);
+int format_variables_prometheus_remote_write(struct instance *instance, RRDHOST *host);
int format_batch_prometheus_remote_write(struct instance *instance);
void prometheus_remote_write_prepare_header(struct instance *instance);
diff --git a/exporting/prometheus/remote_write/remote_write_request.cc b/exporting/prometheus/remote_write/remote_write_request.cc
index cfd61271e..ecfa11fa8 100644
--- a/exporting/prometheus/remote_write/remote_write_request.cc
+++ b/exporting/prometheus/remote_write/remote_write_request.cc
@@ -138,6 +138,38 @@ void add_metric(
}
/**
+ * Adds a metric to a write request
+ *
+ * @param write_request_p the write request
+ * @param name the name of the metric
+ * @param instance the name of the host, the metric belongs to
+ * @param value the value of the metric
+ * @param timestamp the timestamp for the metric in milliseconds
+ */
+void add_variable(
+ void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp)
+{
+ WriteRequest *write_request = (WriteRequest *)write_request_p;
+ TimeSeries *timeseries;
+ Sample *sample;
+ Label *label;
+
+ timeseries = write_request->add_timeseries();
+
+ label = timeseries->add_labels();
+ label->set_name("__name__");
+ label->set_value(name);
+
+ label = timeseries->add_labels();
+ label->set_name("instance");
+ label->set_value(instance);
+
+ sample = timeseries->add_samples();
+ sample->set_value(value);
+ sample->set_timestamp(timestamp);
+}
+
+/**
* Gets the size of a write request
*
* @param write_request_p the write request
diff --git a/exporting/prometheus/remote_write/remote_write_request.h b/exporting/prometheus/remote_write/remote_write_request.h
index 5f242b941..b25370133 100644
--- a/exporting/prometheus/remote_write/remote_write_request.h
+++ b/exporting/prometheus/remote_write/remote_write_request.h
@@ -20,6 +20,9 @@ void add_metric(
const char *name, const char *chart, const char *family, const char *dimension,
const char *instance, const double value, const int64_t timestamp);
+void add_variable(
+ void *write_request_p, const char *name, const char *instance, const double value, const int64_t timestamp);
+
size_t get_write_request_size(void *write_request_p);
int pack_and_clear_write_request(void *write_request_p, char *buffer, size_t *size);
diff --git a/exporting/pubsub/pubsub.c b/exporting/pubsub/pubsub.c
index 5a5afbdc2..b218338f1 100644
--- a/exporting/pubsub/pubsub.c
+++ b/exporting/pubsub/pubsub.c
@@ -23,6 +23,7 @@ int init_pubsub_instance(struct instance *instance)
instance->metric_formatting = format_dimension_stored_json_plaintext;
instance->end_chart_formatting = NULL;
+ instance->variables_formatting = NULL;
instance->end_host_formatting = flush_host_labels;
instance->end_batch_formatting = NULL;
diff --git a/exporting/read_config.c b/exporting/read_config.c
index b834e867d..1cba16826 100644
--- a/exporting/read_config.c
+++ b/exporting/read_config.c
@@ -399,6 +399,11 @@ struct engine *read_exporting_config()
else
tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;
+ if (exporter_get_boolean(instance_name, "send variables", CONFIG_BOOLEAN_YES))
+ tmp_instance->config.options |= EXPORTING_OPTION_SEND_VARIABLES;
+ else
+ tmp_instance->config.options &= ~EXPORTING_OPTION_SEND_VARIABLES;
+
if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) {
struct prometheus_remote_write_specific_config *connector_specific_config =
callocz(1, sizeof(struct prometheus_remote_write_specific_config));
diff --git a/exporting/tests/exporting_doubles.c b/exporting/tests/exporting_doubles.c
index b8c9f3756..75ab7ba43 100644
--- a/exporting/tests/exporting_doubles.c
+++ b/exporting/tests/exporting_doubles.c
@@ -52,11 +52,11 @@ int __wrap_mark_scheduled_instances(struct engine *engine)
return mock_type(int);
}
-calculated_number __real_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);
-calculated_number __wrap_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp)
@@ -67,7 +67,7 @@ calculated_number __wrap_exporting_calculate_value_from_stored_data(
*last_timestamp = 15052;
function_called();
- return mock_type(calculated_number);
+ return mock_type(NETDATA_DOUBLE);
}
int __real_prepare_buffers(struct engine *engine);
@@ -156,6 +156,14 @@ int __mock_end_chart_formatting(struct instance *instance, RRDSET *st)
return mock_type(int);
}
+int __mock_variables_formatting(struct instance *instance, RRDHOST *host)
+{
+ function_called();
+ check_expected_ptr(instance);
+ check_expected_ptr(host);
+ return mock_type(int);
+}
+
int __mock_end_host_formatting(struct instance *instance, RRDHOST *host)
{
function_called();
diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c
index 501fc405c..aae1c53fb 100644
--- a/exporting/tests/exporting_fixtures.c
+++ b/exporting/tests/exporting_fixtures.c
@@ -41,17 +41,9 @@ int setup_rrdhost()
localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2");
- struct label *label = calloc(1, sizeof(struct label));
- label->key = strdupz("key1");
- label->value = strdupz("value1");
- label->label_source = LABEL_SOURCE_NETDATA_CONF;
- localhost->labels.head = label;
-
- label = calloc(1, sizeof(struct label));
- label->key = strdupz("key2");
- label->value = strdupz("value2");
- label->label_source = LABEL_SOURCE_AUTO;
- localhost->labels.head->next = label;
+ localhost->host_labels = rrdlabels_create();
+ rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG);
+ rrdlabels_add(localhost->host_labels, "key2", "value2", RRDLABEL_SRC_CONFIG);
localhost->rrdset_root = calloc(1, sizeof(RRDSET));
RRDSET *st = localhost->rrdset_root;
@@ -71,13 +63,13 @@ int setup_rrdhost()
rd->collections_counter++;
rd->next = NULL;
- rd->state = calloc(1, sizeof(*rd->state));
- rd->state->query_ops.oldest_time = __mock_rrddim_query_oldest_time;
- rd->state->query_ops.latest_time = __mock_rrddim_query_latest_time;
- rd->state->query_ops.init = __mock_rrddim_query_init;
- rd->state->query_ops.is_finished = __mock_rrddim_query_is_finished;
- rd->state->query_ops.next_metric = __mock_rrddim_query_next_metric;
- rd->state->query_ops.finalize = __mock_rrddim_query_finalize;
+ rd->tiers[0] = calloc(1, sizeof(struct rrddim_tier));
+ rd->tiers[0]->query_ops.oldest_time = __mock_rrddim_query_oldest_time;
+ rd->tiers[0]->query_ops.latest_time = __mock_rrddim_query_latest_time;
+ rd->tiers[0]->query_ops.init = __mock_rrddim_query_init;
+ rd->tiers[0]->query_ops.is_finished = __mock_rrddim_query_is_finished;
+ rd->tiers[0]->query_ops.next_metric = __mock_rrddim_query_next_metric;
+ rd->tiers[0]->query_ops.finalize = __mock_rrddim_query_finalize;
return 0;
}
@@ -87,19 +79,14 @@ int teardown_rrdhost()
RRDDIM *rd = localhost->rrdset_root->dimensions;
free((void *)rd->name);
free((void *)rd->id);
- free(rd->state);
+ free(rd->tiers[0]);
free(rd);
RRDSET *st = localhost->rrdset_root;
free((void *)st->name);
free(st);
- free(localhost->labels.head->next->key);
- free(localhost->labels.head->next->value);
- free(localhost->labels.head->next);
- free(localhost->labels.head->key);
- free(localhost->labels.head->value);
- free(localhost->labels.head);
+ rrdlabels_destroy(localhost->host_labels);
free((void *)localhost->tags);
free(localhost);
@@ -124,7 +111,7 @@ int teardown_initialized_engine(void **state)
struct engine *engine = *state;
teardown_rrdhost();
- buffer_free(engine->instance_root->labels);
+ buffer_free(engine->instance_root->labels_buffer);
buffer_free(engine->instance_root->buffer);
teardown_configured_engine(state);
diff --git a/exporting/tests/netdata_doubles.c b/exporting/tests/netdata_doubles.c
index a9a184336..ee36e887a 100644
--- a/exporting/tests/netdata_doubles.c
+++ b/exporting/tests/netdata_doubles.c
@@ -177,7 +177,7 @@ const char *rrd_memory_mode_name(RRD_MEMORY_MODE id)
return RRD_MEMORY_MODE_NONE_NAME;
}
-calculated_number rrdvar2number(RRDVAR *rv)
+NETDATA_DOUBLE rrdvar2number(RRDVAR *rv)
{
(void)rv;
return 0;
@@ -196,26 +196,27 @@ void rrdset_update_heterogeneous_flag(RRDSET *st)
(void)st;
}
-time_t __mock_rrddim_query_oldest_time(RRDDIM *rd)
+time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle)
{
- (void)rd;
+ (void)db_metric_handle;
function_called();
return mock_type(time_t);
}
-time_t __mock_rrddim_query_latest_time(RRDDIM *rd)
+time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle)
{
- (void)rd;
+ (void)db_metric_handle;
function_called();
return mock_type(time_t);
}
-void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time)
+void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type)
{
- (void)rd;
+ (void)db_metric_handle;
(void)handle;
+ (void)tier_query_fetch_type;
function_called();
check_expected(start_time);
@@ -230,13 +231,14 @@ int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle)
return mock_type(int);
}
-storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time)
+STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle)
{
(void)handle;
- (void)current_time;
function_called();
- return mock_type(storage_number);
+
+ STORAGE_POINT sp = {};
+ return sp;
}
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
@@ -245,3 +247,11 @@ void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
function_called();
}
+
+void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value)
+{
+ (void)chart_uuid;
+ (void)source_type;
+ (void)label;
+ (void)value;
+}
diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c
index 6bb7d2efd..56a28059f 100644
--- a/exporting/tests/test_exporting_engine.c
+++ b/exporting/tests/test_exporting_engine.c
@@ -307,19 +307,17 @@ static void test_exporting_calculate_value_from_stored_data(void **state)
expect_function_call(__mock_rrddim_query_is_finished);
will_return(__mock_rrddim_query_is_finished, 0);
expect_function_call(__mock_rrddim_query_next_metric);
- will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_DEFAULT_FLAGS));
expect_function_call(__mock_rrddim_query_is_finished);
will_return(__mock_rrddim_query_is_finished, 0);
expect_function_call(__mock_rrddim_query_next_metric);
- will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_DEFAULT_FLAGS));
expect_function_call(__mock_rrddim_query_is_finished);
will_return(__mock_rrddim_query_is_finished, 1);
expect_function_call(__mock_rrddim_query_finalize);
- assert_int_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, &timestamp), 36);
+ assert_float_equal(__real_exporting_calculate_value_from_stored_data(instance, rd, &timestamp), 36, 0.1);
}
static void test_prepare_buffers(void **state)
@@ -381,7 +379,7 @@ static void test_prepare_buffers(void **state)
expect_value(__mock_end_batch_formatting, instance, instance);
will_return(__mock_end_batch_formatting, 0);
- assert_int_equal(__real_prepare_buffers(engine), 0);
+ __real_prepare_buffers(engine);
assert_int_equal(instance->stats.buffered_metrics, 1);
@@ -393,7 +391,7 @@ static void test_prepare_buffers(void **state)
instance->end_chart_formatting = NULL;
instance->end_host_formatting = NULL;
instance->end_batch_formatting = NULL;
- assert_int_equal(__real_prepare_buffers(engine), 0);
+ __real_prepare_buffers(engine);
assert_int_equal(instance->scheduled, 0);
assert_int_equal(instance->after, 2);
@@ -705,7 +703,7 @@ static void test_format_host_labels_json_plaintext(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_json_plaintext(instance, localhost), 0);
- assert_string_equal(buffer_tostring(instance->labels), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
+ assert_string_equal(buffer_tostring(instance->labels_buffer), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
}
static void test_format_host_labels_graphite_plaintext(void **state)
@@ -717,7 +715,7 @@ static void test_format_host_labels_graphite_plaintext(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_graphite_plaintext(instance, localhost), 0);
- assert_string_equal(buffer_tostring(instance->labels), ";key1=value1;key2=value2");
+ assert_string_equal(buffer_tostring(instance->labels_buffer), ";key1=value1;key2=value2");
}
static void test_format_host_labels_opentsdb_telnet(void **state)
@@ -729,7 +727,7 @@ static void test_format_host_labels_opentsdb_telnet(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_opentsdb_telnet(instance, localhost), 0);
- assert_string_equal(buffer_tostring(instance->labels), " key1=value1 key2=value2");
+ assert_string_equal(buffer_tostring(instance->labels_buffer), " key1=value1 key2=value2");
}
static void test_format_host_labels_opentsdb_http(void **state)
@@ -741,7 +739,7 @@ static void test_format_host_labels_opentsdb_http(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
assert_int_equal(format_host_labels_opentsdb_http(instance, localhost), 0);
- assert_string_equal(buffer_tostring(instance->labels), ",\"key1\":\"value1\",\"key2\":\"value2\"");
+ assert_string_equal(buffer_tostring(instance->labels_buffer), ",\"key1\":\"value1\",\"key2\":\"value2\"");
}
static void test_flush_host_labels(void **state)
@@ -749,12 +747,12 @@ static void test_flush_host_labels(void **state)
struct engine *engine = *state;
struct instance *instance = engine->instance_root;
- instance->labels = buffer_create(12);
- buffer_strcat(instance->labels, "check string");
- assert_int_equal(buffer_strlen(instance->labels), 12);
+ instance->labels_buffer = buffer_create(12);
+ buffer_strcat(instance->labels_buffer, "check string");
+ assert_int_equal(buffer_strlen(instance->labels_buffer), 12);
assert_int_equal(flush_host_labels(instance, localhost), 0);
- assert_int_equal(buffer_strlen(instance->labels), 0);
+ assert_int_equal(buffer_strlen(instance->labels_buffer), 0);
}
static void test_create_main_rusage_chart(void **state)
@@ -1048,7 +1046,7 @@ static void test_format_host_labels_prometheus(void **state)
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
format_host_labels_prometheus(instance, localhost);
- assert_string_equal(buffer_tostring(instance->labels), "key1=\"value1\",key2=\"value2\"");
+ assert_string_equal(buffer_tostring(instance->labels_buffer), "key1=\"value1\",key2=\"value2\"");
}
static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
@@ -1071,9 +1069,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
assert_string_equal(
buffer_tostring(buffer),
- "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n"
- "netdata_host_tags_info{key1=\"value1\",key2=\"value2\"} 1\n"
- "netdata_host_tags{key1=\"value1\",key2=\"value2\"} 1\n"
+ "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n"
"test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\"} 690565856.0000000\n");
buffer_flush(buffer);
@@ -1089,9 +1085,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
assert_string_equal(
buffer_tostring(buffer),
- "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n"
- "netdata_host_tags_info{key1=\"value1\",key2=\"value2\"} 1\n"
- "netdata_host_tags{key1=\"value1\",key2=\"value2\"} 1\n"
+ "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n"
"# TYPE test_prefix_test_context gauge\n"
"test_prefix_test_context{chart=\"chart_name\",family=\"test_family\",dimension=\"dimension_name\"} 690565856.0000000\n");
@@ -1107,9 +1101,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
assert_string_equal(
buffer_tostring(buffer),
- "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\"} 1\n"
- "netdata_host_tags_info{instance=\"test_hostname\",key1=\"value1\",key2=\"value2\"} 1\n"
- "netdata_host_tags{instance=\"test_hostname\",key1=\"value1\",key2=\"value2\"} 1\n"
+ "netdata_info{instance=\"test_hostname\",application=\"(null)\",version=\"(null)\",key1=\"value1\",key2=\"value2\"} 1\n"
"test_prefix_test_context{chart=\"chart_id\",family=\"test_family\",dimension=\"dimension_id\",instance=\"test_hostname\"} 690565856.0000000\n");
free(localhost->rrdset_root->context);
diff --git a/exporting/tests/test_exporting_engine.h b/exporting/tests/test_exporting_engine.h
index 800be1b99..ae0b7df9a 100644
--- a/exporting/tests/test_exporting_engine.h
+++ b/exporting/tests/test_exporting_engine.h
@@ -30,7 +30,14 @@
#include <stddef.h>
#include <setjmp.h>
#include <stdint.h>
+
+#ifndef UNIT_TESTING
+#include <cmocka.h>
+#else
+#undef UNIT_TESTING
#include <cmocka.h>
+#define UNIT_TESTING
+#endif
#define MAX_LOG_LINE 1024
extern char log_line[];
@@ -50,11 +57,11 @@ int __wrap_connect_to_one_of(
void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);
void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line);
void __rrd_check_rdlock(const char *file, const char *function, const unsigned long line);
-time_t __mock_rrddim_query_oldest_time(RRDDIM *rd);
-time_t __mock_rrddim_query_latest_time(RRDDIM *rd);
-void __mock_rrddim_query_init(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
+time_t __mock_rrddim_query_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
+time_t __mock_rrddim_query_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
+void __mock_rrddim_query_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
int __mock_rrddim_query_is_finished(struct rrddim_query_handle *handle);
-storage_number __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle, time_t *current_time);
+STORAGE_POINT __mock_rrddim_query_next_metric(struct rrddim_query_handle *handle);
void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle);
// -----------------------------------------------------------------------
@@ -81,11 +88,11 @@ int __wrap_init_connectors(struct engine *engine);
int __real_mark_scheduled_instances(struct engine *engine);
int __wrap_mark_scheduled_instances(struct engine *engine);
-calculated_number __real_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __real_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);
-calculated_number __wrap_exporting_calculate_value_from_stored_data(
+NETDATA_DOUBLE __wrap_exporting_calculate_value_from_stored_data(
struct instance *instance,
RRDDIM *rd,
time_t *last_timestamp);
@@ -113,6 +120,7 @@ int __mock_start_host_formatting(struct instance *instance, RRDHOST *host);
int __mock_start_chart_formatting(struct instance *instance, RRDSET *st);
int __mock_metric_formatting(struct instance *instance, RRDDIM *rd);
int __mock_end_chart_formatting(struct instance *instance, RRDSET *st);
+int __mock_variables_formatting(struct instance *instance, RRDHOST *host);
int __mock_end_host_formatting(struct instance *instance, RRDHOST *host);
int __mock_end_batch_formatting(struct instance *instance);