summaryrefslogtreecommitdiffstats
path: root/collectors/statsd.plugin/statsd.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--collectors/statsd.plugin/statsd.c (renamed from src/statsd.c)373
1 files changed, 215 insertions, 158 deletions
diff --git a/src/statsd.c b/collectors/statsd.plugin/statsd.c
index 44ebd8894..c92bfd1c2 100644
--- a/src/statsd.c
+++ b/collectors/statsd.plugin/statsd.c
@@ -1,7 +1,10 @@
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "statsd.h"
#define STATSD_CHART_PREFIX "statsd"
-#define STATSD_CHART_PRIORITY 90000
+
+#define PLUGIN_STATSD_NAME "statsd.plugin"
// --------------------------------------------------------------------------------------
@@ -87,7 +90,10 @@ typedef enum statsd_metric_options {
STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED = 0x00000002, // render a private chart for this metric
STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED = 0x00000004, // the metric has been checked if it should get private chart or not
STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT = 0x00000008, // show the count of events for this private chart
- STATSD_METRIC_OPTION_CHECKED_IN_APPS = 0x00000010, // set when this metric has been checked agains apps
+ STATSD_METRIC_OPTION_CHECKED_IN_APPS = 0x00000010, // set when this metric has been checked against apps
+ STATSD_METRIC_OPTION_USED_IN_APPS = 0x00000020, // set when this metric is used in apps
+ STATSD_METRIC_OPTION_CHECKED = 0x00000040, // set when the charting thread checks this metric for use in charts (its usefulness)
+ STATSD_METRIC_OPTION_USEFUL = 0x00000080, // set when the charting thread finds the metric useful (i.e. used in a chart)
} STATS_METRIC_OPTIONS;
typedef enum statsd_metric_type {
@@ -101,7 +107,7 @@ typedef enum statsd_metric_type {
typedef struct statsd_metric {
- avl avl; // indexing
+ avl avl; // indexing - has to be first
const char *name; // the name of the metric
uint32_t hash; // hash of the name
@@ -122,14 +128,15 @@ typedef struct statsd_metric {
// chart related members
STATS_METRIC_OPTIONS options; // STATSD_METRIC_OPTION_* (bitfield)
- char reset; // set to 1 to reset this metric to zero
+ char reset; // set to 1 by the charting thread to instruct the collector thread(s) to reset this metric
collected_number last; // the last value sent to netdata
- RRDSET *st; // the chart of this metric
+ RRDSET *st; // the private chart of this metric
RRDDIM *rd_value; // the dimension of this metric value
RRDDIM *rd_count; // the dimension for the number of events received
// linking, used for walking through all metrics
struct statsd_metric *next;
+ struct statsd_metric *next_useful;
} STATSD_METRIC;
@@ -140,10 +147,12 @@ typedef struct statsd_index {
char *name; // the name of the index of metrics
size_t events; // the number of events processed for this index
size_t metrics; // the number of metrics in this index
+ size_t useful; // the number of useful metrics in this index
STATSD_AVL_TREE index; // the AVL tree
STATSD_METRIC *first; // the linked list of metrics (new metrics are added in front)
+ STATSD_METRIC *first_useful; // the linked list of useful metrics (new metrics are added in front)
STATSD_FIRST_PTR_MUTEX; // when mutli-threading is enabled, a lock to protect the linked list
STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index
@@ -255,13 +264,13 @@ static struct statsd {
SIMPLE_PATTERN *charts_for;
size_t tcp_idle_timeout;
- size_t decimal_detail;
+ collected_number decimal_detail;
size_t private_charts;
size_t max_private_charts;
size_t max_private_charts_hard;
RRD_MEMORY_MODE private_charts_memory_mode;
long private_charts_rrd_history_entries;
- int private_charts_hidden;
+ unsigned int private_charts_hidden:1;
STATSD_APP *apps;
size_t recvmmsg_size;
@@ -344,6 +353,7 @@ static struct statsd {
.threads = 0,
.collection_threads_status = NULL,
.sockets = {
+ .config = &netdata_config,
.config_section = CONFIG_SECTION_STATSD,
.default_bind_to = "udp:localhost tcp:localhost",
.default_port = STATSD_LISTEN_PORT,
@@ -427,6 +437,13 @@ static inline LONG_DOUBLE statsd_parse_float(const char *v, LONG_DOUBLE def) {
return value;
}
+static inline LONG_DOUBLE statsd_parse_sampling_rate(const char *v) {
+ LONG_DOUBLE sampling_rate = statsd_parse_float(v, 1.0);
+ if(unlikely(isless(sampling_rate, 0.001))) sampling_rate = 0.001;
+ if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
+ return sampling_rate;
+}
+
static inline long long statsd_parse_int(const char *v, long long def) {
long long value;
@@ -455,7 +472,12 @@ static inline int value_is_zinit(const char *value) {
return (value && *value == 'z' && *++value == 'i' && *++value == 'n' && *++value == 'i' && *++value == 't' && *++value == '\0');
}
+#define is_metric_checked(m) ((m)->options & STATSD_METRIC_OPTION_CHECKED)
+#define is_metric_useful_for_collection(m) (!is_metric_checked(m) || ((m)->options & STATSD_METRIC_OPTION_USEFUL))
+
static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, const char *sampling) {
+ if(!is_metric_useful_for_collection(m)) return;
+
if(unlikely(!value || !*value)) {
error("STATSD: metric '%s' of type gauge, with empty value is ignored.", m->name);
return;
@@ -471,16 +493,18 @@ static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, con
}
else {
if (unlikely(*value == '+' || *value == '-'))
- m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+ m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_sampling_rate(sampling);
else
- m->gauge.value = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+ m->gauge.value = statsd_parse_float(value, 1.0);
m->events++;
m->count++;
}
}
-static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, const char *sampling) {
+static inline void statsd_process_counter_or_meter(STATSD_METRIC *m, const char *value, const char *sampling) {
+ if(!is_metric_useful_for_collection(m)) return;
+
// we accept empty values for counters
if(unlikely(m->reset)) statsd_reset_metric(m);
@@ -489,21 +513,21 @@ static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, c
// magic loading of metric, without affecting anything
}
else {
- m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_float(sampling, 1.0));
+ m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling));
m->events++;
m->count++;
}
}
-static inline void statsd_process_meter(STATSD_METRIC *m, const char *value, const char *sampling) {
- // this is the same with the counter
- statsd_process_counter(m, value, sampling);
-}
+#define statsd_process_counter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling)
+#define statsd_process_meter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling)
+
+static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const char *value, const char *sampling, const char *type) {
+ if(!is_metric_useful_for_collection(m)) return;
-static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value, const char *sampling) {
if(unlikely(!value || !*value)) {
- error("STATSD: metric '%s' of type histogram, with empty value is ignored.", m->name);
+ error("STATSD: metric of type %s, with empty value is ignored.", type);
return;
}
@@ -516,31 +540,35 @@ static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value,
// magic loading of metric, without affecting anything
}
else {
- if (unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
- netdata_mutex_lock(&m->histogram.ext->mutex);
- m->histogram.ext->size += statsd.histogram_increase_step;
- m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size);
- netdata_mutex_unlock(&m->histogram.ext->mutex);
- }
+ LONG_DOUBLE v = statsd_parse_float(value, 1.0);
+ LONG_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling);
+ if(unlikely(isless(sampling_rate, 0.01))) sampling_rate = 0.01;
+ if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
+
+ long long samples = llrintl(1.0 / sampling_rate);
+ while(samples-- > 0) {
+
+ if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
+ netdata_mutex_lock(&m->histogram.ext->mutex);
+ m->histogram.ext->size += statsd.histogram_increase_step;
+ m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size);
+ netdata_mutex_unlock(&m->histogram.ext->mutex);
+ }
- m->histogram.ext->values[m->histogram.ext->used++] = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+ m->histogram.ext->values[m->histogram.ext->used++] = v;
+ }
m->events++;
m->count++;
}
}
-static inline void statsd_process_timer(STATSD_METRIC *m, const char *value, const char *sampling) {
- if(unlikely(!value || !*value)) {
- error("STATSD: metric of type timer, with empty value is ignored.");
- return;
- }
-
- // timers are a use case of histogram
- statsd_process_histogram(m, value, sampling);
-}
+#define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer")
+#define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram")
static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
+ if(!is_metric_useful_for_collection(m)) return;
+
if(unlikely(!value || !*value)) {
error("STATSD: metric of type set, with empty value is ignored.");
return;
@@ -578,8 +606,10 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
// --------------------------------------------------------------------------------------------------------------------
// statsd parsing
-static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling) {
- debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', rate '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)");
+static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling, const char *tags) {
+ (void)tags;
+
+ debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', sampling '%s', tags '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)", tags?tags:"(null)");
if(unlikely(!name || !*name)) return;
if(unlikely(!type || !*type)) type = "m";
@@ -663,8 +693,8 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
const char *s = buffer;
while(*s) {
- const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL;
- char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL;
+ const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL, *tags = NULL;
+ char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL, *tags_end = NULL;
s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '|');
if(name == name_end) {
@@ -679,10 +709,15 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@');
if(likely(*s == '|' || *s == '@')) {
- s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '\r', '\n');
+ s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '|', '#');
if(*sampling == '@') sampling++;
}
+ if(likely(*s == '|' || *s == '#')) {
+ s = tags_end = (char *) statsd_parse_skip_up_to(tags = ++s, '|', '|');
+ if(*tags == '#') tags++;
+ }
+
// skip everything until the end of the line
while(*s && *s != '\n') s++;
@@ -700,6 +735,7 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
, statsd_parse_field_trim(value, value_end)
, statsd_parse_field_trim(type, type_end)
, statsd_parse_field_trim(sampling, sampling_end)
+ , statsd_parse_field_trim(tags, tags_end)
);
}
@@ -997,7 +1033,7 @@ void *statsd_collector_thread(void *ptr) {
#define STATSD_CONF_LINE_MAX 8192
-static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *path, const char *filename) {
+static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *filename) {
if(!type || !*type) type = "last";
if(!strcmp(type, "events")) return STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS;
@@ -1010,7 +1046,7 @@ static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t
else if(!strcmp(type, "stddev")) return STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV;
else if(!strcmp(type, "percentile")) return STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE;
- error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename);
+ error("STATSD: invalid type '%s' at line %zu of file '%s'. Using 'last'.", type, line, filename);
return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
}
@@ -1076,19 +1112,14 @@ static STATSD_APP_CHART_DIM *add_dimension_to_app_chart(
return dim;
}
-static int statsd_readfile(const char *path, const char *filename, STATSD_APP *app, STATSD_APP_CHART *chart, DICTIONARY *dict) {
- debug(D_STATSD, "STATSD configuration reading file '%s/%s'", path, filename);
+static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHART *chart, DICTIONARY *dict) {
+ debug(D_STATSD, "STATSD configuration reading file '%s'", filename);
char *buffer = mallocz(STATSD_CONF_LINE_MAX + 1);
- if(filename[0] == '/')
- strncpyz(buffer, filename, STATSD_CONF_LINE_MAX);
- else
- snprintfz(buffer, STATSD_CONF_LINE_MAX, "%s/%s", path, filename);
-
- FILE *fp = fopen(buffer, "r");
+ FILE *fp = fopen(filename, "r");
if(!fp) {
- error("STATSD: cannot open file '%s'.", buffer);
+ error("STATSD: cannot open file '%s'.", filename);
freez(buffer);
return -1;
}
@@ -1101,18 +1132,31 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
s = trim(buffer);
if (!s || *s == '#') {
- debug(D_STATSD, "STATSD: ignoring line %zu of file '%s/%s', it is empty.", line, path, filename);
+ debug(D_STATSD, "STATSD: ignoring line %zu of file '%s', it is empty.", line, filename);
continue;
}
- debug(D_STATSD, "STATSD: processing line %zu of file '%s/%s': %s", line, path, filename, buffer);
+ debug(D_STATSD, "STATSD: processing line %zu of file '%s': %s", line, filename, buffer);
if(*s == 'i' && strncmp(s, "include", 7) == 0) {
s = trim(&s[7]);
- if(s && *s)
- statsd_readfile(path, s, app, chart, dict);
+ if(s && *s) {
+ char *tmp;
+ if(*s == '/')
+ tmp = strdupz(s);
+ else {
+ // the file to be included is relative to current file
+ // find the directory name from the file we already read
+ char *filename2 = strdupz(filename); // copy filename, since dirname() will change it
+ char *dir = dirname(filename2); // find the directory part of the filename
+ tmp = strdupz_path_subpath(dir, s); // compose the new filename to read;
+ freez(filename2); // free the filename we copied
+ }
+ statsd_readfile(tmp, app, chart, dict);
+ freez(tmp);
+ }
else
- error("STATSD: ignoring line %zu of file '%s/%s', include filename is empty", line, path, s);
+ error("STATSD: ignoring line %zu of file '%s', include filename is empty", line, filename);
continue;
}
@@ -1160,7 +1204,7 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
chart->context = strdupz(s);
chart->family = strdupz("overview");
chart->units = strdupz("value");
- chart->priority = STATSD_CHART_PRIORITY;
+ chart->priority = NETDATA_CHART_PRIO_STATSD_PRIVATE;
chart->chart_type = RRDSET_TYPE_LINE;
chart->next = app->charts;
@@ -1174,20 +1218,20 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
}
}
else
- error("STATSD: ignoring line %zu ('%s') of file '%s/%s', [app] is not defined.", line, s, path, filename);
+ error("STATSD: ignoring line %zu ('%s') of file '%s', [app] is not defined.", line, s, filename);
continue;
}
if(!app) {
- error("STATSD: ignoring line %zu ('%s') of file '%s/%s', it is outside all sections.", line, s, path, filename);
+ error("STATSD: ignoring line %zu ('%s') of file '%s', it is outside all sections.", line, s, filename);
continue;
}
char *name = s;
char *value = strchr(s, '=');
if(!value) {
- error("STATSD: ignoring line %zu ('%s') of file '%s/%s', there is no = in it.", line, s, path, filename);
+ error("STATSD: ignoring line %zu ('%s') of file '%s', there is no = in it.", line, s, filename);
continue;
}
*value = '\0';
@@ -1197,11 +1241,11 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
value = trim(value);
if(!name || *name == '#') {
- error("STATSD: ignoring line %zu of file '%s/%s', name is empty.", line, path, filename);
+ error("STATSD: ignoring line %zu of file '%s', name is empty.", line, filename);
continue;
}
if(!value) {
- debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s/%s', value is empty.", line, path, filename);
+ debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s', value is empty.", line, filename);
continue;
}
@@ -1241,7 +1285,7 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
app->rrd_history_entries = 5;
}
else {
- error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [app] section.", line, name, path, filename);
+ error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [app] section.", line, name, filename);
continue;
}
}
@@ -1326,14 +1370,14 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
, (multipler && *multipler)?str2l(multipler):1
, (divisor && *divisor)?str2l(divisor):1
, flags
- , string2valuetype(type, line, path, filename)
+ , string2valuetype(type, line, filename)
);
if(pattern)
dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT);
}
else {
- error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id);
+ error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [%s] section.", line, name, filename, chart->id);
continue;
}
}
@@ -1344,49 +1388,13 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
return 0;
}
-static void statsd_readdir(const char *path) {
- size_t pathlen = strlen(path);
-
- debug(D_STATSD, "STATSD configuration reading directory '%s'", path);
-
- DIR *dir = opendir(path);
- if (!dir) {
- error("STATSD configuration cannot open directory '%s'.", path);
- return;
- }
-
- struct dirent *de = NULL;
- while ((de = readdir(dir))) {
- size_t len = strlen(de->d_name);
-
- if(de->d_type == DT_DIR
- && (
- (de->d_name[0] == '.' && de->d_name[1] == '\0')
- || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
- )) {
- debug(D_STATSD, "STATSD: ignoring directory '%s'", de->d_name);
- continue;
- }
-
- else if(de->d_type == DT_DIR) {
- char *s = mallocz(pathlen + strlen(de->d_name) + 2);
- strcpy(s, path);
- strcat(s, "/");
- strcat(s, de->d_name);
- statsd_readdir(s);
- freez(s);
- continue;
- }
-
- else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
- len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
- statsd_readfile(path, de->d_name, NULL, NULL, NULL);
- }
-
- else debug(D_STATSD, "STATSD: ignoring file '%s'", de->d_name);
- }
+static int statsd_file_callback(const char *filename, void *data) {
+ (void)data;
+ return statsd_readfile(filename, NULL, NULL, NULL);
+}
- closedir(dir);
+static inline void statsd_readdir(const char *user_path, const char *stock_path, const char *subpath) {
+ recursive_config_double_dir_load(user_path, stock_path, subpath, statsd_file_callback, NULL, 0);
}
// --------------------------------------------------------------------------------------------------------------------
@@ -1445,7 +1453,7 @@ static inline RRDSET *statsd_private_rrdset_create(
, context // context
, title // title
, units // units
- , "statsd" // plugin
+ , PLUGIN_STATSD_NAME // plugin
, "private_chart" // module
, priority // priority
, update_every // update every
@@ -1484,7 +1492,7 @@ static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
, context // context
, title // title
, "value" // units
- , STATSD_CHART_PRIORITY
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
, statsd.update_every
, RRDSET_TYPE_LINE
);
@@ -1526,7 +1534,7 @@ static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const
, context // context
, title // title
, "events/s" // units
- , STATSD_CHART_PRIORITY
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
, statsd.update_every
, RRDSET_TYPE_AREA
);
@@ -1568,7 +1576,7 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) {
, context // context
, title // title
, "entries" // units
- , STATSD_CHART_PRIORITY
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
, statsd.update_every
, RRDSET_TYPE_LINE
);
@@ -1610,7 +1618,7 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con
, context // context
, title // title
, units // units
- , STATSD_CHART_PRIORITY
+ , NETDATA_CHART_PRIO_STATSD_PRIVATE
, statsd.update_every
, RRDSET_TYPE_AREA
);
@@ -1649,14 +1657,14 @@ static inline void statsd_flush_gauge(STATSD_METRIC *m) {
debug(D_STATSD, "flushing gauge metric '%s'", m->name);
int updated = 0;
- if(m->count && !m->reset) {
+ if(unlikely(!m->reset && m->count)) {
m->last = (collected_number) (m->gauge.value * statsd.decimal_detail);
m->reset = 1;
updated = 1;
}
- if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
statsd_private_chart_gauge(m);
}
@@ -1664,14 +1672,14 @@ static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *d
debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
int updated = 0;
- if(m->count && !m->reset) {
+ if(unlikely(!m->reset && m->count)) {
m->last = m->counter.value;
m->reset = 1;
updated = 1;
}
- if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
statsd_private_chart_counter_or_meter(m, dim, family);
}
@@ -1687,40 +1695,27 @@ static inline void statsd_flush_set(STATSD_METRIC *m) {
debug(D_STATSD, "flushing set metric '%s'", m->name);
int updated = 0;
- if(m->count && !m->reset) {
+ if(unlikely(!m->reset && m->count)) {
m->last = (collected_number)m->set.unique;
m->reset = 1;
updated = 1;
}
+ else {
+ m->last = 0;
+ }
- if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
statsd_private_chart_set(m);
}
static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
- netdata_mutex_lock(&m->histogram.ext->mutex);
-
- if(unlikely(!m->histogram.ext->zeroed)) {
- // reset the metrics
- // if we collected anything, they will be updated below
- // this ensures that we report zeros if nothing is collected
-
- m->histogram.ext->last_min = 0;
- m->histogram.ext->last_max = 0;
- m->last = 0;
- m->histogram.ext->last_median = 0;
- m->histogram.ext->last_stddev = 0;
- m->histogram.ext->last_sum = 0;
- m->histogram.ext->last_percentile = 0;
-
- m->histogram.ext->zeroed = 1;
- }
-
int updated = 0;
- if(m->count && !m->reset && m->histogram.ext->used > 0) {
+ if(unlikely(!m->reset && m->count && m->histogram.ext->used > 0)) {
+ netdata_mutex_lock(&m->histogram.ext->mutex);
+
size_t len = m->histogram.ext->used;
LONG_DOUBLE *series = m->histogram.ext->values;
sort_series(series, len);
@@ -1738,6 +1733,8 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char
else
m->histogram.ext->last_percentile = (collected_number)roundl(series[pct_len - 1] * statsd.decimal_detail);
+ netdata_mutex_unlock(&m->histogram.ext->mutex);
+
debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT,
dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum);
@@ -1745,11 +1742,24 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char
m->reset = 1;
updated = 1;
}
+ else if(unlikely(!m->histogram.ext->zeroed)) {
+ // reset the metrics
+ // if we collected anything, they will be updated below
+ // this ensures that we report zeros if nothing is collected
- if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
- statsd_private_chart_timer_or_histogram(m, dim, family, units);
+ m->histogram.ext->last_min = 0;
+ m->histogram.ext->last_max = 0;
+ m->last = 0;
+ m->histogram.ext->last_median = 0;
+ m->histogram.ext->last_stddev = 0;
+ m->histogram.ext->last_sum = 0;
+ m->histogram.ext->last_percentile = 0;
- netdata_mutex_unlock(&m->histogram.ext->mutex);
+ m->histogram.ext->zeroed = 1;
+ }
+
+ if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+ statsd_private_chart_timer_or_histogram(m, dim, family, units);
}
static inline void statsd_flush_timer(STATSD_METRIC *m) {
@@ -1837,6 +1847,7 @@ static inline void link_metric_to_app_dimension(STATSD_APP *app, STATSD_METRIC *
}
chart->dimensions_linked_count++;
+ m->options |= STATSD_METRIC_OPTION_USED_IN_APPS;
debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm));
}
@@ -1980,7 +1991,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
, chart->context // context
, chart->title // title
, chart->units // units
- , "statsd" // plugin
+ , PLUGIN_STATSD_NAME // plugin
, chart->source // module
, chart->priority // priority
, statsd.update_every // update every
@@ -2043,7 +2054,13 @@ const char *statsd_metric_type_string(STATSD_METRIC_TYPE type) {
static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_metric)(STATSD_METRIC *)) {
STATSD_METRIC *m;
+
+ // find the useful metrics (incremental = each time we are called, we check the new metrics only)
for(m = index->first; m ; m = m->next) {
+ // since we add new metrics at the beginning
+ // check for useful charts, until the point we last checked
+ if(unlikely(is_metric_checked(m))) break;
+
if(unlikely(!(m->options & STATSD_METRIC_OPTION_CHECKED_IN_APPS))) {
log_access("NEW STATSD METRIC '%s': '%s'", statsd_metric_type_string(m->type), m->name);
check_if_metric_is_for_app(index, m);
@@ -2051,7 +2068,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
}
if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) {
- if(statsd.private_charts >= statsd.max_private_charts_hard) {
+ if(unlikely(statsd.private_charts >= statsd.max_private_charts_hard)) {
debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name);
info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts);
m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
@@ -2069,6 +2086,20 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED;
}
+ // mark it as checked
+ m->options |= STATSD_METRIC_OPTION_CHECKED;
+
+ // check if it is used in charts
+ if((m->options & (STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED|STATSD_METRIC_OPTION_USED_IN_APPS)) && !(m->options & STATSD_METRIC_OPTION_USEFUL)) {
+ m->options |= STATSD_METRIC_OPTION_USEFUL;
+ index->useful++;
+ m->next_useful = index->first_useful;
+ index->first_useful = m;
+ }
+ }
+
+ // flush all the useful metrics
+ for(m = index->first_useful; m ; m = m->next_useful) {
flush_metric(m);
}
}
@@ -2130,9 +2161,9 @@ void *statsd_main(void *ptr) {
statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts * 5);
statsd.private_charts_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_STATSD, "private charts memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
- statsd.decimal_detail = (size_t)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
+ statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);
- statsd.private_charts_hidden = (int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);
+ statsd.private_charts_hidden = (unsigned int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);
statsd.histogram_percentile = (double)config_get_float(CONFIG_SECTION_STATSD, "histograms and timers percentile (percentThreshold)", statsd.histogram_percentile);
if(isless(statsd.histogram_percentile, 0) || isgreater(statsd.histogram_percentile, 100)) {
@@ -2186,11 +2217,7 @@ void *statsd_main(void *ptr) {
#endif
// read custom application definitions
- {
- char filename[FILENAME_MAX + 1];
- snprintfz(filename, FILENAME_MAX, "%s/statsd.d", netdata_configured_config_dir);
- statsd_readdir(filename);
- }
+ statsd_readdir(netdata_configured_user_config_dir, netdata_configured_stock_config_dir, "statsd.d");
// ----------------------------------------------------------------------------------------------------------------
// statsd setup
@@ -2224,7 +2251,7 @@ void *statsd_main(void *ptr) {
, NULL
, "Metrics in the netdata statsd database"
, "metrics"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132010
, statsd.update_every
@@ -2237,6 +2264,27 @@ void *statsd_main(void *ptr) {
RRDDIM *rd_metrics_histogram = rrddim_add(st_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
RRDDIM *rd_metrics_set = rrddim_add(st_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDSET *st_useful_metrics = rrdset_create_localhost(
+ "netdata"
+ , "statsd_useful_metrics"
+ , NULL
+ , "statsd"
+ , NULL
+ , "Useful metrics in the netdata statsd database"
+ , "metrics"
+ , PLUGIN_STATSD_NAME
+ , "stats"
+ , 132010
+ , statsd.update_every
+ , RRDSET_TYPE_STACKED
+ );
+ RRDDIM *rd_useful_metrics_gauge = rrddim_add(st_useful_metrics, "gauges", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_useful_metrics_counter = rrddim_add(st_useful_metrics, "counters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_useful_metrics_timer = rrddim_add(st_useful_metrics, "timers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_useful_metrics_meter = rrddim_add(st_useful_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_useful_metrics_histogram = rrddim_add(st_useful_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+ RRDDIM *rd_useful_metrics_set = rrddim_add(st_useful_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
RRDSET *st_events = rrdset_create_localhost(
"netdata"
, "statsd_events"
@@ -2245,7 +2293,7 @@ void *statsd_main(void *ptr) {
, NULL
, "Events processed by the netdata statsd server"
, "events/s"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132011
, statsd.update_every
@@ -2268,7 +2316,7 @@ void *statsd_main(void *ptr) {
, NULL
, "Read operations made by the netdata statsd server"
, "reads/s"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132012
, statsd.update_every
@@ -2285,7 +2333,7 @@ void *statsd_main(void *ptr) {
, NULL
, "Bytes read by the netdata statsd server"
, "kilobits/s"
- , "netdata"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132013
, statsd.update_every
@@ -2302,7 +2350,7 @@ void *statsd_main(void *ptr) {
, NULL
, "Network packets processed by the netdata statsd server"
, "packets/s"
- , "netdata"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132014
, statsd.update_every
@@ -2319,7 +2367,7 @@ void *statsd_main(void *ptr) {
, NULL
, "statsd server TCP connects and disconnects"
, "events"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132015
, statsd.update_every
@@ -2336,7 +2384,7 @@ void *statsd_main(void *ptr) {
, NULL
, "statsd server TCP connected sockets"
, "connected"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132016
, statsd.update_every
@@ -2352,7 +2400,7 @@ void *statsd_main(void *ptr) {
, NULL
, "Private metric charts created by the netdata statsd server"
, "charts"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132020
, statsd.update_every
@@ -2368,7 +2416,7 @@ void *statsd_main(void *ptr) {
, "netdata.statsd_cpu"
, "NetData statsd charting thread CPU usage"
, "milliseconds/s"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132001
, statsd.update_every
@@ -2394,7 +2442,7 @@ void *statsd_main(void *ptr) {
, "netdata.statsd_cpu"
, title
, "milliseconds/s"
- , "statsd"
+ , PLUGIN_STATSD_NAME
, "stats"
, 132002 + i
, statsd.update_every
@@ -2430,6 +2478,7 @@ void *statsd_main(void *ptr) {
if(likely(hb_dt)) {
rrdset_next(st_metrics);
+ rrdset_next(st_useful_metrics);
rrdset_next(st_events);
rrdset_next(st_reads);
rrdset_next(st_bytes);
@@ -2450,6 +2499,14 @@ void *statsd_main(void *ptr) {
rrddim_set_by_pointer(st_metrics, rd_metrics_set, (collected_number)statsd.sets.metrics);
rrdset_done(st_metrics);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_gauge, (collected_number)statsd.gauges.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_counter, (collected_number)statsd.counters.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_timer, (collected_number)statsd.timers.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_meter, (collected_number)statsd.meters.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_histogram, (collected_number)statsd.histograms.useful);
+ rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_set, (collected_number)statsd.sets.useful);
+ rrdset_done(st_useful_metrics);
+
rrddim_set_by_pointer(st_events, rd_events_gauge, (collected_number)statsd.gauges.events);
rrddim_set_by_pointer(st_events, rd_events_counter, (collected_number)statsd.counters.events);
rrddim_set_by_pointer(st_events, rd_events_timer, (collected_number)statsd.timers.events);
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-05 16:56:16 +0000