New upstream version 1.11.0+dfsgupstream/1.11.0+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 215 insertions, 158 deletions

diff --git a/src/statsd.c b/collectors/statsd.plugin/statsd.c
index 44ebd8894..c92bfd1c2 100644
--- a/src/statsd.c
+++ b/collectors/statsd.plugin/statsd.c
@@ -1,7 +1,10 @@
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "statsd.h"
 
 #define STATSD_CHART_PREFIX "statsd"
-#define STATSD_CHART_PRIORITY 90000
+
+#define PLUGIN_STATSD_NAME "statsd.plugin"
 
 // --------------------------------------------------------------------------------------
 
@@ -87,7 +90,10 @@ typedef enum statsd_metric_options {
     STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED        = 0x00000002, // render a private chart for this metric
     STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED        = 0x00000004, // the metric has been checked if it should get private chart or not
     STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT        = 0x00000008, // show the count of events for this private chart
-    STATSD_METRIC_OPTION_CHECKED_IN_APPS              = 0x00000010, // set when this metric has been checked agains apps
+    STATSD_METRIC_OPTION_CHECKED_IN_APPS              = 0x00000010, // set when this metric has been checked against apps
+    STATSD_METRIC_OPTION_USED_IN_APPS                 = 0x00000020, // set when this metric is used in apps
+    STATSD_METRIC_OPTION_CHECKED                      = 0x00000040, // set when the charting thread checks this metric for use in charts (its usefulness)
+    STATSD_METRIC_OPTION_USEFUL                       = 0x00000080, // set when the charting thread finds the metric useful (i.e. used in a chart)
 } STATS_METRIC_OPTIONS;
 
 typedef enum statsd_metric_type {
@@ -101,7 +107,7 @@ typedef enum statsd_metric_type {
 
 
 typedef struct statsd_metric {
-    avl avl;                        // indexing
+    avl avl;                        // indexing - has to be first
 
     const char *name;               // the name of the metric
     uint32_t hash;                  // hash of the name
@@ -122,14 +128,15 @@ typedef struct statsd_metric {
 
     // chart related members
     STATS_METRIC_OPTIONS options;   // STATSD_METRIC_OPTION_* (bitfield)
-    char reset;                     // set to 1 to reset this metric to zero
+    char reset;                     // set to 1 by the charting thread to instruct the collector thread(s) to reset this metric
     collected_number last;          // the last value sent to netdata
-    RRDSET *st;                     // the chart of this metric
+    RRDSET *st;                     // the private chart of this metric
     RRDDIM *rd_value;               // the dimension of this metric value
     RRDDIM *rd_count;               // the dimension for the number of events received
 
     // linking, used for walking through all metrics
     struct statsd_metric *next;
+    struct statsd_metric *next_useful;
 } STATSD_METRIC;
 
 
@@ -140,10 +147,12 @@ typedef struct statsd_index {
     char *name;                     // the name of the index of metrics
     size_t events;                  // the number of events processed for this index
     size_t metrics;                 // the number of metrics in this index
+    size_t useful;                  // the number of useful metrics in this index
 
     STATSD_AVL_TREE index;          // the AVL tree
 
     STATSD_METRIC *first;           // the linked list of metrics (new metrics are added in front)
+    STATSD_METRIC *first_useful;    // the linked list of useful metrics (new metrics are added in front)
     STATSD_FIRST_PTR_MUTEX;         // when mutli-threading is enabled, a lock to protect the linked list
 
     STATS_METRIC_OPTIONS default_options;  // default options for all metrics in this index
@@ -255,13 +264,13 @@ static struct statsd {
     SIMPLE_PATTERN *charts_for;
 
     size_t tcp_idle_timeout;
-    size_t decimal_detail;
+    collected_number decimal_detail;
     size_t private_charts;
     size_t max_private_charts;
     size_t max_private_charts_hard;
     RRD_MEMORY_MODE private_charts_memory_mode;
     long private_charts_rrd_history_entries;
-    int private_charts_hidden;
+    unsigned int private_charts_hidden:1;
 
     STATSD_APP *apps;
     size_t recvmmsg_size;
@@ -344,6 +353,7 @@ static struct statsd {
         .threads = 0,
         .collection_threads_status = NULL,
         .sockets = {
+                .config          = &netdata_config,
                 .config_section  = CONFIG_SECTION_STATSD,
                 .default_bind_to = "udp:localhost tcp:localhost",
                 .default_port    = STATSD_LISTEN_PORT,
@@ -427,6 +437,13 @@ static inline LONG_DOUBLE statsd_parse_float(const char *v, LONG_DOUBLE def) {
     return value;
 }
 
+static inline LONG_DOUBLE statsd_parse_sampling_rate(const char *v) {
+    LONG_DOUBLE sampling_rate = statsd_parse_float(v, 1.0);
+    if(unlikely(isless(sampling_rate, 0.001))) sampling_rate = 0.001;
+    if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
+    return sampling_rate;
+}
+
 static inline long long statsd_parse_int(const char *v, long long def) {
     long long value;
 
@@ -455,7 +472,12 @@ static inline int value_is_zinit(const char *value) {
     return (value && *value == 'z' && *++value == 'i' && *++value == 'n' && *++value == 'i' && *++value == 't' && *++value == '\0');
 }
 
+#define is_metric_checked(m) ((m)->options & STATSD_METRIC_OPTION_CHECKED)
+#define is_metric_useful_for_collection(m) (!is_metric_checked(m) || ((m)->options & STATSD_METRIC_OPTION_USEFUL))
+
 static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, const char *sampling) {
+    if(!is_metric_useful_for_collection(m)) return;
+
     if(unlikely(!value || !*value)) {
         error("STATSD: metric '%s' of type gauge, with empty value is ignored.", m->name);
         return;
@@ -471,16 +493,18 @@ static inline void statsd_process_gauge(STATSD_METRIC *m, const char *value, con
     }
     else {
         if (unlikely(*value == '+' || *value == '-'))
-            m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+            m->gauge.value += statsd_parse_float(value, 1.0) / statsd_parse_sampling_rate(sampling);
         else
-            m->gauge.value = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+            m->gauge.value = statsd_parse_float(value, 1.0);
 
         m->events++;
         m->count++;
     }
 }
 
-static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, const char *sampling) {
+static inline void statsd_process_counter_or_meter(STATSD_METRIC *m, const char *value, const char *sampling) {
+    if(!is_metric_useful_for_collection(m)) return;
+
     // we accept empty values for counters
 
     if(unlikely(m->reset)) statsd_reset_metric(m);
@@ -489,21 +513,21 @@ static inline void statsd_process_counter(STATSD_METRIC *m, const char *value, c
         // magic loading of metric, without affecting anything
     }
     else {
-        m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_float(sampling, 1.0));
+        m->counter.value += llrintl((LONG_DOUBLE) statsd_parse_int(value, 1) / statsd_parse_sampling_rate(sampling));
 
         m->events++;
         m->count++;
     }
 }
 
-static inline void statsd_process_meter(STATSD_METRIC *m, const char *value, const char *sampling) {
-    // this is the same with the counter
-    statsd_process_counter(m, value, sampling);
-}
+#define statsd_process_counter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling)
+#define statsd_process_meter(m, value, sampling) statsd_process_counter_or_meter(m, value, sampling)
+
+static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const char *value, const char *sampling, const char *type) {
+    if(!is_metric_useful_for_collection(m)) return;
 
-static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value, const char *sampling) {
     if(unlikely(!value || !*value)) {
-        error("STATSD: metric '%s' of type histogram, with empty value is ignored.", m->name);
+        error("STATSD: metric of type %s, with empty value is ignored.", type);
         return;
     }
 
@@ -516,31 +540,35 @@ static inline void statsd_process_histogram(STATSD_METRIC *m, const char *value,
         // magic loading of metric, without affecting anything
     }
     else {
-        if (unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
-            netdata_mutex_lock(&m->histogram.ext->mutex);
-            m->histogram.ext->size += statsd.histogram_increase_step;
-            m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size);
-            netdata_mutex_unlock(&m->histogram.ext->mutex);
-        }
+        LONG_DOUBLE v = statsd_parse_float(value, 1.0);
+        LONG_DOUBLE sampling_rate = statsd_parse_sampling_rate(sampling);
+        if(unlikely(isless(sampling_rate, 0.01))) sampling_rate = 0.01;
+        if(unlikely(isgreater(sampling_rate, 1.0))) sampling_rate = 1.0;
+
+        long long samples = llrintl(1.0 / sampling_rate);
+        while(samples-- > 0) {
+
+            if(unlikely(m->histogram.ext->used == m->histogram.ext->size)) {
+                netdata_mutex_lock(&m->histogram.ext->mutex);
+                m->histogram.ext->size += statsd.histogram_increase_step;
+                m->histogram.ext->values = reallocz(m->histogram.ext->values, sizeof(LONG_DOUBLE) * m->histogram.ext->size);
+                netdata_mutex_unlock(&m->histogram.ext->mutex);
+            }
 
-        m->histogram.ext->values[m->histogram.ext->used++] = statsd_parse_float(value, 1.0) / statsd_parse_float(sampling, 1.0);
+            m->histogram.ext->values[m->histogram.ext->used++] = v;
+        }
 
         m->events++;
         m->count++;
     }
 }
 
-static inline void statsd_process_timer(STATSD_METRIC *m, const char *value, const char *sampling) {
-    if(unlikely(!value || !*value)) {
-        error("STATSD: metric of type timer, with empty value is ignored.");
-        return;
-    }
-
-    // timers are a use case of histogram
-    statsd_process_histogram(m, value, sampling);
-}
+#define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer")
+#define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram")
 
 static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
+    if(!is_metric_useful_for_collection(m)) return;
+
     if(unlikely(!value || !*value)) {
         error("STATSD: metric of type set, with empty value is ignored.");
         return;
@@ -578,8 +606,10 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
 // --------------------------------------------------------------------------------------------------------------------
 // statsd parsing
 
-static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling) {
-    debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', rate '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)");
+static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling, const char *tags) {
+    (void)tags;
+
+    debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', sampling '%s', tags '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)", tags?tags:"(null)");
 
     if(unlikely(!name || !*name)) return;
     if(unlikely(!type || !*type)) type = "m";
@@ -663,8 +693,8 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
 
     const char *s = buffer;
     while(*s) {
-        const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL;
-        char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL;
+        const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL, *tags = NULL;
+        char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL, *tags_end = NULL;
 
         s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '|');
         if(name == name_end) {
@@ -679,10 +709,15 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
             s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@');
 
         if(likely(*s == '|' || *s == '@')) {
-            s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '\r', '\n');
+            s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '|', '#');
             if(*sampling == '@') sampling++;
         }
 
+        if(likely(*s == '|' || *s == '#')) {
+            s = tags_end = (char *) statsd_parse_skip_up_to(tags = ++s, '|', '|');
+            if(*tags == '#') tags++;
+        }
+
         // skip everything until the end of the line
         while(*s && *s != '\n') s++;
 
@@ -700,6 +735,7 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
                 , statsd_parse_field_trim(value, value_end)
                 , statsd_parse_field_trim(type, type_end)
                 , statsd_parse_field_trim(sampling, sampling_end)
+                , statsd_parse_field_trim(tags, tags_end)
         );
     }
 
@@ -997,7 +1033,7 @@ void *statsd_collector_thread(void *ptr) {
 
 #define STATSD_CONF_LINE_MAX 8192
 
-static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *path, const char *filename) {
+static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t line, const char *filename) {
     if(!type || !*type) type = "last";
 
     if(!strcmp(type, "events")) return STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS;
@@ -1010,7 +1046,7 @@ static STATSD_APP_CHART_DIM_VALUE_TYPE string2valuetype(const char *type, size_t
     else if(!strcmp(type, "stddev")) return STATSD_APP_CHART_DIM_VALUE_TYPE_STDDEV;
     else if(!strcmp(type, "percentile")) return STATSD_APP_CHART_DIM_VALUE_TYPE_PERCENTILE;
 
-    error("STATSD: invalid type '%s' at line %zu of file '%s/%s'. Using 'last'.", type, line, path, filename);
+    error("STATSD: invalid type '%s' at line %zu of file '%s'. Using 'last'.", type, line, filename);
     return STATSD_APP_CHART_DIM_VALUE_TYPE_LAST;
 }
 
@@ -1076,19 +1112,14 @@ static STATSD_APP_CHART_DIM *add_dimension_to_app_chart(
     return dim;
 }
 
-static int statsd_readfile(const char *path, const char *filename, STATSD_APP *app, STATSD_APP_CHART *chart, DICTIONARY *dict) {
-    debug(D_STATSD, "STATSD configuration reading file '%s/%s'", path, filename);
+static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHART *chart, DICTIONARY *dict) {
+    debug(D_STATSD, "STATSD configuration reading file '%s'", filename);
 
     char *buffer = mallocz(STATSD_CONF_LINE_MAX + 1);
 
-    if(filename[0] == '/')
-        strncpyz(buffer, filename, STATSD_CONF_LINE_MAX);
-    else
-        snprintfz(buffer, STATSD_CONF_LINE_MAX, "%s/%s", path, filename);
-
-    FILE *fp = fopen(buffer, "r");
+    FILE *fp = fopen(filename, "r");
     if(!fp) {
-        error("STATSD: cannot open file '%s'.", buffer);
+        error("STATSD: cannot open file '%s'.", filename);
         freez(buffer);
         return -1;
     }
@@ -1101,18 +1132,31 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
 
         s = trim(buffer);
         if (!s || *s == '#') {
-            debug(D_STATSD, "STATSD: ignoring line %zu of file '%s/%s', it is empty.", line, path, filename);
+            debug(D_STATSD, "STATSD: ignoring line %zu of file '%s', it is empty.", line, filename);
             continue;
         }
 
-        debug(D_STATSD, "STATSD: processing line %zu of file '%s/%s': %s", line, path, filename, buffer);
+        debug(D_STATSD, "STATSD: processing line %zu of file '%s': %s", line, filename, buffer);
 
         if(*s == 'i' && strncmp(s, "include", 7) == 0) {
             s = trim(&s[7]);
-            if(s && *s)
-                statsd_readfile(path, s, app, chart, dict);
+            if(s && *s) {
+                char *tmp;
+                if(*s == '/')
+                    tmp = strdupz(s);
+                else {
+                    // the file to be included is relative to current file
+                    // find the directory name from the file we already read
+                    char *filename2 = strdupz(filename); // copy filename, since dirname() will change it
+                    char *dir = dirname(filename2);      // find the directory part of the filename
+                    tmp = strdupz_path_subpath(dir, s);  // compose the new filename to read;
+                    freez(filename2);                    // free the filename we copied
+                }
+                statsd_readfile(tmp, app, chart, dict);
+                freez(tmp);
+            }
             else
-                error("STATSD: ignoring line %zu of file '%s/%s', include filename is empty", line, path, s);
+                error("STATSD: ignoring line %zu of file '%s', include filename is empty", line, filename);
 
             continue;
         }
@@ -1160,7 +1204,7 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
                     chart->context    = strdupz(s);
                     chart->family     = strdupz("overview");
                     chart->units      = strdupz("value");
-                    chart->priority   = STATSD_CHART_PRIORITY;
+                    chart->priority   = NETDATA_CHART_PRIO_STATSD_PRIVATE;
                     chart->chart_type = RRDSET_TYPE_LINE;
 
                     chart->next = app->charts;
@@ -1174,20 +1218,20 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
                 }
             }
             else
-                error("STATSD: ignoring line %zu ('%s') of file '%s/%s', [app] is not defined.", line, s, path, filename);
+                error("STATSD: ignoring line %zu ('%s') of file '%s', [app] is not defined.", line, s, filename);
 
             continue;
         }
 
         if(!app) {
-            error("STATSD: ignoring line %zu ('%s') of file '%s/%s', it is outside all sections.", line, s, path, filename);
+            error("STATSD: ignoring line %zu ('%s') of file '%s', it is outside all sections.", line, s, filename);
             continue;
         }
 
         char *name = s;
         char *value = strchr(s, '=');
         if(!value) {
-            error("STATSD: ignoring line %zu ('%s') of file '%s/%s', there is no = in it.", line, s, path, filename);
+            error("STATSD: ignoring line %zu ('%s') of file '%s', there is no = in it.", line, s, filename);
             continue;
         }
         *value = '\0';
@@ -1197,11 +1241,11 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
         value = trim(value);
 
         if(!name || *name == '#') {
-            error("STATSD: ignoring line %zu of file '%s/%s', name is empty.", line, path, filename);
+            error("STATSD: ignoring line %zu of file '%s', name is empty.", line, filename);
             continue;
         }
         if(!value) {
-            debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s/%s', value is empty.", line, path, filename);
+            debug(D_CONFIG, "STATSD: ignoring line %zu of file '%s', value is empty.", line, filename);
             continue;
         }
 
@@ -1241,7 +1285,7 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
                     app->rrd_history_entries = 5;
             }
             else {
-                error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [app] section.", line, name, path, filename);
+                error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [app] section.", line, name, filename);
                 continue;
             }
         }
@@ -1326,14 +1370,14 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
                         , (multipler && *multipler)?str2l(multipler):1
                         , (divisor && *divisor)?str2l(divisor):1
                         , flags
-                        , string2valuetype(type, line, path, filename)
+                        , string2valuetype(type, line, filename)
                 );
 
                 if(pattern)
                     dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT);
             }
             else {
-                error("STATSD: ignoring line %zu ('%s') of file '%s/%s'. Unknown keyword for the [%s] section.", line, name, path, filename, chart->id);
+                error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [%s] section.", line, name, filename, chart->id);
                 continue;
             }
         }
@@ -1344,49 +1388,13 @@ static int statsd_readfile(const char *path, const char *filename, STATSD_APP *a
     return 0;
 }
 
-static void statsd_readdir(const char *path) {
-    size_t pathlen = strlen(path);
-
-    debug(D_STATSD, "STATSD configuration reading directory '%s'", path);
-
-    DIR *dir = opendir(path);
-    if (!dir) {
-        error("STATSD configuration cannot open directory '%s'.", path);
-        return;
-    }
-
-    struct dirent *de = NULL;
-    while ((de = readdir(dir))) {
-        size_t len = strlen(de->d_name);
-
-        if(de->d_type == DT_DIR
-           && (
-                   (de->d_name[0] == '.' && de->d_name[1] == '\0')
-                   || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
-           )) {
-            debug(D_STATSD, "STATSD: ignoring directory '%s'", de->d_name);
-            continue;
-        }
-
-        else if(de->d_type == DT_DIR) {
-            char *s = mallocz(pathlen + strlen(de->d_name) + 2);
-            strcpy(s, path);
-            strcat(s, "/");
-            strcat(s, de->d_name);
-            statsd_readdir(s);
-            freez(s);
-            continue;
-        }
-
-        else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
-                len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
-            statsd_readfile(path, de->d_name, NULL, NULL, NULL);
-        }
-
-        else debug(D_STATSD, "STATSD: ignoring file '%s'", de->d_name);
-    }
+static int statsd_file_callback(const char *filename, void *data) {
+    (void)data;
+    return statsd_readfile(filename, NULL, NULL, NULL);
+}
 
-    closedir(dir);
+static inline void statsd_readdir(const char *user_path, const char *stock_path, const char *subpath) {
+    recursive_config_double_dir_load(user_path, stock_path, subpath, statsd_file_callback, NULL, 0);
 }
 
 // --------------------------------------------------------------------------------------------------------------------
@@ -1445,7 +1453,7 @@ static inline RRDSET *statsd_private_rrdset_create(
             , context         // context
             , title           // title
             , units           // units
-            , "statsd"        // plugin
+            , PLUGIN_STATSD_NAME // plugin
             , "private_chart" // module
             , priority        // priority
             , update_every    // update every
@@ -1484,7 +1492,7 @@ static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
                 , context       // context
                 , title         // title
                 , "value"       // units
-                , STATSD_CHART_PRIORITY
+                , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_LINE
         );
@@ -1526,7 +1534,7 @@ static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const
                 , context       // context
                 , title         // title
                 , "events/s"    // units
-                , STATSD_CHART_PRIORITY
+                , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_AREA
         );
@@ -1568,7 +1576,7 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) {
                 , context       // context
                 , title         // title
                 , "entries"     // units
-                , STATSD_CHART_PRIORITY
+                , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_LINE
         );
@@ -1610,7 +1618,7 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con
                 , context       // context
                 , title         // title
                 , units         // units
-                , STATSD_CHART_PRIORITY
+                , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_AREA
         );
@@ -1649,14 +1657,14 @@ static inline void statsd_flush_gauge(STATSD_METRIC *m) {
     debug(D_STATSD, "flushing gauge metric '%s'", m->name);
 
     int updated = 0;
-    if(m->count && !m->reset) {
+    if(unlikely(!m->reset && m->count)) {
         m->last = (collected_number) (m->gauge.value * statsd.decimal_detail);
 
         m->reset = 1;
         updated = 1;
     }
 
-    if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+    if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
         statsd_private_chart_gauge(m);
 }
 
@@ -1664,14 +1672,14 @@ static inline void statsd_flush_counter_or_meter(STATSD_METRIC *m, const char *d
     debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
 
     int updated = 0;
-    if(m->count && !m->reset) {
+    if(unlikely(!m->reset && m->count)) {
         m->last = m->counter.value;
 
         m->reset = 1;
         updated = 1;
     }
 
-    if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+    if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
         statsd_private_chart_counter_or_meter(m, dim, family);
 }
 
@@ -1687,40 +1695,27 @@ static inline void statsd_flush_set(STATSD_METRIC *m) {
     debug(D_STATSD, "flushing set metric '%s'", m->name);
 
     int updated = 0;
-    if(m->count && !m->reset) {
+    if(unlikely(!m->reset && m->count)) {
         m->last = (collected_number)m->set.unique;
 
         m->reset = 1;
         updated = 1;
     }
+    else {
+        m->last = 0;
+    }
 
-    if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
+    if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
         statsd_private_chart_set(m);
 }
 
 static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
     debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
 
-    netdata_mutex_lock(&m->histogram.ext->mutex);
-
-    if(unlikely(!m->histogram.ext->zeroed)) {
-        // reset the metrics
-        // if we collected anything, they will be updated below
-        // this ensures that we report zeros if nothing is collected
-
-        m->histogram.ext->last_min = 0;
-        m->histogram.ext->last_max = 0;
-        m->last = 0;
-        m->histogram.ext->last_median = 0;
-        m->histogram.ext->last_stddev = 0;
-        m->histogram.ext->last_sum = 0;
-        m->histogram.ext->last_percentile = 0;
-
-        m->histogram.ext->zeroed = 1;
-    }
-
     int updated = 0;
-    if(m->count && !m->reset && m->histogram.ext->used > 0) {
+    if(unlikely(!m->reset && m->count && m->histogram.ext->used > 0)) {
+        netdata_mutex_lock(&m->histogram.ext->mutex);
+
         size_t len = m->histogram.ext->used;
         LONG_DOUBLE *series = m->histogram.ext->values;
         sort_series(series, len);
@@ -1738,6 +1733,8 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char
         else
             m->histogram.ext->last_percentile = (collected_number)roundl(series[pct_len - 1] * statsd.decimal_detail);
 
+        netdata_mutex_unlock(&m->histogram.ext->mutex);
+
         debug(D_STATSD, "STATSD %s metric %s: min " COLLECTED_NUMBER_FORMAT ", max " COLLECTED_NUMBER_FORMAT ", last " COLLECTED_NUMBER_FORMAT ", pcent " COLLECTED_NUMBER_FORMAT ", median " COLLECTED_NUMBER_FORMAT ", stddev " COLLECTED_NUMBER_FORMAT ", sum " COLLECTED_NUMBER_FORMAT,
               dim, m->name, m->histogram.ext->last_min, m->histogram.ext->last_max, m->last, m->histogram.ext->last_percentile, m->histogram.ext->last_median, m->histogram.ext->last_stddev, m->histogram.ext->last_sum);
 
@@ -1745,11 +1742,24 @@ static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char
         m->reset = 1;
         updated = 1;
     }
+    else if(unlikely(!m->histogram.ext->zeroed)) {
+        // reset the metrics
+        // if we collected anything, they will be updated below
+        // this ensures that we report zeros if nothing is collected
 
-    if(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED)))
-        statsd_private_chart_timer_or_histogram(m, dim, family, units);
+        m->histogram.ext->last_min = 0;
+        m->histogram.ext->last_max = 0;
+        m->last = 0;
+        m->histogram.ext->last_median = 0;
+        m->histogram.ext->last_stddev = 0;
+        m->histogram.ext->last_sum = 0;
+        m->histogram.ext->last_percentile = 0;
 
-    netdata_mutex_unlock(&m->histogram.ext->mutex);
+        m->histogram.ext->zeroed = 1;
+    }
+
+    if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+        statsd_private_chart_timer_or_histogram(m, dim, family, units);
 }
 
 static inline void statsd_flush_timer(STATSD_METRIC *m) {
@@ -1837,6 +1847,7 @@ static inline void link_metric_to_app_dimension(STATSD_APP *app, STATSD_METRIC *
     }
 
     chart->dimensions_linked_count++;
+    m->options |= STATSD_METRIC_OPTION_USED_IN_APPS;
     debug(D_STATSD, "metric '%s' of type %u linked with app '%s', chart '%s', dimension '%s', algorithm '%s'", m->name, m->type, app->name, chart->id, dim->name, rrd_algorithm_name(dim->algorithm));
 }
 
@@ -1980,7 +1991,7 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
                 , chart->context            // context
                 , chart->title              // title
                 , chart->units              // units
-                , "statsd"                  // plugin
+                , PLUGIN_STATSD_NAME        // plugin
                 , chart->source             // module
                 , chart->priority           // priority
                 , statsd.update_every       // update every
@@ -2043,7 +2054,13 @@ const char *statsd_metric_type_string(STATSD_METRIC_TYPE type) {
 
 static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_metric)(STATSD_METRIC *)) {
     STATSD_METRIC *m;
+
+    // find the useful metrics (incremental = each time we are called, we check the new metrics only)
     for(m = index->first; m ; m = m->next) {
+        // since we add new metrics at the beginning
+        // check for useful charts, until the point we last checked
+        if(unlikely(is_metric_checked(m))) break;
+
         if(unlikely(!(m->options & STATSD_METRIC_OPTION_CHECKED_IN_APPS))) {
             log_access("NEW STATSD METRIC '%s': '%s'", statsd_metric_type_string(m->type), m->name);
             check_if_metric_is_for_app(index, m);
@@ -2051,7 +2068,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
         }
 
         if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) {
-            if(statsd.private_charts >= statsd.max_private_charts_hard) {
+            if(unlikely(statsd.private_charts >= statsd.max_private_charts_hard)) {
                 debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name);
                 info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts);
                 m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
@@ -2069,6 +2086,20 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
             m->options |= STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED;
         }
 
+        // mark it as checked
+        m->options |= STATSD_METRIC_OPTION_CHECKED;
+
+        // check if it is used in charts
+        if((m->options & (STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED|STATSD_METRIC_OPTION_USED_IN_APPS)) && !(m->options & STATSD_METRIC_OPTION_USEFUL)) {
+            m->options |= STATSD_METRIC_OPTION_USEFUL;
+            index->useful++;
+            m->next_useful = index->first_useful;
+            index->first_useful = m;
+        }
+    }
+
+    // flush all the useful metrics
+    for(m = index->first_useful; m ; m = m->next_useful) {
         flush_metric(m);
     }
 }
@@ -2130,9 +2161,9 @@ void *statsd_main(void *ptr) {
     statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts * 5);
     statsd.private_charts_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_STATSD, "private charts memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
     statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
-    statsd.decimal_detail = (size_t)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
+    statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
     statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);
-    statsd.private_charts_hidden = (int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);
+    statsd.private_charts_hidden = (unsigned int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);
 
     statsd.histogram_percentile = (double)config_get_float(CONFIG_SECTION_STATSD, "histograms and timers percentile (percentThreshold)", statsd.histogram_percentile);
     if(isless(statsd.histogram_percentile, 0) || isgreater(statsd.histogram_percentile, 100)) {
@@ -2186,11 +2217,7 @@ void *statsd_main(void *ptr) {
 #endif
 
     // read custom application definitions
-    {
-        char filename[FILENAME_MAX + 1];
-        snprintfz(filename, FILENAME_MAX, "%s/statsd.d", netdata_configured_config_dir);
-        statsd_readdir(filename);
-    }
+    statsd_readdir(netdata_configured_user_config_dir, netdata_configured_stock_config_dir, "statsd.d");
 
     // ----------------------------------------------------------------------------------------------------------------
     // statsd setup
@@ -2224,7 +2251,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "Metrics in the netdata statsd database"
             , "metrics"
-            , "statsd"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132010
             , statsd.update_every
@@ -2237,6 +2264,27 @@ void *statsd_main(void *ptr) {
     RRDDIM *rd_metrics_histogram = rrddim_add(st_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
     RRDDIM *rd_metrics_set       = rrddim_add(st_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
 
+    RRDSET *st_useful_metrics = rrdset_create_localhost(
+            "netdata"
+            , "statsd_useful_metrics"
+            , NULL
+            , "statsd"
+            , NULL
+            , "Useful metrics in the netdata statsd database"
+            , "metrics"
+            , PLUGIN_STATSD_NAME
+            , "stats"
+            , 132010
+            , statsd.update_every
+            , RRDSET_TYPE_STACKED
+    );
+    RRDDIM *rd_useful_metrics_gauge     = rrddim_add(st_useful_metrics, "gauges", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    RRDDIM *rd_useful_metrics_counter   = rrddim_add(st_useful_metrics, "counters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    RRDDIM *rd_useful_metrics_timer     = rrddim_add(st_useful_metrics, "timers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    RRDDIM *rd_useful_metrics_meter     = rrddim_add(st_useful_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    RRDDIM *rd_useful_metrics_histogram = rrddim_add(st_useful_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    RRDDIM *rd_useful_metrics_set       = rrddim_add(st_useful_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+
     RRDSET *st_events = rrdset_create_localhost(
             "netdata"
             , "statsd_events"
@@ -2245,7 +2293,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "Events processed by the netdata statsd server"
             , "events/s"
-            , "statsd"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132011
             , statsd.update_every
@@ -2268,7 +2316,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "Read operations made by the netdata statsd server"
             , "reads/s"
-            , "statsd"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132012
             , statsd.update_every
@@ -2285,7 +2333,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "Bytes read by the netdata statsd server"
             , "kilobits/s"
-            , "netdata"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132013
             , statsd.update_every
@@ -2302,7 +2350,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "Network packets processed by the netdata statsd server"
             , "packets/s"
-            , "netdata"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132014
             , statsd.update_every
@@ -2319,7 +2367,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "statsd server TCP connects and disconnects"
             , "events"
-            , "statsd"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132015
             , statsd.update_every
@@ -2336,7 +2384,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "statsd server TCP connected sockets"
             , "connected"
-            , "statsd"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132016
             , statsd.update_every
@@ -2352,7 +2400,7 @@ void *statsd_main(void *ptr) {
             , NULL
             , "Private metric charts created by the netdata statsd server"
             , "charts"
-            , "statsd"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132020
             , statsd.update_every
@@ -2368,7 +2416,7 @@ void *statsd_main(void *ptr) {
             , "netdata.statsd_cpu"
             , "NetData statsd charting thread CPU usage"
             , "milliseconds/s"
-            , "statsd"
+            , PLUGIN_STATSD_NAME
             , "stats"
             , 132001
             , statsd.update_every
@@ -2394,7 +2442,7 @@ void *statsd_main(void *ptr) {
                 , "netdata.statsd_cpu"
                 , title
                 , "milliseconds/s"
-                , "statsd"
+                , PLUGIN_STATSD_NAME
                 , "stats"
                 , 132002 + i
                 , statsd.update_every
@@ -2430,6 +2478,7 @@ void *statsd_main(void *ptr) {
 
         if(likely(hb_dt)) {
             rrdset_next(st_metrics);
+            rrdset_next(st_useful_metrics);
             rrdset_next(st_events);
             rrdset_next(st_reads);
             rrdset_next(st_bytes);
@@ -2450,6 +2499,14 @@ void *statsd_main(void *ptr) {
         rrddim_set_by_pointer(st_metrics, rd_metrics_set,          (collected_number)statsd.sets.metrics);
         rrdset_done(st_metrics);
 
+        rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_gauge,        (collected_number)statsd.gauges.useful);
+        rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_counter,      (collected_number)statsd.counters.useful);
+        rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_timer,        (collected_number)statsd.timers.useful);
+        rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_meter,        (collected_number)statsd.meters.useful);
+        rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_histogram,    (collected_number)statsd.histograms.useful);
+        rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_set,          (collected_number)statsd.sets.useful);
+        rrdset_done(st_useful_metrics);
+
         rrddim_set_by_pointer(st_events,  rd_events_gauge,         (collected_number)statsd.gauges.events);
         rrddim_set_by_pointer(st_events,  rd_events_counter,       (collected_number)statsd.counters.events);
         rrddim_set_by_pointer(st_events,  rd_events_timer,         (collected_number)statsd.timers.events);