Adding upstream version 1.35.0.upstream/1.35.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 522 insertions, 277 deletions

diff --git a/collectors/statsd.plugin/statsd.c b/collectors/statsd.plugin/statsd.c
index a630d00d0..63e3316cb 100644
--- a/collectors/statsd.plugin/statsd.c
+++ b/collectors/statsd.plugin/statsd.c
@@ -9,31 +9,24 @@
 #define STATSD_LISTEN_PORT 8125
 #define STATSD_LISTEN_BACKLOG 4096
 
+#define WORKER_JOB_TYPE_TCP_CONNECTED 0
+#define WORKER_JOB_TYPE_TCP_DISCONNECTED 1
+#define WORKER_JOB_TYPE_RCV_DATA 2
+#define WORKER_JOB_TYPE_SND_DATA 3
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 4
+#error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least 4
+#endif
+
 // --------------------------------------------------------------------------------------
 
 // #define STATSD_MULTITHREADED 1
 
 #ifdef STATSD_MULTITHREADED
 // DO NOT ENABLE MULTITHREADING - IT IS NOT WELL TESTED
-#define STATSD_AVL_TREE avl_tree_lock
-#define STATSD_AVL_INSERT avl_insert_lock
-#define STATSD_AVL_SEARCH avl_search_lock
-#define STATSD_AVL_INDEX_INIT { .avl_tree = { NULL, statsd_metric_compare }, .rwlock = AVL_LOCK_INITIALIZER }
-#define STATSD_FIRST_PTR_MUTEX netdata_mutex_t first_mutex
-#define STATSD_FIRST_PTR_MUTEX_INIT .first_mutex = NETDATA_MUTEX_INITIALIZER
-#define STATSD_FIRST_PTR_MUTEX_LOCK(index) netdata_mutex_lock(&((index)->first_mutex))
-#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index) netdata_mutex_unlock(&((index)->first_mutex))
-#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DEFAULT
+#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DONT_OVERWRITE_VALUE|DICTIONARY_FLAG_ADD_IN_FRONT
 #else
-#define STATSD_AVL_TREE avl_tree_type
-#define STATSD_AVL_INSERT avl_insert
-#define STATSD_AVL_SEARCH avl_search
-#define STATSD_AVL_INDEX_INIT { .root = NULL, .compar = statsd_metric_compare }
-#define STATSD_FIRST_PTR_MUTEX
-#define STATSD_FIRST_PTR_MUTEX_INIT
-#define STATSD_FIRST_PTR_MUTEX_LOCK(index)
-#define STATSD_FIRST_PTR_MUTEX_UNLOCK(index)
-#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_SINGLE_THREADED
+#define STATSD_DICTIONARY_OPTIONS DICTIONARY_FLAG_DONT_OVERWRITE_VALUE|DICTIONARY_FLAG_ADD_IN_FRONT|DICTIONARY_FLAG_SINGLE_THREADED
 #endif
 
 #define STATSD_DECIMAL_DETAIL 1000 // floating point values get multiplied by this, with the same divisor
@@ -67,7 +60,7 @@ typedef struct statsd_histogram_extensions {
     RRDDIM *rd_percentile;
     RRDDIM *rd_median;
     RRDDIM *rd_stddev;
-    RRDDIM *rd_sum;
+    //RRDDIM *rd_sum;
 
     size_t size;
     size_t used;
@@ -83,6 +76,16 @@ typedef struct statsd_metric_set {
     size_t unique;
 } STATSD_METRIC_SET;
 
+typedef struct statsd_metric_dictionary_item {
+    size_t count;
+    RRDDIM *rd;
+} STATSD_METRIC_DICTIONARY_ITEM;
+
+typedef struct statsd_metric_dictionary {
+    DICTIONARY *dict;
+    size_t unique;
+} STATSD_METRIC_DICTIONARY;
+
 
 // --------------------------------------------------------------------------------------------------------------------
 // this is a metric - for all types of metrics
@@ -97,6 +100,7 @@ typedef enum statsd_metric_options {
     STATSD_METRIC_OPTION_USED_IN_APPS                 = 0x00000020, // set when this metric is used in apps
     STATSD_METRIC_OPTION_CHECKED                      = 0x00000040, // set when the charting thread checks this metric for use in charts (its usefulness)
     STATSD_METRIC_OPTION_USEFUL                       = 0x00000080, // set when the charting thread finds the metric useful (i.e. used in a chart)
+    STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED       = 0x00000100, // set when the collection is full for this metric
 } STATS_METRIC_OPTIONS;
 
 typedef enum statsd_metric_type {
@@ -105,14 +109,13 @@ typedef enum statsd_metric_type {
     STATSD_METRIC_TYPE_METER,
     STATSD_METRIC_TYPE_TIMER,
     STATSD_METRIC_TYPE_HISTOGRAM,
-    STATSD_METRIC_TYPE_SET
+    STATSD_METRIC_TYPE_SET,
+    STATSD_METRIC_TYPE_DICTIONARY
 } STATSD_METRIC_TYPE;
 
 
 typedef struct statsd_metric {
-    avl_t avl;                      // indexing - has to be first
-
-    const char *name;               // the name of the metric
+    const char *name;               // the name of the metric - linked to dictionary name
     uint32_t hash;                  // hash of the name
 
     STATSD_METRIC_TYPE type;
@@ -127,8 +130,13 @@ typedef struct statsd_metric {
         STATSD_METRIC_COUNTER counter;
         STATSD_METRIC_HISTOGRAM histogram;
         STATSD_METRIC_SET set;
+        STATSD_METRIC_DICTIONARY dictionary;
     };
 
+    char *units;
+    char *dimname;
+    char *family;
+
     // chart related members
     STATS_METRIC_OPTIONS options;   // STATSD_METRIC_OPTION_* (bitfield)
     char reset;                     // set to 1 by the charting thread to instruct the collector thread(s) to reset this metric
@@ -138,7 +146,6 @@ typedef struct statsd_metric {
     RRDDIM *rd_count;               // the dimension for the number of events received
 
     // linking, used for walking through all metrics
-    struct statsd_metric *next;
     struct statsd_metric *next_useful;
 } STATSD_METRIC;
 
@@ -152,17 +159,14 @@ typedef struct statsd_index {
     size_t metrics;                 // the number of metrics in this index
     size_t useful;                  // the number of useful metrics in this index
 
-    STATSD_AVL_TREE index;          // the AVL tree
+    STATSD_METRIC_TYPE type;        // the type of index
+    DICTIONARY *dict;
 
-    STATSD_METRIC *first;           // the linked list of metrics (new metrics are added in front)
     STATSD_METRIC *first_useful;    // the linked list of useful metrics (new metrics are added in front)
-    STATSD_FIRST_PTR_MUTEX;         // when multi-threading is enabled, a lock to protect the linked list
 
     STATS_METRIC_OPTIONS default_options;  // default options for all metrics in this index
 } STATSD_INDEX;
 
-static int statsd_metric_compare(void* a, void* b);
-
 // --------------------------------------------------------------------------------------------------------------------
 // synthetic charts
 
@@ -237,10 +241,6 @@ struct collection_thread_status {
     size_t max_sockets;
 
     netdata_thread_t thread;
-    struct rusage rusage;
-    RRDSET *st_cpu;
-    RRDDIM *rd_user;
-    RRDDIM *rd_system;
 };
 
 static struct statsd {
@@ -250,6 +250,8 @@ static struct statsd {
     STATSD_INDEX histograms;
     STATSD_INDEX meters;
     STATSD_INDEX sets;
+    STATSD_INDEX dictionaries;
+
     size_t unknown_types;
     size_t socket_errors;
     size_t tcp_socket_connects;
@@ -280,6 +282,7 @@ static struct statsd {
     size_t histogram_increase_step;
     double histogram_percentile;
     char *histogram_percentile_str;
+    size_t dictionary_max_unique;
 
     int threads;
     struct collection_thread_status *collection_threads_status;
@@ -297,55 +300,57 @@ static struct statsd {
                 .name = "gauge",
                 .events = 0,
                 .metrics = 0,
-                .index = STATSD_AVL_INDEX_INIT,
-                .default_options = STATSD_METRIC_OPTION_NONE,
-                .first = NULL,
-                STATSD_FIRST_PTR_MUTEX_INIT
+                .dict = NULL,
+                .type = STATSD_METRIC_TYPE_GAUGE,
+                .default_options = STATSD_METRIC_OPTION_NONE
         },
         .counters   = {
                 .name = "counter",
                 .events = 0,
                 .metrics = 0,
-                .index = STATSD_AVL_INDEX_INIT,
-                .default_options = STATSD_METRIC_OPTION_NONE,
-                .first = NULL,
-                STATSD_FIRST_PTR_MUTEX_INIT
+                .dict = NULL,
+                .type = STATSD_METRIC_TYPE_COUNTER,
+                .default_options = STATSD_METRIC_OPTION_NONE
         },
         .timers     = {
                 .name = "timer",
                 .events = 0,
                 .metrics = 0,
-                .index = STATSD_AVL_INDEX_INIT,
-                .default_options = STATSD_METRIC_OPTION_NONE,
-                .first = NULL,
-                STATSD_FIRST_PTR_MUTEX_INIT
+                .dict = NULL,
+                .type = STATSD_METRIC_TYPE_TIMER,
+                .default_options = STATSD_METRIC_OPTION_NONE
         },
         .histograms = {
                 .name = "histogram",
                 .events = 0,
                 .metrics = 0,
-                .index = STATSD_AVL_INDEX_INIT,
-                .default_options = STATSD_METRIC_OPTION_NONE,
-                .first = NULL,
-                STATSD_FIRST_PTR_MUTEX_INIT
+                .dict = NULL,
+                .type = STATSD_METRIC_TYPE_HISTOGRAM,
+                .default_options = STATSD_METRIC_OPTION_NONE
         },
         .meters     = {
                 .name = "meter",
                 .events = 0,
                 .metrics = 0,
-                .index = STATSD_AVL_INDEX_INIT,
-                .default_options = STATSD_METRIC_OPTION_NONE,
-                .first = NULL,
-                STATSD_FIRST_PTR_MUTEX_INIT
+                .dict = NULL,
+                .type = STATSD_METRIC_TYPE_METER,
+                .default_options = STATSD_METRIC_OPTION_NONE
         },
         .sets       = {
                 .name = "set",
                 .events = 0,
                 .metrics = 0,
-                .index = STATSD_AVL_INDEX_INIT,
-                .default_options = STATSD_METRIC_OPTION_NONE,
-                .first = NULL,
-                STATSD_FIRST_PTR_MUTEX_INIT
+                .dict = NULL,
+                .type = STATSD_METRIC_TYPE_SET,
+                .default_options = STATSD_METRIC_OPTION_NONE
+        },
+        .dictionaries = {
+                .name = "dictionary",
+                .events = 0,
+                .metrics = 0,
+                .dict = NULL,
+                .type = STATSD_METRIC_TYPE_DICTIONARY,
+                .default_options = STATSD_METRIC_OPTION_NONE
         },
 
         .tcp_idle_timeout = 600,
@@ -353,6 +358,7 @@ static struct statsd {
         .apps = NULL,
         .histogram_percentile = 95.0,
         .histogram_increase_step = 10,
+        .dictionary_max_unique = 200,
         .threads = 0,
         .collection_threads_status = NULL,
         .sockets = {
@@ -368,54 +374,54 @@ static struct statsd {
 // --------------------------------------------------------------------------------------------------------------------
 // statsd index management - add/find metrics
 
-static int statsd_metric_compare(void* a, void* b) {
-    if(((STATSD_METRIC *)a)->hash < ((STATSD_METRIC *)b)->hash) return -1;
-    else if(((STATSD_METRIC *)a)->hash > ((STATSD_METRIC *)b)->hash) return 1;
-    else return strcmp(((STATSD_METRIC *)a)->name, ((STATSD_METRIC *)b)->name);
-}
+static void dictionary_metric_insert_callback(const char *name, void *value, void *data) {
+    STATSD_INDEX *index = (STATSD_INDEX *)data;
+    STATSD_METRIC *m = (STATSD_METRIC *)value;
+
+    debug(D_STATSD, "Creating new %s metric '%s'", index->name, name);
 
-static inline STATSD_METRIC *statsd_metric_index_find(STATSD_INDEX *index, const char *name, uint32_t hash) {
-    STATSD_METRIC tmp;
-    tmp.name = name;
-    tmp.hash = (hash)?hash:simple_hash(tmp.name);
+    m->name = name;
+    m->hash = simple_hash(name);
+    m->type = index->type;
+    m->options = index->default_options;
+
+    if (m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
+        m->histogram.ext = callocz(1,sizeof(STATSD_METRIC_HISTOGRAM_EXTENSIONS));
+        netdata_mutex_init(&m->histogram.ext->mutex);
+    }
 
-    return (STATSD_METRIC *)STATSD_AVL_SEARCH(&index->index, (avl_t *)&tmp);
+    __atomic_fetch_add(&index->metrics, 1, __ATOMIC_SEQ_CST);
 }
 
-static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, const char *name, STATSD_METRIC_TYPE type) {
-    debug(D_STATSD, "searching for metric '%s' under '%s'", name, index->name);
+static void dictionary_metric_delete_callback(const char *name, void *value, void *data) {
+    (void)data; // STATSD_INDEX *index = (STATSD_INDEX *)data;
+    (void)name;
+    STATSD_METRIC *m = (STATSD_METRIC *)value;
 
-    uint32_t hash = simple_hash(name);
+    if(m->type == STATSD_METRIC_TYPE_HISTOGRAM || m->type == STATSD_METRIC_TYPE_TIMER) {
+        freez(m->histogram.ext);
+        m->histogram.ext = NULL;
+    }
 
-    STATSD_METRIC *m = statsd_metric_index_find(index, name, hash);
-    if(unlikely(!m)) {
-        debug(D_STATSD, "Creating new %s metric '%s'", index->name, name);
+    freez(m->units);
+    freez(m->family);
+    freez(m->dimname);
+}
 
-        m = (STATSD_METRIC *)callocz(sizeof(STATSD_METRIC), 1);
-        m->name = strdupz(name);
-        m->hash = hash;
-        m->type = type;
-        m->options = index->default_options;
+static inline STATSD_METRIC *statsd_find_or_add_metric(STATSD_INDEX *index, const char *name) {
+    debug(D_STATSD, "searching for metric '%s' under '%s'", name, index->name);
 
-        if(type == STATSD_METRIC_TYPE_HISTOGRAM || type == STATSD_METRIC_TYPE_TIMER) {
-            m->histogram.ext = callocz(sizeof(STATSD_METRIC_HISTOGRAM_EXTENSIONS), 1);
-            netdata_mutex_init(&m->histogram.ext->mutex);
-        }
-        STATSD_METRIC *n = (STATSD_METRIC *)STATSD_AVL_INSERT(&index->index, (avl_t *)m);
-        if(unlikely(n != m)) {
-            freez((void *)m->histogram.ext);
-            freez((void *)m->name);
-            freez((void *)m);
-            m = n;
-        }
-        else {
-            STATSD_FIRST_PTR_MUTEX_LOCK(index);
-            index->metrics++;
-            m->next = index->first;
-            index->first = m;
-            STATSD_FIRST_PTR_MUTEX_UNLOCK(index);
-        }
-    }
+#ifdef STATSD_MULTITHREADED
+    // avoid the write lock of dictionary_set() for existing metrics
+    STATSD_METRIC *m = dictionary_get(index->dict, name);
+    if(!m) m = dictionary_set(index->dict, name, NULL, sizeof(STATSD_METRIC));
+#else
+    // no locks here, go faster
+    // this will call the dictionary_metric_insert_callback() if an item
+    // is inserted, otherwise it will return the existing one.
+    // We used the flag DICTIONARY_FLAG_DONT_OVERWRITE_VALUE to support this.
+    STATSD_METRIC *m = dictionary_set(index->dict, name, NULL, sizeof(STATSD_METRIC));
+#endif
 
     index->events++;
     return m;
@@ -569,6 +575,13 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha
 #define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer")
 #define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram")
 
+static void dictionary_metric_set_value_insert_callback(const char *name, void *value, void *data) {
+    (void)name;
+    (void)value;
+    STATSD_METRIC *m = (STATSD_METRIC *)data;
+    m->set.unique++;
+}
+
 static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
     if(!is_metric_useful_for_collection(m)) return;
 
@@ -580,13 +593,14 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
     if(unlikely(m->reset)) {
         if(likely(m->set.dict)) {
             dictionary_destroy(m->set.dict);
+            dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m);
             m->set.dict = NULL;
         }
         statsd_reset_metric(m);
     }
 
     if (unlikely(!m->set.dict)) {
-        m->set.dict   = dictionary_create(STATSD_DICTIONARY_OPTIONS | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
+        m->set.dict   = dictionary_create(STATSD_DICTIONARY_OPTIONS);
         m->set.unique = 0;
     }
 
@@ -594,12 +608,56 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
         // magic loading of metric, without affecting anything
     }
     else {
-        void *t = dictionary_get(m->set.dict, value);
+#ifdef STATSD_MULTITHREADED
+        // avoid the write lock to check if something is already there
+        if(!dictionary_get(m->set.dict, value))
+            dictionary_set(m->set.dict, value, NULL, 0);
+#else
+        dictionary_set(m->set.dict, value, NULL, 0);
+#endif
+        m->events++;
+        m->count++;
+    }
+}
+
+static void dictionary_metric_dict_value_insert_callback(const char *name, void *value, void *data) {
+    (void)name;
+    (void)value;
+    STATSD_METRIC *m = (STATSD_METRIC *)data;
+    m->dictionary.unique++;
+}
+
+static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value) {
+    if(!is_metric_useful_for_collection(m)) return;
+
+    if(unlikely(!value || !*value)) {
+        error("STATSD: metric of type set, with empty value is ignored.");
+        return;
+    }
+
+    if(unlikely(m->reset))
+        statsd_reset_metric(m);
+
+    if (unlikely(!m->dictionary.dict)) {
+        m->dictionary.dict   = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+        dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m);
+        m->dictionary.unique = 0;
+    }
+
+    if(unlikely(value_is_zinit(value))) {
+        // magic loading of metric, without affecting anything
+    }
+    else {
+        STATSD_METRIC_DICTIONARY_ITEM *t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_get(m->dictionary.dict, value);
+
         if (unlikely(!t)) {
-            dictionary_set(m->set.dict, value, NULL, 1);
-            m->set.unique++;
+            if(!t && m->dictionary.unique >= statsd.dictionary_max_unique)
+                value = "other";
+
+            t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_set(m->dictionary.dict, value, NULL, sizeof(STATSD_METRIC_DICTIONARY_ITEM));
         }
 
+        t->count++;
         m->events++;
         m->count++;
     }
@@ -609,85 +667,125 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
 // --------------------------------------------------------------------------------------------------------------------
 // statsd parsing
 
-static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling, const char *tags) {
-    (void)tags;
+static inline const char *statsd_parse_skip_up_to(const char *s, char d1, char d2, char d3) {
+    char c;
+
+    for(c = *s; c && c != d1 && c != d2 && c != d3 && c != '\r' && c != '\n'; c = *++s) ;
 
+    return s;
+}
+
+const char *statsd_parse_skip_spaces(const char *s) {
+    char c;
+
+    for(c = *s; c && ( c == ' ' || c == '\t' || c == '\r' || c == '\n' ); c = *++s) ;
+
+    return s;
+}
+
+static inline const char *statsd_parse_field_trim(const char *start, char *end) {
+    if(unlikely(!start || !*start)) {
+        start = end;
+        return start;
+    }
+
+    while(start <= end && (*start == ' ' || *start == '\t'))
+        start++;
+
+    *end = '\0';
+    end--;
+    while(end >= start && (*end == ' ' || *end == '\t'))
+        *end-- = '\0';
+
+    return start;
+}
+
+static void statsd_process_metric(const char *name, const char *value, const char *type, const char *sampling, const char *tags) {
     debug(D_STATSD, "STATSD: raw metric '%s', value '%s', type '%s', sampling '%s', tags '%s'", name?name:"(null)", value?value:"(null)", type?type:"(null)", sampling?sampling:"(null)", tags?tags:"(null)");
 
     if(unlikely(!name || !*name)) return;
     if(unlikely(!type || !*type)) type = "m";
 
-    char t0 = type[0], t1 = type[1];
+    STATSD_METRIC *m = NULL;
 
+    char t0 = type[0], t1 = type[1];
     if(unlikely(t0 == 'g' && t1 == '\0')) {
         statsd_process_gauge(
-                statsd_find_or_add_metric(&statsd.gauges, name, STATSD_METRIC_TYPE_GAUGE),
-                value, sampling);
+            m = statsd_find_or_add_metric(&statsd.gauges, name),
+            value, sampling);
     }
     else if(unlikely((t0 == 'c' || t0 == 'C') && t1 == '\0')) {
         // etsy/statsd uses 'c'
         // brubeck     uses 'C'
         statsd_process_counter(
-                statsd_find_or_add_metric(&statsd.counters, name, STATSD_METRIC_TYPE_COUNTER),
-                value, sampling);
+            m = statsd_find_or_add_metric(&statsd.counters, name),
+            value, sampling);
     }
     else if(unlikely(t0 == 'm' && t1 == '\0')) {
         statsd_process_meter(
-                statsd_find_or_add_metric(&statsd.meters, name, STATSD_METRIC_TYPE_METER),
-                value, sampling);
+            m = statsd_find_or_add_metric(&statsd.meters, name),
+            value, sampling);
     }
     else if(unlikely(t0 == 'h' && t1 == '\0')) {
         statsd_process_histogram(
-                statsd_find_or_add_metric(&statsd.histograms, name, STATSD_METRIC_TYPE_HISTOGRAM),
-                value, sampling);
+            m = statsd_find_or_add_metric(&statsd.histograms, name),
+            value, sampling);
     }
     else if(unlikely(t0 == 's' && t1 == '\0')) {
         statsd_process_set(
-                statsd_find_or_add_metric(&statsd.sets, name, STATSD_METRIC_TYPE_SET),
-                value);
+            m = statsd_find_or_add_metric(&statsd.sets, name),
+            value);
+    }
+    else if(unlikely(t0 == 'd' && t1 == '\0')) {
+        statsd_process_dictionary(
+            m = statsd_find_or_add_metric(&statsd.dictionaries, name),
+            value);
     }
     else if(unlikely(t0 == 'm' && t1 == 's' && type[2] == '\0')) {
         statsd_process_timer(
-                statsd_find_or_add_metric(&statsd.timers, name, STATSD_METRIC_TYPE_TIMER),
-                value, sampling);
+            m = statsd_find_or_add_metric(&statsd.timers, name),
+            value, sampling);
     }
     else {
         statsd.unknown_types++;
         error("STATSD: metric '%s' with value '%s' is sent with unknown metric type '%s'", name, value?value:"", type);
     }
-}
-
-static inline const char *statsd_parse_skip_up_to(const char *s, char d1, char d2) {
-    char c;
 
-    for(c = *s; c && c != d1 && c != d2 && c != '\r' && c != '\n'; c = *++s) ;
-
-    return s;
-}
+    if(m && tags && *tags) {
+        const char *s = tags;
+        while(*s) {
+            const char *tagkey = NULL, *tagvalue = NULL;
+            char *tagkey_end = NULL, *tagvalue_end = NULL;
 
-const char *statsd_parse_skip_spaces(const char *s) {
-    char c;
+            s = tagkey_end = (char *)statsd_parse_skip_up_to(tagkey = s, ':', '=', ',');
+            if(tagkey == tagkey_end) {
+                if (*s) {
+                    s++;
+                    s = statsd_parse_skip_spaces(s);
+                }
+                continue;
+            }
 
-    for(c = *s; c && ( c == ' ' || c == '\t' || c == '\r' || c == '\n' ); c = *++s) ;
+            if(likely(*s == ':' || *s == '='))
+                s = tagvalue_end = (char *) statsd_parse_skip_up_to(tagvalue = ++s, ',', '\0', '\0');
 
-    return s;
-}
+            if(*s == ',') s++;
 
-static inline const char *statsd_parse_field_trim(const char *start, char *end) {
-    if(unlikely(!start)) {
-        start = end;
-        return start;
-    }
+            statsd_parse_field_trim(tagkey, tagkey_end);
+            statsd_parse_field_trim(tagvalue, tagvalue_end);
 
-    while(start <= end && (*start == ' ' || *start == '\t'))
-        start++;
+            if(tagkey && *tagkey && tagvalue && *tagvalue) {
+                if (!m->units && strcmp(tagkey, "units") == 0)
+                    m->units = strdupz(tagvalue);
 
-    *end = '\0';
-    end--;
-    while(end >= start && (*end == ' ' || *end == '\t'))
-        *end-- = '\0';
+                if (!m->dimname && strcmp(tagkey, "name") == 0)
+                    m->dimname = strdupz(tagvalue);
 
-    return start;
+                if (!m->family && strcmp(tagkey, "family") == 0)
+                    m->family = strdupz(tagvalue);
+            }
+        }
+    }
 }
 
 static inline size_t statsd_process(char *buffer, size_t size, int require_newlines) {
@@ -699,7 +797,7 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
         const char *name = NULL, *value = NULL, *type = NULL, *sampling = NULL, *tags = NULL;
         char *name_end = NULL, *value_end = NULL, *type_end = NULL, *sampling_end = NULL, *tags_end = NULL;
 
-        s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '|');
+        s = name_end = (char *)statsd_parse_skip_up_to(name = s, ':', '=', '|');
         if(name == name_end) {
             if (*s) {
                 s++;
@@ -708,20 +806,27 @@ static inline size_t statsd_process(char *buffer, size_t size, int require_newli
             continue;
         }
 
-        if(likely(*s == ':'))
-            s = value_end = (char *) statsd_parse_skip_up_to(value = ++s, '|', '|');
+        if(likely(*s == ':' || *s == '='))
+            s = value_end = (char *) statsd_parse_skip_up_to(value = ++s, '|', '@', '#');
 
         if(likely(*s == '|'))
-            s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@');
+            s = type_end = (char *) statsd_parse_skip_up_to(type = ++s, '|', '@', '#');
 
-        if(likely(*s == '|' || *s == '@')) {
-            s = sampling_end = (char *) statsd_parse_skip_up_to(sampling = ++s, '|', '#');
-            if(*sampling == '@') sampling++;
-        }
+        while(*s == '|' || *s == '@' || *s == '#') {
+            // parse all the fields that may be appended
 
-        if(likely(*s == '|' || *s == '#')) {
-            s = tags_end = (char *) statsd_parse_skip_up_to(tags = ++s, '|', '|');
-            if(*tags == '#') tags++;
+            if ((*s == '|' && s[1] == '@') || *s == '@') {
+                s = sampling_end = (char *)statsd_parse_skip_up_to(sampling = ++s, '|', '@', '#');
+                if (*sampling == '@') sampling++;
+            }
+            else if ((*s == '|' && s[1] == '#') || *s == '#') {
+                s = tags_end = (char *)statsd_parse_skip_up_to(tags = ++s, '|', '@', '#');
+                if (*tags == '#') tags++;
+            }
+            else {
+                // unknown field, skip it
+                s = (char *)statsd_parse_skip_up_to(++s, '|', '@', '#');
+            }
         }
 
         // skip everything until the end of the line
@@ -788,6 +893,7 @@ static void *statsd_add_callback(POLLINFO *pi, short int *events, void *data) {
     (void)pi;
     (void)data;
 
+    worker_is_busy(WORKER_JOB_TYPE_TCP_CONNECTED);
     *events = POLLIN;
 
     struct statsd_tcp *t = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1);
@@ -796,11 +902,14 @@ static void *statsd_add_callback(POLLINFO *pi, short int *events, void *data) {
     statsd.tcp_socket_connects++;
     statsd.tcp_socket_connected++;
 
+    worker_is_idle();
     return t;
 }
 
 // TCP client disconnected
 static void statsd_del_callback(POLLINFO *pi) {
+    worker_is_busy(WORKER_JOB_TYPE_TCP_DISCONNECTED);
+
     struct statsd_tcp *t = pi->data;
 
     if(likely(t)) {
@@ -818,10 +927,15 @@ static void statsd_del_callback(POLLINFO *pi) {
 
         freez(t);
     }
+
+    worker_is_idle();
 }
 
 // Receive data
 static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
+    int retval = -1;
+    worker_is_busy(WORKER_JOB_TYPE_RCV_DATA);
+
     *events = POLLIN;
 
     int fd = pi->fd;
@@ -832,14 +946,16 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
             if(unlikely(!d)) {
                 error("STATSD: internal error: expected TCP data pointer is NULL");
                 statsd.socket_errors++;
-                return -1;
+                retval = -1;
+                goto cleanup;
             }
 
 #ifdef NETDATA_INTERNAL_CHECKS
             if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_TCP)) {
                 error("STATSD: internal error: socket data type should be %d, but it is %d", (int)STATSD_SOCKET_DATA_TYPE_TCP, (int)d->type);
                 statsd.socket_errors++;
-                return -1;
+                retval = -1;
+                goto cleanup;
             }
 #endif
 
@@ -872,8 +988,10 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
                     d->len = statsd_process(d->buffer, d->len, 1);
                 }
 
-                if(unlikely(ret == -1))
-                    return -1;
+                if(unlikely(ret == -1)) {
+                    retval = -1;
+                    goto cleanup;
+                }
 
             } while (rc != -1);
             break;
@@ -884,14 +1002,16 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
             if(unlikely(!d)) {
                 error("STATSD: internal error: expected UDP data pointer is NULL");
                 statsd.socket_errors++;
-                return -1;
+                retval = -1;
+                goto cleanup;
             }
 
 #ifdef NETDATA_INTERNAL_CHECKS
             if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) {
                 error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_UDP);
                 statsd.socket_errors++;
-                return -1;
+                retval = -1;
+                goto cleanup;
             }
 #endif
 
@@ -904,7 +1024,8 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
                     if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
                         error("STATSD: recvmmsg() on UDP socket %d failed.", fd);
                         statsd.socket_errors++;
-                        return -1;
+                        retval = -1;
+                        goto cleanup;
                     }
                 } else if (rc) {
                     // data received
@@ -929,7 +1050,8 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
                     if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
                         error("STATSD: recv() on UDP socket %d failed.", fd);
                         statsd.socket_errors++;
-                        return -1;
+                        retval = -1;
+                        goto cleanup;
                     }
                 } else if (rc) {
                     // data received
@@ -947,24 +1069,26 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
         default: {
             error("STATSD: internal error: unknown socktype %d on socket %d", pi->socktype, fd);
             statsd.socket_errors++;
-            return -1;
+            retval = -1;
+            goto cleanup;
         }
     }
 
-    return 0;
+    retval = 0;
+cleanup:
+    worker_is_idle();
+    return retval;
 }
 
 static int statsd_snd_callback(POLLINFO *pi, short int *events) {
     (void)pi;
     (void)events;
 
+    worker_is_busy(WORKER_JOB_TYPE_SND_DATA);
     error("STATSD: snd_callback() called, but we never requested to send data to statsd clients.");
-    return -1;
-}
+    worker_is_idle();
 
-static void statsd_timer_callback(void *timer_data) {
-    struct collection_thread_status *status = timer_data;
-    getrusage(RUSAGE_THREAD, &status->rusage);
+    return -1;
 }
 
 // --------------------------------------------------------------------------------------------------------------------
@@ -986,12 +1110,19 @@ void statsd_collector_thread_cleanup(void *data) {
 #endif
 
     freez(d);
+    worker_unregister();
 }
 
 void *statsd_collector_thread(void *ptr) {
     struct collection_thread_status *status = ptr;
     status->status = 1;
 
+    worker_register("STATSD");
+    worker_register_job_name(WORKER_JOB_TYPE_TCP_CONNECTED, "tcp connect");
+    worker_register_job_name(WORKER_JOB_TYPE_TCP_DISCONNECTED, "tcp disconnect");
+    worker_register_job_name(WORKER_JOB_TYPE_RCV_DATA, "receive");
+    worker_register_job_name(WORKER_JOB_TYPE_SND_DATA, "send");
+
     info("STATSD collector thread started with taskid %d", gettid());
 
     struct statsd_udp *d = callocz(sizeof(struct statsd_udp), 1);
@@ -1019,7 +1150,7 @@ void *statsd_collector_thread(void *ptr) {
             , statsd_del_callback
             , statsd_rcv_callback
             , statsd_snd_callback
-            , statsd_timer_callback
+            , NULL
             , NULL                     // No access control pattern
             , 0                        // No dns lookups for access control pattern
             , (void *)d
@@ -1413,23 +1544,50 @@ static inline void statsd_readdir(const char *user_path, const char *stock_path,
 // send metrics to netdata - in private charts - called from the main thread
 
 // extract chart type and chart id from metric name
-static inline void statsd_get_metric_type_and_id(STATSD_METRIC *m, char *type, char *id, const char *defid, size_t len) {
-    char *s;
+static inline void statsd_get_metric_type_and_id(STATSD_METRIC *m, char *type, char *id, char *context, const char *metrictype, size_t len) {
+
+    // The full chart type.id looks like this:
+    // ${STATSD_CHART_PREFIX} + "_" + ${METRIC_NAME} + "_" + ${METRIC_TYPE}
+    //
+    // where:
+    // STATSD_CHART_PREFIX = "statsd" as defined above
+    // METRIC_NAME = whatever the user gave to statsd
+    // METRIC_TYPE = "gauge", "counter", "meter", "timer", "histogram", "set", "dictionary"
+
+    // for chart type, we want:
+    // ${STATSD_CHART_PREFIX} + "_" + the first word of ${METRIC_NAME}
+
+    // find the first word of ${METRIC_NAME}
+    char firstword[len + 1], *s = "";
+    strncpyz(firstword, m->name, len);
+    for (s = firstword; *s ; s++) {
+        if (unlikely(*s == '.' || *s == '_')) {
+            *s = '\0';
+            s++;
+            break;
+        }
+    }
+    // firstword has the first word of ${METRIC_NAME}
+    // s has the remaining, if any
 
-    snprintfz(type, len, "%s_%s_%s", STATSD_CHART_PREFIX, defid, m->name);
-    for(s = type; *s ;s++)
-        if(unlikely(*s == '.')) break;
+    // create the chart type:
+    snprintfz(type, len, STATSD_CHART_PREFIX "_%s", firstword);
 
-    if(*s == '.') {
-        *s++ = '\0';
-        strncpyz(id, s, len);
-    }
-    else {
-        strncpyz(id, defid, len);
-    }
+    // for chart id, we want:
+    // the remaining of the words of ${METRIC_NAME} + "_" + ${METRIC_TYPE}
+    // or the ${METRIC_NAME} has no remaining words, the ${METRIC_TYPE} alone
+    if(*s)
+        snprintfz(id, len, "%s_%s", s, metrictype);
+    else
+        snprintfz(id, len, "%s", metrictype);
+
+    // for the context, we want the full of both the above, separated with a dot (type.id):
+    snprintfz(context, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
 
+    // make sure they don't have illegal characters
     netdata_fix_chart_id(type);
     netdata_fix_chart_id(id);
+    netdata_fix_chart_id(context);
 }
 
 static inline RRDSET *statsd_private_rrdset_create(
@@ -1486,11 +1644,8 @@ static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
     debug(D_STATSD, "updating private chart for gauge metric '%s'", m->name);
 
     if(unlikely(!m->st)) {
-        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
-        statsd_get_metric_type_and_id(m, type, id, "gauge", RRD_ID_LENGTH_MAX);
-
-        char context[RRD_ID_LENGTH_MAX + 1];
-        snprintfz(context, RRD_ID_LENGTH_MAX, "statsd_gauge.%s", m->name);
+        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+        statsd_get_metric_type_and_id(m, type, id, context, "gauge", RRD_ID_LENGTH_MAX);
 
         char title[RRD_ID_LENGTH_MAX + 1];
         snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for gauge %s", m->name);
@@ -1500,16 +1655,16 @@ static inline void statsd_private_chart_gauge(STATSD_METRIC *m) {
                 , type
                 , id
                 , NULL          // name
-                , "gauges"      // family (submenu)
+                , m->family?m->family:"gauges"      // family (submenu)
                 , context       // context
                 , title         // title
-                , "value"       // units
+                , m->units?m->units:"value"       // units
                 , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_LINE
         );
 
-        m->rd_value = rrddim_add(m->st, "gauge",  NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+        m->rd_value = rrddim_add(m->st, "gauge",  m->dimname?m->dimname:NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
 
         if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
             m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1,    RRD_ALGORITHM_INCREMENTAL);
@@ -1528,11 +1683,8 @@ static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const
     debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);
 
     if(unlikely(!m->st)) {
-        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
-        statsd_get_metric_type_and_id(m, type, id, dim, RRD_ID_LENGTH_MAX);
-
-        char context[RRD_ID_LENGTH_MAX + 1];
-        snprintfz(context, RRD_ID_LENGTH_MAX, "statsd_%s.%s", dim, m->name);
+        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+        statsd_get_metric_type_and_id(m, type, id, context, dim, RRD_ID_LENGTH_MAX);
 
         char title[RRD_ID_LENGTH_MAX + 1];
         snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for %s %s", dim, m->name);
@@ -1542,16 +1694,16 @@ static inline void statsd_private_chart_counter_or_meter(STATSD_METRIC *m, const
                 , type
                 , id
                 , NULL          // name
-                , family        // family (submenu)
+                , m->family?m->family:family        // family (submenu)
                 , context       // context
                 , title         // title
-                , "events/s"    // units
+                , m->units?m->units:"events/s"    // units
                 , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_AREA
         );
 
-        m->rd_value = rrddim_add(m->st, dim, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+        m->rd_value = rrddim_add(m->st, dim, m->dimname?m->dimname:NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
 
         if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
             m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1570,11 +1722,8 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) {
     debug(D_STATSD, "updating private chart for set metric '%s'", m->name);
 
     if(unlikely(!m->st)) {
-        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
-        statsd_get_metric_type_and_id(m, type, id, "set", RRD_ID_LENGTH_MAX);
-
-        char context[RRD_ID_LENGTH_MAX + 1];
-        snprintfz(context, RRD_ID_LENGTH_MAX, "statsd_set.%s", m->name);
+        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+        statsd_get_metric_type_and_id(m, type, id, context, "set", RRD_ID_LENGTH_MAX);
 
         char title[RRD_ID_LENGTH_MAX + 1];
         snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for set %s", m->name);
@@ -1584,16 +1733,16 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) {
                 , type
                 , id
                 , NULL          // name
-                , "sets"        // family (submenu)
+                , m->family?m->family:"sets"        // family (submenu)
                 , context       // context
                 , title         // title
-                , "entries"     // units
+                , m->units?m->units:"entries"     // units
                 , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_LINE
         );
 
-        m->rd_value = rrddim_add(m->st, "set", "set size", 1, 1, RRD_ALGORITHM_ABSOLUTE);
+        m->rd_value = rrddim_add(m->st, "set", m->dimname?m->dimname:"unique", 1, 1, RRD_ALGORITHM_ABSOLUTE);
 
         if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
             m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1608,15 +1757,54 @@ static inline void statsd_private_chart_set(STATSD_METRIC *m) {
     rrdset_done(m->st);
 }
 
+static inline void statsd_private_chart_dictionary(STATSD_METRIC *m) {
+    debug(D_STATSD, "updating private chart for dictionary metric '%s'", m->name);
+
+    if(unlikely(!m->st)) {
+        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+        statsd_get_metric_type_and_id(m, type, id, context, "dictionary", RRD_ID_LENGTH_MAX);
+
+        char title[RRD_ID_LENGTH_MAX + 1];
+        snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for dictionary %s", m->name);
+
+        m->st = statsd_private_rrdset_create(
+            m
+            , type
+            , id
+            , NULL          // name
+            , m->family?m->family:"dictionaries" // family (submenu)
+            , context       // context
+            , title         // title
+            , m->units?m->units:"events/s"     // units
+            , NETDATA_CHART_PRIO_STATSD_PRIVATE
+            , statsd.update_every
+            , RRDSET_TYPE_STACKED
+        );
+
+        if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
+            m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+    }
+    else rrdset_next(m->st);
+
+    STATSD_METRIC_DICTIONARY_ITEM *t;
+    dfe_start_read(m->dictionary.dict, t) {
+        if (!t->rd) t->rd = rrddim_add(m->st, t_name, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+        rrddim_set_by_pointer(m->st, t->rd, (collected_number)t->count);
+    }
+    dfe_done(t);
+
+    if(m->rd_count)
+        rrddim_set_by_pointer(m->st, m->rd_count, m->events);
+
+    rrdset_done(m->st);
+}
+
 static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
     debug(D_STATSD, "updating private chart for %s metric '%s'", dim, m->name);
 
     if(unlikely(!m->st)) {
-        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1];
-        statsd_get_metric_type_and_id(m, type, id, dim, RRD_ID_LENGTH_MAX);
-
-        char context[RRD_ID_LENGTH_MAX + 1];
-        snprintfz(context, RRD_ID_LENGTH_MAX, "statsd_%s.%s", dim, m->name);
+        char type[RRD_ID_LENGTH_MAX + 1], id[RRD_ID_LENGTH_MAX + 1], context[RRD_ID_LENGTH_MAX + 1];
+        statsd_get_metric_type_and_id(m, type, id, context, dim, RRD_ID_LENGTH_MAX);
 
         char title[RRD_ID_LENGTH_MAX + 1];
         snprintfz(title, RRD_ID_LENGTH_MAX, "statsd private chart for %s %s", dim, m->name);
@@ -1626,10 +1814,10 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con
                 , type
                 , id
                 , NULL          // name
-                , family        // family (submenu)
+                , m->family?m->family:family        // family (submenu)
                 , context       // context
                 , title         // title
-                , units         // units
+                , m->units?m->units:units         // units
                 , NETDATA_CHART_PRIO_STATSD_PRIVATE
                 , statsd.update_every
                 , RRDSET_TYPE_AREA
@@ -1641,7 +1829,7 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con
         m->histogram.ext->rd_percentile = rrddim_add(m->st, statsd.histogram_percentile_str, NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
         m->histogram.ext->rd_median = rrddim_add(m->st, "median", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
         m->histogram.ext->rd_stddev = rrddim_add(m->st, "stddev", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
-        m->histogram.ext->rd_sum = rrddim_add(m->st, "sum", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
+        //m->histogram.ext->rd_sum = rrddim_add(m->st, "sum", NULL, 1, statsd.decimal_detail, RRD_ALGORITHM_ABSOLUTE);
 
         if(m->options & STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT)
             m->rd_count = rrddim_add(m->st, "events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1653,7 +1841,7 @@ static inline void statsd_private_chart_timer_or_histogram(STATSD_METRIC *m, con
     rrddim_set_by_pointer(m->st, m->histogram.ext->rd_percentile, m->histogram.ext->last_percentile);
     rrddim_set_by_pointer(m->st, m->histogram.ext->rd_median, m->histogram.ext->last_median);
     rrddim_set_by_pointer(m->st, m->histogram.ext->rd_stddev, m->histogram.ext->last_stddev);
-    rrddim_set_by_pointer(m->st, m->histogram.ext->rd_sum, m->histogram.ext->last_sum);
+    //rrddim_set_by_pointer(m->st, m->histogram.ext->rd_sum, m->histogram.ext->last_sum);
     rrddim_set_by_pointer(m->st, m->rd_value, m->last);
 
     if(m->rd_count)
@@ -1721,6 +1909,34 @@ static inline void statsd_flush_set(STATSD_METRIC *m) {
         statsd_private_chart_set(m);
 }
 
+static inline void statsd_flush_dictionary(STATSD_METRIC *m) {
+    debug(D_STATSD, "flushing dictionary metric '%s'", m->name);
+
+    int updated = 0;
+    if(unlikely(!m->reset && m->count)) {
+        m->last = (collected_number)m->dictionary.unique;
+
+        m->reset = 1;
+        updated = 1;
+    }
+    else {
+        m->last = 0;
+    }
+
+    if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
+        statsd_private_chart_dictionary(m);
+
+    if(m->dictionary.unique >= statsd.dictionary_max_unique) {
+        if(!(m->options & STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED)) {
+            m->options |= STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED;
+            info(
+                "STATSD dictionary '%s' reach max of %zu items - try increasing 'dictionaries max unique dimensions' in netdata.conf",
+                m->name,
+                m->dictionary.unique);
+        }
+    }
+}
+
 static inline void statsd_flush_timer_or_histogram(STATSD_METRIC *m, const char *dim, const char *family, const char *units) {
     debug(D_STATSD, "flushing %s metric '%s'", dim, m->name);
 
@@ -1793,6 +2009,7 @@ static inline RRD_ALGORITHM statsd_algorithm_for_metric(STATSD_METRIC *m) {
 
         case STATSD_METRIC_TYPE_METER:
         case STATSD_METRIC_TYPE_COUNTER:
+        case STATSD_METRIC_TYPE_DICTIONARY:
             return RRD_ALGORITHM_INCREMENTAL;
     }
 }
@@ -2059,6 +2276,7 @@ const char *statsd_metric_type_string(STATSD_METRIC_TYPE type) {
         case STATSD_METRIC_TYPE_HISTOGRAM: return "histogram";
         case STATSD_METRIC_TYPE_METER: return "meter";
         case STATSD_METRIC_TYPE_SET: return "set";
+        case STATSD_METRIC_TYPE_DICTIONARY: return "dictionary";
         case STATSD_METRIC_TYPE_TIMER: return "timer";
         default: return "unknown";
     }
@@ -2068,7 +2286,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
     STATSD_METRIC *m;
 
     // find the useful metrics (incremental = each time we are called, we check the new metrics only)
-    for(m = index->first; m ; m = m->next) {
+    dfe_start_read(index->dict, m) {
         // since we add new metrics at the beginning
         // check for useful charts, until the point we last checked
         if(unlikely(is_metric_checked(m))) break;
@@ -2109,6 +2327,7 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
             index->first_useful = m;
         }
     }
+    dfe_done(m);
 
     // flush all the useful metrics
     for(m = index->first_useful; m ; m = m->next_useful) {
@@ -2145,17 +2364,75 @@ static void statsd_main_cleanup(void *data) {
     info("STATSD: closing sockets...");
     listen_sockets_close(&statsd.sockets);
 
+    // destroy the dictionaries
+    dictionary_destroy(statsd.gauges.dict);
+    dictionary_destroy(statsd.meters.dict);
+    dictionary_destroy(statsd.counters.dict);
+    dictionary_destroy(statsd.histograms.dict);
+    dictionary_destroy(statsd.dictionaries.dict);
+    dictionary_destroy(statsd.sets.dict);
+    dictionary_destroy(statsd.timers.dict);
+
     info("STATSD: cleanup completed.");
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+
+    worker_unregister();
 }
 
+#define WORKER_STATSD_FLUSH_GAUGES 0
+#define WORKER_STATSD_FLUSH_COUNTERS 1
+#define WORKER_STATSD_FLUSH_METERS 2
+#define WORKER_STATSD_FLUSH_TIMERS 3
+#define WORKER_STATSD_FLUSH_HISTOGRAMS 4
+#define WORKER_STATSD_FLUSH_SETS 5
+#define WORKER_STATSD_FLUSH_DICTIONARIES 6
+#define WORKER_STATSD_FLUSH_STATS 7
+
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 8
+#endif
+
 void *statsd_main(void *ptr) {
+    worker_register("STATSDFLUSH");
+    worker_register_job_name(WORKER_STATSD_FLUSH_GAUGES, "gauges");
+    worker_register_job_name(WORKER_STATSD_FLUSH_COUNTERS, "counters");
+    worker_register_job_name(WORKER_STATSD_FLUSH_METERS, "meters");
+    worker_register_job_name(WORKER_STATSD_FLUSH_TIMERS, "timers");
+    worker_register_job_name(WORKER_STATSD_FLUSH_HISTOGRAMS, "histograms");
+    worker_register_job_name(WORKER_STATSD_FLUSH_SETS, "sets");
+    worker_register_job_name(WORKER_STATSD_FLUSH_DICTIONARIES, "dictionaries");
+    worker_register_job_name(WORKER_STATSD_FLUSH_STATS, "statistics");
+
     netdata_thread_cleanup_push(statsd_main_cleanup, ptr);
 
+    statsd.gauges.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+    statsd.meters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+    statsd.counters.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+    statsd.histograms.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+    statsd.dictionaries.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+    statsd.sets.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+    statsd.timers.dict = dictionary_create(STATSD_DICTIONARY_OPTIONS);
+
+    dictionary_register_insert_callback(statsd.gauges.dict, dictionary_metric_insert_callback, &statsd.gauges);
+    dictionary_register_insert_callback(statsd.meters.dict, dictionary_metric_insert_callback, &statsd.meters);
+    dictionary_register_insert_callback(statsd.counters.dict, dictionary_metric_insert_callback, &statsd.counters);
+    dictionary_register_insert_callback(statsd.histograms.dict, dictionary_metric_insert_callback, &statsd.histograms);
+    dictionary_register_insert_callback(statsd.dictionaries.dict, dictionary_metric_insert_callback, &statsd.dictionaries);
+    dictionary_register_insert_callback(statsd.sets.dict, dictionary_metric_insert_callback, &statsd.sets);
+    dictionary_register_insert_callback(statsd.timers.dict, dictionary_metric_insert_callback, &statsd.timers);
+
+    dictionary_register_delete_callback(statsd.gauges.dict, dictionary_metric_delete_callback, &statsd.gauges);
+    dictionary_register_delete_callback(statsd.meters.dict, dictionary_metric_delete_callback, &statsd.meters);
+    dictionary_register_delete_callback(statsd.counters.dict, dictionary_metric_delete_callback, &statsd.counters);
+    dictionary_register_delete_callback(statsd.histograms.dict, dictionary_metric_delete_callback, &statsd.histograms);
+    dictionary_register_delete_callback(statsd.dictionaries.dict, dictionary_metric_delete_callback, &statsd.dictionaries);
+    dictionary_register_delete_callback(statsd.sets.dict, dictionary_metric_delete_callback, &statsd.sets);
+    dictionary_register_delete_callback(statsd.timers.dict, dictionary_metric_delete_callback, &statsd.timers);
+
     // ----------------------------------------------------------------------------------------------------------------
     // statsd configuration
 
-    statsd.enabled = config_get_boolean(CONFIG_SECTION_STATSD, "enabled", statsd.enabled);
+    statsd.enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, "statsd", statsd.enabled);
 
     statsd.update_every = default_rrd_update_every;
     statsd.update_every = (int)config_get_number(CONFIG_SECTION_STATSD, "update every (flushInterval)", statsd.update_every);
@@ -2188,13 +2465,16 @@ void *statsd_main(void *ptr) {
         statsd.histogram_percentile_str = strdupz(buffer);
     }
 
-    if(config_get_boolean(CONFIG_SECTION_STATSD, "add dimension for number of events received", 1)) {
+    statsd.dictionary_max_unique = config_get_number(CONFIG_SECTION_STATSD, "dictionaries max unique dimensions", statsd.dictionary_max_unique);
+
+    if(config_get_boolean(CONFIG_SECTION_STATSD, "add dimension for number of events received", 0)) {
         statsd.gauges.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
         statsd.counters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
         statsd.meters.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
         statsd.sets.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
         statsd.histograms.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
         statsd.timers.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
+        statsd.dictionaries.default_options |= STATSD_METRIC_OPTION_CHART_DIMENSION_COUNT;
     }
 
     if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on gauges (deleteGauges)", 0))
@@ -2215,6 +2495,9 @@ void *statsd_main(void *ptr) {
     if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on timers (deleteTimers)", 0))
         statsd.timers.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
 
+    if(config_get_boolean(CONFIG_SECTION_STATSD, "gaps on dictionaries (deleteDictionaries)", 0))
+        statsd.dictionaries.default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
+
     size_t max_sockets = (size_t)config_get_number(CONFIG_SECTION_STATSD, "statsd server max TCP sockets", (long long int)(rlimit_nofile.rlim_cur / 4));
 
 #ifdef STATSD_MULTITHREADED
@@ -2275,6 +2558,7 @@ void *statsd_main(void *ptr) {
     RRDDIM *rd_metrics_meter     = rrddim_add(st_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
     RRDDIM *rd_metrics_histogram = rrddim_add(st_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
     RRDDIM *rd_metrics_set       = rrddim_add(st_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    RRDDIM *rd_metrics_dictionary= rrddim_add(st_metrics, "dictionaries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
 
     RRDSET *st_useful_metrics = rrdset_create_localhost(
             "netdata"
@@ -2296,6 +2580,7 @@ void *statsd_main(void *ptr) {
     RRDDIM *rd_useful_metrics_meter     = rrddim_add(st_useful_metrics, "meters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
     RRDDIM *rd_useful_metrics_histogram = rrddim_add(st_useful_metrics, "histograms", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
     RRDDIM *rd_useful_metrics_set       = rrddim_add(st_useful_metrics, "sets", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    RRDDIM *rd_useful_metrics_dictionary= rrddim_add(st_useful_metrics, "dictionaries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
 
     RRDSET *st_events = rrdset_create_localhost(
             "netdata"
@@ -2317,6 +2602,7 @@ void *statsd_main(void *ptr) {
     RRDDIM *rd_events_meter     = rrddim_add(st_events, "meters", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
     RRDDIM *rd_events_histogram = rrddim_add(st_events, "histograms", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
     RRDDIM *rd_events_set       = rrddim_add(st_events, "sets", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+    RRDDIM *rd_events_dictionary= rrddim_add(st_events, "dictionaries", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
     RRDDIM *rd_events_unknown   = rrddim_add(st_events, "unknown", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
     RRDDIM *rd_events_errors    = rrddim_add(st_events, "errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
 
@@ -2420,70 +2706,39 @@ void *statsd_main(void *ptr) {
     );
     RRDDIM *rd_pcharts = rrddim_add(st_pcharts, "charts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
 
-    RRDSET *stcpu_thread = rrdset_create_localhost(
-            "netdata"
-            , "plugin_statsd_charting_cpu"
-            , NULL
-            , "statsd"
-            , "netdata.statsd_cpu"
-            , "Netdata statsd charting thread CPU usage"
-            , "milliseconds/s"
-            , PLUGIN_STATSD_NAME
-            , "stats"
-            , 132001
-            , statsd.update_every
-            , RRDSET_TYPE_STACKED
-    );
-
-    RRDDIM *rd_user   = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
-    RRDDIM *rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
-    struct rusage thread;
-
-    for(i = 0; i < statsd.threads ;i++) {
-        char id[100 + 1];
-        char title[100 + 1];
-
-        snprintfz(id, 100, "plugin_statsd_collector%d_cpu", i + 1);
-        snprintfz(title, 100, "Netdata statsd collector thread No %d CPU usage", i + 1);
-
-        statsd.collection_threads_status[i].st_cpu = rrdset_create_localhost(
-                "netdata"
-                , id
-                , NULL
-                , "statsd"
-                , "netdata.statsd_cpu"
-                , title
-                , "milliseconds/s"
-                , PLUGIN_STATSD_NAME
-                , "stats"
-                , 132002 + i
-                , statsd.update_every
-                , RRDSET_TYPE_STACKED
-        );
-
-        statsd.collection_threads_status[i].rd_user   = rrddim_add(statsd.collection_threads_status[i].st_cpu, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
-        statsd.collection_threads_status[i].rd_system = rrddim_add(statsd.collection_threads_status[i].st_cpu, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
-    }
-
-            // ----------------------------------------------------------------------------------------------------------------
+    // ----------------------------------------------------------------------------------------------------------------
     // statsd thread to turn metrics into charts
 
     usec_t step = statsd.update_every * USEC_PER_SEC;
     heartbeat_t hb;
     heartbeat_init(&hb);
     while(!netdata_exit) {
+        worker_is_idle();
         usec_t hb_dt = heartbeat_next(&hb, step);
 
+        worker_is_busy(WORKER_STATSD_FLUSH_GAUGES);
         statsd_flush_index_metrics(&statsd.gauges,     statsd_flush_gauge);
+
+        worker_is_busy(WORKER_STATSD_FLUSH_COUNTERS);
         statsd_flush_index_metrics(&statsd.counters,   statsd_flush_counter);
+
+        worker_is_busy(WORKER_STATSD_FLUSH_METERS);
         statsd_flush_index_metrics(&statsd.meters,     statsd_flush_meter);
+
+        worker_is_busy(WORKER_STATSD_FLUSH_TIMERS);
         statsd_flush_index_metrics(&statsd.timers,     statsd_flush_timer);
+
+        worker_is_busy(WORKER_STATSD_FLUSH_HISTOGRAMS);
         statsd_flush_index_metrics(&statsd.histograms, statsd_flush_histogram);
+
+        worker_is_busy(WORKER_STATSD_FLUSH_SETS);
         statsd_flush_index_metrics(&statsd.sets,       statsd_flush_set);
 
-        statsd_update_all_app_charts();
+        worker_is_busy(WORKER_STATSD_FLUSH_DICTIONARIES);
+        statsd_flush_index_metrics(&statsd.dictionaries,statsd_flush_dictionary);
 
-        getrusage(RUSAGE_THREAD, &thread);
+        worker_is_busy(WORKER_STATSD_FLUSH_STATS);
+        statsd_update_all_app_charts();
 
         if(unlikely(netdata_exit))
             break;
@@ -2498,9 +2753,6 @@ void *statsd_main(void *ptr) {
             rrdset_next(st_tcp_connects);
             rrdset_next(st_tcp_connected);
             rrdset_next(st_pcharts);
-            rrdset_next(stcpu_thread);
-            for(i = 0; i < statsd.threads ;i++)
-                rrdset_next(statsd.collection_threads_status[i].st_cpu);
         }
 
         rrddim_set_by_pointer(st_metrics, rd_metrics_gauge,        (collected_number)statsd.gauges.metrics);
@@ -2509,6 +2761,7 @@ void *statsd_main(void *ptr) {
         rrddim_set_by_pointer(st_metrics, rd_metrics_meter,        (collected_number)statsd.meters.metrics);
         rrddim_set_by_pointer(st_metrics, rd_metrics_histogram,    (collected_number)statsd.histograms.metrics);
         rrddim_set_by_pointer(st_metrics, rd_metrics_set,          (collected_number)statsd.sets.metrics);
+        rrddim_set_by_pointer(st_metrics, rd_metrics_dictionary,   (collected_number)statsd.dictionaries.metrics);
         rrdset_done(st_metrics);
 
         rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_gauge,        (collected_number)statsd.gauges.useful);
@@ -2517,6 +2770,7 @@ void *statsd_main(void *ptr) {
         rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_meter,        (collected_number)statsd.meters.useful);
         rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_histogram,    (collected_number)statsd.histograms.useful);
         rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_set,          (collected_number)statsd.sets.useful);
+        rrddim_set_by_pointer(st_useful_metrics, rd_useful_metrics_dictionary,   (collected_number)statsd.dictionaries.useful);
         rrdset_done(st_useful_metrics);
 
         rrddim_set_by_pointer(st_events,  rd_events_gauge,         (collected_number)statsd.gauges.events);
@@ -2525,6 +2779,7 @@ void *statsd_main(void *ptr) {
         rrddim_set_by_pointer(st_events,  rd_events_meter,         (collected_number)statsd.meters.events);
         rrddim_set_by_pointer(st_events,  rd_events_histogram,     (collected_number)statsd.histograms.events);
         rrddim_set_by_pointer(st_events,  rd_events_set,           (collected_number)statsd.sets.events);
+        rrddim_set_by_pointer(st_events,  rd_events_dictionary,    (collected_number)statsd.dictionaries.events);
         rrddim_set_by_pointer(st_events,  rd_events_unknown,       (collected_number)statsd.unknown_types);
         rrddim_set_by_pointer(st_events,  rd_events_errors,        (collected_number)statsd.socket_errors);
         rrdset_done(st_events);
@@ -2550,16 +2805,6 @@ void *statsd_main(void *ptr) {
 
         rrddim_set_by_pointer(st_pcharts, rd_pcharts,              (collected_number)statsd.private_charts);
         rrdset_done(st_pcharts);
-
-        rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
-        rrddim_set_by_pointer(stcpu_thread, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
-        rrdset_done(stcpu_thread);
-
-        for(i = 0; i < statsd.threads ;i++) {
-            rrddim_set_by_pointer(statsd.collection_threads_status[i].st_cpu, statsd.collection_threads_status[i].rd_user, statsd.collection_threads_status[i].rusage.ru_utime.tv_sec * 1000000ULL + statsd.collection_threads_status[i].rusage.ru_utime.tv_usec);
-            rrddim_set_by_pointer(statsd.collection_threads_status[i].st_cpu, statsd.collection_threads_status[i].rd_system, statsd.collection_threads_status[i].rusage.ru_stime.tv_sec * 1000000ULL + statsd.collection_threads_status[i].rusage.ru_stime.tv_usec);
-            rrdset_done(statsd.collection_threads_status[i].st_cpu);
-        }
     }
 
 cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement