New upstream version 1.6.0+dfsg

1 files changed, 398 insertions, 158 deletions

diff --git a/src/backends.c b/src/backends.c
index 1272d0473..3e385cab5 100644
--- a/src/backends.c
+++ b/src/backends.c
@@ -1,16 +1,51 @@
 #include "common.h"
 
+// ----------------------------------------------------------------------------
+// How backends work in netdata:
+//
+// 1. There is an independent thread that runs at the required interval
+//    (for example, once every 10 seconds)
+//
+// 2. Every time it wakes, it calls the backend formatting functions to build
+//    a buffer of data. This is a very fast, memory only operation.
+//
+// 3. If the buffer already includes data, the new data are appended.
+//    If the buffer becomes too big, because the data cannot be sent, a
+//    log is written and the buffer is discarded.
+//
+// 4. Then it tries to send all the data. It blocks until all the data are sent
+//    or the socket returns an error.
+//    If the time required for this is above the interval, it starts skipping
+//    intervals, but the calculated values include the entire database, without
+//    gaps (it remembers the timestamps and continues from where it stopped).
+//
+// 5. repeats the above forever.
+//
+
 #define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001
 #define BACKEND_SOURCE_DATA_AVERAGE      0x00000002
 #define BACKEND_SOURCE_DATA_SUM          0x00000004
 
-static inline calculated_number backend_calculate_value_from_stored_data(RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) {
+
+// ----------------------------------------------------------------------------
+// helper functions for backends
+
+// calculate the SUM or AVERAGE of a dimension, for any timeframe
+// may return NAN if the database does not have any value in the give timeframe
+
+static inline calculated_number backend_calculate_value_from_stored_data(
+          RRDSET *st                // the chart
+        , RRDDIM *rd                // the dimension
+        , time_t after              // the start timestamp
+        , time_t before             // the end timestamp
+        , uint32_t options          // BACKEND_SOURCE_* bitmap
+) {
+    // find the edges of the rrd database for this chart
     time_t first_t = rrdset_first_entry_t(st);
-    time_t last_t = rrdset_last_entry_t(st);
+    time_t last_t  = rrdset_last_entry_t(st);
 
-    if(unlikely(before - after < st->update_every && after != after - after % st->update_every))
-        // when st->update_every is bigger than the frequency we send data to backend
-        // skip the iterations that are not aligned to the database
+    if(unlikely(before < first_t || after > last_t))
+        // the chart has not been updated in the wanted timeframe
         return NAN;
 
     // align the time-frame
@@ -22,7 +57,7 @@ static inline calculated_number backend_calculate_value_from_stored_data(RRDSET
         after = first_t;
 
     if(unlikely(after > before))
-        // this can happen when the st->update_every > before - after
+        // this can happen when st->update_every > before - after
         before = after;
 
     if(unlikely(before > last_t))
@@ -36,14 +71,20 @@ static inline calculated_number backend_calculate_value_from_stored_data(RRDSET
             slot, stop_now = 0;
 
     for(slot = start_at_slot; !stop_now ; slot--) {
+
         if(unlikely(slot < 0)) slot = st->entries - 1;
         if(unlikely(slot == stop_at_slot)) stop_now = 1;
 
         storage_number n = rd->values[slot];
-        if(unlikely(!does_storage_number_exist(n))) continue;
+
+        if(unlikely(!does_storage_number_exist(n))) {
+            // not collected
+            continue;
+        }
 
         calculated_number value = unpack_storage_number(n);
         sum += value;
+
         counter++;
     }
 
@@ -56,84 +97,295 @@ static inline calculated_number backend_calculate_value_from_stored_data(RRDSET
     return sum / (calculated_number)counter;
 }
 
-static inline int format_dimension_collected_graphite_plaintext(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) {
+
+// discard a response received by a backend
+// after logging a simple of it to error.log
+
+static inline int discard_response(BUFFER *b, const char *backend) {
+    char sample[1024];
+    const char *s = buffer_tostring(b);
+    char *d = sample, *e = &sample[sizeof(sample) - 1];
+
+    for(; *s && d < e ;s++) {
+        char c = *s;
+        if(unlikely(!isprint(c))) c = ' ';
+        *d++ = c;
+    }
+    *d = '\0';
+
+    info("Received %zu bytes from %s backend. Ignoring them. Sample: '%s'", buffer_strlen(b), backend, sample);
+    buffer_flush(b);
+    return 0;
+}
+
+
+// ----------------------------------------------------------------------------
+// graphite backend
+
+static inline int format_dimension_collected_graphite_plaintext(
+          BUFFER *b                 // the buffer to write data to
+        , const char *prefix        // the prefix to use
+        , RRDHOST *host             // the host this chart comes from
+        , const char *hostname      // the hostname (to override host->hostname)
+        , RRDSET *st                // the chart
+        , RRDDIM *rd                // the dimension
+        , time_t after              // the start timestamp
+        , time_t before             // the end timestamp
+        , uint32_t options          // BACKEND_SOURCE_* bitmap
+) {
     (void)host;
     (void)after;
     (void)before;
     (void)options;
-    buffer_sprintf(b, "%s.%s.%s.%s " COLLECTED_NUMBER_FORMAT " %u\n", prefix, hostname, st->id, rd->id, rd->last_collected_value, (uint32_t)rd->last_collected_time.tv_sec);
+
+    buffer_sprintf(
+            b
+            , "%s.%s.%s.%s " COLLECTED_NUMBER_FORMAT " %u\n"
+            , prefix
+            , hostname
+            , st->id
+            , rd->id
+            , rd->last_collected_value
+            , (uint32_t)rd->last_collected_time.tv_sec
+    );
+
     return 1;
 }
 
-static inline int format_dimension_stored_graphite_plaintext(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) {
+static inline int format_dimension_stored_graphite_plaintext(
+          BUFFER *b                 // the buffer to write data to
+        , const char *prefix        // the prefix to use
+        , RRDHOST *host             // the host this chart comes from
+        , const char *hostname      // the hostname (to override host->hostname)
+        , RRDSET *st                // the chart
+        , RRDDIM *rd                // the dimension
+        , time_t after              // the start timestamp
+        , time_t before             // the end timestamp
+        , uint32_t options          // BACKEND_SOURCE_* bitmap
+) {
     (void)host;
+
     calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options);
+
     if(!isnan(value)) {
-        buffer_sprintf(b, "%s.%s.%s.%s " CALCULATED_NUMBER_FORMAT " %u\n", prefix, hostname, st->id, rd->id, value, (uint32_t) before);
+
+        buffer_sprintf(
+                b
+                , "%s.%s.%s.%s " CALCULATED_NUMBER_FORMAT " %u\n"
+                , prefix
+                , hostname
+                , st->id
+                , rd->id
+                , value
+                , (uint32_t) before
+        );
+
         return 1;
     }
     return 0;
 }
 
-static inline int format_dimension_collected_opentsdb_telnet(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) {
+static inline int process_graphite_response(BUFFER *b) {
+    return discard_response(b, "graphite");
+}
+
+
+// ----------------------------------------------------------------------------
+// opentsdb backend
+
+static inline int format_dimension_collected_opentsdb_telnet(
+          BUFFER *b                 // the buffer to write data to
+        , const char *prefix        // the prefix to use
+        , RRDHOST *host             // the host this chart comes from
+        , const char *hostname      // the hostname (to override host->hostname)
+        , RRDSET *st                // the chart
+        , RRDDIM *rd                // the dimension
+        , time_t after              // the start timestamp
+        , time_t before             // the end timestamp
+        , uint32_t options          // BACKEND_SOURCE_* bitmap
+) {
     (void)host;
     (void)after;
     (void)before;
     (void)options;
-    buffer_sprintf(b, "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s\n", prefix, st->id, rd->id, (uint32_t)rd->last_collected_time.tv_sec, rd->last_collected_value, hostname);
+
+    buffer_sprintf(
+            b
+            , "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s\n"
+            , prefix
+            , st->id
+            , rd->id
+            , (uint32_t)rd->last_collected_time.tv_sec
+            , rd->last_collected_value
+            , hostname
+    );
+
     return 1;
 }
 
-static inline int format_dimension_stored_opentsdb_telnet(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) {
+static inline int format_dimension_stored_opentsdb_telnet(
+          BUFFER *b                 // the buffer to write data to
+        , const char *prefix        // the prefix to use
+        , RRDHOST *host             // the host this chart comes from
+        , const char *hostname      // the hostname (to override host->hostname)
+        , RRDSET *st                // the chart
+        , RRDDIM *rd                // the dimension
+        , time_t after              // the start timestamp
+        , time_t before             // the end timestamp
+        , uint32_t options          // BACKEND_SOURCE_* bitmap
+) {
     (void)host;
+
     calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options);
+
     if(!isnan(value)) {
-        buffer_sprintf(b, "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s\n", prefix, st->id, rd->id, (uint32_t) before, value, hostname);
+
+        buffer_sprintf(
+                b
+                , "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s\n"
+                , prefix
+                , st->id
+                , rd->id
+                , (uint32_t) before
+                , value
+                , hostname
+        );
+
         return 1;
     }
     return 0;
 }
 
-static inline int process_graphite_response(BUFFER *b) {
-    char sample[1024];
-    const char *s = buffer_tostring(b);
-    char *d = sample, *e = &sample[sizeof(sample) - 1];
+static inline int process_opentsdb_response(BUFFER *b) {
+    return discard_response(b, "opentsdb");
+}
 
-    for(; *s && d < e ;s++) {
-        char c = *s;
-        if(unlikely(!isprint(c))) c = ' ';
-        *d++ = c;
-    }
-    *d = '\0';
 
-    info("Received %zu bytes from graphite backend. Ignoring them. Sample: '%s'", buffer_strlen(b), sample);
-    buffer_flush(b);
-    return 0;
+// ----------------------------------------------------------------------------
+// json backend
+
+static inline int format_dimension_collected_json_plaintext(
+          BUFFER *b                 // the buffer to write data to
+        , const char *prefix        // the prefix to use
+        , RRDHOST *host             // the host this chart comes from
+        , const char *hostname      // the hostname (to override host->hostname)
+        , RRDSET *st                // the chart
+        , RRDDIM *rd                // the dimension
+        , time_t after              // the start timestamp
+        , time_t before             // the end timestamp
+        , uint32_t options          // BACKEND_SOURCE_* bitmap
+) {
+    (void)host;
+    (void)after;
+    (void)before;
+    (void)options;
+
+    buffer_sprintf(b, "{"
+        "\"prefix\":\"%s\","
+        "\"hostname\":\"%s\","
+
+        "\"chart_id\":\"%s\","
+        "\"chart_name\":\"%s\","
+        "\"chart_family\":\"%s\","
+        "\"chart_context\": \"%s\","
+        "\"chart_type\":\"%s\","
+        "\"units\": \"%s\","
+
+        "\"id\":\"%s\","
+        "\"name\":\"%s\","
+        "\"value\":" COLLECTED_NUMBER_FORMAT ","
+
+        "\"timestamp\": %u}\n", 
+            prefix,
+            hostname,
+
+            st->id,
+            st->name,
+            st->family,
+            st->context,
+            st->type,
+            st->units,
+
+            rd->id,
+            rd->name,
+            rd->last_collected_value,
+
+            (uint32_t)rd->last_collected_time.tv_sec
+    );
+
+    return 1;
 }
 
-static inline int process_opentsdb_response(BUFFER *b) {
-    char sample[1024];
-    const char *s = buffer_tostring(b);
-    char *d = sample, *e = &sample[sizeof(sample) - 1];
+static inline int format_dimension_stored_json_plaintext(
+          BUFFER *b                 // the buffer to write data to
+        , const char *prefix        // the prefix to use
+        , RRDHOST *host             // the host this chart comes from
+        , const char *hostname      // the hostname (to override host->hostname)
+        , RRDSET *st                // the chart
+        , RRDDIM *rd                // the dimension
+        , time_t after              // the start timestamp
+        , time_t before             // the end timestamp
+        , uint32_t options          // BACKEND_SOURCE_* bitmap
+) {
+    (void)host;
 
-    for(; *s && d < e ;s++) {
-        char c = *s;
-        if(unlikely(!isprint(c))) c = ' ';
-        *d++ = c;
-    }
-    *d = '\0';
+    calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options);
 
-    info("Received %zu bytes from opentsdb backend. Ignoring them. Sample: '%s'", buffer_strlen(b), sample);
-    buffer_flush(b);
+    if(!isnan(value)) {
+        buffer_sprintf(b, "{"
+            "\"prefix\":\"%s\","
+            "\"hostname\":\"%s\","
+
+            "\"chart_id\":\"%s\","
+            "\"chart_name\":\"%s\","
+            "\"chart_family\":\"%s\","
+            "\"chart_context\": \"%s\","
+            "\"chart_type\":\"%s\","
+            "\"units\": \"%s\","
+
+            "\"id\":\"%s\","
+            "\"name\":\"%s\","
+            "\"value\":" CALCULATED_NUMBER_FORMAT ","
+
+            "\"timestamp\": %u}\n", 
+                prefix,
+                hostname,
+                
+                st->id,
+                st->name,
+                st->family,
+                st->context,
+                st->type,
+                st->units,
+
+                rd->id,
+                rd->name,
+                value, 
+                
+                (uint32_t)before
+        );
+        
+        return 1;
+    }
     return 0;
 }
 
+static inline int process_json_response(BUFFER *b) {
+    return discard_response(b, "json");
+}
+
+
+// ----------------------------------------------------------------------------
+// the backend thread
+
 void *backends_main(void *ptr) {
+    int default_port = 0;
+    int sock = -1;
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
 
     BUFFER *b = buffer_create(1), *response = buffer_create(1);
-    int (*backend_request_formatter)(BUFFER *b, const char *prefix, RRDHOST *host, const char *hostname, RRDSET *st, RRDDIM *rd, time_t after, time_t before, uint32_t options) = NULL;
-    int (*backend_response_checker)(BUFFER *b) = NULL;
+    int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, uint32_t) = NULL;
+    int (*backend_response_checker)(BUFFER *) = NULL;
 
     info("BACKEND thread created with task id %d", gettid());
 
@@ -150,22 +402,21 @@ void *backends_main(void *ptr) {
             .tv_sec = 0,
             .tv_usec = 0
     };
-    int default_port = 0;
-    int sock = -1;
     uint32_t options;
-    int enabled = config_get_boolean("backend", "enabled", 0);
-    const char *source = config_get("backend", "data source", "average");
-    const char *type = config_get("backend", "type", "graphite");
-    const char *destination = config_get("backend", "destination", "localhost");
-    const char *prefix = config_get("backend", "prefix", "netdata");
-    const char *hostname = config_get("backend", "hostname", localhost.hostname);
-    int frequency = (int)config_get_number("backend", "update every", 10);
-    int buffer_on_failures = (int)config_get_number("backend", "buffer on failures", 10);
-    long timeoutms = config_get_number("backend", "timeout ms", frequency * 2 * 1000);
+    int enabled             = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0);
+    const char *source      = config_get(CONFIG_SECTION_BACKEND, "data source", "average");
+    const char *type        = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");
+    const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost");
+    const char *prefix      = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");
+    const char *hostname    = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname);
+    int frequency           = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", 10);
+    int buffer_on_failures  = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10);
+    long timeoutms          = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", frequency * 2 * 1000);
 
     // ------------------------------------------------------------------------
     // validate configuration options
     // and prepare for sending data to our backend
+
     if(!enabled || frequency < 1)
         goto cleanup;
 
@@ -183,23 +434,49 @@ void *backends_main(void *ptr) {
         goto cleanup;
     }
 
+    if(timeoutms < 1) {
+        error("BACKED invalid timeout %ld ms given. Assuming %d ms.", timeoutms, frequency * 2 * 1000);
+        timeoutms = frequency * 2 * 1000;
+    }
+    timeout.tv_sec  = (timeoutms * 1000) / 1000000;
+    timeout.tv_usec = (timeoutms * 1000) % 1000000;
+
+
+    // ------------------------------------------------------------------------
+    // select the backend type
+
     if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
+
         default_port = 2003;
+        backend_response_checker = process_graphite_response;
+
         if(options == BACKEND_SOURCE_DATA_AS_COLLECTED)
             backend_request_formatter = format_dimension_collected_graphite_plaintext;
         else
             backend_request_formatter = format_dimension_stored_graphite_plaintext;
 
-        backend_response_checker = process_graphite_response;
     }
     else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
+
         default_port = 4242;
+        backend_response_checker = process_opentsdb_response;
+
         if(options == BACKEND_SOURCE_DATA_AS_COLLECTED)
             backend_request_formatter = format_dimension_collected_opentsdb_telnet;
         else
             backend_request_formatter = format_dimension_stored_opentsdb_telnet;
 
-        backend_response_checker = process_opentsdb_response;
+    }
+    else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
+
+        default_port = 5448;
+        backend_response_checker = process_json_response;
+
+        if (options == BACKEND_SOURCE_DATA_AS_COLLECTED)
+            backend_request_formatter = format_dimension_collected_json_plaintext;
+        else
+            backend_request_formatter = format_dimension_stored_json_plaintext;
+
     }
     else {
         error("Unknown backend type '%s'", type);
@@ -211,15 +488,9 @@ void *backends_main(void *ptr) {
         goto cleanup;
     }
 
-    if(timeoutms < 1) {
-        error("BACKED invalid timeout %ld ms given. Assuming %d ms.", timeoutms, frequency * 2 * 1000);
-        timeoutms = frequency * 2 * 1000;
-    }
-    timeout.tv_sec  = (timeoutms * 1000) / 1000000;
-    timeout.tv_usec = (timeoutms * 1000) % 1000000;
 
     // ------------------------------------------------------------------------
-    // prepare the charts for monitoring the backend
+    // prepare the charts for monitoring the backend operation
 
     struct rusage thread;
 
@@ -238,32 +509,23 @@ void *backends_main(void *ptr) {
             chart_backend_reconnects = 0,
             chart_backend_latency = 0;
 
-    RRDSET *chart_metrics = rrdset_find("netdata.backend_metrics");
-    if(!chart_metrics) {
-        chart_metrics = rrdset_create("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, frequency, RRDSET_TYPE_LINE);
-        rrddim_add(chart_metrics, "buffered", NULL,  1, 1, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_metrics, "lost",     NULL,  1, 1, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_metrics, "sent",     NULL,  1, 1, RRDDIM_ABSOLUTE);
-    }
+    RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", 130600, frequency, RRDSET_TYPE_LINE);
+    rrddim_add(chart_metrics, "buffered", NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_metrics, "lost",     NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_metrics, "sent",     NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
 
-    RRDSET *chart_bytes = rrdset_find("netdata.backend_bytes");
-    if(!chart_bytes) {
-        chart_bytes = rrdset_create("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, frequency, RRDSET_TYPE_AREA);
-        rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_bytes, "lost",     NULL, 1, 1024, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_bytes, "sent",     NULL, 1, 1024, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRDDIM_ABSOLUTE);
-    }
+    RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", 130610, frequency, RRDSET_TYPE_AREA);
+    rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_bytes, "lost",     NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_bytes, "sent",     NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
 
-    RRDSET *chart_ops = rrdset_find("netdata.backend_ops");
-    if(!chart_ops) {
-        chart_ops = rrdset_create("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, frequency, RRDSET_TYPE_LINE);
-        rrddim_add(chart_ops, "write",     NULL, 1, 1, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_ops, "discard",   NULL, 1, 1, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_ops, "failure",   NULL, 1, 1, RRDDIM_ABSOLUTE);
-        rrddim_add(chart_ops, "read",      NULL, 1, 1, RRDDIM_ABSOLUTE);
-    }
+    RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", 130630, frequency, RRDSET_TYPE_LINE);
+    rrddim_add(chart_ops, "write",     NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_ops, "discard",   NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_ops, "failure",   NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+    rrddim_add(chart_ops, "read",      NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
 
     /*
      * this is misleading - we can only measure the time we need to send data
@@ -272,19 +534,14 @@ void *backends_main(void *ptr) {
      *
      * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html
      *
-    RRDSET *chart_latency = rrdset_find("netdata.backend_latency");
-    if(!chart_latency) {
-        chart_latency = rrdset_create("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, frequency, RRDSET_TYPE_AREA);
-        rrddim_add(chart_latency, "latency",   NULL,  1, 1000, RRDDIM_ABSOLUTE);
-    }
+    RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", 130620, frequency, RRDSET_TYPE_AREA);
+    rrddim_add(chart_latency, "latency",   NULL,  1, 1000, RRD_ALGORITHM_ABSOLUTE);
     */
 
-    RRDSET *chart_rusage = rrdset_find("netdata.backend_thread_cpu");
-    if(!chart_rusage) {
-        chart_rusage = rrdset_create("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, frequency, RRDSET_TYPE_STACKED);
-        rrddim_add(chart_rusage, "user",   NULL, 1, 1000, RRDDIM_INCREMENTAL);
-        rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
-    }
+    RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", 130630, frequency, RRDSET_TYPE_STACKED);
+    rrddim_add(chart_rusage, "user",   NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
+    rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
+
 
     // ------------------------------------------------------------------------
     // prepare the backend main loop
@@ -292,49 +549,49 @@ void *backends_main(void *ptr) {
     info("BACKEND configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, frequency, hostname, prefix);
 
     usec_t step_ut = frequency * USEC_PER_SEC;
-    usec_t random_ut = now_realtime_usec() % (step_ut / 2);
-    time_t before = (time_t)((now_realtime_usec() - step_ut) / USEC_PER_SEC);
-    time_t after = before;
+    time_t after = now_realtime_sec();
     int failures = 0;
+    heartbeat_t hb;
+    heartbeat_init(&hb);
 
     for(;;) {
-        // ------------------------------------------------------------------------
-        // wait for the next iteration point
 
-        usec_t now_ut = now_realtime_usec();
-        usec_t next_ut = now_ut - (now_ut % step_ut) + step_ut;
-        before = (time_t)(next_ut / USEC_PER_SEC);
-
-        // add a little delay (1/4 of the step) plus some randomness
-        next_ut += (step_ut / 4) + random_ut;
+        // ------------------------------------------------------------------------
+        // Wait for the next iteration point.
+        heartbeat_next(&hb, step_ut);
+        time_t before = now_realtime_sec();
 
-        while(now_ut < next_ut) {
-            sleep_usec(next_ut - now_ut);
-            now_ut = now_realtime_usec();
-        }
 
         // ------------------------------------------------------------------------
         // add to the buffer the data we need to send to the backend
 
-        RRDSET *st;
         int pthreadoldcancelstate;
 
         if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &pthreadoldcancelstate) != 0))
             error("Cannot set pthread cancel state to DISABLE.");
 
-        rrdhost_rdlock(&localhost);
-        for(st = localhost.rrdset_root; st ;st = st->next) {
-            pthread_rwlock_rdlock(&st->rwlock);
+        rrd_rdlock();
+        RRDHOST *host;
+        rrdhost_foreach_read(host) {
+            if(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE)
+                continue;
 
-            RRDDIM *rd;
-            for(rd = st->dimensions; rd ;rd = rd->next) {
-                if(rd->last_collected_time.tv_sec >= after)
-                    chart_buffered_metrics += backend_request_formatter(b, prefix, &localhost, hostname, st, rd, after, before, options);
-            }
+            rrdhost_rdlock(host);
 
-            pthread_rwlock_unlock(&st->rwlock);
+            RRDSET *st;
+            rrdset_foreach_read(st, host) {
+                rrdset_rdlock(st);
+
+                RRDDIM *rd;
+                rrddim_foreach_read(rd, st) {
+                    if(rd->last_collected_time.tv_sec >= after)
+                        chart_buffered_metrics += backend_request_formatter(b, prefix, host, (host == localhost)?hostname:host->hostname, st, rd, after, before, options);
+                }
+                rrdset_unlock(st);
+            }
+            rrdhost_unlock(host);
         }
-        rrdhost_unlock(&localhost);
+        rrd_unlock();
 
         if(unlikely(pthread_setcancelstate(pthreadoldcancelstate, NULL) != 0))
             error("Cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate);
@@ -360,6 +617,10 @@ void *backends_main(void *ptr) {
         //fprintf(stderr, "\nBACKEND BEGIN:\n%s\nBACKEND END\n", buffer_tostring(b)); // FIXME
         //fprintf(stderr, "after = %lu, before = %lu\n", after, before);
 
+        // prepare for the next iteration
+        // to add incrementally data to buffer
+        after = before;
+
         // ------------------------------------------------------------------------
         // if we are connected, receive a response, without blocking
 
@@ -399,28 +660,13 @@ void *backends_main(void *ptr) {
         // if we are not connected, connect to a backend server
 
         if(unlikely(sock == -1)) {
-            usec_t start_ut = now_realtime_usec();
-            const char *s = destination;
-            while(*s) {
-                const char *e = s;
-
-                // skip separators, moving both s(tart) and e(nd)
-                while(isspace(*e) || *e == ',') s = ++e;
-
-                // move e(nd) to the first separator
-                while(*e && !isspace(*e) && *e != ',') e++;
-
-                // is there anything?
-                if(!*s || s == e) break;
-
-                char buf[e - s + 1];
-                strncpyz(buf, s, e - s);
-                chart_backend_reconnects++;
-                sock = connect_to(buf, default_port, &timeout);
-                if(sock != -1) break;
-                s = e;
-            }
-            chart_backend_latency += now_realtime_usec() - start_ut;
+            usec_t start_ut = now_monotonic_usec();
+            size_t reconnects = 0;
+
+            sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
+
+            chart_backend_reconnects += reconnects;
+            chart_backend_latency += now_monotonic_usec() - start_ut;
         }
 
         if(unlikely(netdata_exit)) break;
@@ -430,14 +676,14 @@ void *backends_main(void *ptr) {
 
         if(likely(sock != -1)) {
             size_t len = buffer_strlen(b);
-            usec_t start_ut = now_realtime_usec();
+            usec_t start_ut = now_monotonic_usec();
             int flags = 0;
 #ifdef MSG_NOSIGNAL
             flags += MSG_NOSIGNAL;
 #endif
 
             ssize_t written = send(sock, buffer_tostring(b), len, flags);
-            chart_backend_latency += now_realtime_usec() - start_ut;
+            chart_backend_latency += now_monotonic_usec() - start_ut;
             if(written != -1 && (size_t)written == len) {
                 // we sent the data successfully
                 chart_transmission_successes++;
@@ -465,12 +711,6 @@ void *backends_main(void *ptr) {
                 close(sock);
                 sock = -1;
             }
-
-            // either the buffer is empty
-            // or is holding the data we couldn't send
-            // so, make sure the next iteration will continue
-            // from where we are now
-            after = before;
         }
         else {
             error("Failed to update database backend '%s'", destination);
@@ -495,7 +735,7 @@ void *backends_main(void *ptr) {
         // ------------------------------------------------------------------------
         // update the monitoring charts
 
-        if(chart_ops->counter_done) rrdset_next(chart_ops);
+        if(likely(chart_ops->counter_done)) rrdset_next(chart_ops);
         rrddim_set(chart_ops, "read",         chart_receptions);
         rrddim_set(chart_ops, "write",        chart_transmission_successes);
         rrddim_set(chart_ops, "discard",      chart_data_lost_events);
@@ -503,13 +743,13 @@ void *backends_main(void *ptr) {
         rrddim_set(chart_ops, "reconnect",    chart_backend_reconnects);
         rrdset_done(chart_ops);
 
-        if(chart_metrics->counter_done) rrdset_next(chart_metrics);
+        if(likely(chart_metrics->counter_done)) rrdset_next(chart_metrics);
         rrddim_set(chart_metrics, "buffered", chart_buffered_metrics);
         rrddim_set(chart_metrics, "lost",     chart_lost_metrics);
         rrddim_set(chart_metrics, "sent",     chart_sent_metrics);
         rrdset_done(chart_metrics);
 
-        if(chart_bytes->counter_done) rrdset_next(chart_bytes);
+        if(likely(chart_bytes->counter_done)) rrdset_next(chart_bytes);
         rrddim_set(chart_bytes, "buffered",   chart_buffered_bytes);
         rrddim_set(chart_bytes, "lost",       chart_lost_bytes);
         rrddim_set(chart_bytes, "sent",       chart_sent_bytes);
@@ -517,13 +757,13 @@ void *backends_main(void *ptr) {
         rrdset_done(chart_bytes);
 
         /*
-        if(chart_latency->counter_done) rrdset_next(chart_latency);
+        if(likely(chart_latency->counter_done)) rrdset_next(chart_latency);
         rrddim_set(chart_latency, "latency",  chart_backend_latency);
         rrdset_done(chart_latency);
         */
 
         getrusage(RUSAGE_THREAD, &thread);
-        if(chart_rusage->counter_done) rrdset_next(chart_rusage);
+        if(likely(chart_rusage->counter_done)) rrdset_next(chart_rusage);
         rrddim_set(chart_rusage, "user",   thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
         rrddim_set(chart_rusage, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
         rrdset_done(chart_rusage);