New upstream version 1.6.0+dfsgupstream/1.6.0+dfsg

1 files changed, 224 insertions, 531 deletions

diff --git a/src/rrd2json.c b/src/rrd2json.c
index 06747500..4d853930 100644
--- a/src/rrd2json.c
+++ b/src/rrd2json.c
@@ -2,7 +2,7 @@
 
 void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used)
 {
-    pthread_rwlock_rdlock(&st->rwlock);
+    rrdset_rdlock(st);
 
     buffer_sprintf(wb,
         "\t\t{\n"
@@ -29,7 +29,7 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions
         , st->context
         , st->title
         , st->priority
-        , st->enabled?"true":"false"
+        , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?"true":"false"
         , st->units
         , st->name
         , rrdset_type_name(st->chart_type)
@@ -43,8 +43,8 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions
 
     size_t dimensions = 0;
     RRDDIM *rd;
-    for(rd = st->dimensions; rd ; rd = rd->next) {
-        if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
+    rrddim_foreach_read(rd, st) {
+        if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) continue;
 
         memory += rd->memsize;
 
@@ -71,58 +71,99 @@ void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions
         "\n\t\t}"
         );
 
-    pthread_rwlock_unlock(&st->rwlock);
+    rrdset_unlock(st);
 }
 
 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
     rrd_stats_api_v1_chart_with_data(st, wb, NULL, NULL);
 }
 
-void rrd_stats_api_v1_charts(BUFFER *wb)
-{
+void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) {
+    static char *custom_dashboard_info_js_filename = NULL;
     size_t c, dimensions = 0, memory = 0, alarms = 0;
     RRDSET *st;
 
+    time_t now = now_realtime_sec();
+
+    if(unlikely(!custom_dashboard_info_js_filename))
+        custom_dashboard_info_js_filename = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", "");
+
     buffer_sprintf(wb, "{\n"
            "\t\"hostname\": \"%s\""
+        ",\n\t\"version\": \"%s\""
+        ",\n\t\"os\": \"%s\""
         ",\n\t\"update_every\": %d"
-        ",\n\t\"history\": %d"
+        ",\n\t\"history\": %ld"
+        ",\n\t\"custom_info\": \"%s\""
         ",\n\t\"charts\": {"
-        , localhost.hostname
-        , rrd_update_every
-        , rrd_default_history_entries
+        , host->hostname
+        , program_version
+        , host->os
+        , host->rrd_update_every
+        , host->rrd_history_entries
+        , custom_dashboard_info_js_filename
         );
 
-    pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
-    for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
-        if(st->enabled && st->dimensions) {
+    c = 0;
+    rrdhost_rdlock(host);
+    rrdset_foreach_read(st, host) {
+        if(rrdset_is_available_for_viewers(st)) {
             if(c) buffer_strcat(wb, ",");
             buffer_strcat(wb, "\n\t\t\"");
             buffer_strcat(wb, st->id);
             buffer_strcat(wb, "\": ");
             rrd_stats_api_v1_chart_with_data(st, wb, &dimensions, &memory);
+
             c++;
+            st->last_accessed_time = now;
         }
     }
 
     RRDCALC *rc;
-    for(rc = localhost.alarms; rc ; rc = rc->next) {
+    for(rc = host->alarms; rc ; rc = rc->next) {
         if(rc->rrdset)
             alarms++;
     }
-    pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
+    rrdhost_unlock(host);
 
     buffer_sprintf(wb, "\n\t}"
                     ",\n\t\"charts_count\": %zu"
                     ",\n\t\"dimensions_count\": %zu"
                     ",\n\t\"alarms_count\": %zu"
                     ",\n\t\"rrd_memory_bytes\": %zu"
-                    "\n}\n"
+                    ",\n\t\"hosts_count\": %zu"
+                    ",\n\t\"hosts\": ["
                    , c
                    , dimensions
                    , alarms
                    , memory
+                   , rrd_hosts_available
     );
+
+    if(unlikely(rrd_hosts_available > 1)) {
+        rrd_rdlock();
+        RRDHOST *h;
+        rrdhost_foreach_read(h) {
+            buffer_sprintf(wb,
+                   "%s\n\t\t{"
+                   "\n\t\t\t\"hostname\": \"%s\""
+                   "\n\t\t}"
+                   , (h != localhost) ? "," : ""
+                   , h->hostname
+            );
+        }
+        rrd_unlock();
+    }
+    else {
+        buffer_sprintf(wb,
+                "\n\t\t{"
+                "\n\t\t\t\"hostname\": \"%s\""
+                "\n\t\t}"
+                , host->hostname
+        );
+    }
+
+    buffer_sprintf(wb, "\n\t]\n}\n");
 }
 
 // ----------------------------------------------------------------------------
@@ -145,35 +186,34 @@ static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable)
 
 #define PROMETHEUS_ELEMENT_MAX 256
 
-void rrd_stats_api_v1_charts_allmetrics_prometheus(BUFFER *wb)
-{
-    pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
+void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb) {
+    rrdhost_rdlock(host);
 
-    char host[PROMETHEUS_ELEMENT_MAX + 1];
-    prometheus_name_copy(host, config_get("global", "hostname", "localhost"), PROMETHEUS_ELEMENT_MAX);
+    char hostname[PROMETHEUS_ELEMENT_MAX + 1];
+    prometheus_name_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
 
     // for each chart
     RRDSET *st;
-    for(st = localhost.rrdset_root; st ; st = st->next) {
+    rrdset_foreach_read(st, host) {
         char chart[PROMETHEUS_ELEMENT_MAX + 1];
         prometheus_name_copy(chart, st->id, PROMETHEUS_ELEMENT_MAX);
 
         buffer_strcat(wb, "\n");
-        if(st->enabled && st->dimensions) {
-            pthread_rwlock_rdlock(&st->rwlock);
+        if(rrdset_is_available_for_viewers(st)) {
+            rrdset_rdlock(st);
 
             // for each dimension
             RRDDIM *rd;
-            for(rd = st->dimensions; rd ; rd = rd->next) {
-                if(rd->counter) {
+            rrddim_foreach_read(rd, st) {
+                if(rd->collections_counter) {
                     char dimension[PROMETHEUS_ELEMENT_MAX + 1];
                     prometheus_name_copy(dimension, rd->id, PROMETHEUS_ELEMENT_MAX);
 
                     // buffer_sprintf(wb, "# HELP %s.%s %s\n", st->id, rd->id, st->units);
 
                     switch(rd->algorithm) {
-                        case RRDDIM_INCREMENTAL:
-                        case RRDDIM_PCENT_OVER_DIFF_TOTAL:
+                        case RRD_ALGORITHM_INCREMENTAL:
+                        case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
                             buffer_sprintf(wb, "# TYPE %s_%s counter\n", chart, dimension);
                             break;
 
@@ -183,21 +223,20 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus(BUFFER *wb)
                     }
 
                     // calculated_number n = (calculated_number)rd->last_collected_value * (calculated_number)(abs(rd->multiplier)) / (calculated_number)(abs(rd->divisor));
-                    // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n,
-                    //        (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000)));
+                    // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n, timeval_msec(&rd->last_collected_time));
 
                     buffer_sprintf(wb, "%s_%s{instance=\"%s\"} " COLLECTED_NUMBER_FORMAT " %llu\n",
-                            chart, dimension, host, rd->last_collected_value,
-                            (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000)));
+                            chart, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)
+                    );
 
                 }
             }
 
-            pthread_rwlock_unlock(&st->rwlock);
+            rrdset_unlock(st);
         }
     }
 
-    pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
+    rrdhost_unlock(host);
 }
 
 // ----------------------------------------------------------------------------
@@ -220,28 +259,24 @@ static inline size_t shell_name_copy(char *d, const char *s, size_t usable) {
 
 #define SHELL_ELEMENT_MAX 100
 
-void rrd_stats_api_v1_charts_allmetrics_shell(BUFFER *wb)
-{
-    pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
-
-    char host[SHELL_ELEMENT_MAX + 1];
-    shell_name_copy(host, config_get("global", "hostname", "localhost"), SHELL_ELEMENT_MAX);
+void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) {
+    rrdhost_rdlock(host);
 
     // for each chart
     RRDSET *st;
-    for(st = localhost.rrdset_root; st ; st = st->next) {
+    rrdset_foreach_read(st, host) {
         calculated_number total = 0.0;
         char chart[SHELL_ELEMENT_MAX + 1];
         shell_name_copy(chart, st->id, SHELL_ELEMENT_MAX);
 
         buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name);
-        if(st->enabled && st->dimensions) {
-            pthread_rwlock_rdlock(&st->rwlock);
+        if(rrdset_is_available_for_viewers(st)) {
+            rrdset_rdlock(st);
 
             // for each dimension
             RRDDIM *rd;
-            for(rd = st->dimensions; rd ; rd = rd->next) {
-                if(rd->counter) {
+            rrddim_foreach_read(rd, st) {
+                if(rd->collections_counter) {
                     char dimension[SHELL_ELEMENT_MAX + 1];
                     shell_name_copy(dimension, rd->id, SHELL_ELEMENT_MAX);
 
@@ -252,7 +287,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(BUFFER *wb)
                     else {
                         if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
                         n = roundl(n);
-                        if(!(rd->flags & RRDDIM_FLAG_HIDDEN)) total += n;
+                        if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
                         buffer_sprintf(wb, "NETDATA_%s_%s=\"%0.0Lf\"      # %s\n", chart, dimension, n, st->units);
                     }
                 }
@@ -260,14 +295,14 @@ void rrd_stats_api_v1_charts_allmetrics_shell(BUFFER *wb)
 
             total = roundl(total);
             buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"%0.0Lf\"      # %s\n", chart, total, st->units);
-            pthread_rwlock_unlock(&st->rwlock);
+            rrdset_unlock(st);
         }
     }
 
     buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n");
 
     RRDCALC *rc;
-    for(rc = localhost.alarms; rc ;rc = rc->next) {
+    for(rc = host->alarms; rc ;rc = rc->next) {
         if(!rc->rrdset) continue;
 
         char chart[SHELL_ELEMENT_MAX + 1];
@@ -288,159 +323,76 @@ void rrd_stats_api_v1_charts_allmetrics_shell(BUFFER *wb)
         buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
     }
 
-    pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
+    rrdhost_unlock(host);
 }
 
 // ----------------------------------------------------------------------------
 
-unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
-{
-    time_t now = now_realtime_sec();
+void rrd_stats_api_v1_charts_allmetrics_json(RRDHOST *host, BUFFER *wb) {
+    rrdhost_rdlock(host);
 
-    pthread_rwlock_rdlock(&st->rwlock);
+    buffer_strcat(wb, "{");
 
-    buffer_sprintf(wb,
-        "\t\t{\n"
-        "\t\t\t\"id\": \"%s\",\n"
-        "\t\t\t\"name\": \"%s\",\n"
-        "\t\t\t\"type\": \"%s\",\n"
-        "\t\t\t\"family\": \"%s\",\n"
-        "\t\t\t\"context\": \"%s\",\n"
-        "\t\t\t\"title\": \"%s\",\n"
-        "\t\t\t\"priority\": %ld,\n"
-        "\t\t\t\"enabled\": %d,\n"
-        "\t\t\t\"units\": \"%s\",\n"
-        "\t\t\t\"url\": \"/data/%s/%s\",\n"
-        "\t\t\t\"chart_type\": \"%s\",\n"
-        "\t\t\t\"counter\": %lu,\n"
-        "\t\t\t\"entries\": %ld,\n"
-        "\t\t\t\"first_entry_t\": %ld,\n"
-        "\t\t\t\"last_entry\": %lu,\n"
-        "\t\t\t\"last_entry_t\": %ld,\n"
-        "\t\t\t\"last_entry_secs_ago\": %ld,\n"
-        "\t\t\t\"update_every\": %d,\n"
-        "\t\t\t\"isdetail\": %d,\n"
-        "\t\t\t\"usec_since_last_update\": %llu,\n"
-        "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
-        "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
-        "\t\t\t\"dimensions\": [\n"
-        , st->id
-        , st->name
-        , st->type
-        , st->family
-        , st->context
-        , st->title
-        , st->priority
-        , st->enabled
-        , st->units
-        , st->name, options?options:""
-        , rrdset_type_name(st->chart_type)
-        , st->counter
-        , st->entries
-        , rrdset_first_entry_t(st)
-        , rrdset_last_slot(st)
-        , rrdset_last_entry_t(st)
-        , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
-        , st->update_every
-        , st->isdetail
-        , st->usec_since_last_update
-        , st->collected_total
-        , st->last_collected_total
-        );
-
-    unsigned long memory = st->memsize;
-
-    RRDDIM *rd;
-    for(rd = st->dimensions; rd ; rd = rd->next) {
-
-        memory += rd->memsize;
+    size_t chart_counter = 0;
+    size_t dimension_counter = 0;
 
-        buffer_sprintf(wb,
-            "\t\t\t\t{\n"
-            "\t\t\t\t\t\"id\": \"%s\",\n"
-            "\t\t\t\t\t\"name\": \"%s\",\n"
-            "\t\t\t\t\t\"entries\": %ld,\n"
-            "\t\t\t\t\t\"isHidden\": %d,\n"
-            "\t\t\t\t\t\"algorithm\": \"%s\",\n"
-            "\t\t\t\t\t\"multiplier\": %ld,\n"
-            "\t\t\t\t\t\"divisor\": %ld,\n"
-            "\t\t\t\t\t\"last_entry_t\": %ld,\n"
-            "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
-            "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
-            "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
-            "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
-            "\t\t\t\t\t\"memory\": %lu\n"
-            "\t\t\t\t}%s\n"
-            , rd->id
-            , rd->name
-            , rd->entries
-            , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
-            , rrddim_algorithm_name(rd->algorithm)
-            , rd->multiplier
-            , rd->divisor
-            , rd->last_collected_time.tv_sec
-            , rd->collected_value
-            , rd->calculated_value
-            , rd->last_collected_value
-            , rd->last_calculated_value
-            , rd->memsize
-            , rd->next?",":""
+    // for each chart
+    RRDSET *st;
+    rrdset_foreach_read(st, host) {
+        if(rrdset_is_available_for_viewers(st)) {
+            rrdset_rdlock(st);
+
+            buffer_sprintf(wb, "%s\n"
+                    "\t\"%s\": {\n"
+                    "\t\t\"name\":\"%s\",\n"
+                    "\t\t\"context\":\"%s\",\n"
+                    "\t\t\"units\":\"%s\",\n"
+                    "\t\t\"last_updated\": %ld,\n"
+                    "\t\t\"dimensions\": {"
+                    , chart_counter?",":""
+                    , st->id
+                    , st->name
+                    , st->context
+                    , st->units
+                    , rrdset_last_entry_t(st)
             );
-    }
-
-    buffer_sprintf(wb,
-        "\t\t\t],\n"
-        "\t\t\t\"memory\" : %lu\n"
-        "\t\t}"
-        , memory
-        );
 
-    pthread_rwlock_unlock(&st->rwlock);
-    return memory;
-}
+            chart_counter++;
+            dimension_counter = 0;
 
-#define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
-#define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
+            // for each dimension
+            RRDDIM *rd;
+            rrddim_foreach_read(rd, st) {
+                if(rd->collections_counter) {
+
+                    buffer_sprintf(wb, "%s\n"
+                            "\t\t\t\"%s\": {\n"
+                            "\t\t\t\t\"name\": \"%s\",\n"
+                            "\t\t\t\t\"value\": "
+                            , dimension_counter?",":""
+                            , rd->id
+                            , rd->name
+                    );
 
-void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
-{
-    buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
-    rrd_stats_one_json(st, options, wb);
-    buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
-}
+                    if(isnan(rd->last_stored_value))
+                        buffer_strcat(wb, "null");
+                    else
+                        buffer_sprintf(wb, CALCULATED_NUMBER_FORMAT, rd->last_stored_value);
 
-void rrd_stats_all_json(BUFFER *wb)
-{
-    unsigned long memory = 0;
-    long c;
-    RRDSET *st;
+                    buffer_strcat(wb, "\n\t\t\t}");
 
-    buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
+                    dimension_counter++;
+                }
+            }
 
-    pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
-    for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
-        if(st->enabled && st->dimensions) {
-            if(c) buffer_strcat(wb, ",\n");
-            memory += rrd_stats_one_json(st, NULL, wb);
-            c++;
+            buffer_strcat(wb, "\n\t\t}\n\t}");
+            rrdset_unlock(st);
         }
     }
-    pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
-
-    buffer_sprintf(wb, "\n\t],\n"
-        "\t\"hostname\": \"%s\",\n"
-        "\t\"update_every\": %d,\n"
-        "\t\"history\": %d,\n"
-        "\t\"memory\": %lu\n"
-        "}\n"
-        , localhost.hostname
-        , rrd_update_every
-        , rrd_default_history_entries
-        , memory
-        );
-}
-
 
+    buffer_strcat(wb, "\n}");
+    rrdhost_unlock(host);
+}
 
 // ----------------------------------------------------------------------------
 
@@ -449,6 +401,7 @@ void rrd_stats_all_json(BUFFER *wb)
 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
+#define RRDR_SELECTED   0x10 // the dimension is selected
 
 // RRDR result options
 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
@@ -541,13 +494,16 @@ static void rrdr_dump(RRDR *r)
 }
 */
 
-void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
-{
+void rrdr_disable_not_selected_dimensions(RRDR *r, uint32_t options, const char *dims) {
+    rrdset_check_rdlock(r->st);
+
+    if(unlikely(!dims || !*dims)) return;
+
     char b[strlen(dims) + 1];
     char *o = b, *tok;
     strcpy(o, dims);
 
-    long c;
+    long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
     RRDDIM *d;
 
     // disable all of them
@@ -561,16 +517,38 @@ void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 
         // find it and enable it
         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
-            if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || !strcmp(d->name, tok))) {
-                r->od[c] &= ~RRDR_HIDDEN;
+            if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || (hash == d->hash_name && !strcmp(d->name, tok)))) {
+
+                if(likely(r->od[c] & RRDR_HIDDEN)) {
+                    r->od[c] |= RRDR_SELECTED;
+                    r->od[c] &= ~RRDR_HIDDEN;
+                    dims_selected++;
+                }
 
                 // since the user needs this dimension
                 // make it appear as NONZERO, to return it
                 // even if the dimension has only zeros
-                r->od[c] |= RRDR_NONZERO;
+                // unless option non_zero is set
+                if(likely(!(options & RRDR_OPTION_NONZERO)))
+                    r->od[c] |= RRDR_NONZERO;
+
+                // count the visible dimensions
+                if(likely(r->od[c] & RRDR_NONZERO))
+                    dims_not_hidden_not_zero++;
             }
         }
     }
+
+    // check if all dimensions are hidden
+    if(unlikely(!dims_not_hidden_not_zero && dims_selected)) {
+        // there are a few selected dimensions
+        // but they are all zero
+        // enable the selected ones
+        // to avoid returning an empty chart
+        for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
+            if(unlikely(r->od[c] & RRDR_SELECTED))
+                r->od[c] |= RRDR_NONZERO;
+    }
 }
 
 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
@@ -624,6 +602,8 @@ void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 
 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 {
+    rrdset_check_rdlock(r->st);
+
     (void)dims;
 
     if(options & RRDR_OPTION_NONZERO) {
@@ -659,6 +639,8 @@ uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 {
+    rrdset_check_rdlock(r->st);
+
     long rows = rrdr_rows(r);
     long c, i;
     RRDDIM *rd;
@@ -840,6 +822,8 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t option
 
 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 {
+    rrdset_check_rdlock(r->st);
+
     //info("RRD2JSON(): %s: BEGIN", r->st->id);
     int row_annotations = 0, dates, dates_with_new = 0;
     char kq[2] = "",                    // key quote
@@ -893,14 +877,14 @@ static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
     else {
         kq[0] = '"';
         sq[0] = '"';
-        if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
-            dates = JSON_DATES_TIMESTAMP;
-            dates_with_new = 0;
-        }
-        else {
+        if(options & RRDR_OPTION_GOOGLE_JSON) {
             dates = JSON_DATES_JS;
             dates_with_new = 1;
         }
+        else {
+            dates = JSON_DATES_TIMESTAMP;
+            dates_with_new = 0;
+        }
         if( options & RRDR_OPTION_OBJECTSROWS )
             strcpy(pre_date, "      { ");
         else
@@ -1066,6 +1050,8 @@ static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 
 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 {
+    rrdset_check_rdlock(r->st);
+
     //info("RRD2CSV(): %s: BEGIN", r->st->id);
     long c, i;
     RRDDIM *d;
@@ -1171,6 +1157,8 @@ static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startlin
 }
 
 inline static calculated_number rrdr2value(RRDR *r, long i, uint32_t options, int *all_values_are_null) {
+    rrdset_check_rdlock(r->st);
+
     long c;
     RRDDIM *d;
 
@@ -1321,7 +1309,7 @@ inline static void rrdr_lock_rrdset(RRDR *r) {
         return;
     }
 
-    pthread_rwlock_rdlock(&r->st->rwlock);
+    rrdset_rdlock(r->st);
     r->has_st_lock = 1;
 }
 
@@ -1332,7 +1320,7 @@ inline static void rrdr_unlock_rrdset(RRDR *r) {
     }
 
     if(likely(r->has_st_lock)) {
-        pthread_rwlock_unlock(&r->st->rwlock);
+        rrdset_unlock(r->st);
         r->has_st_lock = 0;
     }
 }
@@ -1371,7 +1359,7 @@ static RRDR *rrdr_create(RRDSET *st, long n)
     rrdr_lock_rrdset(r);
 
     RRDDIM *rd;
-    for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
+    rrddim_foreach_read(rd, st) r->d++;
 
     r->n = n;
 
@@ -1383,8 +1371,10 @@ static RRDR *rrdr_create(RRDSET *st, long n)
     // set the hidden flag on hidden dimensions
     int c;
     for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
-        if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
-        else r->od[c] = 0;
+        if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)))
+            r->od[c] = RRDR_HIDDEN;
+        else
+            r->od[c] = 0;
     }
 
     r->c = -1;
@@ -1396,7 +1386,7 @@ static RRDR *rrdr_create(RRDSET *st, long n)
 
 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, int aligned)
 {
-    int debug = st->debug;
+    int debug = rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?1:0;
     int absolute_period_requested = -1;
 
     time_t first_entry_t = rrdset_first_entry_t(st);
@@ -1572,6 +1562,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
     // initialize them
     RRDDIM *rd;
     long c;
+    rrdset_check_rdlock(st);
     for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
         last_values[c] = 0;
         group_values[c] = (group_method == GROUP_MAX || group_method == GROUP_MIN)?NAN:0;
@@ -1600,7 +1591,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
             );
 
     r->group = group;
-    r->update_every = group * st->update_every;
+    r->update_every = (int)group * st->update_every;
     r->before = now;
     r->after = now;
 
@@ -1752,8 +1743,20 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
     return r;
 }
 
-int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *db_after, time_t *db_before, int *value_is_null)
-{
+int rrdset2value_api_v1(
+          RRDSET *st
+        , BUFFER *wb
+        , calculated_number *n
+        , const char *dimensions
+        , long points
+        , long long after
+        , long long before
+        , int group_method
+        , uint32_t options
+        , time_t *db_after
+        , time_t *db_before
+        , int *value_is_null
+) {
     RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
     if(!r) {
         if(value_is_null) *value_is_null = 1;
@@ -1778,7 +1781,7 @@ int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensio
     options = rrdr_check_options(r, options, dimensions);
 
     if(dimensions)
-        rrdr_disable_not_selected_dimensions(r, dimensions);
+        rrdr_disable_not_selected_dimensions(r, options, dimensions);
 
     if(db_after)  *db_after  = r->after;
     if(db_before) *db_before = r->before;
@@ -1790,8 +1793,20 @@ int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensio
     return 200;
 }
 
-int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
-{
+int rrdset2anything_api_v1(
+          RRDSET *st
+        , BUFFER *wb
+        , BUFFER *dimensions
+        , uint32_t format
+        , long points
+        , long long after
+        , long long before
+        , int group_method
+        , uint32_t options
+        , time_t *latest_timestamp
+) {
+    st->last_accessed_time = now_realtime_sec();
+
     RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
@@ -1806,7 +1821,7 @@ int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long
     options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
 
     if(dimensions)
-        rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
+        rrdr_disable_not_selected_dimensions(r, options, buffer_tostring(dimensions));
 
     if(latest_timestamp && rrdr_rows(r) > 0)
         *latest_timestamp = r->before;
@@ -1963,325 +1978,3 @@ int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long
     rrdr_free(r);
     return 200;
 }
-
-time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
-{
-    int c;
-    pthread_rwlock_rdlock(&st->rwlock);
-
-
-    // -------------------------------------------------------------------------
-    // switch from JSON to google JSON
-
-    char kq[2] = "\"";
-    char sq[2] = "\"";
-    switch(type) {
-        case DATASOURCE_DATATABLE_JSON:
-        case DATASOURCE_DATATABLE_JSONP:
-            kq[0] = '\0';
-            sq[0] = '\'';
-            break;
-
-        case DATASOURCE_JSON:
-        default:
-            break;
-    }
-
-
-    // -------------------------------------------------------------------------
-    // validate the parameters
-
-    if(points < 1) points = 1;
-    if(group < 1) group = 1;
-
-    if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
-    if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
-
-    // ---
-
-    // our return value (the last timestamp printed)
-    // this is required to detect re-transmit in google JSONP
-    time_t last_timestamp = 0;
-
-
-    // -------------------------------------------------------------------------
-    // find how many dimensions we have
-
-    int dimensions = 0;
-    RRDDIM *rd;
-    for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
-    if(!dimensions) {
-        pthread_rwlock_unlock(&st->rwlock);
-        buffer_strcat(wb, "No dimensions yet.");
-        return 0;
-    }
-
-
-    // -------------------------------------------------------------------------
-    // prepare various strings, to speed up the loop
-
-    char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
-    char normal_annotation[201];   snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
-    char pre_date[51];             snprintfz(pre_date,             50, "        {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
-    char post_date[21];            snprintfz(post_date,            20, "%s}", sq);
-    char pre_value[21];            snprintfz(pre_value,            20, ",{%sv%s:", kq, kq);
-    char post_value[21];           strcpy(post_value,                  "}");
-
-
-    // -------------------------------------------------------------------------
-    // checks for debugging
-
-    if(st->debug) {
-        debug(D_RRD_STATS, "%s first_entry_t = %ld, last_entry_t = %ld, duration = %ld, after = %ld, before = %ld, duration = %ld, entries_to_show = %ld, group = %ld"
-            , st->id
-            , rrdset_first_entry_t(st)
-            , rrdset_last_entry_t(st)
-            , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
-            , after
-            , before
-            , before - after
-            , points
-            , group
-            );
-
-        if(before < after)
-            debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%ld) is earlier than the oldest (%ld)", st->name, before, after);
-
-        if((before - after) > st->entries * st->update_every)
-            debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%ld) is higher than the capacity of the database (%ld)", st->name, before - after, st->entries * st->update_every);
-    }
-
-
-    // -------------------------------------------------------------------------
-    // temp arrays for keeping values per dimension
-
-    calculated_number group_values[dimensions]; // keep sums when grouping
-    int               print_hidden[dimensions]; // keep hidden flags
-    int               found_non_zero[dimensions];
-    int               found_non_existing[dimensions];
-
-    // initialize them
-    for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
-        group_values[c] = 0;
-        print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
-        found_non_zero[c] = 0;
-        found_non_existing[c] = 0;
-    }
-
-
-    // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
-    // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
-    // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
-
-    // -------------------------------------------------------------------------
-    // remove dimensions that contain only zeros
-
-    int max_loop = 1;
-    if(only_non_zero) max_loop = 2;
-
-    for(; max_loop ; max_loop--) {
-
-        // -------------------------------------------------------------------------
-        // print the JSON header
-
-        buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq);
-        buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
-        buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
-        buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
-
-        // print the header for each dimension
-        // and update the print_hidden array for the dimensions that should be hidden
-        int pc = 0;
-        for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
-            if(!print_hidden[c]) {
-                pc++;
-                buffer_sprintf(wb, ",\n     {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
-            }
-        }
-        if(!pc) {
-            buffer_sprintf(wb, ",\n     {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
-        }
-
-        // print the begin of row data
-        buffer_sprintf(wb, "\n  ],\n    %srows%s:\n [\n", kq, kq);
-
-
-        // -------------------------------------------------------------------------
-        // the main loop
-
-        int annotate_reset = 0;
-        int annotation_count = 0;
-
-        long    t = rrdset_time2slot(st, before),
-                stop_at_t = rrdset_time2slot(st, after),
-                stop_now = 0;
-
-        t -= t % group;
-
-        time_t  now = rrdset_slot2time(st, t),
-                dt = st->update_every;
-
-        long count = 0, printed = 0, group_count = 0;
-        last_timestamp = 0;
-
-        if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld"
-                    , st->id
-                    , (uint32_t)after
-                    , (uint32_t)before
-                    , points
-                    , group
-                    , st->current_entry
-                    , (uint32_t)rrdset_first_entry_t(st)
-                    , (uint32_t)rrdset_last_entry_t(st)
-                    , t
-                    , stop_at_t
-                    );
-
-        long counter = 0;
-        for(; !stop_now ; now -= dt, t--, counter++) {
-            if(t < 0) t = st->entries - 1;
-            if(t == stop_at_t) stop_now = counter;
-
-            int print_this = 0;
-
-            if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s"
-                    , st->id
-                    , t
-                    , count + 1
-                    , group_count + 1
-                    , printed
-                    , now
-                    , (group_count + 1 == group)?"PRINT":"  -  "
-                    , (now >= after && now <= before)?"RANGE":"  -  "
-                    );
-
-
-            // make sure we return data in the proper time range
-            if(now > before) continue;
-            if(now < after) break;
-
-            //if(rrdset_slot2time(st, t) != now)
-            //  error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
-
-            count++;
-            group_count++;
-
-            // check if we have to print this now
-            if(group_count == group) {
-                if(printed >= points) {
-                    // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
-                    break;
-                }
-
-                // generate the local date time
-                struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
-                if(!tm) { error("localtime() failed."); continue; }
-                if(now > last_timestamp) last_timestamp = now;
-
-                if(printed) buffer_strcat(wb, "]},\n");
-                buffer_strcat(wb, pre_date);
-                buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
-                buffer_strcat(wb, post_date);
-
-                print_this = 1;
-            }
-
-            // do the calculations
-            for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
-                storage_number n = rd->values[t];
-                calculated_number value = unpack_storage_number(n);
-
-                if(!does_storage_number_exist(n)) {
-                    value = 0.0;
-                    found_non_existing[c]++;
-                }
-                if(did_storage_number_reset(n)) annotate_reset = 1;
-
-                switch(group_method) {
-                    case GROUP_MAX:
-                        if(abs(value) > abs(group_values[c])) group_values[c] = value;
-                        break;
-
-                    case GROUP_SUM:
-                        group_values[c] += value;
-                        break;
-
-                    default:
-                    case GROUP_AVERAGE:
-                        group_values[c] += value;
-                        if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
-                        break;
-                }
-            }
-
-            if(print_this) {
-                if(annotate_reset) {
-                    annotation_count++;
-                    buffer_strcat(wb, overflow_annotation);
-                    annotate_reset = 0;
-                }
-                else
-                    buffer_strcat(wb, normal_annotation);
-
-                pc = 0;
-                for(c = 0 ; c < dimensions ; c++) {
-                    if(found_non_existing[c] == group_count) {
-                        // all entries are non-existing
-                        pc++;
-                        buffer_strcat(wb, pre_value);
-                        buffer_strcat(wb, "null");
-                        buffer_strcat(wb, post_value);
-                    }
-                    else if(!print_hidden[c]) {
-                        pc++;
-                        buffer_strcat(wb, pre_value);
-                        buffer_rrd_value(wb, group_values[c]);
-                        buffer_strcat(wb, post_value);
-
-                        if(group_values[c]) found_non_zero[c]++;
-                    }
-
-                    // reset them for the next loop
-                    group_values[c] = 0;
-                    found_non_existing[c] = 0;
-                }
-
-                // if all dimensions are hidden, print a null
-                if(!pc) {
-                    buffer_strcat(wb, pre_value);
-                    buffer_strcat(wb, "null");
-                    buffer_strcat(wb, post_value);
-                }
-
-                printed++;
-                group_count = 0;
-            }
-        }
-
-        if(printed) buffer_strcat(wb, "]}");
-        buffer_strcat(wb, "\n   ]\n}\n");
-
-        if(only_non_zero && max_loop > 1) {
-            int changed = 0;
-            for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
-                group_values[c] = 0;
-                found_non_existing[c] = 0;
-
-                if(!print_hidden[c] && !found_non_zero[c]) {
-                    changed = 1;
-                    print_hidden[c] = 1;
-                }
-            }
-
-            if(changed) buffer_flush(wb);
-            else break;
-        }
-        else break;
-
-    } // max_loop
-
-    debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %lu bytes", st->name, wb->len);
-
-    pthread_rwlock_unlock(&st->rwlock);
-    return last_timestamp;
-}