Merging upstream version 1.37.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 168 insertions, 119 deletions

diff --git a/daemon/unit_test.c b/daemon/unit_test.c
index 8ba251b9a..f69861869 100644
--- a/daemon/unit_test.c
+++ b/daemon/unit_test.c
@@ -2,6 +2,61 @@
 
 #include "common.h"
 
+static bool cmd_arg_sanitization_test(const char *expected, const char *src, char *dst, size_t dst_size) {
+    bool ok = sanitize_command_argument_string(dst, src, dst_size);
+
+    if (!expected)
+        return ok == false;
+
+    return strcmp(expected, dst) == 0;
+}
+
+bool command_argument_sanitization_tests() {
+    char dst[1024];
+
+    for (size_t i = 0; i != 5; i++)  {
+        const char *expected = i == 4 ? "'\\''" : NULL;
+        if (cmd_arg_sanitization_test(expected, "'", dst, i) == false) {
+            fprintf(stderr, "expected: >>>%s<<<, got: >>>%s<<<\n", expected, dst);
+            return 1;
+        }
+    }
+
+    for (size_t i = 0; i != 9; i++)  {
+        const char *expected = i == 8 ? "'\\'''\\''" : NULL;
+        if (cmd_arg_sanitization_test(expected, "''", dst, i) == false) {
+            fprintf(stderr, "expected: >>>%s<<<, got: >>>%s<<<\n", expected, dst);
+            return 1;
+        }
+    }
+
+    for (size_t i = 0; i != 7; i++)  {
+        const char *expected = i == 6 ? "'\\''a" : NULL;
+        if (cmd_arg_sanitization_test(expected, "'a", dst, i) == false) {
+            fprintf(stderr, "expected: >>>%s<<<, got: >>>%s<<<\n", expected, dst);
+            return 1;
+        }
+    }
+
+    for (size_t i = 0; i != 7; i++)  {
+        const char *expected = i == 6 ? "a'\\''" : NULL;
+        if (cmd_arg_sanitization_test(expected, "a'", dst, i) == false) {
+            fprintf(stderr, "expected: >>>%s<<<, got: >>>%s<<<\n", expected, dst);
+            return 1;
+        }
+    }
+
+    for (size_t i = 0; i != 22; i++)  {
+        const char *expected = i == 21 ? "foo'\\''a'\\'''\\'''\\''b" : NULL;
+        if (cmd_arg_sanitization_test(expected, "--foo'a'''b", dst, i) == false) {
+            fprintf(stderr, "expected: >>>%s<<<, got: >>>%s<<<\n length: %zu\n", expected, dst, strlen(dst));
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
 static int check_number_printing(void) {
     struct {
         NETDATA_DOUBLE n;
@@ -1197,16 +1252,18 @@ int run_test(struct test *test)
             fprintf(stderr, "    > %s: feeding position %lu\n", test->name, c+1);
         }
 
-        fprintf(stderr, "       >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd->name, test->feed[c].value);
+        fprintf(stderr, "       >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rrddim_name(rd), test->feed[c].value);
         rrddim_set(st, "dim1", test->feed[c].value);
         last = test->feed[c].value;
 
         if(rd2) {
-            fprintf(stderr, "       >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rd2->name, test->feed2[c]);
+            fprintf(stderr, "       >> %s with value " COLLECTED_NUMBER_FORMAT "\n", rrddim_name(rd2), test->feed2[c]);
             rrddim_set(st, "dim2", test->feed2[c]);
         }
 
-        rrdset_done(st);
+        struct timeval now;
+        now_realtime_timeval(&now);
+        rrdset_timed_done(st, now, false);
 
         // align the first entry to second boundary
         if(!c) {
@@ -1231,7 +1288,7 @@ int run_test(struct test *test)
         int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
         fprintf(stderr, "    %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
             ", found " NETDATA_DOUBLE_FORMAT ", %s\n",
-            test->name, rd->name, c+1,
+            test->name, rrddim_name(rd), c+1,
             (int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
             n, v, (same)?"OK":"### E R R O R ###");
 
@@ -1243,7 +1300,7 @@ int run_test(struct test *test)
             same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
             fprintf(stderr, "    %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
                 ", found " NETDATA_DOUBLE_FORMAT ", %s\n",
-                test->name, rd2->name, c+1,
+                test->name, rrddim_name(rd2), c+1,
                 (int64_t)((rrdset_first_entry_t(st) + c * st->update_every) - time_start),
                 n, v, (same)?"OK":"### E R R O R ###");
             if(!same) errors++;
@@ -1258,39 +1315,39 @@ static int test_variable_renames(void) {
 
     fprintf(stderr, "Creating chart\n");
     RRDSET *st = rrdset_create_localhost("chart", "ID", NULL, "family", "context", "Unit Testing", "a value", "unittest", NULL, 1, 1, RRDSET_TYPE_LINE);
-    fprintf(stderr, "Created chart with id '%s', name '%s'\n", st->id, st->name);
+    fprintf(stderr, "Created chart with id '%s', name '%s'\n", rrdset_id(st), rrdset_name(st));
 
     fprintf(stderr, "Creating dimension DIM1\n");
     RRDDIM *rd1 = rrddim_add(st, "DIM1", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
-    fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd1->id, rd1->name);
+    fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rrddim_id(rd1), rrddim_name(rd1));
 
     fprintf(stderr, "Creating dimension DIM2\n");
     RRDDIM *rd2 = rrddim_add(st, "DIM2", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
-    fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rd2->id, rd2->name);
+    fprintf(stderr, "Created dimension with id '%s', name '%s'\n", rrddim_id(rd2), rrddim_name(rd2));
 
     fprintf(stderr, "Renaming chart to CHARTNAME1\n");
-    rrdset_set_name(st, "CHARTNAME1");
-    fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name);
+    rrdset_reset_name(st, "CHARTNAME1");
+    fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st));
 
     fprintf(stderr, "Renaming chart to CHARTNAME2\n");
-    rrdset_set_name(st, "CHARTNAME2");
-    fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", st->id, st->name);
+    rrdset_reset_name(st, "CHARTNAME2");
+    fprintf(stderr, "Renamed chart with id '%s' to name '%s'\n", rrdset_id(st), rrdset_name(st));
 
     fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME1\n");
-    rrddim_set_name(st, rd1, "DIM1NAME1");
-    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name);
+    rrddim_reset_name(st, rd1, "DIM1NAME1");
+    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1));
 
     fprintf(stderr, "Renaming dimension DIM1 to DIM1NAME2\n");
-    rrddim_set_name(st, rd1, "DIM1NAME2");
-    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd1->id, rd1->name);
+    rrddim_reset_name(st, rd1, "DIM1NAME2");
+    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd1), rrddim_name(rd1));
 
     fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME1\n");
-    rrddim_set_name(st, rd2, "DIM2NAME1");
-    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name);
+    rrddim_reset_name(st, rd2, "DIM2NAME1");
+    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2));
 
     fprintf(stderr, "Renaming dimension DIM2 to DIM2NAME2\n");
-    rrddim_set_name(st, rd2, "DIM2NAME2");
-    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rd2->id, rd2->name);
+    rrddim_reset_name(st, rd2, "DIM2NAME2");
+    fprintf(stderr, "Renamed dimension with id '%s' to name '%s'\n", rrddim_id(rd2), rrddim_name(rd2));
 
     BUFFER *buf = buffer_create(1);
     health_api_v1_chart_variables2json(st, buf);
@@ -1447,9 +1504,8 @@ int unit_test(long delay, long shift)
     long increment = 1000;
     collected_number i = 0;
 
-    unsigned long c, dimensions = 0;
+    unsigned long c, dimensions = rrdset_number_of_dimensions(st);
     RRDDIM *rd;
-    for(rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
 
     for(c = 0; c < 20 ;c++) {
         i += increment;
@@ -1470,8 +1526,10 @@ int unit_test(long delay, long shift)
         }
 
         // prevent it from deleting the dimensions
-        for(rd = st->dimensions ; rd ; rd = rd->next)
+        rrddim_foreach_read(rd, st) {
             rd->last_collected_time.tv_sec = st->last_collected_time.tv_sec;
+        }
+        rrddim_foreach_done(rd);
 
         rrdset_done(st);
     }
@@ -1486,10 +1544,10 @@ int unit_test(long delay, long shift)
     for(c = 0 ; c < st->counter ; c++) {
         fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10);
 
-        for(rd = st->dimensions ; rd ; rd = rd->next) {
+        rrddim_foreach_read(rd, st) {
             sn = rd->db[c];
             cn = unpack_storage_number(sn);
-            fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ")   ->   ", rd->id, cn, sn);
+            fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ")   ->   ", rrddim_id(rd), cn, sn);
 
             if(rd == rdabs) v =
                 (     oincrement
@@ -1508,6 +1566,7 @@ int unit_test(long delay, long shift)
                 ret = 1;
             }
         }
+        rrddim_foreach_done(rd);
     }
 
     if(ret)
@@ -1527,19 +1586,19 @@ int test_sqlite(void) {
         return 1;
     }
 
-    rc = sqlite3_exec(db_meta, "CREATE TABLE IF NOT EXISTS mine (id1, id2);", 0, 0, NULL);
+    rc = sqlite3_exec_monitored(db_meta, "CREATE TABLE IF NOT EXISTS mine (id1, id2);", 0, 0, NULL);
     if (rc != SQLITE_OK) {
         fprintf(stderr,"Failed to test SQLite: Create table failed\n");
         return 1;
     }
 
-    rc = sqlite3_exec(db_meta, "DELETE FROM MINE LIMIT 1;", 0, 0, NULL);
+    rc = sqlite3_exec_monitored(db_meta, "DELETE FROM MINE LIMIT 1;", 0, 0, NULL);
     if (rc != SQLITE_OK) {
         fprintf(stderr,"Failed to test SQLite: Delete with LIMIT failed\n");
         return 1;
     }
 
-    rc = sqlite3_exec(db_meta, "UPDATE MINE SET id1=1 LIMIT 1;", 0, 0, NULL);
+    rc = sqlite3_exec_monitored(db_meta, "UPDATE MINE SET id1=1 LIMIT 1;", 0, 0, NULL);
     if (rc != SQLITE_OK) {
         fprintf(stderr,"Failed to test SQLite: Update with LIMIT failed\n");
         return 1;
@@ -1548,58 +1607,24 @@ int test_sqlite(void) {
     BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE);
     char *uuid_str = "0000_000";
 
-    buffer_sprintf(sql, TABLE_ACLK_CHART, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
-    buffer_flush(sql);
-    if (rc != SQLITE_OK)
-        goto error;
-
-    buffer_sprintf(sql, TABLE_ACLK_CHART_PAYLOAD, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
-    buffer_flush(sql);
-    if (rc != SQLITE_OK)
-        goto error;
-
-    buffer_sprintf(sql, TABLE_ACLK_CHART_LATEST, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
-    if (rc != SQLITE_OK)
-        goto error;
-    buffer_flush(sql);
-
-    buffer_sprintf(sql, INDEX_ACLK_CHART, uuid_str, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
-    if (rc != SQLITE_OK)
-        goto error;
-    buffer_flush(sql);
-
-    buffer_sprintf(sql, INDEX_ACLK_CHART_LATEST, uuid_str, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
-    if (rc != SQLITE_OK)
-        goto error;
-    buffer_flush(sql);
-
-    buffer_sprintf(sql, TRIGGER_ACLK_CHART_PAYLOAD, uuid_str, uuid_str, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
-    if (rc != SQLITE_OK)
-        goto error;
-    buffer_flush(sql);
-
     buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
+    rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
     if (rc != SQLITE_OK)
         goto error;
     buffer_flush(sql);
 
     buffer_sprintf(sql, INDEX_ACLK_ALERT, uuid_str, uuid_str);
-    rc = sqlite3_exec(db_meta, buffer_tostring(sql), 0, 0, NULL);
+    rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
     if (rc != SQLITE_OK)
         goto error;
     buffer_flush(sql);
 
     buffer_free(sql);
     fprintf(stderr,"SQLite is OK\n");
+    rc = sqlite3_close_v2(db_meta);
     return 0;
 error:
+    rc = sqlite3_close_v2(db_meta);
     fprintf(stderr,"SQLite statement failed: %s\n", buffer_tostring(sql));
     buffer_free(sql);
     fprintf(stderr,"SQLite tests failed\n");
@@ -1634,28 +1659,28 @@ int unit_test_bitmap256(void) {
     if (test_bitmap.data[0] == 0xffffffffffffffff)
         fprintf(stderr, "%s() INDEX 0 is fully set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 0 is %lx expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
+        fprintf(stderr, "%s() INDEX 0 is %"PRIu64" expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
         return 1;
     }
 
     if (test_bitmap.data[1] == 0xffffffffffffffff)
         fprintf(stderr, "%s() INDEX 1 is fully set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 1 is %lx expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
+        fprintf(stderr, "%s() INDEX 1 is %"PRIu64" expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
         return 1;
     }
 
     if (test_bitmap.data[2] == 0xffffffffffffffff)
         fprintf(stderr, "%s() INDEX 2 is fully set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 2 is %lx expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
+        fprintf(stderr, "%s() INDEX 2 is %"PRIu64" expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
         return 1;
     }
 
     if (test_bitmap.data[3] == 0xffffffffffffffff)
         fprintf(stderr, "%s() INDEX 3 is fully set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 3 is %lx expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
+        fprintf(stderr, "%s() INDEX 3 is %"PRIu64" expected 0xffffffffffffffff\n", __FUNCTION__, test_bitmap.data[0]);
         return 1;
     }
 
@@ -1704,28 +1729,28 @@ int unit_test_bitmap256(void) {
     if (test_bitmap.data[0] == 0x1111111111111111)
         fprintf(stderr, "%s() INDEX 0 is 0x1111111111111111 set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 0 is %lx expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[0]);
+        fprintf(stderr, "%s() INDEX 0 is %"PRIu64" expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[0]);
         return 1;
     }
 
     if (test_bitmap.data[1] == 0x1111111111111111)
         fprintf(stderr, "%s() INDEX 1 is 0x1111111111111111 set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 1 is %lx expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[1]);
+        fprintf(stderr, "%s() INDEX 1 is %"PRIu64" expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[1]);
         return 1;
     }
 
     if (test_bitmap.data[2] == 0x1111111111111111)
         fprintf(stderr, "%s() INDEX 2 is 0x1111111111111111 set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 2 is %lx expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[2]);
+        fprintf(stderr, "%s() INDEX 2 is %"PRIu64" expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[2]);
         return 1;
     }
 
     if (test_bitmap.data[3] == 0x1111111111111111)
         fprintf(stderr, "%s() INDEX 3 is 0x1111111111111111 set OK\n", __FUNCTION__);
     else {
-        fprintf(stderr, "%s() INDEX 3 is %lx expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[3]);
+        fprintf(stderr, "%s() INDEX 3 is %"PRIu64" expected 0x1111111111111111\n", __FUNCTION__, test_bitmap.data[3]);
         return 1;
     }
 
@@ -1771,6 +1796,9 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name)
             , default_rrdpush_destination
             , default_rrdpush_api_key
             , default_rrdpush_send_charts_matching
+            , default_rrdpush_enable_replication
+            , default_rrdpush_seconds_to_replicate
+            , default_rrdpush_replication_step
             , NULL
             , 0
     );
@@ -1826,7 +1854,10 @@ static void test_dbengine_create_charts(RRDHOST *host, RRDSET *st[CHARTS], RRDDI
         for (j = 0; j < DIMS; ++j) {
             rrddim_set_by_pointer_fake_time(rd[i][j], 69, 2 * API_RELATIVE_TIME_MAX); // set first value to 69
         }
-        rrdset_done(st[i]);
+
+        struct timeval now;
+        now_realtime_timeval(&now);
+        rrdset_timed_done(st[i], now, false);
     }
     // Fluh pages for subsequent real values
     for (i = 0 ; i < CHARTS ; ++i) {
@@ -1850,6 +1881,8 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
     // feed it with the test data
     for (i = 0 ; i < CHARTS ; ++i) {
         for (j = 0 ; j < DIMS ; ++j) {
+            rd[i][j]->tiers[0]->collect_ops->change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every);
+
             rd[i][j]->last_collected_time.tv_sec =
             st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
             rd[i][j]->last_collected_time.tv_usec =
@@ -1858,7 +1891,8 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
     }
     for (c = 0; c < REGION_POINTS[current_region] ; ++c) {
         time_now += update_every; // time_now = start + (c + 1) * update_every
-        for (i = 0 ; i < CHARTS ; ++i) {
+
+         for (i = 0 ; i < CHARTS ; ++i) {
             st[i]->usec_since_last_update = USEC_PER_SEC * update_every;
 
             for (j = 0; j < DIMS; ++j) {
@@ -1866,7 +1900,12 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
                        j * REGION_POINTS[current_region] + c;
                 rrddim_set_by_pointer_fake_time(rd[i][j], next, time_now);
             }
-            rrdset_done(st[i]);
+
+            struct timeval now;
+            now.tv_sec = time_now;
+            now.tv_usec = 0;
+
+            rrdset_timed_done(st[i], now, false);
         }
     }
     return time_now; //time_end
@@ -1882,7 +1921,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
     int i, j, k, c, errors, update_every;
     collected_number last;
     NETDATA_DOUBLE value, expected;
-    struct rrddim_query_handle handle;
+    struct storage_engine_query_handle handle;
     size_t value_errors = 0, time_errors = 0;
 
     update_every = REGION_UPDATE_EVERY[current_region];
@@ -1893,13 +1932,13 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
         time_now = time_start + (c + 1) * update_every;
         for (i = 0 ; i < CHARTS ; ++i) {
             for (j = 0; j < DIMS; ++j) {
-                rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, TIER_QUERY_FETCH_SUM);
+                rd[i][j]->tiers[0]->query_ops->init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every);
                 for (k = 0; k < QUERY_BATCH; ++k) {
                     last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
                            j * REGION_POINTS[current_region] + c + k;
                     expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE)last, SN_DEFAULT_FLAGS));
 
-                    STORAGE_POINT sp = rd[i][j]->tiers[0]->query_ops.next_metric(&handle);
+                    STORAGE_POINT sp = rd[i][j]->tiers[0]->query_ops->next_metric(&handle);
                     value = sp.sum;
                     time_retrieved = sp.start_time;
                     end_time = sp.end_time;
@@ -1909,19 +1948,19 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
                         if(!value_errors)
                             fprintf(stderr, "    DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
                                 ", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
-                                st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, expected, value);
+                                    rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now + k * update_every, expected, value);
                         value_errors++;
                         errors++;
                     }
                     if(end_time != time_now + k * update_every) {
                         if(!time_errors)
                             fprintf(stderr, "    DB-engine unittest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n",
-                                st[i]->name, rd[i][j]->name, (unsigned long)time_now + k * update_every, (unsigned long)time_retrieved);
+                                    rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now + k * update_every, (unsigned long)time_retrieved);
                         time_errors++;
                         errors++;
                     }
                 }
-                rd[i][j]->tiers[0]->query_ops.finalize(&handle);
+                rd[i][j]->tiers[0]->query_ops->finalize(&handle);
             }
         }
     }
@@ -1940,7 +1979,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
                                     int current_region, time_t time_start, time_t time_end)
 {
     int update_every = REGION_UPDATE_EVERY[current_region];
-    fprintf(stderr, "%s() running on region %d, start time %ld, end time %ld, update every %d...\n", __FUNCTION__, current_region, time_start, time_end, update_every);
+    fprintf(stderr, "%s() running on region %d, start time %lld, end time %lld, update every %d...\n", __FUNCTION__, current_region, (long long)time_start, (long long)time_end, update_every);
     uint8_t same;
     time_t time_now, time_retrieved;
     int i, j, errors, value_errors = 0, time_errors = 0;
@@ -1952,22 +1991,25 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
     long points = (time_end - time_start) / update_every;
     for (i = 0 ; i < CHARTS ; ++i) {
         ONEWAYALLOC *owa = onewayalloc_create(0);
-        RRDR *r = rrd2rrdr(owa, st[i], points, time_start, time_end,
-                           RRDR_GROUPING_AVERAGE, 0, RRDR_OPTION_NATURAL_POINTS,
-                           NULL, NULL, NULL, 0, 0);
-
+        RRDR *r = rrd2rrdr_legacy(owa, st[i], points, time_start, time_end,
+                                  RRDR_GROUPING_AVERAGE, 0, RRDR_OPTION_NATURAL_POINTS,
+                                  NULL, NULL, 0, 0, QUERY_SOURCE_UNITTEST);
         if (!r) {
-            fprintf(stderr, "    DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", st[i]->name, current_region);
+            fprintf(stderr, "    DB-engine unittest %s: empty RRDR on region %d ### E R R O R ###\n", rrdset_name(st[i]), current_region);
             return ++errors;
         } else {
-            assert(r->st == st[i]);
-            for (c = 0; c != rrdr_rows(r) ; ++c) {
+            assert(r->internal.qt->request.st == st[i]);
+            for (c = 0; c != (long)rrdr_rows(r) ; ++c) {
                 RRDDIM *d;
                 time_now = time_start + (c + 1) * update_every;
                 time_retrieved = r->t[c];
 
                 // for each dimension
-                for (j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
+                rrddim_foreach_read(d, r->internal.qt->request.st) {
+                    if(unlikely(d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe
+
+                    j = (int)d_dfe.counter;
+
                     NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
                     value = cn[j];
                     assert(rd[i][j] == d);
@@ -1980,16 +2022,17 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
                         if(value_errors < 20)
                             fprintf(stderr, "    DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
                                 ", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
-                                st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
+                                    rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, expected, value);
                         value_errors++;
                     }
                     if(time_retrieved != time_now) {
                         if(time_errors < 20)
                             fprintf(stderr, "    DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",
-                                st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved);
+                                    rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, (unsigned long)time_retrieved);
                         time_errors++;
                     }
                 }
+                rrddim_foreach_done(d);
             }
             rrdr_free(owa, r);
         }
@@ -2086,24 +2129,29 @@ int test_dbengine(void)
     long point_offset = (time_start[current_region] - time_start[0]) / update_every;
     for (i = 0 ; i < CHARTS ; ++i) {
         ONEWAYALLOC *owa = onewayalloc_create(0);
-        RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every,
-                           time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0,
-                           RRDR_OPTION_NATURAL_POINTS, NULL, NULL, NULL, 0, 0);
+        RRDR *r = rrd2rrdr_legacy(owa, st[i], points, time_start[0] + update_every,
+                                  time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0,
+                                  RRDR_OPTION_NATURAL_POINTS, NULL, NULL, 0, 0, QUERY_SOURCE_UNITTEST);
+
         if (!r) {
-            fprintf(stderr, "    DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name);
+            fprintf(stderr, "    DB-engine unittest %s: empty RRDR ### E R R O R ###\n", rrdset_name(st[i]));
             ++errors;
         } else {
             long c;
 
-            assert(r->st == st[i]);
+            assert(r->internal.qt->request.st == st[i]);
             // test current region values only, since they must be left unchanged
-            for (c = point_offset ; c < point_offset + rrdr_rows(r) / REGIONS / 2 ; ++c) {
+            for (c = point_offset ; c < (long)(point_offset + rrdr_rows(r) / REGIONS / 2) ; ++c) {
                 RRDDIM *d;
                 time_t time_now = time_start[current_region] + (c - point_offset + 2) * update_every;
                 time_t time_retrieved = r->t[c];
 
                 // for each dimension
-                for(j = 0, d = r->st->dimensions ; d && j < r->d ; ++j, d = d->next) {
+                rrddim_foreach_read(d, r->internal.qt->request.st) {
+                    if(unlikely(d_dfe.counter >= r->d)) break; // d_counter is provided by the dictionary dfe
+
+                    j = (int)d_dfe.counter;
+
                     NETDATA_DOUBLE *cn = &r->v[ c * r->d ];
                     NETDATA_DOUBLE value = cn[j];
                     assert(rd[i][j] == d);
@@ -2116,16 +2164,17 @@ int test_dbengine(void)
                         if(!value_errors)
                             fprintf(stderr, "    DB-engine unittest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
                                 ", RRDR found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
-                                st[i]->name, rd[i][j]->name, (unsigned long)time_now, expected, value);
+                                    rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, expected, value);
                         value_errors++;
                     }
                     if(time_retrieved != time_now) {
                         if(!time_errors)
                             fprintf(stderr, "    DB-engine unittest %s/%s: at %lu secs, found RRDR timestamp %lu ### E R R O R ###\n",
-                                st[i]->name, rd[i][j]->name, (unsigned long)time_now, (unsigned long)time_retrieved);
+                                    rrdset_name(st[i]), rrddim_name(rd[i][j]), (unsigned long)time_now, (unsigned long)time_retrieved);
                         time_errors++;
                     }
                 }
+                rrddim_foreach_done(d);
             }
             rrdr_free(owa, r);
         }
@@ -2133,9 +2182,9 @@ int test_dbengine(void)
     }
 error_out:
     rrd_wrlock();
-    rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[0]);
+    rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
     rrdhost_delete_charts(host);
-    rrdeng_exit((struct rrdengine_instance *)host->storage_instance[0]);
+    rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
     rrd_unlock();
 
     return errors + value_errors + time_errors;
@@ -2313,7 +2362,7 @@ static void query_dbengine_chart(void *arg)
     time_t time_now, time_retrieved, end_time;
     collected_number generatedv;
     NETDATA_DOUBLE value, expected;
-    struct rrddim_query_handle handle;
+    struct storage_engine_query_handle handle;
     size_t value_errors = 0, time_errors = 0;
 
     do {
@@ -2340,23 +2389,23 @@ static void query_dbengine_chart(void *arg)
             time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
         }
 
-        rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before, TIER_QUERY_FETCH_SUM);
+        rd->tiers[0]->query_ops->init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before);
         ++thread_info->queries_nr;
         for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
             generatedv = generate_dbengine_chart_value(i, j, time_now);
             expected = unpack_storage_number(pack_storage_number((NETDATA_DOUBLE) generatedv, SN_DEFAULT_FLAGS));
 
-            if (unlikely(rd->tiers[0]->query_ops.is_finished(&handle))) {
+            if (unlikely(rd->tiers[0]->query_ops->is_finished(&handle))) {
                 if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
                     fprintf(stderr, "    DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
                         ", found data gap, ### E R R O R ###\n",
-                            st->name, rd->name, (unsigned long) time_now, expected);
+                            rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected);
                     ++thread_info->errors;
                 }
                 break;
             }
 
-            STORAGE_POINT sp = rd->tiers[0]->query_ops.next_metric(&handle);
+            STORAGE_POINT sp = rd->tiers[0]->query_ops->next_metric(&handle);
             value = sp.sum;
             time_retrieved = sp.start_time;
             end_time = sp.end_time;
@@ -2365,7 +2414,7 @@ static void query_dbengine_chart(void *arg)
                 if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
                     fprintf(stderr, "    DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
                         ", found data gap, ### E R R O R ###\n",
-                            st->name, rd->name, (unsigned long) time_now, expected);
+                            rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected);
                     ++thread_info->errors;
                 }
                 break;
@@ -2378,7 +2427,7 @@ static void query_dbengine_chart(void *arg)
                     if(!value_errors)
                        fprintf(stderr, "    DB-engine stresstest %s/%s: at %lu secs, expecting value " NETDATA_DOUBLE_FORMAT
                             ", found " NETDATA_DOUBLE_FORMAT ", ### E R R O R ###\n",
-                            st->name, rd->name, (unsigned long) time_now, expected, value);
+                                rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, expected, value);
                     value_errors++;
                     thread_info->errors++;
                 }
@@ -2388,13 +2437,13 @@ static void query_dbengine_chart(void *arg)
                     if(!time_errors)
                         fprintf(stderr,
                             "    DB-engine stresstest %s/%s: at %lu secs, found timestamp %lu ### E R R O R ###\n",
-                            st->name, rd->name, (unsigned long) time_now, (unsigned long) time_retrieved);
+                                rrdset_name(st), rrddim_name(rd), (unsigned long) time_now, (unsigned long) time_retrieved);
                     time_errors++;
                     thread_info->errors++;
                 }
             }
         }
-        rd->tiers[0]->query_ops.finalize(&handle);
+        rd->tiers[0]->query_ops->finalize(&handle);
     } while(!thread_info->done);
 
     if(value_errors)
@@ -2515,7 +2564,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
     test_duration = now_realtime_sec() - (time_start - HISTORY_SECONDS);
     if (!test_duration)
         test_duration = 1;
-    fprintf(stderr, "\nDB-engine stress test finished in %ld seconds.\n", test_duration);
+    fprintf(stderr, "\nDB-engine stress test finished in %lld seconds.\n", (long long)test_duration);
     unsigned long stored_metrics_nr = 0;
     for (i = 0 ; i < DSET_CHARTS ; ++i) {
         stored_metrics_nr += chart_threads[i]->stored_metrics_nr;
@@ -2530,8 +2579,8 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
     fprintf(stderr, "%lu metric data points were queried by %u reader threads.\n", queried_metrics_nr, QUERY_THREADS);
     fprintf(stderr, "Query starting time is randomly chosen from the beginning of the time-series up to the time of\n"
                     "the latest data point, and ending time from 1 second up to 1 hour after the starting time.\n");
-    fprintf(stderr, "Performance is %lu written data points/sec and %lu read data points/sec.\n",
-            stored_metrics_nr / test_duration, queried_metrics_nr / test_duration);
+    fprintf(stderr, "Performance is %lld written data points/sec and %lld read data points/sec.\n",
+            (long long)(stored_metrics_nr / test_duration), (long long)(queried_metrics_nr / test_duration));
 
     for (i = 0 ; i < DSET_CHARTS ; ++i) {
         freez(chart_threads[i]);
@@ -2542,9 +2591,9 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
     }
     freez(query_threads);
     rrd_wrlock();
-    rrdeng_prepare_exit((struct rrdengine_instance *)host->storage_instance[0]);
+    rrdeng_prepare_exit((struct rrdengine_instance *)host->db[0].instance);
     rrdhost_delete_charts(host);
-    rrdeng_exit((struct rrdengine_instance *)host->storage_instance[0]);
+    rrdeng_exit((struct rrdengine_instance *)host->db[0].instance);
     rrd_unlock();
 }