Merging upstream version 1.11.1+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 40 insertions, 18 deletions

diff --git a/database/README.md b/database/README.md
index 8f5e3a6df..68156f8a4 100644
--- a/database/README.md
+++ b/database/README.md
@@ -1,4 +1,4 @@
-# netdata database
+# Netdata database
 
 Although `netdata` does all its calculations using `long double`, it stores all values using
 a [custom-made 32-bit number](../libnetdata/storage_number/).
@@ -26,17 +26,17 @@ Currently netdata supports 5 memory modes:
 
 1. `ram`, data are purely in memory. Data are never saved on disk. This mode uses `mmap()` and
    supports [KSM](#ksm).
-   
+
 2. `save`, (the default) data are only in RAM while netdata runs and are saved to / loaded from
    disk on netdata restart. It also uses `mmap()` and supports [KSM](#ksm).
-   
+
 3. `map`, data are in memory mapped files. This works like the swap. Keep in mind though, this
    will have a constant write on your disk. When netdata writes data on its memory, the Linux kernel
    marks the related memory pages as dirty and automatically starts updating them on disk.
    Unfortunately we cannot control how frequently this works. The Linux kernel uses exactly the
    same algorithm it uses for its swap memory. Check below for additional information on running a
    dedicated central netdata server. This mode uses `mmap()` but does not support [KSM](#ksm).
-   
+
 4. `none`, without a database (collected metrics can only be streamed to another netdata).
 
 5. `alloc`, like `ram` but it uses `calloc()` and does not support [KSM](#ksm). This mode is the
@@ -73,9 +73,9 @@ by netdata. Of course experiment a bit. On very weak devices you might have to u
 You can also disable [data collection plugins](../collectors) you don't need.
 Disabling such plugins will also free both CPU and RAM resources.
 
-## running a dedicated central netdata server
+## Running a dedicated central netdata server
 
-netdata allows streaming data between netdata nodes. This allows us to have a central netdata
+Netdata allows streaming data between netdata nodes. This allows us to have a central netdata
 server that will maintain the entire database for all nodes, and will also run health checks/alarms
 for all nodes.
 
@@ -166,7 +166,7 @@ netdata, each byte at the in-memory database will be updated just once per day).
 
 KSM is a solution that will provide 60+% memory savings to netdata.
 
-#### Enable KSM in kernel
+### Enable KSM in kernel
 
 You need to run a kernel compiled with:
 
@@ -186,7 +186,7 @@ The files that `CONFIG_KSM=y` offers include:
 
 So, by default `ksmd` is just disabled. It will not harm performance and the user/admin can control the CPU resources he/she is willing `ksmd` to use.
 
-#### Run `ksmd` kernel daemon
+### Run `ksmd` kernel daemon
 
 To activate / run `ksmd` you need to run:
 
diff --git a/database/rrd.h b/database/rrd.h
index 19eb100cd..24705ebb1 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -176,7 +176,9 @@ struct rrddim {
     char *cache_filename;                           // the filename we load/save from/to this set
 
     size_t collections_counter;                     // the number of times we added values to this rrdim
-    size_t unused[10];
+    size_t unused[9];
+
+    collected_number collected_value_max;           // the absolute maximum of the collected value
 
     unsigned int updated:1;                         // 1 when the dimension has been updated since the last processing
     unsigned int exposed:1;                         // 1 when set what have sent this dimension to the central netdata
diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h
index 2235ecaa1..b8996bc14 100644
--- a/database/rrdcalctemplate.h
+++ b/database/rrdcalctemplate.h
@@ -58,7 +58,7 @@ struct rrdcalctemplate {
     struct rrdcalctemplate *next;
 };
 
-#define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after)
+#define RRDCALCTEMPLATE_HAS_DB_LOOKUP(rt) ((rt)->after)
 
 extern void rrdcalctemplate_link_matching(RRDSET *st);
 
diff --git a/database/rrddim.c b/database/rrddim.c
index 95e485106..e98f702fe 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -239,6 +239,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
     rd->last_calculated_value = 0;
     rd->collected_value = 0;
     rd->last_collected_value = 0;
+    rd->collected_value_max = 0;
     rd->collected_volume = 0;
     rd->stored_volume = 0;
     rd->last_stored_value = 0;
@@ -380,6 +381,9 @@ inline collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_
 
     rd->collections_counter++;
 
+    collected_number v = (value >= 0) ? value : -value;
+    if(unlikely(v > rd->collected_value_max)) rd->collected_value_max = v;
+
     // fprintf(stderr, "%s.%s %llu " COLLECTED_NUMBER_FORMAT " dt %0.6f" " rate " CALCULATED_NUMBER_FORMAT "\n", st->name, rd->name, st->usec_since_last_update, value, (float)((double)st->usec_since_last_update / (double)1000000), (calculated_number)((value - rd->last_collected_value) * (calculated_number)rd->multiplier / (calculated_number)rd->divisor * 1000000.0 / (calculated_number)st->usec_since_last_update));
 
     return rd->last_collected_value;
diff --git a/database/rrdset.c b/database/rrdset.c
index 3f5ba73b6..d74ac91ab 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -1080,7 +1080,7 @@ static inline size_t rrdset_done_interpolate(
                       , get_storage_number_flags(rd->values[current_entry])
                       , t1
                       , accuracy
-                      , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
+                      , (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : ""
                 );
 
                 rd->collected_volume += t1;
@@ -1093,7 +1093,7 @@ static inline size_t rrdset_done_interpolate(
                       , rd->stored_volume
                       , rd->collected_volume
                       , accuracy
-                      , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
+                      , (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : ""
                 );
             }
             #endif
@@ -1381,13 +1381,29 @@ void rrdset_done(RRDSET *st) {
                     if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)))
                         storage_flags = SN_EXISTS_RESET;
 
-                    rd->last_collected_value = rd->collected_value;
-                }
+                    uint64_t last = (uint64_t)rd->last_collected_value;
+                    uint64_t new = (uint64_t)rd->collected_value;
+                    uint64_t max = (uint64_t)rd->collected_value_max;
+                    uint64_t cap = 0;
+
+                         if(max > 0x00000000FFFFFFFFULL) cap = 0xFFFFFFFFFFFFFFFFULL;
+                    else if(max > 0x000000000000FFFFULL) cap = 0x00000000FFFFFFFFULL;
+                    else if(max > 0x00000000000000FFULL) cap = 0x000000000000FFFFULL;
+                    else                                 cap = 0x00000000000000FFULL;
+
+                    uint64_t delta = cap - last + new;
 
-                rd->calculated_value +=
-                        (calculated_number)(rd->collected_value - rd->last_collected_value)
-                        * (calculated_number)rd->multiplier
-                        / (calculated_number)rd->divisor;
+                    rd->calculated_value +=
+                            (calculated_number) delta
+                            * (calculated_number) rd->multiplier
+                            / (calculated_number) rd->divisor;
+                }
+                else {
+                    rd->calculated_value +=
+                            (calculated_number) (rd->collected_value - rd->last_collected_value)
+                            * (calculated_number) rd->multiplier
+                            / (calculated_number) rd->divisor;
+                }
 
                 #ifdef NETDATA_INTERNAL_CHECKS
                 rrdset_debug(st, "%s: CALC INC PRE "