Adding upstream version 1.39.0.upstream/1.39.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 183 insertions, 61 deletions

diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
index c504bd4b..c2449493 100644
--- a/libnetdata/libnetdata.h
+++ b/libnetdata/libnetdata.h
@@ -32,6 +32,9 @@ extern "C" {
 #define OS_FREEBSD 2
 #define OS_MACOS   3
 
+#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2)
+#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED)
+#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED)
 
 // ----------------------------------------------------------------------------
 // system include files for all netdata C programs
@@ -363,15 +366,131 @@ size_t judy_aral_structures(void);
 
 // ---------------------------------------------------------------------------------------------
 
+#include "storage_number/storage_number.h"
+
+typedef struct storage_point {
+    NETDATA_DOUBLE min;     // when count > 1, this is the minimum among them
+    NETDATA_DOUBLE max;     // when count > 1, this is the maximum among them
+    NETDATA_DOUBLE sum;     // the point sum - divided by count gives the average
+
+    // end_time - start_time = point duration
+    time_t start_time_s;    // the time the point starts
+    time_t end_time_s;      // the time the point ends
+
+    uint32_t count;         // the number of original points aggregated
+    uint32_t anomaly_count; // the number of original points found anomalous
+
+    SN_FLAGS flags;         // flags stored with the point
+} STORAGE_POINT;
+
+#define storage_point_unset(x)                     do { \
+    (x).min = (x).max = (x).sum = NAN;                  \
+    (x).count = 0;                                      \
+    (x).anomaly_count = 0;                              \
+    (x).flags = SN_FLAG_NONE;                           \
+    (x).start_time_s = 0;                               \
+    (x).end_time_s = 0;                                 \
+    } while(0)
+
+#define storage_point_empty(x, start_s, end_s)     do { \
+    (x).min = (x).max = (x).sum = NAN;                  \
+    (x).count = 1;                                      \
+    (x).anomaly_count = 0;                              \
+    (x).flags = SN_FLAG_NONE;                           \
+    (x).start_time_s = start_s;                         \
+    (x).end_time_s = end_s;                             \
+    } while(0)
+
+#define STORAGE_POINT_UNSET (STORAGE_POINT){ .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }
+
+#define storage_point_is_unset(x) (!(x).count)
+#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))
+#define storage_point_is_zero(x) (!(x).count || (netdata_double_is_zero((x).min) && netdata_double_is_zero((x).max) && netdata_double_is_zero((x).sum) && (x).anomaly_count == 0))
+
+#define storage_point_merge_to(dst, src) do {           \
+        if(storage_point_is_unset(dst))                 \
+            (dst) = (src);                              \
+                                                        \
+        else if(!storage_point_is_unset(src) &&         \
+                !storage_point_is_gap(src)) {           \
+                                                        \
+            if((src).start_time_s < (dst).start_time_s) \
+                (dst).start_time_s = (src).start_time_s;\
+                                                        \
+            if((src).end_time_s > (dst).end_time_s)     \
+                (dst).end_time_s = (src).end_time_s;    \
+                                                        \
+            if((src).min < (dst).min)                   \
+                (dst).min = (src).min;                  \
+                                                        \
+            if((src).max > (dst).max)                   \
+                (dst).max = (src).max;                  \
+                                                        \
+            (dst).sum += (src).sum;                     \
+                                                        \
+            (dst).count += (src).count;                 \
+            (dst).anomaly_count += (src).anomaly_count; \
+                                                        \
+            (dst).flags |= (src).flags & SN_FLAG_RESET; \
+        }                                               \
+} while(0)
+
+#define storage_point_add_to(dst, src) do {             \
+        if(storage_point_is_unset(dst))                 \
+            (dst) = (src);                              \
+                                                        \
+        else if(!storage_point_is_unset(src) &&         \
+                !storage_point_is_gap(src)) {           \
+                                                        \
+            if((src).start_time_s < (dst).start_time_s) \
+                (dst).start_time_s = (src).start_time_s;\
+                                                        \
+            if((src).end_time_s > (dst).end_time_s)     \
+                (dst).end_time_s = (src).end_time_s;    \
+                                                        \
+            (dst).min += (src).min;                     \
+            (dst).max += (src).max;                     \
+            (dst).sum += (src).sum;                     \
+                                                        \
+            (dst).count += (src).count;                 \
+            (dst).anomaly_count += (src).anomaly_count; \
+                                                        \
+            (dst).flags |= (src).flags & SN_FLAG_RESET; \
+        }                                               \
+} while(0)
+
+#define storage_point_make_positive(sp) do {            \
+        if(!storage_point_is_unset(sp) &&               \
+           !storage_point_is_gap(sp)) {                 \
+                                                        \
+            if(unlikely(signbit((sp).sum)))             \
+                (sp).sum = -(sp).sum;                   \
+                                                        \
+            if(unlikely(signbit((sp).min)))             \
+                (sp).min = -(sp).min;                   \
+                                                        \
+            if(unlikely(signbit((sp).max)))             \
+                (sp).max = -(sp).max;                   \
+                                                        \
+            if(unlikely((sp).min > (sp).max)) {         \
+                NETDATA_DOUBLE t = (sp).min;            \
+                (sp).min = (sp).max;                    \
+                (sp).max = t;                           \
+            }                                           \
+        }                                               \
+} while(0)
+
+#define storage_point_anomaly_rate(sp) \
+    (NETDATA_DOUBLE)(storage_point_is_unset(sp) ? 0.0 : (NETDATA_DOUBLE)((sp).anomaly_count) * 100.0 / (NETDATA_DOUBLE)((sp).count))
+
+#define storage_point_average_value(sp) \
+    ((sp).count ? (sp).sum / (NETDATA_DOUBLE)((sp).count) : 0.0)
+
+// ---------------------------------------------------------------------------------------------
 
 void netdata_fix_chart_id(char *s);
 void netdata_fix_chart_name(char *s);
 
-void strreverse(char* begin, char* end);
-char *mystrsep(char **ptr, char *s);
-char *trim(char *s); // remove leading and trailing spaces; may return NULL
-char *trim_all(char *buffer); // like trim(), but also remove duplicate spaces inside the string; may return NULL
-
 int madvise_sequential(void *mem, size_t len);
 int madvise_random(void *mem, size_t len);
 int madvise_dontfork(void *mem, size_t len);
@@ -485,8 +604,8 @@ void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value);
 int config_isspace(char c);
 int pluginsd_space(char c);
 
-size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char), char *recover_input, char **recover_location, int max_recover);
-size_t pluginsd_split_words(char *str, char **words, size_t max_words, char *recover_string, char **recover_location, int max_recover);
+size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char));
+size_t pluginsd_split_words(char *str, char **words, size_t max_words);
 
 static inline char *get_word(char **words, size_t num_words, size_t index) {
     if (index >= num_words)
@@ -514,7 +633,6 @@ extern char *netdata_configured_host_prefix;
 #include "libjudy/src/Judy.h"
 #include "july/july.h"
 #include "os.h"
-#include "storage_number/storage_number.h"
 #include "threads/threads.h"
 #include "buffer/buffer.h"
 #include "locks/locks.h"
@@ -547,10 +665,11 @@ extern char *netdata_configured_host_prefix;
 #include "libnetdata/aral/aral.h"
 #include "onewayalloc/onewayalloc.h"
 #include "worker_utilization/worker_utilization.h"
+#include "parser/parser.h"
+#include "yaml.h"
 
-// BEWARE: Outside of the C code this also exists in alarm-notify.sh
-#define DEFAULT_CLOUD_BASE_URL "https://api.netdata.cloud"
-#define DEFAULT_CLOUD_UI_URL "https://app.netdata.cloud"
+// BEWARE: this exists in alarm-notify.sh
+#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
 
 #define RRD_STORAGE_TIERS 5
 
@@ -609,58 +728,61 @@ static inline PPvoid_t JudyLLastThenPrev(Pcvoid_t PArray, Word_t * PIndex, bool
     return JudyLPrev(PArray, PIndex, PJE0);
 }
 
-static inline size_t indexing_partition_old(Word_t ptr, Word_t modulo) {
-    size_t total = 0;
-
-    total += (ptr & 0xff) >> 0;
-    total += (ptr & 0xff00) >> 8;
-    total += (ptr & 0xff0000) >> 16;
-    total += (ptr & 0xff000000) >> 24;
-
-    if(sizeof(Word_t) > 4) {
-        total += (ptr & 0xff00000000) >> 32;
-        total += (ptr & 0xff0000000000) >> 40;
-        total += (ptr & 0xff000000000000) >> 48;
-        total += (ptr & 0xff00000000000000) >> 56;
-    }
-
-    return (total % modulo);
-}
-
-static uint32_t murmur32(uint32_t h) __attribute__((const));
-static inline uint32_t murmur32(uint32_t h) {
-    h ^= h >> 16;
-    h *= 0x85ebca6b;
-    h ^= h >> 13;
-    h *= 0xc2b2ae35;
-    h ^= h >> 16;
-
-    return h;
-}
-
-static uint64_t murmur64(uint64_t h) __attribute__((const));
-static inline uint64_t murmur64(uint64_t k) {
-    k ^= k >> 33;
-    k *= 0xff51afd7ed558ccdUL;
-    k ^= k >> 33;
-    k *= 0xc4ceb9fe1a85ec53UL;
-    k ^= k >> 33;
-
-    return k;
-}
+typedef enum {
+    TIMING_STEP_INTERNAL = 0,
+
+    TIMING_STEP_BEGIN2_PREPARE,
+    TIMING_STEP_BEGIN2_FIND_CHART,
+    TIMING_STEP_BEGIN2_PARSE,
+    TIMING_STEP_BEGIN2_ML,
+    TIMING_STEP_BEGIN2_PROPAGATE,
+    TIMING_STEP_BEGIN2_STORE,
+
+    TIMING_STEP_SET2_PREPARE,
+    TIMING_STEP_SET2_LOOKUP_DIMENSION,
+    TIMING_STEP_SET2_PARSE,
+    TIMING_STEP_SET2_ML,
+    TIMING_STEP_SET2_PROPAGATE,
+    TIMING_STEP_RRDSET_STORE_METRIC,
+    TIMING_STEP_DBENGINE_FIRST_CHECK,
+    TIMING_STEP_DBENGINE_CHECK_DATA,
+    TIMING_STEP_DBENGINE_PACK,
+    TIMING_STEP_DBENGINE_PAGE_FIN,
+    TIMING_STEP_DBENGINE_MRG_UPDATE,
+    TIMING_STEP_DBENGINE_PAGE_ALLOC,
+    TIMING_STEP_DBENGINE_CREATE_NEW_PAGE,
+    TIMING_STEP_DBENGINE_FLUSH_PAGE,
+    TIMING_STEP_SET2_STORE,
+
+    TIMING_STEP_END2_PREPARE,
+    TIMING_STEP_END2_PUSH_V1,
+    TIMING_STEP_END2_ML,
+    TIMING_STEP_END2_RRDSET,
+    TIMING_STEP_END2_PROPAGATE,
+    TIMING_STEP_END2_STORE,
+
+    // terminator
+    TIMING_STEP_MAX,
+} TIMING_STEP;
 
-static inline size_t indexing_partition(Word_t ptr, Word_t modulo) __attribute__((const));
-static inline size_t indexing_partition(Word_t ptr, Word_t modulo) {
-    if(sizeof(Word_t) == 8) {
-        uint64_t hash = murmur64(ptr);
-        return hash % modulo;
-    }
-    else {
-        uint32_t hash = murmur32(ptr);
-        return hash % modulo;
-    }
-}
+typedef enum {
+    TIMING_ACTION_INIT,
+    TIMING_ACTION_STEP,
+    TIMING_ACTION_FINISH,
+} TIMING_ACTION;
+
+#ifdef NETDATA_TIMING_REPORT
+#define timing_init() timing_action(TIMING_ACTION_INIT, TIMING_STEP_INTERNAL)
+#define timing_step(step) timing_action(TIMING_ACTION_STEP, step)
+#define timing_report() timing_action(TIMING_ACTION_FINISH, TIMING_STEP_INTERNAL)
+#else
+#define timing_init() debug_dummy()
+#define timing_step(step) debug_dummy()
+#define timing_report() debug_dummy()
+#endif
+void timing_action(TIMING_ACTION action, TIMING_STEP step);
 
+int hash256_string(const unsigned char *string, size_t size, char *hash);
 # ifdef __cplusplus
 }
 # endif