Merging upstream version 1.39.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 149 insertions, 187 deletions

diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c
index 666344a9..a8f26c33 100644
--- a/libnetdata/libnetdata.c
+++ b/libnetdata/libnetdata.c
@@ -225,10 +225,6 @@ void posix_memfree(void *ptr) {
     libc_free(ptr);
 }
 
-#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2)
-#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED)
-#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED)
-
 struct malloc_header_signature {
     uint32_t magic;
     uint32_t size;
@@ -1046,171 +1042,6 @@ void netdata_fix_chart_id(char *s) {
     while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++;
 }
 
-/*
-// http://stackoverflow.com/questions/7666509/hash-function-for-string
-uint32_t simple_hash(const char *name)
-{
-    const char *s = name;
-    uint32_t hash = 5381;
-    int i;
-
-    while((i = *s++)) hash = ((hash << 5) + hash) + i;
-
-    // fprintf(stderr, "HASH: %lu %s\n", hash, name);
-
-    return hash;
-}
-*/
-
-/*
-// http://isthe.com/chongo/tech/comp/fnv/#FNV-1a
-uint32_t simple_hash(const char *name) {
-    unsigned char *s = (unsigned char *) name;
-    uint32_t hval = 0x811c9dc5;
-
-    // FNV-1a algorithm
-    while (*s) {
-        // multiply by the 32 bit FNV magic prime mod 2^32
-        // NOTE: No need to optimize with left shifts.
-        //       GCC will use imul instruction anyway.
-        //       Tested with 'gcc -O3 -S'
-        //hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24);
-        hval *= 16777619;
-
-        // xor the bottom with the current octet
-        hval ^= (uint32_t) *s++;
-    }
-
-    // fprintf(stderr, "HASH: %u = %s\n", hval, name);
-    return hval;
-}
-
-uint32_t simple_uhash(const char *name) {
-    unsigned char *s = (unsigned char *) name;
-    uint32_t hval = 0x811c9dc5, c;
-
-    // FNV-1a algorithm
-    while ((c = *s++)) {
-        if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A';
-        hval *= 16777619;
-        hval ^= c;
-    }
-    return hval;
-}
-*/
-
-/*
-// http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx
-// one at a time hash
-uint32_t simple_hash(const char *name) {
-    unsigned char *s = (unsigned char *)name;
-    uint32_t h = 0;
-
-    while(*s) {
-        h += *s++;
-        h += (h << 10);
-        h ^= (h >> 6);
-    }
-
-    h += (h << 3);
-    h ^= (h >> 11);
-    h += (h << 15);
-
-    // fprintf(stderr, "HASH: %u = %s\n", h, name);
-
-    return h;
-}
-*/
-
-void strreverse(char *begin, char *end) {
-    while (end > begin) {
-        // clearer code.
-        char aux = *end;
-        *end-- = *begin;
-        *begin++ = aux;
-    }
-}
-
-char *strsep_on_1char(char **ptr, char c) {
-    if(unlikely(!ptr || !*ptr))
-        return NULL;
-
-    // remember the position we started
-    char *s = *ptr;
-
-    // skip separators in front
-    while(*s == c) s++;
-    char *ret = s;
-
-    // find the next separator
-    while(*s++) {
-        if(unlikely(*s == c)) {
-            *s++ = '\0';
-            *ptr = s;
-            return ret;
-        }
-    }
-
-    *ptr = NULL;
-    return ret;
-}
-
-char *mystrsep(char **ptr, char *s) {
-    char *p = "";
-    while (p && !p[0] && *ptr) p = strsep(ptr, s);
-    return (p);
-}
-
-char *trim(char *s) {
-    // skip leading spaces
-    while (*s && isspace(*s)) s++;
-    if (!*s) return NULL;
-
-    // skip tailing spaces
-    // this way is way faster. Writes only one NUL char.
-    ssize_t l = strlen(s);
-    if (--l >= 0) {
-        char *p = s + l;
-        while (p > s && isspace(*p)) p--;
-        *++p = '\0';
-    }
-
-    if (!*s) return NULL;
-
-    return s;
-}
-
-inline char *trim_all(char *buffer) {
-    char *d = buffer, *s = buffer;
-
-    // skip spaces
-    while(isspace(*s)) s++;
-
-    while(*s) {
-        // copy the non-space part
-        while(*s && !isspace(*s)) *d++ = *s++;
-
-        // add a space if we have to
-        if(*s && isspace(*s)) {
-            *d++ = ' ';
-            s++;
-        }
-
-        // skip spaces
-        while(isspace(*s)) s++;
-    }
-
-    *d = '\0';
-
-    if(d > buffer) {
-        d--;
-        if(isspace(*d)) *d = '\0';
-    }
-
-    if(!buffer[0]) return NULL;
-    return buffer;
-}
-
 static int memory_file_open(const char *filename, size_t size) {
     // info("memory_file_open('%s', %zu", filename, size);
 
@@ -1883,17 +1714,20 @@ inline int config_isspace(char c)
 }
 
 // split a text into words, respecting quotes
-inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char), char *recover_input, char **recover_location, int max_recover)
+inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char))
 {
     char *s = str, quote = 0;
     size_t i = 0;
-    int rec = 0;
-    char *recover = recover_input;
 
     // skip all white space
     while (unlikely(custom_isspace(*s)))
         s++;
 
+    if(unlikely(!*s)) {
+        words[i] = NULL;
+        return 0;
+    }
+
     // check for quote
     if (unlikely(*s == '\'' || *s == '"')) {
         quote = *s; // remember the quote
@@ -1905,19 +1739,15 @@ inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words,
 
     // while we have something
     while (likely(*s)) {
-        // if it is escape
+        // if it is an escape
         if (unlikely(*s == '\\' && s[1])) {
             s += 2;
             continue;
         }
 
-        // if it is quote
+        // if it is a quote
         else if (unlikely(*s == quote)) {
             quote = 0;
-            if (recover && rec < max_recover) {
-                recover_location[rec++] = s;
-                *recover++ = *s;
-            }
             *s = ' ';
             continue;
         }
@@ -1925,19 +1755,13 @@ inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words,
         // if it is a space
         else if (unlikely(quote == 0 && custom_isspace(*s))) {
             // terminate the word
-            if (recover && rec < max_recover) {
-                if (!rec || recover_location[rec-1] != s) {
-                    recover_location[rec++] = s;
-                    *recover++ = *s;
-                }
-            }
             *s++ = '\0';
 
             // skip all white space
             while (likely(custom_isspace(*s)))
                 s++;
 
-            // check for quote
+            // check for a quote
             if (unlikely(*s == '\'' || *s == '"')) {
                 quote = *s; // remember the quote
                 s++;        // skip the quote
@@ -1965,9 +1789,9 @@ inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words,
     return i;
 }
 
-inline size_t pluginsd_split_words(char *str, char **words, size_t max_words, char *recover_input, char **recover_location, int max_recover)
+inline size_t pluginsd_split_words(char *str, char **words, size_t max_words)
 {
-    return quoted_strings_splitter(str, words, max_words, pluginsd_space, recover_input, recover_location, max_recover);
+    return quoted_strings_splitter(str, words, max_words, pluginsd_space);
 }
 
 bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx) {
@@ -2072,3 +1896,141 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){
         closedir(dir);
     }
 }
+
+struct timing_steps {
+    const char *name;
+    usec_t time;
+    size_t count;
+} timing_steps[TIMING_STEP_MAX + 1] = {
+        [TIMING_STEP_INTERNAL] = { .name = "internal", .time = 0, },
+
+        [TIMING_STEP_BEGIN2_PREPARE] = { .name = "BEGIN2 prepare", .time = 0, },
+        [TIMING_STEP_BEGIN2_FIND_CHART] = { .name = "BEGIN2 find chart", .time = 0, },
+        [TIMING_STEP_BEGIN2_PARSE] = { .name = "BEGIN2 parse", .time = 0, },
+        [TIMING_STEP_BEGIN2_ML] = { .name = "BEGIN2 ml", .time = 0, },
+        [TIMING_STEP_BEGIN2_PROPAGATE] = { .name = "BEGIN2 propagate", .time = 0, },
+        [TIMING_STEP_BEGIN2_STORE] = { .name = "BEGIN2 store", .time = 0, },
+
+        [TIMING_STEP_SET2_PREPARE] = { .name = "SET2 prepare", .time = 0, },
+        [TIMING_STEP_SET2_LOOKUP_DIMENSION] = { .name = "SET2 find dimension", .time = 0, },
+        [TIMING_STEP_SET2_PARSE] = { .name = "SET2 parse", .time = 0, },
+        [TIMING_STEP_SET2_ML] = { .name = "SET2 ml", .time = 0, },
+        [TIMING_STEP_SET2_PROPAGATE] = { .name = "SET2 propagate", .time = 0, },
+        [TIMING_STEP_RRDSET_STORE_METRIC] = { .name = "SET2 rrdset store", .time = 0, },
+        [TIMING_STEP_DBENGINE_FIRST_CHECK] = { .name = "db 1st check", .time = 0, },
+        [TIMING_STEP_DBENGINE_CHECK_DATA] = { .name = "db check data", .time = 0, },
+        [TIMING_STEP_DBENGINE_PACK] = { .name = "db pack", .time = 0, },
+        [TIMING_STEP_DBENGINE_PAGE_FIN] = { .name = "db page fin", .time = 0, },
+        [TIMING_STEP_DBENGINE_MRG_UPDATE] = { .name = "db mrg update", .time = 0, },
+        [TIMING_STEP_DBENGINE_PAGE_ALLOC] = { .name = "db page alloc", .time = 0, },
+        [TIMING_STEP_DBENGINE_CREATE_NEW_PAGE] = { .name = "db new page", .time = 0, },
+        [TIMING_STEP_DBENGINE_FLUSH_PAGE] = { .name = "db page flush", .time = 0, },
+        [TIMING_STEP_SET2_STORE] = { .name = "SET2 store", .time = 0, },
+
+        [TIMING_STEP_END2_PREPARE] = { .name = "END2 prepare", .time = 0, },
+        [TIMING_STEP_END2_PUSH_V1] = { .name = "END2 push v1", .time = 0, },
+        [TIMING_STEP_END2_ML] = { .name = "END2 ml", .time = 0, },
+        [TIMING_STEP_END2_RRDSET] = { .name = "END2 rrdset", .time = 0, },
+        [TIMING_STEP_END2_PROPAGATE] = { .name = "END2 propagate", .time = 0, },
+        [TIMING_STEP_END2_STORE] = { .name = "END2 store", .time = 0, },
+
+        // terminator
+        [TIMING_STEP_MAX] = { .name = NULL, .time = 0, },
+};
+
+void timing_action(TIMING_ACTION action, TIMING_STEP step) {
+    static __thread usec_t last_action_time = 0;
+    static struct timing_steps timings2[TIMING_STEP_MAX + 1] = {};
+
+    switch(action) {
+        case TIMING_ACTION_INIT:
+            last_action_time = now_monotonic_usec();
+            break;
+
+        case TIMING_ACTION_STEP: {
+            if(!last_action_time)
+                return;
+
+            usec_t now = now_monotonic_usec();
+            __atomic_add_fetch(&timing_steps[step].time, now - last_action_time, __ATOMIC_RELAXED);
+            __atomic_add_fetch(&timing_steps[step].count, 1, __ATOMIC_RELAXED);
+            last_action_time = now;
+            break;
+        }
+
+        case TIMING_ACTION_FINISH: {
+            if(!last_action_time)
+                return;
+
+            usec_t expected = __atomic_load_n(&timing_steps[TIMING_STEP_INTERNAL].time, __ATOMIC_RELAXED);
+            if(last_action_time - expected < 10 * USEC_PER_SEC) {
+                last_action_time = 0;
+                return;
+            }
+
+            if(!__atomic_compare_exchange_n(&timing_steps[TIMING_STEP_INTERNAL].time, &expected, last_action_time, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                last_action_time = 0;
+                return;
+            }
+
+            struct timing_steps timings3[TIMING_STEP_MAX + 1];
+            memcpy(timings3, timing_steps, sizeof(timings3));
+
+            size_t total_reqs = 0;
+            usec_t total_usec = 0;
+            for(size_t t = 1; t < TIMING_STEP_MAX ; t++) {
+                total_usec += timings3[t].time - timings2[t].time;
+                total_reqs += timings3[t].count - timings2[t].count;
+            }
+
+            BUFFER *wb = buffer_create(1024, NULL);
+
+            for(size_t t = 1; t < TIMING_STEP_MAX ; t++) {
+                size_t requests = timings3[t].count - timings2[t].count;
+                if(!requests) continue;
+
+                buffer_sprintf(wb, "TIMINGS REPORT: [%3zu. %-20s]: # %10zu, t %11.2f ms (%6.2f %%), avg %6.2f usec/run\n",
+                               t,
+                               timing_steps[t].name ? timing_steps[t].name : "x",
+                               requests,
+                               (double) (timings3[t].time - timings2[t].time) / (double)USEC_PER_MS,
+                               (double) (timings3[t].time - timings2[t].time) * 100.0 / (double) total_usec,
+                               (double) (timings3[t].time - timings2[t].time) / (double)requests
+                );
+            }
+
+            info("TIMINGS REPORT:\n%sTIMINGS REPORT:                        total # %10zu, t %11.2f ms",
+                 buffer_tostring(wb), total_reqs, (double)total_usec / USEC_PER_MS);
+
+            memcpy(timings2, timings3, sizeof(timings2));
+
+            last_action_time = 0;
+            buffer_free(wb);
+        }
+    }
+}
+
+int hash256_string(const unsigned char *string, size_t size, char *hash) {
+    EVP_MD_CTX *ctx;
+    ctx = EVP_MD_CTX_create();
+
+    if (!ctx)
+        return 0;
+
+    if (!EVP_DigestInit(ctx, EVP_sha256())) {
+        EVP_MD_CTX_destroy(ctx);
+        return 0;
+    }
+
+    if (!EVP_DigestUpdate(ctx, string, size)) {
+        EVP_MD_CTX_destroy(ctx);
+        return 0;
+    }
+
+    if (!EVP_DigestFinal(ctx, (unsigned char *)hash, NULL)) {
+        EVP_MD_CTX_destroy(ctx);
+        return 0;
+    }
+
+    return 1;
+}