New upstream version 1.11.0+dfsgupstream/1.11.0+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 391 insertions, 305 deletions

diff --git a/src/common.c b/libnetdata/libnetdata.c
index 94fd5e42..de330cae 100644
--- a/src/common.c
+++ b/libnetdata/libnetdata.c
@@ -1,4 +1,6 @@
-#include "common.h"
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata.h"
 
 #ifdef __APPLE__
 #define INHERIT_NONE 0
@@ -8,17 +10,6 @@
 #    define MADV_DONTFORK INHERIT_NONE
 #endif /* __FreeBSD__ || __APPLE__*/
 
-char *netdata_configured_hostname    = NULL;
-char *netdata_configured_config_dir  = NULL;
-char *netdata_configured_log_dir     = NULL;
-char *netdata_configured_plugins_dir = NULL;
-char *netdata_configured_web_dir     = NULL;
-char *netdata_configured_cache_dir   = NULL;
-char *netdata_configured_varlib_dir  = NULL;
-char *netdata_configured_home_dir    = NULL;
-char *netdata_configured_host_prefix = NULL;
-char *netdata_configured_timezone    = NULL;
-
 struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };
 int enable_ksm = 1;
 
@@ -36,182 +27,172 @@ const char *program_version = VERSION;
 // routines.
 
 #ifdef NETDATA_LOG_ALLOCATIONS
-static struct memory_statistics {
-    volatile size_t malloc_calls_made;
-    volatile size_t calloc_calls_made;
-    volatile size_t realloc_calls_made;
-    volatile size_t strdup_calls_made;
-    volatile size_t free_calls_made;
-    volatile size_t memory_calls_made;
-    volatile size_t allocated_memory;
-    volatile size_t mmapped_memory;
-} memory_statistics;
-
-static inline void print_allocations(const char *file, const char *function, const unsigned long line) {
-    static struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
-
-    //if(unlikely(!(memory_statistics.memory_calls_made % 5))) {
-        fprintf(stderr, "(%04lu@%-10.10s:%-15.15s): Allocated %zu KB (+%zu B), mmapped %zu KB (+%zu B): malloc %zu (+%zu), calloc %zu (+%zu), realloc %zu (+%zu), strdup %zu (+%zu), free %zu (+%zu)\n",
-                line, file, function,
-                (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
-                (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
-                memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
-                memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
-                memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
-                memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made,
-                memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made
-        );
-
-        memcpy(&old, &memory_statistics, sizeof(struct memory_statistics));
-    //}
+static __thread struct memory_statistics {
+    volatile ssize_t malloc_calls_made;
+    volatile ssize_t calloc_calls_made;
+    volatile ssize_t realloc_calls_made;
+    volatile ssize_t strdup_calls_made;
+    volatile ssize_t free_calls_made;
+    volatile ssize_t memory_calls_made;
+    volatile ssize_t allocated_memory;
+    volatile ssize_t mmapped_memory;
+} memory_statistics = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+__thread size_t log_thread_memory_allocations = 0;
+
+static inline void print_allocations(const char *file, const char *function, const unsigned long line, const char *type, size_t size) {
+    static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+    fprintf(stderr, "%s iteration %zu MEMORY TRACE: %lu@%s : %s : %s : %zu\n",
+            netdata_thread_tag(),
+            log_thread_memory_allocations,
+            line, file, function,
+            type, size
+    );
+
+    fprintf(stderr, "%s iteration %zu MEMORY ALLOCATIONS: (%04lu@%-40.40s:%-40.40s): Allocated %zd KB (%+zd B), mmapped %zd KB (%+zd B): %s : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
+            netdata_thread_tag(),
+            log_thread_memory_allocations,
+            line, file, function,
+            (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
+            (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
+            type,
+            memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
+            memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
+            memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
+            memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made,
+            memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made
+    );
+
+    memcpy(&old, &memory_statistics, sizeof(struct memory_statistics));
 }
 
-static inline void malloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) {
-#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
-    __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.malloc_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
-#else
-    // this is for debugging - we don't care locking it
-    memory_statistics.memory_calls_made++;
-    memory_statistics.malloc_calls_made++;
-    memory_statistics.allocated_memory += size;
-#endif
-    print_allocations(file, function, line);
+static inline void mmap_accounting(size_t size) {
+    if(log_thread_memory_allocations) {
+        memory_statistics.memory_calls_made++;
+        memory_statistics.mmapped_memory += size;
+    }
 }
 
-static inline void mmap_accounting(size_t size) {
-#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
-    __atomic_fetch_add(&memory_statistics.malloc_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.mmapped_memory, size, __ATOMIC_SEQ_CST);
-#else
-    // this is for debugging - we don't care locking it
-    memory_statistics.memory_calls_made++;
-    memory_statistics.mmapped_memory += size;
-#endif
+void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
+    if(log_thread_memory_allocations) {
+        memory_statistics.memory_calls_made++;
+        memory_statistics.malloc_calls_made++;
+        memory_statistics.allocated_memory += size;
+        print_allocations(file, function, line, "malloc()", size);
+    }
+
+    size_t *n = (size_t *)malloc(sizeof(size_t) + size);
+    if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size);
+    *n = size;
+    return (void *)&n[1];
 }
 
-static inline void calloc_accounting(const char *file, const char *function, const unsigned long line, size_t size) {
-#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
-    __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.calloc_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
-#else
-    // this is for debugging - we don't care locking it
-    memory_statistics.memory_calls_made++;
-    memory_statistics.calloc_calls_made++;
-    memory_statistics.allocated_memory += size;
-#endif
-    print_allocations(file, function, line);
+void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
+    size = nmemb * size;
+
+    if(log_thread_memory_allocations) {
+        memory_statistics.memory_calls_made++;
+        memory_statistics.calloc_calls_made++;
+        memory_statistics.allocated_memory += size;
+        print_allocations(file, function, line, "calloc()", size);
+    }
+
+    size_t *n = (size_t *)calloc(1, sizeof(size_t) + size);
+    if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size);
+    *n = size;
+    return (void *)&n[1];
 }
 
-static inline void realloc_accounting(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
-    (void)ptr;
+void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
+    if(!ptr) return mallocz_int(file, function, line, size);
 
-#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
-    __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.realloc_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
-#else
-    // this is for debugging - we don't care locking it
-    memory_statistics.memory_calls_made++;
-    memory_statistics.realloc_calls_made++;
-    memory_statistics.allocated_memory += size;
-#endif
-    print_allocations(file, function, line);
+    size_t *n = (size_t *)ptr;
+    n--;
+    size_t old_size = *n;
+
+    n = realloc(n, sizeof(size_t) + size);
+    if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size);
+
+    if(log_thread_memory_allocations) {
+        memory_statistics.memory_calls_made++;
+        memory_statistics.realloc_calls_made++;
+        memory_statistics.allocated_memory += (size - old_size);
+        print_allocations(file, function, line, "realloc()", size - old_size);
+    }
+
+    *n = size;
+    return (void *)&n[1];
 }
 
-static inline void strdup_accounting(const char *file, const char *function, const unsigned long line, const char *s) {
+char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
     size_t size = strlen(s) + 1;
 
-#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
-    __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.strdup_calls_made, 1, __ATOMIC_SEQ_CST);
-    __atomic_fetch_add(&memory_statistics.allocated_memory, size, __ATOMIC_SEQ_CST);
-#else
-    // this is for debugging - we don't care locking it
-    memory_statistics.memory_calls_made++;
-    memory_statistics.strdup_calls_made++;
-    memory_statistics.allocated_memory += size;
-#endif
-    print_allocations(file, function, line);
+    if(log_thread_memory_allocations) {
+        memory_statistics.memory_calls_made++;
+        memory_statistics.strdup_calls_made++;
+        memory_statistics.allocated_memory += size;
+        print_allocations(file, function, line, "strdup()", size);
+    }
+
+    size_t *n = (size_t *)malloc(sizeof(size_t) + size);
+    if (unlikely(!n)) fatal("strdupz() cannot allocate %zu bytes of memory.", size);
+
+    *n = size;
+    char *t = (char *)&n[1];
+    strcpy(t, s);
+    return t;
 }
 
-static inline void free_accounting(const char *file, const char *function, const unsigned long line, void *ptr) {
-    (void)file;
-    (void)function;
-    (void)line;
+void freez_int(const char *file, const char *function, const unsigned long line, void *ptr) {
+    if(unlikely(!ptr)) return;
 
-    if(likely(ptr)) {
-#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
-        __atomic_fetch_add(&memory_statistics.memory_calls_made, 1, __ATOMIC_SEQ_CST);
-        __atomic_fetch_add(&memory_statistics.free_calls_made, 1, __ATOMIC_SEQ_CST);
-#else
-        // this is for debugging - we don't care locking it
+    size_t *n = (size_t *)ptr;
+    n--;
+    size_t size = *n;
+
+    if(log_thread_memory_allocations) {
         memory_statistics.memory_calls_made++;
         memory_statistics.free_calls_made++;
-#endif
+        memory_statistics.allocated_memory -= size;
+        print_allocations(file, function, line, "free()", size);
     }
-}
-#endif
 
-#ifdef NETDATA_LOG_ALLOCATIONS
-char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
-    strdup_accounting(file, function, line, s);
+    free(n);
+}
 #else
-char *strdupz(const char *s) {
-#endif
 
+char *strdupz(const char *s) {
     char *t = strdup(s);
     if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s);
     return t;
 }
 
-#ifdef NETDATA_LOG_ALLOCATIONS
-void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
-    malloc_accounting(file, function, line, size);
-#else
-void *mallocz(size_t size) {
-#endif
+void freez(void *ptr) {
+    free(ptr);
+}
 
+void *mallocz(size_t size) {
     void *p = malloc(size);
     if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size);
     return p;
 }
 
-#ifdef NETDATA_LOG_ALLOCATIONS
-void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
-    calloc_accounting(file, function, line, nmemb * size);
-#else
 void *callocz(size_t nmemb, size_t size) {
-#endif
-
     void *p = calloc(nmemb, size);
     if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size);
     return p;
 }
 
-#ifdef NETDATA_LOG_ALLOCATIONS
-void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
-    realloc_accounting(file, function, line, ptr, size);
-#else
 void *reallocz(void *ptr, size_t size) {
-#endif
-
     void *p = realloc(ptr, size);
     if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size);
     return p;
 }
 
-#ifdef NETDATA_LOG_ALLOCATIONS
-void freez_int(const char *file, const char *function, const unsigned long line, void *ptr) {
-    free_accounting(file, function, line, ptr);
-#else
-void freez(void *ptr) {
 #endif
 
-    free(ptr);
-}
+// --------------------------------------------------------------------------------------------------------------------
 
 void json_escape_string(char *dst, const char *src, size_t size) {
     const char *t;
@@ -244,52 +225,6 @@ void json_fix_string(char *s) {
     }
 }
 
-int sleep_usec(usec_t usec) {
-
-#ifndef NETDATA_WITH_USLEEP
-    // we expect microseconds (1.000.000 per second)
-    // but timespec is nanoseconds (1.000.000.000 per second)
-    struct timespec rem, req = {
-            .tv_sec = (time_t) (usec / 1000000),
-            .tv_nsec = (suseconds_t) ((usec % 1000000) * 1000)
-    };
-
-    while (nanosleep(&req, &rem) == -1) {
-        if (likely(errno == EINTR)) {
-            debug(D_SYSTEM, "nanosleep() interrupted (while sleeping for %llu microseconds).", usec);
-            req.tv_sec = rem.tv_sec;
-            req.tv_nsec = rem.tv_nsec;
-        } else {
-            error("Cannot nanosleep() for %llu microseconds.", usec);
-            break;
-        }
-    }
-
-    return 0;
-#else
-    int ret = usleep(usec);
-    if(unlikely(ret == -1 && errno == EINVAL)) {
-        // on certain systems, usec has to be up to 999999
-        if(usec > 999999) {
-            int counter = usec / 999999;
-            while(counter--)
-                usleep(999999);
-
-            usleep(usec % 999999);
-        }
-        else {
-            error("Cannot usleep() for %llu microseconds.", usec);
-            return ret;
-        }
-    }
-
-    if(ret != 0)
-        error("usleep() failed for %llu microseconds.", usec);
-
-    return ret;
-#endif
-}
-
 unsigned char netdata_map_chart_names[256] = {
         [0] = '\0', //
         [1] = '_', //
@@ -1183,124 +1118,6 @@ int snprintfz(char *dst, size_t n, const char *fmt, ...) {
     return ret;
 }
 
-// ----------------------------------------------------------------------------
-// system functions
-// to retrieve settings of the system
-
-int processors = 1;
-long get_system_cpus(void) {
-    processors = 1;
-
-    #ifdef __APPLE__
-        int32_t tmp_processors;
-
-        if (unlikely(GETSYSCTL_BY_NAME("hw.logicalcpu", tmp_processors))) {
-            error("Assuming system has %d processors.", processors);
-        } else {
-            processors = tmp_processors;
-        }
-
-        return processors;
-    #elif __FreeBSD__
-        int32_t tmp_processors;
-
-        if (unlikely(GETSYSCTL_BY_NAME("hw.ncpu", tmp_processors))) {
-            error("Assuming system has %d processors.", processors);
-        } else {
-            processors = tmp_processors;
-        }
-
-        return processors;
-    #else
-
-    char filename[FILENAME_MAX + 1];
-    snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix);
-
-    procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
-    if(!ff) {
-        error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
-        return processors;
-    }
-
-    ff = procfile_readall(ff);
-    if(!ff) {
-        error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
-        return processors;
-    }
-
-    processors = 0;
-    unsigned int i;
-    for(i = 0; i < procfile_lines(ff); i++) {
-        if(!procfile_linewords(ff, i)) continue;
-
-        if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) processors++;
-    }
-    processors--;
-    if(processors < 1) processors = 1;
-
-    procfile_close(ff);
-
-    debug(D_SYSTEM, "System has %d processors.", processors);
-    return processors;
-
-    #endif /* __APPLE__, __FreeBSD__ */
-}
-
-pid_t pid_max = 32768;
-pid_t get_system_pid_max(void) {
-    #ifdef __APPLE__
-        // As we currently do not know a solution to query pid_max from the os
-        // we use the number defined in bsd/sys/proc_internal.h in XNU sources
-        pid_max = 99999;
-        return pid_max;
-    #elif __FreeBSD__
-        int32_t tmp_pid_max;
-
-        if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) {
-            pid_max = 99999;
-            error("Assuming system's maximum pid is %d.", pid_max);
-        } else {
-            pid_max = tmp_pid_max;
-        }
-
-        return pid_max;
-    #else
-
-    static char read = 0;
-    if(unlikely(read)) return pid_max;
-    read = 1;
-
-    char filename[FILENAME_MAX + 1];
-    snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix);
-
-    unsigned long long max = 0;
-    if(read_single_number_file(filename, &max) != 0) {
-        error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max);
-        return pid_max;
-    }
-
-    if(!max) {
-        error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max);
-        return pid_max;
-    }
-
-    pid_max = (pid_t) max;
-    return pid_max;
-
-    #endif /* __APPLE__, __FreeBSD__ */
-}
-
-unsigned int hz;
-void get_system_HZ(void) {
-    long ticks;
-
-    if ((ticks = sysconf(_SC_CLK_TCK)) == -1) {
-        error("Cannot get system clock ticks");
-    }
-
-    hz = (unsigned int) ticks;
-}
-
 /*
 // poor man cycle counting
 static unsigned long tsc;
@@ -1359,3 +1176,272 @@ int recursively_delete_dir(const char *path, const char *reason) {
 
     return ret;
 }
+
+static int is_virtual_filesystem(const char *path, char **reason) {
+
+#if defined(__APPLE__) || defined(__FreeBSD__)
+    (void)path;
+    (void)reason;
+#else
+    struct statfs stat;
+    // stat.f_fsid.__val[0] is a file system id
+    // stat.f_fsid.__val[1] is the inode
+    // so their combination uniquely identifies the file/dir
+
+    if (statfs(path, &stat) == -1) {
+        if(reason) *reason = "failed to statfs()";
+        return -1;
+    }
+
+    if(stat.f_fsid.__val[0] != 0 || stat.f_fsid.__val[1] != 0) {
+        errno = EINVAL;
+        if(reason) *reason = "is not a virtual file system";
+        return -1;
+    }
+#endif
+
+    return 0;
+}
+
+int verify_netdata_host_prefix() {
+    if(!netdata_configured_host_prefix)
+        netdata_configured_host_prefix = "";
+
+    if(!*netdata_configured_host_prefix)
+        return 0;
+
+    char buffer[FILENAME_MAX + 1];
+    char *path = netdata_configured_host_prefix;
+    char *reason = "unknown reason";
+    errno = 0;
+
+    struct stat sb;
+    if (stat(path, &sb) == -1) {
+        reason = "failed to stat()";
+        goto failed;
+    }
+
+    if((sb.st_mode & S_IFMT) != S_IFDIR) {
+        errno = EINVAL;
+        reason = "is not a directory";
+        goto failed;
+    }
+
+    path = buffer;
+    snprintfz(path, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix);
+    if(is_virtual_filesystem(path, &reason) == -1)
+        goto failed;
+
+    snprintfz(path, FILENAME_MAX, "%s/sys", netdata_configured_host_prefix);
+    if(is_virtual_filesystem(path, &reason) == -1)
+        goto failed;
+
+    if(netdata_configured_host_prefix && *netdata_configured_host_prefix)
+        info("Using host prefix directory '%s'", netdata_configured_host_prefix);
+
+    return 0;
+
+failed:
+    error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason);
+    netdata_configured_host_prefix = "";
+    return -1;
+}
+
+char *strdupz_path_subpath(const char *path, const char *subpath) {
+    if(unlikely(!path || !*path)) path = ".";
+    if(unlikely(!subpath)) subpath = "";
+
+    // skip trailing slashes in path
+    size_t len = strlen(path);
+    while(len > 0 && path[len - 1] == '/') len--;
+
+    // skip leading slashes in subpath
+    while(subpath[0] == '/') subpath++;
+
+    // if the last character in path is / and (there is a subpath or path is now empty)
+    // keep the trailing slash in path and remove the additional slash
+    char *slash = "/";
+    if(path[len] == '/' && (*subpath || len == 0)) {
+        slash = "";
+        len++;
+    }
+    else if(!*subpath) {
+        // there is no subpath
+        // no need for trailing slash
+        slash = "";
+    }
+
+    char buffer[FILENAME_MAX + 1];
+    snprintfz(buffer, FILENAME_MAX, "%.*s%s%s", (int)len, path, slash, subpath);
+    return strdupz(buffer);
+}
+
+int path_is_dir(const char *path, const char *subpath) {
+    char *s = strdupz_path_subpath(path, subpath);
+
+    size_t max_links = 100;
+
+    int is_dir = 0;
+    struct stat statbuf;
+    while(max_links-- && stat(s, &statbuf) == 0) {
+        if((statbuf.st_mode & S_IFMT) == S_IFDIR) {
+            is_dir = 1;
+            break;
+        }
+        else if((statbuf.st_mode & S_IFMT) == S_IFLNK) {
+            char buffer[FILENAME_MAX + 1];
+            ssize_t l = readlink(s, buffer, FILENAME_MAX);
+            if(l > 0) {
+                buffer[l] = '\0';
+                freez(s);
+                s = strdupz(buffer);
+                continue;
+            }
+            else {
+                is_dir = 0;
+                break;
+            }
+        }
+        else {
+            is_dir = 0;
+            break;
+        }
+    }
+
+    freez(s);
+    return is_dir;
+}
+
+int path_is_file(const char *path, const char *subpath) {
+    char *s = strdupz_path_subpath(path, subpath);
+
+    size_t max_links = 100;
+
+    int is_file = 0;
+    struct stat statbuf;
+    while(max_links-- && stat(s, &statbuf) == 0) {
+        if((statbuf.st_mode & S_IFMT) == S_IFREG) {
+            is_file = 1;
+            break;
+        }
+        else if((statbuf.st_mode & S_IFMT) == S_IFLNK) {
+            char buffer[FILENAME_MAX + 1];
+            ssize_t l = readlink(s, buffer, FILENAME_MAX);
+            if(l > 0) {
+                buffer[l] = '\0';
+                freez(s);
+                s = strdupz(buffer);
+                continue;
+            }
+            else {
+                is_file = 0;
+                break;
+            }
+        }
+        else {
+            is_file = 0;
+            break;
+        }
+    }
+
+    freez(s);
+    return is_file;
+}
+
+void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data, size_t depth) {
+    if(depth > 3) {
+        error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath);
+        return;
+    }
+
+    char *udir = strdupz_path_subpath(user_path, subpath);
+    char *sdir = strdupz_path_subpath(stock_path, subpath);
+
+    debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir);
+
+    DIR *dir = opendir(udir);
+    if (!dir) {
+        error("CONFIG cannot open user-config directory '%s'.", udir);
+    }
+    else {
+        struct dirent *de = NULL;
+        while((de = readdir(dir))) {
+            if(de->d_type == DT_DIR || de->d_type == DT_LNK) {
+                if( !de->d_name[0] ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '\0') ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+                        ) {
+                    debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name);
+                    continue;
+                }
+
+                if(path_is_dir(udir, de->d_name)) {
+                    recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1);
+                    continue;
+                }
+            }
+
+            if(de->d_type == DT_REG || de->d_type == DT_LNK) {
+                size_t len = strlen(de->d_name);
+                if(path_is_file(udir, de->d_name) &&
+                   len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
+                    char *filename = strdupz_path_subpath(udir, de->d_name);
+                    callback(filename, data);
+                    freez(filename);
+                }
+                else
+                    debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s'", udir, de->d_name);
+            }
+        }
+
+        closedir(dir);
+    }
+
+    debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir);
+
+    dir = opendir(sdir);
+    if (!dir) {
+        error("CONFIG cannot open stock config directory '%s'.", sdir);
+    }
+    else {
+        struct dirent *de = NULL;
+        while((de = readdir(dir))) {
+            if(de->d_type == DT_DIR || de->d_type == DT_LNK) {
+                if( !de->d_name[0] ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '\0') ||
+                    (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+                        ) {
+                    debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name);
+                    continue;
+                }
+
+                if(path_is_dir(sdir, de->d_name)) {
+                    // we recurse in stock subdirectory, only when there is no corresponding
+                    // user subdirectory - to avoid reading the files twice
+
+                    if(!path_is_dir(udir, de->d_name))
+                        recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1);
+
+                    continue;
+                }
+            }
+
+            if(de->d_type == DT_REG || de->d_type == DT_LNK) {
+                size_t len = strlen(de->d_name);
+                if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) &&
+                   len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
+                    char *filename = strdupz_path_subpath(sdir, de->d_name);
+                    callback(filename, data);
+                    freez(filename);
+                }
+                else
+                    debug(D_HEALTH, "CONFIG ignoring stock config file '%s/%s'", sdir, de->d_name);
+            }
+        }
+
+        closedir(dir);
+    }
+
+    freez(udir);
+    freez(sdir);
+}