summaryrefslogtreecommitdiffstats
path: root/libnetdata
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--libnetdata/Makefile.am32
-rw-r--r--libnetdata/README.md10
-rw-r--r--libnetdata/adaptive_resortable_list/Makefile.am8
-rw-r--r--libnetdata/adaptive_resortable_list/README.md99
-rw-r--r--libnetdata/adaptive_resortable_list/adaptive_resortable_list.c280
-rw-r--r--libnetdata/adaptive_resortable_list/adaptive_resortable_list.h138
-rw-r--r--libnetdata/avl/Makefile.am8
-rw-r--r--libnetdata/avl/README.md17
-rw-r--r--libnetdata/avl/avl.c420
-rw-r--r--libnetdata/avl/avl.h92
-rw-r--r--libnetdata/buffer/Makefile.am8
-rw-r--r--libnetdata/buffer/README.md17
-rw-r--r--libnetdata/buffer/buffer.c426
-rw-r--r--libnetdata/buffer/buffer.h86
-rw-r--r--libnetdata/circular_buffer/Makefile.am8
-rw-r--r--libnetdata/circular_buffer/README.md12
-rw-r--r--libnetdata/circular_buffer/circular_buffer.c85
-rw-r--r--libnetdata/circular_buffer/circular_buffer.h16
-rw-r--r--libnetdata/clocks/Makefile.am8
-rw-r--r--libnetdata/clocks/README.md5
-rw-r--r--libnetdata/clocks/clocks.c300
-rw-r--r--libnetdata/clocks/clocks.h169
-rw-r--r--libnetdata/config/Makefile.am8
-rw-r--r--libnetdata/config/README.md54
-rw-r--r--libnetdata/config/appconfig.c883
-rw-r--r--libnetdata/config/appconfig.h209
-rw-r--r--libnetdata/dictionary/Makefile.am8
-rw-r--r--libnetdata/dictionary/README.md5
-rw-r--r--libnetdata/dictionary/dictionary.c329
-rw-r--r--libnetdata/dictionary/dictionary.h49
-rw-r--r--libnetdata/ebpf/Makefile.am8
-rw-r--r--libnetdata/ebpf/README.md5
-rw-r--r--libnetdata/ebpf/ebpf.c324
-rw-r--r--libnetdata/ebpf/ebpf.h105
-rw-r--r--libnetdata/eval/Makefile.am8
-rw-r--r--libnetdata/eval/README.md1
-rw-r--r--libnetdata/eval/eval.c1190
-rw-r--r--libnetdata/eval/eval.h88
-rw-r--r--libnetdata/health/Makefile.am7
-rw-r--r--libnetdata/health/health.c173
-rw-r--r--libnetdata/health/health.h55
-rw-r--r--libnetdata/inlined.h318
-rw-r--r--libnetdata/json/Makefile.am8
-rw-r--r--libnetdata/json/README.md10
-rw-r--r--libnetdata/json/jsmn.c328
-rw-r--r--libnetdata/json/jsmn.h75
-rw-r--r--libnetdata/json/json.c556
-rw-r--r--libnetdata/json/json.h72
-rw-r--r--libnetdata/libnetdata.c1493
-rw-r--r--libnetdata/libnetdata.h340
-rw-r--r--libnetdata/locks/Makefile.am8
-rw-r--r--libnetdata/locks/README.md5
-rw-r--r--libnetdata/locks/locks.c343
-rw-r--r--libnetdata/locks/locks.h75
-rw-r--r--libnetdata/log/Makefile.am8
-rw-r--r--libnetdata/log/README.md5
-rw-r--r--libnetdata/log/log.c872
-rw-r--r--libnetdata/log/log.h109
-rw-r--r--libnetdata/os.c217
-rw-r--r--libnetdata/os.h70
-rw-r--r--libnetdata/popen/Makefile.am8
-rw-r--r--libnetdata/popen/README.md5
-rw-r--r--libnetdata/popen/popen.c337
-rw-r--r--libnetdata/popen/popen.h23
-rw-r--r--libnetdata/procfile/Makefile.am8
-rw-r--r--libnetdata/procfile/README.md67
-rw-r--r--libnetdata/procfile/procfile.c472
-rw-r--r--libnetdata/procfile/procfile.h106
-rw-r--r--libnetdata/required_dummies.h38
-rw-r--r--libnetdata/simple_pattern/Makefile.am8
-rw-r--r--libnetdata/simple_pattern/README.md43
-rw-r--r--libnetdata/simple_pattern/simple_pattern.c365
-rw-r--r--libnetdata/simple_pattern/simple_pattern.h40
-rw-r--r--libnetdata/socket/Makefile.am8
-rw-r--r--libnetdata/socket/README.md5
-rw-r--r--libnetdata/socket/security.c385
-rw-r--r--libnetdata/socket/security.h55
-rw-r--r--libnetdata/socket/socket.c1733
-rw-r--r--libnetdata/socket/socket.h207
-rw-r--r--libnetdata/statistical/Makefile.am8
-rw-r--r--libnetdata/statistical/README.md5
-rw-r--r--libnetdata/statistical/statistical.c452
-rw-r--r--libnetdata/statistical/statistical.h26
-rw-r--r--libnetdata/storage_number/Makefile.am12
-rw-r--r--libnetdata/storage_number/README.md17
-rw-r--r--libnetdata/storage_number/storage_number.c252
-rw-r--r--libnetdata/storage_number/storage_number.h94
-rw-r--r--libnetdata/storage_number/tests/Makefile.am4
-rw-r--r--libnetdata/storage_number/tests/test_storage_number.c52
-rw-r--r--libnetdata/string/utf8.h9
-rw-r--r--libnetdata/tests/Makefile.am4
-rw-r--r--libnetdata/tests/test_str2ld.c48
-rw-r--r--libnetdata/threads/Makefile.am8
-rw-r--r--libnetdata/threads/README.md5
-rw-r--r--libnetdata/threads/threads.c241
-rw-r--r--libnetdata/threads/threads.h42
-rw-r--r--libnetdata/url/Makefile.am8
-rw-r--r--libnetdata/url/README.md5
-rw-r--r--libnetdata/url/url.c391
-rw-r--r--libnetdata/url/url.h35
100 files changed, 16296 insertions, 0 deletions
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am
new file mode 100644
index 0000000..598b72f
--- /dev/null
+++ b/libnetdata/Makefile.am
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+SUBDIRS = \
+ adaptive_resortable_list \
+ avl \
+ buffer \
+ clocks \
+ config \
+ dictionary \
+ ebpf \
+ eval \
+ json \
+ health \
+ locks \
+ log \
+ popen \
+ procfile \
+ simple_pattern \
+ socket \
+ statistical \
+ storage_number \
+ threads \
+ url \
+ tests \
+ $(NULL)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/README.md b/libnetdata/README.md
new file mode 100644
index 0000000..cdb199f
--- /dev/null
+++ b/libnetdata/README.md
@@ -0,0 +1,10 @@
+<!--
+title: "libnetdata"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/README.md
+-->
+
+# libnetdata
+
+`libnetdata` is a collection of library code that is used by all Netdata `C` programs.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/adaptive_resortable_list/Makefile.am b/libnetdata/adaptive_resortable_list/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/adaptive_resortable_list/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/adaptive_resortable_list/README.md b/libnetdata/adaptive_resortable_list/README.md
new file mode 100644
index 0000000..74e379a
--- /dev/null
+++ b/libnetdata/adaptive_resortable_list/README.md
@@ -0,0 +1,99 @@
+<!--
+title: "Adaptive Re-sortable List (ARL)"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/adaptive_resortable_list/README.md
+-->
+
+# Adaptive Re-sortable List (ARL)
+
+This library allows Netdata to read a series of `name - value` pairs
+in the **fastest possible way**.
+
+ARLs are used all over Netdata, as they are the most
+CPU utilization efficient way to process `/proc` files. They are used to
+process both vertical (csv like) and horizontal (one pair per line) `name - value` pairs.
+
+## How ARL works
+
+It maintains a linked list of all `NAME` (keywords), sorted in the
+order found in the data source. The linked list is kept
+sorted at all times - the data source may change at any time, the
+linked list will adapt at the next iteration.
+
+### Initialization
+
+During initialization (just once), the caller:
+
+- calls `arl_create()` to create the ARL
+
+- calls `arl_expect()` multiple times to register the expected keywords
+
+The library will call the `processor()` function (given to
+`arl_create()`), for each expected keyword found.
+The default `processor()` expects `dst` to be an `unsigned long long *`.
+
+Each `name` keyword may have a different `processor()` (by calling
+`arl_expect_custom()` instead of `arl_expect()`).
+
+### Data collection iterations
+
+For each iteration through the data source, the caller:
+
+- calls `arl_begin()` to initiate a data collection iteration.
+ This is to be called just ONCE every time the source is re-evaluated.
+
+- calls `arl_check()` for each entry read from the file.
+
+### Cleanup
+
+When the caller exits:
+
+- calls `arl_free()` to destroy this and free all memory.
+
+### Performance
+
+ARL maintains a list of `name` keywords found in the data source (even the ones
+that are not useful for data collection).
+
+If the data source maintains the same order on the `name-value` pairs, for each
+each call to `arl_check()` only an `strcmp()` is executed to verify the
+expected order has not changed, a counter is incremented and a pointer is changed.
+So, if the data source has 100 `name-value` pairs, and their order remains constant
+over time, 100 successful `strcmp()` are executed.
+
+In the unlikely event that an iteration sees the data source with a different order,
+for each out-of-order keyword, a full search of the remaining keywords is made. But
+this search uses 32bit hashes, not string comparisons, so it should also be fast.
+
+When all expectations are satisfied (even in the middle of an iteration),
+the call to `arl_check()` will return 1, to signal the caller to stop the loop,
+saving valuable CPU resources for the rest of the data source.
+
+In the following test we used alternative methods to process, **1M times**,
+a data source like `/proc/meminfo`, already tokenized, in memory,
+to extract the same number of expected metrics:
+
+|test|code|string comparison|number parsing|duration|
+|:--:|:--:|:---------------:|:------------:|:------:|
+|1|if-else-if-else-if|`strcmp()`|`strtoull()`|4630.337 ms|
+|2|nested loops|inline `simple_hash()` and `strcmp()`|`strtoull()`|1597.481 ms|
+|3|nested loops|inline `simple_hash()` and `strcmp()`|`str2ull()`|923.523 ms|
+|4|if-else-if-else-if|inline `simple_hash()` and `strcmp()`|`strtoull()`|854.574 ms|
+|5|if-else-if-else-if|statement expression `simple_hash()` and `strcmp()`|`strtoull()`|912.013 ms|
+|6|if-continue|inline `simple_hash()` and `strcmp()`|`strtoull()`|842.279 ms|
+|7|if-else-if-else-if|inline `simple_hash()` and `strcmp()`|`str2ull()`|602.837 ms|
+|8|ARL|ARL|`strtoull()`|350.360 ms|
+|9|ARL|ARL|`str2ull()`|157.237 ms|
+
+Compared to unoptimized code (test No 1: 4.6sec):
+
+- before ARL Netdata was using test No **7** with hashing and a custom `str2ull()` to achieve 602ms.
+- the current ARL implementation is test No **9** that needs only 157ms (29 times faster vs unoptimized code, about 4 times faster vs optimized code).
+
+[Check the source code of this test](https://raw.githubusercontent.com/netdata/netdata/master/tests/profile/benchmark-value-pairs.c).
+
+## Limitations
+
+Do not use ARL if the a name/keyword may appear more than once in the
+source data.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fadaptive_resortable_list%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
new file mode 100644
index 0000000..7f4c6c5
--- /dev/null
+++ b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+// the default processor() of the ARL
+// can be overwritten at arl_create()
+inline void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name;
+ (void)hash;
+
+ register unsigned long long *d = dst;
+ *d = str2ull(value);
+ // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, *d);
+}
+
+inline void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name;
+ (void)hash;
+
+ register kernel_uint_t *d = dst;
+ *d = str2kernel_uint_t(value);
+ // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, (unsigned long long)*d);
+}
+
+inline void arl_callback_ssize_t(const char *name, uint32_t hash, const char *value, void *dst) {
+ (void)name;
+ (void)hash;
+
+ register ssize_t *d = dst;
+ *d = (ssize_t)str2ll(value, NULL);
+ // fprintf(stderr, "name '%s' with hash %u and value '%s' is %zd\n", name, hash, value, *d);
+}
+
+// create a new ARL
+ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks) {
+ ARL_BASE *base = callocz(1, sizeof(ARL_BASE));
+
+ base->name = strdupz(name);
+
+ if(!processor)
+ base->processor = arl_callback_str2ull;
+ else
+ base->processor = processor;
+
+ base->rechecks = rechecks;
+
+ return base;
+}
+
+void arl_free(ARL_BASE *arl_base) {
+ if(unlikely(!arl_base))
+ return;
+
+ while(arl_base->head) {
+ ARL_ENTRY *e = arl_base->head;
+ arl_base->head = e->next;
+
+ freez(e->name);
+#ifdef NETDATA_INTERNAL_CHECKS
+ memset(e, 0, sizeof(ARL_ENTRY));
+#endif
+ freez(e);
+ }
+
+ freez(arl_base->name);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ memset(arl_base, 0, sizeof(ARL_BASE));
+#endif
+
+ freez(arl_base);
+}
+
+void arl_begin(ARL_BASE *base) {
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(likely(base->iteration > 10)) {
+ // do these checks after the ARL has been sorted
+
+ if(unlikely(base->relinkings > (base->expected + base->allocated)))
+ info("ARL '%s' has %zu relinkings with %zu expected and %zu allocated entries. Is the source changing so fast?"
+ , base->name, base->relinkings, base->expected, base->allocated);
+
+ if(unlikely(base->slow > base->fast))
+ info("ARL '%s' has %zu fast searches and %zu slow searches. Is the source really changing so fast?"
+ , base->name, base->fast, base->slow);
+
+ /*
+ if(unlikely(base->iteration % 60 == 0)) {
+ info("ARL '%s' statistics: iteration %zu, expected %zu, wanted %zu, allocated %zu, fred %zu, relinkings %zu, found %zu, added %zu, fast %zu, slow %zu"
+ , base->name
+ , base->iteration
+ , base->expected
+ , base->wanted
+ , base->allocated
+ , base->fred
+ , base->relinkings
+ , base->found
+ , base->added
+ , base->fast
+ , base->slow
+ );
+ // for(e = base->head; e; e = e->next) fprintf(stderr, "%s ", e->name);
+ // fprintf(stderr, "\n");
+ }
+ */
+ }
+#endif
+
+ if(unlikely(base->iteration > 0 && (base->added || (base->iteration % base->rechecks) == 0))) {
+ int wanted_equals_expected = ((base->iteration % base->rechecks) == 0);
+
+ // fprintf(stderr, "\n\narl_begin() rechecking, added %zu, iteration %zu, rechecks %zu, wanted_equals_expected %d\n\n\n", base->added, base->iteration, base->rechecks, wanted_equals_expected);
+
+ base->added = 0;
+ base->wanted = (wanted_equals_expected)?base->expected:0;
+
+ ARL_ENTRY *e = base->head;
+ while(e) {
+ if(e->flags & ARL_ENTRY_FLAG_FOUND) {
+
+ // remove the found flag
+ e->flags &= ~ARL_ENTRY_FLAG_FOUND;
+
+ // count it in wanted
+ if(!wanted_equals_expected && e->flags & ARL_ENTRY_FLAG_EXPECTED)
+ base->wanted++;
+
+ }
+ else if(e->flags & ARL_ENTRY_FLAG_DYNAMIC && !(base->head == e && !e->next)) { // not last entry
+ // we can remove this entry
+ // it is not found, and it was created because
+ // it was found in the source file
+
+ // remember the next one
+ ARL_ENTRY *t = e->next;
+
+ // remove it from the list
+ if(e->next) e->next->prev = e->prev;
+ if(e->prev) e->prev->next = e->next;
+ if(base->head == e) base->head = e->next;
+
+ // free it
+ freez(e->name);
+ freez(e);
+
+ // count it
+ base->fred++;
+
+ // continue
+ e = t;
+ continue;
+ }
+
+ e = e->next;
+ }
+ }
+
+ if(unlikely(!base->head)) {
+ // hm... no nodes at all in the list #1700
+ // add a fake one to prevent a crash
+ // this is better than checking for the existence of nodes all the time
+ arl_expect(base, "a-really-not-existing-source-keyword", NULL);
+ }
+
+ base->iteration++;
+ base->next_keyword = base->head;
+ base->found = 0;
+
+}
+
+// register an expected keyword to the ARL
+// together with its destination ( i.e. the output of the processor() )
+ARL_ENTRY *arl_expect_custom(ARL_BASE *base, const char *keyword, void (*processor)(const char *name, uint32_t hash, const char *value, void *dst), void *dst) {
+ ARL_ENTRY *e = callocz(1, sizeof(ARL_ENTRY));
+ e->name = strdupz(keyword);
+ e->hash = simple_hash(e->name);
+ e->processor = (processor)?processor:base->processor;
+ e->dst = dst;
+ e->flags = ARL_ENTRY_FLAG_EXPECTED;
+ e->prev = NULL;
+ e->next = base->head;
+
+ if(base->head) base->head->prev = e;
+ else base->next_keyword = e;
+
+ base->head = e;
+ base->expected++;
+ base->allocated++;
+
+ base->wanted = base->expected;
+
+ return e;
+}
+
+int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const char *value) {
+ ARL_ENTRY *e;
+
+ uint32_t hash = simple_hash(s);
+
+ // find if it already exists in the data
+ for(e = base->head; e ; e = e->next)
+ if(e->hash == hash && !strcmp(e->name, s))
+ break;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(base->next_keyword && e == base->next_keyword))
+ fatal("Internal Error: e == base->last");
+#endif
+
+ if(e) {
+ // found it in the keywords
+
+ base->relinkings++;
+
+ // run the processor for it
+ if(unlikely(e->dst)) {
+ e->processor(e->name, hash, value, e->dst);
+ base->found++;
+ }
+
+ // unlink it - we will relink it below
+ if(e->next) e->next->prev = e->prev;
+ if(e->prev) e->prev->next = e->next;
+
+ // make sure the head is properly linked
+ if(base->head == e)
+ base->head = e->next;
+ }
+ else {
+ // not found
+
+ // create it
+ e = callocz(1, sizeof(ARL_ENTRY));
+ e->name = strdupz(s);
+ e->hash = hash;
+ e->flags = ARL_ENTRY_FLAG_DYNAMIC;
+
+ base->allocated++;
+ base->added++;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely(base->iteration % 60 == 0 && e->flags & ARL_ENTRY_FLAG_FOUND))
+ info("ARL '%s': entry '%s' is already found. Did you forget to call arl_begin()?", base->name, s);
+#endif
+
+ e->flags |= ARL_ENTRY_FLAG_FOUND;
+
+ // link it here
+ e->next = base->next_keyword;
+ if(base->next_keyword) {
+ e->prev = base->next_keyword->prev;
+ base->next_keyword->prev = e;
+
+ if(e->prev)
+ e->prev->next = e;
+
+ if(base->head == base->next_keyword)
+ base->head = e;
+ }
+ else {
+ e->prev = NULL;
+
+ if(!base->head)
+ base->head = e;
+ }
+
+ // prepare the next iteration
+ base->next_keyword = e->next;
+ if(unlikely(!base->next_keyword))
+ base->next_keyword = base->head;
+
+ if(unlikely(base->found == base->wanted)) {
+ // fprintf(stderr, "FOUND ALL WANTED 1: found = %zu, wanted = %zu, expected %zu\n", base->found, base->wanted, base->expected);
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h
new file mode 100644
index 0000000..011ee73
--- /dev/null
+++ b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+#ifndef NETDATA_ADAPTIVE_RESORTABLE_LIST_H
+#define NETDATA_ADAPTIVE_RESORTABLE_LIST_H 1
+
+#define ARL_ENTRY_FLAG_FOUND 0x01 // the entry has been found in the source data
+#define ARL_ENTRY_FLAG_EXPECTED 0x02 // the entry is expected by the program
+#define ARL_ENTRY_FLAG_DYNAMIC 0x04 // the entry was dynamically allocated, from source data
+
+typedef struct arl_entry {
+ char *name; // the keywords
+ uint32_t hash; // the hash of the keyword
+
+ void *dst; // the dst to pass to the processor
+
+ uint8_t flags; // ARL_ENTRY_FLAG_*
+
+ // the processor to do the job
+ void (*processor)(const char *name, uint32_t hash, const char *value, void *dst);
+
+ // double linked list for fast re-linkings
+ struct arl_entry *prev, *next;
+} ARL_ENTRY;
+
+typedef struct arl_base {
+ char *name;
+
+ size_t iteration; // incremented on each iteration (arl_begin())
+ size_t found; // the number of expected keywords found in this iteration
+ size_t expected; // the number of expected keywords
+ size_t wanted; // the number of wanted keywords
+ // i.e. the number of keywords found and expected
+
+ size_t relinkings; // the number of relinkings we have made so far
+
+ size_t allocated; // the number of keywords allocated
+ size_t fred; // the number of keywords cleaned up
+
+ size_t rechecks; // the number of iterations between re-checks of the
+ // wanted number of keywords
+ // this is only needed in cases where the source
+ // is having less lines over time.
+
+ size_t added; // it is non-zero if new keywords have been added
+ // this is only needed to detect new lines have
+ // been added to the file, over time.
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ size_t fast; // the number of times we have taken the fast path
+ size_t slow; // the number of times we have taken the slow path
+#endif
+
+ // the processor to do the job
+ void (*processor)(const char *name, uint32_t hash, const char *value, void *dst);
+
+ // the linked list of the keywords
+ ARL_ENTRY *head;
+
+ // since we keep the list of keywords sorted (as found in the source data)
+ // this is next keyword that we expect to find in the source data.
+ ARL_ENTRY *next_keyword;
+} ARL_BASE;
+
+// create a new ARL
+extern ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks);
+
+// free an ARL
+extern void arl_free(ARL_BASE *arl_base);
+
+// register an expected keyword to the ARL
+// together with its destination ( i.e. the output of the processor() )
+extern ARL_ENTRY *arl_expect_custom(ARL_BASE *base, const char *keyword, void (*processor)(const char *name, uint32_t hash, const char *value, void *dst), void *dst);
+#define arl_expect(base, keyword, dst) arl_expect_custom(base, keyword, NULL, dst)
+
+// an internal call to complete the check() call
+extern int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const char *value);
+
+// begin an ARL iteration
+extern void arl_begin(ARL_BASE *base);
+
+extern void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst);
+extern void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, const char *value, void *dst);
+extern void arl_callback_ssize_t(const char *name, uint32_t hash, const char *value, void *dst);
+
+// check a keyword against the ARL
+// this is to be called for each keyword read from source data
+// s = the keyword, as collected
+// src = the src data to be passed to the processor
+// it is defined in the header file in order to be inlined
+static inline int arl_check(ARL_BASE *base, const char *keyword, const char *value) {
+ ARL_ENTRY *e = base->next_keyword;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ if(unlikely((base->fast + base->slow) % (base->expected + base->allocated) == 0 && (base->fast + base->slow) > (base->expected + base->allocated) * base->iteration))
+ info("ARL '%s': Did you forget to call arl_begin()?", base->name);
+#endif
+
+ // it should be the first entry (pointed by base->next_keyword)
+ if(likely(!strcmp(keyword, e->name))) {
+ // it is
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ base->fast++;
+#endif
+
+ e->flags |= ARL_ENTRY_FLAG_FOUND;
+
+ // execute the processor
+ if(unlikely(e->dst)) {
+ e->processor(e->name, e->hash, value, e->dst);
+ base->found++;
+ }
+
+ // be prepared for the next iteration
+ base->next_keyword = e->next;
+ if(unlikely(!base->next_keyword))
+ base->next_keyword = base->head;
+
+ // stop if we collected all the values for this iteration
+ if(unlikely(base->found == base->wanted)) {
+ // fprintf(stderr, "FOUND ALL WANTED 2: found = %zu, wanted = %zu, expected %zu\n", base->found, base->wanted, base->expected);
+ return 1;
+ }
+
+ return 0;
+ }
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ base->slow++;
+#endif
+
+ // we read from source, a not-expected keyword
+ return arl_find_or_create_and_relink(base, keyword, value);
+}
+
+#endif //NETDATA_ADAPTIVE_RESORTABLE_LIST_H
diff --git a/libnetdata/avl/Makefile.am b/libnetdata/avl/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/avl/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/avl/README.md b/libnetdata/avl/README.md
new file mode 100644
index 0000000..2097be5
--- /dev/null
+++ b/libnetdata/avl/README.md
@@ -0,0 +1,17 @@
+<!--
+title: "AVL"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/avl/README.md
+-->
+
+# AVL
+
+AVL is a library indexing objects in B-Trees.
+
+`avl_insert()`, `avl_remove()` and `avl_search()` are adaptations
+of the AVL algorithm found in `libavl` v2.0.3, so that they do not
+use any memory allocations and their memory footprint is optimized
+(by eliminating non-necessary data members).
+
+In addition to the above, this version of AVL, provides versions using locks
+and traversal functions.
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Favl%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/avl/avl.c b/libnetdata/avl/avl.c
new file mode 100644
index 0000000..5219851
--- /dev/null
+++ b/libnetdata/avl/avl.c
@@ -0,0 +1,420 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+/* ------------------------------------------------------------------------- */
+/*
+ * avl_insert(), avl_remove() and avl_search()
+ * are adaptations (by Costa Tsaousis) of the AVL algorithm found in libavl
+ * v2.0.3, so that they do not use any memory allocations and their memory
+ * footprint is optimized (by eliminating non-necessary data members).
+ *
+ * libavl - library for manipulation of binary trees.
+ * Copyright (C) 1998, 1999, 2000, 2001, 2002, 2004 Free Software
+ * Foundation, Inc.
+*/
+
+
+/* Search |tree| for an item matching |item|, and return it if found.
+ Otherwise return |NULL|. */
+avl *avl_search(avl_tree_type *tree, avl *item) {
+ avl *p;
+
+ // assert (tree != NULL && item != NULL);
+
+ for (p = tree->root; p != NULL; ) {
+ int cmp = tree->compar(item, p);
+
+ if (cmp < 0)
+ p = p->avl_link[0];
+ else if (cmp > 0)
+ p = p->avl_link[1];
+ else /* |cmp == 0| */
+ return p;
+ }
+
+ return NULL;
+}
+
+/* Inserts |item| into |tree| and returns a pointer to |item|'s address.
+ If a duplicate item is found in the tree,
+ returns a pointer to the duplicate without inserting |item|.
+ */
+avl *avl_insert(avl_tree_type *tree, avl *item) {
+ avl *y, *z; /* Top node to update balance factor, and parent. */
+ avl *p, *q; /* Iterator, and parent. */
+ avl *n; /* Newly inserted node. */
+ avl *w; /* New root of rebalanced subtree. */
+ unsigned char dir; /* Direction to descend. */
+
+ unsigned char da[AVL_MAX_HEIGHT]; /* Cached comparison results. */
+ int k = 0; /* Number of cached results. */
+
+ // assert(tree != NULL && item != NULL);
+
+ z = (avl *) &tree->root;
+ y = tree->root;
+ dir = 0;
+ for (q = z, p = y; p != NULL; q = p, p = p->avl_link[dir]) {
+ int cmp = tree->compar(item, p);
+ if (cmp == 0)
+ return p;
+
+ if (p->avl_balance != 0)
+ z = q, y = p, k = 0;
+ da[k++] = dir = (unsigned char)(cmp > 0);
+ }
+
+ n = q->avl_link[dir] = item;
+
+ // tree->avl_count++;
+ n->avl_link[0] = n->avl_link[1] = NULL;
+ n->avl_balance = 0;
+ if (y == NULL) return n;
+
+ for (p = y, k = 0; p != n; p = p->avl_link[da[k]], k++)
+ if (da[k] == 0)
+ p->avl_balance--;
+ else
+ p->avl_balance++;
+
+ if (y->avl_balance == -2) {
+ avl *x = y->avl_link[0];
+ if (x->avl_balance == -1) {
+ w = x;
+ y->avl_link[0] = x->avl_link[1];
+ x->avl_link[1] = y;
+ x->avl_balance = y->avl_balance = 0;
+ }
+ else {
+ // assert (x->avl_balance == +1);
+ w = x->avl_link[1];
+ x->avl_link[1] = w->avl_link[0];
+ w->avl_link[0] = x;
+ y->avl_link[0] = w->avl_link[1];
+ w->avl_link[1] = y;
+ if (w->avl_balance == -1)
+ x->avl_balance = 0, y->avl_balance = +1;
+ else if (w->avl_balance == 0)
+ x->avl_balance = y->avl_balance = 0;
+ else /* |w->avl_balance == +1| */
+ x->avl_balance = -1, y->avl_balance = 0;
+ w->avl_balance = 0;
+ }
+ }
+ else if (y->avl_balance == +2) {
+ avl *x = y->avl_link[1];
+ if (x->avl_balance == +1) {
+ w = x;
+ y->avl_link[1] = x->avl_link[0];
+ x->avl_link[0] = y;
+ x->avl_balance = y->avl_balance = 0;
+ }
+ else {
+ // assert (x->avl_balance == -1);
+ w = x->avl_link[0];
+ x->avl_link[0] = w->avl_link[1];
+ w->avl_link[1] = x;
+ y->avl_link[1] = w->avl_link[0];
+ w->avl_link[0] = y;
+ if (w->avl_balance == +1)
+ x->avl_balance = 0, y->avl_balance = -1;
+ else if (w->avl_balance == 0)
+ x->avl_balance = y->avl_balance = 0;
+ else /* |w->avl_balance == -1| */
+ x->avl_balance = +1, y->avl_balance = 0;
+ w->avl_balance = 0;
+ }
+ }
+ else return n;
+
+ z->avl_link[y != z->avl_link[0]] = w;
+
+ // tree->avl_generation++;
+ return n;
+}
+
+/* Deletes from |tree| and returns an item matching |item|.
+ Returns a null pointer if no matching item found. */
+avl *avl_remove(avl_tree_type *tree, avl *item) {
+ /* Stack of nodes. */
+ avl *pa[AVL_MAX_HEIGHT]; /* Nodes. */
+ unsigned char da[AVL_MAX_HEIGHT]; /* |avl_link[]| indexes. */
+ int k; /* Stack pointer. */
+
+ avl *p; /* Traverses tree to find node to delete. */
+ int cmp; /* Result of comparison between |item| and |p|. */
+
+ // assert (tree != NULL && item != NULL);
+
+ k = 0;
+ p = (avl *) &tree->root;
+ for(cmp = -1; cmp != 0; cmp = tree->compar(item, p)) {
+ unsigned char dir = (unsigned char)(cmp > 0);
+
+ pa[k] = p;
+ da[k++] = dir;
+
+ p = p->avl_link[dir];
+ if(p == NULL) return NULL;
+ }
+
+ item = p;
+
+ if (p->avl_link[1] == NULL)
+ pa[k - 1]->avl_link[da[k - 1]] = p->avl_link[0];
+ else {
+ avl *r = p->avl_link[1];
+ if (r->avl_link[0] == NULL) {
+ r->avl_link[0] = p->avl_link[0];
+ r->avl_balance = p->avl_balance;
+ pa[k - 1]->avl_link[da[k - 1]] = r;
+ da[k] = 1;
+ pa[k++] = r;
+ }
+ else {
+ avl *s;
+ int j = k++;
+
+ for (;;) {
+ da[k] = 0;
+ pa[k++] = r;
+ s = r->avl_link[0];
+ if (s->avl_link[0] == NULL) break;
+
+ r = s;
+ }
+
+ s->avl_link[0] = p->avl_link[0];
+ r->avl_link[0] = s->avl_link[1];
+ s->avl_link[1] = p->avl_link[1];
+ s->avl_balance = p->avl_balance;
+
+ pa[j - 1]->avl_link[da[j - 1]] = s;
+ da[j] = 1;
+ pa[j] = s;
+ }
+ }
+
+ // assert (k > 0);
+ while (--k > 0) {
+ avl *y = pa[k];
+
+ if (da[k] == 0) {
+ y->avl_balance++;
+ if (y->avl_balance == +1) break;
+ else if (y->avl_balance == +2) {
+ avl *x = y->avl_link[1];
+ if (x->avl_balance == -1) {
+ avl *w;
+ // assert (x->avl_balance == -1);
+ w = x->avl_link[0];
+ x->avl_link[0] = w->avl_link[1];
+ w->avl_link[1] = x;
+ y->avl_link[1] = w->avl_link[0];
+ w->avl_link[0] = y;
+ if (w->avl_balance == +1)
+ x->avl_balance = 0, y->avl_balance = -1;
+ else if (w->avl_balance == 0)
+ x->avl_balance = y->avl_balance = 0;
+ else /* |w->avl_balance == -1| */
+ x->avl_balance = +1, y->avl_balance = 0;
+ w->avl_balance = 0;
+ pa[k - 1]->avl_link[da[k - 1]] = w;
+ }
+ else {
+ y->avl_link[1] = x->avl_link[0];
+ x->avl_link[0] = y;
+ pa[k - 1]->avl_link[da[k - 1]] = x;
+ if (x->avl_balance == 0) {
+ x->avl_balance = -1;
+ y->avl_balance = +1;
+ break;
+ }
+ else x->avl_balance = y->avl_balance = 0;
+ }
+ }
+ }
+ else
+ {
+ y->avl_balance--;
+ if (y->avl_balance == -1) break;
+ else if (y->avl_balance == -2) {
+ avl *x = y->avl_link[0];
+ if (x->avl_balance == +1) {
+ avl *w;
+ // assert (x->avl_balance == +1);
+ w = x->avl_link[1];
+ x->avl_link[1] = w->avl_link[0];
+ w->avl_link[0] = x;
+ y->avl_link[0] = w->avl_link[1];
+ w->avl_link[1] = y;
+ if (w->avl_balance == -1)
+ x->avl_balance = 0, y->avl_balance = +1;
+ else if (w->avl_balance == 0)
+ x->avl_balance = y->avl_balance = 0;
+ else /* |w->avl_balance == +1| */
+ x->avl_balance = -1, y->avl_balance = 0;
+ w->avl_balance = 0;
+ pa[k - 1]->avl_link[da[k - 1]] = w;
+ }
+ else {
+ y->avl_link[0] = x->avl_link[1];
+ x->avl_link[1] = y;
+ pa[k - 1]->avl_link[da[k - 1]] = x;
+ if (x->avl_balance == 0) {
+ x->avl_balance = +1;
+ y->avl_balance = -1;
+ break;
+ }
+ else x->avl_balance = y->avl_balance = 0;
+ }
+ }
+ }
+ }
+
+ // tree->avl_count--;
+ // tree->avl_generation++;
+ return item;
+}
+
+/* ------------------------------------------------------------------------- */
+// below are functions by (C) Costa Tsaousis
+
+// ---------------------------
+// traversing
+
+int avl_walker(avl *node, int (*callback)(void * /*entry*/, void * /*data*/), void *data) {
+ int total = 0, ret = 0;
+
+ if(node->avl_link[0]) {
+ ret = avl_walker(node->avl_link[0], callback, data);
+ if(ret < 0) return ret;
+ total += ret;
+ }
+
+ ret = callback(node, data);
+ if(ret < 0) return ret;
+ total += ret;
+
+ if(node->avl_link[1]) {
+ ret = avl_walker(node->avl_link[1], callback, data);
+ if (ret < 0) return ret;
+ total += ret;
+ }
+
+ return total;
+}
+
+int avl_traverse(avl_tree_type *tree, int (*callback)(void * /*entry*/, void * /*data*/), void *data) {
+ if(tree->root)
+ return avl_walker(tree->root, callback, data);
+ else
+ return 0;
+}
+
+// ---------------------------
+// locks
+
+void avl_read_lock(avl_tree_lock *t) {
+#ifndef AVL_WITHOUT_PTHREADS
+#ifdef AVL_LOCK_WITH_MUTEX
+ netdata_mutex_lock(&t->mutex);
+#else
+ netdata_rwlock_rdlock(&t->rwlock);
+#endif
+#endif /* AVL_WITHOUT_PTHREADS */
+}
+
+void avl_write_lock(avl_tree_lock *t) {
+#ifndef AVL_WITHOUT_PTHREADS
+#ifdef AVL_LOCK_WITH_MUTEX
+ netdata_mutex_lock(&t->mutex);
+#else
+ netdata_rwlock_wrlock(&t->rwlock);
+#endif
+#endif /* AVL_WITHOUT_PTHREADS */
+}
+
+void avl_unlock(avl_tree_lock *t) {
+#ifndef AVL_WITHOUT_PTHREADS
+#ifdef AVL_LOCK_WITH_MUTEX
+ netdata_mutex_unlock(&t->mutex);
+#else
+ netdata_rwlock_unlock(&t->rwlock);
+#endif
+#endif /* AVL_WITHOUT_PTHREADS */
+}
+
+// ---------------------------
+// operations with locking
+
+void avl_init_lock(avl_tree_lock *tree, int (*compar)(void * /*a*/, void * /*b*/)) {
+ avl_init(&tree->avl_tree, compar);
+
+#ifndef AVL_WITHOUT_PTHREADS
+ int lock;
+
+#ifdef AVL_LOCK_WITH_MUTEX
+ lock = netdata_mutex_init(&tree->mutex, NULL);
+#else
+ lock = netdata_rwlock_init(&tree->rwlock);
+#endif
+
+ if(lock != 0)
+ fatal("Failed to initialize AVL mutex/rwlock, error: %d", lock);
+
+#endif /* AVL_WITHOUT_PTHREADS */
+}
+
+void avl_destroy_lock(avl_tree_lock *tree) {
+#ifndef AVL_WITHOUT_PTHREADS
+ int lock;
+
+#ifdef AVL_LOCK_WITH_MUTEX
+ lock = pthread_mutex_destroy(&tree->mutex);
+#else
+ lock = pthread_rwlock_destroy(&tree->rwlock);
+#endif
+
+ if(lock != 0)
+ fatal("Failed to destroy AVL mutex/rwlock, error: %d", lock);
+
+#endif /* AVL_WITHOUT_PTHREADS */
+}
+
+avl *avl_search_lock(avl_tree_lock *tree, avl *item) {
+ avl_read_lock(tree);
+ avl *ret = avl_search(&tree->avl_tree, item);
+ avl_unlock(tree);
+ return ret;
+}
+
+avl * avl_remove_lock(avl_tree_lock *tree, avl *item) {
+ avl_write_lock(tree);
+ avl *ret = avl_remove(&tree->avl_tree, item);
+ avl_unlock(tree);
+ return ret;
+}
+
+avl *avl_insert_lock(avl_tree_lock *tree, avl *item) {
+ avl_write_lock(tree);
+ avl * ret = avl_insert(&tree->avl_tree, item);
+ avl_unlock(tree);
+ return ret;
+}
+
+int avl_traverse_lock(avl_tree_lock *tree, int (*callback)(void * /*entry*/, void * /*data*/), void *data) {
+ int ret;
+ avl_read_lock(tree);
+ ret = avl_traverse(&tree->avl_tree, callback, data);
+ avl_unlock(tree);
+ return ret;
+}
+
+void avl_init(avl_tree_type *tree, int (*compar)(void * /*a*/, void * /*b*/)) {
+ tree->root = NULL;
+ tree->compar = compar;
+}
+
+// ------------------
diff --git a/libnetdata/avl/avl.h b/libnetdata/avl/avl.h
new file mode 100644
index 0000000..32e3f27
--- /dev/null
+++ b/libnetdata/avl/avl.h
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+
+#ifndef _AVL_H
+#define _AVL_H 1
+
+#include "../libnetdata.h"
+
+/* Maximum AVL tree height. */
+#ifndef AVL_MAX_HEIGHT
+#define AVL_MAX_HEIGHT 92
+#endif
+
+#ifndef AVL_WITHOUT_PTHREADS
+#include <pthread.h>
+
+// #define AVL_LOCK_WITH_MUTEX 1
+
+#ifdef AVL_LOCK_WITH_MUTEX
+#define AVL_LOCK_INITIALIZER NETDATA_MUTEX_INITIALIZER
+#else /* AVL_LOCK_WITH_MUTEX */
+#define AVL_LOCK_INITIALIZER NETDATA_RWLOCK_INITIALIZER
+#endif /* AVL_LOCK_WITH_MUTEX */
+
+#else /* AVL_WITHOUT_PTHREADS */
+#define AVL_LOCK_INITIALIZER
+#endif /* AVL_WITHOUT_PTHREADS */
+
+/* Data structures */
+
+/* One element of the AVL tree */
+typedef struct avl {
+ struct avl *avl_link[2]; /* Subtrees. */
+ signed char avl_balance; /* Balance factor. */
+} avl;
+
+/* An AVL tree */
+typedef struct avl_tree_type {
+ avl *root;
+ int (*compar)(void *a, void *b);
+} avl_tree_type;
+
+typedef struct avl_tree_lock {
+ avl_tree_type avl_tree;
+
+#ifndef AVL_WITHOUT_PTHREADS
+#ifdef AVL_LOCK_WITH_MUTEX
+ netdata_mutex_t mutex;
+#else /* AVL_LOCK_WITH_MUTEX */
+ netdata_rwlock_t rwlock;
+#endif /* AVL_LOCK_WITH_MUTEX */
+#endif /* AVL_WITHOUT_PTHREADS */
+} avl_tree_lock;
+
+/* Public methods */
+
+/* Insert element a into the AVL tree t
+ * returns the added element a, or a pointer the
+ * element that is equal to a (as returned by t->compar())
+ * a is linked directly to the tree, so it has to
+ * be properly allocated by the caller.
+ */
+avl *avl_insert_lock(avl_tree_lock *tree, avl *item) NEVERNULL WARNUNUSED;
+avl *avl_insert(avl_tree_type *tree, avl *item) NEVERNULL WARNUNUSED;
+
+/* Remove an element a from the AVL tree t
+ * returns a pointer to the removed element
+ * or NULL if an element equal to a is not found
+ * (equal as returned by t->compar())
+ */
+avl *avl_remove_lock(avl_tree_lock *tree, avl *item) WARNUNUSED;
+avl *avl_remove(avl_tree_type *tree, avl *item) WARNUNUSED;
+
+/* Find the element into the tree that equal to a
+ * (equal as returned by t->compar())
+ * returns NULL is no element is equal to a
+ */
+avl *avl_search_lock(avl_tree_lock *tree, avl *item);
+avl *avl_search(avl_tree_type *tree, avl *item);
+
+/* Initialize the avl_tree_lock
+ */
+void avl_init_lock(avl_tree_lock *tree, int (*compar)(void *a, void *b));
+void avl_init(avl_tree_type *tree, int (*compar)(void *a, void *b));
+
+/* Destroy the avl_tree_lock locks
+ */
+void avl_destroy_lock(avl_tree_lock *tree);
+
+int avl_traverse_lock(avl_tree_lock *tree, int (*callback)(void *entry, void *data), void *data);
+int avl_traverse(avl_tree_type *tree, int (*callback)(void *entry, void *data), void *data);
+
+#endif /* avl.h */
diff --git a/libnetdata/buffer/Makefile.am b/libnetdata/buffer/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/buffer/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/buffer/README.md b/libnetdata/buffer/README.md
new file mode 100644
index 0000000..5a9d7b0
--- /dev/null
+++ b/libnetdata/buffer/README.md
@@ -0,0 +1,17 @@
+<!--
+title: "BUFFER"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/buffer/README.md
+-->
+
+# BUFFER
+
+`BUFFER` is a convenience library for working with strings in `C`.
+Mainly, `BUFFER`s eliminate the need for tracking the string length, thus providing
+a safe alternative for string operations.
+
+Also, they are super fast in printing and appending data to the string and its `buffer_strlen()`
+is just a lookup (it does not traverse the string).
+
+Netdata uses `BUFFER`s for preparing web responses and buffering data to be sent upstream or
+to backend databases.
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fbuffer%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c
new file mode 100644
index 0000000..c37da21
--- /dev/null
+++ b/libnetdata/buffer/buffer.c
@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+#define BUFFER_OVERFLOW_EOF "EOF"
+
+static inline void buffer_overflow_init(BUFFER *b)
+{
+ b->buffer[b->size] = '\0';
+ strcpy(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF);
+}
+
+#ifdef NETDATA_INTERNAL_CHECKS
+#define buffer_overflow_check(b) _buffer_overflow_check(b, __FILE__, __FUNCTION__, __LINE__)
+#else
+#define buffer_overflow_check(b)
+#endif
+
+static inline void _buffer_overflow_check(BUFFER *b, const char *file, const char *function, const unsigned long line)
+{
+ if(b->len > b->size) {
+ error("BUFFER: length %zu is above size %zu, at line %lu, at function %s() of file '%s'.", b->len, b->size, line, function, file);
+ b->len = b->size;
+ }
+
+ if(b->buffer[b->size] != '\0' || strcmp(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF) != 0) {
+ error("BUFFER: detected overflow at line %lu, at function %s() of file '%s'.", line, function, file);
+ buffer_overflow_init(b);
+ }
+}
+
+
+void buffer_reset(BUFFER *wb)
+{
+ buffer_flush(wb);
+
+ wb->contenttype = CT_TEXT_PLAIN;
+ wb->options = 0;
+ wb->date = 0;
+ wb->expires = 0;
+
+ buffer_overflow_check(wb);
+}
+
+const char *buffer_tostring(BUFFER *wb)
+{
+ buffer_need_bytes(wb, 1);
+ wb->buffer[wb->len] = '\0';
+
+ buffer_overflow_check(wb);
+
+ return(wb->buffer);
+}
+
+void buffer_char_replace(BUFFER *wb, char from, char to)
+{
+ char *s = wb->buffer, *end = &wb->buffer[wb->len];
+
+ while(s != end) {
+ if(*s == from) *s = to;
+ s++;
+ }
+
+ buffer_overflow_check(wb);
+}
+
+// This trick seems to give an 80% speed increase in 32bit systems
+// print_calculated_number_llu_r() will just print the digits up to the
+// point the remaining value fits in 32 bits, and then calls
+// print_calculated_number_lu_r() to print the rest with 32 bit arithmetic.
+
+inline char *print_number_lu_r(char *str, unsigned long uvalue) {
+ char *wstr = str;
+
+ // print each digit
+ do *wstr++ = (char)('0' + (uvalue % 10)); while(uvalue /= 10);
+ return wstr;
+}
+
+inline char *print_number_llu_r(char *str, unsigned long long uvalue) {
+ char *wstr = str;
+
+ // print each digit
+ do *wstr++ = (char)('0' + (uvalue % 10)); while((uvalue /= 10) && uvalue > (unsigned long long)0xffffffff);
+ if(uvalue) return print_number_lu_r(wstr, uvalue);
+ return wstr;
+}
+
+inline char *print_number_llu_r_smart(char *str, unsigned long long uvalue) {
+#ifdef ENVIRONMENT32
+ if(uvalue > (unsigned long long)0xffffffff)
+ str = print_number_llu_r(str, uvalue);
+ else
+ str = print_number_lu_r(str, uvalue);
+#else
+ do *str++ = (char)('0' + (uvalue % 10)); while(uvalue /= 10);
+#endif
+
+ return str;
+}
+
+void buffer_print_llu(BUFFER *wb, unsigned long long uvalue)
+{
+ buffer_need_bytes(wb, 50);
+
+ char *str = &wb->buffer[wb->len];
+ char *wstr = str;
+
+#ifdef ENVIRONMENT32
+ if(uvalue > (unsigned long long)0xffffffff)
+ wstr = print_number_llu_r(wstr, uvalue);
+ else
+ wstr = print_number_lu_r(wstr, uvalue);
+#else
+ do *wstr++ = (char)('0' + (uvalue % 10)); while(uvalue /= 10);
+#endif
+
+ // terminate it
+ *wstr = '\0';
+
+ // reverse it
+ char *begin = str, *end = wstr - 1, aux;
+ while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux;
+
+ // return the buffer length
+ wb->len += wstr - str;
+}
+
+void buffer_strcat(BUFFER *wb, const char *txt)
+{
+ // buffer_sprintf(wb, "%s", txt);
+
+ if(unlikely(!txt || !*txt)) return;
+
+ buffer_need_bytes(wb, 1);
+
+ char *s = &wb->buffer[wb->len], *start, *end = &wb->buffer[wb->size];
+ size_t len = wb->len;
+
+ start = s;
+ while(*txt && s != end)
+ *s++ = *txt++;
+
+ len += s - start;
+
+ wb->len = len;
+ buffer_overflow_check(wb);
+
+ if(*txt) {
+ debug(D_WEB_BUFFER, "strcat(): increasing web_buffer at position %zu, size = %zu\n", wb->len, wb->size);
+ len = strlen(txt);
+ buffer_increase(wb, len);
+ buffer_strcat(wb, txt);
+ }
+ else {
+ // terminate the string
+ // without increasing the length
+ buffer_need_bytes(wb, (size_t)1);
+ wb->buffer[wb->len] = '\0';
+ }
+}
+
+void buffer_strcat_jsonescape(BUFFER *wb, const char *txt)
+{
+ while(*txt) {
+ switch(*txt) {
+ case '\\':
+ buffer_need_bytes(wb, 2);
+ wb->buffer[wb->len++] = '\\';
+ wb->buffer[wb->len++] = '\\';
+ break;
+ case '"':
+ buffer_need_bytes(wb, 2);
+ wb->buffer[wb->len++] = '\\';
+ wb->buffer[wb->len++] = '"';
+ break;
+ default: {
+ buffer_need_bytes(wb, 1);
+ wb->buffer[wb->len++] = *txt;
+ }
+ }
+ txt++;
+ }
+
+ buffer_overflow_check(wb);
+}
+
+void buffer_strcat_htmlescape(BUFFER *wb, const char *txt)
+{
+ while(*txt) {
+ switch(*txt) {
+ case '&': buffer_strcat(wb, "&amp;"); break;
+ case '<': buffer_strcat(wb, "&lt;"); break;
+ case '>': buffer_strcat(wb, "&gt;"); break;
+ case '"': buffer_strcat(wb, "&quot;"); break;
+ case '/': buffer_strcat(wb, "&#x2F;"); break;
+ case '\'': buffer_strcat(wb, "&#x27;"); break;
+ default: {
+ buffer_need_bytes(wb, 1);
+ wb->buffer[wb->len++] = *txt;
+ }
+ }
+ txt++;
+ }
+
+ buffer_overflow_check(wb);
+}
+
+void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...)
+{
+ if(unlikely(!fmt || !*fmt)) return;
+
+ buffer_need_bytes(wb, len + 1);
+
+ va_list args;
+ va_start(args, fmt);
+ wb->len += vsnprintfz(&wb->buffer[wb->len], len, fmt, args);
+ va_end(args);
+
+ buffer_overflow_check(wb);
+
+ // the buffer is \0 terminated by vsnprintfz
+}
+
+void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args)
+{
+ if(unlikely(!fmt || !*fmt)) return;
+
+ buffer_need_bytes(wb, 2);
+
+ size_t len = wb->size - wb->len - 1;
+
+ wb->len += vsnprintfz(&wb->buffer[wb->len], len, fmt, args);
+
+ buffer_overflow_check(wb);
+
+ // the buffer is \0 terminated by vsnprintfz
+}
+
+void buffer_sprintf(BUFFER *wb, const char *fmt, ...)
+{
+ if(unlikely(!fmt || !*fmt)) return;
+
+ va_list args;
+ size_t wrote = 0, need = 2, multiplier = 0, len;
+
+ do {
+ need += wrote + multiplier * WEB_DATA_LENGTH_INCREASE_STEP;
+ multiplier++;
+
+ debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote);
+ buffer_need_bytes(wb, need);
+
+ len = wb->size - wb->len - 1;
+
+ va_start(args, fmt);
+ wrote = (size_t) vsnprintfz(&wb->buffer[wb->len], len, fmt, args);
+ va_end(args);
+
+ } while(wrote >= len);
+
+ wb->len += wrote;
+
+ // the buffer is \0 terminated by vsnprintf
+}
+
+
+void buffer_rrd_value(BUFFER *wb, calculated_number value)
+{
+ buffer_need_bytes(wb, 50);
+
+ if(isnan(value) || isinf(value)) {
+ buffer_strcat(wb, "null");
+ return;
+ }
+ else
+ wb->len += print_calculated_number(&wb->buffer[wb->len], value);
+
+ // terminate it
+ buffer_need_bytes(wb, 1);
+ wb->buffer[wb->len] = '\0';
+
+ buffer_overflow_check(wb);
+}
+
+// generate a javascript date, the fastest possible way...
+void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds)
+{
+ // 10 20 30 = 35
+ // 01234567890123456789012345678901234
+ // Date(2014,04,01,03,28,20)
+
+ buffer_need_bytes(wb, 30);
+
+ char *b = &wb->buffer[wb->len], *p;
+ unsigned int *q = (unsigned int *)b;
+
+ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ *q++ = 0x65746144; // "Date" backwards.
+ #else
+ *q++ = 0x44617465; // "Date"
+ #endif
+ p = (char *)q;
+
+ *p++ = '(';
+ *p++ = '0' + year / 1000; year %= 1000;
+ *p++ = '0' + year / 100; year %= 100;
+ *p++ = '0' + year / 10;
+ *p++ = '0' + year % 10;
+ *p++ = ',';
+ *p = '0' + month / 10; if (*p != '0') p++;
+ *p++ = '0' + month % 10;
+ *p++ = ',';
+ *p = '0' + day / 10; if (*p != '0') p++;
+ *p++ = '0' + day % 10;
+ *p++ = ',';
+ *p = '0' + hours / 10; if (*p != '0') p++;
+ *p++ = '0' + hours % 10;
+ *p++ = ',';
+ *p = '0' + minutes / 10; if (*p != '0') p++;
+ *p++ = '0' + minutes % 10;
+ *p++ = ',';
+ *p = '0' + seconds / 10; if (*p != '0') p++;
+ *p++ = '0' + seconds % 10;
+
+ unsigned short *r = (unsigned short *)p;
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ *r++ = 0x0029; // ")\0" backwards.
+ #else
+ *r++ = 0x2900; // ")\0"
+ #endif
+
+ wb->len += (size_t)((char *)r - b - 1);
+
+ // terminate it
+ wb->buffer[wb->len] = '\0';
+ buffer_overflow_check(wb);
+}
+
+// generate a date, the fastest possible way...
+void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds)
+{
+ // 10 20 30 = 35
+ // 01234567890123456789012345678901234
+ // 2014-04-01 03:28:20
+
+ buffer_need_bytes(wb, 36);
+
+ char *b = &wb->buffer[wb->len];
+ char *p = b;
+
+ *p++ = '0' + year / 1000; year %= 1000;
+ *p++ = '0' + year / 100; year %= 100;
+ *p++ = '0' + year / 10;
+ *p++ = '0' + year % 10;
+ *p++ = '-';
+ *p++ = '0' + month / 10;
+ *p++ = '0' + month % 10;
+ *p++ = '-';
+ *p++ = '0' + day / 10;
+ *p++ = '0' + day % 10;
+ *p++ = ' ';
+ *p++ = '0' + hours / 10;
+ *p++ = '0' + hours % 10;
+ *p++ = ':';
+ *p++ = '0' + minutes / 10;
+ *p++ = '0' + minutes % 10;
+ *p++ = ':';
+ *p++ = '0' + seconds / 10;
+ *p++ = '0' + seconds % 10;
+ *p = '\0';
+
+ wb->len += (size_t)(p - b);
+
+ // terminate it
+ wb->buffer[wb->len] = '\0';
+ buffer_overflow_check(wb);
+}
+
+BUFFER *buffer_create(size_t size)
+{
+ BUFFER *b;
+
+ debug(D_WEB_BUFFER, "Creating new web buffer of size %zu.", size);
+
+ b = callocz(1, sizeof(BUFFER));
+ b->buffer = mallocz(size + sizeof(BUFFER_OVERFLOW_EOF) + 2);
+ b->buffer[0] = '\0';
+ b->size = size;
+ b->contenttype = CT_TEXT_PLAIN;
+ buffer_overflow_init(b);
+ buffer_overflow_check(b);
+
+ return(b);
+}
+
+void buffer_free(BUFFER *b) {
+ if(unlikely(!b)) return;
+
+ buffer_overflow_check(b);
+
+ debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size);
+
+ freez(b->buffer);
+ freez(b);
+}
+
+void buffer_increase(BUFFER *b, size_t free_size_required) {
+ buffer_overflow_check(b);
+
+ size_t left = b->size - b->len;
+
+ if(left >= free_size_required) return;
+
+ size_t increase = free_size_required - left;
+ if(increase < WEB_DATA_LENGTH_INCREASE_STEP) increase = WEB_DATA_LENGTH_INCREASE_STEP;
+
+ debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + increase);
+
+ b->buffer = reallocz(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2);
+ b->size += increase;
+
+ buffer_overflow_init(b);
+ buffer_overflow_check(b);
+}
diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
new file mode 100644
index 0000000..d50910c
--- /dev/null
+++ b/libnetdata/buffer/buffer.h
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_WEB_BUFFER_H
+#define NETDATA_WEB_BUFFER_H 1
+
+#include "../libnetdata.h"
+
+#define WEB_DATA_LENGTH_INCREASE_STEP 1024
+
+typedef struct web_buffer {
+ size_t size; // allocation size of buffer, in bytes
+ size_t len; // current data length in buffer, in bytes
+ char *buffer; // the buffer itself
+ uint8_t contenttype; // the content type of the data in the buffer
+ uint8_t options; // options related to the content
+ time_t date; // the timestamp this content has been generated
+ time_t expires; // the timestamp this content expires
+} BUFFER;
+
+// options
+#define WB_CONTENT_CACHEABLE 1
+#define WB_CONTENT_NO_CACHEABLE 2
+
+// content-types
+#define CT_APPLICATION_JSON 1
+#define CT_TEXT_PLAIN 2
+#define CT_TEXT_HTML 3
+#define CT_APPLICATION_X_JAVASCRIPT 4
+#define CT_TEXT_CSS 5
+#define CT_TEXT_XML 6
+#define CT_APPLICATION_XML 7
+#define CT_TEXT_XSL 8
+#define CT_APPLICATION_OCTET_STREAM 9
+#define CT_APPLICATION_X_FONT_TRUETYPE 10
+#define CT_APPLICATION_X_FONT_OPENTYPE 11
+#define CT_APPLICATION_FONT_WOFF 12
+#define CT_APPLICATION_FONT_WOFF2 13
+#define CT_APPLICATION_VND_MS_FONTOBJ 14
+#define CT_IMAGE_SVG_XML 15
+#define CT_IMAGE_PNG 16
+#define CT_IMAGE_JPG 17
+#define CT_IMAGE_GIF 18
+#define CT_IMAGE_XICON 19
+#define CT_IMAGE_ICNS 20
+#define CT_IMAGE_BMP 21
+#define CT_PROMETHEUS 22
+
+#define buffer_cacheable(wb) do { (wb)->options |= WB_CONTENT_CACHEABLE; if((wb)->options & WB_CONTENT_NO_CACHEABLE) (wb)->options &= ~WB_CONTENT_NO_CACHEABLE; } while(0)
+#define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE) (wb)->options &= ~WB_CONTENT_CACHEABLE; (wb)->expires = 0; } while(0)
+
+#define buffer_strlen(wb) ((wb)->len)
+extern const char *buffer_tostring(BUFFER *wb);
+
+#define buffer_flush(wb) wb->buffer[(wb)->len = 0] = '\0'
+extern void buffer_reset(BUFFER *wb);
+
+extern void buffer_strcat(BUFFER *wb, const char *txt);
+extern void buffer_rrd_value(BUFFER *wb, calculated_number value);
+
+extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
+extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
+
+extern BUFFER *buffer_create(size_t size);
+extern void buffer_free(BUFFER *b);
+extern void buffer_increase(BUFFER *b, size_t free_size_required);
+
+extern void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) PRINTFLIKE(3, 4);
+extern void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args);
+extern void buffer_sprintf(BUFFER *wb, const char *fmt, ...) PRINTFLIKE(2,3);
+extern void buffer_strcat_jsonescape(BUFFER *wb, const char *txt);
+extern void buffer_strcat_htmlescape(BUFFER *wb, const char *txt);
+
+extern void buffer_char_replace(BUFFER *wb, char from, char to);
+
+extern char *print_number_lu_r(char *str, unsigned long uvalue);
+extern char *print_number_llu_r(char *str, unsigned long long uvalue);
+extern char *print_number_llu_r_smart(char *str, unsigned long long uvalue);
+
+extern void buffer_print_llu(BUFFER *wb, unsigned long long uvalue);
+
+static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
+ if(unlikely(buffer->size - buffer->len < needed_free_size))
+ buffer_increase(buffer, needed_free_size);
+}
+
+#endif /* NETDATA_WEB_BUFFER_H */
diff --git a/libnetdata/circular_buffer/Makefile.am b/libnetdata/circular_buffer/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/circular_buffer/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/circular_buffer/README.md b/libnetdata/circular_buffer/README.md
new file mode 100644
index 0000000..d36c05f
--- /dev/null
+++ b/libnetdata/circular_buffer/README.md
@@ -0,0 +1,12 @@
+<!--
+title: "circular_buffer"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/circular_buffer/README.md
+-->
+
+# Circular Buffer
+
+`struct circular_buffer` is an adaptive circular buffer. It will start at an initial size
+and grow up to a maximum size as it fills. Two indices within the structure track the current
+`read` and `write` position for data.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fcircular_buffer%2README&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]()
diff --git a/libnetdata/circular_buffer/circular_buffer.c b/libnetdata/circular_buffer/circular_buffer.c
new file mode 100644
index 0000000..998008d
--- /dev/null
+++ b/libnetdata/circular_buffer/circular_buffer.c
@@ -0,0 +1,85 @@
+#include "../libnetdata.h"
+
+struct circular_buffer *cbuffer_new(size_t initial, size_t max) {
+ struct circular_buffer *result = mallocz(sizeof(*result));
+ result->size = initial;
+ result->data = mallocz(initial);
+ result->write = 0;
+ result->read = 0;
+ result->max_size = max;
+ return result;
+}
+
+void cbuffer_free(struct circular_buffer *buf) {
+ freez(buf->data);
+ freez(buf);
+}
+
+static int cbuffer_realloc_unsafe(struct circular_buffer *buf) {
+ // Check that we can grow
+ if (buf->size >= buf->max_size)
+ return 1;
+ size_t new_size = buf->size * 2;
+ if (new_size > buf->max_size)
+ new_size = buf->max_size;
+
+ // We know that: size < new_size <= max_size
+ // For simplicity align the current data at the bottom of the new buffer
+ char *new_data = mallocz(new_size);
+ if (buf->read == buf->write)
+ buf->write = 0; // buffer is empty
+ else if (buf->read < buf->write) {
+ memcpy(new_data, buf->data + buf->read, buf->write - buf->read);
+ buf->write -= buf->read;
+ } else {
+ size_t top_part = buf->size - buf->read;
+ memcpy(new_data, buf->data + buf->read, top_part);
+ memcpy(new_data + top_part, buf->data, buf->write);
+ buf->write = top_part + buf->write;
+ }
+ buf->read = 0;
+
+ // Switch buffers
+ freez(buf->data);
+ buf->data = new_data;
+ buf->size = new_size;
+ return 0;
+}
+
+int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len) {
+ size_t len = (buf->write >= buf->read) ? (buf->write - buf->read) : (buf->size - buf->read + buf->write);
+ while (d_len + len >= buf->size) {
+ if (cbuffer_realloc_unsafe(buf)) {
+ return 1;
+ }
+ }
+ // Guarantee: write + d_len cannot hit read
+ if (buf->write + d_len < buf->size) {
+ memcpy(buf->data + buf->write, d, d_len);
+ buf->write += d_len;
+ }
+ else {
+ size_t top_part = buf->size - buf->write;
+ memcpy(buf->data + buf->write, d, top_part);
+ memcpy(buf->data, d + top_part, d_len - top_part);
+ buf->write = d_len - top_part;
+ }
+ return 0;
+}
+
+// Assume caller does not remove too many bytes (i.e. read will jump over write)
+void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num) {
+ buf->read += num;
+ // Assume num < size (i.e. caller cannot remove more bytes than are in the buffer)
+ if (buf->read >= buf->size)
+ buf->read -= buf->size;
+}
+
+size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start) {
+ if (start != NULL)
+ *start = buf->data + buf->read;
+ if (buf->read <= buf->write) {
+ return buf->write - buf->read; // Includes empty case
+ }
+ return buf->size - buf->read;
+}
diff --git a/libnetdata/circular_buffer/circular_buffer.h b/libnetdata/circular_buffer/circular_buffer.h
new file mode 100644
index 0000000..ba37e0e
--- /dev/null
+++ b/libnetdata/circular_buffer/circular_buffer.h
@@ -0,0 +1,16 @@
+#ifndef CIRCULAR_BUFFER_H
+#define CIRCULAR_BUFFER_H 1
+
+#include <string.h>
+
+struct circular_buffer {
+ size_t size, write, read, max_size;
+ char *data;
+};
+
+extern struct circular_buffer *cbuffer_new(size_t initial, size_t max);
+extern void cbuffer_free(struct circular_buffer *buf);
+extern int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len);
+extern void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num);
+extern size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start);
+#endif
diff --git a/libnetdata/clocks/Makefile.am b/libnetdata/clocks/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/clocks/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/clocks/README.md b/libnetdata/clocks/README.md
new file mode 100644
index 0000000..e53b86a
--- /dev/null
+++ b/libnetdata/clocks/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/clocks/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fclocks%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/clocks/clocks.c b/libnetdata/clocks/clocks.c
new file mode 100644
index 0000000..4ec5fa9
--- /dev/null
+++ b/libnetdata/clocks/clocks.c
@@ -0,0 +1,300 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+static int clock_boottime_valid = 1;
+static int clock_monotonic_coarse_valid = 1;
+
+#ifndef HAVE_CLOCK_GETTIME
+inline int clock_gettime(clockid_t clk_id, struct timespec *ts) {
+ struct timeval tv;
+ if(unlikely(gettimeofday(&tv, NULL) == -1)) {
+ error("gettimeofday() failed.");
+ return -1;
+ }
+ ts->tv_sec = tv.tv_sec;
+ ts->tv_nsec = (tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC;
+ return 0;
+}
+#endif
+
+void test_clock_boottime(void) {
+ struct timespec ts;
+ if(clock_gettime(CLOCK_BOOTTIME, &ts) == -1 && errno == EINVAL)
+ clock_boottime_valid = 0;
+}
+
+void test_clock_monotonic_coarse(void) {
+ struct timespec ts;
+ if(clock_gettime(CLOCK_MONOTONIC_COARSE, &ts) == -1 && errno == EINVAL)
+ clock_monotonic_coarse_valid = 0;
+}
+
+static inline time_t now_sec(clockid_t clk_id) {
+ struct timespec ts;
+ if(unlikely(clock_gettime(clk_id, &ts) == -1)) {
+ error("clock_gettime(%d, &timespec) failed.", clk_id);
+ return 0;
+ }
+ return ts.tv_sec;
+}
+
+static inline usec_t now_usec(clockid_t clk_id) {
+ struct timespec ts;
+ if(unlikely(clock_gettime(clk_id, &ts) == -1)) {
+ error("clock_gettime(%d, &timespec) failed.", clk_id);
+ return 0;
+ }
+ return (usec_t)ts.tv_sec * USEC_PER_SEC + (ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC;
+}
+
+static inline int now_timeval(clockid_t clk_id, struct timeval *tv) {
+ struct timespec ts;
+
+ if(unlikely(clock_gettime(clk_id, &ts) == -1)) {
+ error("clock_gettime(%d, &timespec) failed.", clk_id);
+ tv->tv_sec = 0;
+ tv->tv_usec = 0;
+ return -1;
+ }
+
+ tv->tv_sec = ts.tv_sec;
+ tv->tv_usec = (suseconds_t)((ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC);
+ return 0;
+}
+
+inline time_t now_realtime_sec(void) {
+ return now_sec(CLOCK_REALTIME);
+}
+
+inline usec_t now_realtime_usec(void) {
+ return now_usec(CLOCK_REALTIME);
+}
+
+inline int now_realtime_timeval(struct timeval *tv) {
+ return now_timeval(CLOCK_REALTIME, tv);
+}
+
+inline time_t now_monotonic_sec(void) {
+ return now_sec(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
+}
+
+inline usec_t now_monotonic_usec(void) {
+ return now_usec(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
+}
+
+inline int now_monotonic_timeval(struct timeval *tv) {
+ return now_timeval(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC, tv);
+}
+
+inline time_t now_monotonic_high_precision_sec(void) {
+ return now_sec(CLOCK_MONOTONIC);
+}
+
+inline usec_t now_monotonic_high_precision_usec(void) {
+ return now_usec(CLOCK_MONOTONIC);
+}
+
+inline int now_monotonic_high_precision_timeval(struct timeval *tv) {
+ return now_timeval(CLOCK_MONOTONIC, tv);
+}
+
+inline time_t now_boottime_sec(void) {
+ return now_sec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME :
+ likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
+}
+
+inline usec_t now_boottime_usec(void) {
+ return now_usec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME :
+ likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
+}
+
+inline int now_boottime_timeval(struct timeval *tv) {
+ return now_timeval(likely(clock_boottime_valid) ? CLOCK_BOOTTIME :
+ likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC,
+ tv);
+}
+
+inline usec_t timeval_usec(struct timeval *tv) {
+ return (usec_t)tv->tv_sec * USEC_PER_SEC + (tv->tv_usec % USEC_PER_SEC);
+}
+
+inline msec_t timeval_msec(struct timeval *tv) {
+ return (msec_t)tv->tv_sec * MSEC_PER_SEC + ((tv->tv_usec % USEC_PER_SEC) / MSEC_PER_SEC);
+}
+
+inline susec_t dt_usec_signed(struct timeval *now, struct timeval *old) {
+ usec_t ts1 = timeval_usec(now);
+ usec_t ts2 = timeval_usec(old);
+
+ if(likely(ts1 >= ts2)) return (susec_t)(ts1 - ts2);
+ return -((susec_t)(ts2 - ts1));
+}
+
+inline usec_t dt_usec(struct timeval *now, struct timeval *old) {
+ usec_t ts1 = timeval_usec(now);
+ usec_t ts2 = timeval_usec(old);
+ return (ts1 > ts2) ? (ts1 - ts2) : (ts2 - ts1);
+}
+
+inline void heartbeat_init(heartbeat_t *hb)
+{
+ hb->monotonic = hb->realtime = 0ULL;
+}
+
+// waits for the next heartbeat
+// it waits using the monotonic clock
+// it returns the dt using the realtime clock
+
+usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) {
+ heartbeat_t now;
+ now.monotonic = now_monotonic_usec();
+ now.realtime = now_realtime_usec();
+
+ usec_t next_monotonic = now.monotonic - (now.monotonic % tick) + tick;
+
+ while(now.monotonic < next_monotonic) {
+ sleep_usec(next_monotonic - now.monotonic);
+ now.monotonic = now_monotonic_usec();
+ now.realtime = now_realtime_usec();
+ }
+
+ if(likely(hb->realtime != 0ULL)) {
+ usec_t dt_monotonic = now.monotonic - hb->monotonic;
+ usec_t dt_realtime = now.realtime - hb->realtime;
+
+ hb->monotonic = now.monotonic;
+ hb->realtime = now.realtime;
+
+ if(unlikely(dt_monotonic >= tick + tick / 2)) {
+ errno = 0;
+ error("heartbeat missed %llu monotonic microseconds", dt_monotonic - tick);
+ }
+
+ return dt_realtime;
+ }
+ else {
+ hb->monotonic = now.monotonic;
+ hb->realtime = now.realtime;
+ return 0ULL;
+ }
+}
+
+// returned the elapsed time, since the last heartbeat
+// using the monotonic clock
+
+inline usec_t heartbeat_monotonic_dt_to_now_usec(heartbeat_t *hb) {
+ if(!hb || !hb->monotonic) return 0ULL;
+ return now_monotonic_usec() - hb->monotonic;
+}
+
+int sleep_usec(usec_t usec) {
+
+#ifndef NETDATA_WITH_USLEEP
+ // we expect microseconds (1.000.000 per second)
+ // but timespec is nanoseconds (1.000.000.000 per second)
+ struct timespec rem, req = {
+ .tv_sec = (time_t) (usec / 1000000),
+ .tv_nsec = (suseconds_t) ((usec % 1000000) * 1000)
+ };
+
+ while (nanosleep(&req, &rem) == -1) {
+ if (likely(errno == EINTR)) {
+ debug(D_SYSTEM, "nanosleep() interrupted (while sleeping for %llu microseconds).", usec);
+ req.tv_sec = rem.tv_sec;
+ req.tv_nsec = rem.tv_nsec;
+ } else {
+ error("Cannot nanosleep() for %llu microseconds.", usec);
+ break;
+ }
+ }
+
+ return 0;
+#else
+ int ret = usleep(usec);
+ if(unlikely(ret == -1 && errno == EINVAL)) {
+ // on certain systems, usec has to be up to 999999
+ if(usec > 999999) {
+ int counter = usec / 999999;
+ while(counter--)
+ usleep(999999);
+
+ usleep(usec % 999999);
+ }
+ else {
+ error("Cannot usleep() for %llu microseconds.", usec);
+ return ret;
+ }
+ }
+
+ if(ret != 0)
+ error("usleep() failed for %llu microseconds.", usec);
+
+ return ret;
+#endif
+}
+
+static inline collected_number uptime_from_boottime(void) {
+#ifdef CLOCK_BOOTTIME_IS_AVAILABLE
+ return now_boottime_usec() / 1000;
+#else
+ error("uptime cannot be read from CLOCK_BOOTTIME on this system.");
+ return 0;
+#endif
+}
+
+static procfile *read_proc_uptime_ff = NULL;
+static inline collected_number read_proc_uptime(char *filename) {
+ if(unlikely(!read_proc_uptime_ff)) {
+ read_proc_uptime_ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
+ if(unlikely(!read_proc_uptime_ff)) return 0;
+ }
+
+ read_proc_uptime_ff = procfile_readall(read_proc_uptime_ff);
+ if(unlikely(!read_proc_uptime_ff)) return 0;
+
+ if(unlikely(procfile_lines(read_proc_uptime_ff) < 1)) {
+ error("/proc/uptime has no lines.");
+ return 0;
+ }
+ if(unlikely(procfile_linewords(read_proc_uptime_ff, 0) < 1)) {
+ error("/proc/uptime has less than 1 word in it.");
+ return 0;
+ }
+
+ return (collected_number)(strtold(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0);
+}
+
+inline collected_number uptime_msec(char *filename){
+ static int use_boottime = -1;
+
+ if(unlikely(use_boottime == -1)) {
+ collected_number uptime_boottime = uptime_from_boottime();
+ collected_number uptime_proc = read_proc_uptime(filename);
+
+ long long delta = (long long)uptime_boottime - (long long)uptime_proc;
+ if(delta < 0) delta = -delta;
+
+ if(delta <= 1000 && uptime_boottime != 0) {
+ procfile_close(read_proc_uptime_ff);
+ info("Using now_boottime_usec() for uptime (dt is %lld ms)", delta);
+ use_boottime = 1;
+ }
+ else if(uptime_proc != 0) {
+ info("Using /proc/uptime for uptime (dt is %lld ms)", delta);
+ use_boottime = 0;
+ }
+ else {
+ error("Cannot find any way to read uptime on this system.");
+ return 1;
+ }
+ }
+
+ collected_number uptime;
+ if(use_boottime)
+ uptime = uptime_from_boottime();
+ else
+ uptime = read_proc_uptime(filename);
+
+ return uptime;
+}
diff --git a/libnetdata/clocks/clocks.h b/libnetdata/clocks/clocks.h
new file mode 100644
index 0000000..cfe99f5
--- /dev/null
+++ b/libnetdata/clocks/clocks.h
@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_CLOCKS_H
+#define NETDATA_CLOCKS_H 1
+
+#include "../libnetdata.h"
+
+#ifndef HAVE_STRUCT_TIMESPEC
+struct timespec {
+ time_t tv_sec; /* seconds */
+ long tv_nsec; /* nanoseconds */
+};
+#endif
+
+#ifndef HAVE_CLOCKID_T
+typedef int clockid_t;
+#endif
+
+typedef unsigned long long nsec_t;
+typedef unsigned long long msec_t;
+typedef unsigned long long usec_t;
+typedef long long susec_t;
+
+typedef struct heartbeat {
+ usec_t monotonic;
+ usec_t realtime;
+} heartbeat_t;
+
+/* Linux value is as good as any other */
+#ifndef CLOCK_REALTIME
+#define CLOCK_REALTIME 0
+#endif
+
+#ifndef CLOCK_MONOTONIC
+/* fallback to CLOCK_REALTIME if not available */
+#define CLOCK_MONOTONIC CLOCK_REALTIME
+#endif
+
+/* Prefer CLOCK_MONOTONIC_COARSE where available to reduce overhead. It has the same semantics as CLOCK_MONOTONIC */
+#ifndef CLOCK_MONOTONIC_COARSE
+/* fallback to CLOCK_MONOTONIC if not available */
+#define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
+#endif
+
+#ifndef CLOCK_BOOTTIME
+
+#ifdef CLOCK_UPTIME
+/* CLOCK_BOOTTIME falls back to CLOCK_UPTIME on FreeBSD */
+#define CLOCK_BOOTTIME CLOCK_UPTIME
+#else // CLOCK_UPTIME
+/* CLOCK_BOOTTIME falls back to CLOCK_MONOTONIC */
+#define CLOCK_BOOTTIME CLOCK_MONOTONIC_COARSE
+#endif // CLOCK_UPTIME
+
+#else // CLOCK_BOOTTIME
+
+#ifdef HAVE_CLOCK_GETTIME
+#define CLOCK_BOOTTIME_IS_AVAILABLE 1 // required for /proc/uptime
+#endif // HAVE_CLOCK_GETTIME
+
+#endif // CLOCK_BOOTTIME
+
+#ifndef NSEC_PER_MSEC
+#define NSEC_PER_MSEC 1000000ULL
+#endif
+
+#ifndef NSEC_PER_SEC
+#define NSEC_PER_SEC 1000000000ULL
+#endif
+#ifndef NSEC_PER_USEC
+#define NSEC_PER_USEC 1000ULL
+#endif
+
+#ifndef USEC_PER_SEC
+#define USEC_PER_SEC 1000000ULL
+#endif
+#ifndef MSEC_PER_SEC
+#define MSEC_PER_SEC 1000ULL
+#endif
+
+#define USEC_PER_MS 1000ULL
+
+#ifndef HAVE_CLOCK_GETTIME
+/* Fallback function for POSIX.1-2001 clock_gettime() function.
+ *
+ * We use a realtime clock from gettimeofday(), this will
+ * make systems without clock_gettime() support sensitive
+ * to time jumps or hibernation/suspend side effects.
+ */
+extern int clock_gettime(clockid_t clk_id, struct timespec *ts);
+#endif
+
+/*
+ * Three clocks are available (cf. man 3 clock_gettime):
+ *
+ * REALTIME clock (i.e. wall-clock):
+ * This clock is affected by discontinuous jumps in the system time
+ * (e.g., if the system administrator manually changes the clock), and by the incremental adjustments performed by adjtime(3) and NTP.
+ *
+ * MONOTONIC clock
+ * Clock that cannot be set and represents monotonic time since some unspecified starting point.
+ * This clock is not affected by discontinuous jumps in the system time
+ * (e.g., if the system administrator manually changes the clock), but is affected by the incremental adjustments performed by adjtime(3) and NTP.
+ * If not available on the system, this clock falls back to REALTIME clock.
+ *
+ * BOOTTIME clock
+ * Identical to CLOCK_MONOTONIC, except it also includes any time that the system is suspended.
+ * This allows applications to get a suspend-aware monotonic clock without having to deal with the complications of CLOCK_REALTIME,
+ * which may have discontinuities if the time is changed using settimeofday(2).
+ * If not available on the system, this clock falls back to MONOTONIC clock.
+ *
+ * All now_*_timeval() functions fill the `struct timeval` with the time from the appropriate clock.
+ * Those functions return 0 on success, -1 else with errno set appropriately.
+ *
+ * All now_*_sec() functions return the time in seconds from the approriate clock, or 0 on error.
+ * All now_*_usec() functions return the time in microseconds from the approriate clock, or 0 on error.
+ *
+ * Most functions will attempt to use CLOCK_MONOTONIC_COARSE if available to reduce contention overhead and improve
+ * performance scaling. If high precision is required please use one of the available now_*_high_precision_* functions.
+ */
+extern int now_realtime_timeval(struct timeval *tv);
+extern time_t now_realtime_sec(void);
+extern usec_t now_realtime_usec(void);
+
+extern int now_monotonic_timeval(struct timeval *tv);
+extern time_t now_monotonic_sec(void);
+extern usec_t now_monotonic_usec(void);
+extern int now_monotonic_high_precision_timeval(struct timeval *tv);
+extern time_t now_monotonic_high_precision_sec(void);
+extern usec_t now_monotonic_high_precision_usec(void);
+
+extern int now_boottime_timeval(struct timeval *tv);
+extern time_t now_boottime_sec(void);
+extern usec_t now_boottime_usec(void);
+
+
+extern usec_t timeval_usec(struct timeval *tv);
+extern msec_t timeval_msec(struct timeval *tv);
+
+extern usec_t dt_usec(struct timeval *now, struct timeval *old);
+extern susec_t dt_usec_signed(struct timeval *now, struct timeval *old);
+
+extern void heartbeat_init(heartbeat_t *hb);
+
+/* Sleeps until next multiple of tick using monotonic clock.
+ * Returns elapsed time in microseconds since previous heartbeat
+ */
+extern usec_t heartbeat_next(heartbeat_t *hb, usec_t tick);
+
+/* Returns elapsed time in microseconds since last heartbeat */
+extern usec_t heartbeat_monotonic_dt_to_now_usec(heartbeat_t *hb);
+
+extern int sleep_usec(usec_t usec);
+
+/*
+ * When running a binary with CLOCK_BOOTTIME defined on a system with a linux kernel older than Linux 2.6.39 the
+ * clock_gettime(2) system call fails with EINVAL. In that case it must fall-back to CLOCK_MONOTONIC.
+ */
+void test_clock_boottime(void);
+
+/*
+ * When running a binary with CLOCK_MONOTONIC_COARSE defined on a system with a linux kernel older than Linux 2.6.32 the
+ * clock_gettime(2) system call fails with EINVAL. In that case it must fall-back to CLOCK_MONOTONIC.
+ */
+void test_clock_monotonic_coarse(void);
+
+extern collected_number uptime_msec(char *filename);
+
+#endif /* NETDATA_CLOCKS_H */
diff --git a/libnetdata/config/Makefile.am b/libnetdata/config/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/config/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/config/README.md b/libnetdata/config/README.md
new file mode 100644
index 0000000..a71f1ee
--- /dev/null
+++ b/libnetdata/config/README.md
@@ -0,0 +1,54 @@
+<!--
+title: "Netdata ini config files"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/config/README.md
+-->
+
+# Netdata ini config files
+
+Configuration files `netdata.conf` and `stream.conf` are Netdata ini files.
+
+## Motivation
+
+The whole idea came up when we were evaluating the documentation involved
+in maintaining a complex configuration system. Our intention was to give
+configuration options for everything imaginable. But then, documenting all
+these options would require a tremendous amount of time, users would have
+to search through endless pages for the option they need, etc.
+
+We concluded then that **configuring software like that is a waste of time
+and effort**. Of course there must be plenty of configuration options, but
+the implementation itself should require a lot less effort for both the
+developers and the users.
+
+So, we did this:
+
+1. No configuration is required to run Netdata
+2. There are plenty of options to tweak
+3. There is minimal documentation (or no at all)
+
+## Why this works?
+
+The configuration file is a `name = value` dictionary with `[sections]`.
+Write whatever you like there as long as it follows this simple format.
+
+Netdata loads this dictionary and then when the code needs a value from
+it, it just looks up the `name` in the dictionary at the proper `section`.
+In all places, in the code, there are both the `names` and their
+`default values`, so if something is not found in the configuration
+file, the default is used. The lookup is made using B-Trees and hashes
+(no string comparisons), so they are super fast. Also the `names` of the
+settings can be `my super duper setting that once set to yes, will turn the world upside down = no`
+
+- so goodbye to most of the documentation involved.
+
+Next, Netdata can generate a valid configuration for the user to edit.
+No need to remember anything or copy and paste settings. Just get the
+configuration from the server (`/netdata.conf` on your Netdata server),
+edit it and save it.
+
+Last, what about options you believe you have set, but you misspelled?
+When you get the configuration file from the server, there will be a
+comment above all `name = value` pairs the server does not use.
+So you know that whatever you wrote there, is not used.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fconfig%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/config/appconfig.c b/libnetdata/config/appconfig.c
new file mode 100644
index 0000000..d9dcde7
--- /dev/null
+++ b/libnetdata/config/appconfig.c
@@ -0,0 +1,883 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+/*
+ * @Input:
+ * Connector / instance to add to an internal structure
+ * @Return
+ * The current head of the linked list of connector_instance
+ *
+ */
+
+_CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance)
+{
+ static struct _connector_instance *global_connector_instance = NULL;
+ struct _connector_instance *local_ci, *local_ci_tmp;
+
+ if (unlikely(!connector)) {
+ if (unlikely(!instance))
+ return global_connector_instance;
+
+ local_ci = global_connector_instance;
+ while (local_ci) {
+ local_ci_tmp = local_ci->next;
+ freez(local_ci);
+ local_ci = local_ci_tmp;
+ }
+ global_connector_instance = NULL;
+ return NULL;
+ }
+
+ local_ci = callocz(1, sizeof(struct _connector_instance));
+ local_ci->instance = instance;
+ local_ci->connector = connector;
+ strncpy(local_ci->instance_name, instance->name, CONFIG_MAX_NAME);
+ strncpy(local_ci->connector_name, connector->name, CONFIG_MAX_NAME);
+ local_ci->next = global_connector_instance;
+ global_connector_instance = local_ci;
+
+ return global_connector_instance;
+}
+
+int is_valid_connector(char *type, int check_reserved)
+{
+ int rc = 1;
+
+ if (unlikely(!type))
+ return 0;
+
+ if (!check_reserved) {
+ if (unlikely(is_valid_connector(type,1))) {
+ return 0;
+ }
+ //if (unlikely(*type == ':')
+ // return 0;
+ char *separator = strrchr(type, ':');
+ if (likely(separator)) {
+ *separator = '\0';
+ rc = separator - type;
+ } else
+ return 0;
+ }
+// else {
+// if (unlikely(is_valid_connector(type,1))) {
+// error("Section %s invalid -- reserved name", type);
+// return 0;
+// }
+// }
+
+ if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
+ return rc;
+ } else if (!strcmp(type, "graphite:http") || !strcmp(type, "graphite:https")) {
+ return rc;
+ } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
+ return rc;
+ } else if (!strcmp(type, "json:http") || !strcmp(type, "json:https")) {
+ return rc;
+ } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
+ return rc;
+ } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
+ return rc;
+ } else if (!strcmp(type, "prometheus_remote_write")) {
+ return rc;
+ } else if (!strcmp(type, "prometheus_remote_write:http") || !strcmp(type, "prometheus_remote_write:https")) {
+ return rc;
+ } else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
+ return rc;
+ } else if (!strcmp(type, "pubsub") || !strcmp(type, "pubsub:plaintext")) {
+ return rc;
+ } else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) {
+ return rc;
+ }
+
+ return 0;
+}
+
+// ----------------------------------------------------------------------------
+// locking
+
+inline void appconfig_wrlock(struct config *root) {
+ netdata_mutex_lock(&root->mutex);
+}
+
+inline void appconfig_unlock(struct config *root) {
+ netdata_mutex_unlock(&root->mutex);
+}
+
+inline void config_section_wrlock(struct section *co) {
+ netdata_mutex_lock(&co->mutex);
+}
+
+inline void config_section_unlock(struct section *co) {
+ netdata_mutex_unlock(&co->mutex);
+}
+
+
+// ----------------------------------------------------------------------------
+// config name-value index
+
+static int appconfig_option_compare(void *a, void *b) {
+ if(((struct config_option *)a)->hash < ((struct config_option *)b)->hash) return -1;
+ else if(((struct config_option *)a)->hash > ((struct config_option *)b)->hash) return 1;
+ else return strcmp(((struct config_option *)a)->name, ((struct config_option *)b)->name);
+}
+
+#define appconfig_option_index_add(co, cv) (struct config_option *)avl_insert_lock(&((co)->values_index), (avl *)(cv))
+#define appconfig_option_index_del(co, cv) (struct config_option *)avl_remove_lock(&((co)->values_index), (avl *)(cv))
+
+static struct config_option *appconfig_option_index_find(struct section *co, const char *name, uint32_t hash) {
+ struct config_option tmp;
+ tmp.hash = (hash)?hash:simple_hash(name);
+ tmp.name = (char *)name;
+
+ return (struct config_option *)avl_search_lock(&(co->values_index), (avl *) &tmp);
+}
+
+
+// ----------------------------------------------------------------------------
+// config sections index
+
+int appconfig_section_compare(void *a, void *b) {
+ if(((struct section *)a)->hash < ((struct section *)b)->hash) return -1;
+ else if(((struct section *)a)->hash > ((struct section *)b)->hash) return 1;
+ else return strcmp(((struct section *)a)->name, ((struct section *)b)->name);
+}
+
+#define appconfig_index_add(root, cfg) (struct section *)avl_insert_lock(&(root)->index, (avl *)(cfg))
+#define appconfig_index_del(root, cfg) (struct section *)avl_remove_lock(&(root)->index, (avl *)(cfg))
+
+static struct section *appconfig_index_find(struct config *root, const char *name, uint32_t hash) {
+ struct section tmp;
+ tmp.hash = (hash)?hash:simple_hash(name);
+ tmp.name = (char *)name;
+
+ return (struct section *)avl_search_lock(&root->index, (avl *) &tmp);
+}
+
+
+// ----------------------------------------------------------------------------
+// config section methods
+
+static inline struct section *appconfig_section_find(struct config *root, const char *section) {
+ return appconfig_index_find(root, section, 0);
+}
+
+static inline struct section *appconfig_section_create(struct config *root, const char *section) {
+ debug(D_CONFIG, "Creating section '%s'.", section);
+
+ struct section *co = callocz(1, sizeof(struct section));
+ co->name = strdupz(section);
+ co->hash = simple_hash(co->name);
+ netdata_mutex_init(&co->mutex);
+
+ avl_init_lock(&co->values_index, appconfig_option_compare);
+
+ if(unlikely(appconfig_index_add(root, co) != co))
+ error("INTERNAL ERROR: indexing of section '%s', already exists.", co->name);
+
+ appconfig_wrlock(root);
+ struct section *co2 = root->last_section;
+ if(co2) {
+ co2->next = co;
+ } else {
+ root->first_section = co;
+ }
+ root->last_section = co;
+ appconfig_unlock(root);
+
+ return co;
+}
+
+void appconfig_section_destroy_non_loaded(struct config *root, const char *section)
+{
+ struct section *co;
+ struct config_option *cv, *cv_next;
+
+ debug(D_CONFIG, "Destroying section '%s'.", section);
+
+ co = appconfig_section_find(root, section);
+ if(!co) {
+ error("Could not destroy section '%s'. Not found.", section);
+ return;
+ }
+
+ config_section_wrlock(co);
+ for(cv = co->values; cv ; cv = cv->next) {
+ if (cv->flags & CONFIG_VALUE_LOADED) {
+ /* Do not destroy values that were loaded from the configuration files. */
+ config_section_unlock(co);
+ return;
+ }
+ }
+ for(cv = co->values ; cv ; cv = cv_next) {
+ cv_next = cv->next;
+ if(unlikely(!appconfig_option_index_del(co, cv)))
+ error("Cannot remove config option '%s' from section '%s'.", cv->name, co->name);
+ freez(cv->value);
+ freez(cv->name);
+ freez(cv);
+ }
+ co->values = NULL;
+ config_section_unlock(co);
+
+ if (unlikely(!appconfig_index_del(root, co))) {
+ error("Cannot remove section '%s' from config.", section);
+ return;
+ }
+
+ avl_destroy_lock(&co->values_index);
+ freez(co->name);
+ pthread_mutex_destroy(&co->mutex);
+ freez(co);
+}
+
+
+// ----------------------------------------------------------------------------
+// config name-value methods
+
+static inline struct config_option *appconfig_value_create(struct section *co, const char *name, const char *value) {
+ debug(D_CONFIG, "Creating config entry for name '%s', value '%s', in section '%s'.", name, value, co->name);
+
+ struct config_option *cv = callocz(1, sizeof(struct config_option));
+ cv->name = strdupz(name);
+ cv->hash = simple_hash(cv->name);
+ cv->value = strdupz(value);
+
+ struct config_option *found = appconfig_option_index_add(co, cv);
+ if(found != cv) {
+ error("indexing of config '%s' in section '%s': already exists - using the existing one.", cv->name, co->name);
+ freez(cv->value);
+ freez(cv->name);
+ freez(cv);
+ return found;
+ }
+
+ config_section_wrlock(co);
+ struct config_option *cv2 = co->values;
+ if(cv2) {
+ while (cv2->next) cv2 = cv2->next;
+ cv2->next = cv;
+ }
+ else co->values = cv;
+ config_section_unlock(co);
+
+ return cv;
+}
+
+int appconfig_exists(struct config *root, const char *section, const char *name) {
+ struct config_option *cv;
+
+ debug(D_CONFIG, "request to get config in section '%s', name '%s'", section, name);
+
+ struct section *co = appconfig_section_find(root, section);
+ if(!co) return 0;
+
+ cv = appconfig_option_index_find(co, name, 0);
+ if(!cv) return 0;
+
+ return 1;
+}
+
+int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new) {
+ struct config_option *cv_old, *cv_new;
+ int ret = -1;
+
+ debug(D_CONFIG, "request to rename config in section '%s', old name '%s', to section '%s', new name '%s'", section_old, name_old, section_new, name_new);
+
+ struct section *co_old = appconfig_section_find(root, section_old);
+ if(!co_old) return ret;
+
+ struct section *co_new = appconfig_section_find(root, section_new);
+ if(!co_new) co_new = appconfig_section_create(root, section_new);
+
+ config_section_wrlock(co_old);
+ if(co_old != co_new)
+ config_section_wrlock(co_new);
+
+ cv_old = appconfig_option_index_find(co_old, name_old, 0);
+ if(!cv_old) goto cleanup;
+
+ cv_new = appconfig_option_index_find(co_new, name_new, 0);
+ if(cv_new) goto cleanup;
+
+ if(unlikely(appconfig_option_index_del(co_old, cv_old) != cv_old))
+ error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted tge wrong config entry.", cv_old->name, co_old->name);
+
+ if(co_old->values == cv_old) {
+ co_old->values = cv_old->next;
+ }
+ else {
+ struct config_option *t;
+ for(t = co_old->values; t && t->next != cv_old ;t = t->next) ;
+ if(!t || t->next != cv_old)
+ error("INTERNAL ERROR: cannot find variable '%s' in section '%s' of the config - but it should be there.", cv_old->name, co_old->name);
+ else
+ t->next = cv_old->next;
+ }
+
+ freez(cv_old->name);
+ cv_old->name = strdupz(name_new);
+ cv_old->hash = simple_hash(cv_old->name);
+
+ cv_new = cv_old;
+ cv_new->next = co_new->values;
+ co_new->values = cv_new;
+
+ if(unlikely(appconfig_option_index_add(co_new, cv_old) != cv_old))
+ error("INTERNAL ERROR: re-indexing of config '%s' in section '%s', already exists.", cv_old->name, co_new->name);
+
+ ret = 0;
+
+cleanup:
+ if(co_old != co_new)
+ config_section_unlock(co_new);
+ config_section_unlock(co_old);
+ return ret;
+}
+
+char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value)
+{
+ struct config_option *cv;
+
+ // Only calls internal to this file check for a NULL result and they do not supply a NULL arg.
+ // External caller should treat NULL as an error case.
+ cv = appconfig_option_index_find(co, name, 0);
+ if (!cv) {
+ if (!default_value) return NULL;
+ cv = appconfig_value_create(co, name, default_value);
+ if (!cv) return NULL;
+ }
+ cv->flags |= CONFIG_VALUE_USED;
+
+ if((cv->flags & CONFIG_VALUE_LOADED) || (cv->flags & CONFIG_VALUE_CHANGED)) {
+ // this is a loaded value from the config file
+ // if it is different than the default, mark it
+ if(!(cv->flags & CONFIG_VALUE_CHECKED)) {
+ if(default_value && strcmp(cv->value, default_value) != 0) cv->flags |= CONFIG_VALUE_CHANGED;
+ cv->flags |= CONFIG_VALUE_CHECKED;
+ }
+ }
+
+ return(cv->value);
+}
+
+
+char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value)
+{
+ if (default_value == NULL)
+ debug(D_CONFIG, "request to get config in section '%s', name '%s' or fail", section, name);
+ else
+ debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value);
+
+ struct section *co = appconfig_section_find(root, section);
+ if (!co && !default_value)
+ return NULL;
+ if(!co) co = appconfig_section_create(root, section);
+
+ return appconfig_get_by_section(co, name, default_value);
+}
+
+long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value)
+{
+ char buffer[100], *s;
+ sprintf(buffer, "%lld", value);
+
+ s = appconfig_get(root, section, name, buffer);
+ if(!s) return value;
+
+ return strtoll(s, NULL, 0);
+}
+
+LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value)
+{
+ char buffer[100], *s;
+ sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value);
+
+ s = appconfig_get(root, section, name, buffer);
+ if(!s) return value;
+
+ return str2ld(s, NULL);
+}
+
+static inline int appconfig_test_boolean_value(char *s) {
+ if(!strcasecmp(s, "yes") || !strcasecmp(s, "true") || !strcasecmp(s, "on")
+ || !strcasecmp(s, "auto") || !strcasecmp(s, "on demand"))
+ return 1;
+
+ return 0;
+}
+
+int appconfig_get_boolean_by_section(struct section *co, const char *name, int value) {
+ char *s;
+
+ s = appconfig_get_by_section(co, name, (!value)?"no":"yes");
+ if(!s) return value;
+
+ return appconfig_test_boolean_value(s);
+}
+
+int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value)
+{
+ char *s;
+ if(value) s = "yes";
+ else s = "no";
+
+ s = appconfig_get(root, section, name, s);
+ if(!s) return value;
+
+ return appconfig_test_boolean_value(s);
+}
+
+int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value)
+{
+ char *s;
+
+ if(value == CONFIG_BOOLEAN_AUTO)
+ s = "auto";
+
+ else if(value == CONFIG_BOOLEAN_NO)
+ s = "no";
+
+ else
+ s = "yes";
+
+ s = appconfig_get(root, section, name, s);
+ if(!s) return value;
+
+ if(!strcmp(s, "yes"))
+ return CONFIG_BOOLEAN_YES;
+ else if(!strcmp(s, "no"))
+ return CONFIG_BOOLEAN_NO;
+ else if(!strcmp(s, "auto") || !strcmp(s, "on demand"))
+ return CONFIG_BOOLEAN_AUTO;
+
+ return value;
+}
+
+const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value)
+{
+ struct config_option *cv;
+
+ debug(D_CONFIG, "request to set default config in section '%s', name '%s', value '%s'", section, name, value);
+
+ struct section *co = appconfig_section_find(root, section);
+ if(!co) return appconfig_set(root, section, name, value);
+
+ cv = appconfig_option_index_find(co, name, 0);
+ if(!cv) return appconfig_set(root, section, name, value);
+
+ cv->flags |= CONFIG_VALUE_USED;
+
+ if(cv->flags & CONFIG_VALUE_LOADED)
+ return cv->value;
+
+ if(strcmp(cv->value, value) != 0) {
+ cv->flags |= CONFIG_VALUE_CHANGED;
+
+ freez(cv->value);
+ cv->value = strdupz(value);
+ }
+
+ return cv->value;
+}
+
+const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value)
+{
+ struct config_option *cv;
+
+ debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value);
+
+ struct section *co = appconfig_section_find(root, section);
+ if(!co) co = appconfig_section_create(root, section);
+
+ cv = appconfig_option_index_find(co, name, 0);
+ if(!cv) cv = appconfig_value_create(co, name, value);
+ cv->flags |= CONFIG_VALUE_USED;
+
+ if(strcmp(cv->value, value) != 0) {
+ cv->flags |= CONFIG_VALUE_CHANGED;
+
+ freez(cv->value);
+ cv->value = strdupz(value);
+ }
+
+ return value;
+}
+
+long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value)
+{
+ char buffer[100];
+ sprintf(buffer, "%lld", value);
+
+ appconfig_set(root, section, name, buffer);
+
+ return value;
+}
+
+LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value)
+{
+ char buffer[100];
+ sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value);
+
+ appconfig_set(root, section, name, buffer);
+
+ return value;
+}
+
+int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value)
+{
+ char *s;
+ if(value) s = "yes";
+ else s = "no";
+
+ appconfig_set(root, section, name, s);
+
+ return value;
+}
+
+int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value)
+{
+ int result = 0;
+ const char *s;
+
+ s = appconfig_get(root, section, name, value);
+ if(!s) goto fallback;
+
+ if(!config_parse_duration(s, &result)) {
+ error("config option '[%s].%s = %s' is configured with an valid duration", section, name, s);
+ goto fallback;
+ }
+
+ return result;
+
+ fallback:
+ if(!config_parse_duration(value, &result))
+ error("INTERNAL ERROR: default duration supplied for option '[%s].%s = %s' is not a valid duration", section, name, value);
+
+ return result;
+}
+
+// ----------------------------------------------------------------------------
+// config load/save
+
+int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name)
+{
+ int line = 0;
+ struct section *co = NULL;
+ int is_exporter_config = 0;
+ int _backends = 0; // number of backend sections we have
+ char working_instance[CONFIG_MAX_NAME + 1];
+ char working_connector[CONFIG_MAX_NAME + 1];
+ struct section *working_connector_section = NULL;
+ int global_exporting_section = 0;
+
+ char buffer[CONFIG_FILE_LINE_MAX + 1], *s;
+
+ if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME;
+
+ debug(D_CONFIG, "CONFIG: opening config file '%s'", filename);
+
+ FILE *fp = fopen(filename, "r");
+ if(!fp) {
+ // info("CONFIG: cannot open file '%s'. Using internal defaults.", filename);
+ return 0;
+ }
+
+ uint32_t section_hash = 0;
+ if(section_name) {
+ section_hash = simple_hash(section_name);
+ }
+ is_exporter_config = (strstr(filename, EXPORTING_CONF) != NULL);
+
+ while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {
+ buffer[CONFIG_FILE_LINE_MAX] = '\0';
+ line++;
+
+ s = trim(buffer);
+ if(!s || *s == '#') {
+ debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename);
+ continue;
+ }
+
+ int len = (int) strlen(s);
+ if(*s == '[' && s[len - 1] == ']') {
+ // new section
+ s[len - 1] = '\0';
+ s++;
+
+ if (is_exporter_config) {
+ global_exporting_section =
+ !(strcmp(s, CONFIG_SECTION_EXPORTING)) || !(strcmp(s, CONFIG_SECTION_PROMETHEUS));
+ if (unlikely(!global_exporting_section)) {
+ int rc;
+ rc = is_valid_connector(s, 0);
+ if (likely(rc)) {
+ strncpy(working_connector, s, CONFIG_MAX_NAME);
+ s = s + rc + 1;
+ if (unlikely(!(*s))) {
+ _backends++;
+ sprintf(buffer, "instance_%d", _backends);
+ s = buffer;
+ }
+ strncpy(working_instance, s, CONFIG_MAX_NAME);
+ working_connector_section = NULL;
+ if (unlikely(appconfig_section_find(root, working_instance))) {
+ error("Instance (%s) already exists", working_instance);
+ co = NULL;
+ continue;
+ }
+ } else {
+ co = NULL;
+ error("Section (%s) does not specify a valid connector", s);
+ continue;
+ }
+ }
+ }
+
+ co = appconfig_section_find(root, s);
+ if(!co) co = appconfig_section_create(root, s);
+
+ if(co && section_name && overwrite_used && section_hash == co->hash && !strcmp(section_name, co->name)) {
+ config_section_wrlock(co);
+ struct config_option *cv2 = co->values;
+ while (cv2) {
+ struct config_option *save = cv2->next;
+ struct config_option *found = appconfig_option_index_del(co, cv2);
+ if(found != cv2)
+ error("INTERNAL ERROR: Cannot remove '%s' from section '%s', it was not inserted before.",
+ cv2->name, co->name);
+
+ freez(cv2->name);
+ freez(cv2->value);
+ freez(cv2);
+ cv2 = save;
+ }
+ co->values = NULL;
+ config_section_unlock(co);
+ }
+
+ continue;
+ }
+
+ if(!co) {
+ // line outside a section
+ error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename);
+ continue;
+ }
+
+ if(section_name && overwrite_used && section_hash != co->hash && strcmp(section_name, co->name)) {
+ continue;
+ }
+
+ char *name = s;
+ char *value = strchr(s, '=');
+ if(!value) {
+ error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename);
+ continue;
+ }
+ *value = '\0';
+ value++;
+
+ name = trim(name);
+ value = trim(value);
+
+ if(!name || *name == '#') {
+ error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename);
+ continue;
+ }
+
+ if(!value) value = "";
+
+ struct config_option *cv = appconfig_option_index_find(co, name, 0);
+
+ if (!cv) {
+ cv = appconfig_value_create(co, name, value);
+ if (likely(is_exporter_config) && unlikely(!global_exporting_section)) {
+ if (unlikely(!working_connector_section)) {
+ working_connector_section = appconfig_section_find(root, working_connector);
+ if (!working_connector_section)
+ working_connector_section = appconfig_section_create(root, working_connector);
+ if (likely(working_connector_section)) {
+ add_connector_instance(working_connector_section, co);
+ }
+ }
+ }
+ } else {
+ if (((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) {
+ debug(
+ D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name);
+ freez(cv->value);
+ cv->value = strdupz(value);
+ } else
+ debug(
+ D_CONFIG,
+ "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.",
+ line,
+ filename,
+ co->name,
+ cv->name);
+ }
+ cv->flags |= CONFIG_VALUE_LOADED;
+ }
+
+ fclose(fp);
+
+ return 1;
+}
+
+void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
+{
+ int i, pri;
+ struct section *co;
+ struct config_option *cv;
+
+ for(i = 0; i < 3 ;i++) {
+ switch(i) {
+ case 0:
+ buffer_strcat(wb,
+ "# netdata configuration\n"
+ "#\n"
+ "# You can download the latest version of this file, using:\n"
+ "#\n"
+ "# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n"
+ "# or\n"
+ "# curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n"
+ "#\n"
+ "# You can uncomment and change any of the options below.\n"
+ "# The value shown in the commented settings, is the default value.\n"
+ "#\n"
+ "\n# global netdata configuration\n");
+ break;
+
+ case 1:
+ buffer_strcat(wb, "\n\n# per plugin configuration\n");
+ break;
+
+ case 2:
+ buffer_strcat(wb, "\n\n# per chart configuration\n");
+ break;
+ }
+
+ appconfig_wrlock(root);
+ for(co = root->first_section; co ; co = co->next) {
+ if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)
+ || !strcmp(co->name, CONFIG_SECTION_WEB)
+ || !strcmp(co->name, CONFIG_SECTION_STATSD)
+ || !strcmp(co->name, CONFIG_SECTION_PLUGINS)
+ || !strcmp(co->name, CONFIG_SECTION_CLOUD)
+ || !strcmp(co->name, CONFIG_SECTION_REGISTRY)
+ || !strcmp(co->name, CONFIG_SECTION_HEALTH)
+ || !strcmp(co->name, CONFIG_SECTION_BACKEND)
+ || !strcmp(co->name, CONFIG_SECTION_STREAM)
+ || !strcmp(co->name, CONFIG_SECTION_HOST_LABEL)
+ )
+ pri = 0;
+ else if(!strncmp(co->name, "plugin:", 7)) pri = 1;
+ else pri = 2;
+
+ if(i == pri) {
+ int loaded = 0;
+ int used = 0;
+ int changed = 0;
+ int count = 0;
+
+ config_section_wrlock(co);
+ for(cv = co->values; cv ; cv = cv->next) {
+ used += (cv->flags & CONFIG_VALUE_USED)?1:0;
+ loaded += (cv->flags & CONFIG_VALUE_LOADED)?1:0;
+ changed += (cv->flags & CONFIG_VALUE_CHANGED)?1:0;
+ count++;
+ }
+ config_section_unlock(co);
+
+ if(!count) continue;
+ if(only_changed && !changed && !loaded) continue;
+
+ if(!used) {
+ buffer_sprintf(wb, "\n# section '%s' is not used.", co->name);
+ }
+
+ buffer_sprintf(wb, "\n[%s]\n", co->name);
+
+ config_section_wrlock(co);
+ for(cv = co->values; cv ; cv = cv->next) {
+
+ if(used && !(cv->flags & CONFIG_VALUE_USED)) {
+ buffer_sprintf(wb, "\n\t# option '%s' is not used.\n", cv->name);
+ }
+ buffer_sprintf(wb, "\t%s%s = %s\n", ((!(cv->flags & CONFIG_VALUE_LOADED)) && (!(cv->flags & CONFIG_VALUE_CHANGED)) && (cv->flags & CONFIG_VALUE_USED))?"# ":"", cv->name, cv->value);
+ }
+ config_section_unlock(co);
+ }
+ }
+ appconfig_unlock(root);
+ }
+}
+
+/**
+ * Parse Duration
+ *
+ * Parse the string setting the result
+ *
+ * @param string the timestamp string
+ * @param result the output variable
+ *
+ * @return It returns 1 on success and 0 otherwise
+ */
+int config_parse_duration(const char* string, int* result) {
+ while(*string && isspace(*string)) string++;
+
+ if(unlikely(!*string)) goto fallback;
+
+ if(*string == 'n' && !strcmp(string, "never")) {
+ // this is a valid option
+ *result = 0;
+ return 1;
+ }
+
+ // make sure it is a number
+ if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback;
+
+ char *e = NULL;
+ calculated_number n = str2ld(string, &e);
+ if(e && *e) {
+ switch (*e) {
+ case 'Y':
+ *result = (int) (n * 31536000);
+ break;
+ case 'M':
+ *result = (int) (n * 2592000);
+ break;
+ case 'w':
+ *result = (int) (n * 604800);
+ break;
+ case 'd':
+ *result = (int) (n * 86400);
+ break;
+ case 'h':
+ *result = (int) (n * 3600);
+ break;
+ case 'm':
+ *result = (int) (n * 60);
+ break;
+ case 's':
+ default:
+ *result = (int) (n);
+ break;
+ }
+ }
+ else
+ *result = (int)(n);
+
+ return 1;
+
+ fallback:
+ *result = 0;
+ return 0;
+}
+
+struct section *appconfig_get_section(struct config *root, const char *name)
+{
+ return appconfig_section_find(root, name);
+}
diff --git a/libnetdata/config/appconfig.h b/libnetdata/config/appconfig.h
new file mode 100644
index 0000000..9d02e4a
--- /dev/null
+++ b/libnetdata/config/appconfig.h
@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+/*
+ * This section manages ini config files, like netdata.conf and stream.conf
+ *
+ * It is organized like this:
+ *
+ * struct config (i.e. netdata.conf or stream.conf)
+ * .sections = a linked list of struct section
+ * .mutex = a mutex to protect the above linked list due to multi-threading
+ * .index = an AVL tree of struct section
+ *
+ * struct section (i.e. [global] or [health] of netdata.conf)
+ * .value = a linked list of struct config_option
+ * .mutex = a mutex to protect the above linked list due to multi-threading
+ * .value_index = an AVL tree of struct config_option
+ *
+ * struct config_option (ie. a name-value pair for each ini file option)
+ *
+ * The following operations on name-value options are supported:
+ * SET to set the value of an option
+ * SET DEFAULT to set the value and the default value of an option
+ * GET to get the value of an option
+ * EXISTS to check if an option exists
+ * MOVE to move an option from a section to another section, and/or rename it
+ *
+ * GET and SET operations are provided for the following data types:
+ * STRING
+ * NUMBER (long long)
+ * FLOAT (long double)
+ * BOOLEAN (false, true)
+ * BOOLEAN ONDEMAND (false, true, auto)
+ *
+ * GET and SET operations create struct config_option, if it is not already present.
+ * This allows netdata to run even without netdata.conf and stream.conf. The internal
+ * defaults are used to create the structure that should exist in the ini file and the config
+ * file can be downloaded from the server.
+ *
+ * Also 2 operations are supported for the whole config file:
+ *
+ * LOAD To load the ini file from disk
+ * GENERATE To generate the ini file (this is used to download the ini file from the server)
+ *
+ * For each option (name-value pair), the system maintains 4 flags:
+ * LOADED to indicate that the value has been loaded from the file
+ * USED to indicate that netdata used the value
+ * CHANGED to indicate that the value has been changed from the loaded value or the internal default value
+ * CHECKED is used internally for optimization (to avoid an strcmp() every time GET is called).
+ *
+ * TODO:
+ * 1. The linked lists and the mutexes can be removed and the AVL trees can become DICTIONARY.
+ * This part of the code was written before we add traversal to AVL.
+ *
+ * 2. High level data types could be supported, to simplify the rest of the code:
+ * MULTIPLE CHOICE to let the user select one of the supported keywords
+ * this would allow users see in comments the available options
+ *
+ * SIMPLE PATTERN to let the user define netdata SIMPLE PATTERNS
+ *
+ * 3. Sorting of options should be supported.
+ * Today, when the ini file is downloaded from the server, the options are shown in the order
+ * they appear in the linked list (the order they were added, listing changed options first).
+ * If we remove the linked list, the order they appear in the AVL tree will be used (which is
+ * random due to simple_hash()).
+ * Ideally, we support sorting of options when generating the ini file.
+ *
+ * 4. There is no free() operation. So, memory is freed on netdata exit.
+ *
+ * 5. Avoid memory fragmentation
+ * Since entries are created from multiple threads and a lot of allocations are required
+ * for each config_option, fragmentation can be a problem for IoT.
+ *
+ * 6. Although this way of managing options is quite flexible and dynamic, it wastes memory
+ * for the names of the options. Since most of the option names are static, we could provide
+ * a method to allocate only the dynamic option names.
+ */
+
+#ifndef NETDATA_CONFIG_H
+#define NETDATA_CONFIG_H 1
+
+#include "../libnetdata.h"
+
+#define CONFIG_FILENAME "netdata.conf"
+
+#define CONFIG_SECTION_GLOBAL "global"
+#define CONFIG_SECTION_WEB "web"
+#define CONFIG_SECTION_STATSD "statsd"
+#define CONFIG_SECTION_PLUGINS "plugins"
+#define CONFIG_SECTION_CLOUD "cloud"
+#define CONFIG_SECTION_REGISTRY "registry"
+#define CONFIG_SECTION_HEALTH "health"
+#define CONFIG_SECTION_BACKEND "backend"
+#define CONFIG_SECTION_STREAM "stream"
+#define CONFIG_SECTION_EXPORTING "exporting:global"
+#define CONFIG_SECTION_PROMETHEUS "prometheus:exporter"
+#define CONFIG_SECTION_HOST_LABEL "host labels"
+#define EXPORTING_CONF "exporting.conf"
+
+// these are used to limit the configuration names and values lengths
+// they are not enforced by config.c functions (they will strdup() all strings, no matter of their length)
+#define CONFIG_MAX_NAME 1024
+#define CONFIG_MAX_VALUE 2048
+
+// ----------------------------------------------------------------------------
+// Config definitions
+#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2)
+
+#define CONFIG_VALUE_LOADED 0x01 // has been loaded from the config
+#define CONFIG_VALUE_USED 0x02 // has been accessed from the program
+#define CONFIG_VALUE_CHANGED 0x04 // has been changed from the loaded value or the internal default value
+#define CONFIG_VALUE_CHECKED 0x08 // has been checked if the value is different from the default
+
+struct config_option {
+ avl avl_node; // the index entry of this entry - this has to be first!
+
+ uint8_t flags;
+ uint32_t hash; // a simple hash to speed up searching
+ // we first compare hashes, and only if the hashes are equal we do string comparisons
+
+ char *name;
+ char *value;
+
+ struct config_option *next; // config->mutex protects just this
+};
+
+struct section {
+ avl avl_node; // the index entry of this section - this has to be first!
+
+ uint32_t hash; // a simple hash to speed up searching
+ // we first compare hashes, and only if the hashes are equal we do string comparisons
+
+ char *name;
+
+ struct section *next; // gloabl config_mutex protects just this
+
+ struct config_option *values;
+ avl_tree_lock values_index;
+
+ netdata_mutex_t mutex; // this locks only the writers, to ensure atomic updates
+ // readers are protected using the rwlock in avl_tree_lock
+};
+
+struct config {
+ struct section *first_section;
+ struct section *last_section; // optimize inserting at the end
+ netdata_mutex_t mutex;
+ avl_tree_lock index;
+};
+
+#define CONFIG_BOOLEAN_INVALID 100 // an invalid value to check for validity (used as default initialization when needed)
+
+#define CONFIG_BOOLEAN_NO 0 // disabled
+#define CONFIG_BOOLEAN_YES 1 // enabled
+
+#ifndef CONFIG_BOOLEAN_AUTO
+#define CONFIG_BOOLEAN_AUTO 2 // enabled if it has useful info when enabled
+#endif
+
+extern int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name);
+extern void config_section_wrlock(struct section *co);
+extern void config_section_unlock(struct section *co);
+
+extern char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value);
+extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value);
+extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value);
+extern LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value);
+extern int appconfig_get_boolean_by_section(struct section *co, const char *name, int value);
+extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value);
+extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value);
+extern int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value);
+
+extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value);
+extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value);
+extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value);
+extern LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value);
+extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value);
+
+extern int appconfig_exists(struct config *root, const char *section, const char *name);
+extern int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new);
+
+extern void appconfig_generate(struct config *root, BUFFER *wb, int only_changed);
+
+extern int appconfig_section_compare(void *a, void *b);
+
+extern void appconfig_section_destroy_non_loaded(struct config *root, const char *section);
+
+extern int config_parse_duration(const char* string, int* result);
+
+extern struct section *appconfig_get_section(struct config *root, const char *name);
+
+extern void appconfig_wrlock(struct config *root);
+extern void appconfig_unlock(struct config *root);
+
+struct connector_instance {
+ char instance_name[CONFIG_MAX_NAME + 1];
+ char connector_name[CONFIG_MAX_NAME + 1];
+};
+
+typedef struct _connector_instance {
+ struct section *connector; // actual connector
+ struct section *instance; // This instance
+ char instance_name[CONFIG_MAX_NAME + 1];
+ char connector_name[CONFIG_MAX_NAME + 1];
+ struct _connector_instance *next; // Next instance
+} _CONNECTOR_INSTANCE;
+
+extern _CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance);
+
+#endif /* NETDATA_CONFIG_H */
diff --git a/libnetdata/dictionary/Makefile.am b/libnetdata/dictionary/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/dictionary/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/dictionary/README.md b/libnetdata/dictionary/README.md
new file mode 100644
index 0000000..234ff18
--- /dev/null
+++ b/libnetdata/dictionary/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/dictionary/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fdictionary%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c
new file mode 100644
index 0000000..cfcf1fb
--- /dev/null
+++ b/libnetdata/dictionary/dictionary.c
@@ -0,0 +1,329 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+// ----------------------------------------------------------------------------
+// dictionary statistics
+
+static inline void NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(DICTIONARY *dict) {
+ if(likely(dict->stats))
+ dict->stats->inserts++;
+}
+static inline void NETDATA_DICTIONARY_STATS_DELETES_PLUS1(DICTIONARY *dict) {
+ if(likely(dict->stats))
+ dict->stats->deletes++;
+}
+static inline void NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) {
+ if(likely(dict->stats))
+ dict->stats->searches++;
+}
+static inline void NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict) {
+ if(likely(dict->stats))
+ dict->stats->entries++;
+}
+static inline void NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict) {
+ if(likely(dict->stats))
+ dict->stats->entries--;
+}
+
+
+// ----------------------------------------------------------------------------
+// dictionary locks
+
+static inline void dictionary_read_lock(DICTIONARY *dict) {
+ if(likely(dict->rwlock)) {
+ // debug(D_DICTIONARY, "Dictionary READ lock");
+ netdata_rwlock_rdlock(dict->rwlock);
+ }
+}
+
+static inline void dictionary_write_lock(DICTIONARY *dict) {
+ if(likely(dict->rwlock)) {
+ // debug(D_DICTIONARY, "Dictionary WRITE lock");
+ netdata_rwlock_wrlock(dict->rwlock);
+ }
+}
+
+static inline void dictionary_unlock(DICTIONARY *dict) {
+ if(likely(dict->rwlock)) {
+ // debug(D_DICTIONARY, "Dictionary UNLOCK lock");
+ netdata_rwlock_unlock(dict->rwlock);
+ }
+}
+
+
+// ----------------------------------------------------------------------------
+// avl index
+
+static int name_value_compare(void* a, void* b) {
+ if(((NAME_VALUE *)a)->hash < ((NAME_VALUE *)b)->hash) return -1;
+ else if(((NAME_VALUE *)a)->hash > ((NAME_VALUE *)b)->hash) return 1;
+ else return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name);
+}
+
+static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *dict, const char *name, uint32_t hash) {
+ NAME_VALUE tmp;
+ tmp.hash = (hash)?hash:simple_hash(name);
+ tmp.name = (char *)name;
+
+ NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(dict);
+ return (NAME_VALUE *)avl_search(&(dict->values_index), (avl *) &tmp);
+}
+
+// ----------------------------------------------------------------------------
+// internal methods
+
+static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const char *name, void *value, size_t value_len, uint32_t hash) {
+ debug(D_DICTIONARY, "Creating name value entry for name '%s'.", name);
+
+ NAME_VALUE *nv = callocz(1, sizeof(NAME_VALUE));
+
+ if(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)
+ nv->name = (char *)name;
+ else {
+ nv->name = strdupz(name);
+ }
+
+ nv->hash = (hash)?hash:simple_hash(nv->name);
+
+ if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)
+ nv->value = value;
+ else {
+ nv->value = mallocz(value_len);
+ memcpy(nv->value, value, value_len);
+ }
+
+ // index it
+ NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(dict);
+ if(unlikely(avl_insert(&((dict)->values_index), (avl *)(nv)) != (avl *)nv))
+ error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary.");
+
+ NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(dict);
+
+ return nv;
+}
+
+static void dictionary_name_value_destroy_nolock(DICTIONARY *dict, NAME_VALUE *nv) {
+ debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name);
+
+ NETDATA_DICTIONARY_STATS_DELETES_PLUS1(dict);
+ if(unlikely(avl_remove(&(dict->values_index), (avl *)(nv)) != (avl *)nv))
+ error("dictionary: INTERNAL ERROR: dictionary invalid removal of node.");
+
+ NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(dict);
+
+ if(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) {
+ debug(D_REGISTRY, "Dictionary freeing value of '%s'", nv->name);
+ freez(nv->value);
+ }
+
+ if(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)) {
+ debug(D_REGISTRY, "Dictionary freeing name '%s'", nv->name);
+ freez(nv->name);
+ }
+
+ freez(nv);
+}
+
+// ----------------------------------------------------------------------------
+// API - basic methods
+
+DICTIONARY *dictionary_create(uint8_t flags) {
+ debug(D_DICTIONARY, "Creating dictionary.");
+
+ DICTIONARY *dict = callocz(1, sizeof(DICTIONARY));
+
+ if(flags & DICTIONARY_FLAG_WITH_STATISTICS)
+ dict->stats = callocz(1, sizeof(struct dictionary_stats));
+
+ if(!(flags & DICTIONARY_FLAG_SINGLE_THREADED)) {
+ dict->rwlock = callocz(1, sizeof(netdata_rwlock_t));
+ netdata_rwlock_init(dict->rwlock);
+ }
+
+ avl_init(&dict->values_index, name_value_compare);
+ dict->flags = flags;
+
+ return dict;
+}
+
+void dictionary_destroy(DICTIONARY *dict) {
+ debug(D_DICTIONARY, "Destroying dictionary.");
+
+ dictionary_write_lock(dict);
+
+ while(dict->values_index.root)
+ dictionary_name_value_destroy_nolock(dict, (NAME_VALUE *)dict->values_index.root);
+
+ dictionary_unlock(dict);
+
+ if(dict->stats)
+ freez(dict->stats);
+
+ if(dict->rwlock) {
+ netdata_rwlock_destroy(dict->rwlock);
+ freez(dict->rwlock);
+ }
+
+ freez(dict);
+}
+
+// ----------------------------------------------------------------------------
+
+void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) {
+ debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name);
+
+ uint32_t hash = simple_hash(name);
+
+ dictionary_write_lock(dict);
+
+ NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, hash);
+ if(unlikely(!nv)) {
+ debug(D_DICTIONARY, "Dictionary entry with name '%s' not found. Creating a new one.", name);
+
+ nv = dictionary_name_value_create_nolock(dict, name, value, value_len, hash);
+ if(unlikely(!nv))
+ fatal("Cannot create name_value.");
+ }
+ else {
+ debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", name);
+
+ if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE) {
+ debug(D_REGISTRY, "Dictionary: linking value to '%s'", name);
+ nv->value = value;
+ }
+ else {
+ debug(D_REGISTRY, "Dictionary: cloning value to '%s'", name);
+
+ // copy the new value without breaking
+ // any other thread accessing the same entry
+ void *new = mallocz(value_len),
+ *old = nv->value;
+
+ memcpy(new, value, value_len);
+ nv->value = new;
+
+ debug(D_REGISTRY, "Dictionary: freeing old value of '%s'", name);
+ freez(old);
+ }
+ }
+
+ dictionary_unlock(dict);
+
+ return nv->value;
+}
+
+void *dictionary_get(DICTIONARY *dict, const char *name) {
+ debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name);
+
+ dictionary_read_lock(dict);
+ NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0);
+ dictionary_unlock(dict);
+
+ if(unlikely(!nv)) {
+ debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name);
+ return NULL;
+ }
+
+ debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name);
+ return nv->value;
+}
+
+int dictionary_del(DICTIONARY *dict, const char *name) {
+ int ret;
+
+ debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name);
+
+ dictionary_write_lock(dict);
+
+ NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0);
+ if(unlikely(!nv)) {
+ debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name);
+ ret = -1;
+ }
+ else {
+ debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name);
+ dictionary_name_value_destroy_nolock(dict, nv);
+ ret = 0;
+ }
+
+ dictionary_unlock(dict);
+
+ return ret;
+}
+
+
+// ----------------------------------------------------------------------------
+// API - walk through the dictionary
+// the dictionary is locked for reading while this happens
+// do not user other dictionary calls while walking the dictionary - deadlock!
+
+static int dictionary_walker(avl *a, int (*callback)(void *entry, void *data), void *data) {
+ int total = 0, ret = 0;
+
+ if(a->avl_link[0]) {
+ ret = dictionary_walker(a->avl_link[0], callback, data);
+ if(ret < 0) return ret;
+ total += ret;
+ }
+
+ ret = callback(((NAME_VALUE *)a)->value, data);
+ if(ret < 0) return ret;
+ total += ret;
+
+ if(a->avl_link[1]) {
+ ret = dictionary_walker(a->avl_link[1], callback, data);
+ if (ret < 0) return ret;
+ total += ret;
+ }
+
+ return total;
+}
+
+int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) {
+ int ret = 0;
+
+ dictionary_read_lock(dict);
+
+ if(likely(dict->values_index.root))
+ ret = dictionary_walker(dict->values_index.root, callback, data);
+
+ dictionary_unlock(dict);
+
+ return ret;
+}
+
+static int dictionary_walker_name_value(avl *a, int (*callback)(char *name, void *entry, void *data), void *data) {
+ int total = 0, ret = 0;
+
+ if(a->avl_link[0]) {
+ ret = dictionary_walker_name_value(a->avl_link[0], callback, data);
+ if(ret < 0) return ret;
+ total += ret;
+ }
+
+ ret = callback(((NAME_VALUE *)a)->name, ((NAME_VALUE *)a)->value, data);
+ if(ret < 0) return ret;
+ total += ret;
+
+ if(a->avl_link[1]) {
+ ret = dictionary_walker_name_value(a->avl_link[1], callback, data);
+ if (ret < 0) return ret;
+ total += ret;
+ }
+
+ return total;
+}
+
+int dictionary_get_all_name_value(DICTIONARY *dict, int (*callback)(char *name, void *entry, void *data), void *data) {
+ int ret = 0;
+
+ dictionary_read_lock(dict);
+
+ if(likely(dict->values_index.root))
+ ret = dictionary_walker_name_value(dict->values_index.root, callback, data);
+
+ dictionary_unlock(dict);
+
+ return ret;
+}
diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h
new file mode 100644
index 0000000..fc24ec2
--- /dev/null
+++ b/libnetdata/dictionary/dictionary.h
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_DICTIONARY_H
+#define NETDATA_DICTIONARY_H 1
+
+#include "../libnetdata.h"
+
+struct dictionary_stats {
+ unsigned long long inserts;
+ unsigned long long deletes;
+ unsigned long long searches;
+ unsigned long long entries;
+};
+
+typedef struct name_value {
+ avl avl_node; // the index - this has to be first!
+
+ uint32_t hash; // a simple hash to speed up searching
+ // we first compare hashes, and only if the hashes are equal we do string comparisons
+
+ char *name;
+ void *value;
+} NAME_VALUE;
+
+typedef struct dictionary {
+ avl_tree_type values_index;
+
+ uint8_t flags;
+
+ struct dictionary_stats *stats;
+ netdata_rwlock_t *rwlock;
+} DICTIONARY;
+
+#define DICTIONARY_FLAG_DEFAULT 0x00000000
+#define DICTIONARY_FLAG_SINGLE_THREADED 0x00000001
+#define DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE 0x00000002
+#define DICTIONARY_FLAG_NAME_LINK_DONT_CLONE 0x00000004
+#define DICTIONARY_FLAG_WITH_STATISTICS 0x00000008
+
+extern DICTIONARY *dictionary_create(uint8_t flags);
+extern void dictionary_destroy(DICTIONARY *dict);
+extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) NEVERNULL;
+extern void *dictionary_get(DICTIONARY *dict, const char *name);
+extern int dictionary_del(DICTIONARY *dict, const char *name);
+
+extern int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *d), void *data);
+extern int dictionary_get_all_name_value(DICTIONARY *dict, int (*callback)(char *name, void *entry, void *d), void *data);
+
+#endif /* NETDATA_DICTIONARY_H */
diff --git a/libnetdata/ebpf/Makefile.am b/libnetdata/ebpf/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/ebpf/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/ebpf/README.md b/libnetdata/ebpf/README.md
new file mode 100644
index 0000000..09c6607
--- /dev/null
+++ b/libnetdata/ebpf/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/ebpf/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Febpf%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c
new file mode 100644
index 0000000..a9ff21f
--- /dev/null
+++ b/libnetdata/ebpf/ebpf.c
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <dlfcn.h>
+#include <sys/utsname.h>
+
+#include "../libnetdata.h"
+
+/*
+static int clean_kprobe_event(FILE *out, char *filename, char *father_pid, netdata_ebpf_events_t *ptr)
+{
+ int fd = open(filename, O_WRONLY | O_APPEND, 0);
+ if (fd < 0) {
+ if (out) {
+ fprintf(out, "Cannot open %s : %s\n", filename, strerror(errno));
+ }
+ return 1;
+ }
+
+ char cmd[1024];
+ int length = snprintf(cmd, 1023, "-:kprobes/%c_netdata_%s_%s", ptr->type, ptr->name, father_pid);
+ int ret = 0;
+ if (length > 0) {
+ ssize_t written = write(fd, cmd, strlen(cmd));
+ if (written < 0) {
+ if (out) {
+ fprintf(
+ out, "Cannot remove the event (%d, %d) '%s' from %s : %s\n", getppid(), getpid(), cmd, filename,
+ strerror((int)errno));
+ }
+ ret = 1;
+ }
+ }
+
+ close(fd);
+
+ return ret;
+}
+
+int clean_kprobe_events(FILE *out, int pid, netdata_ebpf_events_t *ptr)
+{
+ debug(D_EXIT, "Cleaning parent process events.");
+ char filename[FILENAME_MAX + 1];
+ snprintf(filename, FILENAME_MAX, "%s%s", NETDATA_DEBUGFS, "kprobe_events");
+
+ char removeme[16];
+ snprintf(removeme, 15, "%d", pid);
+
+ int i;
+ for (i = 0; ptr[i].name; i++) {
+ if (clean_kprobe_event(out, filename, removeme, &ptr[i])) {
+ break;
+ }
+ }
+
+ return 0;
+}
+*/
+
+//----------------------------------------------------------------------------------------------------------------------
+
+int get_kernel_version(char *out, int size)
+{
+ char major[16], minor[16], patch[16];
+ char ver[VERSION_STRING_LEN];
+ char *version = ver;
+
+ out[0] = '\0';
+ int fd = open("/proc/sys/kernel/osrelease", O_RDONLY);
+ if (fd < 0)
+ return -1;
+
+ ssize_t len = read(fd, ver, sizeof(ver));
+ if (len < 0) {
+ close(fd);
+ return -1;
+ }
+
+ close(fd);
+
+ char *move = major;
+ while (*version && *version != '.')
+ *move++ = *version++;
+ *move = '\0';
+
+ version++;
+ move = minor;
+ while (*version && *version != '.')
+ *move++ = *version++;
+ *move = '\0';
+
+ if (*version)
+ version++;
+ else
+ return -1;
+
+ move = patch;
+ while (*version && *version != '\n')
+ *move++ = *version++;
+ *move = '\0';
+
+ fd = snprintf(out, (size_t)size, "%s.%s.%s", major, minor, patch);
+ if (fd > size)
+ error("The buffer to store kernel version is not smaller than necessary.");
+
+ return ((int)(str2l(major) * 65536) + (int)(str2l(minor) * 256) + (int)str2l(patch));
+}
+
+int get_redhat_release()
+{
+ char buffer[VERSION_STRING_LEN + 1];
+ int major, minor;
+ FILE *fp = fopen("/etc/redhat-release", "r");
+
+ if (fp) {
+ major = 0;
+ minor = -1;
+ size_t length = fread(buffer, sizeof(char), VERSION_STRING_LEN, fp);
+ if (length > 4) {
+ buffer[length] = '\0';
+ char *end = strchr(buffer, '.');
+ char *start;
+ if (end) {
+ *end = 0x0;
+
+ if (end > buffer) {
+ start = end - 1;
+
+ major = strtol(start, NULL, 10);
+ start = ++end;
+
+ end++;
+ if (end) {
+ end = 0x00;
+ minor = strtol(start, NULL, 10);
+ } else {
+ minor = -1;
+ }
+ }
+ }
+ }
+
+ fclose(fp);
+ return ((major * 256) + minor);
+ } else {
+ return -1;
+ }
+}
+
+/**
+ * Check if the kernel is in a list of rejected ones
+ *
+ * @return Returns 1 if the kernel is rejected, 0 otherwise.
+ */
+static int kernel_is_rejected()
+{
+ // Get kernel version from system
+ char version_string[VERSION_STRING_LEN + 1];
+ int version_string_len = 0;
+
+ if (read_file("/proc/version_signature", version_string, VERSION_STRING_LEN)) {
+ if (read_file("/proc/version", version_string, VERSION_STRING_LEN)) {
+ struct utsname uname_buf;
+ if (!uname(&uname_buf)) {
+ info("Cannot check kernel version");
+ return 0;
+ }
+ version_string_len =
+ snprintfz(version_string, VERSION_STRING_LEN, "%s %s", uname_buf.release, uname_buf.version);
+ }
+ }
+
+ if (!version_string_len)
+ version_string_len = strlen(version_string);
+
+ // Open a file with a list of rejected kernels
+ char *config_dir = getenv("NETDATA_USER_CONFIG_DIR");
+ if (config_dir == NULL) {
+ config_dir = CONFIG_DIR;
+ }
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/%s", config_dir, EBPF_KERNEL_REJECT_LIST_FILE);
+ FILE *kernel_reject_list = fopen(filename, "r");
+
+ if (!kernel_reject_list) {
+ config_dir = getenv("NETDATA_STOCK_CONFIG_DIR");
+ if (config_dir == NULL) {
+ config_dir = LIBCONFIG_DIR;
+ }
+
+ snprintfz(filename, FILENAME_MAX, "%s/%s", config_dir, EBPF_KERNEL_REJECT_LIST_FILE);
+ kernel_reject_list = fopen(filename, "r");
+
+ if (!kernel_reject_list)
+ return 0;
+ }
+
+ // Find if the kernel is in the reject list
+ char *reject_string = NULL;
+ size_t buf_len = 0;
+ ssize_t reject_string_len;
+ while ((reject_string_len = getline(&reject_string, &buf_len, kernel_reject_list) - 1) > 0) {
+ if (version_string_len >= reject_string_len) {
+ if (!strncmp(version_string, reject_string, reject_string_len)) {
+ info("A buggy kernel is detected");
+ fclose(kernel_reject_list);
+ freez(reject_string);
+ return 1;
+ }
+ }
+ }
+
+ fclose(kernel_reject_list);
+ freez(reject_string);
+
+ return 0;
+}
+
+static int has_ebpf_kernel_version(int version)
+{
+ if (kernel_is_rejected())
+ return 0;
+
+ // Kernel 4.11.0 or RH > 7.5
+ return (version >= NETDATA_MINIMUM_EBPF_KERNEL || get_redhat_release() >= NETDATA_MINIMUM_RH_VERSION);
+}
+
+int has_condition_to_run(int version)
+{
+ if (!has_ebpf_kernel_version(version))
+ return 0;
+
+ return 1;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+char *ebpf_kernel_suffix(int version, int isrh)
+{
+ if (isrh) {
+ if (version >= NETDATA_EBPF_KERNEL_4_11)
+ return "4.18";
+ else
+ return "3.10";
+ } else {
+ if (version >= NETDATA_EBPF_KERNEL_5_10)
+ return "5.10";
+ else if (version >= NETDATA_EBPF_KERNEL_4_17)
+ return "5.4";
+ else if (version >= NETDATA_EBPF_KERNEL_4_15)
+ return "4.16";
+ else if (version >= NETDATA_EBPF_KERNEL_4_11)
+ return "4.14";
+ }
+
+ return NULL;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+int ebpf_update_kernel(ebpf_data_t *ed)
+{
+ char *kernel = ebpf_kernel_suffix(ed->running_on_kernel, (ed->isrh < 0) ? 0 : 1);
+ size_t length = strlen(kernel);
+ strncpyz(ed->kernel_string, kernel, length);
+ ed->kernel_string[length] = '\0';
+
+ return 0;
+}
+
+static int select_file(char *name, const char *program, size_t length, int mode, char *kernel_string)
+{
+ int ret = -1;
+ if (!mode)
+ ret = snprintf(name, length, "rnetdata_ebpf_%s.%s.o", program, kernel_string);
+ else if (mode == 1)
+ ret = snprintf(name, length, "dnetdata_ebpf_%s.%s.o", program, kernel_string);
+ else if (mode == 2)
+ ret = snprintf(name, length, "pnetdata_ebpf_%s.%s.o", program, kernel_string);
+
+ return ret;
+}
+
+struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, char *kernel_string, struct bpf_object **obj, int *map_fd)
+{
+ char lpath[4096];
+ char lname[128];
+ int prog_fd;
+
+ int test = select_file(lname, em->thread_name, (size_t)127, em->mode, kernel_string);
+ if (test < 0 || test > 127)
+ return NULL;
+
+ snprintf(lpath, 4096, "%s/%s", plugins_dir, lname);
+ if (bpf_prog_load(lpath, BPF_PROG_TYPE_KPROBE, obj, &prog_fd)) {
+ em->enabled = CONFIG_BOOLEAN_NO;
+ info("Cannot load program: %s", lpath);
+ return NULL;
+ } else {
+ info("The eBPF program %s was loaded with success.", em->thread_name);
+ }
+
+ struct bpf_map *map;
+ size_t i = 0;
+ bpf_map__for_each(map, *obj)
+ {
+ map_fd[i] = bpf_map__fd(map);
+ i++;
+ }
+
+ struct bpf_program *prog;
+ struct bpf_link **links = callocz(NETDATA_MAX_PROBES , sizeof(struct bpf_link *));
+ i = 0;
+ bpf_object__for_each_program(prog, *obj)
+ {
+ links[i] = bpf_program__attach(prog);
+ i++;
+ }
+
+ return links;
+}
diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h
new file mode 100644
index 0000000..d4faccf
--- /dev/null
+++ b/libnetdata/ebpf/ebpf.h
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EBPF_H
+#define NETDATA_EBPF_H 1
+
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+
+#define NETDATA_DEBUGFS "/sys/kernel/debug/tracing/"
+
+/**
+ * The next magic number is got doing the following math:
+ * 294960 = 4*65536 + 11*256 + 0
+ *
+ * For more details, please, read /usr/include/linux/version.h
+ */
+#define NETDATA_MINIMUM_EBPF_KERNEL 264960
+
+/**
+ * The RedHat magic number was got doing:
+ *
+ * 1797 = 7*256 + 5
+ *
+ * For more details, please, read /usr/include/linux/version.h
+ * in any Red Hat installation.
+ */
+#define NETDATA_MINIMUM_RH_VERSION 1797
+
+/**
+ * 2048 = 8*256 + 0
+ */
+#define NETDATA_RH_8 2048
+
+/**
+ * Kernel 5.10
+ *
+ * 330240 = 5*65536 + 10*256
+ */
+#define NETDATA_EBPF_KERNEL_5_10 330240
+
+/**
+ * Kernel 4.17
+ *
+ * 266496 = 4*65536 + 17*256
+ */
+#define NETDATA_EBPF_KERNEL_4_17 266496
+
+/**
+ * Kernel 4.15
+ *
+ * 265984 = 4*65536 + 15*256
+ */
+#define NETDATA_EBPF_KERNEL_4_15 265984
+
+/**
+ * Kernel 4.11
+ *
+ * 264960 = 4*65536 + 15*256
+ */
+#define NETDATA_EBPF_KERNEL_4_11 264960
+
+#define VERSION_STRING_LEN 256
+#define EBPF_KERNEL_REJECT_LIST_FILE "ebpf_kernel_reject_list.txt"
+
+typedef struct ebpf_data {
+ int *map_fd;
+
+ char *kernel_string;
+ uint32_t running_on_kernel;
+ int isrh;
+} ebpf_data_t;
+
+typedef enum {
+ MODE_RETURN = 0, // This attaches kprobe when the function returns
+ MODE_DEVMODE, // This stores log given description about the errors raised
+ MODE_ENTRY // This attaches kprobe when the function is called
+} netdata_run_mode_t;
+
+typedef struct ebpf_module {
+ const char *thread_name;
+ const char *config_name;
+ int enabled;
+ void *(*start_routine)(void *);
+ int update_time;
+ int global_charts;
+ int apps_charts;
+ netdata_run_mode_t mode;
+ uint32_t thread_id;
+ int optional;
+} ebpf_module_t;
+
+#define NETDATA_MAX_PROBES 64
+
+extern int get_kernel_version(char *out, int size);
+extern int get_redhat_release();
+extern int has_condition_to_run(int version);
+extern char *ebpf_kernel_suffix(int version, int isrh);
+extern int ebpf_update_kernel(ebpf_data_t *ef);
+extern struct bpf_link **ebpf_load_program(char *plugins_dir,
+ ebpf_module_t *em,
+ char *kernel_string,
+ struct bpf_object **obj,
+ int *map_fd);
+
+#endif /* NETDATA_EBPF_H */
diff --git a/libnetdata/eval/Makefile.am b/libnetdata/eval/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/eval/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/eval/README.md b/libnetdata/eval/README.md
new file mode 100644
index 0000000..72b4089
--- /dev/null
+++ b/libnetdata/eval/README.md
@@ -0,0 +1 @@
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Feval%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/eval/eval.c b/libnetdata/eval/eval.c
new file mode 100644
index 0000000..b53b070
--- /dev/null
+++ b/libnetdata/eval/eval.c
@@ -0,0 +1,1190 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+// ----------------------------------------------------------------------------
+// data structures for storing the parsed expression in memory
+
+typedef struct eval_value {
+ int type;
+
+ union {
+ calculated_number number;
+ EVAL_VARIABLE *variable;
+ struct eval_node *expression;
+ };
+} EVAL_VALUE;
+
+typedef struct eval_node {
+ int id;
+ unsigned char operator;
+ int precedence;
+
+ int count;
+ EVAL_VALUE ops[];
+} EVAL_NODE;
+
+// these are used for EVAL_NODE.operator
+// they are used as internal IDs to identify an operator
+// THEY ARE NOT USED FOR PARSING OPERATORS LIKE THAT
+#define EVAL_OPERATOR_NOP '\0'
+#define EVAL_OPERATOR_EXPRESSION_OPEN '('
+#define EVAL_OPERATOR_EXPRESSION_CLOSE ')'
+#define EVAL_OPERATOR_NOT '!'
+#define EVAL_OPERATOR_PLUS '+'
+#define EVAL_OPERATOR_MINUS '-'
+#define EVAL_OPERATOR_AND '&'
+#define EVAL_OPERATOR_OR '|'
+#define EVAL_OPERATOR_GREATER_THAN_OR_EQUAL 'G'
+#define EVAL_OPERATOR_LESS_THAN_OR_EQUAL 'L'
+#define EVAL_OPERATOR_NOT_EQUAL '~'
+#define EVAL_OPERATOR_EQUAL '='
+#define EVAL_OPERATOR_LESS '<'
+#define EVAL_OPERATOR_GREATER '>'
+#define EVAL_OPERATOR_MULTIPLY '*'
+#define EVAL_OPERATOR_DIVIDE '/'
+#define EVAL_OPERATOR_SIGN_PLUS 'P'
+#define EVAL_OPERATOR_SIGN_MINUS 'M'
+#define EVAL_OPERATOR_ABS 'A'
+#define EVAL_OPERATOR_IF_THEN_ELSE '?'
+
+// ----------------------------------------------------------------------------
+// forward function definitions
+
+static inline void eval_node_free(EVAL_NODE *op);
+static inline EVAL_NODE *parse_full_expression(const char **string, int *error);
+static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error);
+static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
+static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error);
+static inline void print_parsed_as_constant(BUFFER *out, calculated_number n);
+
+// ----------------------------------------------------------------------------
+// evaluation of expressions
+
+static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) {
+ static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0;
+ calculated_number n;
+
+ if(unlikely(this_hash == 0)) {
+ this_hash = simple_hash("this");
+ now_hash = simple_hash("now");
+ after_hash = simple_hash("after");
+ before_hash = simple_hash("before");
+ status_hash = simple_hash("status");
+ removed_hash = simple_hash("REMOVED");
+ uninitialized_hash = simple_hash("UNINITIALIZED");
+ undefined_hash = simple_hash("UNDEFINED");
+ clear_hash = simple_hash("CLEAR");
+ warning_hash = simple_hash("WARNING");
+ critical_hash = simple_hash("CRITICAL");
+ }
+
+ if(unlikely(v->hash == this_hash && !strcmp(v->name, "this"))) {
+ n = (exp->myself)?*exp->myself:NAN;
+ buffer_strcat(exp->error_msg, "[ $this = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == after_hash && !strcmp(v->name, "after"))) {
+ n = (exp->after && *exp->after)?*exp->after:NAN;
+ buffer_strcat(exp->error_msg, "[ $after = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == before_hash && !strcmp(v->name, "before"))) {
+ n = (exp->before && *exp->before)?*exp->before:NAN;
+ buffer_strcat(exp->error_msg, "[ $before = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == now_hash && !strcmp(v->name, "now"))) {
+ n = now_realtime_sec();
+ buffer_strcat(exp->error_msg, "[ $now = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == status_hash && !strcmp(v->name, "status"))) {
+ n = (exp->status)?*exp->status:RRDCALC_STATUS_UNINITIALIZED;
+ buffer_strcat(exp->error_msg, "[ $status = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == removed_hash && !strcmp(v->name, "REMOVED"))) {
+ n = RRDCALC_STATUS_REMOVED;
+ buffer_strcat(exp->error_msg, "[ $REMOVED = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == uninitialized_hash && !strcmp(v->name, "UNINITIALIZED"))) {
+ n = RRDCALC_STATUS_UNINITIALIZED;
+ buffer_strcat(exp->error_msg, "[ $UNINITIALIZED = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == undefined_hash && !strcmp(v->name, "UNDEFINED"))) {
+ n = RRDCALC_STATUS_UNDEFINED;
+ buffer_strcat(exp->error_msg, "[ $UNDEFINED = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == clear_hash && !strcmp(v->name, "CLEAR"))) {
+ n = RRDCALC_STATUS_CLEAR;
+ buffer_strcat(exp->error_msg, "[ $CLEAR = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == warning_hash && !strcmp(v->name, "WARNING"))) {
+ n = RRDCALC_STATUS_WARNING;
+ buffer_strcat(exp->error_msg, "[ $WARNING = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(unlikely(v->hash == critical_hash && !strcmp(v->name, "CRITICAL"))) {
+ n = RRDCALC_STATUS_CRITICAL;
+ buffer_strcat(exp->error_msg, "[ $CRITICAL = ");
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ if(exp->rrdcalc && health_variable_lookup(v->name, v->hash, exp->rrdcalc, &n)) {
+ buffer_sprintf(exp->error_msg, "[ ${%s} = ", v->name);
+ print_parsed_as_constant(exp->error_msg, n);
+ buffer_strcat(exp->error_msg, " ] ");
+ return n;
+ }
+
+ *error = EVAL_ERROR_UNKNOWN_VARIABLE;
+ buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", v->name);
+ return NAN;
+}
+
+static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) {
+ calculated_number n;
+
+ switch(v->type) {
+ case EVAL_VALUE_EXPRESSION:
+ n = eval_node(exp, v->expression, error);
+ break;
+
+ case EVAL_VALUE_NUMBER:
+ n = v->number;
+ break;
+
+ case EVAL_VALUE_VARIABLE:
+ n = eval_variable(exp, v->variable, error);
+ break;
+
+ default:
+ *error = EVAL_ERROR_INVALID_VALUE;
+ n = 0;
+ break;
+ }
+
+ return n;
+}
+
+static inline int is_true(calculated_number n) {
+ if(isnan(n)) return 0;
+ if(isinf(n)) return 1;
+ if(n == 0) return 0;
+ return 1;
+}
+
+calculated_number eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ return is_true(eval_value(exp, &op->ops[0], error)) && is_true(eval_value(exp, &op->ops[1], error));
+}
+calculated_number eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ return is_true(eval_value(exp, &op->ops[0], error)) || is_true(eval_value(exp, &op->ops[1], error));
+}
+calculated_number eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ return isgreaterequal(n1, n2);
+}
+calculated_number eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ return islessequal(n1, n2);
+}
+calculated_number eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ if(isnan(n1) && isnan(n2)) return 1;
+ if(isinf(n1) && isinf(n2)) return 1;
+ if(isnan(n1) || isnan(n2)) return 0;
+ if(isinf(n1) || isinf(n2)) return 0;
+ return calculated_number_equal(n1, n2);
+}
+calculated_number eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ return !eval_equal(exp, op, error);
+}
+calculated_number eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ return isless(n1, n2);
+}
+calculated_number eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ return isgreater(n1, n2);
+}
+calculated_number eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ if(isnan(n1) || isnan(n2)) return NAN;
+ if(isinf(n1) || isinf(n2)) return INFINITY;
+ return n1 + n2;
+}
+calculated_number eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ if(isnan(n1) || isnan(n2)) return NAN;
+ if(isinf(n1) || isinf(n2)) return INFINITY;
+ return n1 - n2;
+}
+calculated_number eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ if(isnan(n1) || isnan(n2)) return NAN;
+ if(isinf(n1) || isinf(n2)) return INFINITY;
+ return n1 * n2;
+}
+calculated_number eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ calculated_number n2 = eval_value(exp, &op->ops[1], error);
+ if(isnan(n1) || isnan(n2)) return NAN;
+ if(isinf(n1) || isinf(n2)) return INFINITY;
+ return n1 / n2;
+}
+calculated_number eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ return eval_value(exp, &op->ops[0], error);
+}
+calculated_number eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ return !is_true(eval_value(exp, &op->ops[0], error));
+}
+calculated_number eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ return eval_value(exp, &op->ops[0], error);
+}
+calculated_number eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ if(isnan(n1)) return NAN;
+ if(isinf(n1)) return INFINITY;
+ return -n1;
+}
+calculated_number eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ calculated_number n1 = eval_value(exp, &op->ops[0], error);
+ if(isnan(n1)) return NAN;
+ if(isinf(n1)) return INFINITY;
+ return abs(n1);
+}
+calculated_number eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ if(is_true(eval_value(exp, &op->ops[0], error)))
+ return eval_value(exp, &op->ops[1], error);
+ else
+ return eval_value(exp, &op->ops[2], error);
+}
+
+static struct operator {
+ const char *print_as;
+ char precedence;
+ char parameters;
+ char isfunction;
+ calculated_number (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
+} operators[256] = {
+ // this is a random access array
+ // we always access it with a known EVAL_OPERATOR_X
+
+ [EVAL_OPERATOR_AND] = { "&&", 2, 2, 0, eval_and },
+ [EVAL_OPERATOR_OR] = { "||", 2, 2, 0, eval_or },
+ [EVAL_OPERATOR_GREATER_THAN_OR_EQUAL] = { ">=", 3, 2, 0, eval_greater_than_or_equal },
+ [EVAL_OPERATOR_LESS_THAN_OR_EQUAL] = { "<=", 3, 2, 0, eval_less_than_or_equal },
+ [EVAL_OPERATOR_NOT_EQUAL] = { "!=", 3, 2, 0, eval_not_equal },
+ [EVAL_OPERATOR_EQUAL] = { "==", 3, 2, 0, eval_equal },
+ [EVAL_OPERATOR_LESS] = { "<", 3, 2, 0, eval_less },
+ [EVAL_OPERATOR_GREATER] = { ">", 3, 2, 0, eval_greater },
+ [EVAL_OPERATOR_PLUS] = { "+", 4, 2, 0, eval_plus },
+ [EVAL_OPERATOR_MINUS] = { "-", 4, 2, 0, eval_minus },
+ [EVAL_OPERATOR_MULTIPLY] = { "*", 5, 2, 0, eval_multiply },
+ [EVAL_OPERATOR_DIVIDE] = { "/", 5, 2, 0, eval_divide },
+ [EVAL_OPERATOR_NOT] = { "!", 6, 1, 0, eval_not },
+ [EVAL_OPERATOR_SIGN_PLUS] = { "+", 6, 1, 0, eval_sign_plus },
+ [EVAL_OPERATOR_SIGN_MINUS] = { "-", 6, 1, 0, eval_sign_minus },
+ [EVAL_OPERATOR_ABS] = { "abs(",6,1, 1, eval_abs },
+ [EVAL_OPERATOR_IF_THEN_ELSE] = { "?", 7, 3, 0, eval_if_then_else },
+ [EVAL_OPERATOR_NOP] = { NULL, 8, 1, 0, eval_nop },
+ [EVAL_OPERATOR_EXPRESSION_OPEN] = { NULL, 8, 1, 0, eval_nop },
+
+ // this should exist in our evaluation list
+ [EVAL_OPERATOR_EXPRESSION_CLOSE] = { NULL, 99, 1, 0, eval_nop }
+};
+
+#define eval_precedence(operator) (operators[(unsigned char)(operator)].precedence)
+
+static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
+ if(unlikely(op->count != operators[op->operator].parameters)) {
+ *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS;
+ return 0;
+ }
+
+ calculated_number n = operators[op->operator].eval(exp, op, error);
+
+ return n;
+}
+
+// ----------------------------------------------------------------------------
+// parsed-as generation
+
+static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *error) {
+ (void)error;
+ buffer_sprintf(out, "${%s}", v->name);
+}
+
+static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) {
+ if(unlikely(isnan(n))) {
+ buffer_strcat(out, "nan");
+ return;
+ }
+
+ if(unlikely(isinf(n))) {
+ buffer_strcat(out, "inf");
+ return;
+ }
+
+ char b[100+1], *s;
+ snprintfz(b, 100, CALCULATED_NUMBER_FORMAT, n);
+
+ s = &b[strlen(b) - 1];
+ while(s > b && *s == '0') {
+ *s ='\0';
+ s--;
+ }
+
+ if(s > b && *s == '.')
+ *s = '\0';
+
+ buffer_strcat(out, b);
+}
+
+static inline void print_parsed_as_value(BUFFER *out, EVAL_VALUE *v, int *error) {
+ switch(v->type) {
+ case EVAL_VALUE_EXPRESSION:
+ print_parsed_as_node(out, v->expression, error);
+ break;
+
+ case EVAL_VALUE_NUMBER:
+ print_parsed_as_constant(out, v->number);
+ break;
+
+ case EVAL_VALUE_VARIABLE:
+ print_parsed_as_variable(out, v->variable, error);
+ break;
+
+ default:
+ *error = EVAL_ERROR_INVALID_VALUE;
+ break;
+ }
+}
+
+static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error) {
+ if(unlikely(op->count != operators[op->operator].parameters)) {
+ *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS;
+ return;
+ }
+
+ if(operators[op->operator].parameters == 1) {
+
+ if(operators[op->operator].print_as)
+ buffer_sprintf(out, "%s", operators[op->operator].print_as);
+
+ //if(op->operator == EVAL_OPERATOR_EXPRESSION_OPEN)
+ // buffer_strcat(out, "(");
+
+ print_parsed_as_value(out, &op->ops[0], error);
+
+ //if(op->operator == EVAL_OPERATOR_EXPRESSION_OPEN)
+ // buffer_strcat(out, ")");
+ }
+
+ else if(operators[op->operator].parameters == 2) {
+ buffer_strcat(out, "(");
+ print_parsed_as_value(out, &op->ops[0], error);
+
+ if(operators[op->operator].print_as)
+ buffer_sprintf(out, " %s ", operators[op->operator].print_as);
+
+ print_parsed_as_value(out, &op->ops[1], error);
+ buffer_strcat(out, ")");
+ }
+ else if(op->operator == EVAL_OPERATOR_IF_THEN_ELSE && operators[op->operator].parameters == 3) {
+ buffer_strcat(out, "(");
+ print_parsed_as_value(out, &op->ops[0], error);
+
+ if(operators[op->operator].print_as)
+ buffer_sprintf(out, " %s ", operators[op->operator].print_as);
+
+ print_parsed_as_value(out, &op->ops[1], error);
+ buffer_strcat(out, " : ");
+ print_parsed_as_value(out, &op->ops[2], error);
+ buffer_strcat(out, ")");
+ }
+
+ if(operators[op->operator].isfunction)
+ buffer_strcat(out, ")");
+}
+
+// ----------------------------------------------------------------------------
+// parsing expressions
+
+// skip spaces
+static inline void skip_spaces(const char **string) {
+ const char *s = *string;
+ while(isspace(*s)) s++;
+ *string = s;
+}
+
+// what character can appear just after an operator keyword
+// like NOT AND OR ?
+static inline int isoperatorterm_word(const char s) {
+ if(isspace(s) || s == '(' || s == '$' || s == '!' || s == '-' || s == '+' || isdigit(s) || !s)
+ return 1;
+
+ return 0;
+}
+
+// what character can appear just after an operator symbol?
+static inline int isoperatorterm_symbol(const char s) {
+ if(isoperatorterm_word(s) || isalpha(s))
+ return 1;
+
+ return 0;
+}
+
+// return 1 if the character should never appear in a variable
+static inline int isvariableterm(const char s) {
+ if(isalnum(s) || s == '.' || s == '_')
+ return 0;
+
+ return 1;
+}
+
+// ----------------------------------------------------------------------------
+// parse operators
+
+static inline int parse_and(const char **string) {
+ const char *s = *string;
+
+ // AND
+ if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') && isoperatorterm_word(s[3])) {
+ *string = &s[4];
+ return 1;
+ }
+
+ // &&
+ if(s[0] == '&' && s[1] == '&' && isoperatorterm_symbol(s[2])) {
+ *string = &s[2];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_or(const char **string) {
+ const char *s = *string;
+
+ // OR
+ if((s[0] == 'O' || s[0] == 'o') && (s[1] == 'R' || s[1] == 'r') && isoperatorterm_word(s[2])) {
+ *string = &s[3];
+ return 1;
+ }
+
+ // ||
+ if(s[0] == '|' && s[1] == '|' && isoperatorterm_symbol(s[2])) {
+ *string = &s[2];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_greater_than_or_equal(const char **string) {
+ const char *s = *string;
+
+ // >=
+ if(s[0] == '>' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+ *string = &s[2];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_less_than_or_equal(const char **string) {
+ const char *s = *string;
+
+ // <=
+ if (s[0] == '<' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+ *string = &s[2];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_greater(const char **string) {
+ const char *s = *string;
+
+ // >
+ if(s[0] == '>' && isoperatorterm_symbol(s[1])) {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_less(const char **string) {
+ const char *s = *string;
+
+ // <
+ if(s[0] == '<' && isoperatorterm_symbol(s[1])) {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_equal(const char **string) {
+ const char *s = *string;
+
+ // ==
+ if(s[0] == '=' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+ *string = &s[2];
+ return 1;
+ }
+
+ // =
+ if(s[0] == '=' && isoperatorterm_symbol(s[1])) {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_not_equal(const char **string) {
+ const char *s = *string;
+
+ // !=
+ if(s[0] == '!' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+ *string = &s[2];
+ return 1;
+ }
+
+ // <>
+ if(s[0] == '<' && s[1] == '>' && isoperatorterm_symbol(s[2])) {
+ *string = &s[2];
+ }
+
+ return 0;
+}
+
+static inline int parse_not(const char **string) {
+ const char *s = *string;
+
+ // NOT
+ if((s[0] == 'N' || s[0] == 'n') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') && isoperatorterm_word(s[3])) {
+ *string = &s[3];
+ return 1;
+ }
+
+ if(s[0] == '!') {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_multiply(const char **string) {
+ const char *s = *string;
+
+ // *
+ if(s[0] == '*' && isoperatorterm_symbol(s[1])) {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_divide(const char **string) {
+ const char *s = *string;
+
+ // /
+ if(s[0] == '/' && isoperatorterm_symbol(s[1])) {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_minus(const char **string) {
+ const char *s = *string;
+
+ // -
+ if(s[0] == '-' && isoperatorterm_symbol(s[1])) {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_plus(const char **string) {
+ const char *s = *string;
+
+ // +
+ if(s[0] == '+' && isoperatorterm_symbol(s[1])) {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_open_subexpression(const char **string) {
+ const char *s = *string;
+
+ // (
+ if(s[0] == '(') {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+#define parse_close_function(x) parse_close_subexpression(x)
+
+static inline int parse_close_subexpression(const char **string) {
+ const char *s = *string;
+
+ // )
+ if(s[0] == ')') {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_variable(const char **string, char *buffer, size_t len) {
+ const char *s = *string;
+
+ // $
+ if(*s == '$') {
+ size_t i = 0;
+ s++;
+
+ if(*s == '{') {
+ // ${variable_name}
+
+ s++;
+ while (*s && *s != '}' && i < len)
+ buffer[i++] = *s++;
+
+ if(*s == '}')
+ s++;
+ }
+ else {
+ // $variable_name
+
+ while (*s && !isvariableterm(*s) && i < len)
+ buffer[i++] = *s++;
+ }
+
+ buffer[i] = '\0';
+
+ if (buffer[0]) {
+ *string = s;
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static inline int parse_constant(const char **string, calculated_number *number) {
+ char *end = NULL;
+ calculated_number n = str2ld(*string, &end);
+ if(unlikely(!end || *string == end)) {
+ *number = 0;
+ return 0;
+ }
+ *number = n;
+ *string = end;
+ return 1;
+}
+
+static inline int parse_abs(const char **string) {
+ const char *s = *string;
+
+ // ABS
+ if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'B' || s[1] == 'b') && (s[2] == 'S' || s[2] == 's') && s[3] == '(') {
+ *string = &s[3];
+ return 1;
+ }
+
+ return 0;
+}
+
+static inline int parse_if_then_else(const char **string) {
+ const char *s = *string;
+
+ // ?
+ if(s[0] == '?') {
+ *string = &s[1];
+ return 1;
+ }
+
+ return 0;
+}
+
+static struct operator_parser {
+ unsigned char id;
+ int (*parse)(const char **);
+} operator_parsers[] = {
+ // the order in this list is important!
+ // the first matching will be used
+ // so place the longer of overlapping ones
+ // at the top
+
+ { EVAL_OPERATOR_AND, parse_and },
+ { EVAL_OPERATOR_OR, parse_or },
+ { EVAL_OPERATOR_GREATER_THAN_OR_EQUAL, parse_greater_than_or_equal },
+ { EVAL_OPERATOR_LESS_THAN_OR_EQUAL, parse_less_than_or_equal },
+ { EVAL_OPERATOR_NOT_EQUAL, parse_not_equal },
+ { EVAL_OPERATOR_EQUAL, parse_equal },
+ { EVAL_OPERATOR_LESS, parse_less },
+ { EVAL_OPERATOR_GREATER, parse_greater },
+ { EVAL_OPERATOR_PLUS, parse_plus },
+ { EVAL_OPERATOR_MINUS, parse_minus },
+ { EVAL_OPERATOR_MULTIPLY, parse_multiply },
+ { EVAL_OPERATOR_DIVIDE, parse_divide },
+ { EVAL_OPERATOR_IF_THEN_ELSE, parse_if_then_else },
+
+ /* we should not put in this list the following:
+ *
+ * - NOT
+ * - (
+ * - )
+ *
+ * these are handled in code
+ */
+
+ // termination
+ { EVAL_OPERATOR_NOP, NULL }
+};
+
+static inline unsigned char parse_operator(const char **string, int *precedence) {
+ skip_spaces(string);
+
+ int i;
+ for(i = 0 ; operator_parsers[i].parse != NULL ; i++)
+ if(operator_parsers[i].parse(string)) {
+ if(precedence) *precedence = eval_precedence(operator_parsers[i].id);
+ return operator_parsers[i].id;
+ }
+
+ return EVAL_OPERATOR_NOP;
+}
+
+// ----------------------------------------------------------------------------
+// memory management
+
+static inline EVAL_NODE *eval_node_alloc(int count) {
+ static int id = 1;
+
+ EVAL_NODE *op = callocz(1, sizeof(EVAL_NODE) + (sizeof(EVAL_VALUE) * count));
+
+ op->id = id++;
+ op->operator = EVAL_OPERATOR_NOP;
+ op->precedence = eval_precedence(EVAL_OPERATOR_NOP);
+ op->count = count;
+ return op;
+}
+
+static inline void eval_node_set_value_to_node(EVAL_NODE *op, int pos, EVAL_NODE *value) {
+ if(pos >= op->count)
+ fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
+
+ op->ops[pos].type = EVAL_VALUE_EXPRESSION;
+ op->ops[pos].expression = value;
+}
+
+static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, calculated_number value) {
+ if(pos >= op->count)
+ fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
+
+ op->ops[pos].type = EVAL_VALUE_NUMBER;
+ op->ops[pos].number = value;
+}
+
+static inline void eval_node_set_value_to_variable(EVAL_NODE *op, int pos, const char *variable) {
+ if(pos >= op->count)
+ fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
+
+ op->ops[pos].type = EVAL_VALUE_VARIABLE;
+ op->ops[pos].variable = callocz(1, sizeof(EVAL_VARIABLE));
+ op->ops[pos].variable->name = strdupz(variable);
+ op->ops[pos].variable->hash = simple_hash(op->ops[pos].variable->name);
+}
+
+static inline void eval_variable_free(EVAL_VARIABLE *v) {
+ freez(v->name);
+ freez(v);
+}
+
+static inline void eval_value_free(EVAL_VALUE *v) {
+ switch(v->type) {
+ case EVAL_VALUE_EXPRESSION:
+ eval_node_free(v->expression);
+ break;
+
+ case EVAL_VALUE_VARIABLE:
+ eval_variable_free(v->variable);
+ break;
+
+ default:
+ break;
+ }
+}
+
+static inline void eval_node_free(EVAL_NODE *op) {
+ if(op->count) {
+ int i;
+ for(i = op->count - 1; i >= 0 ;i--)
+ eval_value_free(&op->ops[i]);
+ }
+
+ freez(op);
+}
+
+// ----------------------------------------------------------------------------
+// the parsing logic
+
+// helper function to avoid allocations all over the place
+static inline EVAL_NODE *parse_next_operand_given_its_operator(const char **string, unsigned char operator_type, int *error) {
+ EVAL_NODE *sub = parse_one_full_operand(string, error);
+ if(!sub) return NULL;
+
+ EVAL_NODE *op = eval_node_alloc(1);
+ op->operator = operator_type;
+ eval_node_set_value_to_node(op, 0, sub);
+ return op;
+}
+
+// parse a full operand, including its sign or other associative operator (e.g. NOT)
+static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error) {
+ char variable_buffer[EVAL_MAX_VARIABLE_NAME_LENGTH + 1];
+ EVAL_NODE *op1 = NULL;
+ calculated_number number;
+
+ *error = EVAL_ERROR_OK;
+
+ skip_spaces(string);
+ if(!(**string)) {
+ *error = EVAL_ERROR_MISSING_OPERAND;
+ return NULL;
+ }
+
+ if(parse_not(string)) {
+ op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_NOT, error);
+ op1->precedence = eval_precedence(EVAL_OPERATOR_NOT);
+ }
+ else if(parse_plus(string)) {
+ op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_SIGN_PLUS, error);
+ op1->precedence = eval_precedence(EVAL_OPERATOR_SIGN_PLUS);
+ }
+ else if(parse_minus(string)) {
+ op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_SIGN_MINUS, error);
+ op1->precedence = eval_precedence(EVAL_OPERATOR_SIGN_MINUS);
+ }
+ else if(parse_abs(string)) {
+ op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_ABS, error);
+ op1->precedence = eval_precedence(EVAL_OPERATOR_ABS);
+ }
+ else if(parse_open_subexpression(string)) {
+ EVAL_NODE *sub = parse_full_expression(string, error);
+ if(sub) {
+ op1 = eval_node_alloc(1);
+ op1->operator = EVAL_OPERATOR_EXPRESSION_OPEN;
+ op1->precedence = eval_precedence(EVAL_OPERATOR_EXPRESSION_OPEN);
+ eval_node_set_value_to_node(op1, 0, sub);
+ if(!parse_close_subexpression(string)) {
+ *error = EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION;
+ eval_node_free(op1);
+ return NULL;
+ }
+ }
+ }
+ else if(parse_variable(string, variable_buffer, EVAL_MAX_VARIABLE_NAME_LENGTH)) {
+ op1 = eval_node_alloc(1);
+ op1->operator = EVAL_OPERATOR_NOP;
+ eval_node_set_value_to_variable(op1, 0, variable_buffer);
+ }
+ else if(parse_constant(string, &number)) {
+ op1 = eval_node_alloc(1);
+ op1->operator = EVAL_OPERATOR_NOP;
+ eval_node_set_value_to_constant(op1, 0, number);
+ }
+ else if(**string)
+ *error = EVAL_ERROR_UNKNOWN_OPERAND;
+ else
+ *error = EVAL_ERROR_MISSING_OPERAND;
+
+ return op1;
+}
+
+// parse an operator and the rest of the expression
+// precedence processing is handled here
+static inline EVAL_NODE *parse_rest_of_expression(const char **string, int *error, EVAL_NODE *op1) {
+ EVAL_NODE *op2 = NULL;
+ unsigned char operator;
+ int precedence;
+
+ operator = parse_operator(string, &precedence);
+ skip_spaces(string);
+
+ if(operator != EVAL_OPERATOR_NOP) {
+ op2 = parse_one_full_operand(string, error);
+ if(!op2) {
+ // error is already reported
+ eval_node_free(op1);
+ return NULL;
+ }
+
+ EVAL_NODE *op = eval_node_alloc(operators[operator].parameters);
+ op->operator = operator;
+ op->precedence = precedence;
+
+ if(operator == EVAL_OPERATOR_IF_THEN_ELSE && op->count == 3) {
+ skip_spaces(string);
+
+ if(**string != ':') {
+ eval_node_free(op);
+ eval_node_free(op1);
+ eval_node_free(op2);
+ *error = EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE;
+ return NULL;
+ }
+ (*string)++;
+
+ skip_spaces(string);
+
+ EVAL_NODE *op3 = parse_one_full_operand(string, error);
+ if(!op3) {
+ eval_node_free(op);
+ eval_node_free(op1);
+ eval_node_free(op2);
+ // error is already reported
+ return NULL;
+ }
+
+ eval_node_set_value_to_node(op, 2, op3);
+ }
+
+ eval_node_set_value_to_node(op, 1, op2);
+
+ // precedence processing
+ // if this operator has a higher precedence compared to its next
+ // put the next operator on top of us (top = evaluated later)
+ // function recursion does the rest...
+ if(op->precedence > op1->precedence && op1->count == 2 && op1->operator != '(' && op1->ops[1].type == EVAL_VALUE_EXPRESSION) {
+ eval_node_set_value_to_node(op, 0, op1->ops[1].expression);
+ op1->ops[1].expression = op;
+ op = op1;
+ }
+ else
+ eval_node_set_value_to_node(op, 0, op1);
+
+ return parse_rest_of_expression(string, error, op);
+ }
+ else if(**string == ')') {
+ ;
+ }
+ else if(**string) {
+ eval_node_free(op1);
+ op1 = NULL;
+ *error = EVAL_ERROR_MISSING_OPERATOR;
+ }
+
+ return op1;
+}
+
+// high level function to parse an expression or a sub-expression
+static inline EVAL_NODE *parse_full_expression(const char **string, int *error) {
+ EVAL_NODE *op1 = parse_one_full_operand(string, error);
+ if(!op1) {
+ *error = EVAL_ERROR_MISSING_OPERAND;
+ return NULL;
+ }
+
+ return parse_rest_of_expression(string, error, op1);
+}
+
+// ----------------------------------------------------------------------------
+// public API
+
+int expression_evaluate(EVAL_EXPRESSION *expression) {
+ expression->error = EVAL_ERROR_OK;
+
+ buffer_reset(expression->error_msg);
+ expression->result = eval_node(expression, (EVAL_NODE *)expression->nodes, &expression->error);
+
+ if(unlikely(isnan(expression->result))) {
+ if(expression->error == EVAL_ERROR_OK)
+ expression->error = EVAL_ERROR_VALUE_IS_NAN;
+ }
+ else if(unlikely(isinf(expression->result))) {
+ if(expression->error == EVAL_ERROR_OK)
+ expression->error = EVAL_ERROR_VALUE_IS_INFINITE;
+ }
+ else if(unlikely(expression->error == EVAL_ERROR_UNKNOWN_VARIABLE)) {
+ // although there is an unknown variable
+ // the expression was evaluated successfully
+ expression->error = EVAL_ERROR_OK;
+ }
+
+ if(expression->error != EVAL_ERROR_OK) {
+ expression->result = NAN;
+
+ if(buffer_strlen(expression->error_msg))
+ buffer_strcat(expression->error_msg, "; ");
+
+ buffer_sprintf(expression->error_msg, "failed to evaluate expression with error %d (%s)", expression->error, expression_strerror(expression->error));
+ return 0;
+ }
+
+ return 1;
+}
+
+EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error) {
+ const char *s = string;
+ int err = EVAL_ERROR_OK;
+
+ EVAL_NODE *op = parse_full_expression(&s, &err);
+
+ if(*s) {
+ if(op) {
+ eval_node_free(op);
+ op = NULL;
+ }
+ err = EVAL_ERROR_REMAINING_GARBAGE;
+ }
+
+ if (failed_at) *failed_at = s;
+ if (error) *error = err;
+
+ if(!op) {
+ unsigned long pos = s - string + 1;
+ error("failed to parse expression '%s': %s at character %lu (i.e.: '%s').", string, expression_strerror(err), pos, s);
+ return NULL;
+ }
+
+ BUFFER *out = buffer_create(1024);
+ print_parsed_as_node(out, op, &err);
+ if(err != EVAL_ERROR_OK) {
+ error("failed to re-generate expression '%s' with reason: %s", string, expression_strerror(err));
+ eval_node_free(op);
+ buffer_free(out);
+ return NULL;
+ }
+
+ EVAL_EXPRESSION *exp = callocz(1, sizeof(EVAL_EXPRESSION));
+
+ exp->source = strdupz(string);
+ exp->parsed_as = strdupz(buffer_tostring(out));
+ buffer_free(out);
+
+ exp->error_msg = buffer_create(100);
+ exp->nodes = (void *)op;
+
+ return exp;
+}
+
+void expression_free(EVAL_EXPRESSION *expression) {
+ if(!expression) return;
+
+ if(expression->nodes) eval_node_free((EVAL_NODE *)expression->nodes);
+ freez((void *)expression->source);
+ freez((void *)expression->parsed_as);
+ buffer_free(expression->error_msg);
+ freez(expression);
+}
+
+const char *expression_strerror(int error) {
+ switch(error) {
+ case EVAL_ERROR_OK:
+ return "success";
+
+ case EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION:
+ return "missing closing parenthesis";
+
+ case EVAL_ERROR_UNKNOWN_OPERAND:
+ return "unknown operand";
+
+ case EVAL_ERROR_MISSING_OPERAND:
+ return "expected operand";
+
+ case EVAL_ERROR_MISSING_OPERATOR:
+ return "expected operator";
+
+ case EVAL_ERROR_REMAINING_GARBAGE:
+ return "remaining characters after expression";
+
+ case EVAL_ERROR_INVALID_VALUE:
+ return "invalid value structure - internal error";
+
+ case EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS:
+ return "wrong number of operands for operation - internal error";
+
+ case EVAL_ERROR_VALUE_IS_NAN:
+ return "value is unset";
+
+ case EVAL_ERROR_VALUE_IS_INFINITE:
+ return "computed value is infinite";
+
+ case EVAL_ERROR_UNKNOWN_VARIABLE:
+ return "undefined variable";
+
+ case EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE:
+ return "missing second sub-expression of inline conditional";
+
+ default:
+ return "unknown error";
+ }
+}
diff --git a/libnetdata/eval/eval.h b/libnetdata/eval/eval.h
new file mode 100644
index 0000000..fc03d7c
--- /dev/null
+++ b/libnetdata/eval/eval.h
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_EVAL_H
+#define NETDATA_EVAL_H 1
+
+#include "../libnetdata.h"
+
+#define EVAL_MAX_VARIABLE_NAME_LENGTH 300
+
+typedef enum rrdcalc_status {
+ RRDCALC_STATUS_REMOVED = -2,
+ RRDCALC_STATUS_UNDEFINED = -1,
+ RRDCALC_STATUS_UNINITIALIZED = 0,
+ RRDCALC_STATUS_CLEAR = 1,
+ RRDCALC_STATUS_RAISED = 2,
+ RRDCALC_STATUS_WARNING = 3,
+ RRDCALC_STATUS_CRITICAL = 4
+} RRDCALC_STATUS;
+
+typedef struct eval_variable {
+ char *name;
+ uint32_t hash;
+ struct eval_variable *next;
+} EVAL_VARIABLE;
+
+typedef struct eval_expression {
+ const char *source;
+ const char *parsed_as;
+
+ RRDCALC_STATUS *status;
+ calculated_number *myself;
+ time_t *after;
+ time_t *before;
+
+ calculated_number result;
+
+ int error;
+ BUFFER *error_msg;
+
+ // hidden EVAL_NODE *
+ void *nodes;
+
+ // custom data to be used for looking up variables
+ struct rrdcalc *rrdcalc;
+} EVAL_EXPRESSION;
+
+#define EVAL_VALUE_INVALID 0
+#define EVAL_VALUE_NUMBER 1
+#define EVAL_VALUE_VARIABLE 2
+#define EVAL_VALUE_EXPRESSION 3
+
+// parsing and evaluation
+#define EVAL_ERROR_OK 0
+
+// parsing errors
+#define EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION 1
+#define EVAL_ERROR_UNKNOWN_OPERAND 2
+#define EVAL_ERROR_MISSING_OPERAND 3
+#define EVAL_ERROR_MISSING_OPERATOR 4
+#define EVAL_ERROR_REMAINING_GARBAGE 5
+#define EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE 6
+
+// evaluation errors
+#define EVAL_ERROR_INVALID_VALUE 101
+#define EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS 102
+#define EVAL_ERROR_VALUE_IS_NAN 103
+#define EVAL_ERROR_VALUE_IS_INFINITE 104
+#define EVAL_ERROR_UNKNOWN_VARIABLE 105
+
+// parse the given string as an expression and return:
+// a pointer to an expression if it parsed OK
+// NULL in which case the pointer to error has the error code
+extern EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error);
+
+// free all resources allocated for an expression
+extern void expression_free(EVAL_EXPRESSION *expression);
+
+// convert an error code to a message
+extern const char *expression_strerror(int error);
+
+// evaluate an expression and return
+// 1 = OK, the result is in: expression->result
+// 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg)
+extern int expression_evaluate(EVAL_EXPRESSION *expression);
+
+extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result);
+
+#endif //NETDATA_EVAL_H
diff --git a/libnetdata/health/Makefile.am b/libnetdata/health/Makefile.am
new file mode 100644
index 0000000..643458b
--- /dev/null
+++ b/libnetdata/health/Makefile.am
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ $(NULL)
diff --git a/libnetdata/health/health.c b/libnetdata/health/health.c
new file mode 100644
index 0000000..74f7a02
--- /dev/null
+++ b/libnetdata/health/health.c
@@ -0,0 +1,173 @@
+#include "health.h"
+
+SILENCERS *silencers;
+
+/**
+ * Create Silencer
+ *
+ * Allocate a new silencer to Netdata.
+ *
+ * @return It returns the address off the silencer on success and NULL otherwise
+ */
+SILENCER *create_silencer(void) {
+ SILENCER *t = callocz(1, sizeof(SILENCER));
+ debug(D_HEALTH, "HEALTH command API: Created empty silencer");
+
+ return t;
+}
+
+/**
+ * Health Silencers add
+ *
+ * Add more one silencer to the list of silenecers.
+ *
+ * @param silencer
+ */
+void health_silencers_add(SILENCER *silencer) {
+ // Add the created instance to the linked list in silencers
+ silencer->next = silencers->silencers;
+ silencers->silencers = silencer;
+ debug(D_HEALTH, "HEALTH command API: Added silencer %s:%s:%s:%s:%s", silencer->alarms,
+ silencer->charts, silencer->contexts, silencer->hosts, silencer->families
+ );
+}
+
+/**
+ * Silencers Add Parameter
+ *
+ * Create a new silencer and adjust the variables
+ *
+ * @param silencer a pointer to the silencer that will be adjusted
+ * @param key the key value sent by client
+ * @param value the value sent to the key
+ *
+ * @return It returns the silencer configured on success and NULL otherwise
+ */
+SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) {
+ static uint32_t
+ hash_alarm = 0,
+ hash_template = 0,
+ hash_chart = 0,
+ hash_context = 0,
+ hash_host = 0,
+ hash_families = 0;
+
+ if (unlikely(!hash_alarm)) {
+ hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
+ hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
+ hash_chart = simple_uhash(HEALTH_CHART_KEY);
+ hash_context = simple_uhash(HEALTH_CONTEXT_KEY);
+ hash_host = simple_uhash(HEALTH_HOST_KEY);
+ hash_families = simple_uhash(HEALTH_FAMILIES_KEY);
+ }
+
+ uint32_t hash = simple_uhash(key);
+ if (unlikely(silencer == NULL)) {
+ if (
+ (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) ||
+ (hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) ||
+ (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) ||
+ (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) ||
+ (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) ||
+ (hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY))
+ ) {
+ silencer = create_silencer();
+ if(!silencer) {
+ error("Cannot add a new silencer to Netdata");
+ return NULL;
+ }
+ }
+ }
+
+ if (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
+ silencer->alarms = strdupz(value);
+ silencer->alarms_pattern = simple_pattern_create(silencer->alarms, NULL, SIMPLE_PATTERN_EXACT);
+ } else if (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) {
+ silencer->charts = strdupz(value);
+ silencer->charts_pattern = simple_pattern_create(silencer->charts, NULL, SIMPLE_PATTERN_EXACT);
+ } else if (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) {
+ silencer->contexts = strdupz(value);
+ silencer->contexts_pattern = simple_pattern_create(silencer->contexts, NULL, SIMPLE_PATTERN_EXACT);
+ } else if (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) {
+ silencer->hosts = strdupz(value);
+ silencer->hosts_pattern = simple_pattern_create(silencer->hosts, NULL, SIMPLE_PATTERN_EXACT);
+ } else if (hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) {
+ silencer->families = strdupz(value);
+ silencer->families_pattern = simple_pattern_create(silencer->families, NULL, SIMPLE_PATTERN_EXACT);
+ }
+
+ return silencer;
+}
+
+/**
+ * JSON Read Callback
+ *
+ * Callback called by netdata to create the silencer.
+ *
+ * @param e the main json structure
+ *
+ * @return It always return 0.
+ */
+int health_silencers_json_read_callback(JSON_ENTRY *e)
+{
+ switch(e->type) {
+ case JSON_OBJECT:
+#ifndef ENABLE_JSONC
+ e->callback_function = health_silencers_json_read_callback;
+ if(strcmp(e->name,"")) {
+ // init silencer
+ debug(D_HEALTH, "JSON: Got object with a name, initializing new silencer for %s",e->name);
+#endif
+ e->callback_data = create_silencer();
+ if(e->callback_data) {
+ health_silencers_add(e->callback_data);
+ }
+#ifndef ENABLE_JSONC
+ }
+#endif
+ break;
+
+ case JSON_ARRAY:
+ e->callback_function = health_silencers_json_read_callback;
+ break;
+
+ case JSON_STRING:
+ if(!strcmp(e->name,"type")) {
+ debug(D_HEALTH, "JSON: Processing type=%s",e->data.string);
+ if (!strcmp(e->data.string,"SILENCE")) silencers->stype = STYPE_SILENCE_NOTIFICATIONS;
+ else if (!strcmp(e->data.string,"DISABLE")) silencers->stype = STYPE_DISABLE_ALARMS;
+ } else {
+ debug(D_HEALTH, "JSON: Adding %s=%s", e->name, e->data.string);
+ if (e->callback_data)
+ (void)health_silencers_addparam(e->callback_data, e->name, e->data.string);
+ }
+ break;
+
+ case JSON_BOOLEAN:
+ debug(D_HEALTH, "JSON: Processing all_alarms");
+ silencers->all_alarms=e->data.boolean?1:0;
+ break;
+
+ case JSON_NUMBER:
+ case JSON_NULL:
+ break;
+ }
+
+ return 0;
+}
+
+/**
+ * Initialize Global Silencers
+ *
+ * Initialize the silencer for the whole netdata system.
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+int health_initialize_global_silencers() {
+ silencers = mallocz(sizeof(SILENCERS));
+ silencers->all_alarms=0;
+ silencers->stype=STYPE_NONE;
+ silencers->silencers=NULL;
+
+ return 0;
+} \ No newline at end of file
diff --git a/libnetdata/health/health.h b/libnetdata/health/health.h
new file mode 100644
index 0000000..f7580ed
--- /dev/null
+++ b/libnetdata/health/health.h
@@ -0,0 +1,55 @@
+#ifndef NETDATA_HEALTH_LIB
+# define NETDATA_HEALTH_LIB 1
+
+# include "../libnetdata.h"
+
+#define HEALTH_ALARM_KEY "alarm"
+#define HEALTH_TEMPLATE_KEY "template"
+#define HEALTH_CONTEXT_KEY "context"
+#define HEALTH_CHART_KEY "chart"
+#define HEALTH_HOST_KEY "hosts"
+#define HEALTH_OS_KEY "os"
+#define HEALTH_FAMILIES_KEY "families"
+#define HEALTH_LOOKUP_KEY "lookup"
+#define HEALTH_CALC_KEY "calc"
+
+typedef struct silencer {
+ char *alarms;
+ SIMPLE_PATTERN *alarms_pattern;
+
+ char *hosts;
+ SIMPLE_PATTERN *hosts_pattern;
+
+ char *contexts;
+ SIMPLE_PATTERN *contexts_pattern;
+
+ char *charts;
+ SIMPLE_PATTERN *charts_pattern;
+
+ char *families;
+ SIMPLE_PATTERN *families_pattern;
+
+ struct silencer *next;
+} SILENCER;
+
+typedef enum silence_type {
+ STYPE_NONE,
+ STYPE_DISABLE_ALARMS,
+ STYPE_SILENCE_NOTIFICATIONS
+} SILENCE_TYPE;
+
+typedef struct silencers {
+ int all_alarms;
+ SILENCE_TYPE stype;
+ SILENCER *silencers;
+} SILENCERS;
+
+extern SILENCERS *silencers;
+
+extern SILENCER *create_silencer(void);
+extern int health_silencers_json_read_callback(JSON_ENTRY *e);
+extern void health_silencers_add(SILENCER *silencer);
+extern SILENCER * health_silencers_addparam(SILENCER *silencer, char *key, char *value);
+extern int health_initialize_global_silencers();
+
+#endif
diff --git a/libnetdata/inlined.h b/libnetdata/inlined.h
new file mode 100644
index 0000000..6f236b8
--- /dev/null
+++ b/libnetdata/inlined.h
@@ -0,0 +1,318 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_INLINED_H
+#define NETDATA_INLINED_H 1
+
+#include "libnetdata.h"
+
+#ifdef KERNEL_32BIT
+typedef uint32_t kernel_uint_t;
+#define str2kernel_uint_t(string) str2uint32_t(string)
+#define KERNEL_UINT_FORMAT "%u"
+#else
+typedef uint64_t kernel_uint_t;
+#define str2kernel_uint_t(string) str2uint64_t(string)
+#define KERNEL_UINT_FORMAT "%" PRIu64
+#endif
+
+#define str2pid_t(string) str2uint32_t(string)
+
+
+// for faster execution, allow the compiler to inline
+// these functions that are called thousands of times per second
+
+static inline uint32_t simple_hash(const char *name) {
+ unsigned char *s = (unsigned char *) name;
+ uint32_t hval = 0x811c9dc5;
+ while (*s) {
+ hval *= 16777619;
+ hval ^= (uint32_t) *s++;
+ }
+ return hval;
+}
+
+static inline uint32_t simple_uhash(const char *name) {
+ unsigned char *s = (unsigned char *) name;
+ uint32_t hval = 0x811c9dc5, c;
+ while ((c = *s++)) {
+ if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A';
+ hval *= 16777619;
+ hval ^= c;
+ }
+ return hval;
+}
+
+static inline int simple_hash_strcmp(const char *name, const char *b, uint32_t *hash) {
+ unsigned char *s = (unsigned char *) name;
+ uint32_t hval = 0x811c9dc5;
+ int ret = 0;
+ while (*s) {
+ if(!ret) ret = *s - *b++;
+ hval *= 16777619;
+ hval ^= (uint32_t) *s++;
+ }
+ *hash = hval;
+ return ret;
+}
+
+static inline int str2i(const char *s) {
+ int n = 0;
+ char c, negative = (*s == '-');
+
+ for(c = (negative)?*(++s):*s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+
+ if(unlikely(negative))
+ return -n;
+
+ return n;
+}
+
+static inline long str2l(const char *s) {
+ long n = 0;
+ char c, negative = (*s == '-');
+
+ for(c = (negative)?*(++s):*s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+
+ if(unlikely(negative))
+ return -n;
+
+ return n;
+}
+
+static inline uint32_t str2uint32_t(const char *s) {
+ uint32_t n = 0;
+ char c;
+ for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+ return n;
+}
+
+static inline uint64_t str2uint64_t(const char *s) {
+ uint64_t n = 0;
+ char c;
+ for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+ return n;
+}
+
+static inline unsigned long str2ul(const char *s) {
+ unsigned long n = 0;
+ char c;
+ for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+ return n;
+}
+
+static inline unsigned long long str2ull(const char *s) {
+ unsigned long long n = 0;
+ char c;
+ for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+ return n;
+}
+
+static inline long long str2ll(const char *s, char **endptr) {
+ int negative = 0;
+
+ if(unlikely(*s == '-')) {
+ s++;
+ negative = 1;
+ }
+ else if(unlikely(*s == '+'))
+ s++;
+
+ long long n = 0;
+ char c;
+ for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) {
+ n *= 10;
+ n += c - '0';
+ }
+
+ if(unlikely(endptr))
+ *endptr = (char *)s;
+
+ if(unlikely(negative))
+ return -n;
+ else
+ return n;
+}
+
+static inline long double str2ld(const char *s, char **endptr) {
+ int negative = 0;
+ const char *start = s;
+ unsigned long long integer_part = 0;
+ unsigned long decimal_part = 0;
+ size_t decimal_digits = 0;
+
+ switch(*s) {
+ case '-':
+ s++;
+ negative = 1;
+ break;
+
+ case '+':
+ s++;
+ break;
+
+ case 'n':
+ if(s[1] == 'a' && s[2] == 'n') {
+ if(endptr) *endptr = (char *)&s[3];
+ return NAN;
+ }
+ break;
+
+ case 'i':
+ if(s[1] == 'n' && s[2] == 'f') {
+ if(endptr) *endptr = (char *)&s[3];
+ return INFINITY;
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ while (*s >= '0' && *s <= '9') {
+ integer_part = (integer_part * 10) + (*s - '0');
+ s++;
+ }
+
+ if(unlikely(*s == '.')) {
+ decimal_part = 0;
+ s++;
+
+ while (*s >= '0' && *s <= '9') {
+ decimal_part = (decimal_part * 10) + (*s - '0');
+ s++;
+ decimal_digits++;
+ }
+ }
+
+ if(unlikely(*s == 'e' || *s == 'E'))
+ return strtold(start, endptr);
+
+ if(unlikely(endptr))
+ *endptr = (char *)s;
+
+ if(unlikely(negative)) {
+ if(unlikely(decimal_digits))
+ return -((long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits));
+ else
+ return -((long double)integer_part);
+ }
+ else {
+ if(unlikely(decimal_digits))
+ return (long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits);
+ else
+ return (long double)integer_part;
+ }
+}
+
+#ifdef NETDATA_STRCMP_OVERRIDE
+#ifdef strcmp
+#undef strcmp
+#endif
+#define strcmp(a, b) strsame(a, b)
+#endif // NETDATA_STRCMP_OVERRIDE
+
+static inline int strsame(const char *a, const char *b) {
+ if(unlikely(a == b)) return 0;
+ while(*a && *a == *b) { a++; b++; }
+ return *a - *b;
+}
+
+static inline char *strncpyz(char *dst, const char *src, size_t n) {
+ char *p = dst;
+
+ while (*src && n--)
+ *dst++ = *src++;
+
+ *dst = '\0';
+
+ return p;
+}
+
+static inline void sanitize_json_string(char *dst, char *src, size_t len) {
+ while (*src != '\0' && len > 1) {
+ if (*src == '\\' || *src == '\"' || *src < 0x1F) {
+ if (*src < 0x1F) {
+ *dst++ = '_';
+ src++;
+ len--;
+ } else {
+ *dst++ = '\\';
+ *dst++ = *src++;
+ len -= 2;
+ }
+ } else {
+ *dst++ = *src++;
+ len--;
+ }
+ }
+ *dst = '\0';
+}
+
+static inline int read_file(const char *filename, char *buffer, size_t size) {
+ if(unlikely(!size)) return 3;
+
+ int fd = open(filename, O_RDONLY, 0666);
+ if(unlikely(fd == -1)) {
+ buffer[0] = '\0';
+ return 1;
+ }
+
+ ssize_t r = read(fd, buffer, size);
+ if(unlikely(r == -1)) {
+ buffer[0] = '\0';
+ close(fd);
+ return 2;
+ }
+ buffer[r] = '\0';
+
+ close(fd);
+ return 0;
+}
+
+static inline int read_single_number_file(const char *filename, unsigned long long *result) {
+ char buffer[30 + 1];
+
+ int ret = read_file(filename, buffer, 30);
+ if(unlikely(ret)) {
+ *result = 0;
+ return ret;
+ }
+
+ buffer[30] = '\0';
+ *result = str2ull(buffer);
+ return 0;
+}
+
+static inline int read_single_signed_number_file(const char *filename, long long *result) {
+ char buffer[30 + 1];
+
+ int ret = read_file(filename, buffer, 30);
+ if(unlikely(ret)) {
+ *result = 0;
+ return ret;
+ }
+
+ buffer[30] = '\0';
+ *result = atoll(buffer);
+ return 0;
+}
+
+#endif //NETDATA_INLINED_H
diff --git a/libnetdata/json/Makefile.am b/libnetdata/json/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/json/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/json/README.md b/libnetdata/json/README.md
new file mode 100644
index 0000000..79a48e2
--- /dev/null
+++ b/libnetdata/json/README.md
@@ -0,0 +1,10 @@
+<!--
+title: "json"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/json/README.md
+-->
+
+# json
+
+`json` contains a parser for json strings, based on `jsmn` (<https://github.com/zserge/jsmn>), but case you have installed the JSON-C library, the installation script will prefer it, you can also force its use with `--enable-jsonc` in the compilation time.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fjson%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/json/jsmn.c b/libnetdata/json/jsmn.c
new file mode 100644
index 0000000..9525358
--- /dev/null
+++ b/libnetdata/json/jsmn.c
@@ -0,0 +1,328 @@
+#include <stdlib.h>
+
+#include "jsmn.h"
+
+/**
+ * Alloc token
+ *
+ * Allocates a fresh unused token from the token pull.
+ *
+ * @param parser the controller
+ * @param tokens the tokens I am working
+ * @param num_tokens the number total of tokens.
+ *
+ * @return it returns the next token to work.
+ */
+static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser,
+ jsmntok_t *tokens, size_t num_tokens) {
+ jsmntok_t *tok;
+ if (parser->toknext >= num_tokens) {
+ return NULL;
+ }
+ tok = &tokens[parser->toknext++];
+ tok->start = tok->end = -1;
+ tok->size = 0;
+#ifdef JSMN_PARENT_LINKS
+ tok->parent = -1;
+#endif
+ return tok;
+}
+
+/**
+ * Fill Token
+ *
+ * Fills token type and boundaries.
+ *
+ * @param token the structure to set the values
+ * @param type is the token type
+ * @param start is the first position of the value
+ * @param end is the end of the value
+ */
+static void jsmn_fill_token(jsmntok_t *token, jsmntype_t type,
+ int start, int end) {
+ token->type = type;
+ token->start = start;
+ token->end = end;
+ token->size = 0;
+}
+
+/**
+ * Parse primitive
+ *
+ * Fills next available token with JSON primitive.
+ *
+ * @param parser is the control structure
+ * @param js is the json string
+ * @param type is the token type
+ */
+static jsmnerr_t jsmn_parse_primitive(jsmn_parser *parser, const char *js,
+ size_t len, jsmntok_t *tokens, size_t num_tokens) {
+ jsmntok_t *token;
+ int start;
+
+ start = parser->pos;
+
+ for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+ switch (js[parser->pos]) {
+#ifndef JSMN_STRICT
+ /* In strict mode primitive must be followed by "," or "}" or "]" */
+ case ':':
+#endif
+ case '\t' : case '\r' : case '\n' : case ' ' :
+ case ',' : case ']' : case '}' :
+ goto found;
+ }
+ if (js[parser->pos] < 32 || js[parser->pos] >= 127) {
+ parser->pos = start;
+ return JSMN_ERROR_INVAL;
+ }
+ }
+#ifdef JSMN_STRICT
+ /* In strict mode primitive must be followed by a comma/object/array */
+ parser->pos = start;
+ return JSMN_ERROR_PART;
+#endif
+
+ found:
+ if (tokens == NULL) {
+ parser->pos--;
+ return 0;
+ }
+ token = jsmn_alloc_token(parser, tokens, num_tokens);
+ if (token == NULL) {
+ parser->pos = start;
+ return JSMN_ERROR_NOMEM;
+ }
+ jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos);
+#ifdef JSMN_PARENT_LINKS
+ token->parent = parser->toksuper;
+#endif
+ parser->pos--;
+ return 0;
+}
+
+/**
+ * Parse string
+ *
+ * Fills next token with JSON string.
+ *
+ * @param parser is the control structure
+ * @param js is the json string
+ * @param len is the js length
+ * @param tokens is structure with the tokens mapped.
+ * @param num_tokens is the total number of tokens
+ *
+ * @return It returns 0 on success and another integer otherwise
+ */
+static jsmnerr_t jsmn_parse_string(jsmn_parser *parser, const char *js,
+ size_t len, jsmntok_t *tokens, size_t num_tokens) {
+ jsmntok_t *token;
+
+ int start = parser->pos;
+
+ parser->pos++;
+
+ /* Skip starting quote */
+ for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+ char c = js[parser->pos];
+
+ /* Quote: end of string */
+ if (c == '\"') {
+ if (tokens == NULL) {
+ return 0;
+ }
+ token = jsmn_alloc_token(parser, tokens, num_tokens);
+ if (token == NULL) {
+ parser->pos = start;
+ return JSMN_ERROR_NOMEM;
+ }
+ jsmn_fill_token(token, JSMN_STRING, start+1, parser->pos);
+#ifdef JSMN_PARENT_LINKS
+ token->parent = parser->toksuper;
+#endif
+ return 0;
+ }
+
+ /* Backslash: Quoted symbol expected */
+ if (c == '\\') {
+ parser->pos++;
+ switch (js[parser->pos]) {
+ /* Allowed escaped symbols */
+ case '\"': case '/' : case '\\' : case 'b' :
+ case 'f' : case 'r' : case 'n' : case 't' :
+ break;
+ /* Allows escaped symbol \uXXXX */
+ case 'u':
+ parser->pos++;
+ int i = 0;
+ for(; i < 4 && js[parser->pos] != '\0'; i++) {
+ /* If it isn't a hex character we have an error */
+ if(!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */
+ (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */
+ (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */
+ parser->pos = start;
+ return JSMN_ERROR_INVAL;
+ }
+ parser->pos++;
+ }
+ parser->pos--;
+ break;
+ /* Unexpected symbol */
+ default:
+ parser->pos = start;
+ return JSMN_ERROR_INVAL;
+ }
+ }
+ }
+ parser->pos = start;
+ return JSMN_ERROR_PART;
+}
+
+/**
+ * JSMN Parse
+ *
+ * Parse JSON string and fill tokens.
+ *
+ * @param parser the auxiliar vector used to parser
+ * @param js the string to parse
+ * @param len the string length
+ * @param tokens the place to map the tokens
+ * @param num_tokens the number of tokens present in the tokens structure.
+ *
+ * @return It returns the number of tokens present in the string on success or a negative number otherwise
+ */
+jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
+ jsmntok_t *tokens, unsigned int num_tokens) {
+ jsmnerr_t r;
+ int i;
+ jsmntok_t *token;
+ int count = 0;
+
+ for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+ char c;
+ jsmntype_t type;
+
+ c = js[parser->pos];
+ switch (c) {
+ case '{': case '[':
+ count++;
+ if (tokens == NULL) {
+ break;
+ }
+ token = jsmn_alloc_token(parser, tokens, num_tokens);
+ if (token == NULL)
+ return JSMN_ERROR_NOMEM;
+ if (parser->toksuper != -1) {
+ tokens[parser->toksuper].size++;
+#ifdef JSMN_PARENT_LINKS
+ token->parent = parser->toksuper;
+#endif
+ }
+ token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY);
+ token->start = parser->pos;
+ parser->toksuper = parser->toknext - 1;
+ break;
+ case '}': case ']':
+ if (tokens == NULL)
+ break;
+ type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY);
+#ifdef JSMN_PARENT_LINKS
+ if (parser->toknext < 1) {
+ return JSMN_ERROR_INVAL;
+ }
+ token = &tokens[parser->toknext - 1];
+ for (;;) {
+ if (token->start != -1 && token->end == -1) {
+ if (token->type != type) {
+ return JSMN_ERROR_INVAL;
+ }
+ token->end = parser->pos + 1;
+ parser->toksuper = token->parent;
+ break;
+ }
+ if (token->parent == -1) {
+ break;
+ }
+ token = &tokens[token->parent];
+ }
+#else
+ for (i = parser->toknext - 1; i >= 0; i--) {
+ token = &tokens[i];
+ if (token->start != -1 && token->end == -1) {
+ if (token->type != type) {
+ return JSMN_ERROR_INVAL;
+ }
+ parser->toksuper = -1;
+ token->end = parser->pos + 1;
+ break;
+ }
+ }
+ /* Error if unmatched closing bracket */
+ if (i == -1) return JSMN_ERROR_INVAL;
+ for (; i >= 0; i--) {
+ token = &tokens[i];
+ if (token->start != -1 && token->end == -1) {
+ parser->toksuper = i;
+ break;
+ }
+ }
+#endif
+ break;
+ case '\"':
+ r = jsmn_parse_string(parser, js, len, tokens, num_tokens);
+ if (r < 0) return r;
+ count++;
+ if (parser->toksuper != -1 && tokens != NULL)
+ tokens[parser->toksuper].size++;
+ break;
+ case '\t' : case '\r' : case '\n' : case ':' : case ',': case ' ':
+ break;
+#ifdef JSMN_STRICT
+ /* In strict mode primitives are: numbers and booleans */
+ case '-': case '0': case '1' : case '2': case '3' : case '4':
+ case '5': case '6': case '7' : case '8': case '9':
+ case 't': case 'f': case 'n' :
+#else
+ /* In non-strict mode every unquoted value is a primitive */
+ default:
+#endif
+ r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens);
+ if (r < 0) return r;
+ count++;
+ if (parser->toksuper != -1 && tokens != NULL)
+ tokens[parser->toksuper].size++;
+ break;
+
+#ifdef JSMN_STRICT
+ /* Unexpected char in strict mode */
+ default:
+ return JSMN_ERROR_INVAL;
+#endif
+ }
+ }
+
+ if (tokens) {
+ for (i = parser->toknext - 1; i >= 0; i--) {
+ /* Unmatched opened object or array */
+ if (tokens[i].start != -1 && tokens[i].end == -1) {
+ return JSMN_ERROR_PART;
+ }
+ }
+ }
+
+ return count;
+}
+
+/**
+ * JSMN Init
+ *
+ * Creates a new parser based over a given buffer with an array of tokens
+ * available.
+ *
+ * @param parser is the structure with values to reset
+ */
+void jsmn_init(jsmn_parser *parser) {
+ parser->pos = 0;
+ parser->toknext = 0;
+ parser->toksuper = -1;
+} \ No newline at end of file
diff --git a/libnetdata/json/jsmn.h b/libnetdata/json/jsmn.h
new file mode 100644
index 0000000..beff586
--- /dev/null
+++ b/libnetdata/json/jsmn.h
@@ -0,0 +1,75 @@
+#ifndef __JSMN_H_
+#define __JSMN_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+/**
+ * JSON type identifier. Basic types are:
+ * o Object
+ * o Array
+ * o String
+ * o Other primitive: number, boolean (true/false) or null
+ */
+typedef enum {
+ JSMN_PRIMITIVE = 0,
+ JSMN_OBJECT = 1,
+ JSMN_ARRAY = 2,
+ JSMN_STRING = 3
+} jsmntype_t;
+
+typedef enum {
+ /* Not enough tokens were provided */
+ JSMN_ERROR_NOMEM = -1,
+ /* Invalid character inside JSON string */
+ JSMN_ERROR_INVAL = -2,
+ /* The string is not a full JSON packet, more bytes expected */
+ JSMN_ERROR_PART = -3,
+} jsmnerr_t;
+
+/**
+ * JSON token description.
+ *
+ * @param type type (object, array, string etc.)
+ * @param start start position in JSON data string
+ * @param end end position in JSON data string
+ */
+typedef struct {
+ jsmntype_t type;
+ int start;
+ int end;
+ int size;
+#ifdef JSMN_PARENT_LINKS
+ int parent;
+#endif
+} jsmntok_t;
+
+/**
+ * JSON parser. Contains an array of token blocks available. Also stores
+ * the string being parsed now and current position in that string
+ */
+typedef struct {
+ unsigned int pos; /* offset in the JSON string */
+ unsigned int toknext; /* next token to allocate */
+ int toksuper; /* superior token node, e.g parent object or array */
+} jsmn_parser;
+
+/**
+ * Create JSON parser over an array of tokens
+ */
+void jsmn_init(jsmn_parser *parser);
+
+/**
+ * Run JSON parser. It parses a JSON data string into and array of tokens, each describing
+ * a single JSON object.
+ */
+jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
+ jsmntok_t *tokens, unsigned int num_tokens);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __JSMN_H_ */ \ No newline at end of file
diff --git a/libnetdata/json/json.c b/libnetdata/json/json.c
new file mode 100644
index 0000000..bd164ae
--- /dev/null
+++ b/libnetdata/json/json.c
@@ -0,0 +1,556 @@
+#include "jsmn.h"
+#include "../libnetdata.h"
+#include "json.h"
+#include "libnetdata/libnetdata.h"
+#include "../../health/health.h"
+
+#define JSON_TOKENS 1024
+
+int json_tokens = JSON_TOKENS;
+
+/**
+ * Json Tokenise
+ *
+ * Map the string given inside tokens.
+ *
+ * @param js is the string used to create the tokens
+ * @param len is the string length
+ * @param count the number of tokens present in the string
+ *
+ * @return it returns the json parsed in tokens
+ */
+#ifdef ENABLE_JSONC
+json_object *json_tokenise(char *js) {
+ if(!js) {
+ error("JSON: json string is empty.");
+ return NULL;
+ }
+
+ json_object *token = json_tokener_parse(js);
+ if(!token) {
+ error("JSON: Invalid json string.");
+ return NULL;
+ }
+
+ return token;
+}
+#else
+jsmntok_t *json_tokenise(char *js, size_t len, size_t *count)
+{
+ int n = json_tokens;
+ if(!js || !len) {
+ error("JSON: json string is empty.");
+ return NULL;
+ }
+
+ jsmn_parser parser;
+ jsmn_init(&parser);
+
+ jsmntok_t *tokens = mallocz(sizeof(jsmntok_t) * n);
+ if(!tokens) return NULL;
+
+ int ret = jsmn_parse(&parser, js, len, tokens, n);
+ while (ret == JSMN_ERROR_NOMEM) {
+ n *= 2;
+ jsmntok_t *new = reallocz(tokens, sizeof(jsmntok_t) * n);
+ if(!new) {
+ freez(tokens);
+ return NULL;
+ }
+ tokens = new;
+ ret = jsmn_parse(&parser, js, len, tokens, n);
+ }
+
+ if (ret == JSMN_ERROR_INVAL) {
+ error("JSON: Invalid json string.");
+ freez(tokens);
+ return NULL;
+ }
+ else if (ret == JSMN_ERROR_PART) {
+ error("JSON: Truncated JSON string.");
+ freez(tokens);
+ return NULL;
+ }
+
+ if(count) *count = (size_t)ret;
+
+ if(json_tokens < n) json_tokens = n;
+ return tokens;
+}
+#endif
+
+/**
+ * Callback Print
+ *
+ * Set callback print case necesary and wrinte an information inside a buffer to write in the log.
+ *
+ * @param e a pointer for a structure that has the complete information about json structure.
+ *
+ * @return It always return 0
+ */
+int json_callback_print(JSON_ENTRY *e)
+{
+ BUFFER *wb=buffer_create(300);
+
+ buffer_sprintf(wb,"%s = ", e->name);
+ char txt[50];
+ switch(e->type) {
+ case JSON_OBJECT:
+ e->callback_function = json_callback_print;
+ buffer_strcat(wb,"OBJECT");
+ break;
+
+ case JSON_ARRAY:
+ e->callback_function = json_callback_print;
+ sprintf(txt,"ARRAY[%lu]", e->data.items);
+ buffer_strcat(wb, txt);
+ break;
+
+ case JSON_STRING:
+ buffer_strcat(wb, e->data.string);
+ break;
+
+ case JSON_NUMBER:
+ sprintf(txt,"%Lf", e->data.number);
+ buffer_strcat(wb,txt);
+
+ break;
+
+ case JSON_BOOLEAN:
+ buffer_strcat(wb, e->data.boolean?"TRUE":"FALSE");
+ break;
+
+ case JSON_NULL:
+ buffer_strcat(wb,"NULL");
+ break;
+ }
+ info("JSON: %s", buffer_tostring(wb));
+ buffer_free(wb);
+ return 0;
+}
+
+/**
+ * JSONC Set String
+ *
+ * Set the string value of the structure JSON_ENTRY.
+ *
+ * @param e the output structure
+ */
+static inline void json_jsonc_set_string(JSON_ENTRY *e,char *key,const char *value) {
+ size_t len = strlen(key);
+ if(len > JSON_NAME_LEN)
+ len = JSON_NAME_LEN;
+ e->type = JSON_STRING;
+ memcpy(e->name,key,len);
+ e->name[len] = 0x00;
+ e->data.string = (char *) value;
+}
+
+
+#ifdef ENABLE_JSONC
+/**
+ * JSONC set Boolean
+ *
+ * Set the boolean value of the structure JSON_ENTRY
+ *
+ * @param e the output structure
+ * @param value the input value
+ */
+static inline void json_jsonc_set_boolean(JSON_ENTRY *e,int value) {
+ e->type = JSON_BOOLEAN;
+ e->data.boolean = value;
+}
+
+static inline void json_jsonc_set_integer(JSON_ENTRY *e, char *key, int64_t value) {
+ size_t len = strlen(key);
+ if(len > JSON_NAME_LEN)
+ len = JSON_NAME_LEN;
+ e->type = JSON_NUMBER;
+ memcpy(e->name, key, len);
+ e->name[len] = 0;
+ e->data.number = value;
+}
+
+/**
+ * Parse Array
+ *
+ * Parse the array object.
+ *
+ * @param ptr the pointer for the object that we will parse.
+ * @param callback_data additional data to be used together the callback function
+ * @param callback_function function used to create a silencer.
+ */
+static inline void json_jsonc_parse_array(json_object *ptr, void *callback_data,int (*callback_function)(struct json_entry *)) {
+ int end = json_object_array_length(ptr);
+ JSON_ENTRY e;
+
+ if(end) {
+ int i;
+ i = 0;
+
+ enum json_type type;
+ do {
+ json_object *jvalue = json_object_array_get_idx(ptr, i);
+ if(jvalue) {
+ e.callback_data = callback_data;
+ e.type = JSON_OBJECT;
+ callback_function(&e);
+ json_object_object_foreach(jvalue, key, val) {
+ type = json_object_get_type(val);
+ if (type == json_type_array) {
+ e.type = JSON_ARRAY;
+ json_jsonc_parse_array(val, callback_data, callback_function);
+ } else if (type == json_type_object) {
+ json_walk(val,callback_data,callback_function);
+ } else if (type == json_type_string) {
+ json_jsonc_set_string(&e,key,json_object_get_string(val));
+ callback_function(&e);
+ } else if (type == json_type_boolean) {
+ json_jsonc_set_boolean(&e,json_object_get_boolean(val));
+ callback_function(&e);
+ }
+ }
+ }
+
+ } while (++i < end);
+ }
+}
+#else
+
+/**
+ * Walk string
+ *
+ * Set JSON_ENTRY to string and map the values from jsmntok_t.
+ *
+ * @param js the original string
+ * @param t the tokens
+ * @param start the first position
+ * @param e the output structure.
+ *
+ * @return It always return 1
+ */
+size_t json_walk_string(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e)
+{
+ char old = js[t[start].end];
+ js[t[start].end] = '\0';
+ e->original_string = &js[t[start].start];
+
+ e->type = JSON_STRING;
+ e->data.string = e->original_string;
+ if(e->callback_function) e->callback_function(e);
+ js[t[start].end] = old;
+ return 1;
+}
+
+/**
+ * Walk Primitive
+ *
+ * Define the data type of the string
+ *
+ * @param js the original string
+ * @param t the tokens
+ * @param start the first position
+ * @param e the output structure.
+ *
+ * @return It always return 1
+ */
+size_t json_walk_primitive(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e)
+{
+ char old = js[t[start].end];
+ js[t[start].end] = '\0';
+ e->original_string = &js[t[start].start];
+
+ switch(e->original_string[0]) {
+ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7':
+ case '8': case '9': case '-': case '.':
+ e->type = JSON_NUMBER;
+ e->data.number = strtold(e->original_string, NULL);
+ break;
+
+ case 't': case 'T':
+ e->type = JSON_BOOLEAN;
+ e->data.boolean = 1;
+ break;
+
+ case 'f': case 'F':
+ e->type = JSON_BOOLEAN;
+ e->data.boolean = 0;
+ break;
+
+ case 'n': case 'N':
+ default:
+ e->type = JSON_NULL;
+ break;
+ }
+ if(e->callback_function) e->callback_function(e);
+ js[t[start].end] = old;
+ return 1;
+}
+
+/**
+ * Array
+ *
+ * Measure the array length
+ *
+ * @param js the original string
+ * @param t the tokens
+ * @param nest the length of structure t
+ * @param start the first position
+ * @param e the structure with values and callback to be used inside the function.
+ *
+ * @return It returns the array length
+ */
+size_t json_walk_array(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e)
+{
+ JSON_ENTRY ne;
+
+ char old = js[t[start].end];
+ js[t[start].end] = '\0';
+ ne.original_string = &js[t[start].start];
+
+ memcpy(&ne, e, sizeof(JSON_ENTRY));
+ ne.type = JSON_ARRAY;
+ ne.data.items = t[start].size;
+ ne.callback_function = e->callback_function;
+ ne.name[0]='\0';
+ ne.fullname[0]='\0';
+ if(e->callback_function) e->callback_function(&ne);
+ js[t[start].end] = old;
+
+ size_t i, init = start, size = t[start].size;
+
+ start++;
+ for(i = 0; i < size ; i++) {
+ ne.pos = i;
+ if (strlen(e->name) > JSON_NAME_LEN - 24 || strlen(e->fullname) > JSON_FULLNAME_LEN -24) {
+ info("JSON: JSON walk_array ignoring element with name:%s fullname:%s",e->name, e->fullname);
+ continue;
+ }
+ snprintfz(ne.name, JSON_NAME_LEN, "%s[%lu]", e->name, i);
+ snprintfz(ne.fullname, JSON_FULLNAME_LEN, "%s[%lu]", e->fullname, i);
+
+ switch(t[start].type) {
+ case JSMN_PRIMITIVE:
+ start += json_walk_primitive(js, t, start, &ne);
+ break;
+
+ case JSMN_OBJECT:
+ start += json_walk_object(js, t, nest + 1, start, &ne);
+ break;
+
+ case JSMN_ARRAY:
+ start += json_walk_array(js, t, nest + 1, start, &ne);
+ break;
+
+ case JSMN_STRING:
+ start += json_walk_string(js, t, start, &ne);
+ break;
+ }
+ }
+ return start - init;
+}
+
+/**
+ * Object
+ *
+ * Measure the Object length
+ *
+ * @param js the original string
+ * @param t the tokens
+ * @param nest the length of structure t
+ * @param start the first position
+ * @param e the output structure.
+ *
+ * @return It returns the Object length
+ */
+size_t json_walk_object(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e)
+{
+ JSON_ENTRY ne = {
+ .name = "",
+ .fullname = "",
+ .callback_data = NULL,
+ .callback_function = NULL
+ };
+
+ char old = js[t[start].end];
+ js[t[start].end] = '\0';
+ ne.original_string = &js[t[start].start];
+ memcpy(&ne, e, sizeof(JSON_ENTRY));
+ ne.type = JSON_OBJECT;
+ ne.callback_function = e->callback_function;
+ if(e->callback_function) e->callback_function(&ne);
+ js[t[start].end] = old;
+
+ int key = 1;
+ size_t i, init = start, size = t[start].size;
+
+ start++;
+ for(i = 0; i < size ; i++) {
+ switch(t[start].type) {
+ case JSMN_PRIMITIVE:
+ start += json_walk_primitive(js, t, start, &ne);
+ key = 1;
+ break;
+
+ case JSMN_OBJECT:
+ start += json_walk_object(js, t, nest + 1, start, &ne);
+ key = 1;
+ break;
+
+ case JSMN_ARRAY:
+ start += json_walk_array(js, t, nest + 1, start, &ne);
+ key = 1;
+ break;
+
+ case JSMN_STRING:
+ default:
+ if(key) {
+ int len = t[start].end - t[start].start;
+ if (unlikely(len>JSON_NAME_LEN)) len=JSON_NAME_LEN;
+ strncpy(ne.name, &js[t[start].start], len);
+ ne.name[len] = '\0';
+ len=strlen(e->fullname) + strlen(e->fullname[0]?".":"") + strlen(ne.name);
+ char *c = mallocz((len+1)*sizeof(char));
+ sprintf(c,"%s%s%s", e->fullname, e->fullname[0]?".":"", ne.name);
+ if (unlikely(len>JSON_FULLNAME_LEN)) len=JSON_FULLNAME_LEN;
+ strncpy(ne.fullname, c, len);
+ freez(c);
+ start++;
+ key = 0;
+ }
+ else {
+ start += json_walk_string(js, t, start, &ne);
+ key = 1;
+ }
+ break;
+ }
+ }
+ return start - init;
+}
+#endif
+
+/**
+ * Tree
+ *
+ * Call the correct walk function according its type.
+ *
+ * @param t the json object to work
+ * @param callback_data additional data to be used together the callback function
+ * @param callback_function function used to create a silencer.
+ *
+ * @return It always return 1
+ */
+#ifdef ENABLE_JSONC
+size_t json_walk(json_object *t, void *callback_data, int (*callback_function)(struct json_entry *)) {
+ JSON_ENTRY e;
+
+ e.callback_data = callback_data;
+ enum json_type type;
+ json_object_object_foreach(t, key, val) {
+ type = json_object_get_type(val);
+ if (type == json_type_array) {
+ e.type = JSON_ARRAY;
+ json_jsonc_parse_array(val,NULL,health_silencers_json_read_callback);
+ } else if (type == json_type_object) {
+ e.type = JSON_OBJECT;
+ } else if (type == json_type_string) {
+ json_jsonc_set_string(&e,key,json_object_get_string(val));
+ callback_function(&e);
+ } else if (type == json_type_boolean) {
+ json_jsonc_set_boolean(&e,json_object_get_boolean(val));
+ callback_function(&e);
+ } else if (type == json_type_int) {
+ json_jsonc_set_integer(&e,key,json_object_get_int64(val));
+ callback_function(&e);
+ }
+ }
+
+ return 1;
+}
+#else
+/**
+ * Tree
+ *
+ * Call the correct walk function according its type.
+ *
+ * @param js the original string
+ * @param t the tokens
+ * @param callback_data additional data to be used together the callback function
+ * @param callback_function function used to create a silencer.
+ *
+ * @return It always return 1
+ */
+size_t json_walk_tree(char *js, jsmntok_t *t, void *callback_data, int (*callback_function)(struct json_entry *))
+{
+ JSON_ENTRY e = {
+ .name = "",
+ .fullname = "",
+ .callback_data = callback_data,
+ .callback_function = callback_function
+ };
+
+ switch (t[0].type) {
+ case JSMN_OBJECT:
+ e.type = JSON_OBJECT;
+ json_walk_object(js, t, 0, 0, &e);
+ break;
+
+ case JSMN_ARRAY:
+ e.type = JSON_ARRAY;
+ json_walk_array(js, t, 0, 0, &e);
+ break;
+
+ case JSMN_PRIMITIVE:
+ case JSMN_STRING:
+ break;
+ }
+
+ return 1;
+}
+#endif
+
+/**
+ * JSON Parse
+ *
+ * Parse the json message with the callback function
+ *
+ * @param js the string that the callback function will parse
+ * @param callback_data additional data to be used together the callback function
+ * @param callback_function function used to create a silencer.
+ *
+ * @return JSON_OK case everything happend as expected, JSON_CANNOT_PARSE case there were errors in the
+ * parsing procces and JSON_CANNOT_DOWNLOAD case the string given(js) is NULL.
+ */
+int json_parse(char *js, void *callback_data, int (*callback_function)(JSON_ENTRY *))
+{
+ if(js) {
+#ifdef ENABLE_JSONC
+ json_object *tokens = json_tokenise(js);
+#else
+ size_t count;
+ jsmntok_t *tokens = json_tokenise(js, strlen(js), &count);
+#endif
+
+ if(tokens) {
+#ifdef ENABLE_JSONC
+ json_walk(tokens, callback_data, callback_function);
+ json_object_put(tokens);
+#else
+ json_walk_tree(js, tokens, callback_data, callback_function);
+ freez(tokens);
+#endif
+ return JSON_OK;
+ }
+
+ return JSON_CANNOT_PARSE;
+ }
+
+ return JSON_CANNOT_DOWNLOAD;
+}
+
+/*
+int json_test(char *str)
+{
+ return json_parse(str, NULL, json_callback_print);
+}
+ */ \ No newline at end of file
diff --git a/libnetdata/json/json.h b/libnetdata/json/json.h
new file mode 100644
index 0000000..79b58b1
--- /dev/null
+++ b/libnetdata/json/json.h
@@ -0,0 +1,72 @@
+#ifndef CHECKIN_JSON_H
+#define CHECKIN_JSON_H 1
+
+
+#if ENABLE_JSONC
+# include <json-c/json.h>
+#endif
+
+#include "jsmn.h"
+
+//https://www.ibm.com/support/knowledgecenter/en/SS9H2Y_7.6.0/com.ibm.dp.doc/json_parserlimits.html
+#define JSON_NAME_LEN 256
+#define JSON_FULLNAME_LEN 1024
+
+typedef enum {
+ JSON_OBJECT = 0,
+ JSON_ARRAY = 1,
+ JSON_STRING = 2,
+ JSON_NUMBER = 3,
+ JSON_BOOLEAN = 4,
+ JSON_NULL = 5,
+} JSON_ENTRY_TYPE;
+
+typedef struct json_entry {
+ JSON_ENTRY_TYPE type;
+ char name[JSON_NAME_LEN + 1];
+ char fullname[JSON_FULLNAME_LEN + 1];
+ union {
+ char *string; // type == JSON_STRING
+ long double number; // type == JSON_NUMBER
+ int boolean; // type == JSON_BOOLEAN
+ size_t items; // type == JSON_ARRAY
+ } data;
+ size_t pos; // the position of this item in its parent
+
+ char *original_string;
+
+ void *callback_data;
+ int (*callback_function)(struct json_entry *);
+} JSON_ENTRY;
+
+// ----------------------------------------------------------------------------
+// public functions
+
+#define JSON_OK 0
+#define JSON_CANNOT_DOWNLOAD 1
+#define JSON_CANNOT_PARSE 2
+
+int json_parse(char *js, void *callback_data, int (*callback_function)(JSON_ENTRY *));
+
+
+// ----------------------------------------------------------------------------
+// private functions
+
+#ifdef ENABLE_JSONC
+json_object *json_tokenise(char *js);
+size_t json_walk(json_object *t, void *callback_data, int (*callback_function)(struct json_entry *));
+#else
+jsmntok_t *json_tokenise(char *js, size_t len, size_t *count);
+size_t json_walk_tree(char *js, jsmntok_t *t, void *callback_data, int (*callback_function)(struct json_entry *));
+#endif
+
+size_t json_walk_object(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e);
+size_t json_walk_array(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e);
+size_t json_walk_string(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e);
+size_t json_walk_primitive(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e);
+
+int json_callback_print(JSON_ENTRY *e);
+
+
+
+#endif \ No newline at end of file
diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c
new file mode 100644
index 0000000..325df3f
--- /dev/null
+++ b/libnetdata/libnetdata.c
@@ -0,0 +1,1493 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "libnetdata.h"
+
+#ifdef __APPLE__
+#define INHERIT_NONE 0
+#endif /* __APPLE__ */
+#if defined(__FreeBSD__) || defined(__APPLE__)
+# define O_NOATIME 0
+# define MADV_DONTFORK INHERIT_NONE
+#endif /* __FreeBSD__ || __APPLE__*/
+
+struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };
+int enable_ksm = 1;
+
+volatile sig_atomic_t netdata_exit = 0;
+const char *os_type = NETDATA_OS_TYPE;
+const char *program_version = VERSION;
+
+// ----------------------------------------------------------------------------
+// memory allocation functions that handle failures
+
+// although netdata does not use memory allocations too often (netdata tries to
+// maintain its memory footprint stable during runtime, i.e. all buffers are
+// allocated during initialization and are adapted to current use throughout
+// its lifetime), these can be used to override the default system allocation
+// routines.
+
+#ifdef NETDATA_LOG_ALLOCATIONS
+static __thread struct memory_statistics {
+ volatile ssize_t malloc_calls_made;
+ volatile ssize_t calloc_calls_made;
+ volatile ssize_t realloc_calls_made;
+ volatile ssize_t strdup_calls_made;
+ volatile ssize_t free_calls_made;
+ volatile ssize_t memory_calls_made;
+ volatile ssize_t allocated_memory;
+ volatile ssize_t mmapped_memory;
+} memory_statistics = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+__thread size_t log_thread_memory_allocations = 0;
+
+static inline void print_allocations(const char *file, const char *function, const unsigned long line, const char *type, size_t size) {
+ static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+ fprintf(stderr, "%s iteration %zu MEMORY TRACE: %lu@%s : %s : %s : %zu\n",
+ netdata_thread_tag(),
+ log_thread_memory_allocations,
+ line, file, function,
+ type, size
+ );
+
+ fprintf(stderr, "%s iteration %zu MEMORY ALLOCATIONS: (%04lu@%-40.40s:%-40.40s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): %s : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
+ netdata_thread_tag(),
+ log_thread_memory_allocations,
+ line, file, function,
+ (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
+ (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
+ type,
+ memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
+ memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
+ memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
+ memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made,
+ memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made
+ );
+
+ memcpy(&old, &memory_statistics, sizeof(struct memory_statistics));
+}
+
+static inline void mmap_accounting(size_t size) {
+ if(log_thread_memory_allocations) {
+ memory_statistics.memory_calls_made++;
+ memory_statistics.mmapped_memory += size;
+ }
+}
+
+void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
+ if(log_thread_memory_allocations) {
+ memory_statistics.memory_calls_made++;
+ memory_statistics.malloc_calls_made++;
+ memory_statistics.allocated_memory += size;
+ print_allocations(file, function, line, "malloc()", size);
+ }
+
+ size_t *n = (size_t *)malloc(sizeof(size_t) + size);
+ if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size);
+ *n = size;
+ return (void *)&n[1];
+}
+
+void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
+ size = nmemb * size;
+
+ if(log_thread_memory_allocations) {
+ memory_statistics.memory_calls_made++;
+ memory_statistics.calloc_calls_made++;
+ memory_statistics.allocated_memory += size;
+ print_allocations(file, function, line, "calloc()", size);
+ }
+
+ size_t *n = (size_t *)calloc(1, sizeof(size_t) + size);
+ if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size);
+ *n = size;
+ return (void *)&n[1];
+}
+
+void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) {
+ if(!ptr) return mallocz_int(file, function, line, size);
+
+ size_t *n = (size_t *)ptr;
+ n--;
+ size_t old_size = *n;
+
+ n = realloc(n, sizeof(size_t) + size);
+ if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size);
+
+ if(log_thread_memory_allocations) {
+ memory_statistics.memory_calls_made++;
+ memory_statistics.realloc_calls_made++;
+ memory_statistics.allocated_memory += (size - old_size);
+ print_allocations(file, function, line, "realloc()", size - old_size);
+ }
+
+ *n = size;
+ return (void *)&n[1];
+}
+
+char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
+ size_t size = strlen(s) + 1;
+
+ if(log_thread_memory_allocations) {
+ memory_statistics.memory_calls_made++;
+ memory_statistics.strdup_calls_made++;
+ memory_statistics.allocated_memory += size;
+ print_allocations(file, function, line, "strdup()", size);
+ }
+
+ size_t *n = (size_t *)malloc(sizeof(size_t) + size);
+ if (unlikely(!n)) fatal("strdupz() cannot allocate %zu bytes of memory.", size);
+
+ *n = size;
+ char *t = (char *)&n[1];
+ strcpy(t, s);
+ return t;
+}
+
+void freez_int(const char *file, const char *function, const unsigned long line, void *ptr) {
+ if(unlikely(!ptr)) return;
+
+ size_t *n = (size_t *)ptr;
+ n--;
+ size_t size = *n;
+
+ if(log_thread_memory_allocations) {
+ memory_statistics.memory_calls_made++;
+ memory_statistics.free_calls_made++;
+ memory_statistics.allocated_memory -= size;
+ print_allocations(file, function, line, "free()", size);
+ }
+
+ free(n);
+}
+#else
+
+char *strdupz(const char *s) {
+ char *t = strdup(s);
+ if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s);
+ return t;
+}
+
+// If ptr is NULL, no operation is performed.
+void freez(void *ptr) {
+ free(ptr);
+}
+
+void *mallocz(size_t size) {
+ void *p = malloc(size);
+ if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size);
+ return p;
+}
+
+void *callocz(size_t nmemb, size_t size) {
+ void *p = calloc(nmemb, size);
+ if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size);
+ return p;
+}
+
+void *reallocz(void *ptr, size_t size) {
+ void *p = realloc(ptr, size);
+ if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size);
+ return p;
+}
+
+#endif
+
+// --------------------------------------------------------------------------------------------------------------------
+
+void json_escape_string(char *dst, const char *src, size_t size) {
+ const char *t;
+ char *d = dst, *e = &dst[size - 1];
+
+ for(t = src; *t && d < e ;t++) {
+ if(unlikely(*t == '\\' || *t == '"')) {
+ if(unlikely(d + 1 >= e)) break;
+ *d++ = '\\';
+ }
+ *d++ = *t;
+ }
+
+ *d = '\0';
+}
+
+void json_fix_string(char *s) {
+ unsigned char c;
+ while((c = (unsigned char)*s)) {
+ if(unlikely(c == '\\'))
+ *s++ = '/';
+ else if(unlikely(c == '"'))
+ *s++ = '\'';
+ else if(unlikely(isspace(c) || iscntrl(c)))
+ *s++ = ' ';
+ else if(unlikely(!isprint(c) || c > 127))
+ *s++ = '_';
+ else
+ s++;
+ }
+}
+
+unsigned char netdata_map_chart_names[256] = {
+ [0] = '\0', //
+ [1] = '_', //
+ [2] = '_', //
+ [3] = '_', //
+ [4] = '_', //
+ [5] = '_', //
+ [6] = '_', //
+ [7] = '_', //
+ [8] = '_', //
+ [9] = '_', //
+ [10] = '_', //
+ [11] = '_', //
+ [12] = '_', //
+ [13] = '_', //
+ [14] = '_', //
+ [15] = '_', //
+ [16] = '_', //
+ [17] = '_', //
+ [18] = '_', //
+ [19] = '_', //
+ [20] = '_', //
+ [21] = '_', //
+ [22] = '_', //
+ [23] = '_', //
+ [24] = '_', //
+ [25] = '_', //
+ [26] = '_', //
+ [27] = '_', //
+ [28] = '_', //
+ [29] = '_', //
+ [30] = '_', //
+ [31] = '_', //
+ [32] = '_', //
+ [33] = '_', // !
+ [34] = '_', // "
+ [35] = '_', // #
+ [36] = '_', // $
+ [37] = '_', // %
+ [38] = '_', // &
+ [39] = '_', // '
+ [40] = '_', // (
+ [41] = '_', // )
+ [42] = '_', // *
+ [43] = '_', // +
+ [44] = '.', // ,
+ [45] = '-', // -
+ [46] = '.', // .
+ [47] = '/', // /
+ [48] = '0', // 0
+ [49] = '1', // 1
+ [50] = '2', // 2
+ [51] = '3', // 3
+ [52] = '4', // 4
+ [53] = '5', // 5
+ [54] = '6', // 6
+ [55] = '7', // 7
+ [56] = '8', // 8
+ [57] = '9', // 9
+ [58] = '_', // :
+ [59] = '_', // ;
+ [60] = '_', // <
+ [61] = '_', // =
+ [62] = '_', // >
+ [63] = '_', // ?
+ [64] = '_', // @
+ [65] = 'a', // A
+ [66] = 'b', // B
+ [67] = 'c', // C
+ [68] = 'd', // D
+ [69] = 'e', // E
+ [70] = 'f', // F
+ [71] = 'g', // G
+ [72] = 'h', // H
+ [73] = 'i', // I
+ [74] = 'j', // J
+ [75] = 'k', // K
+ [76] = 'l', // L
+ [77] = 'm', // M
+ [78] = 'n', // N
+ [79] = 'o', // O
+ [80] = 'p', // P
+ [81] = 'q', // Q
+ [82] = 'r', // R
+ [83] = 's', // S
+ [84] = 't', // T
+ [85] = 'u', // U
+ [86] = 'v', // V
+ [87] = 'w', // W
+ [88] = 'x', // X
+ [89] = 'y', // Y
+ [90] = 'z', // Z
+ [91] = '_', // [
+ [92] = '/', // backslash
+ [93] = '_', // ]
+ [94] = '_', // ^
+ [95] = '_', // _
+ [96] = '_', // `
+ [97] = 'a', // a
+ [98] = 'b', // b
+ [99] = 'c', // c
+ [100] = 'd', // d
+ [101] = 'e', // e
+ [102] = 'f', // f
+ [103] = 'g', // g
+ [104] = 'h', // h
+ [105] = 'i', // i
+ [106] = 'j', // j
+ [107] = 'k', // k
+ [108] = 'l', // l
+ [109] = 'm', // m
+ [110] = 'n', // n
+ [111] = 'o', // o
+ [112] = 'p', // p
+ [113] = 'q', // q
+ [114] = 'r', // r
+ [115] = 's', // s
+ [116] = 't', // t
+ [117] = 'u', // u
+ [118] = 'v', // v
+ [119] = 'w', // w
+ [120] = 'x', // x
+ [121] = 'y', // y
+ [122] = 'z', // z
+ [123] = '_', // {
+ [124] = '_', // |
+ [125] = '_', // }
+ [126] = '_', // ~
+ [127] = '_', //
+ [128] = '_', //
+ [129] = '_', //
+ [130] = '_', //
+ [131] = '_', //
+ [132] = '_', //
+ [133] = '_', //
+ [134] = '_', //
+ [135] = '_', //
+ [136] = '_', //
+ [137] = '_', //
+ [138] = '_', //
+ [139] = '_', //
+ [140] = '_', //
+ [141] = '_', //
+ [142] = '_', //
+ [143] = '_', //
+ [144] = '_', //
+ [145] = '_', //
+ [146] = '_', //
+ [147] = '_', //
+ [148] = '_', //
+ [149] = '_', //
+ [150] = '_', //
+ [151] = '_', //
+ [152] = '_', //
+ [153] = '_', //
+ [154] = '_', //
+ [155] = '_', //
+ [156] = '_', //
+ [157] = '_', //
+ [158] = '_', //
+ [159] = '_', //
+ [160] = '_', //
+ [161] = '_', //
+ [162] = '_', //
+ [163] = '_', //
+ [164] = '_', //
+ [165] = '_', //
+ [166] = '_', //
+ [167] = '_', //
+ [168] = '_', //
+ [169] = '_', //
+ [170] = '_', //
+ [171] = '_', //
+ [172] = '_', //
+ [173] = '_', //
+ [174] = '_', //
+ [175] = '_', //
+ [176] = '_', //
+ [177] = '_', //
+ [178] = '_', //
+ [179] = '_', //
+ [180] = '_', //
+ [181] = '_', //
+ [182] = '_', //
+ [183] = '_', //
+ [184] = '_', //
+ [185] = '_', //
+ [186] = '_', //
+ [187] = '_', //
+ [188] = '_', //
+ [189] = '_', //
+ [190] = '_', //
+ [191] = '_', //
+ [192] = '_', //
+ [193] = '_', //
+ [194] = '_', //
+ [195] = '_', //
+ [196] = '_', //
+ [197] = '_', //
+ [198] = '_', //
+ [199] = '_', //
+ [200] = '_', //
+ [201] = '_', //
+ [202] = '_', //
+ [203] = '_', //
+ [204] = '_', //
+ [205] = '_', //
+ [206] = '_', //
+ [207] = '_', //
+ [208] = '_', //
+ [209] = '_', //
+ [210] = '_', //
+ [211] = '_', //
+ [212] = '_', //
+ [213] = '_', //
+ [214] = '_', //
+ [215] = '_', //
+ [216] = '_', //
+ [217] = '_', //
+ [218] = '_', //
+ [219] = '_', //
+ [220] = '_', //
+ [221] = '_', //
+ [222] = '_', //
+ [223] = '_', //
+ [224] = '_', //
+ [225] = '_', //
+ [226] = '_', //
+ [227] = '_', //
+ [228] = '_', //
+ [229] = '_', //
+ [230] = '_', //
+ [231] = '_', //
+ [232] = '_', //
+ [233] = '_', //
+ [234] = '_', //
+ [235] = '_', //
+ [236] = '_', //
+ [237] = '_', //
+ [238] = '_', //
+ [239] = '_', //
+ [240] = '_', //
+ [241] = '_', //
+ [242] = '_', //
+ [243] = '_', //
+ [244] = '_', //
+ [245] = '_', //
+ [246] = '_', //
+ [247] = '_', //
+ [248] = '_', //
+ [249] = '_', //
+ [250] = '_', //
+ [251] = '_', //
+ [252] = '_', //
+ [253] = '_', //
+ [254] = '_', //
+ [255] = '_' //
+};
+
+// make sure the supplied string
+// is good for a netdata chart/dimension ID/NAME
+void netdata_fix_chart_name(char *s) {
+ while ((*s = netdata_map_chart_names[(unsigned char) *s])) s++;
+}
+
+unsigned char netdata_map_chart_ids[256] = {
+ [0] = '\0', //
+ [1] = '_', //
+ [2] = '_', //
+ [3] = '_', //
+ [4] = '_', //
+ [5] = '_', //
+ [6] = '_', //
+ [7] = '_', //
+ [8] = '_', //
+ [9] = '_', //
+ [10] = '_', //
+ [11] = '_', //
+ [12] = '_', //
+ [13] = '_', //
+ [14] = '_', //
+ [15] = '_', //
+ [16] = '_', //
+ [17] = '_', //
+ [18] = '_', //
+ [19] = '_', //
+ [20] = '_', //
+ [21] = '_', //
+ [22] = '_', //
+ [23] = '_', //
+ [24] = '_', //
+ [25] = '_', //
+ [26] = '_', //
+ [27] = '_', //
+ [28] = '_', //
+ [29] = '_', //
+ [30] = '_', //
+ [31] = '_', //
+ [32] = '_', //
+ [33] = '_', // !
+ [34] = '_', // "
+ [35] = '_', // #
+ [36] = '_', // $
+ [37] = '_', // %
+ [38] = '_', // &
+ [39] = '_', // '
+ [40] = '_', // (
+ [41] = '_', // )
+ [42] = '_', // *
+ [43] = '_', // +
+ [44] = '.', // ,
+ [45] = '-', // -
+ [46] = '.', // .
+ [47] = '_', // /
+ [48] = '0', // 0
+ [49] = '1', // 1
+ [50] = '2', // 2
+ [51] = '3', // 3
+ [52] = '4', // 4
+ [53] = '5', // 5
+ [54] = '6', // 6
+ [55] = '7', // 7
+ [56] = '8', // 8
+ [57] = '9', // 9
+ [58] = '_', // :
+ [59] = '_', // ;
+ [60] = '_', // <
+ [61] = '_', // =
+ [62] = '_', // >
+ [63] = '_', // ?
+ [64] = '_', // @
+ [65] = 'a', // A
+ [66] = 'b', // B
+ [67] = 'c', // C
+ [68] = 'd', // D
+ [69] = 'e', // E
+ [70] = 'f', // F
+ [71] = 'g', // G
+ [72] = 'h', // H
+ [73] = 'i', // I
+ [74] = 'j', // J
+ [75] = 'k', // K
+ [76] = 'l', // L
+ [77] = 'm', // M
+ [78] = 'n', // N
+ [79] = 'o', // O
+ [80] = 'p', // P
+ [81] = 'q', // Q
+ [82] = 'r', // R
+ [83] = 's', // S
+ [84] = 't', // T
+ [85] = 'u', // U
+ [86] = 'v', // V
+ [87] = 'w', // W
+ [88] = 'x', // X
+ [89] = 'y', // Y
+ [90] = 'z', // Z
+ [91] = '_', // [
+ [92] = '/', // backslash
+ [93] = '_', // ]
+ [94] = '_', // ^
+ [95] = '_', // _
+ [96] = '_', // `
+ [97] = 'a', // a
+ [98] = 'b', // b
+ [99] = 'c', // c
+ [100] = 'd', // d
+ [101] = 'e', // e
+ [102] = 'f', // f
+ [103] = 'g', // g
+ [104] = 'h', // h
+ [105] = 'i', // i
+ [106] = 'j', // j
+ [107] = 'k', // k
+ [108] = 'l', // l
+ [109] = 'm', // m
+ [110] = 'n', // n
+ [111] = 'o', // o
+ [112] = 'p', // p
+ [113] = 'q', // q
+ [114] = 'r', // r
+ [115] = 's', // s
+ [116] = 't', // t
+ [117] = 'u', // u
+ [118] = 'v', // v
+ [119] = 'w', // w
+ [120] = 'x', // x
+ [121] = 'y', // y
+ [122] = 'z', // z
+ [123] = '_', // {
+ [124] = '_', // |
+ [125] = '_', // }
+ [126] = '_', // ~
+ [127] = '_', //
+ [128] = '_', //
+ [129] = '_', //
+ [130] = '_', //
+ [131] = '_', //
+ [132] = '_', //
+ [133] = '_', //
+ [134] = '_', //
+ [135] = '_', //
+ [136] = '_', //
+ [137] = '_', //
+ [138] = '_', //
+ [139] = '_', //
+ [140] = '_', //
+ [141] = '_', //
+ [142] = '_', //
+ [143] = '_', //
+ [144] = '_', //
+ [145] = '_', //
+ [146] = '_', //
+ [147] = '_', //
+ [148] = '_', //
+ [149] = '_', //
+ [150] = '_', //
+ [151] = '_', //
+ [152] = '_', //
+ [153] = '_', //
+ [154] = '_', //
+ [155] = '_', //
+ [156] = '_', //
+ [157] = '_', //
+ [158] = '_', //
+ [159] = '_', //
+ [160] = '_', //
+ [161] = '_', //
+ [162] = '_', //
+ [163] = '_', //
+ [164] = '_', //
+ [165] = '_', //
+ [166] = '_', //
+ [167] = '_', //
+ [168] = '_', //
+ [169] = '_', //
+ [170] = '_', //
+ [171] = '_', //
+ [172] = '_', //
+ [173] = '_', //
+ [174] = '_', //
+ [175] = '_', //
+ [176] = '_', //
+ [177] = '_', //
+ [178] = '_', //
+ [179] = '_', //
+ [180] = '_', //
+ [181] = '_', //
+ [182] = '_', //
+ [183] = '_', //
+ [184] = '_', //
+ [185] = '_', //
+ [186] = '_', //
+ [187] = '_', //
+ [188] = '_', //
+ [189] = '_', //
+ [190] = '_', //
+ [191] = '_', //
+ [192] = '_', //
+ [193] = '_', //
+ [194] = '_', //
+ [195] = '_', //
+ [196] = '_', //
+ [197] = '_', //
+ [198] = '_', //
+ [199] = '_', //
+ [200] = '_', //
+ [201] = '_', //
+ [202] = '_', //
+ [203] = '_', //
+ [204] = '_', //
+ [205] = '_', //
+ [206] = '_', //
+ [207] = '_', //
+ [208] = '_', //
+ [209] = '_', //
+ [210] = '_', //
+ [211] = '_', //
+ [212] = '_', //
+ [213] = '_', //
+ [214] = '_', //
+ [215] = '_', //
+ [216] = '_', //
+ [217] = '_', //
+ [218] = '_', //
+ [219] = '_', //
+ [220] = '_', //
+ [221] = '_', //
+ [222] = '_', //
+ [223] = '_', //
+ [224] = '_', //
+ [225] = '_', //
+ [226] = '_', //
+ [227] = '_', //
+ [228] = '_', //
+ [229] = '_', //
+ [230] = '_', //
+ [231] = '_', //
+ [232] = '_', //
+ [233] = '_', //
+ [234] = '_', //
+ [235] = '_', //
+ [236] = '_', //
+ [237] = '_', //
+ [238] = '_', //
+ [239] = '_', //
+ [240] = '_', //
+ [241] = '_', //
+ [242] = '_', //
+ [243] = '_', //
+ [244] = '_', //
+ [245] = '_', //
+ [246] = '_', //
+ [247] = '_', //
+ [248] = '_', //
+ [249] = '_', //
+ [250] = '_', //
+ [251] = '_', //
+ [252] = '_', //
+ [253] = '_', //
+ [254] = '_', //
+ [255] = '_' //
+};
+
+// make sure the supplied string
+// is good for a netdata chart/dimension ID/NAME
+void netdata_fix_chart_id(char *s) {
+ while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++;
+}
+
+/*
+// http://stackoverflow.com/questions/7666509/hash-function-for-string
+uint32_t simple_hash(const char *name)
+{
+ const char *s = name;
+ uint32_t hash = 5381;
+ int i;
+
+ while((i = *s++)) hash = ((hash << 5) + hash) + i;
+
+ // fprintf(stderr, "HASH: %lu %s\n", hash, name);
+
+ return hash;
+}
+*/
+
+/*
+// http://isthe.com/chongo/tech/comp/fnv/#FNV-1a
+uint32_t simple_hash(const char *name) {
+ unsigned char *s = (unsigned char *) name;
+ uint32_t hval = 0x811c9dc5;
+
+ // FNV-1a algorithm
+ while (*s) {
+ // multiply by the 32 bit FNV magic prime mod 2^32
+ // NOTE: No need to optimize with left shifts.
+ // GCC will use imul instruction anyway.
+ // Tested with 'gcc -O3 -S'
+ //hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24);
+ hval *= 16777619;
+
+ // xor the bottom with the current octet
+ hval ^= (uint32_t) *s++;
+ }
+
+ // fprintf(stderr, "HASH: %u = %s\n", hval, name);
+ return hval;
+}
+
+uint32_t simple_uhash(const char *name) {
+ unsigned char *s = (unsigned char *) name;
+ uint32_t hval = 0x811c9dc5, c;
+
+ // FNV-1a algorithm
+ while ((c = *s++)) {
+ if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A';
+ hval *= 16777619;
+ hval ^= c;
+ }
+ return hval;
+}
+*/
+
+/*
+// http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx
+// one at a time hash
+uint32_t simple_hash(const char *name) {
+ unsigned char *s = (unsigned char *)name;
+ uint32_t h = 0;
+
+ while(*s) {
+ h += *s++;
+ h += (h << 10);
+ h ^= (h >> 6);
+ }
+
+ h += (h << 3);
+ h ^= (h >> 11);
+ h += (h << 15);
+
+ // fprintf(stderr, "HASH: %u = %s\n", h, name);
+
+ return h;
+}
+*/
+
+void strreverse(char *begin, char *end) {
+ while (end > begin) {
+ // clearer code.
+ char aux = *end;
+ *end-- = *begin;
+ *begin++ = aux;
+ }
+}
+
+char *strsep_on_1char(char **ptr, char c) {
+ if(unlikely(!ptr || !*ptr))
+ return NULL;
+
+ // remember the position we started
+ char *s = *ptr;
+
+ // skip separators in front
+ while(*s == c) s++;
+ char *ret = s;
+
+ // find the next separator
+ while(*s++) {
+ if(unlikely(*s == c)) {
+ *s++ = '\0';
+ *ptr = s;
+ return ret;
+ }
+ }
+
+ *ptr = NULL;
+ return ret;
+}
+
+char *mystrsep(char **ptr, char *s) {
+ char *p = "";
+ while (p && !p[0] && *ptr) p = strsep(ptr, s);
+ return (p);
+}
+
+char *trim(char *s) {
+ // skip leading spaces
+ while (*s && isspace(*s)) s++;
+ if (!*s) return NULL;
+
+ // skip tailing spaces
+ // this way is way faster. Writes only one NUL char.
+ ssize_t l = strlen(s);
+ if (--l >= 0) {
+ char *p = s + l;
+ while (p > s && isspace(*p)) p--;
+ *++p = '\0';
+ }
+
+ if (!*s) return NULL;
+
+ return s;
+}
+
+inline char *trim_all(char *buffer) {
+ char *d = buffer, *s = buffer;
+
+ // skip spaces
+ while(isspace(*s)) s++;
+
+ while(*s) {
+ // copy the non-space part
+ while(*s && !isspace(*s)) *d++ = *s++;
+
+ // add a space if we have to
+ if(*s && isspace(*s)) {
+ *d++ = ' ';
+ s++;
+ }
+
+ // skip spaces
+ while(isspace(*s)) s++;
+ }
+
+ *d = '\0';
+
+ if(d > buffer) {
+ d--;
+ if(isspace(*d)) *d = '\0';
+ }
+
+ if(!buffer[0]) return NULL;
+ return buffer;
+}
+
+static int memory_file_open(const char *filename, size_t size) {
+ // info("memory_file_open('%s', %zu", filename, size);
+
+ int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664);
+ if (fd != -1) {
+ if (lseek(fd, size, SEEK_SET) == (off_t) size) {
+ if (write(fd, "", 1) == 1) {
+ if (ftruncate(fd, size))
+ error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size);
+ }
+ else error("Cannot write to file '%s' at position %zu.", filename, size);
+ }
+ else error("Cannot seek file '%s' to size %zu.", filename, size);
+ }
+ else error("Cannot create/open file '%s'.", filename);
+
+ return fd;
+}
+
+// mmap_shared is used for memory mode = map
+static void *memory_file_mmap(const char *filename, size_t size, int flags) {
+ // info("memory_file_mmap('%s', %zu", filename, size);
+ static int log_madvise = 1;
+
+ int fd = -1;
+ if(filename) {
+ fd = memory_file_open(filename, size);
+ if(fd == -1) return MAP_FAILED;
+ }
+
+ void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0);
+ if (mem != MAP_FAILED) {
+#ifdef NETDATA_LOG_ALLOCATIONS
+ mmap_accounting(size);
+#endif
+ int advise = MADV_SEQUENTIAL | MADV_DONTFORK;
+ if (flags & MAP_SHARED) advise |= MADV_WILLNEED;
+
+ if (madvise(mem, size, advise) != 0 && log_madvise) {
+ error("Cannot advise the kernel about shared memory usage.");
+ log_madvise--;
+ }
+ }
+
+ if(fd != -1)
+ close(fd);
+
+ return mem;
+}
+
+#ifdef MADV_MERGEABLE
+static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) {
+ // info("memory_file_mmap_ksm('%s', %zu", filename, size);
+ static int log_madvise_2 = 1, log_madvise_3 = 1;
+
+ int fd = -1;
+ if(filename) {
+ fd = memory_file_open(filename, size);
+ if(fd == -1) return MAP_FAILED;
+ }
+
+ void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0);
+ if (mem != MAP_FAILED) {
+#ifdef NETDATA_LOG_ALLOCATIONS
+ mmap_accounting(size);
+#endif
+ if(fd != -1) {
+ if (lseek(fd, 0, SEEK_SET) == 0) {
+ if (read(fd, mem, size) != (ssize_t) size)
+ error("Cannot read from file '%s'", filename);
+ }
+ else error("Cannot seek to beginning of file '%s'.", filename);
+ }
+
+ // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE
+ if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) {
+ error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.", filename);
+ log_madvise_2--;
+ }
+
+ if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) {
+ error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.", filename);
+ log_madvise_3--;
+ }
+ }
+
+ if(fd != -1)
+ close(fd);
+
+ return mem;
+}
+#else
+static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) {
+ // info("memory_file_mmap_ksm FALLBACK ('%s', %zu", filename, size);
+
+ if(filename)
+ return memory_file_mmap(filename, size, flags);
+
+ // when KSM is not available and no filename is given (memory mode = ram),
+ // we just report failure
+ return MAP_FAILED;
+}
+#endif
+
+void *mymmap(const char *filename, size_t size, int flags, int ksm) {
+ void *mem = NULL;
+
+ if (filename && (flags & MAP_SHARED || !enable_ksm || !ksm))
+ // memory mode = map | save
+ // when KSM is not enabled
+ // MAP_SHARED is used for memory mode = map (no KSM possible)
+ mem = memory_file_mmap(filename, size, flags);
+
+ else
+ // memory mode = save | ram
+ // when KSM is enabled
+ // for memory mode = ram, the filename is NULL
+ mem = memory_file_mmap_ksm(filename, size, flags);
+
+ if(mem == MAP_FAILED) return NULL;
+
+ errno = 0;
+ return mem;
+}
+
+int memory_file_save(const char *filename, void *mem, size_t size) {
+ char tmpfilename[FILENAME_MAX + 1];
+
+ snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid());
+
+ int fd = open(tmpfilename, O_RDWR | O_CREAT | O_NOATIME, 0664);
+ if (fd < 0) {
+ error("Cannot create/open file '%s'.", filename);
+ return -1;
+ }
+
+ if (write(fd, mem, size) != (ssize_t) size) {
+ error("Cannot write to file '%s' %ld bytes.", filename, (long) size);
+ close(fd);
+ return -1;
+ }
+
+ close(fd);
+
+ if (rename(tmpfilename, filename)) {
+ error("Cannot rename '%s' to '%s'", tmpfilename, filename);
+ return -1;
+ }
+
+ return 0;
+}
+
+int fd_is_valid(int fd) {
+ return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
+}
+
+char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {
+ char *s = fgets(buf, (int)buf_size, fp);
+ if (!s) return NULL;
+
+ char *t = s;
+ if (*t != '\0') {
+ // find the string end
+ while (*++t != '\0');
+
+ // trim trailing spaces/newlines/tabs
+ while (--t > s && *t == '\n')
+ *t = '\0';
+ }
+
+ if (len)
+ *len = t - s + 1;
+
+ return s;
+}
+
+int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args) {
+ int size = vsnprintf(dst, n, fmt, args);
+
+ if (unlikely((size_t) size > n)) {
+ // truncated
+ size = (int)n;
+ }
+
+ dst[size] = '\0';
+ return size;
+}
+
+int snprintfz(char *dst, size_t n, const char *fmt, ...) {
+ va_list args;
+
+ va_start(args, fmt);
+ int ret = vsnprintfz(dst, n, fmt, args);
+ va_end(args);
+
+ return ret;
+}
+
+/*
+// poor man cycle counting
+static unsigned long tsc;
+void begin_tsc(void) {
+ unsigned long a, d;
+ asm volatile ("cpuid\nrdtsc" : "=a" (a), "=d" (d) : "0" (0) : "ebx", "ecx");
+ tsc = ((unsigned long)d << 32) | (unsigned long)a;
+}
+unsigned long end_tsc(void) {
+ unsigned long a, d;
+ asm volatile ("rdtscp" : "=a" (a), "=d" (d) : : "ecx");
+ return (((unsigned long)d << 32) | (unsigned long)a) - tsc;
+}
+*/
+
+int recursively_delete_dir(const char *path, const char *reason) {
+ DIR *dir = opendir(path);
+ if(!dir) {
+ error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path);
+ return -1;
+ }
+
+ int ret = 0;
+ struct dirent *de = NULL;
+ while((de = readdir(dir))) {
+ if(de->d_type == DT_DIR
+ && (
+ (de->d_name[0] == '.' && de->d_name[1] == '\0')
+ || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ ))
+ continue;
+
+ char fullpath[FILENAME_MAX + 1];
+ snprintfz(fullpath, FILENAME_MAX, "%s/%s", path, de->d_name);
+
+ if(de->d_type == DT_DIR) {
+ int r = recursively_delete_dir(fullpath, reason);
+ if(r > 0) ret += r;
+ continue;
+ }
+
+ info("Deleting %s file '%s'", reason?reason:"", fullpath);
+ if(unlikely(unlink(fullpath) == -1))
+ error("Cannot delete %s file '%s'", reason?reason:"", fullpath);
+ else
+ ret++;
+ }
+
+ info("Deleting empty directory '%s'", path);
+ if(unlikely(rmdir(path) == -1))
+ error("Cannot delete empty directory '%s'", path);
+ else
+ ret++;
+
+ closedir(dir);
+
+ return ret;
+}
+
+static int is_virtual_filesystem(const char *path, char **reason) {
+
+#if defined(__APPLE__) || defined(__FreeBSD__)
+ (void)path;
+ (void)reason;
+#else
+ struct statfs stat;
+ // stat.f_fsid.__val[0] is a file system id
+ // stat.f_fsid.__val[1] is the inode
+ // so their combination uniquely identifies the file/dir
+
+ if (statfs(path, &stat) == -1) {
+ if(reason) *reason = "failed to statfs()";
+ return -1;
+ }
+
+ if(stat.f_fsid.__val[0] != 0 || stat.f_fsid.__val[1] != 0) {
+ errno = EINVAL;
+ if(reason) *reason = "is not a virtual file system";
+ return -1;
+ }
+#endif
+
+ return 0;
+}
+
+int verify_netdata_host_prefix() {
+ if(!netdata_configured_host_prefix)
+ netdata_configured_host_prefix = "";
+
+ if(!*netdata_configured_host_prefix)
+ return 0;
+
+ char buffer[FILENAME_MAX + 1];
+ char *path = netdata_configured_host_prefix;
+ char *reason = "unknown reason";
+ errno = 0;
+
+ struct stat sb;
+ if (stat(path, &sb) == -1) {
+ reason = "failed to stat()";
+ goto failed;
+ }
+
+ if((sb.st_mode & S_IFMT) != S_IFDIR) {
+ errno = EINVAL;
+ reason = "is not a directory";
+ goto failed;
+ }
+
+ path = buffer;
+ snprintfz(path, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix);
+ if(is_virtual_filesystem(path, &reason) == -1)
+ goto failed;
+
+ snprintfz(path, FILENAME_MAX, "%s/sys", netdata_configured_host_prefix);
+ if(is_virtual_filesystem(path, &reason) == -1)
+ goto failed;
+
+ if(netdata_configured_host_prefix && *netdata_configured_host_prefix)
+ info("Using host prefix directory '%s'", netdata_configured_host_prefix);
+
+ return 0;
+
+failed:
+ error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason);
+ netdata_configured_host_prefix = "";
+ return -1;
+}
+
+char *strdupz_path_subpath(const char *path, const char *subpath) {
+ if(unlikely(!path || !*path)) path = ".";
+ if(unlikely(!subpath)) subpath = "";
+
+ // skip trailing slashes in path
+ size_t len = strlen(path);
+ while(len > 0 && path[len - 1] == '/') len--;
+
+ // skip leading slashes in subpath
+ while(subpath[0] == '/') subpath++;
+
+ // if the last character in path is / and (there is a subpath or path is now empty)
+ // keep the trailing slash in path and remove the additional slash
+ char *slash = "/";
+ if(path[len] == '/' && (*subpath || len == 0)) {
+ slash = "";
+ len++;
+ }
+ else if(!*subpath) {
+ // there is no subpath
+ // no need for trailing slash
+ slash = "";
+ }
+
+ char buffer[FILENAME_MAX + 1];
+ snprintfz(buffer, FILENAME_MAX, "%.*s%s%s", (int)len, path, slash, subpath);
+ return strdupz(buffer);
+}
+
+int path_is_dir(const char *path, const char *subpath) {
+ char *s = strdupz_path_subpath(path, subpath);
+
+ size_t max_links = 100;
+
+ int is_dir = 0;
+ struct stat statbuf;
+ while(max_links-- && stat(s, &statbuf) == 0) {
+ if((statbuf.st_mode & S_IFMT) == S_IFDIR) {
+ is_dir = 1;
+ break;
+ }
+ else if((statbuf.st_mode & S_IFMT) == S_IFLNK) {
+ char buffer[FILENAME_MAX + 1];
+ ssize_t l = readlink(s, buffer, FILENAME_MAX);
+ if(l > 0) {
+ buffer[l] = '\0';
+ freez(s);
+ s = strdupz(buffer);
+ continue;
+ }
+ else {
+ is_dir = 0;
+ break;
+ }
+ }
+ else {
+ is_dir = 0;
+ break;
+ }
+ }
+
+ freez(s);
+ return is_dir;
+}
+
+int path_is_file(const char *path, const char *subpath) {
+ char *s = strdupz_path_subpath(path, subpath);
+
+ size_t max_links = 100;
+
+ int is_file = 0;
+ struct stat statbuf;
+ while(max_links-- && stat(s, &statbuf) == 0) {
+ if((statbuf.st_mode & S_IFMT) == S_IFREG) {
+ is_file = 1;
+ break;
+ }
+ else if((statbuf.st_mode & S_IFMT) == S_IFLNK) {
+ char buffer[FILENAME_MAX + 1];
+ ssize_t l = readlink(s, buffer, FILENAME_MAX);
+ if(l > 0) {
+ buffer[l] = '\0';
+ freez(s);
+ s = strdupz(buffer);
+ continue;
+ }
+ else {
+ is_file = 0;
+ break;
+ }
+ }
+ else {
+ is_file = 0;
+ break;
+ }
+ }
+
+ freez(s);
+ return is_file;
+}
+
+void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data, size_t depth) {
+ if(depth > 3) {
+ error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath);
+ return;
+ }
+
+ char *udir = strdupz_path_subpath(user_path, subpath);
+ char *sdir = strdupz_path_subpath(stock_path, subpath);
+
+ debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir);
+
+ DIR *dir = opendir(udir);
+ if (!dir) {
+ error("CONFIG cannot open user-config directory '%s'.", udir);
+ }
+ else {
+ struct dirent *de = NULL;
+ while((de = readdir(dir))) {
+ if(de->d_type == DT_DIR || de->d_type == DT_LNK) {
+ if( !de->d_name[0] ||
+ (de->d_name[0] == '.' && de->d_name[1] == '\0') ||
+ (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ ) {
+ debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name);
+ continue;
+ }
+
+ if(path_is_dir(udir, de->d_name)) {
+ recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1);
+ continue;
+ }
+ }
+
+ if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) {
+ size_t len = strlen(de->d_name);
+ if(path_is_file(udir, de->d_name) &&
+ len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
+ char *filename = strdupz_path_subpath(udir, de->d_name);
+ debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename);
+ callback(filename, data);
+ freez(filename);
+ continue;
+ }
+ }
+
+ debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type);
+ }
+
+ closedir(dir);
+ }
+
+ debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir);
+
+ dir = opendir(sdir);
+ if (!dir) {
+ error("CONFIG cannot open stock config directory '%s'.", sdir);
+ }
+ else if (strcmp(udir, sdir)) {
+ struct dirent *de = NULL;
+ while((de = readdir(dir))) {
+ if(de->d_type == DT_DIR || de->d_type == DT_LNK) {
+ if( !de->d_name[0] ||
+ (de->d_name[0] == '.' && de->d_name[1] == '\0') ||
+ (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
+ ) {
+ debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name);
+ continue;
+ }
+
+ if(path_is_dir(sdir, de->d_name)) {
+ // we recurse in stock subdirectory, only when there is no corresponding
+ // user subdirectory - to avoid reading the files twice
+
+ if(!path_is_dir(udir, de->d_name))
+ recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1);
+
+ continue;
+ }
+ }
+
+ if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) {
+ size_t len = strlen(de->d_name);
+ if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) &&
+ len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
+ char *filename = strdupz_path_subpath(sdir, de->d_name);
+ debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename);
+ callback(filename, data);
+ freez(filename);
+ continue;
+ }
+
+ }
+
+ debug(D_HEALTH, "CONFIG ignoring stock-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type);
+ }
+
+ closedir(dir);
+ }
+
+ debug(D_HEALTH, "CONFIG done traversing user-config directory '%s', stock config directory '%s'", udir, sdir);
+
+ freez(udir);
+ freez(sdir);
+}
+
+// Returns the number of bytes read from the file if file_size is not NULL.
+// The actual buffer has an extra byte set to zero (not included in the count).
+char *read_by_filename(char *filename, long *file_size)
+{
+ FILE *f = fopen(filename, "r");
+ if (!f)
+ return NULL;
+ if (fseek(f, 0, SEEK_END) < 0) {
+ fclose(f);
+ return NULL;
+ }
+ long size = ftell(f);
+ if (size <= 0 || fseek(f, 0, SEEK_END) < 0) {
+ fclose(f);
+ return NULL;
+ }
+ char *contents = callocz(size + 1, 1);
+ if (!contents) {
+ fclose(f);
+ return NULL;
+ }
+ if (fseek(f, 0, SEEK_SET) < 0) {
+ fclose(f);
+ freez(contents);
+ return NULL;
+ }
+ size_t res = fread(contents, 1, size, f);
+ if ( res != (size_t)size) {
+ freez(contents);
+ fclose(f);
+ return NULL;
+ }
+ fclose(f);
+ if (file_size)
+ *file_size = size;
+ return contents;
+}
diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
new file mode 100644
index 0000000..50568b5
--- /dev/null
+++ b/libnetdata/libnetdata.h
@@ -0,0 +1,340 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LIB_H
+#define NETDATA_LIB_H 1
+
+# ifdef __cplusplus
+extern "C" {
+# endif
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#define OS_LINUX 1
+#define OS_FREEBSD 2
+#define OS_MACOS 3
+
+
+// ----------------------------------------------------------------------------
+// system include files for all netdata C programs
+
+/* select the memory allocator, based on autoconf findings */
+#if defined(ENABLE_JEMALLOC)
+
+#if defined(HAVE_JEMALLOC_JEMALLOC_H)
+#include <jemalloc/jemalloc.h>
+#else // !defined(HAVE_JEMALLOC_JEMALLOC_H)
+#include <malloc.h>
+#endif // !defined(HAVE_JEMALLOC_JEMALLOC_H)
+
+#elif defined(ENABLE_TCMALLOC)
+
+#include <google/tcmalloc.h>
+
+#else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */
+
+#if !(defined(__FreeBSD__) || defined(__APPLE__))
+#include <malloc.h>
+#endif /* __FreeBSD__ || __APPLE__ */
+
+#endif /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */
+
+// ----------------------------------------------------------------------------
+
+#if defined(__FreeBSD__)
+#include <pthread_np.h>
+#define NETDATA_OS_TYPE "freebsd"
+#elif defined(__APPLE__)
+#define NETDATA_OS_TYPE "macos"
+#else
+#define NETDATA_OS_TYPE "linux"
+#endif /* __FreeBSD__, __APPLE__*/
+
+#include <pthread.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <ctype.h>
+#include <string.h>
+#include <strings.h>
+#include <arpa/inet.h>
+#include <netinet/tcp.h>
+#include <sys/ioctl.h>
+#include <libgen.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <grp.h>
+#include <pwd.h>
+#include <limits.h>
+#include <locale.h>
+#include <net/if.h>
+#include <poll.h>
+#include <signal.h>
+#include <syslog.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/un.h>
+#include <time.h>
+#include <unistd.h>
+#include <uuid/uuid.h>
+#include <spawn.h>
+#include <uv.h>
+#include <assert.h>
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#ifdef HAVE_RESOLV_H
+#include <resolv.h>
+#endif
+
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+
+#ifdef HAVE_SYS_PRCTL_H
+#include <sys/prctl.h>
+#endif
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#ifdef HAVE_SYS_VFS_H
+#include <sys/vfs.h>
+#endif
+
+#ifdef HAVE_SYS_STATFS_H
+#include <sys/statfs.h>
+#endif
+
+#ifdef HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+
+#ifdef HAVE_SYS_STATVFS_H
+#include <sys/statvfs.h>
+#endif
+
+// #1408
+#ifdef MAJOR_IN_MKDEV
+#include <sys/mkdev.h>
+#endif
+#ifdef MAJOR_IN_SYSMACROS
+#include <sys/sysmacros.h>
+#endif
+
+#ifdef STORAGE_WITH_MATH
+#include <math.h>
+#include <float.h>
+#endif
+
+#if defined(HAVE_INTTYPES_H)
+#include <inttypes.h>
+#elif defined(HAVE_STDINT_H)
+#include <stdint.h>
+#endif
+
+#ifdef NETDATA_WITH_ZLIB
+#include <zlib.h>
+#endif
+
+#ifdef HAVE_CAPABILITY
+#include <sys/capability.h>
+#endif
+
+
+// ----------------------------------------------------------------------------
+// netdata common definitions
+
+#if (SIZEOF_VOID_P == 8)
+#define ENVIRONMENT64
+#elif (SIZEOF_VOID_P == 4)
+#define ENVIRONMENT32
+#else
+#error "Cannot detect if this is a 32 or 64 bit CPU"
+#endif
+
+#ifdef __GNUC__
+#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
+#endif // __GNUC__
+
+#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL
+#define NEVERNULL __attribute__((returns_nonnull))
+#else
+#define NEVERNULL
+#endif
+
+#ifdef HAVE_FUNC_ATTRIBUTE_NOINLINE
+#define NOINLINE __attribute__((noinline))
+#else
+#define NOINLINE
+#endif
+
+#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC
+#define MALLOCLIKE __attribute__((malloc))
+#else
+#define MALLOCLIKE
+#endif
+
+#ifdef HAVE_FUNC_ATTRIBUTE_FORMAT
+#define PRINTFLIKE(f, a) __attribute__ ((format(__printf__, f, a)))
+#else
+#define PRINTFLIKE(f, a)
+#endif
+
+#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN
+#define NORETURN __attribute__ ((noreturn))
+#else
+#define NORETURN
+#endif
+
+#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT
+#define WARNUNUSED __attribute__ ((warn_unused_result))
+#else
+#define WARNUNUSED
+#endif
+
+#ifdef abs
+#undef abs
+#endif
+#define abs(x) (((x) < 0)? (-(x)) : (x))
+
+#define MIN(a,b) (((a)<(b))?(a):(b))
+#define MAX(a,b) (((a)>(b))?(a):(b))
+
+#define GUID_LEN 36
+
+extern void netdata_fix_chart_id(char *s);
+extern void netdata_fix_chart_name(char *s);
+
+extern void strreverse(char* begin, char* end);
+extern char *mystrsep(char **ptr, char *s);
+extern char *trim(char *s); // remove leading and trailing spaces; may return NULL
+extern char *trim_all(char *buffer); // like trim(), but also remove duplicate spaces inside the string; may return NULL
+
+extern int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args);
+extern int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4);
+
+// memory allocation functions that handle failures
+#ifdef NETDATA_LOG_ALLOCATIONS
+extern __thread size_t log_thread_memory_allocations;
+#define strdupz(s) strdupz_int(__FILE__, __FUNCTION__, __LINE__, s)
+#define callocz(nmemb, size) callocz_int(__FILE__, __FUNCTION__, __LINE__, nmemb, size)
+#define mallocz(size) mallocz_int(__FILE__, __FUNCTION__, __LINE__, size)
+#define reallocz(ptr, size) reallocz_int(__FILE__, __FUNCTION__, __LINE__, ptr, size)
+#define freez(ptr) freez_int(__FILE__, __FUNCTION__, __LINE__, ptr)
+
+extern char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s);
+extern void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size);
+extern void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size);
+extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size);
+extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr);
+#else // NETDATA_LOG_ALLOCATIONS
+extern char *strdupz(const char *s) MALLOCLIKE NEVERNULL;
+extern void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;
+extern void *mallocz(size_t size) MALLOCLIKE NEVERNULL;
+extern void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL;
+extern void freez(void *ptr);
+#endif // NETDATA_LOG_ALLOCATIONS
+
+extern void json_escape_string(char *dst, const char *src, size_t size);
+extern void json_fix_string(char *s);
+
+extern void *mymmap(const char *filename, size_t size, int flags, int ksm);
+extern int memory_file_save(const char *filename, void *mem, size_t size);
+
+extern int fd_is_valid(int fd);
+
+extern struct rlimit rlimit_nofile;
+
+extern int enable_ksm;
+
+extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);
+
+extern int verify_netdata_host_prefix();
+
+extern int recursively_delete_dir(const char *path, const char *reason);
+
+extern volatile sig_atomic_t netdata_exit;
+extern const char *os_type;
+
+extern const char *program_version;
+
+extern char *strdupz_path_subpath(const char *path, const char *subpath);
+extern int path_is_dir(const char *path, const char *subpath);
+extern int path_is_file(const char *path, const char *subpath);
+extern void recursive_config_double_dir_load(
+ const char *user_path
+ , const char *stock_path
+ , const char *subpath
+ , int (*callback)(const char *filename, void *data)
+ , void *data
+ , size_t depth
+);
+extern char *read_by_filename(char *filename, long *file_size);
+
+/* fix for alpine linux */
+#ifndef RUSAGE_THREAD
+#ifdef RUSAGE_CHILDREN
+#define RUSAGE_THREAD RUSAGE_CHILDREN
+#endif
+#endif
+
+#define BITS_IN_A_KILOBIT 1000
+
+/* misc. */
+#define UNUSED(x) (void)(x)
+#define error_report(x, args...) do { errno = 0; error(x, ##args); } while(0)
+
+extern void netdata_cleanup_and_exit(int ret) NORETURN;
+extern void send_statistics(const char *action, const char *action_result, const char *action_data);
+extern char *netdata_configured_host_prefix;
+#include "os.h"
+#include "storage_number/storage_number.h"
+#include "threads/threads.h"
+#include "buffer/buffer.h"
+#include "locks/locks.h"
+#include "circular_buffer/circular_buffer.h"
+#include "avl/avl.h"
+#include "inlined.h"
+#include "clocks/clocks.h"
+#include "popen/popen.h"
+#include "simple_pattern/simple_pattern.h"
+#ifdef ENABLE_HTTPS
+# include "socket/security.h"
+#endif
+#include "socket/socket.h"
+#include "config/appconfig.h"
+#include "log/log.h"
+#include "procfile/procfile.h"
+#include "dictionary/dictionary.h"
+#ifdef HAVE_LIBBPF
+#include "ebpf/ebpf.h"
+#endif
+#include "eval/eval.h"
+#include "statistical/statistical.h"
+#include "adaptive_resortable_list/adaptive_resortable_list.h"
+#include "url/url.h"
+#include "json/json.h"
+#include "health/health.h"
+#include "string/utf8.h"
+
+// BEWARE: Outside of the C code this also exists in alarm-notify.sh
+#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
+
+# ifdef __cplusplus
+}
+# endif
+
+#endif // NETDATA_LIB_H
diff --git a/libnetdata/locks/Makefile.am b/libnetdata/locks/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/locks/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/locks/README.md b/libnetdata/locks/README.md
new file mode 100644
index 0000000..b525c97
--- /dev/null
+++ b/libnetdata/locks/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/locks/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Flocks%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/locks/locks.c b/libnetdata/locks/locks.c
new file mode 100644
index 0000000..ca9a5ae
--- /dev/null
+++ b/libnetdata/locks/locks.c
@@ -0,0 +1,343 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+// ----------------------------------------------------------------------------
+// automatic thread cancelability management, based on locks
+
+static __thread int netdata_thread_first_cancelability = 0;
+static __thread int netdata_thread_lock_cancelability = 0;
+
+inline void netdata_thread_disable_cancelability(void) {
+ int old;
+ int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old);
+ if(ret != 0)
+ error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret);
+ else {
+ if(!netdata_thread_lock_cancelability)
+ netdata_thread_first_cancelability = old;
+
+ netdata_thread_lock_cancelability++;
+ }
+}
+
+inline void netdata_thread_enable_cancelability(void) {
+ if(netdata_thread_lock_cancelability < 1) {
+ error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d on thread %s - results will be undefined - please report this!", netdata_thread_lock_cancelability, netdata_thread_tag());
+ }
+ else if(netdata_thread_lock_cancelability == 1) {
+ int old = 1;
+ int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old);
+ if(ret != 0)
+ error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret);
+ else {
+ if(old != PTHREAD_CANCEL_DISABLE)
+ error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, (old == PTHREAD_CANCEL_ENABLE)?"ENABLED":"UNKNOWN", old);
+ }
+
+ netdata_thread_lock_cancelability = 0;
+ }
+ else
+ netdata_thread_lock_cancelability--;
+}
+
+// ----------------------------------------------------------------------------
+// mutex
+
+int __netdata_mutex_init(netdata_mutex_t *mutex) {
+ int ret = pthread_mutex_init(mutex, NULL);
+ if(unlikely(ret != 0))
+ error("MUTEX_LOCK: failed to initialize (code %d).", ret);
+ return ret;
+}
+
+int __netdata_mutex_lock(netdata_mutex_t *mutex) {
+ netdata_thread_disable_cancelability();
+
+ int ret = pthread_mutex_lock(mutex);
+ if(unlikely(ret != 0)) {
+ netdata_thread_enable_cancelability();
+ error("MUTEX_LOCK: failed to get lock (code %d)", ret);
+ }
+ return ret;
+}
+
+int __netdata_mutex_trylock(netdata_mutex_t *mutex) {
+ netdata_thread_disable_cancelability();
+
+ int ret = pthread_mutex_trylock(mutex);
+ if(ret != 0)
+ netdata_thread_enable_cancelability();
+
+ return ret;
+}
+
+int __netdata_mutex_unlock(netdata_mutex_t *mutex) {
+ int ret = pthread_mutex_unlock(mutex);
+ if(unlikely(ret != 0))
+ error("MUTEX_LOCK: failed to unlock (code %d).", ret);
+ else
+ netdata_thread_enable_cancelability();
+
+ return ret;
+}
+
+int netdata_mutex_init_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_mutex_t *mutex) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+ }
+
+ int ret = __netdata_mutex_init(mutex);
+
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_mutex_lock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_mutex_t *mutex) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+ }
+
+ int ret = __netdata_mutex_lock(mutex);
+
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_mutex_trylock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_mutex_t *mutex) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+ }
+
+ int ret = __netdata_mutex_trylock(mutex);
+
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_mutex_t *mutex) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+ }
+
+ int ret = __netdata_mutex_unlock(mutex);
+
+ debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+
+// ----------------------------------------------------------------------------
+// r/w lock
+
+int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) {
+ int ret = pthread_rwlock_destroy(rwlock);
+ if(unlikely(ret != 0))
+ error("RW_LOCK: failed to destroy lock (code %d)", ret);
+ return ret;
+}
+
+int __netdata_rwlock_init(netdata_rwlock_t *rwlock) {
+ int ret = pthread_rwlock_init(rwlock, NULL);
+ if(unlikely(ret != 0))
+ error("RW_LOCK: failed to initialize lock (code %d)", ret);
+ return ret;
+}
+
+int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {
+ netdata_thread_disable_cancelability();
+
+ int ret = pthread_rwlock_rdlock(rwlock);
+ if(unlikely(ret != 0)) {
+ netdata_thread_enable_cancelability();
+ error("RW_LOCK: failed to obtain read lock (code %d)", ret);
+ }
+
+ return ret;
+}
+
+int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {
+ netdata_thread_disable_cancelability();
+
+ int ret = pthread_rwlock_wrlock(rwlock);
+ if(unlikely(ret != 0)) {
+ error("RW_LOCK: failed to obtain write lock (code %d)", ret);
+ netdata_thread_enable_cancelability();
+ }
+
+ return ret;
+}
+
+int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) {
+ int ret = pthread_rwlock_unlock(rwlock);
+ if(unlikely(ret != 0))
+ error("RW_LOCK: failed to release lock (code %d)", ret);
+ else
+ netdata_thread_enable_cancelability();
+
+ return ret;
+}
+
+int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {
+ netdata_thread_disable_cancelability();
+
+ int ret = pthread_rwlock_tryrdlock(rwlock);
+ if(ret != 0)
+ netdata_thread_enable_cancelability();
+
+ return ret;
+}
+
+int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {
+ netdata_thread_disable_cancelability();
+
+ int ret = pthread_rwlock_trywrlock(rwlock);
+ if(ret != 0)
+ netdata_thread_enable_cancelability();
+
+ return ret;
+}
+
+
+int netdata_rwlock_destroy_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+ }
+
+ int ret = __netdata_rwlock_destroy(rwlock);
+
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_rwlock_init_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+ }
+
+ int ret = __netdata_rwlock_init(rwlock);
+
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_rwlock_rdlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+ }
+
+ int ret = __netdata_rwlock_rdlock(rwlock);
+
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_rwlock_wrlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+ }
+
+ int ret = __netdata_rwlock_wrlock(rwlock);
+
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_rwlock_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+ }
+
+ int ret = __netdata_rwlock_unlock(rwlock);
+
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_rwlock_tryrdlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+ }
+
+ int ret = __netdata_rwlock_tryrdlock(rwlock);
+
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
+
+int netdata_rwlock_trywrlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+ usec_t start = 0;
+ (void)start;
+
+ if(unlikely(debug_flags & D_LOCKS)) {
+ start = now_boottime_usec();
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+ }
+
+ int ret = __netdata_rwlock_trywrlock(rwlock);
+
+ debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+ return ret;
+}
diff --git a/libnetdata/locks/locks.h b/libnetdata/locks/locks.h
new file mode 100644
index 0000000..850dd7e
--- /dev/null
+++ b/libnetdata/locks/locks.h
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LOCKS_H
+#define NETDATA_LOCKS_H 1
+
+#include "../libnetdata.h"
+
+typedef pthread_mutex_t netdata_mutex_t;
+#define NETDATA_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+
+typedef pthread_rwlock_t netdata_rwlock_t;
+#define NETDATA_RWLOCK_INITIALIZER PTHREAD_RWLOCK_INITIALIZER
+
+extern int __netdata_mutex_init(netdata_mutex_t *mutex);
+extern int __netdata_mutex_lock(netdata_mutex_t *mutex);
+extern int __netdata_mutex_trylock(netdata_mutex_t *mutex);
+extern int __netdata_mutex_unlock(netdata_mutex_t *mutex);
+
+extern int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_init(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock);
+extern int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock);
+
+extern int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+extern int netdata_mutex_lock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+extern int netdata_mutex_trylock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+extern int netdata_mutex_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
+
+extern int netdata_rwlock_destroy_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+extern int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+
+extern void netdata_thread_disable_cancelability(void);
+extern void netdata_thread_enable_cancelability(void);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+
+#define netdata_mutex_init(mutex) netdata_mutex_init_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
+#define netdata_mutex_lock(mutex) netdata_mutex_lock_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
+#define netdata_mutex_trylock(mutex) netdata_mutex_trylock_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
+#define netdata_mutex_unlock(mutex) netdata_mutex_unlock_debug(__FILE__, __FUNCTION__, __LINE__, mutex)
+
+#define netdata_rwlock_destroy(rwlock) netdata_rwlock_destroy_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_init(rwlock) netdata_rwlock_init_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_rdlock(rwlock) netdata_rwlock_rdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_wrlock(rwlock) netdata_rwlock_wrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_unlock(rwlock) netdata_rwlock_unlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_tryrdlock(rwlock) netdata_rwlock_tryrdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_trywrlock(rwlock) netdata_rwlock_trywrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+
+#else // !NETDATA_INTERNAL_CHECKS
+
+#define netdata_mutex_init(mutex) __netdata_mutex_init(mutex)
+#define netdata_mutex_lock(mutex) __netdata_mutex_lock(mutex)
+#define netdata_mutex_trylock(mutex) __netdata_mutex_trylock(mutex)
+#define netdata_mutex_unlock(mutex) __netdata_mutex_unlock(mutex)
+
+#define netdata_rwlock_destroy(rwlock) __netdata_rwlock_destroy(rwlock)
+#define netdata_rwlock_init(rwlock) __netdata_rwlock_init(rwlock)
+#define netdata_rwlock_rdlock(rwlock) __netdata_rwlock_rdlock(rwlock)
+#define netdata_rwlock_wrlock(rwlock) __netdata_rwlock_wrlock(rwlock)
+#define netdata_rwlock_unlock(rwlock) __netdata_rwlock_unlock(rwlock)
+#define netdata_rwlock_tryrdlock(rwlock) __netdata_rwlock_tryrdlock(rwlock)
+#define netdata_rwlock_trywrlock(rwlock) __netdata_rwlock_trywrlock(rwlock)
+
+#endif // NETDATA_INTERNAL_CHECKS
+
+#endif //NETDATA_LOCKS_H
diff --git a/libnetdata/log/Makefile.am b/libnetdata/log/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/log/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/log/README.md b/libnetdata/log/README.md
new file mode 100644
index 0000000..c0d7a34
--- /dev/null
+++ b/libnetdata/log/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/log/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Flog%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c
new file mode 100644
index 0000000..95d3d98
--- /dev/null
+++ b/libnetdata/log/log.c
@@ -0,0 +1,872 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <daemon/main.h>
+#include "../libnetdata.h"
+
+int web_server_is_multithreaded = 1;
+
+const char *program_name = "";
+uint64_t debug_flags = DEBUG;
+
+int access_log_syslog = 1;
+int error_log_syslog = 1;
+int output_log_syslog = 1; // debug log
+
+int stdaccess_fd = -1;
+FILE *stdaccess = NULL;
+
+const char *stdaccess_filename = NULL;
+const char *stderr_filename = NULL;
+const char *stdout_filename = NULL;
+const char *facility_log = NULL;
+
+// ----------------------------------------------------------------------------
+// Log facility(https://tools.ietf.org/html/rfc5424)
+//
+// The facilities accepted in the Netdata are in according with the following
+// header files for their respective operate system:
+// sys/syslog.h (Linux )
+// sys/sys/syslog.h (FreeBSD)
+// bsd/sys/syslog.h (darwin-xnu)
+
+#define LOG_AUTH_KEY "auth"
+#define LOG_AUTHPRIV_KEY "authpriv"
+#ifdef __FreeBSD__
+# define LOG_CONSOLE_KEY "console"
+#endif
+#define LOG_CRON_KEY "cron"
+#define LOG_DAEMON_KEY "daemon"
+#define LOG_FTP_KEY "ftp"
+#ifdef __APPLE__
+# define LOG_INSTALL_KEY "install"
+#endif
+#define LOG_KERN_KEY "kern"
+#define LOG_LPR_KEY "lpr"
+#define LOG_MAIL_KEY "mail"
+//#define LOG_INTERNAL_MARK_KEY "mark"
+#ifdef __APPLE__
+# define LOG_NETINFO_KEY "netinfo"
+# define LOG_RAS_KEY "ras"
+# define LOG_REMOTEAUTH_KEY "remoteauth"
+#endif
+#define LOG_NEWS_KEY "news"
+#ifdef __FreeBSD__
+# define LOG_NTP_KEY "ntp"
+#endif
+#define LOG_SECURITY_KEY "security"
+#define LOG_SYSLOG_KEY "syslog"
+#define LOG_USER_KEY "user"
+#define LOG_UUCP_KEY "uucp"
+#ifdef __APPLE__
+# define LOG_LAUNCHD_KEY "launchd"
+#endif
+#define LOG_LOCAL0_KEY "local0"
+#define LOG_LOCAL1_KEY "local1"
+#define LOG_LOCAL2_KEY "local2"
+#define LOG_LOCAL3_KEY "local3"
+#define LOG_LOCAL4_KEY "local4"
+#define LOG_LOCAL5_KEY "local5"
+#define LOG_LOCAL6_KEY "local6"
+#define LOG_LOCAL7_KEY "local7"
+
+static int log_facility_id(const char *facility_name)
+{
+ static int
+ hash_auth = 0,
+ hash_authpriv = 0,
+#ifdef __FreeBSD__
+ hash_console = 0,
+#endif
+ hash_cron = 0,
+ hash_daemon = 0,
+ hash_ftp = 0,
+#ifdef __APPLE__
+ hash_install = 0,
+#endif
+ hash_kern = 0,
+ hash_lpr = 0,
+ hash_mail = 0,
+// hash_mark = 0,
+#ifdef __APPLE__
+ hash_netinfo = 0,
+ hash_ras = 0,
+ hash_remoteauth = 0,
+#endif
+ hash_news = 0,
+#ifdef __FreeBSD__
+ hash_ntp = 0,
+#endif
+ hash_security = 0,
+ hash_syslog = 0,
+ hash_user = 0,
+ hash_uucp = 0,
+#ifdef __APPLE__
+ hash_launchd = 0,
+#endif
+ hash_local0 = 0,
+ hash_local1 = 0,
+ hash_local2 = 0,
+ hash_local3 = 0,
+ hash_local4 = 0,
+ hash_local5 = 0,
+ hash_local6 = 0,
+ hash_local7 = 0;
+
+ if(unlikely(!hash_auth))
+ {
+ hash_auth = simple_hash(LOG_AUTH_KEY);
+ hash_authpriv = simple_hash(LOG_AUTHPRIV_KEY);
+#ifdef __FreeBSD__
+ hash_console = simple_hash(LOG_CONSOLE_KEY);
+#endif
+ hash_cron = simple_hash(LOG_CRON_KEY);
+ hash_daemon = simple_hash(LOG_DAEMON_KEY);
+ hash_ftp = simple_hash(LOG_FTP_KEY);
+#ifdef __APPLE__
+ hash_install = simple_hash(LOG_INSTALL_KEY);
+#endif
+ hash_kern = simple_hash(LOG_KERN_KEY);
+ hash_lpr = simple_hash(LOG_LPR_KEY);
+ hash_mail = simple_hash(LOG_MAIL_KEY);
+// hash_mark = simple_uhash();
+#ifdef __APPLE__
+ hash_netinfo = simple_hash(LOG_NETINFO_KEY);
+ hash_ras = simple_hash(LOG_RAS_KEY);
+ hash_remoteauth = simple_hash(LOG_REMOTEAUTH_KEY);
+#endif
+ hash_news = simple_hash(LOG_NEWS_KEY);
+#ifdef __FreeBSD__
+ hash_ntp = simple_hash(LOG_NTP_KEY);
+#endif
+ hash_security = simple_hash(LOG_SECURITY_KEY);
+ hash_syslog = simple_hash(LOG_SYSLOG_KEY);
+ hash_user = simple_hash(LOG_USER_KEY);
+ hash_uucp = simple_hash(LOG_UUCP_KEY);
+#ifdef __APPLE__
+ hash_launchd = simple_hash(LOG_LAUNCHD_KEY);
+#endif
+ hash_local0 = simple_hash(LOG_LOCAL0_KEY);
+ hash_local1 = simple_hash(LOG_LOCAL1_KEY);
+ hash_local2 = simple_hash(LOG_LOCAL2_KEY);
+ hash_local3 = simple_hash(LOG_LOCAL3_KEY);
+ hash_local4 = simple_hash(LOG_LOCAL4_KEY);
+ hash_local5 = simple_hash(LOG_LOCAL5_KEY);
+ hash_local6 = simple_hash(LOG_LOCAL6_KEY);
+ hash_local7 = simple_hash(LOG_LOCAL7_KEY);
+ }
+
+ int hash = simple_hash(facility_name);
+ if ( hash == hash_auth )
+ {
+ return LOG_AUTH;
+ }
+ else if ( hash == hash_authpriv )
+ {
+ return LOG_AUTHPRIV;
+ }
+#ifdef __FreeBSD__
+ else if ( hash == hash_console )
+ {
+ return LOG_CONSOLE;
+ }
+#endif
+ else if ( hash == hash_cron )
+ {
+ return LOG_CRON;
+ }
+ else if ( hash == hash_daemon )
+ {
+ return LOG_DAEMON;
+ }
+ else if ( hash == hash_ftp )
+ {
+ return LOG_FTP;
+ }
+#ifdef __APPLE__
+ else if ( hash == hash_install )
+ {
+ return LOG_INSTALL;
+ }
+#endif
+ else if ( hash == hash_kern )
+ {
+ return LOG_KERN;
+ }
+ else if ( hash == hash_lpr )
+ {
+ return LOG_LPR;
+ }
+ else if ( hash == hash_mail )
+ {
+ return LOG_MAIL;
+ }
+ /*
+ else if ( hash == hash_mark )
+ {
+ //this is internal for all OS
+ return INTERNAL_MARK;
+ }
+ */
+#ifdef __APPLE__
+ else if ( hash == hash_netinfo )
+ {
+ return LOG_NETINFO;
+ }
+ else if ( hash == hash_ras )
+ {
+ return LOG_RAS;
+ }
+ else if ( hash == hash_remoteauth )
+ {
+ return LOG_REMOTEAUTH;
+ }
+#endif
+ else if ( hash == hash_news )
+ {
+ return LOG_NEWS;
+ }
+#ifdef __FreeBSD__
+ else if ( hash == hash_ntp )
+ {
+ return LOG_NTP;
+ }
+#endif
+ else if ( hash == hash_security )
+ {
+ //FreeBSD is the unique that does not consider
+ //this facility deprecated. We are keeping
+ //it for other OS while they are kept in their headers.
+#ifdef __FreeBSD__
+ return LOG_SECURITY;
+#else
+ return LOG_AUTH;
+#endif
+ }
+ else if ( hash == hash_syslog )
+ {
+ return LOG_SYSLOG;
+ }
+ else if ( hash == hash_user )
+ {
+ return LOG_USER;
+ }
+ else if ( hash == hash_uucp )
+ {
+ return LOG_UUCP;
+ }
+ else if ( hash == hash_local0 )
+ {
+ return LOG_LOCAL0;
+ }
+ else if ( hash == hash_local1 )
+ {
+ return LOG_LOCAL1;
+ }
+ else if ( hash == hash_local2 )
+ {
+ return LOG_LOCAL2;
+ }
+ else if ( hash == hash_local3 )
+ {
+ return LOG_LOCAL3;
+ }
+ else if ( hash == hash_local4 )
+ {
+ return LOG_LOCAL4;
+ }
+ else if ( hash == hash_local5 )
+ {
+ return LOG_LOCAL5;
+ }
+ else if ( hash == hash_local6 )
+ {
+ return LOG_LOCAL6;
+ }
+ else if ( hash == hash_local7 )
+ {
+ return LOG_LOCAL7;
+ }
+#ifdef __APPLE__
+ else if ( hash == hash_launchd )
+ {
+ return LOG_LAUNCHD;
+ }
+#endif
+
+ return LOG_DAEMON;
+}
+
+//we do not need to use this now, but I already created this function to be
+//used case necessary.
+/*
+char *log_facility_name(int code)
+{
+ char *defvalue = { "daemon" };
+ switch(code)
+ {
+ case LOG_AUTH:
+ {
+ return "auth";
+ }
+ case LOG_AUTHPRIV:
+ {
+ return "authpriv";
+ }
+#ifdef __FreeBSD__
+ case LOG_CONSOLE:
+ {
+ return "console";
+ }
+#endif
+ case LOG_CRON:
+ {
+ return "cron";
+ }
+ case LOG_DAEMON:
+ {
+ return defvalue;
+ }
+ case LOG_FTP:
+ {
+ return "ftp";
+ }
+#ifdef __APPLE__
+ case LOG_INSTALL:
+ {
+ return "install";
+ }
+#endif
+ case LOG_KERN:
+ {
+ return "kern";
+ }
+ case LOG_LPR:
+ {
+ return "lpr";
+ }
+ case LOG_MAIL:
+ {
+ return "mail";
+ }
+#ifdef __APPLE__
+ case LOG_NETINFO:
+ {
+ return "netinfo" ;
+ }
+ case LOG_RAS:
+ {
+ return "ras";
+ }
+ case LOG_REMOTEAUTH:
+ {
+ return "remoteauth";
+ }
+#endif
+ case LOG_NEWS:
+ {
+ return "news";
+ }
+#ifdef __FreeBSD__
+ case LOG_NTP:
+ {
+ return "ntp" ;
+ }
+ case LOG_SECURITY:
+ {
+ return "security";
+ }
+#endif
+ case LOG_SYSLOG:
+ {
+ return "syslog";
+ }
+ case LOG_USER:
+ {
+ return "user";
+ }
+ case LOG_UUCP:
+ {
+ return "uucp";
+ }
+ case LOG_LOCAL0:
+ {
+ return "local0";
+ }
+ case LOG_LOCAL1:
+ {
+ return "local1";
+ }
+ case LOG_LOCAL2:
+ {
+ return "local2";
+ }
+ case LOG_LOCAL3:
+ {
+ return "local3";
+ }
+ case LOG_LOCAL4:
+ {
+ return "local4" ;
+ }
+ case LOG_LOCAL5:
+ {
+ return "local5";
+ }
+ case LOG_LOCAL6:
+ {
+ return "local6";
+ }
+ case LOG_LOCAL7:
+ {
+ return "local7" ;
+ }
+#ifdef __APPLE__
+ case LOG_LAUNCHD:
+ {
+ return "launchd";
+ }
+#endif
+ }
+
+ return defvalue;
+}
+*/
+
+// ----------------------------------------------------------------------------
+
+void syslog_init() {
+ static int i = 0;
+
+ if(!i) {
+ openlog(program_name, LOG_PID,log_facility_id(facility_log));
+ i = 1;
+ }
+}
+
+#define LOG_DATE_LENGTH 26
+
+static inline void log_date(char *buffer, size_t len) {
+ if(unlikely(!buffer || !len))
+ return;
+
+ time_t t;
+ struct tm *tmp, tmbuf;
+
+ t = now_realtime_sec();
+ tmp = localtime_r(&t, &tmbuf);
+
+ if (tmp == NULL) {
+ buffer[0] = '\0';
+ return;
+ }
+
+ if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
+ buffer[0] = '\0';
+
+ buffer[len - 1] = '\0';
+}
+
+static netdata_mutex_t log_mutex = NETDATA_MUTEX_INITIALIZER;
+static inline void log_lock() {
+ netdata_mutex_lock(&log_mutex);
+}
+static inline void log_unlock() {
+ netdata_mutex_unlock(&log_mutex);
+}
+
+static FILE *open_log_file(int fd, FILE *fp, const char *filename, int *enabled_syslog, int is_stdaccess, int *fd_ptr) {
+ int f, devnull = 0;
+
+ if(!filename || !*filename || !strcmp(filename, "none") || !strcmp(filename, "/dev/null")) {
+ filename = "/dev/null";
+ devnull = 1;
+ }
+
+ if(!strcmp(filename, "syslog")) {
+ filename = "/dev/null";
+ devnull = 1;
+
+ syslog_init();
+ if(enabled_syslog) *enabled_syslog = 1;
+ }
+ else if(enabled_syslog) *enabled_syslog = 0;
+
+ // don't do anything if the user is willing
+ // to have the standard one
+ if(!strcmp(filename, "system")) {
+ if(fd != -1 && !is_stdaccess) {
+ if(fd_ptr) *fd_ptr = fd;
+ return fp;
+ }
+
+ filename = "stderr";
+ }
+
+ if(!strcmp(filename, "stdout"))
+ f = STDOUT_FILENO;
+
+ else if(!strcmp(filename, "stderr"))
+ f = STDERR_FILENO;
+
+ else {
+ f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
+ if(f == -1) {
+ error("Cannot open file '%s'. Leaving %d to its default.", filename, fd);
+ if(fd_ptr) *fd_ptr = fd;
+ return fp;
+ }
+ }
+
+ // if there is a level-2 file pointer
+ // flush it before switching the level-1 fds
+ if(fp)
+ fflush(fp);
+
+ if(devnull && is_stdaccess) {
+ fd = -1;
+ fp = NULL;
+ }
+
+ if(fd != f && fd != -1) {
+ // it automatically closes
+ int t = dup2(f, fd);
+ if (t == -1) {
+ error("Cannot dup2() new fd %d to old fd %d for '%s'", f, fd, filename);
+ close(f);
+ if(fd_ptr) *fd_ptr = fd;
+ return fp;
+ }
+ // info("dup2() new fd %d to old fd %d for '%s'", f, fd, filename);
+ close(f);
+ }
+ else fd = f;
+
+ if(!fp) {
+ fp = fdopen(fd, "a");
+ if (!fp)
+ error("Cannot fdopen() fd %d ('%s')", fd, filename);
+ else {
+ if (setvbuf(fp, NULL, _IOLBF, 0) != 0)
+ error("Cannot set line buffering on fd %d ('%s')", fd, filename);
+ }
+ }
+
+ if(fd_ptr) *fd_ptr = fd;
+ return fp;
+}
+
+void reopen_all_log_files() {
+ if(stdout_filename)
+ open_log_file(STDOUT_FILENO, stdout, stdout_filename, &output_log_syslog, 0, NULL);
+
+ if(stderr_filename)
+ open_log_file(STDERR_FILENO, stderr, stderr_filename, &error_log_syslog, 0, NULL);
+
+ if(stdaccess_filename)
+ stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd);
+}
+
+void open_all_log_files() {
+ // disable stdin
+ open_log_file(STDIN_FILENO, stdin, "/dev/null", NULL, 0, NULL);
+
+ open_log_file(STDOUT_FILENO, stdout, stdout_filename, &output_log_syslog, 0, NULL);
+ open_log_file(STDERR_FILENO, stderr, stderr_filename, &error_log_syslog, 0, NULL);
+ stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd);
+}
+
+// ----------------------------------------------------------------------------
+// error log throttling
+
+time_t error_log_throttle_period = 1200;
+unsigned long error_log_errors_per_period = 200;
+unsigned long error_log_errors_per_period_backup = 0;
+
+int error_log_limit(int reset) {
+ static time_t start = 0;
+ static unsigned long counter = 0, prevented = 0;
+
+ // fprintf(stderr, "FLOOD: counter=%lu, allowed=%lu, backup=%lu, period=%llu\n", counter, error_log_errors_per_period, error_log_errors_per_period_backup, (unsigned long long)error_log_throttle_period);
+
+ // do not throttle if the period is 0
+ if(error_log_throttle_period == 0)
+ return 0;
+
+ // prevent all logs if the errors per period is 0
+ if(error_log_errors_per_period == 0)
+#ifdef NETDATA_INTERNAL_CHECKS
+ return 0;
+#else
+ return 1;
+#endif
+
+ time_t now = now_monotonic_sec();
+ if(!start) start = now;
+
+ if(reset) {
+ if(prevented) {
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stderr, "%s: %s LOG FLOOD PROTECTION reset for process '%s' (prevented %lu logs in the last %ld seconds).\n"
+ , date
+ , program_name
+ , program_name
+ , prevented
+ , now - start
+ );
+ }
+
+ start = now;
+ counter = 0;
+ prevented = 0;
+ }
+
+ // detect if we log too much
+ counter++;
+
+ if(now - start > error_log_throttle_period) {
+ if(prevented) {
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stderr, "%s: %s LOG FLOOD PROTECTION resuming logging from process '%s' (prevented %lu logs in the last %ld seconds).\n"
+ , date
+ , program_name
+ , program_name
+ , prevented
+ , error_log_throttle_period
+ );
+ }
+
+ // restart the period accounting
+ start = now;
+ counter = 1;
+ prevented = 0;
+
+ // log this error
+ return 0;
+ }
+
+ if(counter > error_log_errors_per_period) {
+ if(!prevented) {
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stderr, "%s: %s LOG FLOOD PROTECTION too many logs (%lu logs in %ld seconds, threshold is set to %lu logs in %ld seconds). Preventing more logs from process '%s' for %ld seconds.\n"
+ , date
+ , program_name
+ , counter
+ , now - start
+ , error_log_errors_per_period
+ , error_log_throttle_period
+ , program_name
+ , start + error_log_throttle_period - now);
+ }
+
+ prevented++;
+
+ // prevent logging this error
+#ifdef NETDATA_INTERNAL_CHECKS
+ return 0;
+#else
+ return 1;
+#endif
+ }
+
+ return 0;
+}
+
+// ----------------------------------------------------------------------------
+// debug log
+
+void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
+ va_list args;
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+
+ va_start( args, fmt );
+ printf("%s: %s DEBUG : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function);
+ vprintf(fmt, args);
+ va_end( args );
+ putchar('\n');
+
+ if(output_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_ERR, fmt, args );
+ va_end( args );
+ }
+
+ fflush(stdout);
+}
+
+// ----------------------------------------------------------------------------
+// info log
+
+void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
+{
+ va_list args;
+
+ // prevent logging too much
+ if(error_log_limit(0)) return;
+
+ if(error_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_INFO, fmt, args );
+ va_end( args );
+ }
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+
+ log_lock();
+
+ va_start( args, fmt );
+ if(debug_flags) fprintf(stderr, "%s: %s INFO : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function);
+ else fprintf(stderr, "%s: %s INFO : %s : ", date, program_name, netdata_thread_tag());
+ vfprintf( stderr, fmt, args );
+ va_end( args );
+
+ fputc('\n', stderr);
+
+ log_unlock();
+}
+
+// ----------------------------------------------------------------------------
+// error log
+
+#if defined(STRERROR_R_CHAR_P)
+// GLIBC version of strerror_r
+static const char *strerror_result(const char *a, const char *b) { (void)b; return a; }
+#elif defined(HAVE_STRERROR_R)
+// POSIX version of strerror_r
+static const char *strerror_result(int a, const char *b) { (void)a; return b; }
+#elif defined(HAVE_C__GENERIC)
+
+// what a trick!
+// http://stackoverflow.com/questions/479207/function-overloading-in-c
+static const char *strerror_result_int(int a, const char *b) { (void)a; return b; }
+static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; }
+
+#define strerror_result(a, b) _Generic((a), \
+ int: strerror_result_int, \
+ char *: strerror_result_string \
+ )(a, b)
+
+#else
+#error "cannot detect the format of function strerror_r()"
+#endif
+
+void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
+ // save a copy of errno - just in case this function generates a new error
+ int __errno = errno;
+
+ va_list args;
+
+ // prevent logging too much
+ if(error_log_limit(0)) return;
+
+ if(error_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_ERR, fmt, args );
+ va_end( args );
+ }
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+
+ log_lock();
+
+ va_start( args, fmt );
+ if(debug_flags) fprintf(stderr, "%s: %s %-5.5s : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, prefix, netdata_thread_tag(), line, file, function);
+ else fprintf(stderr, "%s: %s %-5.5s : %s : ", date, program_name, prefix, netdata_thread_tag());
+ vfprintf( stderr, fmt, args );
+ va_end( args );
+
+ if(__errno) {
+ char buf[1024];
+ fprintf(stderr, " (errno %d, %s)\n", __errno, strerror_result(strerror_r(__errno, buf, 1023), buf));
+ errno = 0;
+ }
+ else
+ fputc('\n', stderr);
+
+ log_unlock();
+}
+
+void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
+ // save a copy of errno - just in case this function generates a new error
+ int __errno = errno;
+ va_list args;
+ const char *thread_tag;
+ char os_threadname[NETDATA_THREAD_NAME_MAX + 1];
+
+ if(error_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_CRIT, fmt, args );
+ va_end( args );
+ }
+
+ thread_tag = netdata_thread_tag();
+ if (!netdata_thread_tag_exists()) {
+ os_thread_get_current_name_np(os_threadname);
+ if ('\0' != os_threadname[0]) { /* If it is not an empty string replace "MAIN" thread_tag */
+ thread_tag = os_threadname;
+ }
+ }
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+
+ log_lock();
+
+ va_start( args, fmt );
+ if(debug_flags) fprintf(stderr, "%s: %s FATAL : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, thread_tag, line, file, function);
+ else fprintf(stderr, "%s: %s FATAL : %s : ", date, program_name, thread_tag);
+ vfprintf( stderr, fmt, args );
+ va_end( args );
+
+ perror(" # ");
+ fputc('\n', stderr);
+
+ log_unlock();
+
+ char action_data[70+1];
+ snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, __errno);
+ char action_result[60+1];
+
+ snprintfz(action_result, 60, "%s:%s", program_name, strncmp(thread_tag, "STREAM_RECEIVER", strlen("STREAM_RECEIVER")) ? thread_tag : "[x]");
+ send_statistics("FATAL", action_result, action_data);
+
+ netdata_cleanup_and_exit(1);
+}
+
+// ----------------------------------------------------------------------------
+// access log
+
+void log_access( const char *fmt, ... ) {
+ va_list args;
+
+ if(access_log_syslog) {
+ va_start( args, fmt );
+ vsyslog(LOG_INFO, fmt, args );
+ va_end( args );
+ }
+
+ if(stdaccess) {
+ static netdata_mutex_t access_mutex = NETDATA_MUTEX_INITIALIZER;
+
+ if(web_server_is_multithreaded)
+ netdata_mutex_lock(&access_mutex);
+
+ char date[LOG_DATE_LENGTH];
+ log_date(date, LOG_DATE_LENGTH);
+ fprintf(stdaccess, "%s: ", date);
+
+ va_start( args, fmt );
+ vfprintf( stdaccess, fmt, args );
+ va_end( args );
+ fputc('\n', stdaccess);
+
+ if(web_server_is_multithreaded)
+ netdata_mutex_unlock(&access_mutex);
+ }
+}
diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h
new file mode 100644
index 0000000..c99c151
--- /dev/null
+++ b/libnetdata/log/log.h
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LOG_H
+#define NETDATA_LOG_H 1
+
+# ifdef __cplusplus
+extern "C" {
+# endif
+
+#include "../libnetdata.h"
+
+#define D_WEB_BUFFER 0x0000000000000001
+#define D_WEB_CLIENT 0x0000000000000002
+#define D_LISTENER 0x0000000000000004
+#define D_WEB_DATA 0x0000000000000008
+#define D_OPTIONS 0x0000000000000010
+#define D_PROCNETDEV_LOOP 0x0000000000000020
+#define D_RRD_STATS 0x0000000000000040
+#define D_WEB_CLIENT_ACCESS 0x0000000000000080
+#define D_TC_LOOP 0x0000000000000100
+#define D_DEFLATE 0x0000000000000200
+#define D_CONFIG 0x0000000000000400
+#define D_PLUGINSD 0x0000000000000800
+#define D_CHILDS 0x0000000000001000
+#define D_EXIT 0x0000000000002000
+#define D_CHECKS 0x0000000000004000
+#define D_NFACCT_LOOP 0x0000000000008000
+#define D_PROCFILE 0x0000000000010000
+#define D_RRD_CALLS 0x0000000000020000
+#define D_DICTIONARY 0x0000000000040000
+#define D_MEMORY 0x0000000000080000
+#define D_CGROUP 0x0000000000100000
+#define D_REGISTRY 0x0000000000200000
+#define D_VARIABLES 0x0000000000400000
+#define D_HEALTH 0x0000000000800000
+#define D_CONNECT_TO 0x0000000001000000
+#define D_RRDHOST 0x0000000002000000
+#define D_LOCKS 0x0000000004000000
+#define D_BACKEND 0x0000000008000000
+#define D_STATSD 0x0000000010000000
+#define D_POLLFD 0x0000000020000000
+#define D_STREAM 0x0000000040000000
+#define D_RRDENGINE 0x0000000100000000
+#define D_ACLK 0x0000000200000000
+#define D_METADATALOG 0x0000000400000000
+#define D_GUIDLOG 0x0000000800000000
+#define D_SYSTEM 0x8000000000000000
+
+//#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)
+//#define DEBUG 0xffffffff
+#define DEBUG (0)
+
+extern int web_server_is_multithreaded;
+
+extern uint64_t debug_flags;
+
+extern const char *program_name;
+
+extern int stdaccess_fd;
+extern FILE *stdaccess;
+
+extern const char *stdaccess_filename;
+extern const char *stderr_filename;
+extern const char *stdout_filename;
+extern const char *facility_log;
+
+extern int access_log_syslog;
+extern int error_log_syslog;
+extern int output_log_syslog;
+
+extern time_t error_log_throttle_period;
+extern unsigned long error_log_errors_per_period, error_log_errors_per_period_backup;
+extern int error_log_limit(int reset);
+
+extern void open_all_log_files();
+extern void reopen_all_log_files();
+
+static inline void debug_dummy(void) {}
+
+#define error_log_limit_reset() do { error_log_errors_per_period = error_log_errors_per_period_backup; error_log_limit(1); } while(0)
+#define error_log_limit_unlimited() do { \
+ error_log_limit_reset(); \
+ error_log_errors_per_period = ((error_log_errors_per_period_backup * 10) < 10000) ? 10000 : (error_log_errors_per_period_backup * 10); \
+ } while(0)
+
+#ifdef NETDATA_INTERNAL_CHECKS
+#define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
+#else
+#define debug(type, args...) debug_dummy()
+#endif
+
+#define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args)
+#define infoerr(args...) error_int("INFO", __FILE__, __FUNCTION__, __LINE__, ##args)
+#define error(args...) error_int("ERROR", __FILE__, __FUNCTION__, __LINE__, ##args)
+#define fatal(args...) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args)
+#define fatal_assert(expr) ((expr) ? (void)(0) : fatal_int(__FILE__, __FUNCTION__, __LINE__, "Assertion `%s' failed", #expr))
+
+extern void send_statistics(const char *action, const char *action_result, const char *action_data);
+extern void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5);
+extern void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5);
+extern void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(5, 6);
+extern void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5);
+extern void log_access( const char *fmt, ... ) PRINTFLIKE(1, 2);
+
+# ifdef __cplusplus
+}
+# endif
+
+#endif /* NETDATA_LOG_H */
diff --git a/libnetdata/os.c b/libnetdata/os.c
new file mode 100644
index 0000000..4271a91
--- /dev/null
+++ b/libnetdata/os.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "os.h"
+
+// ----------------------------------------------------------------------------
+// system functions
+// to retrieve settings of the system
+
+int processors = 1;
+long get_system_cpus(void) {
+ processors = 1;
+
+#ifdef __APPLE__
+ int32_t tmp_processors;
+
+ if (unlikely(GETSYSCTL_BY_NAME("hw.logicalcpu", tmp_processors))) {
+ error("Assuming system has %d processors.", processors);
+ } else {
+ processors = tmp_processors;
+ }
+
+ return processors;
+#elif __FreeBSD__
+ int32_t tmp_processors;
+
+ if (unlikely(GETSYSCTL_BY_NAME("hw.ncpu", tmp_processors))) {
+ error("Assuming system has %d processors.", processors);
+ } else {
+ processors = tmp_processors;
+ }
+
+ return processors;
+#else
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix);
+
+ procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
+ if(!ff) {
+ error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
+ return processors;
+ }
+
+ ff = procfile_readall(ff);
+ if(!ff) {
+ error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors);
+ return processors;
+ }
+
+ processors = 0;
+ unsigned int i;
+ for(i = 0; i < procfile_lines(ff); i++) {
+ if(!procfile_linewords(ff, i)) continue;
+
+ if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) processors++;
+ }
+ processors--;
+ if(processors < 1) processors = 1;
+
+ procfile_close(ff);
+
+ debug(D_SYSTEM, "System has %d processors.", processors);
+ return processors;
+
+#endif /* __APPLE__, __FreeBSD__ */
+}
+
+pid_t pid_max = 32768;
+pid_t get_system_pid_max(void) {
+#ifdef __APPLE__
+ // As we currently do not know a solution to query pid_max from the os
+ // we use the number defined in bsd/sys/proc_internal.h in XNU sources
+ pid_max = 99999;
+ return pid_max;
+#elif __FreeBSD__
+ int32_t tmp_pid_max;
+
+ if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) {
+ pid_max = 99999;
+ error("Assuming system's maximum pid is %d.", pid_max);
+ } else {
+ pid_max = tmp_pid_max;
+ }
+
+ return pid_max;
+#else
+
+ static char read = 0;
+ if(unlikely(read)) return pid_max;
+ read = 1;
+
+ char filename[FILENAME_MAX + 1];
+ snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix);
+
+ unsigned long long max = 0;
+ if(read_single_number_file(filename, &max) != 0) {
+ error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max);
+ return pid_max;
+ }
+
+ if(!max) {
+ error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max);
+ return pid_max;
+ }
+
+ pid_max = (pid_t) max;
+ return pid_max;
+
+#endif /* __APPLE__, __FreeBSD__ */
+}
+
+unsigned int system_hz;
+void get_system_HZ(void) {
+ long ticks;
+
+ if ((ticks = sysconf(_SC_CLK_TCK)) == -1) {
+ error("Cannot get system clock ticks");
+ }
+
+ system_hz = (unsigned int) ticks;
+}
+
+// =====================================================================================================================
+// FreeBSD
+
+#if (TARGET_OS == OS_FREEBSD)
+
+int getsysctl_by_name(const char *name, void *ptr, size_t len) {
+ size_t nlen = len;
+
+ if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) {
+ error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
+ return 1;
+ }
+ if (unlikely(nlen != len)) {
+ error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
+ return 1;
+ }
+ return 0;
+}
+
+int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len) {
+ size_t nlen = len;
+
+ if (unlikely(!mib[0]))
+ if (unlikely(getsysctl_mib(name, mib, miblen)))
+ return 1;
+
+ if (unlikely(sysctl(mib, miblen, ptr, &nlen, NULL, 0) == -1)) {
+ error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
+ return 1;
+ }
+ if (unlikely(nlen != len)) {
+ error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
+ return 1;
+ }
+
+ return 0;
+}
+
+int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len) {
+ size_t nlen = *len;
+
+ if (unlikely(!mib[0]))
+ if (unlikely(getsysctl_mib(name, mib, miblen)))
+ return 1;
+
+ if (unlikely(sysctl(mib, miblen, ptr, len, NULL, 0) == -1)) {
+ error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
+ return 1;
+ }
+ if (unlikely(ptr != NULL && nlen != *len)) {
+ error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)*len, (unsigned long)nlen);
+ return 1;
+ }
+
+ return 0;
+}
+
+int getsysctl_mib(const char *name, int *mib, size_t len) {
+ size_t nlen = len;
+
+ if (unlikely(sysctlnametomib(name, mib, &nlen) == -1)) {
+ error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
+ return 1;
+ }
+ if (unlikely(nlen != len)) {
+ error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
+ return 1;
+ }
+ return 0;
+}
+
+
+#endif
+
+
+// =====================================================================================================================
+// MacOS
+
+#if (TARGET_OS == OS_MACOS)
+
+int getsysctl_by_name(const char *name, void *ptr, size_t len) {
+ size_t nlen = len;
+
+ if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) {
+ error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno));
+ return 1;
+ }
+ if (unlikely(nlen != len)) {
+ error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
+ return 1;
+ }
+ return 0;
+}
+
+#endif // (TARGET_OS == OS_MACOS)
diff --git a/libnetdata/os.h b/libnetdata/os.h
new file mode 100644
index 0000000..2494174
--- /dev/null
+++ b/libnetdata/os.h
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_OS_H
+#define NETDATA_OS_H
+
+#include "libnetdata.h"
+
+// =====================================================================================================================
+// Linux
+
+#if (TARGET_OS == OS_LINUX)
+
+
+// =====================================================================================================================
+// FreeBSD
+
+#elif (TARGET_OS == OS_FREEBSD)
+
+#include <sys/sysctl.h>
+
+#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var))
+extern int getsysctl_by_name(const char *name, void *ptr, size_t len);
+
+#define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int))
+
+extern int getsysctl_mib(const char *name, int *mib, size_t len);
+
+#define GETSYSCTL_SIMPLE(name, mib, var) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), &(var), sizeof(var))
+#define GETSYSCTL_WSIZE(name, mib, var, size) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), var, size)
+
+extern int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len);
+
+#define GETSYSCTL_SIZE(name, mib, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), NULL, &(size))
+#define GETSYSCTL(name, mib, var, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), &(var), &(size))
+
+extern int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len);
+
+
+// =====================================================================================================================
+// MacOS
+
+#elif (TARGET_OS == OS_MACOS)
+
+#include <sys/sysctl.h>
+
+#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var))
+extern int getsysctl_by_name(const char *name, void *ptr, size_t len);
+
+
+// =====================================================================================================================
+// unknown O/S
+
+#else
+#error unsupported operating system
+#endif
+
+
+// =====================================================================================================================
+// common for all O/S
+
+extern int processors;
+extern long get_system_cpus(void);
+
+extern pid_t pid_max;
+extern pid_t get_system_pid_max(void);
+
+extern unsigned int system_hz;
+extern void get_system_HZ(void);
+
+#endif //NETDATA_OS_H
diff --git a/libnetdata/popen/Makefile.am b/libnetdata/popen/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/popen/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/popen/README.md b/libnetdata/popen/README.md
new file mode 100644
index 0000000..e51e205
--- /dev/null
+++ b/libnetdata/popen/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/popen/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fpopen%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/popen/popen.c b/libnetdata/popen/popen.c
new file mode 100644
index 0000000..c0135cf
--- /dev/null
+++ b/libnetdata/popen/popen.c
@@ -0,0 +1,337 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+static pthread_mutex_t myp_lock;
+static int myp_tracking = 0;
+
+struct mypopen {
+ pid_t pid;
+ struct mypopen *next;
+ struct mypopen *prev;
+};
+
+static struct mypopen *mypopen_root = NULL;
+
+// myp_add_lock takes the lock if we're tracking.
+static void myp_add_lock(void) {
+ if (myp_tracking == 0)
+ return;
+
+ netdata_mutex_lock(&myp_lock);
+}
+
+// myp_add_unlock release the lock if we're tracking.
+static void myp_add_unlock(void) {
+ if (myp_tracking == 0)
+ return;
+
+ netdata_mutex_unlock(&myp_lock);
+}
+
+// myp_add_locked adds pid if we're tracking.
+// myp_add_lock must have been called previously.
+static void myp_add_locked(pid_t pid) {
+ struct mypopen *mp;
+
+ if (myp_tracking == 0)
+ return;
+
+ mp = mallocz(sizeof(struct mypopen));
+ mp->pid = pid;
+
+ mp->next = mypopen_root;
+ mp->prev = NULL;
+ if (mypopen_root != NULL)
+ mypopen_root->prev = mp;
+ mypopen_root = mp;
+ netdata_mutex_unlock(&myp_lock);
+}
+
+// myp_del deletes pid if we're tracking.
+static void myp_del(pid_t pid) {
+ struct mypopen *mp;
+
+ if (myp_tracking == 0)
+ return;
+
+ netdata_mutex_lock(&myp_lock);
+ for (mp = mypopen_root; mp != NULL; mp = mp->next) {
+ if (mp->pid == pid) {
+ if (mp->next != NULL)
+ mp->next->prev = mp->prev;
+ if (mp->prev != NULL)
+ mp->prev->next = mp->next;
+ if (mypopen_root == mp)
+ mypopen_root = mp->next;
+ freez(mp);
+ break;
+ }
+ }
+
+ if (mp == NULL)
+ error("Cannot find pid %d.", pid);
+
+ netdata_mutex_unlock(&myp_lock);
+}
+
+#define PIPE_READ 0
+#define PIPE_WRITE 1
+
+/* custom_popene flag definitions */
+#define FLAG_CREATE_PIPE 1 // Create a pipe like popen() when set, otherwise set stdout to /dev/null
+#define FLAG_CLOSE_FD 2 // Close all file descriptors other than STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO
+
+/*
+ * Returns -1 on failure, 0 on success. When FLAG_CREATE_PIPE is set, on success set the FILE *fp pointer.
+ */
+static inline int custom_popene(const char *command, volatile pid_t *pidptr, char **env, uint8_t flags, FILE **fpp) {
+ FILE *fp = NULL;
+ int ret = 0; // success by default
+ int pipefd[2], error;
+ pid_t pid;
+ char *const spawn_argv[] = {
+ "sh",
+ "-c",
+ (char *)command,
+ NULL
+ };
+ posix_spawnattr_t attr;
+ posix_spawn_file_actions_t fa;
+
+ if (flags & FLAG_CREATE_PIPE) {
+ if (pipe(pipefd) == -1)
+ return -1;
+ if ((fp = fdopen(pipefd[PIPE_READ], "r")) == NULL) {
+ goto error_after_pipe;
+ }
+ }
+
+ if (flags & FLAG_CLOSE_FD) {
+ // Mark all files to be closed by the exec() stage of posix_spawn()
+ int i;
+ for (i = (int) (sysconf(_SC_OPEN_MAX) - 1); i >= 0; i--) {
+ if (i != STDIN_FILENO && i != STDERR_FILENO)
+ (void) fcntl(i, F_SETFD, FD_CLOEXEC);
+ }
+ }
+
+ if (!posix_spawn_file_actions_init(&fa)) {
+ if (flags & FLAG_CREATE_PIPE) {
+ // move the pipe to stdout in the child
+ if (posix_spawn_file_actions_adddup2(&fa, pipefd[PIPE_WRITE], STDOUT_FILENO)) {
+ error("posix_spawn_file_actions_adddup2() failed");
+ goto error_after_posix_spawn_file_actions_init;
+ }
+ } else {
+ // set stdout to /dev/null
+ if (posix_spawn_file_actions_addopen(&fa, STDOUT_FILENO, "/dev/null", O_WRONLY, 0)) {
+ error("posix_spawn_file_actions_addopen() failed");
+ // this is not a fatal error
+ }
+ }
+ } else {
+ error("posix_spawn_file_actions_init() failed.");
+ goto error_after_pipe;
+ }
+ if (!(error = posix_spawnattr_init(&attr))) {
+ // reset all signals in the child
+ sigset_t mask;
+
+ if (posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF))
+ error("posix_spawnattr_setflags() failed.");
+ sigemptyset(&mask);
+ if (posix_spawnattr_setsigmask(&attr, &mask))
+ error("posix_spawnattr_setsigmask() failed.");
+ } else {
+ error("posix_spawnattr_init() failed.");
+ }
+
+ // Take the lock while we fork to ensure we don't race with SIGCHLD
+ // delivery on a process which exits quickly.
+ myp_add_lock();
+ if (!posix_spawn(&pid, "/bin/sh", &fa, &attr, spawn_argv, env)) {
+ *pidptr = pid;
+ myp_add_locked(pid);
+ debug(D_CHILDS, "Spawned command: '%s' on pid %d from parent pid %d.", command, pid, getpid());
+ } else {
+ myp_add_unlock();
+ error("Failed to spawn command: '%s' from parent pid %d.", command, getpid());
+ if (flags & FLAG_CREATE_PIPE) {
+ fclose(fp);
+ }
+ ret = -1;
+ }
+ if (flags & FLAG_CREATE_PIPE) {
+ close(pipefd[PIPE_WRITE]);
+ if (0 == ret) // on success set FILE * pointer
+ *fpp = fp;
+ }
+
+ if (!error) {
+ // posix_spawnattr_init() succeeded
+ if (posix_spawnattr_destroy(&attr))
+ error("posix_spawnattr_destroy");
+ }
+ if (posix_spawn_file_actions_destroy(&fa))
+ error("posix_spawn_file_actions_destroy");
+
+ return ret;
+
+error_after_posix_spawn_file_actions_init:
+ if (posix_spawn_file_actions_destroy(&fa))
+ error("posix_spawn_file_actions_destroy");
+error_after_pipe:
+ if (flags & FLAG_CREATE_PIPE) {
+ if (fp)
+ fclose(fp);
+ else
+ close(pipefd[PIPE_READ]);
+
+ close(pipefd[PIPE_WRITE]);
+ }
+ return -1;
+}
+
+// See man environ
+extern char **environ;
+
+// myp_init should be called by apps which act as init
+// (pid 1) so that processes created by mypopen and mypopene
+// are tracked. This enables the reaper to ignore processes
+// which will be handled internally, by calling myp_reap, to
+// avoid issues with already reaped processes during wait calls.
+//
+// Callers should call myp_free() to clean up resources.
+void myp_init(void) {
+ info("process tracking enabled.");
+ myp_tracking = 1;
+
+ if (netdata_mutex_init(&myp_lock) != 0) {
+ fatal("myp_init() mutex init failed.");
+ }
+}
+
+// myp_free cleans up any resources allocated for process
+// tracking.
+void myp_free(void) {
+ struct mypopen *mp, *next;
+
+ if (myp_tracking == 0)
+ return;
+
+ netdata_mutex_lock(&myp_lock);
+ for (mp = mypopen_root; mp != NULL; mp = next) {
+ next = mp->next;
+ freez(mp);
+ }
+
+ mypopen_root = NULL;
+ myp_tracking = 0;
+ netdata_mutex_unlock(&myp_lock);
+}
+
+// myp_reap returns 1 if pid should be reaped, 0 otherwise.
+int myp_reap(pid_t pid) {
+ struct mypopen *mp;
+
+ if (myp_tracking == 0)
+ return 0;
+
+ netdata_mutex_lock(&myp_lock);
+ for (mp = mypopen_root; mp != NULL; mp = mp->next) {
+ if (mp->pid == pid) {
+ netdata_mutex_unlock(&myp_lock);
+ return 0;
+ }
+ }
+ netdata_mutex_unlock(&myp_lock);
+
+ return 1;
+}
+
+FILE *mypopen(const char *command, volatile pid_t *pidptr) {
+ FILE *fp = NULL;
+ (void)custom_popene(command, pidptr, environ, FLAG_CREATE_PIPE | FLAG_CLOSE_FD, &fp);
+ return fp;
+}
+
+FILE *mypopene(const char *command, volatile pid_t *pidptr, char **env) {
+ FILE *fp = NULL;
+ (void)custom_popene(command, pidptr, env, FLAG_CREATE_PIPE | FLAG_CLOSE_FD, &fp);
+ return fp;
+}
+
+// returns 0 on success, -1 on failure
+int netdata_spawn(const char *command, volatile pid_t *pidptr) {
+ return custom_popene(command, pidptr, environ, 0, NULL);
+}
+
+int custom_pclose(FILE *fp, pid_t pid) {
+ int ret;
+ siginfo_t info;
+
+ debug(D_EXIT, "Request to mypclose() on pid %d", pid);
+
+ if (fp) {
+ // close the pipe fd
+ // this is required in musl
+ // without it the childs do not exit
+ close(fileno(fp));
+
+ // close the pipe file pointer
+ fclose(fp);
+ }
+
+ errno = 0;
+
+ ret = waitid(P_PID, (id_t) pid, &info, WEXITED);
+ myp_del(pid);
+
+ if (ret != -1) {
+ switch (info.si_code) {
+ case CLD_EXITED:
+ if(info.si_status)
+ error("child pid %d exited with code %d.", info.si_pid, info.si_status);
+ return(info.si_status);
+
+ case CLD_KILLED:
+ error("child pid %d killed by signal %d.", info.si_pid, info.si_status);
+ return(-1);
+
+ case CLD_DUMPED:
+ error("child pid %d core dumped by signal %d.", info.si_pid, info.si_status);
+ return(-2);
+
+ case CLD_STOPPED:
+ error("child pid %d stopped by signal %d.", info.si_pid, info.si_status);
+ return(0);
+
+ case CLD_TRAPPED:
+ error("child pid %d trapped by signal %d.", info.si_pid, info.si_status);
+ return(-4);
+
+ case CLD_CONTINUED:
+ error("child pid %d continued by signal %d.", info.si_pid, info.si_status);
+ return(0);
+
+ default:
+ error("child pid %d gave us a SIGCHLD with code %d and status %d.", info.si_pid, info.si_code, info.si_status);
+ return(-5);
+ }
+ }
+ else
+ error("Cannot waitid() for pid %d", pid);
+
+ return 0;
+}
+
+int mypclose(FILE *fp, pid_t pid)
+{
+ return custom_pclose(fp, pid);
+}
+
+int netdata_spawn_waitpid(pid_t pid)
+{
+ return custom_pclose(NULL, pid);
+} \ No newline at end of file
diff --git a/libnetdata/popen/popen.h b/libnetdata/popen/popen.h
new file mode 100644
index 0000000..f387cff
--- /dev/null
+++ b/libnetdata/popen/popen.h
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_POPEN_H
+#define NETDATA_POPEN_H 1
+
+#include "../libnetdata.h"
+
+#define PIPE_READ 0
+#define PIPE_WRITE 1
+
+extern FILE *mypopen(const char *command, volatile pid_t *pidptr);
+extern FILE *mypopene(const char *command, volatile pid_t *pidptr, char **env);
+extern int mypclose(FILE *fp, pid_t pid);
+extern int netdata_spawn(const char *command, volatile pid_t *pidptr);
+extern int netdata_spawn_waitpid(pid_t pid);
+extern void myp_init(void);
+extern void myp_free(void);
+extern int myp_reap(pid_t pid);
+
+extern void signals_unblock(void);
+extern void signals_reset(void);
+
+#endif /* NETDATA_POPEN_H */
diff --git a/libnetdata/procfile/Makefile.am b/libnetdata/procfile/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/procfile/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/procfile/README.md b/libnetdata/procfile/README.md
new file mode 100644
index 0000000..3eeb1e2
--- /dev/null
+++ b/libnetdata/procfile/README.md
@@ -0,0 +1,67 @@
+<!--
+title: "PROCFILE"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/procfile/README.md
+-->
+
+# PROCFILE
+
+procfile is a library for reading text data files (i.e `/proc` files) in the fastest possible way.
+
+## How it works
+
+The library automatically adapts (through the iterations) its memory so that each file
+is read with single `read()` call.
+
+Then the library splits the file into words, using the supplied separators.
+The library also supported quoted words (i.e. strings within of which the separators are ignored).
+
+### Initialization
+
+Initially the caller:
+
+- calls `procfile_open()` to open the file and allocate the structures needed.
+
+### Iterations
+
+For each iteration, the caller:
+
+- calls `procfile_readall()` to read updated contents.
+ This call also rewinds (`lseek()` to 0) before reading it.
+
+ For every file, a [BUFFER](/libnetdata/buffer/README.md) is used that is automatically adjusted to fit the entire
+ file contents of the file. So the file is read with a single `read()` call (providing atomicity / consistency when
+ the data are read from the kernel).
+
+ Once the data are read, 2 arrays of pointers are updated:
+
+ - a `words` array, pointing to each word in the data read
+ - a `lines` array, pointing to the first word for each line
+
+ This is highly optimized. Both arrays are automatically adjusted to
+ fit all contents and are updated in a single pass on the data.
+
+ The library provides a number of macros:
+
+ - `procfile_lines()` returns the # of lines read
+ - `procfile_linewords()` returns the # of words in the given line
+ - `procfile_word()` returns a pointer the given word #
+ - `procfile_line()` returns a pointer to the first word of the given line #
+ - `procfile_lineword()` returns a pointer to the given word # of the given line #
+
+### Cleanup
+
+When the caller exits:
+
+- calls `procfile_free()` to close the file and free all memory used.
+
+### Performance
+
+- a **raspberry Pi 1** (the oldest single core one) can process 5.000+ `/proc` files per second.
+- a **J1900 Celeron** processor can process 23.000+ `/proc` files per second per core.
+
+To achieve this kind of performance, the library tries to work in batches so that the code
+and the data are inside the processor's caches.
+
+This library is extensively used in Netdata and its plugins.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fprocfile%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/procfile/procfile.c b/libnetdata/procfile/procfile.c
new file mode 100644
index 0000000..4a812ba
--- /dev/null
+++ b/libnetdata/procfile/procfile.c
@@ -0,0 +1,472 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+#define PF_PREFIX "PROCFILE"
+
+#define PFWORDS_INCREASE_STEP 200
+#define PFLINES_INCREASE_STEP 10
+#define PROCFILE_INCREMENT_BUFFER 512
+
+int procfile_open_flags = O_RDONLY;
+
+int procfile_adaptive_initial_allocation = 0;
+
+// if adaptive allocation is set, these store the
+// max values we have seen so far
+size_t procfile_max_lines = PFLINES_INCREASE_STEP;
+size_t procfile_max_words = PFWORDS_INCREASE_STEP;
+size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER;
+
+
+// ----------------------------------------------------------------------------
+
+char *procfile_filename(procfile *ff) {
+ if(ff->filename[0]) return ff->filename;
+
+ char buffer[FILENAME_MAX + 1];
+ snprintfz(buffer, FILENAME_MAX, "/proc/self/fd/%d", ff->fd);
+
+ ssize_t l = readlink(buffer, ff->filename, FILENAME_MAX);
+ if(unlikely(l == -1))
+ snprintfz(ff->filename, FILENAME_MAX, "unknown filename for fd %d", ff->fd);
+ else
+ ff->filename[l] = '\0';
+
+ // on non-linux systems, something like this will be needed
+ // fcntl(ff->fd, F_GETPATH, ff->filename)
+
+ return ff->filename;
+}
+
+// ----------------------------------------------------------------------------
+// An array of words
+
+static inline void pfwords_add(procfile *ff, char *str) {
+ // debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str);
+
+ pfwords *fw = ff->words;
+ if(unlikely(fw->len == fw->size)) {
+ // debug(D_PROCFILE, PF_PREFIX ": expanding words");
+
+ ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + PFWORDS_INCREASE_STEP) * sizeof(char *));
+ fw->size += PFWORDS_INCREASE_STEP;
+ }
+
+ fw->words[fw->len++] = str;
+}
+
+NEVERNULL
+static inline pfwords *pfwords_new(void) {
+ // debug(D_PROCFILE, PF_PREFIX ": initializing words");
+
+ size_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP;
+
+ pfwords *new = mallocz(sizeof(pfwords) + size * sizeof(char *));
+ new->len = 0;
+ new->size = size;
+ return new;
+}
+
+static inline void pfwords_reset(pfwords *fw) {
+ // debug(D_PROCFILE, PF_PREFIX ": reseting words");
+ fw->len = 0;
+}
+
+static inline void pfwords_free(pfwords *fw) {
+ // debug(D_PROCFILE, PF_PREFIX ": freeing words");
+
+ freez(fw);
+}
+
+
+// ----------------------------------------------------------------------------
+// An array of lines
+
+NEVERNULL
+static inline size_t *pflines_add(procfile *ff) {
+ // debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word);
+
+ pflines *fl = ff->lines;
+ if(unlikely(fl->len == fl->size)) {
+ // debug(D_PROCFILE, PF_PREFIX ": expanding lines");
+
+ ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + PFLINES_INCREASE_STEP) * sizeof(ffline));
+ fl->size += PFLINES_INCREASE_STEP;
+ }
+
+ ffline *ffl = &fl->lines[fl->len++];
+ ffl->words = 0;
+ ffl->first = ff->words->len;
+
+ return &ffl->words;
+}
+
+NEVERNULL
+static inline pflines *pflines_new(void) {
+ // debug(D_PROCFILE, PF_PREFIX ": initializing lines");
+
+ size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_words : PFLINES_INCREASE_STEP;
+
+ pflines *new = mallocz(sizeof(pflines) + size * sizeof(ffline));
+ new->len = 0;
+ new->size = size;
+ return new;
+}
+
+static inline void pflines_reset(pflines *fl) {
+ // debug(D_PROCFILE, PF_PREFIX ": reseting lines");
+
+ fl->len = 0;
+}
+
+static inline void pflines_free(pflines *fl) {
+ // debug(D_PROCFILE, PF_PREFIX ": freeing lines");
+
+ freez(fl);
+}
+
+
+// ----------------------------------------------------------------------------
+// The procfile
+
+void procfile_close(procfile *ff) {
+ if(unlikely(!ff)) return;
+
+ debug(D_PROCFILE, PF_PREFIX ": Closing file '%s'", procfile_filename(ff));
+
+ if(likely(ff->lines)) pflines_free(ff->lines);
+ if(likely(ff->words)) pfwords_free(ff->words);
+
+ if(likely(ff->fd != -1)) close(ff->fd);
+ freez(ff);
+}
+
+NOINLINE
+static void procfile_parser(procfile *ff) {
+ // debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename);
+
+ char *s = ff->data // our current position
+ , *e = &ff->data[ff->len] // the terminating null
+ , *t = ff->data; // the first character of a word (or quoted / parenthesized string)
+
+ // the look up array to find our type of character
+ PF_CHAR_TYPE *separators = ff->separators;
+
+ char quote = 0; // the quote character - only when in quoted string
+ size_t opened = 0; // counts the number of open parenthesis
+
+ size_t *line_words = pflines_add(ff);
+
+ while(s < e) {
+ PF_CHAR_TYPE ct = separators[(unsigned char)(*s)];
+
+ // this is faster than a switch()
+ // read more here: http://lazarenko.me/switch/
+ if(likely(ct == PF_CHAR_IS_WORD)) {
+ s++;
+ }
+ else if(likely(ct == PF_CHAR_IS_SEPARATOR)) {
+ if(!quote && !opened) {
+ if (s != t) {
+ // separator, but we have word before it
+ *s = '\0';
+ pfwords_add(ff, t);
+ (*line_words)++;
+ t = ++s;
+ }
+ else {
+ // separator at the beginning
+ // skip it
+ t = ++s;
+ }
+ }
+ else {
+ // we are inside a quote or parenthesized string
+ s++;
+ }
+ }
+ else if(likely(ct == PF_CHAR_IS_NEWLINE)) {
+ // end of line
+
+ *s = '\0';
+ pfwords_add(ff, t);
+ (*line_words)++;
+ t = ++s;
+
+ // debug(D_PROCFILE, PF_PREFIX ": ended line %d with %d words", l, ff->lines->lines[l].words);
+
+ line_words = pflines_add(ff);
+ }
+ else if(likely(ct == PF_CHAR_IS_QUOTE)) {
+ if(unlikely(!quote && s == t)) {
+ // quote opened at the beginning
+ quote = *s;
+ t = ++s;
+ }
+ else if(unlikely(quote && quote == *s)) {
+ // quote closed
+ quote = 0;
+
+ *s = '\0';
+ pfwords_add(ff, t);
+ (*line_words)++;
+ t = ++s;
+ }
+ else
+ s++;
+ }
+ else if(likely(ct == PF_CHAR_IS_OPEN)) {
+ if(s == t) {
+ opened++;
+ t = ++s;
+ }
+ else if(opened) {
+ opened++;
+ s++;
+ }
+ else
+ s++;
+ }
+ else if(likely(ct == PF_CHAR_IS_CLOSE)) {
+ if(opened) {
+ opened--;
+
+ if(!opened) {
+ *s = '\0';
+ pfwords_add(ff, t);
+ (*line_words)++;
+ t = ++s;
+ }
+ else
+ s++;
+ }
+ else
+ s++;
+ }
+ else
+ fatal("Internal Error: procfile_readall() does not handle all the cases.");
+ }
+
+ if(likely(s > t && t < e)) {
+ // the last word
+ if(unlikely(ff->len >= ff->size)) {
+ // we are going to loose the last byte
+ s = &ff->data[ff->size - 1];
+ }
+
+ *s = '\0';
+ pfwords_add(ff, t);
+ (*line_words)++;
+ // t = ++s;
+ }
+}
+
+procfile *procfile_readall(procfile *ff) {
+ // debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename);
+
+ ff->len = 0; // zero the used size
+ ssize_t r = 1; // read at least once
+ while(r > 0) {
+ ssize_t s = ff->len;
+ ssize_t x = ff->size - s;
+
+ if(unlikely(!x)) {
+ debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", procfile_filename(ff));
+ ff = reallocz(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER);
+ ff->size += PROCFILE_INCREMENT_BUFFER;
+ }
+
+ debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s));
+ r = read(ff->fd, &ff->data[s], ff->size - s);
+ if(unlikely(r == -1)) {
+ if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd);
+ procfile_close(ff);
+ return NULL;
+ }
+
+ ff->len += r;
+ }
+
+ // debug(D_PROCFILE, "Rewinding file '%s'", ff->filename);
+ if(unlikely(lseek(ff->fd, 0, SEEK_SET) == -1)) {
+ if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot rewind on file '%s'.", procfile_filename(ff));
+ procfile_close(ff);
+ return NULL;
+ }
+
+ pflines_reset(ff->lines);
+ pfwords_reset(ff->words);
+ procfile_parser(ff);
+
+ if(unlikely(procfile_adaptive_initial_allocation)) {
+ if(unlikely(ff->len > procfile_max_allocation)) procfile_max_allocation = ff->len;
+ if(unlikely(ff->lines->len > procfile_max_lines)) procfile_max_lines = ff->lines->len;
+ if(unlikely(ff->words->len > procfile_max_words)) procfile_max_words = ff->words->len;
+ }
+
+ // debug(D_PROCFILE, "File '%s' updated.", ff->filename);
+ return ff;
+}
+
+NOINLINE
+static void procfile_set_separators(procfile *ff, const char *separators) {
+ static PF_CHAR_TYPE def[256];
+ static char initilized = 0;
+
+ if(unlikely(!initilized)) {
+ // this is thread safe
+ // if initialized is zero, multiple threads may be executing
+ // this code at the same time, setting in def[] the exact same values
+ int i = 256;
+ while(i--) {
+ if(unlikely(i == '\n' || i == '\r'))
+ def[i] = PF_CHAR_IS_NEWLINE;
+
+ else if(unlikely(isspace(i) || !isprint(i)))
+ def[i] = PF_CHAR_IS_SEPARATOR;
+
+ else
+ def[i] = PF_CHAR_IS_WORD;
+ }
+
+ initilized = 1;
+ }
+
+ // copy the default
+ PF_CHAR_TYPE *ffs = ff->separators, *ffd = def, *ffe = &def[256];
+ while(ffd != ffe)
+ *ffs++ = *ffd++;
+
+ // set the separators
+ if(unlikely(!separators))
+ separators = " \t=|";
+
+ ffs = ff->separators;
+ const char *s = separators;
+ while(*s)
+ ffs[(int)*s++] = PF_CHAR_IS_SEPARATOR;
+}
+
+void procfile_set_quotes(procfile *ff, const char *quotes) {
+ PF_CHAR_TYPE *ffs = ff->separators;
+
+ // remove all quotes
+ int i = 256;
+ while(i--)
+ if(unlikely(ffs[i] == PF_CHAR_IS_QUOTE))
+ ffs[i] = PF_CHAR_IS_WORD;
+
+ // if nothing given, return
+ if(unlikely(!quotes || !*quotes))
+ return;
+
+ // set the quotes
+ const char *s = quotes;
+ while(*s)
+ ffs[(int)*s++] = PF_CHAR_IS_QUOTE;
+}
+
+void procfile_set_open_close(procfile *ff, const char *open, const char *close) {
+ PF_CHAR_TYPE *ffs = ff->separators;
+
+ // remove all open/close
+ int i = 256;
+ while(i--)
+ if(unlikely(ffs[i] == PF_CHAR_IS_OPEN || ffs[i] == PF_CHAR_IS_CLOSE))
+ ffs[i] = PF_CHAR_IS_WORD;
+
+ // if nothing given, return
+ if(unlikely(!open || !*open || !close || !*close))
+ return;
+
+ // set the openings
+ const char *s = open;
+ while(*s)
+ ffs[(int)*s++] = PF_CHAR_IS_OPEN;
+
+ // set the closings
+ s = close;
+ while(*s)
+ ffs[(int)*s++] = PF_CHAR_IS_CLOSE;
+}
+
+procfile *procfile_open(const char *filename, const char *separators, uint32_t flags) {
+ debug(D_PROCFILE, PF_PREFIX ": Opening file '%s'", filename);
+
+ int fd = open(filename, procfile_open_flags, 0666);
+ if(unlikely(fd == -1)) {
+ if(unlikely(!(flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot open file '%s'", filename);
+ return NULL;
+ }
+
+ // info("PROCFILE: opened '%s' on fd %d", filename, fd);
+
+ size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_allocation : PROCFILE_INCREMENT_BUFFER;
+ procfile *ff = mallocz(sizeof(procfile) + size);
+
+ //strncpyz(ff->filename, filename, FILENAME_MAX);
+ ff->filename[0] = '\0';
+
+ ff->fd = fd;
+ ff->size = size;
+ ff->len = 0;
+ ff->flags = flags;
+
+ ff->lines = pflines_new();
+ ff->words = pfwords_new();
+
+ procfile_set_separators(ff, separators);
+
+ debug(D_PROCFILE, "File '%s' opened.", filename);
+ return ff;
+}
+
+procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags) {
+ if(unlikely(!ff)) return procfile_open(filename, separators, flags);
+
+ if(likely(ff->fd != -1)) {
+ // info("PROCFILE: closing fd %d", ff->fd);
+ close(ff->fd);
+ }
+
+ ff->fd = open(filename, procfile_open_flags, 0666);
+ if(unlikely(ff->fd == -1)) {
+ procfile_close(ff);
+ return NULL;
+ }
+
+ // info("PROCFILE: opened '%s' on fd %d", filename, ff->fd);
+
+ //strncpyz(ff->filename, filename, FILENAME_MAX);
+ ff->filename[0] = '\0';
+ ff->flags = flags;
+
+ // do not do the separators again if NULL is given
+ if(likely(separators)) procfile_set_separators(ff, separators);
+
+ return ff;
+}
+
+// ----------------------------------------------------------------------------
+// example parsing of procfile data
+
+void procfile_print(procfile *ff) {
+ size_t lines = procfile_lines(ff), l;
+ char *s;
+ (void)s;
+
+ debug(D_PROCFILE, "File '%s' with %zu lines and %zu words", procfile_filename(ff), ff->lines->len, ff->words->len);
+
+ for(l = 0; likely(l < lines) ;l++) {
+ size_t words = procfile_linewords(ff, l);
+
+ debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words);
+
+ size_t w;
+ for(w = 0; likely(w < words) ;w++) {
+ s = procfile_lineword(ff, l, w);
+ debug(D_PROCFILE, " [%zu.%zu] '%s'", l, w, s);
+ }
+ }
+}
diff --git a/libnetdata/procfile/procfile.h b/libnetdata/procfile/procfile.h
new file mode 100644
index 0000000..b107358
--- /dev/null
+++ b/libnetdata/procfile/procfile.h
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PROCFILE_H
+#define NETDATA_PROCFILE_H 1
+
+#include "../libnetdata.h"
+
+// ----------------------------------------------------------------------------
+// An array of words
+
+typedef struct {
+ size_t len; // used entries
+ size_t size; // capacity
+ char *words[]; // array of pointers
+} pfwords;
+
+
+// ----------------------------------------------------------------------------
+// An array of lines
+
+typedef struct {
+ size_t words; // how many words this line has
+ size_t first; // the id of the first word of this line
+ // in the words array
+} ffline;
+
+typedef struct {
+ size_t len; // used entries
+ size_t size; // capacity
+ ffline lines[]; // array of lines
+} pflines;
+
+
+// ----------------------------------------------------------------------------
+// The procfile
+
+#define PROCFILE_FLAG_DEFAULT 0x00000000
+#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001
+
+typedef enum procfile_separator {
+ PF_CHAR_IS_SEPARATOR,
+ PF_CHAR_IS_NEWLINE,
+ PF_CHAR_IS_WORD,
+ PF_CHAR_IS_QUOTE,
+ PF_CHAR_IS_OPEN,
+ PF_CHAR_IS_CLOSE
+} PF_CHAR_TYPE;
+
+typedef struct {
+ char filename[FILENAME_MAX + 1]; // not populated until profile_filename() is called
+
+ uint32_t flags;
+ int fd; // the file desriptor
+ size_t len; // the bytes we have placed into data
+ size_t size; // the bytes we have allocated for data
+ pflines *lines;
+ pfwords *words;
+ PF_CHAR_TYPE separators[256];
+ char data[]; // allocated buffer to keep file contents
+} procfile;
+
+// close the proc file and free all related memory
+extern void procfile_close(procfile *ff);
+
+// (re)read and parse the proc file
+extern procfile *procfile_readall(procfile *ff);
+
+// open a /proc or /sys file
+extern procfile *procfile_open(const char *filename, const char *separators, uint32_t flags);
+
+// re-open a file
+// if separators == NULL, the last separators are used
+extern procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags);
+
+// example walk-through a procfile parsed file
+extern void procfile_print(procfile *ff);
+
+extern void procfile_set_quotes(procfile *ff, const char *quotes);
+extern void procfile_set_open_close(procfile *ff, const char *open, const char *close);
+
+extern char *procfile_filename(procfile *ff);
+
+// ----------------------------------------------------------------------------
+
+// set to the O_XXXX flags, to have procfile_open and procfile_reopen use them when opening proc files
+extern int procfile_open_flags;
+
+// set this to 1, to have procfile adapt its initial buffer allocation to the max allocation used so far
+extern int procfile_adaptive_initial_allocation;
+
+// return the number of lines present
+#define procfile_lines(ff) ((ff)->lines->len)
+
+// return the number of words of the Nth line
+#define procfile_linewords(ff, line) (((line) < procfile_lines(ff)) ? (ff)->lines->lines[(line)].words : 0)
+
+// return the Nth word of the file, or empty string
+#define procfile_word(ff, word) (((word) < (ff)->words->len) ? (ff)->words->words[(word)] : "")
+
+// return the first word of the Nth line, or empty string
+#define procfile_line(ff, line) (((line) < procfile_lines(ff)) ? procfile_word((ff), (ff)->lines->lines[(line)].first) : "")
+
+// return the Nth word of the current line
+#define procfile_lineword(ff, line, word) (((line) < procfile_lines(ff) && (word) < procfile_linewords((ff), (line))) ? procfile_word((ff), (ff)->lines->lines[(line)].first + (word)) : "")
+
+#endif /* NETDATA_PROCFILE_H */
diff --git a/libnetdata/required_dummies.h b/libnetdata/required_dummies.h
new file mode 100644
index 0000000..aa87e39
--- /dev/null
+++ b/libnetdata/required_dummies.h
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LIB_DUMMIES_H
+#define NETDATA_LIB_DUMMIES_H 1
+
+// callback required by fatal()
+void netdata_cleanup_and_exit(int ret)
+{
+ exit(ret);
+}
+
+void send_statistics(const char *action, const char *action_result, const char *action_data)
+{
+ (void)action;
+ (void)action_result;
+ (void)action_data;
+ return;
+}
+
+// callbacks required by popen()
+void signals_block(void){};
+void signals_unblock(void){};
+void signals_reset(void){};
+
+// callback required by eval()
+int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result)
+{
+ (void)variable;
+ (void)hash;
+ (void)rc;
+ (void)result;
+ return 0;
+};
+
+// required by get_system_cpus()
+char *netdata_configured_host_prefix = "";
+
+#endif // NETDATA_LIB_DUMMIES_H
diff --git a/libnetdata/simple_pattern/Makefile.am b/libnetdata/simple_pattern/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/simple_pattern/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/simple_pattern/README.md b/libnetdata/simple_pattern/README.md
new file mode 100644
index 0000000..a90759c
--- /dev/null
+++ b/libnetdata/simple_pattern/README.md
@@ -0,0 +1,43 @@
+<!--
+title: "Netdata simple patterns"
+description: "Netdata supports simple patterns, which are less cryptic versions of regular expressions. Use familiar notation for powerful results."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/simple_pattern/README.md
+-->
+
+# Netdata simple patterns
+
+Unix prefers regular expressions. But they are just too hard, too cryptic
+to use, write and understand.
+
+So, Netdata supports **simple patterns**.
+
+Simple patterns are a space separated list of words, that can have `*`
+as a wildcard. Each world may use any number of `*`. Simple patterns
+allow **negative** matches by prefixing a word with `!`.
+
+So, `pattern = !*bad* *` will match anything, except all those that
+contain the word `bad`.
+
+Simple patterns are quite powerful: `pattern = *foobar* !foo* !*bar *`
+matches everything containing `foobar`, except strings that start
+with `foo` or end with `bar`.
+
+You can use the Netdata command line to check simple patterns,
+like this:
+
+```sh
+# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world'
+RESULT: MATCHED - pattern '*foobar* !foo* !*bar *' matches 'hello world'
+
+# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world bar'
+RESULT: NOT MATCHED - pattern '*foobar* !foo* !*bar *' does not match 'hello world bar'
+
+# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world foobar'
+RESULT: MATCHED - pattern '*foobar* !foo* !*bar *' matches 'hello world foobar'
+```
+
+Netdata stops processing to the first positive or negative match
+(left to right). If it is not matched by either positive or negative
+patterns, it is denied at the end.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fsimple_pattern%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/simple_pattern/simple_pattern.c b/libnetdata/simple_pattern/simple_pattern.c
new file mode 100644
index 0000000..70b06a2
--- /dev/null
+++ b/libnetdata/simple_pattern/simple_pattern.c
@@ -0,0 +1,365 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+struct simple_pattern {
+ const char *match;
+ size_t len;
+
+ SIMPLE_PREFIX_MODE mode;
+ char negative;
+
+ struct simple_pattern *child;
+
+ struct simple_pattern *next;
+};
+
+static inline struct simple_pattern *parse_pattern(char *str, SIMPLE_PREFIX_MODE default_mode) {
+ // fprintf(stderr, "PARSING PATTERN: '%s'\n", str);
+
+ SIMPLE_PREFIX_MODE mode;
+ struct simple_pattern *child = NULL;
+
+ char *s = str, *c = str;
+
+ // skip asterisks in front
+ while(*c == '*') c++;
+
+ // find the next asterisk
+ while(*c && *c != '*') c++;
+
+ // do we have an asterisk in the middle?
+ if(*c == '*' && c[1] != '\0') {
+ // yes, we have
+ child = parse_pattern(c, default_mode);
+ c[1] = '\0';
+ }
+
+ // check what this one matches
+
+ size_t len = strlen(s);
+ if(len >= 2 && *s == '*' && s[len - 1] == '*') {
+ s[len - 1] = '\0';
+ s++;
+ mode = SIMPLE_PATTERN_SUBSTRING;
+ }
+ else if(len >= 1 && *s == '*') {
+ s++;
+ mode = SIMPLE_PATTERN_SUFFIX;
+ }
+ else if(len >= 1 && s[len - 1] == '*') {
+ s[len - 1] = '\0';
+ mode = SIMPLE_PATTERN_PREFIX;
+ }
+ else
+ mode = default_mode;
+
+ // allocate the structure
+ struct simple_pattern *m = callocz(1, sizeof(struct simple_pattern));
+ if(*s) {
+ m->match = strdupz(s);
+ m->len = strlen(m->match);
+ m->mode = mode;
+ }
+ else {
+ m->mode = SIMPLE_PATTERN_SUBSTRING;
+ }
+
+ m->child = child;
+
+ return m;
+}
+
+SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode) {
+ struct simple_pattern *root = NULL, *last = NULL;
+
+ if(unlikely(!list || !*list)) return root;
+
+ int isseparator[256] = {
+ [' '] = 1 // space
+ , ['\t'] = 1 // tab
+ , ['\r'] = 1 // carriage return
+ , ['\n'] = 1 // new line
+ , ['\f'] = 1 // form feed
+ , ['\v'] = 1 // vertical tab
+ };
+
+ if (unlikely(separators && *separators)) {
+ memset(&isseparator[0], 0, sizeof(isseparator));
+ while(*separators) isseparator[(unsigned char)*separators++] = 1;
+ }
+
+ char *buf = mallocz(strlen(list) + 1);
+ const char *s = list;
+
+ while(s && *s) {
+ buf[0] = '\0';
+ char *c = buf;
+
+ char negative = 0;
+
+ // skip all spaces
+ while(isseparator[(unsigned char)*s])
+ s++;
+
+ if(*s == '!') {
+ negative = 1;
+ s++;
+ }
+
+ // empty string
+ if(unlikely(!*s))
+ break;
+
+ // find the next space
+ char escape = 0;
+ while(*s) {
+ if(*s == '\\' && !escape) {
+ escape = 1;
+ s++;
+ }
+ else {
+ if (isseparator[(unsigned char)*s] && !escape) {
+ s++;
+ break;
+ }
+
+ *c++ = *s++;
+ escape = 0;
+ }
+ }
+
+ // terminate our string
+ *c = '\0';
+
+ // if we matched the empty string, skip it
+ if(unlikely(!*buf))
+ continue;
+
+ // fprintf(stderr, "FOUND PATTERN: '%s'\n", buf);
+ struct simple_pattern *m = parse_pattern(buf, default_mode);
+ m->negative = negative;
+
+ // link it at the end
+ if(unlikely(!root))
+ root = last = m;
+ else {
+ last->next = m;
+ last = m;
+ }
+ }
+
+ freez(buf);
+ return (SIMPLE_PATTERN *)root;
+}
+
+static inline char *add_wildcarded(const char *matched, size_t matched_size, char *wildcarded, size_t *wildcarded_size) {
+ //if(matched_size) {
+ // char buf[matched_size + 1];
+ // strncpyz(buf, matched, matched_size);
+ // fprintf(stderr, "ADD WILDCARDED '%s' of length %zu\n", buf, matched_size);
+ //}
+
+ if(unlikely(wildcarded && *wildcarded_size && matched && *matched && matched_size)) {
+ size_t wss = *wildcarded_size - 1;
+ size_t len = (matched_size < wss)?matched_size:wss;
+ if(likely(len)) {
+ strncpyz(wildcarded, matched, len);
+
+ *wildcarded_size -= len;
+ return &wildcarded[len];
+ }
+ }
+
+ return wildcarded;
+}
+
+static inline int match_pattern(struct simple_pattern *m, const char *str, size_t len, char *wildcarded, size_t *wildcarded_size) {
+ char *s;
+
+ if(m->len <= len) {
+ switch(m->mode) {
+ case SIMPLE_PATTERN_SUBSTRING:
+ if(!m->len) return 1;
+ if((s = strstr(str, m->match))) {
+ wildcarded = add_wildcarded(str, s - str, wildcarded, wildcarded_size);
+ if(!m->child) {
+ wildcarded = add_wildcarded(&s[m->len], len - (&s[m->len] - str), wildcarded, wildcarded_size);
+ return 1;
+ }
+ return match_pattern(m->child, &s[m->len], len - (s - str) - m->len, wildcarded, wildcarded_size);
+ }
+ break;
+
+ case SIMPLE_PATTERN_PREFIX:
+ if(unlikely(strncmp(str, m->match, m->len) == 0)) {
+ if(!m->child) {
+ wildcarded = add_wildcarded(&str[m->len], len - m->len, wildcarded, wildcarded_size);
+ return 1;
+ }
+ return match_pattern(m->child, &str[m->len], len - m->len, wildcarded, wildcarded_size);
+ }
+ break;
+
+ case SIMPLE_PATTERN_SUFFIX:
+ if(unlikely(strcmp(&str[len - m->len], m->match) == 0)) {
+ wildcarded = add_wildcarded(str, len - m->len, wildcarded, wildcarded_size);
+ if(!m->child) return 1;
+ return 0;
+ }
+ break;
+
+ case SIMPLE_PATTERN_EXACT:
+ default:
+ if(unlikely(strcmp(str, m->match) == 0)) {
+ if(!m->child) return 1;
+ return 0;
+ }
+ break;
+ }
+ }
+
+ return 0;
+}
+
+int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) {
+ struct simple_pattern *m, *root = (struct simple_pattern *)list;
+
+ if(unlikely(!root || !str || !*str)) return 0;
+
+ size_t len = strlen(str);
+ for(m = root; m ; m = m->next) {
+ char *ws = wildcarded;
+ size_t wss = wildcarded_size;
+ if(unlikely(ws)) *ws = '\0';
+
+ if (match_pattern(m, str, len, ws, &wss)) {
+
+ //if(ws && wss)
+ // fprintf(stderr, "FINAL WILDCARDED '%s' of length %zu\n", ws, strlen(ws));
+
+ if (m->negative) return 0;
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static inline void free_pattern(struct simple_pattern *m) {
+ if(!m) return;
+
+ free_pattern(m->child);
+ free_pattern(m->next);
+ freez((void *)m->match);
+ freez(m);
+}
+
+void simple_pattern_free(SIMPLE_PATTERN *list) {
+ if(!list) return;
+
+ free_pattern(((struct simple_pattern *)list));
+}
+
+/* Debugging patterns
+
+ This code should be dead - it is useful for debugging but should not be called by production code.
+ Feel free to comment it out, but please leave it in the file.
+*/
+extern void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p)
+{
+ struct simple_pattern *root = (struct simple_pattern *)p;
+ if(root==NULL) {
+ debug(debug_type,"dump_pattern(NULL)");
+ return;
+ }
+ debug(debug_type,"dump_pattern(%p) child=%p next=%p mode=%u match=%s", root, root->child, root->next, root->mode,
+ root->match);
+ if(root->child!=NULL)
+ simple_pattern_dump(debug_type, (SIMPLE_PATTERN*)root->child);
+ if(root->next!=NULL)
+ simple_pattern_dump(debug_type, (SIMPLE_PATTERN*)root->next);
+}
+
+/* Heuristic: decide if the pattern could match a DNS name.
+
+ Although this functionality is used directly by socket.c:connection_allowed() it must be in this file
+ because of the SIMPLE_PATTERN/simple_pattern structure hiding.
+ Based on RFC952 / RFC1123. We need to decide if the pattern may match a DNS name, or not. For the negative
+ cases we need to be sure that it can only match an ipv4 or ipv6 address:
+ * IPv6 addresses contain ':', which are illegal characters in DNS.
+ * IPv4 addresses cannot contain alpha- characters.
+ * DNS TLDs must be alphanumeric to distinguish from IPv4.
+ Some patterns (e.g. "*a*" ) could match multiple cases (i.e. DNS or IPv6).
+ Some patterns will be awkward (e.g. "192.168.*") as they look like they are intended to match IPv4-only
+ but could match DNS (i.e. "192.168.com" is a valid name).
+*/
+static void scan_is_potential_name(struct simple_pattern *p, int *alpha, int *colon, int *wildcards)
+{
+ while (p) {
+ if (p->match) {
+ if(p->mode == SIMPLE_PATTERN_EXACT && !strcmp("localhost", p->match)) {
+ p = p->child;
+ continue;
+ }
+ char const *scan = p->match;
+ while (*scan != 0) {
+ if ((*scan >= 'a' && *scan <= 'z') || (*scan >= 'A' && *scan <= 'Z'))
+ *alpha = 1;
+ if (*scan == ':')
+ *colon = 1;
+ scan++;
+ }
+ if (p->mode != SIMPLE_PATTERN_EXACT)
+ *wildcards = 1;
+ p = p->child;
+ }
+ }
+}
+
+extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p)
+{
+ int alpha=0, colon=0, wildcards=0;
+ struct simple_pattern *root = (struct simple_pattern*)p;
+ while (root != NULL) {
+ if (root->match != NULL) {
+ scan_is_potential_name(root, &alpha, &colon, &wildcards);
+ }
+ if (root->mode != SIMPLE_PATTERN_EXACT)
+ wildcards = 1;
+ root = root->next;
+ }
+ return (alpha || wildcards) && !colon;
+}
+
+char *simple_pattern_trim_around_equal(char *src) {
+ char *store = mallocz(strlen(src) +1);
+ if(!store)
+ return NULL;
+
+ char *dst = store;
+ while (*src) {
+ if (*src == '=') {
+ if (*(dst -1) == ' ')
+ dst--;
+
+ *dst++ = *src++;
+ if (*src == ' ')
+ src++;
+ }
+
+ *dst++ = *src++;
+ }
+ *dst = 0x00;
+
+ return store;
+}
+
+char *simple_pattern_iterate(SIMPLE_PATTERN **p)
+{
+ struct simple_pattern *root = (struct simple_pattern *) *p;
+ struct simple_pattern **Proot = (struct simple_pattern **)p;
+
+ (*Proot) = (*Proot)->next;
+ return (char *) root->match;
+}
diff --git a/libnetdata/simple_pattern/simple_pattern.h b/libnetdata/simple_pattern/simple_pattern.h
new file mode 100644
index 0000000..36fbbde
--- /dev/null
+++ b/libnetdata/simple_pattern/simple_pattern.h
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SIMPLE_PATTERN_H
+#define NETDATA_SIMPLE_PATTERN_H
+
+#include "../libnetdata.h"
+
+
+typedef enum {
+ SIMPLE_PATTERN_EXACT,
+ SIMPLE_PATTERN_PREFIX,
+ SIMPLE_PATTERN_SUFFIX,
+ SIMPLE_PATTERN_SUBSTRING
+} SIMPLE_PREFIX_MODE;
+
+typedef void SIMPLE_PATTERN;
+
+// create a simple_pattern from the string given
+// default_mode is used in cases where EXACT matches, without an asterisk,
+// should be considered PREFIX matches.
+extern SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode);
+
+// test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*'
+extern int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size);
+
+// test if string str is matched from the pattern
+#define simple_pattern_matches(list, str) simple_pattern_matches_extract(list, str, NULL, 0)
+
+// free a simple_pattern that was created with simple_pattern_create()
+// list can be NULL, in which case, this does nothing.
+extern void simple_pattern_free(SIMPLE_PATTERN *list);
+
+extern void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p) ;
+extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p) ;
+extern char *simple_pattern_iterate(SIMPLE_PATTERN **p);
+
+//Auxiliary function to create a pattern
+char *simple_pattern_trim_around_equal(char *src);
+
+#endif //NETDATA_SIMPLE_PATTERN_H
diff --git a/libnetdata/socket/Makefile.am b/libnetdata/socket/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/socket/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/socket/README.md b/libnetdata/socket/README.md
new file mode 100644
index 0000000..6736f13
--- /dev/null
+++ b/libnetdata/socket/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/socket/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fsocket%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/socket/security.c b/libnetdata/socket/security.c
new file mode 100644
index 0000000..53366c4
--- /dev/null
+++ b/libnetdata/socket/security.c
@@ -0,0 +1,385 @@
+#include "../libnetdata.h"
+
+#ifdef ENABLE_HTTPS
+
+SSL_CTX *netdata_exporting_ctx=NULL;
+SSL_CTX *netdata_client_ctx=NULL;
+SSL_CTX *netdata_srv_ctx=NULL;
+const char *security_key=NULL;
+const char *security_cert=NULL;
+const char *tls_version=NULL;
+const char *tls_ciphers=NULL;
+int netdata_validate_server = NETDATA_SSL_VALID_CERTIFICATE;
+
+/**
+ * Info Callback
+ *
+ * Function used as callback for the OpenSSL Library
+ *
+ * @param ssl a pointer to the SSL structure of the client
+ * @param where the variable with the flags set.
+ * @param ret the return of the caller
+ */
+static void security_info_callback(const SSL *ssl, int where, int ret __maybe_unused) {
+ (void)ssl;
+ if (where & SSL_CB_ALERT) {
+ debug(D_WEB_CLIENT,"SSL INFO CALLBACK %s %s", SSL_alert_type_string(ret), SSL_alert_desc_string_long(ret));
+ }
+}
+
+/**
+ * OpenSSL Library
+ *
+ * Starts the openssl library for the Netdata.
+ */
+void security_openssl_library()
+{
+#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
+# if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097)
+ OPENSSL_config(NULL);
+# endif
+
+ SSL_load_error_strings();
+
+ SSL_library_init();
+#else
+ if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) != 1) {
+ error("SSL library cannot be initialized.");
+ }
+#endif
+}
+
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_110
+/**
+ * TLS version
+ *
+ * Returns the TLS version depending of the user input.
+ *
+ * @param lversion is the user input.
+ *
+ * @return it returns the version number.
+ */
+int tls_select_version(const char *lversion) {
+ if (!strcmp(lversion, "1") || !strcmp(lversion, "1.0"))
+ return TLS1_VERSION;
+ else if (!strcmp(lversion, "1.1"))
+ return TLS1_1_VERSION;
+ else if (!strcmp(lversion, "1.2"))
+ return TLS1_2_VERSION;
+#if defined(TLS1_3_VERSION)
+ else if (!strcmp(lversion, "1.3"))
+ return TLS1_3_VERSION;
+#endif
+
+#if defined(TLS_MAX_VERSION)
+ return TLS_MAX_VERSION;
+#else
+ return TLS1_2_VERSION;
+#endif
+}
+#endif
+
+/**
+ * OpenSSL common options
+ *
+ * Clients and SERVER have common options, this function is responsible to set them in the context.
+ *
+ * @param ctx the initialized SSL context.
+ * @param side 0 means server, and 1 client.
+ */
+void security_openssl_common_options(SSL_CTX *ctx, int side) {
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_110
+ if (!side) {
+ int version = tls_select_version(tls_version) ;
+#endif
+#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
+ SSL_CTX_set_options (ctx,SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION);
+#else
+ SSL_CTX_set_min_proto_version(ctx, TLS1_VERSION);
+ SSL_CTX_set_max_proto_version(ctx, version);
+
+ if(tls_ciphers && strcmp(tls_ciphers, "none") != 0) {
+ if (!SSL_CTX_set_cipher_list(ctx, tls_ciphers)) {
+ error("SSL error. cannot set the cipher list");
+ }
+ }
+ }
+#endif
+
+ SSL_CTX_set_mode(ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+}
+
+/**
+ * Initialize Openssl Client
+ *
+ * Starts the client context with TLS 1.2.
+ *
+ * @return It returns the context on success or NULL otherwise
+ */
+SSL_CTX * security_initialize_openssl_client() {
+ SSL_CTX *ctx;
+#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
+ ctx = SSL_CTX_new(SSLv23_client_method());
+#else
+ ctx = SSL_CTX_new(TLS_client_method());
+#endif
+ if(ctx) {
+#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
+ SSL_CTX_set_options (ctx,SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION);
+#else
+ SSL_CTX_set_min_proto_version(ctx, TLS1_VERSION);
+# if defined(TLS_MAX_VERSION)
+ SSL_CTX_set_max_proto_version(ctx, TLS_MAX_VERSION);
+# elif defined(TLS1_3_VERSION)
+ SSL_CTX_set_max_proto_version(ctx, TLS1_3_VERSION);
+# elif defined(TLS1_2_VERSION)
+ SSL_CTX_set_max_proto_version(ctx, TLS1_2_VERSION);
+# endif
+#endif
+ }
+
+ return ctx;
+}
+
+/**
+ * Initialize OpenSSL server
+ *
+ * Starts the server context with TLS 1.2 and load the certificate.
+ *
+ * @return It returns the context on success or NULL otherwise
+ */
+static SSL_CTX * security_initialize_openssl_server() {
+ SSL_CTX *ctx;
+ char lerror[512];
+ static int netdata_id_context = 1;
+
+ //TO DO: Confirm the necessity to check return for other OPENSSL function
+#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
+ ctx = SSL_CTX_new(SSLv23_server_method());
+ if (!ctx) {
+ error("Cannot create a new SSL context, netdata won't encrypt communication");
+ return NULL;
+ }
+
+ SSL_CTX_use_certificate_file(ctx, security_cert, SSL_FILETYPE_PEM);
+#else
+ ctx = SSL_CTX_new(TLS_server_method());
+ if (!ctx) {
+ error("Cannot create a new SSL context, netdata won't encrypt communication");
+ return NULL;
+ }
+
+ SSL_CTX_use_certificate_chain_file(ctx, security_cert);
+#endif
+ security_openssl_common_options(ctx, 0);
+
+ SSL_CTX_use_PrivateKey_file(ctx,security_key,SSL_FILETYPE_PEM);
+
+ if (!SSL_CTX_check_private_key(ctx)) {
+ ERR_error_string_n(ERR_get_error(),lerror,sizeof(lerror));
+ error("SSL cannot check the private key: %s",lerror);
+ SSL_CTX_free(ctx);
+ return NULL;
+ }
+
+ SSL_CTX_set_session_id_context(ctx,(void*)&netdata_id_context,(unsigned int)sizeof(netdata_id_context));
+ SSL_CTX_set_info_callback(ctx,security_info_callback);
+
+#if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_095)
+ SSL_CTX_set_verify_depth(ctx,1);
+#endif
+ debug(D_WEB_CLIENT,"SSL GLOBAL CONTEXT STARTED\n");
+
+ return ctx;
+}
+
+/**
+ * Start SSL
+ *
+ * Call the correct function to start the SSL context.
+ *
+ * @param selector informs the context that must be initialized, the following list has the valid values:
+ * NETDATA_SSL_CONTEXT_SERVER - the server context
+ * NETDATA_SSL_CONTEXT_STREAMING - Starts the streaming context.
+ * NETDATA_SSL_CONTEXT_EXPORTING - Starts the OpenTSDB contextv
+ */
+void security_start_ssl(int selector) {
+ switch (selector) {
+ case NETDATA_SSL_CONTEXT_SERVER: {
+ struct stat statbuf;
+ if (stat(security_key, &statbuf) || stat(security_cert, &statbuf)) {
+ info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n");
+ return;
+ }
+
+ netdata_srv_ctx = security_initialize_openssl_server();
+ break;
+ }
+ case NETDATA_SSL_CONTEXT_STREAMING: {
+ netdata_client_ctx = security_initialize_openssl_client();
+ //This is necessary for the stream, because it is working sometimes with nonblock socket.
+ //It returns the bitmask afte to change, there is not any description of errors in the documentation
+ SSL_CTX_set_mode(netdata_client_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE |SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER |SSL_MODE_AUTO_RETRY);
+ break;
+ }
+ case NETDATA_SSL_CONTEXT_EXPORTING: {
+ netdata_exporting_ctx = security_initialize_openssl_client();
+ break;
+ }
+ }
+}
+
+/**
+ * Clean Open SSL
+ *
+ * Clean all the allocated contexts from netdata.
+ */
+void security_clean_openssl()
+{
+ if (netdata_srv_ctx) {
+ SSL_CTX_free(netdata_srv_ctx);
+ }
+
+ if (netdata_client_ctx) {
+ SSL_CTX_free(netdata_client_ctx);
+ }
+
+ if (netdata_exporting_ctx) {
+ SSL_CTX_free(netdata_exporting_ctx);
+ }
+
+#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
+ ERR_free_strings();
+#endif
+}
+
+/**
+ * Process accept
+ *
+ * Process the SSL handshake with the client case it is necessary.
+ *
+ * @param ssl is a pointer for the SSL structure
+ * @param msg is a copy of the first 8 bytes of the initial message received
+ *
+ * @return it returns 0 case it performs the handshake, 8 case it is clean connection
+ * and another integer power of 2 otherwise.
+ */
+int security_process_accept(SSL *ssl,int msg) {
+ int sock = SSL_get_fd(ssl);
+ int test;
+ if (msg > 0x17)
+ {
+ return NETDATA_SSL_NO_HANDSHAKE;
+ }
+
+ ERR_clear_error();
+ if ((test = SSL_accept(ssl)) <= 0) {
+ int sslerrno = SSL_get_error(ssl, test);
+ switch(sslerrno) {
+ case SSL_ERROR_WANT_READ:
+ {
+ error("SSL handshake did not finish and it wanna read on socket %d!", sock);
+ return NETDATA_SSL_WANT_READ;
+ }
+ case SSL_ERROR_WANT_WRITE:
+ {
+ error("SSL handshake did not finish and it wanna read on socket %d!", sock);
+ return NETDATA_SSL_WANT_WRITE;
+ }
+ case SSL_ERROR_NONE:
+ case SSL_ERROR_SSL:
+ case SSL_ERROR_SYSCALL:
+ default:
+ {
+ u_long err;
+ char buf[256];
+ int counter = 0;
+ while ((err = ERR_get_error()) != 0) {
+ ERR_error_string_n(err, buf, sizeof(buf));
+ info("%d SSL Handshake error (%s) on socket %d ", counter++, ERR_error_string((long)SSL_get_error(ssl, test), NULL), sock);
+ }
+ return NETDATA_SSL_NO_HANDSHAKE;
+ }
+ }
+ }
+
+ if (SSL_is_init_finished(ssl))
+ {
+ debug(D_WEB_CLIENT_ACCESS,"SSL Handshake finished %s errno %d on socket fd %d", ERR_error_string((long)SSL_get_error(ssl, test), NULL), errno, sock);
+ }
+
+ return NETDATA_SSL_HANDSHAKE_COMPLETE;
+}
+
+/**
+ * Test Certificate
+ *
+ * Check the certificate of Netdata parent
+ *
+ * @param ssl is the connection structure
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+int security_test_certificate(SSL *ssl) {
+ X509* cert = SSL_get_peer_certificate(ssl);
+ int ret;
+ long status;
+ if (!cert) {
+ return -1;
+ }
+
+ status = SSL_get_verify_result(ssl);
+ if((X509_V_OK != status))
+ {
+ char error[512];
+ ERR_error_string_n(ERR_get_error(), error, sizeof(error));
+ error("SSL RFC4158 check: We have a invalid certificate, the tests result with %ld and message %s", status, error);
+ ret = -1;
+ } else {
+ ret = 0;
+ }
+
+ return ret;
+}
+
+/**
+ * Location for context
+ *
+ * Case the user give us a directory with the certificates available and
+ * the Netdata parent certificate, we use this function to validate the certificate.
+ *
+ * @param ctx the context where the path will be set.
+ * @param file the file with Netdata parent certificate.
+ * @param path the directory where the certificates are stored.
+ *
+ * @return It returns 0 on success and -1 otherwise.
+ */
+int security_location_for_context(SSL_CTX *ctx, char *file, char *path) {
+ struct stat statbuf;
+ if (stat(file, &statbuf)) {
+ info("Netdata does not have the parent's SSL certificate, so it will use the default OpenSSL configuration to validate certificates!");
+ return 0;
+ }
+
+ ERR_clear_error();
+ u_long err;
+ char buf[256];
+ if(!SSL_CTX_load_verify_locations(ctx, file, path)) {
+ goto slfc;
+ }
+
+ if(!SSL_CTX_set_default_verify_paths(ctx)) {
+ goto slfc;
+ }
+
+ return 0;
+
+slfc:
+ while ((err = ERR_get_error()) != 0) {
+ ERR_error_string_n(err, buf, sizeof(buf));
+ error("Cannot set the directory for the certificates and the parent SSL certificate: %s",buf);
+ }
+ return -1;
+}
+
+#endif
diff --git a/libnetdata/socket/security.h b/libnetdata/socket/security.h
new file mode 100644
index 0000000..17ecc6d
--- /dev/null
+++ b/libnetdata/socket/security.h
@@ -0,0 +1,55 @@
+#ifndef NETDATA_SECURITY_H
+# define NETDATA_SECURITY_H
+
+# define NETDATA_SSL_HANDSHAKE_COMPLETE 0 //All the steps were successful
+# define NETDATA_SSL_START 1 //Starting handshake, conn variable is NULL
+# define NETDATA_SSL_WANT_READ 2 //The connection wanna read from socket
+# define NETDATA_SSL_WANT_WRITE 4 //The connection wanna write on socket
+# define NETDATA_SSL_NO_HANDSHAKE 8 //Continue without encrypt connection.
+# define NETDATA_SSL_OPTIONAL 16 //Flag to define the HTTP request
+# define NETDATA_SSL_FORCE 32 //We only accepts HTTPS request
+# define NETDATA_SSL_INVALID_CERTIFICATE 64 //Accepts invalid certificate
+# define NETDATA_SSL_VALID_CERTIFICATE 128 //Accepts invalid certificate
+# define NETDATA_SSL_PROXY_HTTPS 256 //Proxy is using HTTPS
+
+#define NETDATA_SSL_CONTEXT_SERVER 0
+#define NETDATA_SSL_CONTEXT_STREAMING 1
+#define NETDATA_SSL_CONTEXT_EXPORTING 2
+
+# ifdef ENABLE_HTTPS
+
+#define OPENSSL_VERSION_095 0x00905100L
+#define OPENSSL_VERSION_097 0x0907000L
+#define OPENSSL_VERSION_110 0x10100000L
+#define OPENSSL_VERSION_111 0x10101000L
+
+# include <openssl/ssl.h>
+# include <openssl/err.h>
+# if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) && (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110)
+# include <openssl/conf.h>
+# endif
+
+struct netdata_ssl{
+ SSL *conn; //SSL connection
+ uint32_t flags; //The flags for SSL connection
+};
+
+extern SSL_CTX *netdata_exporting_ctx;
+extern SSL_CTX *netdata_client_ctx;
+extern SSL_CTX *netdata_srv_ctx;
+extern const char *security_key;
+extern const char *security_cert;
+extern const char *tls_version;
+extern const char *tls_ciphers;
+extern int netdata_validate_server;
+extern int security_location_for_context(SSL_CTX *ctx,char *file,char *path);
+
+void security_openssl_library();
+void security_clean_openssl();
+void security_start_ssl(int selector);
+int security_process_accept(SSL *ssl,int msg);
+int security_test_certificate(SSL *ssl);
+SSL_CTX * security_initialize_openssl_client();
+
+# endif //ENABLE_HTTPS
+#endif //NETDATA_SECURITY_H
diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c
new file mode 100644
index 0000000..73eb8e6
--- /dev/null
+++ b/libnetdata/socket/socket.c
@@ -0,0 +1,1733 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+// --------------------------------------------------------------------------------------------------------------------
+// various library calls
+
+#ifdef __gnu_linux__
+#define LARGE_SOCK_SIZE 33554431 // don't ask why - I found it at brubeck source - I guess it is just a large number
+#else
+#define LARGE_SOCK_SIZE 4096
+#endif
+
+int sock_setnonblock(int fd) {
+ int flags;
+
+ flags = fcntl(fd, F_GETFL);
+ flags |= O_NONBLOCK;
+
+ int ret = fcntl(fd, F_SETFL, flags);
+ if(ret < 0)
+ error("Failed to set O_NONBLOCK on socket %d", fd);
+
+ return ret;
+}
+
+int sock_delnonblock(int fd) {
+ int flags;
+
+ flags = fcntl(fd, F_GETFL);
+ flags &= ~O_NONBLOCK;
+
+ int ret = fcntl(fd, F_SETFL, flags);
+ if(ret < 0)
+ error("Failed to remove O_NONBLOCK on socket %d", fd);
+
+ return ret;
+}
+
+int sock_setreuse(int fd, int reuse) {
+ int ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
+
+ if(ret == -1)
+ error("Failed to set SO_REUSEADDR on socket %d", fd);
+
+ return ret;
+}
+
+int sock_setreuse_port(int fd, int reuse) {
+ int ret;
+
+#ifdef SO_REUSEPORT
+ ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &reuse, sizeof(reuse));
+ if(ret == -1 && errno != ENOPROTOOPT)
+ error("failed to set SO_REUSEPORT on socket %d", fd);
+#else
+ ret = -1;
+#endif
+
+ return ret;
+}
+
+int sock_enlarge_in(int fd) {
+ int ret, bs = LARGE_SOCK_SIZE;
+
+ ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs));
+
+ if(ret == -1)
+ error("Failed to set SO_RCVBUF on socket %d", fd);
+
+ return ret;
+}
+
+int sock_enlarge_out(int fd) {
+ int ret, bs = LARGE_SOCK_SIZE;
+ ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs));
+
+ if(ret == -1)
+ error("Failed to set SO_SNDBUF on socket %d", fd);
+
+ return ret;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+
+char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port) {
+ char buffer[100 + 1];
+
+ switch(family) {
+ case AF_INET:
+ snprintfz(buffer, 100, "%s:%s:%d", protocol, ip, port);
+ break;
+
+ case AF_INET6:
+ default:
+ snprintfz(buffer, 100, "%s:[%s]:%d", protocol, ip, port);
+ break;
+
+ case AF_UNIX:
+ snprintfz(buffer, 100, "%s:%s", protocol, ip);
+ break;
+ }
+
+ return strdupz(buffer);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// listening sockets
+
+int create_listen_socket_unix(const char *path, int listen_backlog) {
+ int sock;
+
+ debug(D_LISTENER, "LISTENER: UNIX creating new listening socket on path '%s'", path);
+
+ sock = socket(AF_UNIX, SOCK_STREAM, 0);
+ if(sock < 0) {
+ error("LISTENER: UNIX socket() on path '%s' failed.", path);
+ return -1;
+ }
+
+ sock_setnonblock(sock);
+ sock_enlarge_in(sock);
+
+ struct sockaddr_un name;
+ memset(&name, 0, sizeof(struct sockaddr_un));
+ name.sun_family = AF_UNIX;
+ strncpy(name.sun_path, path, sizeof(name.sun_path)-1);
+
+ errno = 0;
+ if (unlink(path) == -1 && errno != ENOENT)
+ error("LISTENER: failed to remove existing (probably obsolete or left-over) file on UNIX socket path '%s'.", path);
+
+ if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
+ close(sock);
+ error("LISTENER: UNIX bind() on path '%s' failed.", path);
+ return -1;
+ }
+
+ // we have to chmod this to 0777 so that the client will be able
+ // to read from and write to this socket.
+ if(chmod(path, 0777) == -1)
+ error("LISTENER: failed to chmod() socket file '%s'.", path);
+
+ if(listen(sock, listen_backlog) < 0) {
+ close(sock);
+ error("LISTENER: UNIX listen() on path '%s' failed.", path);
+ return -1;
+ }
+
+ debug(D_LISTENER, "LISTENER: Listening on UNIX path '%s'", path);
+ return sock;
+}
+
+int create_listen_socket4(int socktype, const char *ip, uint16_t port, int listen_backlog) {
+ int sock;
+
+ debug(D_LISTENER, "LISTENER: IPv4 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype);
+
+ sock = socket(AF_INET, socktype, 0);
+ if(sock < 0) {
+ error("LISTENER: IPv4 socket() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ sock_setreuse(sock, 1);
+ sock_setreuse_port(sock, 1);
+ sock_setnonblock(sock);
+ sock_enlarge_in(sock);
+
+ struct sockaddr_in name;
+ memset(&name, 0, sizeof(struct sockaddr_in));
+ name.sin_family = AF_INET;
+ name.sin_port = htons (port);
+
+ int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr);
+ if(ret != 1) {
+ error("LISTENER: Failed to convert IP '%s' to a valid IPv4 address.", ip);
+ close(sock);
+ return -1;
+ }
+
+ if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
+ close(sock);
+ error("LISTENER: IPv4 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ if(socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) {
+ close(sock);
+ error("LISTENER: IPv4 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ debug(D_LISTENER, "LISTENER: Listening on IPv4 ip '%s' port %d, socktype %d", ip, port, socktype);
+ return sock;
+}
+
+int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int port, int listen_backlog) {
+ int sock;
+ int ipv6only = 1;
+
+ debug(D_LISTENER, "LISTENER: IPv6 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype);
+
+ sock = socket(AF_INET6, socktype, 0);
+ if (sock < 0) {
+ error("LISTENER: IPv6 socket() on ip '%s' port %d, socktype %d, failed.", ip, port, socktype);
+ return -1;
+ }
+
+ sock_setreuse(sock, 1);
+ sock_setreuse_port(sock, 1);
+ sock_setnonblock(sock);
+ sock_enlarge_in(sock);
+
+ /* IPv6 only */
+ if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0)
+ error("LISTENER: Cannot set IPV6_V6ONLY on ip '%s' port %d, socktype %d.", ip, port, socktype);
+
+ struct sockaddr_in6 name;
+ memset(&name, 0, sizeof(struct sockaddr_in6));
+ name.sin6_family = AF_INET6;
+ name.sin6_port = htons ((uint16_t) port);
+ name.sin6_scope_id = scope_id;
+
+ int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr);
+ if(ret != 1) {
+ error("LISTENER: Failed to convert IP '%s' to a valid IPv6 address.", ip);
+ close(sock);
+ return -1;
+ }
+
+ name.sin6_scope_id = scope_id;
+
+ if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
+ close(sock);
+ error("LISTENER: IPv6 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ if (socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) {
+ close(sock);
+ error("LISTENER: IPv6 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype);
+ return -1;
+ }
+
+ debug(D_LISTENER, "LISTENER: Listening on IPv6 ip '%s' port %d, socktype %d", ip, port, socktype);
+ return sock;
+}
+
+static inline int listen_sockets_add(LISTEN_SOCKETS *sockets, int fd, int family, int socktype, const char *protocol, const char *ip, uint16_t port, int acl_flags) {
+ if(sockets->opened >= MAX_LISTEN_FDS) {
+ error("LISTENER: Too many listening sockets. Failed to add listening %s socket at ip '%s' port %d, protocol %s, socktype %d", protocol, ip, port, protocol, socktype);
+ close(fd);
+ return -1;
+ }
+
+ sockets->fds[sockets->opened] = fd;
+ sockets->fds_types[sockets->opened] = socktype;
+ sockets->fds_families[sockets->opened] = family;
+ sockets->fds_names[sockets->opened] = strdup_client_description(family, protocol, ip, port);
+ sockets->fds_acl_flags[sockets->opened] = acl_flags;
+
+ sockets->opened++;
+ return 0;
+}
+
+int listen_sockets_check_is_member(LISTEN_SOCKETS *sockets, int fd) {
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++)
+ if(sockets->fds[i] == fd) return 1;
+
+ return 0;
+}
+
+static inline void listen_sockets_init(LISTEN_SOCKETS *sockets) {
+ size_t i;
+ for(i = 0; i < MAX_LISTEN_FDS ;i++) {
+ sockets->fds[i] = -1;
+ sockets->fds_names[i] = NULL;
+ sockets->fds_types[i] = -1;
+ }
+
+ sockets->opened = 0;
+ sockets->failed = 0;
+}
+
+void listen_sockets_close(LISTEN_SOCKETS *sockets) {
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++) {
+ close(sockets->fds[i]);
+ sockets->fds[i] = -1;
+
+ freez(sockets->fds_names[i]);
+ sockets->fds_names[i] = NULL;
+
+ sockets->fds_types[i] = -1;
+ }
+
+ sockets->opened = 0;
+ sockets->failed = 0;
+}
+
+/*
+ * SSL ACL
+ *
+ * Search the SSL acl and apply it case it is set.
+ *
+ * @param acl is the acl given by the user.
+ */
+WEB_CLIENT_ACL socket_ssl_acl(char *acl) {
+ char *ssl = strchr(acl,'^');
+ if(ssl) {
+ //Due the format of the SSL command it is always the last command,
+ //we finish it here to avoid problems with the ACLs
+ *ssl = '\0';
+#ifdef ENABLE_HTTPS
+ ssl++;
+ if (!strncmp("SSL=",ssl,4)) {
+ ssl += 4;
+ if (!strcmp(ssl,"optional")) {
+ return WEB_CLIENT_ACL_SSL_OPTIONAL;
+ }
+ else if (!strcmp(ssl,"force")) {
+ return WEB_CLIENT_ACL_SSL_FORCE;
+ }
+ }
+#endif
+ }
+
+ return WEB_CLIENT_ACL_NONE;
+}
+
+WEB_CLIENT_ACL read_acl(char *st) {
+ WEB_CLIENT_ACL ret = socket_ssl_acl(st);
+
+ if (!strcmp(st,"dashboard")) ret |= WEB_CLIENT_ACL_DASHBOARD;
+ if (!strcmp(st,"registry")) ret |= WEB_CLIENT_ACL_REGISTRY;
+ if (!strcmp(st,"badges")) ret |= WEB_CLIENT_ACL_BADGE;
+ if (!strcmp(st,"management")) ret |= WEB_CLIENT_ACL_MGMT;
+ if (!strcmp(st,"streaming")) ret |= WEB_CLIENT_ACL_STREAMING;
+ if (!strcmp(st,"netdata.conf")) ret |= WEB_CLIENT_ACL_NETDATACONF;
+
+ return ret;
+}
+
+static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, uint16_t default_port, int listen_backlog) {
+ int added = 0;
+ WEB_CLIENT_ACL acl_flags = WEB_CLIENT_ACL_NONE;
+
+ struct addrinfo hints;
+ struct addrinfo *result = NULL, *rp = NULL;
+
+ char buffer[strlen(definition) + 1];
+ strcpy(buffer, definition);
+
+ char buffer2[10 + 1];
+ snprintfz(buffer2, 10, "%d", default_port);
+
+ char *ip = buffer, *port = buffer2, *interface = "", *portconfig;;
+
+ int protocol = IPPROTO_TCP, socktype = SOCK_STREAM;
+ const char *protocol_str = "tcp";
+
+ if(strncmp(ip, "tcp:", 4) == 0) {
+ ip += 4;
+ protocol = IPPROTO_TCP;
+ socktype = SOCK_STREAM;
+ protocol_str = "tcp";
+ }
+ else if(strncmp(ip, "udp:", 4) == 0) {
+ ip += 4;
+ protocol = IPPROTO_UDP;
+ socktype = SOCK_DGRAM;
+ protocol_str = "udp";
+ }
+ else if(strncmp(ip, "unix:", 5) == 0) {
+ char *path = ip + 5;
+ socktype = SOCK_STREAM;
+ protocol_str = "unix";
+ int fd = create_listen_socket_unix(path, listen_backlog);
+ if (fd == -1) {
+ error("LISTENER: Cannot create unix socket '%s'", path);
+ sockets->failed++;
+ } else {
+ acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT;
+ listen_sockets_add(sockets, fd, AF_UNIX, socktype, protocol_str, path, 0, acl_flags);
+ added++;
+ }
+ return added;
+ }
+
+ char *e = ip;
+ if(*e == '[') {
+ e = ++ip;
+ while(*e && *e != ']') e++;
+ if(*e == ']') {
+ *e = '\0';
+ e++;
+ }
+ }
+ else {
+ while(*e && *e != ':' && *e != '%' && *e != '=') e++;
+ }
+
+ if(*e == '%') {
+ *e = '\0';
+ e++;
+ interface = e;
+ while(*e && *e != ':' && *e != '=') e++;
+ }
+
+ if(*e == ':') {
+ port = e + 1;
+ *e = '\0';
+ e++;
+ while(*e && *e != '=') e++;
+ }
+
+ if(*e == '=') {
+ *e='\0';
+ e++;
+ portconfig = e;
+ while (*e != '\0') {
+ if (*e == '|') {
+ *e = '\0';
+ acl_flags |= read_acl(portconfig);
+ e++;
+ portconfig = e;
+ continue;
+ }
+ e++;
+ }
+ acl_flags |= read_acl(portconfig);
+ } else {
+ acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT;
+ }
+
+ //Case the user does not set the option SSL in the "bind to", but he has
+ //the certificates, I must redirect, so I am assuming here the default option
+ if(!(acl_flags & WEB_CLIENT_ACL_SSL_OPTIONAL) && !(acl_flags & WEB_CLIENT_ACL_SSL_FORCE)) {
+ acl_flags |= WEB_CLIENT_ACL_SSL_DEFAULT;
+ }
+
+ uint32_t scope_id = 0;
+ if(*interface) {
+ scope_id = if_nametoindex(interface);
+ if(!scope_id)
+ error("LISTENER: Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface);
+ }
+
+ if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all"))
+ ip = NULL;
+
+ if(!*port)
+ port = buffer2;
+
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
+ hints.ai_socktype = socktype;
+ hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */
+ hints.ai_protocol = protocol;
+ hints.ai_canonname = NULL;
+ hints.ai_addr = NULL;
+ hints.ai_next = NULL;
+
+ int r = getaddrinfo(ip, port, &hints, &result);
+ if (r != 0) {
+ error("LISTENER: getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r));
+ return -1;
+ }
+
+ for (rp = result; rp != NULL; rp = rp->ai_next) {
+ int fd = -1;
+ int family;
+
+ char rip[INET_ADDRSTRLEN + INET6_ADDRSTRLEN] = "INVALID";
+ uint16_t rport = default_port;
+
+ family = rp->ai_addr->sa_family;
+ switch (family) {
+ case AF_INET: {
+ struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr;
+ inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN);
+ rport = ntohs(sin->sin_port);
+ // info("Attempting to listen on IPv4 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype);
+ fd = create_listen_socket4(socktype, rip, rport, listen_backlog);
+ break;
+ }
+
+ case AF_INET6: {
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr;
+ inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN);
+ rport = ntohs(sin6->sin6_port);
+ // info("Attempting to listen on IPv6 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype);
+ fd = create_listen_socket6(socktype, scope_id, rip, rport, listen_backlog);
+ break;
+ }
+
+ default:
+ debug(D_LISTENER, "LISTENER: Unknown socket family %d", family);
+ break;
+ }
+
+ if (fd == -1) {
+ error("LISTENER: Cannot bind to ip '%s', port %d", rip, rport);
+ sockets->failed++;
+ }
+ else {
+ listen_sockets_add(sockets, fd, family, socktype, protocol_str, rip, rport, acl_flags);
+ added++;
+ }
+ }
+
+ freeaddrinfo(result);
+
+ return added;
+}
+
+int listen_sockets_setup(LISTEN_SOCKETS *sockets) {
+ listen_sockets_init(sockets);
+
+ sockets->backlog = (int) appconfig_get_number(sockets->config, sockets->config_section, "listen backlog", sockets->backlog);
+
+ long long int old_port = sockets->default_port;
+ long long int new_port = appconfig_get_number(sockets->config, sockets->config_section, "default port", sockets->default_port);
+ if(new_port < 1 || new_port > 65535) {
+ error("LISTENER: Invalid listen port %lld given. Defaulting to %lld.", new_port, old_port);
+ sockets->default_port = (uint16_t) appconfig_set_number(sockets->config, sockets->config_section, "default port", old_port);
+ }
+ else sockets->default_port = (uint16_t)new_port;
+
+ debug(D_OPTIONS, "LISTENER: Default listen port set to %d.", sockets->default_port);
+
+ char *s = appconfig_get(sockets->config, sockets->config_section, "bind to", sockets->default_bind_to);
+ while(*s) {
+ char *e = s;
+
+ // skip separators, moving both s(tart) and e(nd)
+ while(isspace(*e) || *e == ',') s = ++e;
+
+ // move e(nd) to the first separator
+ while(*e && !isspace(*e) && *e != ',') e++;
+
+ // is there anything?
+ if(!*s || s == e) break;
+
+ char buf[e - s + 1];
+ strncpyz(buf, s, e - s);
+ bind_to_this(sockets, buf, sockets->default_port, sockets->backlog);
+
+ s = e;
+ }
+
+ if(sockets->failed) {
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++)
+ info("LISTENER: Listen socket %s opened successfully.", sockets->fds_names[i]);
+ }
+
+ return (int)sockets->opened;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// connect to another host/port
+
+// connect_to_this_unix()
+// path the path of the unix socket
+// timeout the timeout for establishing a connection
+
+static inline int connect_to_unix(const char *path, struct timeval *timeout) {
+ int fd = socket(AF_UNIX, SOCK_STREAM, 0);
+ if(fd == -1) {
+ error("Failed to create UNIX socket() for '%s'", path);
+ return -1;
+ }
+
+ if(timeout) {
+ if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0)
+ error("Failed to set timeout on UNIX socket '%s'", path);
+ }
+
+ struct sockaddr_un addr;
+ memset(&addr, 0, sizeof(addr));
+ addr.sun_family = AF_UNIX;
+ strncpy(addr.sun_path, path, sizeof(addr.sun_path)-1);
+
+ if (connect(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
+ error("Cannot connect to UNIX socket on path '%s'.", path);
+ close(fd);
+ return -1;
+ }
+
+ debug(D_CONNECT_TO, "Connected to UNIX socket on path '%s'.", path);
+
+ return fd;
+}
+
+// connect_to_this_ip46()
+// protocol IPPROTO_TCP, IPPROTO_UDP
+// socktype SOCK_STREAM, SOCK_DGRAM
+// host the destination hostname or IP address (IPv4 or IPv6) to connect to
+// if it resolves to many IPs, all are tried (IPv4 and IPv6)
+// scope_id the if_index id of the interface to use for connecting (0 = any)
+// (used only under IPv6)
+// service the service name or port to connect to
+// timeout the timeout for establishing a connection
+
+int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout) {
+ struct addrinfo hints;
+ struct addrinfo *ai_head = NULL, *ai = NULL;
+
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_family = PF_UNSPEC; /* Allow IPv4 or IPv6 */
+ hints.ai_socktype = socktype;
+ hints.ai_protocol = protocol;
+
+ int ai_err = getaddrinfo(host, service, &hints, &ai_head);
+ if (ai_err != 0) {
+ error("Cannot resolve host '%s', port '%s': %s", host, service, gai_strerror(ai_err));
+ return -1;
+ }
+
+ int fd = -1;
+ for (ai = ai_head; ai != NULL && fd == -1; ai = ai->ai_next) {
+
+ if (ai->ai_family == PF_INET6) {
+ struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr;
+ if(pSadrIn6->sin6_scope_id == 0) {
+ pSadrIn6->sin6_scope_id = scope_id;
+ }
+ }
+
+ char hostBfr[NI_MAXHOST + 1];
+ char servBfr[NI_MAXSERV + 1];
+
+ getnameinfo(ai->ai_addr,
+ ai->ai_addrlen,
+ hostBfr,
+ sizeof(hostBfr),
+ servBfr,
+ sizeof(servBfr),
+ NI_NUMERICHOST | NI_NUMERICSERV);
+
+ debug(D_CONNECT_TO, "Address info: host = '%s', service = '%s', ai_flags = 0x%02X, ai_family = %d (PF_INET = %d, PF_INET6 = %d), ai_socktype = %d (SOCK_STREAM = %d, SOCK_DGRAM = %d), ai_protocol = %d (IPPROTO_TCP = %d, IPPROTO_UDP = %d), ai_addrlen = %lu (sockaddr_in = %lu, sockaddr_in6 = %lu)",
+ hostBfr,
+ servBfr,
+ (unsigned int)ai->ai_flags,
+ ai->ai_family,
+ PF_INET,
+ PF_INET6,
+ ai->ai_socktype,
+ SOCK_STREAM,
+ SOCK_DGRAM,
+ ai->ai_protocol,
+ IPPROTO_TCP,
+ IPPROTO_UDP,
+ (unsigned long)ai->ai_addrlen,
+ (unsigned long)sizeof(struct sockaddr_in),
+ (unsigned long)sizeof(struct sockaddr_in6));
+
+ switch (ai->ai_addr->sa_family) {
+ case PF_INET: {
+ struct sockaddr_in *pSadrIn = (struct sockaddr_in *)ai->ai_addr;
+ (void)pSadrIn;
+
+ debug(D_CONNECT_TO, "ai_addr = sin_family: %d (AF_INET = %d, AF_INET6 = %d), sin_addr: '%s', sin_port: '%s'",
+ pSadrIn->sin_family,
+ AF_INET,
+ AF_INET6,
+ hostBfr,
+ servBfr);
+ break;
+ }
+
+ case PF_INET6: {
+ struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr;
+ (void)pSadrIn6;
+
+ debug(D_CONNECT_TO,"ai_addr = sin6_family: %d (AF_INET = %d, AF_INET6 = %d), sin6_addr: '%s', sin6_port: '%s', sin6_flowinfo: %u, sin6_scope_id: %u",
+ pSadrIn6->sin6_family,
+ AF_INET,
+ AF_INET6,
+ hostBfr,
+ servBfr,
+ pSadrIn6->sin6_flowinfo,
+ pSadrIn6->sin6_scope_id);
+ break;
+ }
+
+ default: {
+ debug(D_CONNECT_TO, "Unknown protocol family %d.", ai->ai_family);
+ continue;
+ }
+ }
+
+ fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
+ if(fd != -1) {
+ if(timeout) {
+ if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0)
+ error("Failed to set timeout on the socket to ip '%s' port '%s'", hostBfr, servBfr);
+ }
+
+ errno = 0;
+ if(connect(fd, ai->ai_addr, ai->ai_addrlen) < 0) {
+ if(errno == EALREADY || errno == EINPROGRESS) {
+ info("Waiting for connection to ip %s port %s to be established", hostBfr, servBfr);
+
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(0, &fds);
+ int rc = select (1, NULL, &fds, NULL, timeout);
+
+ if(rc > 0 && FD_ISSET(fd, &fds)) {
+ info("connect() to ip %s port %s completed successfully", hostBfr, servBfr);
+ }
+ else if(rc == -1) {
+ error("Failed to connect to '%s', port '%s'. select() returned %d", hostBfr, servBfr, rc);
+ close(fd);
+ fd = -1;
+ }
+ else {
+ error("Timed out while connecting to '%s', port '%s'. select() returned %d", hostBfr, servBfr, rc);
+ close(fd);
+ fd = -1;
+ }
+ }
+ else {
+ error("Failed to connect to '%s', port '%s'", hostBfr, servBfr);
+ close(fd);
+ fd = -1;
+ }
+ }
+
+ if(fd != -1)
+ debug(D_CONNECT_TO, "Connected to '%s' on port '%s'.", hostBfr, servBfr);
+ }
+ }
+
+ freeaddrinfo(ai_head);
+
+ return fd;
+}
+
+// connect_to_this()
+//
+// definition format:
+//
+// [PROTOCOL:]IP[%INTERFACE][:PORT]
+//
+// PROTOCOL = tcp or udp
+// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6)
+// INTERFACE = for IPv6 only, the network interface to use
+// PORT = port number or service name
+
+int connect_to_this(const char *definition, int default_port, struct timeval *timeout) {
+ char buffer[strlen(definition) + 1];
+ strcpy(buffer, definition);
+
+ char default_service[10 + 1];
+ snprintfz(default_service, 10, "%d", default_port);
+
+ char *host = buffer, *service = default_service, *interface = "";
+ int protocol = IPPROTO_TCP, socktype = SOCK_STREAM;
+ uint32_t scope_id = 0;
+
+ if(strncmp(host, "tcp:", 4) == 0) {
+ host += 4;
+ protocol = IPPROTO_TCP;
+ socktype = SOCK_STREAM;
+ }
+ else if(strncmp(host, "udp:", 4) == 0) {
+ host += 4;
+ protocol = IPPROTO_UDP;
+ socktype = SOCK_DGRAM;
+ }
+ else if(strncmp(host, "unix:", 5) == 0) {
+ char *path = host + 5;
+ return connect_to_unix(path, timeout);
+ }
+
+ char *e = host;
+ if(*e == '[') {
+ e = ++host;
+ while(*e && *e != ']') e++;
+ if(*e == ']') {
+ *e = '\0';
+ e++;
+ }
+ }
+ else {
+ while(*e && *e != ':' && *e != '%') e++;
+ }
+
+ if(*e == '%') {
+ *e = '\0';
+ e++;
+ interface = e;
+ while(*e && *e != ':') e++;
+ }
+
+ if(*e == ':') {
+ *e = '\0';
+ e++;
+ service = e;
+ }
+
+ debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP);
+
+ if(!*host) {
+ error("Definition '%s' does not specify a host.", definition);
+ return -1;
+ }
+
+ if(*interface) {
+ scope_id = if_nametoindex(interface);
+ if(!scope_id)
+ error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface);
+ }
+
+ if(!*service)
+ service = default_service;
+
+
+ return connect_to_this_ip46(protocol, socktype, host, scope_id, service, timeout);
+}
+
+int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) {
+ int sock = -1;
+
+ const char *s = destination;
+ while(*s) {
+ const char *e = s;
+
+ // skip path, moving both s(tart) and e(nd)
+ if(*e == '/')
+ while(!isspace(*e) && *e != ',') s = ++e;
+
+ // skip separators, moving both s(tart) and e(nd)
+ while(isspace(*e) || *e == ',') s = ++e;
+
+ // move e(nd) to the first separator
+ while(*e && !isspace(*e) && *e != ',' && *e != '/') e++;
+
+ // is there anything?
+ if(!*s || s == e) break;
+
+ char buf[e - s + 1];
+ strncpyz(buf, s, e - s);
+ if(reconnects_counter) *reconnects_counter += 1;
+ sock = connect_to_this(buf, default_port, timeout);
+ if(sock != -1) {
+ if(connected_to && connected_to_size) {
+ strncpy(connected_to, buf, connected_to_size);
+ connected_to[connected_to_size - 1] = '\0';
+ }
+ break;
+ }
+ s = e;
+ }
+
+ return sock;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// helpers to send/receive data in one call, in blocking mode, with a timeout
+
+#ifdef ENABLE_HTTPS
+ssize_t recv_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) {
+#else
+ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) {
+#endif
+
+ for(;;) {
+ struct pollfd fd = {
+ .fd = sockfd,
+ .events = POLLIN,
+ .revents = 0
+ };
+
+ errno = 0;
+ int retval = poll(&fd, 1, timeout * 1000);
+
+ if(retval == -1) {
+ // failed
+
+ if(errno == EINTR || errno == EAGAIN)
+ continue;
+
+ return -1;
+ }
+
+ if(!retval) {
+ // timeout
+ return 0;
+ }
+
+ if(fd.events & POLLIN) break;
+ }
+
+#ifdef ENABLE_HTTPS
+ if (ssl->conn) {
+ if (!ssl->flags) {
+ return SSL_read(ssl->conn,buf,len);
+ }
+ }
+#endif
+ return recv(sockfd, buf, len, flags);
+}
+
+#ifdef ENABLE_HTTPS
+ssize_t send_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) {
+#else
+ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) {
+#endif
+
+ for(;;) {
+ struct pollfd fd = {
+ .fd = sockfd,
+ .events = POLLOUT,
+ .revents = 0
+ };
+
+ errno = 0;
+ int retval = poll(&fd, 1, timeout * 1000);
+
+ if(retval == -1) {
+ // failed
+
+ if(errno == EINTR || errno == EAGAIN)
+ continue;
+
+ return -1;
+ }
+
+ if(!retval) {
+ // timeout
+ return 0;
+ }
+
+ if(fd.events & POLLOUT) break;
+ }
+
+#ifdef ENABLE_HTTPS
+ if(ssl->conn) {
+ if (!ssl->flags) {
+ return SSL_write(ssl->conn, buf, len);
+ }
+ }
+#endif
+ return send(sockfd, buf, len, flags);
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// accept4() replacement for systems that do not have one
+
+#ifndef HAVE_ACCEPT4
+int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) {
+ int fd = accept(sock, addr, addrlen);
+ int newflags = 0;
+
+ if (fd < 0) return fd;
+
+ if (flags & SOCK_NONBLOCK) {
+ newflags |= O_NONBLOCK;
+ flags &= ~SOCK_NONBLOCK;
+ }
+
+#ifdef SOCK_CLOEXEC
+#ifdef O_CLOEXEC
+ if (flags & SOCK_CLOEXEC) {
+ newflags |= O_CLOEXEC;
+ flags &= ~SOCK_CLOEXEC;
+ }
+#endif
+#endif
+
+ if (flags) {
+ close(fd);
+ errno = EINVAL;
+ return -1;
+ }
+
+ if (fcntl(fd, F_SETFL, newflags) < 0) {
+ int saved_errno = errno;
+ close(fd);
+ errno = saved_errno;
+ return -1;
+ }
+
+ return fd;
+}
+#endif
+
+/*
+ * ---------------------------------------------------------------------------------------------------------------------
+ * connection_allowed() - if there is an access list then check the connection matches a pattern.
+ * Numeric patterns are checked against the IP address first, only if they
+ * do not match is the hostname resolved (reverse-DNS) and checked. If the
+ * hostname matches then we perform forward DNS resolution to check the IP
+ * is really associated with the DNS record. This call is repeatable: the
+ * web server may check more refined matches against the connection. Will
+ * update the client_host if uninitialized - ensure the hostsize is the number
+ * of *writable* bytes (i.e. be aware of the strdup used to compact the pollinfo).
+ */
+extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list,
+ const char *patname, int allow_dns)
+{
+ debug(D_LISTENER,"checking %s... (allow_dns=%d)", patname, allow_dns);
+ if (!access_list)
+ return 1;
+ if (simple_pattern_matches(access_list, client_ip))
+ return 1;
+ // If the hostname is unresolved (and needed) then attempt the DNS lookups.
+ //if (client_host[0]==0 && simple_pattern_is_potential_name(access_list))
+ if (client_host[0]==0 && allow_dns)
+ {
+ struct sockaddr_storage sadr;
+ socklen_t addrlen = sizeof(sadr);
+ int err = getpeername(fd, (struct sockaddr*)&sadr, &addrlen);
+ if (err != 0 ||
+ (err = getnameinfo((struct sockaddr *)&sadr, addrlen, client_host, (socklen_t)hostsize,
+ NULL, 0, NI_NAMEREQD)) != 0) {
+ error("Incoming %s on '%s' does not match a numeric pattern, and host could not be resolved (err=%s)",
+ patname, client_ip, gai_strerror(err));
+ if (hostsize >= 8)
+ strcpy(client_host,"UNKNOWN");
+ return 0;
+ }
+ struct addrinfo *addr_infos = NULL;
+ if (getaddrinfo(client_host, NULL, NULL, &addr_infos) !=0 ) {
+ error("LISTENER: cannot validate hostname '%s' from '%s' by resolving it",
+ client_host, client_ip);
+ if (hostsize >= 8)
+ strcpy(client_host,"UNKNOWN");
+ return 0;
+ }
+ struct addrinfo *scan = addr_infos;
+ int validated = 0;
+ while (scan) {
+ char address[INET6_ADDRSTRLEN];
+ address[0] = 0;
+ switch (scan->ai_addr->sa_family) {
+ case AF_INET:
+ inet_ntop(AF_INET, &((struct sockaddr_in*)(scan->ai_addr))->sin_addr, address, INET6_ADDRSTRLEN);
+ break;
+ case AF_INET6:
+ inet_ntop(AF_INET6, &((struct sockaddr_in6*)(scan->ai_addr))->sin6_addr, address, INET6_ADDRSTRLEN);
+ break;
+ }
+ debug(D_LISTENER, "Incoming ip %s rev-resolved onto %s, validating against forward-resolution %s",
+ client_ip, client_host, address);
+ if (!strcmp(client_ip, address)) {
+ validated = 1;
+ break;
+ }
+ scan = scan->ai_next;
+ }
+ if (!validated) {
+ error("LISTENER: Cannot validate '%s' as ip of '%s', not listed in DNS", client_ip, client_host);
+ if (hostsize >= 8)
+ strcpy(client_host,"UNKNOWN");
+ }
+ if (addr_infos!=NULL)
+ freeaddrinfo(addr_infos);
+ }
+ if (!simple_pattern_matches(access_list, client_host)) {
+ debug(D_LISTENER, "Incoming connection on '%s' (%s) does not match allowed pattern for %s",
+ client_ip, client_host, patname);
+ return 0;
+ }
+ return 1;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// accept_socket() - accept a socket and store client IP and port
+int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize,
+ char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns) {
+ struct sockaddr_storage sadr;
+ socklen_t addrlen = sizeof(sadr);
+
+ int nfd = accept4(fd, (struct sockaddr *)&sadr, &addrlen, flags);
+ if (likely(nfd >= 0)) {
+ if (getnameinfo((struct sockaddr *)&sadr, addrlen, client_ip, (socklen_t)ipsize,
+ client_port, (socklen_t)portsize, NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
+ error("LISTENER: cannot getnameinfo() on received client connection.");
+ strncpyz(client_ip, "UNKNOWN", ipsize - 1);
+ strncpyz(client_port, "UNKNOWN", portsize - 1);
+ }
+ if (!strcmp(client_ip, "127.0.0.1") || !strcmp(client_ip, "::1")) {
+ strncpy(client_ip, "localhost", ipsize);
+ client_ip[ipsize - 1] = '\0';
+ }
+
+#ifdef __FreeBSD__
+ if(((struct sockaddr *)&sadr)->sa_family == AF_LOCAL)
+ strncpyz(client_ip, "localhost", ipsize);
+#endif
+
+ client_ip[ipsize - 1] = '\0';
+ client_port[portsize - 1] = '\0';
+
+ switch (((struct sockaddr *)&sadr)->sa_family) {
+ case AF_UNIX:
+ debug(D_LISTENER, "New UNIX domain web client from %s on socket %d.", client_ip, fd);
+ // set the port - certain versions of libc return garbage on unix sockets
+ strncpy(client_port, "UNIX", portsize);
+ client_port[portsize - 1] = '\0';
+ break;
+
+ case AF_INET:
+ debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ break;
+
+ case AF_INET6:
+ if (strncmp(client_ip, "::ffff:", 7) == 0) {
+ memmove(client_ip, &client_ip[7], strlen(&client_ip[7]) + 1);
+ debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ }
+ else
+ debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ break;
+
+ default:
+ debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd);
+ break;
+ }
+ if (!connection_allowed(nfd, client_ip, client_host, hostsize, access_list, "connection", allow_dns)) {
+ errno = 0;
+ error("Permission denied for client '%s', port '%s'", client_ip, client_port);
+ close(nfd);
+ nfd = -1;
+ errno = EPERM;
+ }
+ }
+#ifdef HAVE_ACCEPT4
+ else if (errno == ENOSYS)
+ error("netdata has been compiled with the assumption that the system has the accept4() call, but it is not here. Recompile netdata like this: ./configure --disable-accept4 ...");
+#endif
+
+ return nfd;
+}
+
+
+// --------------------------------------------------------------------------------------------------------------------
+// poll() based listener
+// this should be the fastest possible listener for up to 100 sockets
+// above 100, an epoll() interface is needed on Linux
+
+#define POLL_FDS_INCREASE_STEP 10
+
+inline POLLINFO *poll_add_fd(POLLJOB *p
+ , int fd
+ , int socktype
+ , WEB_CLIENT_ACL port_acl
+ , uint32_t flags
+ , const char *client_ip
+ , const char *client_port
+ , const char *client_host
+ , void *(*add_callback)(POLLINFO * /*pi*/, short int * /*events*/, void * /*data*/)
+ , void (*del_callback)(POLLINFO * /*pi*/)
+ , int (*rcv_callback)(POLLINFO * /*pi*/, short int * /*events*/)
+ , int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/)
+ , void *data
+) {
+ debug(D_POLLFD, "POLLFD: ADD: request to add fd %d, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", fd, p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ if(unlikely(fd < 0)) return NULL;
+
+ //if(p->limit && p->used >= p->limit) {
+ // info("Max sockets limit reached (%zu sockets), dropping connection", p->used);
+ // close(fd);
+ // return NULL;
+ //}
+
+ if(unlikely(!p->first_free)) {
+ size_t new_slots = p->slots + POLL_FDS_INCREASE_STEP;
+ debug(D_POLLFD, "POLLFD: ADD: increasing size (current = %zu, new = %zu, used = %zu, min = %zu, max = %zu)", p->slots, new_slots, p->used, p->min, p->max);
+
+ p->fds = reallocz(p->fds, sizeof(struct pollfd) * new_slots);
+ p->inf = reallocz(p->inf, sizeof(POLLINFO) * new_slots);
+
+ // reset all the newly added slots
+ ssize_t i;
+ for(i = new_slots - 1; i >= (ssize_t)p->slots ; i--) {
+ debug(D_POLLFD, "POLLFD: ADD: resetting new slot %zd", i);
+ p->fds[i].fd = -1;
+ p->fds[i].events = 0;
+ p->fds[i].revents = 0;
+
+ p->inf[i].p = p;
+ p->inf[i].slot = (size_t)i;
+ p->inf[i].flags = 0;
+ p->inf[i].socktype = -1;
+ p->inf[i].port_acl = -1;
+
+ p->inf[i].client_ip = NULL;
+ p->inf[i].client_port = NULL;
+ p->inf[i].client_host = NULL;
+ p->inf[i].del_callback = p->del_callback;
+ p->inf[i].rcv_callback = p->rcv_callback;
+ p->inf[i].snd_callback = p->snd_callback;
+ p->inf[i].data = NULL;
+
+ // link them so that the first free will be earlier in the array
+ // (we loop decrementing i)
+ p->inf[i].next = p->first_free;
+ p->first_free = &p->inf[i];
+ }
+
+ p->slots = new_slots;
+ }
+
+ POLLINFO *pi = p->first_free;
+ p->first_free = p->first_free->next;
+
+ debug(D_POLLFD, "POLLFD: ADD: selected slot %zu, next free is %zd", pi->slot, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ struct pollfd *pf = &p->fds[pi->slot];
+ pf->fd = fd;
+ pf->events = POLLIN;
+ pf->revents = 0;
+
+ pi->fd = fd;
+ pi->p = p;
+ pi->socktype = socktype;
+ pi->port_acl = port_acl;
+ pi->flags = flags;
+ pi->next = NULL;
+ pi->client_ip = strdupz(client_ip);
+ pi->client_port = strdupz(client_port);
+ pi->client_host = strdupz(client_host);
+
+ pi->del_callback = del_callback;
+ pi->rcv_callback = rcv_callback;
+ pi->snd_callback = snd_callback;
+
+ pi->connected_t = now_boottime_sec();
+ pi->last_received_t = 0;
+ pi->last_sent_t = 0;
+ pi->last_sent_t = 0;
+ pi->recv_count = 0;
+ pi->send_count = 0;
+
+ netdata_thread_disable_cancelability();
+ p->used++;
+ if(unlikely(pi->slot > p->max))
+ p->max = pi->slot;
+
+ if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) {
+ pi->data = add_callback(pi, &pf->events, data);
+ }
+
+ if(pi->flags & POLLINFO_FLAG_SERVER_SOCKET) {
+ p->min = pi->slot;
+ }
+ netdata_thread_enable_cancelability();
+
+ debug(D_POLLFD, "POLLFD: ADD: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ return pi;
+}
+
+inline void poll_close_fd(POLLINFO *pi) {
+ POLLJOB *p = pi->p;
+
+ struct pollfd *pf = &p->fds[pi->slot];
+ debug(D_POLLFD, "POLLFD: DEL: request to clear slot %zu (fd %d), old next free was %zd", pi->slot, pf->fd, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+
+ if(unlikely(pf->fd == -1)) return;
+
+ netdata_thread_disable_cancelability();
+
+ if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) {
+ pi->del_callback(pi);
+
+ if(likely(!(pi->flags & POLLINFO_FLAG_DONT_CLOSE))) {
+ if(close(pf->fd) == -1)
+ error("Failed to close() poll_events() socket %d", pf->fd);
+ }
+ }
+
+ pf->fd = -1;
+ pf->events = 0;
+ pf->revents = 0;
+
+ pi->fd = -1;
+ pi->socktype = -1;
+ pi->flags = 0;
+ pi->data = NULL;
+
+ pi->del_callback = NULL;
+ pi->rcv_callback = NULL;
+ pi->snd_callback = NULL;
+
+ freez(pi->client_ip);
+ pi->client_ip = NULL;
+
+ freez(pi->client_port);
+ pi->client_port = NULL;
+
+ freez(pi->client_host);
+ pi->client_host = NULL;
+
+ pi->next = p->first_free;
+ p->first_free = pi;
+
+ p->used--;
+ if(unlikely(p->max == pi->slot)) {
+ p->max = p->min;
+ ssize_t i;
+ for(i = (ssize_t)pi->slot; i > (ssize_t)p->min ;i--) {
+ if (unlikely(p->fds[i].fd != -1)) {
+ p->max = (size_t)i;
+ break;
+ }
+ }
+ }
+ netdata_thread_enable_cancelability();
+
+ debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1);
+}
+
+void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data) {
+ (void)pi;
+ (void)events;
+ (void)data;
+
+ // error("POLLFD: internal error: poll_default_add_callback() called");
+
+ return NULL;
+}
+
+void poll_default_del_callback(POLLINFO *pi) {
+ if(pi->data)
+ error("POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak");
+}
+
+int poll_default_rcv_callback(POLLINFO *pi, short int *events) {
+ *events |= POLLIN;
+
+ char buffer[1024 + 1];
+
+ ssize_t rc;
+ do {
+ rc = recv(pi->fd, buffer, 1024, MSG_DONTWAIT);
+ if (rc < 0) {
+ // read failed
+ if (errno != EWOULDBLOCK && errno != EAGAIN) {
+ error("POLLFD: poll_default_rcv_callback(): recv() failed with %zd.", rc);
+ return -1;
+ }
+ } else if (rc) {
+ // data received
+ info("POLLFD: internal error: poll_default_rcv_callback() is discarding %zd bytes received on socket %d", rc, pi->fd);
+ }
+ } while (rc != -1);
+
+ return 0;
+}
+
+int poll_default_snd_callback(POLLINFO *pi, short int *events) {
+ *events &= ~POLLOUT;
+
+ info("POLLFD: internal error: poll_default_snd_callback(): nothing to send on socket %d", pi->fd);
+ return 0;
+}
+
+void poll_default_tmr_callback(void *timer_data) {
+ (void)timer_data;
+}
+
+static void poll_events_cleanup(void *data) {
+ POLLJOB *p = (POLLJOB *)data;
+
+ size_t i;
+ for(i = 0 ; i <= p->max ; i++) {
+ POLLINFO *pi = &p->inf[i];
+ poll_close_fd(pi);
+ }
+
+ freez(p->fds);
+ freez(p->inf);
+}
+
+static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, short int revents, time_t now) {
+ short int events = pf->events;
+ int fd = pf->fd;
+ pf->revents = 0;
+ size_t i = pi->slot;
+
+ if(unlikely(fd == -1)) {
+ debug(D_POLLFD, "POLLFD: LISTENER: ignoring slot %zu, it does not have an fd", i);
+ return;
+ }
+
+ debug(D_POLLFD, "POLLFD: LISTENER: processing events for slot %zu (events = %d, revents = %d)", i, events, revents);
+
+ if(revents & POLLIN || revents & POLLPRI) {
+ // receiving data
+
+ pi->last_received_t = now;
+ pi->recv_count++;
+
+ if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) {
+ // read data from client TCP socket
+ debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", i, fd);
+
+ pf->events = 0;
+ if (pi->rcv_callback(pi, &pf->events) == -1) {
+ poll_close_fd(&p->inf[i]);
+ return;
+ }
+ pf = &p->fds[i];
+ pi = &p->inf[i];
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ // this is common - it is used for web server file copies
+ if(unlikely(!(pf->events & (POLLIN|POLLOUT)))) {
+ error("POLLFD: LISTENER: after reading, client slot %zu (fd %d) from %s port %s was left without expecting input or output. ", i, fd, pi->client_ip?pi->client_ip:"<undefined-ip>", pi->client_port?pi->client_port:"<undefined-port>");
+ //poll_close_fd(pi);
+ //return;
+ }
+#endif
+ }
+ else if(likely(pi->flags & POLLINFO_FLAG_SERVER_SOCKET)) {
+ // new connection
+ // debug(D_POLLFD, "POLLFD: LISTENER: accepting connections from slot %zu (fd %d)", i, fd);
+
+ switch(pi->socktype) {
+ case SOCK_STREAM: {
+ // a TCP socket
+ // we accept the connection
+
+ int nfd;
+ do {
+ char client_ip[INET6_ADDRSTRLEN];
+ char client_port[NI_MAXSERV];
+ char client_host[NI_MAXHOST];
+ client_host[0] = 0;
+ client_ip[0] = 0;
+ client_port[0] = 0;
+
+ debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", i, fd);
+ nfd = accept_socket(fd, SOCK_NONBLOCK, client_ip, INET6_ADDRSTRLEN, client_port, NI_MAXSERV,
+ client_host, NI_MAXHOST, p->access_list, p->allow_dns);
+ if (unlikely(nfd < 0)) {
+ // accept failed
+
+ debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", i, fd);
+
+ if(unlikely(errno == EMFILE)) {
+ error("POLLFD: LISTENER: too many open files - sleeping for 1ms - used by this thread %zu, max for this thread %zu", p->used, p->limit);
+ usleep(1000); // 10ms
+ }
+ else if(unlikely(errno != EWOULDBLOCK && errno != EAGAIN))
+ error("POLLFD: LISTENER: accept() failed.");
+
+ break;
+ }
+ else {
+ // accept ok
+ // info("POLLFD: LISTENER: client '[%s]:%s' connected to '%s' on fd %d", client_ip, client_port, sockets->fds_names[i], nfd);
+ poll_add_fd(p
+ , nfd
+ , SOCK_STREAM
+ , pi->port_acl
+ , POLLINFO_FLAG_CLIENT_SOCKET
+ , client_ip
+ , client_port
+ , client_host
+ , p->add_callback
+ , p->del_callback
+ , p->rcv_callback
+ , p->snd_callback
+ , NULL
+ );
+
+ // it may have reallocated them, so refresh our pointers
+ pf = &p->fds[i];
+ pi = &p->inf[i];
+ }
+ } while (nfd >= 0 && (!p->limit || p->used < p->limit));
+ break;
+ }
+
+ case SOCK_DGRAM: {
+ // a UDP socket
+ // we read data from the server socket
+
+ debug(D_POLLFD, "POLLFD: LISTENER: reading data from UDP slot %zu (fd %d)", i, fd);
+
+ // TODO: access_list is not applied to UDP
+ // but checking the access list on every UDP packet will destroy
+ // performance, especially for statsd.
+
+ pf->events = 0;
+ pi->rcv_callback(pi, &pf->events);
+ break;
+ }
+
+ default: {
+ error("POLLFD: LISTENER: Unknown socktype %d on slot %zu", pi->socktype, pi->slot);
+ break;
+ }
+ }
+ }
+ }
+
+ if(unlikely(revents & POLLOUT)) {
+ // sending data
+ debug(D_POLLFD, "POLLFD: LISTENER: sending data to socket on slot %zu (fd %d)", i, fd);
+
+ pi->last_sent_t = now;
+ pi->send_count++;
+
+ pf->events = 0;
+ if (pi->snd_callback(pi, &pf->events) == -1) {
+ poll_close_fd(&p->inf[i]);
+ return;
+ }
+ pf = &p->fds[i];
+ pi = &p->inf[i];
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ // this is common - it is used for streaming
+ if(unlikely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET && !(pf->events & (POLLIN|POLLOUT)))) {
+ error("POLLFD: LISTENER: after sending, client slot %zu (fd %d) from %s port %s was left without expecting input or output. ", i, fd, pi->client_ip?pi->client_ip:"<undefined-ip>", pi->client_port?pi->client_port:"<undefined-port>");
+ //poll_close_fd(pi);
+ //return;
+ }
+#endif
+ }
+
+ if(unlikely(revents & POLLERR)) {
+ error("POLLFD: LISTENER: processing POLLERR events for slot %zu fd %d (events = %d, revents = %d)", i, events, revents, fd);
+ pf->events = 0;
+ poll_close_fd(pi);
+ return;
+ }
+
+ if(unlikely(revents & POLLHUP)) {
+ error("POLLFD: LISTENER: processing POLLHUP events for slot %zu fd %d (events = %d, revents = %d)", i, events, revents, fd);
+ pf->events = 0;
+ poll_close_fd(pi);
+ return;
+ }
+
+ if(unlikely(revents & POLLNVAL)) {
+ error("POLLFD: LISTENER: processing POLLNVAL events for slot %zu fd %d (events = %d, revents = %d)", i, events, revents, fd);
+ pf->events = 0;
+ poll_close_fd(pi);
+ return;
+ }
+}
+
+void poll_events(LISTEN_SOCKETS *sockets
+ , void *(*add_callback)(POLLINFO * /*pi*/, short int * /*events*/, void * /*data*/)
+ , void (*del_callback)(POLLINFO * /*pi*/)
+ , int (*rcv_callback)(POLLINFO * /*pi*/, short int * /*events*/)
+ , int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/)
+ , void (*tmr_callback)(void * /*timer_data*/)
+ , SIMPLE_PATTERN *access_list
+ , int allow_dns
+ , void *data
+ , time_t tcp_request_timeout_seconds
+ , time_t tcp_idle_timeout_seconds
+ , time_t timer_milliseconds
+ , void *timer_data
+ , size_t max_tcp_sockets
+) {
+ if(!sockets || !sockets->opened) {
+ error("POLLFD: internal error: no listening sockets are opened");
+ return;
+ }
+
+ if(timer_milliseconds <= 0) timer_milliseconds = 0;
+
+ int retval;
+
+ POLLJOB p = {
+ .slots = 0,
+ .used = 0,
+ .max = 0,
+ .limit = max_tcp_sockets,
+ .fds = NULL,
+ .inf = NULL,
+ .first_free = NULL,
+
+ .complete_request_timeout = tcp_request_timeout_seconds,
+ .idle_timeout = tcp_idle_timeout_seconds,
+ .checks_every = (tcp_idle_timeout_seconds / 3) + 1,
+
+ .access_list = access_list,
+ .allow_dns = allow_dns,
+
+ .timer_milliseconds = timer_milliseconds,
+ .timer_data = timer_data,
+
+ .add_callback = add_callback?add_callback:poll_default_add_callback,
+ .del_callback = del_callback?del_callback:poll_default_del_callback,
+ .rcv_callback = rcv_callback?rcv_callback:poll_default_rcv_callback,
+ .snd_callback = snd_callback?snd_callback:poll_default_snd_callback,
+ .tmr_callback = tmr_callback?tmr_callback:poll_default_tmr_callback
+ };
+
+ size_t i;
+ for(i = 0; i < sockets->opened ;i++) {
+
+ POLLINFO *pi = poll_add_fd(&p
+ , sockets->fds[i]
+ , sockets->fds_types[i]
+ , sockets->fds_acl_flags[i]
+ , POLLINFO_FLAG_SERVER_SOCKET
+ , (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN"
+ , ""
+ , ""
+ , p.add_callback
+ , p.del_callback
+ , p.rcv_callback
+ , p.snd_callback
+ , NULL
+ );
+
+ pi->data = data;
+ info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN");
+ }
+
+ int listen_sockets_active = 1;
+
+ int timeout_ms = 1000; // in milliseconds
+ time_t last_check = now_boottime_sec();
+
+ usec_t timer_usec = timer_milliseconds * USEC_PER_MS;
+ usec_t now_usec = 0, next_timer_usec = 0, last_timer_usec = 0;
+ (void)last_timer_usec;
+
+ if(unlikely(timer_usec)) {
+ now_usec = now_boottime_usec();
+ next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec;
+ }
+
+ netdata_thread_cleanup_push(poll_events_cleanup, &p);
+
+ while(!netdata_exit) {
+ if(unlikely(timer_usec)) {
+ now_usec = now_boottime_usec();
+
+ if(unlikely(timer_usec && now_usec >= next_timer_usec)) {
+ debug(D_POLLFD, "Calling timer callback after %zu usec", (size_t)(now_usec - last_timer_usec));
+ last_timer_usec = now_usec;
+ p.tmr_callback(p.timer_data);
+ now_usec = now_boottime_usec();
+ next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec;
+ }
+
+ usec_t dt_usec = next_timer_usec - now_usec;
+ if(dt_usec < 1000 * USEC_PER_MS)
+ timeout_ms = 1000;
+ else
+ timeout_ms = (int)(dt_usec / USEC_PER_MS);
+ }
+
+ // enable or disable the TCP listening sockets, based on the current number of sockets used and the limit set
+ if((listen_sockets_active && (p.limit && p.used >= p.limit)) || (!listen_sockets_active && (!p.limit || p.used < p.limit))) {
+ listen_sockets_active = !listen_sockets_active;
+ info("%s listening sockets (used TCP sockets %zu, max allowed for this worker %zu)", (listen_sockets_active)?"ENABLING":"DISABLING", p.used, p.limit);
+ for (i = 0; i <= p.max; i++) {
+ if(p.inf[i].flags & POLLINFO_FLAG_SERVER_SOCKET && p.inf[i].socktype == SOCK_STREAM) {
+ p.fds[i].events = (short int) ((listen_sockets_active) ? POLLIN : 0);
+ }
+ }
+ }
+
+ debug(D_POLLFD, "POLLFD: LISTENER: Waiting on %zu sockets for %zu ms...", p.max + 1, (size_t)timeout_ms);
+ retval = poll(p.fds, p.max + 1, timeout_ms);
+ time_t now = now_boottime_sec();
+
+ if(unlikely(retval == -1)) {
+ error("POLLFD: LISTENER: poll() failed while waiting on %zu sockets.", p.max + 1);
+ break;
+ }
+ else if(unlikely(!retval)) {
+ debug(D_POLLFD, "POLLFD: LISTENER: poll() timeout.");
+ }
+ else {
+ for (i = 0; i <= p.max; i++) {
+ struct pollfd *pf = &p.fds[i];
+ short int revents = pf->revents;
+ if (unlikely(revents))
+ poll_events_process(&p, &p.inf[i], pf, revents, now);
+ }
+ }
+
+ if(unlikely(p.checks_every > 0 && now - last_check > p.checks_every)) {
+ last_check = now;
+
+ // security checks
+ for(i = 0; i <= p.max; i++) {
+ POLLINFO *pi = &p.inf[i];
+
+ if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) {
+ if (unlikely(pi->send_count == 0 && p.complete_request_timeout > 0 && (now - pi->connected_t) >= p.complete_request_timeout)) {
+ info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s has not sent a complete request in %zu seconds - closing it. "
+ , i
+ , pi->fd
+ , pi->client_ip ? pi->client_ip : "<undefined-ip>"
+ , pi->client_port ? pi->client_port : "<undefined-port>"
+ , (size_t) p.complete_request_timeout
+ );
+ poll_close_fd(pi);
+ }
+ else if(unlikely(pi->recv_count && p.idle_timeout > 0 && now - ((pi->last_received_t > pi->last_sent_t) ? pi->last_received_t : pi->last_sent_t) >= p.idle_timeout )) {
+ info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s is idle for more than %zu seconds - closing it. "
+ , i
+ , pi->fd
+ , pi->client_ip ? pi->client_ip : "<undefined-ip>"
+ , pi->client_port ? pi->client_port : "<undefined-port>"
+ , (size_t) p.idle_timeout
+ );
+ poll_close_fd(pi);
+ }
+ }
+ }
+ }
+ }
+
+ netdata_thread_cleanup_pop(1);
+ debug(D_POLLFD, "POLLFD: LISTENER: cleanup completed");
+}
diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h
new file mode 100644
index 0000000..a40d801
--- /dev/null
+++ b/libnetdata/socket/socket.h
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SOCKET_H
+#define NETDATA_SOCKET_H
+
+#include "../libnetdata.h"
+
+#ifndef MAX_LISTEN_FDS
+#define MAX_LISTEN_FDS 50
+#endif
+
+typedef enum web_client_acl {
+ WEB_CLIENT_ACL_NONE = 0,
+ WEB_CLIENT_ACL_NOCHECK = 0,
+ WEB_CLIENT_ACL_DASHBOARD = 1 << 0,
+ WEB_CLIENT_ACL_REGISTRY = 1 << 1,
+ WEB_CLIENT_ACL_BADGE = 1 << 2,
+ WEB_CLIENT_ACL_MGMT = 1 << 3,
+ WEB_CLIENT_ACL_STREAMING = 1 << 4,
+ WEB_CLIENT_ACL_NETDATACONF = 1 << 5,
+ WEB_CLIENT_ACL_SSL_OPTIONAL = 1 << 6,
+ WEB_CLIENT_ACL_SSL_FORCE = 1 << 7,
+ WEB_CLIENT_ACL_SSL_DEFAULT = 1 << 8
+} WEB_CLIENT_ACL;
+
+#define web_client_can_access_dashboard(w) ((w)->acl & WEB_CLIENT_ACL_DASHBOARD)
+#define web_client_can_access_registry(w) ((w)->acl & WEB_CLIENT_ACL_REGISTRY)
+#define web_client_can_access_badges(w) ((w)->acl & WEB_CLIENT_ACL_BADGE)
+#define web_client_can_access_mgmt(w) ((w)->acl & WEB_CLIENT_ACL_MGMT)
+#define web_client_can_access_stream(w) ((w)->acl & WEB_CLIENT_ACL_STREAMING)
+#define web_client_can_access_netdataconf(w) ((w)->acl & WEB_CLIENT_ACL_NETDATACONF)
+#define web_client_is_using_ssl_optional(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_OPTIONAL)
+#define web_client_is_using_ssl_force(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_FORCE)
+#define web_client_is_using_ssl_default(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_DEFAULT)
+
+typedef struct listen_sockets {
+ struct config *config; // the config file to use
+ const char *config_section; // the netdata configuration section to read settings from
+ const char *default_bind_to; // the default bind to configuration string
+ uint16_t default_port; // the default port to use
+ int backlog; // the default listen backlog to use
+
+ size_t opened; // the number of sockets opened
+ size_t failed; // the number of sockets attempted to open, but failed
+ int fds[MAX_LISTEN_FDS]; // the open sockets
+ char *fds_names[MAX_LISTEN_FDS]; // descriptions for the open sockets
+ int fds_types[MAX_LISTEN_FDS]; // the socktype for the open sockets (SOCK_STREAM, SOCK_DGRAM)
+ int fds_families[MAX_LISTEN_FDS]; // the family of the open sockets (AF_UNIX, AF_INET, AF_INET6)
+ WEB_CLIENT_ACL fds_acl_flags[MAX_LISTEN_FDS]; // the acl to apply to the open sockets (dashboard, badges, streaming, netdata.conf, management)
+} LISTEN_SOCKETS;
+
+extern char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port);
+
+extern int listen_sockets_setup(LISTEN_SOCKETS *sockets);
+extern void listen_sockets_close(LISTEN_SOCKETS *sockets);
+
+extern int connect_to_this(const char *definition, int default_port, struct timeval *timeout);
+extern int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size);
+int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout);
+
+#ifdef ENABLE_HTTPS
+extern ssize_t recv_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout);
+extern ssize_t send_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout);
+#else
+extern ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
+extern ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
+#endif
+
+extern int sock_setnonblock(int fd);
+extern int sock_delnonblock(int fd);
+extern int sock_setreuse(int fd, int reuse);
+extern int sock_setreuse_port(int fd, int reuse);
+extern int sock_enlarge_in(int fd);
+extern int sock_enlarge_out(int fd);
+
+extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize,
+ SIMPLE_PATTERN *access_list, const char *patname, int allow_dns);
+extern int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize,
+ char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns);
+
+#ifndef HAVE_ACCEPT4
+extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags);
+
+#ifndef SOCK_NONBLOCK
+#define SOCK_NONBLOCK 00004000
+#endif /* #ifndef SOCK_NONBLOCK */
+
+#ifndef SOCK_CLOEXEC
+#define SOCK_CLOEXEC 02000000
+#endif /* #ifndef SOCK_CLOEXEC */
+
+#endif /* #ifndef HAVE_ACCEPT4 */
+
+
+// ----------------------------------------------------------------------------
+// poll() based listener
+
+#define POLLINFO_FLAG_SERVER_SOCKET 0x00000001
+#define POLLINFO_FLAG_CLIENT_SOCKET 0x00000002
+#define POLLINFO_FLAG_DONT_CLOSE 0x00000004
+
+typedef struct poll POLLJOB;
+
+typedef struct pollinfo {
+ POLLJOB *p; // the parent
+ size_t slot; // the slot id
+
+ int fd; // the file descriptor
+ int socktype; // the client socket type
+ WEB_CLIENT_ACL port_acl; // the access lists permitted on this web server port (it's -1 for client sockets)
+ char *client_ip; // Max INET6_ADDRSTRLEN bytes
+ char *client_port; // Max NI_MAXSERV bytes
+ char *client_host; // Max NI_MAXHOST bytes
+
+ time_t connected_t; // the time the socket connected
+ time_t last_received_t; // the time the socket last received data
+ time_t last_sent_t; // the time the socket last sent data
+
+ size_t recv_count; // the number of times the socket was ready for inbound traffic
+ size_t send_count; // the number of times the socket was ready for outbound traffic
+
+ uint32_t flags; // internal flags
+
+ // callbacks for this socket
+ void (*del_callback)(struct pollinfo *pi);
+ int (*rcv_callback)(struct pollinfo *pi, short int *events);
+ int (*snd_callback)(struct pollinfo *pi, short int *events);
+
+ // the user data
+ void *data;
+
+ // linking of free pollinfo structures
+ // for quickly finding the next available
+ // this is like a stack, it grows and shrinks
+ // (with gaps - lower empty slots are preferred)
+ struct pollinfo *next;
+} POLLINFO;
+
+struct poll {
+ size_t slots;
+ size_t used;
+ size_t min;
+ size_t max;
+
+ size_t limit;
+
+ time_t complete_request_timeout;
+ time_t idle_timeout;
+ time_t checks_every;
+
+ time_t timer_milliseconds;
+ void *timer_data;
+
+ struct pollfd *fds;
+ struct pollinfo *inf;
+ struct pollinfo *first_free;
+
+ SIMPLE_PATTERN *access_list;
+ int allow_dns;
+
+ void *(*add_callback)(POLLINFO *pi, short int *events, void *data);
+ void (*del_callback)(POLLINFO *pi);
+ int (*rcv_callback)(POLLINFO *pi, short int *events);
+ int (*snd_callback)(POLLINFO *pi, short int *events);
+ void (*tmr_callback)(void *timer_data);
+};
+
+#define pollinfo_from_slot(p, slot) (&((p)->inf[(slot)]))
+
+extern int poll_default_snd_callback(POLLINFO *pi, short int *events);
+extern int poll_default_rcv_callback(POLLINFO *pi, short int *events);
+extern void poll_default_del_callback(POLLINFO *pi);
+extern void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data);
+
+extern POLLINFO *poll_add_fd(POLLJOB *p
+ , int fd
+ , int socktype
+ , WEB_CLIENT_ACL port_acl
+ , uint32_t flags
+ , const char *client_ip
+ , const char *client_port
+ , const char *client_host
+ , void *(*add_callback)(POLLINFO *pi, short int *events, void *data)
+ , void (*del_callback)(POLLINFO *pi)
+ , int (*rcv_callback)(POLLINFO *pi, short int *events)
+ , int (*snd_callback)(POLLINFO *pi, short int *events)
+ , void *data
+);
+extern void poll_close_fd(POLLINFO *pi);
+
+extern void poll_events(LISTEN_SOCKETS *sockets
+ , void *(*add_callback)(POLLINFO *pi, short int *events, void *data)
+ , void (*del_callback)(POLLINFO *pi)
+ , int (*rcv_callback)(POLLINFO *pi, short int *events)
+ , int (*snd_callback)(POLLINFO *pi, short int *events)
+ , void (*tmr_callback)(void *timer_data)
+ , SIMPLE_PATTERN *access_list
+ , int allow_dns
+ , void *data
+ , time_t tcp_request_timeout_seconds
+ , time_t tcp_idle_timeout_seconds
+ , time_t timer_milliseconds
+ , void *timer_data
+ , size_t max_tcp_sockets
+);
+
+#endif //NETDATA_SOCKET_H
diff --git a/libnetdata/statistical/Makefile.am b/libnetdata/statistical/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/statistical/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/statistical/README.md b/libnetdata/statistical/README.md
new file mode 100644
index 0000000..ee1faba
--- /dev/null
+++ b/libnetdata/statistical/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/statistical/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fstatistical%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/statistical/statistical.c b/libnetdata/statistical/statistical.c
new file mode 100644
index 0000000..ba8f0c4
--- /dev/null
+++ b/libnetdata/statistical/statistical.c
@@ -0,0 +1,452 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+LONG_DOUBLE default_single_exponential_smoothing_alpha = 0.1;
+
+void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg) {
+ const LONG_DOUBLE *value, *end = &series[entries];
+
+ fprintf(stderr, "%s of %zu entries [ ", msg, entries);
+ for(value = series; value < end ;value++) {
+ if(value != series) fprintf(stderr, ", ");
+ fprintf(stderr, "%" LONG_DOUBLE_MODIFIER, *value);
+ }
+ fprintf(stderr, " ] results in " CALCULATED_NUMBER_FORMAT "\n", result);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count) {
+ const LONG_DOUBLE *value, *end = &series[entries];
+ LONG_DOUBLE sum = 0;
+ size_t c = 0;
+
+ for(value = series; value < end ; value++) {
+ if(calculated_number_isnumber(*value)) {
+ sum += *value;
+ c++;
+ }
+ }
+
+ if(unlikely(!c)) sum = NAN;
+ if(likely(count)) *count = c;
+
+ return sum;
+}
+
+inline LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries) {
+ return sum_and_count(series, entries, NULL);
+}
+
+inline LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries) {
+ size_t count = 0;
+ LONG_DOUBLE sum = sum_and_count(series, entries, &count);
+
+ if(unlikely(!count)) return NAN;
+ return sum / (LONG_DOUBLE)count;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period) {
+ if(unlikely(period <= 0))
+ return 0.0;
+
+ size_t i, count;
+ LONG_DOUBLE sum = 0, avg = 0;
+ LONG_DOUBLE p[period];
+
+ for(count = 0; count < period ; count++)
+ p[count] = 0.0;
+
+ for(i = 0, count = 0; i < entries; i++) {
+ LONG_DOUBLE value = series[i];
+ if(unlikely(!calculated_number_isnumber(value))) continue;
+
+ if(unlikely(count < period)) {
+ sum += value;
+ avg = (count == period - 1) ? sum / (LONG_DOUBLE)period : 0;
+ }
+ else {
+ sum = sum - p[count % period] + value;
+ avg = sum / (LONG_DOUBLE)period;
+ }
+
+ p[count % period] = value;
+ count++;
+ }
+
+ return avg;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+static int qsort_compare(const void *a, const void *b) {
+ LONG_DOUBLE *p1 = (LONG_DOUBLE *)a, *p2 = (LONG_DOUBLE *)b;
+ LONG_DOUBLE n1 = *p1, n2 = *p2;
+
+ if(unlikely(isnan(n1) || isnan(n2))) {
+ if(isnan(n1) && !isnan(n2)) return -1;
+ if(!isnan(n1) && isnan(n2)) return 1;
+ return 0;
+ }
+ if(unlikely(isinf(n1) || isinf(n2))) {
+ if(!isinf(n1) && isinf(n2)) return -1;
+ if(isinf(n1) && !isinf(n2)) return 1;
+ return 0;
+ }
+
+ if(unlikely(n1 < n2)) return -1;
+ if(unlikely(n1 > n2)) return 1;
+ return 0;
+}
+
+inline void sort_series(LONG_DOUBLE *series, size_t entries) {
+ qsort(series, entries, sizeof(LONG_DOUBLE), qsort_compare);
+}
+
+inline LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries) {
+ LONG_DOUBLE *copy = mallocz(sizeof(LONG_DOUBLE) * entries);
+ memcpy(copy, series, sizeof(LONG_DOUBLE) * entries);
+ return copy;
+}
+
+LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) {
+ if(unlikely(entries == 0)) return NAN;
+ if(unlikely(entries == 1)) return series[0];
+ if(unlikely(entries == 2)) return (series[0] + series[1]) / 2;
+
+ LONG_DOUBLE average;
+ if(entries % 2 == 0) {
+ size_t m = entries / 2;
+ average = (series[m] + series[m + 1]) / 2;
+ }
+ else {
+ average = series[entries / 2];
+ }
+
+ return average;
+}
+
+LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) {
+ if(unlikely(entries == 0)) return NAN;
+ if(unlikely(entries == 1)) return series[0];
+
+ if(unlikely(entries == 2))
+ return (series[0] + series[1]) / 2;
+
+ LONG_DOUBLE *copy = copy_series(series, entries);
+ sort_series(copy, entries);
+
+ LONG_DOUBLE avg = median_on_sorted_series(copy, entries);
+
+ freez(copy);
+ return avg;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period) {
+ if(entries <= period)
+ return median(series, entries);
+
+ LONG_DOUBLE *data = copy_series(series, entries);
+
+ size_t i;
+ for(i = period; i < entries; i++) {
+ data[i - period] = median(&series[i - period], period);
+ }
+
+ LONG_DOUBLE avg = median(data, entries - period);
+ freez(data);
+ return avg;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+// http://stackoverflow.com/a/15150143/4525767
+LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) {
+ LONG_DOUBLE median = 0.0f;
+ LONG_DOUBLE average = 0.0f;
+ size_t i;
+
+ for(i = 0; i < entries ; i++) {
+ LONG_DOUBLE value = series[i];
+ if(unlikely(!calculated_number_isnumber(value))) continue;
+
+ average += ( value - average ) * 0.1f; // rough running average.
+ median += copysignl( average * 0.01, value - median );
+ }
+
+ return median;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) {
+ if(unlikely(entries == 0)) return NAN;
+ if(unlikely(entries == 1)) return series[0];
+
+ const LONG_DOUBLE *value, *end = &series[entries];
+ size_t count;
+ LONG_DOUBLE sum;
+
+ for(count = 0, sum = 0, value = series ; value < end ;value++) {
+ if(likely(calculated_number_isnumber(*value))) {
+ count++;
+ sum += *value;
+ }
+ }
+
+ if(unlikely(count == 0)) return NAN;
+ if(unlikely(count == 1)) return sum;
+
+ LONG_DOUBLE average = sum / (LONG_DOUBLE)count;
+
+ for(count = 0, sum = 0, value = series ; value < end ;value++) {
+ if(calculated_number_isnumber(*value)) {
+ count++;
+ sum += powl(*value - average, 2);
+ }
+ }
+
+ if(unlikely(count == 0)) return NAN;
+ if(unlikely(count == 1)) return average;
+
+ LONG_DOUBLE variance = sum / (LONG_DOUBLE)(count); // remove -1 from count to have a population stddev
+ LONG_DOUBLE stddev = sqrtl(variance);
+ return stddev;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) {
+ if(unlikely(entries == 0))
+ return NAN;
+
+ if(unlikely(isnan(alpha)))
+ alpha = default_single_exponential_smoothing_alpha;
+
+ const LONG_DOUBLE *value = series, *end = &series[entries];
+ LONG_DOUBLE level = (1.0 - alpha) * (*value);
+
+ for(value++ ; value < end; value++) {
+ if(likely(calculated_number_isnumber(*value)))
+ level = alpha * (*value) + (1.0 - alpha) * level;
+ }
+
+ return level;
+}
+
+LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) {
+ if(unlikely(entries == 0))
+ return NAN;
+
+ if(unlikely(isnan(alpha)))
+ alpha = default_single_exponential_smoothing_alpha;
+
+ const LONG_DOUBLE *value = &series[entries -1];
+ LONG_DOUBLE level = (1.0 - alpha) * (*value);
+
+ for(value++ ; value >= series; value--) {
+ if(likely(calculated_number_isnumber(*value)))
+ level = alpha * (*value) + (1.0 - alpha) * level;
+ }
+
+ return level;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+// http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/
+LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast) {
+ if(unlikely(entries == 0))
+ return NAN;
+
+ LONG_DOUBLE level, trend;
+
+ if(unlikely(isnan(alpha)))
+ alpha = 0.3;
+
+ if(unlikely(isnan(beta)))
+ beta = 0.05;
+
+ level = series[0];
+
+ if(likely(entries > 1))
+ trend = series[1] - series[0];
+ else
+ trend = 0;
+
+ const LONG_DOUBLE *value = series;
+ for(value++ ; value >= series; value--) {
+ if(likely(calculated_number_isnumber(*value))) {
+
+ LONG_DOUBLE last_level = level;
+ level = alpha * *value + (1.0 - alpha) * (level + trend);
+ trend = beta * (level - last_level) + (1.0 - beta) * trend;
+
+ }
+ }
+
+ if(forecast)
+ *forecast = level + trend;
+
+ return level;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+/*
+ * Based on th R implementation
+ *
+ * a: level component
+ * b: trend component
+ * s: seasonal component
+ *
+ * Additive:
+ *
+ * Yhat[t+h] = a[t] + h * b[t] + s[t + 1 + (h - 1) mod p],
+ * a[t] = α (Y[t] - s[t-p]) + (1-α) (a[t-1] + b[t-1])
+ * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1]
+ * s[t] = γ (Y[t] - a[t]) + (1-γ) s[t-p]
+ *
+ * Multiplicative:
+ *
+ * Yhat[t+h] = (a[t] + h * b[t]) * s[t + 1 + (h - 1) mod p],
+ * a[t] = α (Y[t] / s[t-p]) + (1-α) (a[t-1] + b[t-1])
+ * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1]
+ * s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p]
+ */
+static int __HoltWinters(
+ const LONG_DOUBLE *series,
+ int entries, // start_time + h
+
+ LONG_DOUBLE alpha, // alpha parameter of Holt-Winters Filter.
+ LONG_DOUBLE beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing.
+ LONG_DOUBLE gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted.
+
+ const int *seasonal,
+ const int *period,
+ const LONG_DOUBLE *a, // Start value for level (a[0]).
+ const LONG_DOUBLE *b, // Start value for trend (b[0]).
+ LONG_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0])
+
+ /* return values */
+ LONG_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing
+ LONG_DOUBLE *level, // Estimated values for the level component (size entries - t + 2)
+ LONG_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2)
+ LONG_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2)
+)
+{
+ if(unlikely(entries < 4))
+ return 0;
+
+ int start_time = 2;
+
+ LONG_DOUBLE res = 0, xhat = 0, stmp = 0;
+ int i, i0, s0;
+
+ /* copy start values to the beginning of the vectors */
+ level[0] = *a;
+ if(beta > 0) trend[0] = *b;
+ if(gamma > 0) memcpy(season, s, *period * sizeof(LONG_DOUBLE));
+
+ for(i = start_time - 1; i < entries; i++) {
+ /* indices for period i */
+ i0 = i - start_time + 2;
+ s0 = i0 + *period - 1;
+
+ /* forecast *for* period i */
+ xhat = level[i0 - 1] + (beta > 0 ? trend[i0 - 1] : 0);
+ stmp = gamma > 0 ? season[s0 - *period] : (*seasonal != 1);
+ if (*seasonal == 1)
+ xhat += stmp;
+ else
+ xhat *= stmp;
+
+ /* Sum of Squared Errors */
+ res = series[i] - xhat;
+ *SSE += res * res;
+
+ /* estimate of level *in* period i */
+ if (*seasonal == 1)
+ level[i0] = alpha * (series[i] - stmp)
+ + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]);
+ else
+ level[i0] = alpha * (series[i] / stmp)
+ + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]);
+
+ /* estimate of trend *in* period i */
+ if (beta > 0)
+ trend[i0] = beta * (level[i0] - level[i0 - 1])
+ + (1 - beta) * trend[i0 - 1];
+
+ /* estimate of seasonal component *in* period i */
+ if (gamma > 0) {
+ if (*seasonal == 1)
+ season[s0] = gamma * (series[i] - level[i0])
+ + (1 - gamma) * stmp;
+ else
+ season[s0] = gamma * (series[i] / level[i0])
+ + (1 - gamma) * stmp;
+ }
+ }
+
+ return 1;
+}
+
+LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast) {
+ if(unlikely(isnan(alpha)))
+ alpha = 0.3;
+
+ if(unlikely(isnan(beta)))
+ beta = 0.05;
+
+ if(unlikely(isnan(gamma)))
+ gamma = 0;
+
+ int seasonal = 0;
+ int period = 0;
+ LONG_DOUBLE a0 = series[0];
+ LONG_DOUBLE b0 = 0;
+ LONG_DOUBLE s[] = {};
+
+ LONG_DOUBLE errors = 0.0;
+ size_t nb_computations = entries;
+ LONG_DOUBLE *estimated_level = callocz(nb_computations, sizeof(LONG_DOUBLE));
+ LONG_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(LONG_DOUBLE));
+ LONG_DOUBLE *estimated_season = callocz(nb_computations, sizeof(LONG_DOUBLE));
+
+ int ret = __HoltWinters(
+ series,
+ (int)entries,
+ alpha,
+ beta,
+ gamma,
+ &seasonal,
+ &period,
+ &a0,
+ &b0,
+ s,
+ &errors,
+ estimated_level,
+ estimated_trend,
+ estimated_season
+ );
+
+ LONG_DOUBLE value = estimated_level[nb_computations - 1];
+
+ if(forecast)
+ *forecast = 0.0;
+
+ freez(estimated_level);
+ freez(estimated_trend);
+ freez(estimated_season);
+
+ if(!ret)
+ return 0.0;
+
+ return value;
+}
diff --git a/libnetdata/statistical/statistical.h b/libnetdata/statistical/statistical.h
new file mode 100644
index 0000000..bbf241f
--- /dev/null
+++ b/libnetdata/statistical/statistical.h
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_STATISTICAL_H
+#define NETDATA_STATISTICAL_H 1
+
+#include "../libnetdata.h"
+
+extern void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg);
+
+extern LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries);
+extern LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period);
+extern LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries);
+extern LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period);
+extern LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries);
+extern LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries);
+extern LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha);
+extern LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha);
+extern LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast);
+extern LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast);
+extern LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count);
+extern LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries);
+extern LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries);
+extern LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries);
+extern void sort_series(LONG_DOUBLE *series, size_t entries);
+
+#endif //NETDATA_STATISTICAL_H
diff --git a/libnetdata/storage_number/Makefile.am b/libnetdata/storage_number/Makefile.am
new file mode 100644
index 0000000..c5f8450
--- /dev/null
+++ b/libnetdata/storage_number/Makefile.am
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+SUBDIRS = \
+ tests \
+ $(NULL)
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/storage_number/README.md b/libnetdata/storage_number/README.md
new file mode 100644
index 0000000..295b3d6
--- /dev/null
+++ b/libnetdata/storage_number/README.md
@@ -0,0 +1,17 @@
+<!--
+title: "Netdata storage number"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/storage_number/README.md
+-->
+
+# Netdata storage number
+
+Although `netdata` does all its calculations using `long double`, it stores all values using
+a **custom-made 32-bit number**.
+
+This custom-made number can store in 29 bits values from `-167772150000000.0` to `167772150000000.0`
+with a precision of 0.00001 (yes, it's a floating point number, meaning that higher integer values
+have less decimal precision) and 3 bits for flags.
+
+This provides an extremely optimized memory footprint with just 0.0001% max accuracy loss.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fstorage_number%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c
new file mode 100644
index 0000000..8ef1353
--- /dev/null
+++ b/libnetdata/storage_number/storage_number.c
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+storage_number pack_storage_number(calculated_number value, uint32_t flags) {
+ // bit 32 = sign 0:positive, 1:negative
+ // bit 31 = 0:divide, 1:multiply
+ // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
+ // bit 27 SN_EXISTS_100
+ // bit 26 SN_EXISTS_RESET
+ // bit 25 SN_EXISTS
+ // bit 24 to bit 1 = the value
+
+ storage_number r = get_storage_number_flags(flags);
+ if(!value) return r;
+
+ int m = 0;
+ calculated_number n = value, factor = 10;
+
+ // if the value is negative
+ // add the sign bit and make it positive
+ if(n < 0) {
+ r += (1 << 31); // the sign bit 32
+ n = -n;
+ }
+
+ if(n / 10000000.0 > 0x00ffffff) {
+ factor = 100;
+ r |= SN_EXISTS_100;
+ }
+
+ // make its integer part fit in 0x00ffffff
+ // by dividing it by 10 up to 7 times
+ // and increasing the multiplier
+ while(m < 7 && n > (calculated_number)0x00ffffff) {
+ n /= factor;
+ m++;
+ }
+
+ if(m) {
+ // the value was too big and we divided it
+ // so we add a multiplier to unpack it
+ r += (1 << 30) + (m << 27); // the multiplier m
+
+ if(n > (calculated_number)0x00ffffff) {
+ #ifdef NETDATA_INTERNAL_CHECKS
+ error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value);
+ #endif
+ r += 0x00ffffff;
+ return r;
+ }
+ }
+ else {
+ // 0x0019999e is the number that can be multiplied
+ // by 10 to give 0x00ffffff
+ // while the value is below 0x0019999e we can
+ // multiply it by 10, up to 7 times, increasing
+ // the multiplier
+ while(m < 7 && n < (calculated_number)0x0019999e) {
+ n *= 10;
+ m++;
+ }
+
+ if (unlikely(n > (calculated_number) (0x00ffffff))) {
+ n /= 10;
+ m--;
+ }
+ // the value was small enough and we multiplied it
+ // so we add a divider to unpack it
+ r += (0 << 30) + (m << 27); // the divider m
+ }
+
+#ifdef STORAGE_WITH_MATH
+ // without this there are rounding problems
+ // example: 0.9 becomes 0.89
+ r += lrint((double) n);
+#else
+ r += (storage_number)n;
+#endif
+
+ return r;
+}
+
+calculated_number unpack_storage_number(storage_number value) {
+ if(!value) return 0;
+
+ int sign = 0, exp = 0;
+ int factor = 10;
+
+ // bit 32 = 0:positive, 1:negative
+ if(unlikely(value & (1 << 31)))
+ sign = 1;
+
+ // bit 31 = 0:divide, 1:multiply
+ if(unlikely(value & (1 << 30)))
+ exp = 1;
+
+ // bit 27 SN_EXISTS_100
+ if(unlikely(value & (1 << 26)))
+ factor = 100;
+
+ // bit 26 SN_EXISTS_RESET
+ // bit 25 SN_EXISTS
+
+ // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
+ int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27;
+
+ // bit 24 to bit 1 = the value, so remove all other bits
+ value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));
+
+ calculated_number n = value;
+
+ // fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, factor = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, factor, n);
+
+ if(exp) {
+ for(; mul; mul--)
+ n *= factor;
+ }
+ else {
+ for( ; mul ; mul--)
+ n /= 10;
+ }
+
+ if(sign) n = -n;
+ return n;
+}
+
+/*
+int print_calculated_number(char *str, calculated_number value)
+{
+ char *wstr = str;
+
+ int sign = (value < 0) ? 1 : 0;
+ if(sign) value = -value;
+
+#ifdef STORAGE_WITH_MATH
+ // without llrintl() there are rounding problems
+ // for example 0.9 becomes 0.89
+ unsigned long long uvalue = (unsigned long long int) llrintl(value * (calculated_number)100000);
+#else
+ unsigned long long uvalue = value * (calculated_number)100000;
+#endif
+
+ wstr = print_number_llu_r_smart(str, uvalue);
+
+ // make sure we have 6 bytes at least
+ while((wstr - str) < 6) *wstr++ = '0';
+
+ // put the sign back
+ if(sign) *wstr++ = '-';
+
+ // reverse it
+ char *begin = str, *end = --wstr, aux;
+ while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux;
+ // wstr--;
+ // strreverse(str, wstr);
+
+ // remove trailing zeros
+ int decimal = 5;
+ while(decimal > 0 && *wstr == '0') {
+ *wstr-- = '\0';
+ decimal--;
+ }
+
+ // terminate it, one position to the right
+ // to let space for a dot
+ wstr[2] = '\0';
+
+ // make space for the dot
+ int i;
+ for(i = 0; i < decimal ;i++) {
+ wstr[1] = wstr[0];
+ wstr--;
+ }
+
+ // put the dot
+ if(wstr[2] == '\0') { wstr[1] = '\0'; decimal--; }
+ else wstr[1] = '.';
+
+ // return the buffer length
+ return (int) ((wstr - str) + 2 + decimal );
+}
+*/
+
+int print_calculated_number(char *str, calculated_number value) {
+ // info("printing number " CALCULATED_NUMBER_FORMAT, value);
+ char integral_str[50], fractional_str[50];
+
+ char *wstr = str;
+
+ if(unlikely(value < 0)) {
+ *wstr++ = '-';
+ value = -value;
+ }
+
+ calculated_number integral, fractional;
+
+#ifdef STORAGE_WITH_MATH
+ fractional = calculated_number_modf(value, &integral) * 10000000.0;
+#else
+ fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL);
+#endif
+
+ unsigned long long integral_int = (unsigned long long)integral;
+ unsigned long long fractional_int = (unsigned long long)calculated_number_llrint(fractional);
+ if(unlikely(fractional_int >= 10000000)) {
+ integral_int += 1;
+ fractional_int -= 10000000;
+ }
+
+ // info("integral " CALCULATED_NUMBER_FORMAT " (%llu), fractional " CALCULATED_NUMBER_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int);
+
+ char *istre;
+ if(unlikely(integral_int == 0)) {
+ integral_str[0] = '0';
+ istre = &integral_str[1];
+ }
+ else
+ // convert the integral part to string (reversed)
+ istre = print_number_llu_r_smart(integral_str, integral_int);
+
+ // copy reversed the integral string
+ istre--;
+ while( istre >= integral_str ) *wstr++ = *istre--;
+
+ if(likely(fractional_int != 0)) {
+ // add a dot
+ *wstr++ = '.';
+
+ // convert the fractional part to string (reversed)
+ char *fstre = print_number_llu_r_smart(fractional_str, fractional_int);
+
+ // prepend zeros to reach 7 digits length
+ int decimal = 7;
+ int len = (int)(fstre - fractional_str);
+ while(len < decimal) {
+ *wstr++ = '0';
+ len++;
+ }
+
+ char *begin = fractional_str;
+ while(begin < fstre && *begin == '0') begin++;
+
+ // copy reversed the fractional string
+ fstre--;
+ while( fstre >= begin ) *wstr++ = *fstre--;
+ }
+
+ *wstr = '\0';
+ // info("printed number '%s'", str);
+ return (int)(wstr - str);
+}
diff --git a/libnetdata/storage_number/storage_number.h b/libnetdata/storage_number/storage_number.h
new file mode 100644
index 0000000..28b7f26
--- /dev/null
+++ b/libnetdata/storage_number/storage_number.h
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_STORAGE_NUMBER_H
+#define NETDATA_STORAGE_NUMBER_H 1
+
+#include "../libnetdata.h"
+
+#ifdef NETDATA_WITHOUT_LONG_DOUBLE
+
+#define powl pow
+#define modfl modf
+#define llrintl llrint
+#define roundl round
+#define sqrtl sqrt
+#define copysignl copysign
+#define strtold strtod
+
+typedef double calculated_number;
+#define CALCULATED_NUMBER_FORMAT "%0.7f"
+#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0f"
+#define CALCULATED_NUMBER_FORMAT_AUTO "%f"
+
+#define LONG_DOUBLE_MODIFIER "f"
+typedef double LONG_DOUBLE;
+
+#else // NETDATA_WITHOUT_LONG_DOUBLE
+
+typedef long double calculated_number;
+#define CALCULATED_NUMBER_FORMAT "%0.7Lf"
+#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0Lf"
+#define CALCULATED_NUMBER_FORMAT_AUTO "%Lf"
+
+#define LONG_DOUBLE_MODIFIER "Lf"
+typedef long double LONG_DOUBLE;
+
+#endif // NETDATA_WITHOUT_LONG_DOUBLE
+
+//typedef long long calculated_number;
+//#define CALCULATED_NUMBER_FORMAT "%lld"
+
+typedef long long collected_number;
+#define COLLECTED_NUMBER_FORMAT "%lld"
+
+/*
+typedef long double collected_number;
+#define COLLECTED_NUMBER_FORMAT "%0.7Lf"
+*/
+
+#define calculated_number_modf(x, y) modfl(x, y)
+#define calculated_number_llrint(x) llrintl(x)
+#define calculated_number_round(x) roundl(x)
+#define calculated_number_fabs(x) fabsl(x)
+#define calculated_number_pow(x, y) powl(x, y)
+#define calculated_number_epsilon (calculated_number)0.0000001
+
+#define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon)
+
+#define calculated_number_isnumber(a) (!(fpclassify(a) & (FP_NAN|FP_INFINITE)))
+
+typedef uint32_t storage_number;
+#define STORAGE_NUMBER_FORMAT "%u"
+
+#define SN_EXISTS (1 << 24) // the value exists
+#define SN_EXISTS_RESET (1 << 25) // the value has been overflown
+#define SN_EXISTS_100 (1 << 26) // very large value (multipler is 100 instead of 10)
+
+// extract the flags
+#define get_storage_number_flags(value) ((((storage_number)(value)) & (1 << 24)) | (((storage_number)(value)) & (1 << 25)) | (((storage_number)(value)) & (1 << 26)))
+#define SN_EMPTY_SLOT 0x00000000
+
+// checks
+#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0)
+#define did_storage_number_reset(value) ((get_storage_number_flags(value) == SN_EXISTS_RESET)?1:0)
+
+storage_number pack_storage_number(calculated_number value, uint32_t flags);
+calculated_number unpack_storage_number(storage_number value);
+
+int print_calculated_number(char *str, calculated_number value);
+
+// sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE
+#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff )
+#define STORAGE_NUMBER_POSITIVE_MIN_RAW (storage_number)( (0 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 )
+#define STORAGE_NUMBER_NEGATIVE_MAX_RAW (storage_number)( (1 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 )
+#define STORAGE_NUMBER_NEGATIVE_MIN_RAW (storage_number)( (1 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff )
+
+// accepted accuracy loss
+#define ACCURACY_LOSS_ACCEPTED_PERCENT 0.0001
+#define accuracy_loss(t1, t2) (((t1) == (t2) || (t1) == 0.0 || (t2) == 0.0) ? 0.0 : (100.0 - (((t1) > (t2)) ? ((t2) * 100.0 / (t1) ) : ((t1) * 100.0 / (t2)))))
+
+// Maximum acceptable rate of increase for counters. With a rate of 10% netdata can safely detect overflows with a
+// period of at least every other 10 samples.
+#define MAX_INCREMENTAL_PERCENT_RATE 10
+
+#endif /* NETDATA_STORAGE_NUMBER_H */
diff --git a/libnetdata/storage_number/tests/Makefile.am b/libnetdata/storage_number/tests/Makefile.am
new file mode 100644
index 0000000..babdcf0
--- /dev/null
+++ b/libnetdata/storage_number/tests/Makefile.am
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
diff --git a/libnetdata/storage_number/tests/test_storage_number.c b/libnetdata/storage_number/tests/test_storage_number.c
new file mode 100644
index 0000000..0e13208
--- /dev/null
+++ b/libnetdata/storage_number/tests/test_storage_number.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../../libnetdata.h"
+#include "../../required_dummies.h"
+#include <setjmp.h>
+#include <cmocka.h>
+
+static void test_number_pinting(void **state)
+{
+ (void)state;
+
+ char value[50];
+
+ print_calculated_number(value, 0);
+ assert_string_equal(value, "0");
+
+ print_calculated_number(value, 0.0000001);
+ assert_string_equal(value, "0.0000001");
+
+ print_calculated_number(value, 0.00000009);
+ assert_string_equal(value, "0.0000001");
+
+ print_calculated_number(value, 0.000000001);
+ assert_string_equal(value, "0");
+
+ print_calculated_number(value, 99.99999999999999999);
+ assert_string_equal(value, "100");
+
+ print_calculated_number(value, -99.99999999999999999);
+ assert_string_equal(value, "-100");
+
+ print_calculated_number(value, 123.4567890123456789);
+ assert_string_equal(value, "123.456789");
+
+ print_calculated_number(value, 9999.9999999);
+ assert_string_equal(value, "9999.9999999");
+
+ print_calculated_number(value, -9999.9999999);
+ assert_string_equal(value, "-9999.9999999");
+
+ print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_EXISTS)));
+ assert_string_equal(value, "16.77722");
+}
+
+int main(void)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test(test_number_pinting)
+ };
+
+ return cmocka_run_group_tests_name("storage_number", tests, NULL, NULL);
+}
diff --git a/libnetdata/string/utf8.h b/libnetdata/string/utf8.h
new file mode 100644
index 0000000..133ec71
--- /dev/null
+++ b/libnetdata/string/utf8.h
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_STRING_UTF8_H
+#define NETDATA_STRING_UTF8_H 1
+
+#define IS_UTF8_BYTE(x) (x & 0x80)
+#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x)&&(x & 0x40))
+
+#endif /* NETDATA_STRING_UTF8_H */
diff --git a/libnetdata/tests/Makefile.am b/libnetdata/tests/Makefile.am
new file mode 100644
index 0000000..babdcf0
--- /dev/null
+++ b/libnetdata/tests/Makefile.am
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
diff --git a/libnetdata/tests/test_str2ld.c b/libnetdata/tests/test_str2ld.c
new file mode 100644
index 0000000..9d59f6c
--- /dev/null
+++ b/libnetdata/tests/test_str2ld.c
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+#include "../required_dummies.h"
+#include <setjmp.h>
+#include <cmocka.h>
+
+static void test_str2ld(void **state)
+{
+ (void)state;
+ char *values[] = {
+ "1.2345678",
+ "-35.6",
+ "0.00123",
+ "23842384234234.2",
+ ".1",
+ "1.2e-10",
+ "hello",
+ "1wrong",
+ "nan",
+ "inf",
+ NULL
+ };
+
+ for (int i = 0; values[i]; i++) {
+ char *e_mine = "hello", *e_sys = "world";
+ LONG_DOUBLE mine = str2ld(values[i], &e_mine);
+ LONG_DOUBLE sys = strtold(values[i], &e_sys);
+
+ if (isnan(mine))
+ assert_true(isnan(sys));
+ else if (isinf(mine))
+ assert_true(isinf(sys));
+ else if (mine != sys)
+ assert_false(abs(mine - sys) > 0.000001);
+
+ assert_ptr_equal(e_mine, e_sys);
+ }
+}
+
+int main(void)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test(test_str2ld)
+ };
+
+ return cmocka_run_group_tests_name("str2ld", tests, NULL, NULL);
+}
diff --git a/libnetdata/threads/Makefile.am b/libnetdata/threads/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/threads/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/threads/README.md b/libnetdata/threads/README.md
new file mode 100644
index 0000000..377da5e
--- /dev/null
+++ b/libnetdata/threads/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/threads/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Fthreads%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/threads/threads.c b/libnetdata/threads/threads.c
new file mode 100644
index 0000000..d763694
--- /dev/null
+++ b/libnetdata/threads/threads.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+static size_t default_stacksize = 0, wanted_stacksize = 0;
+static pthread_attr_t *attr = NULL;
+
+// ----------------------------------------------------------------------------
+// per thread data
+
+typedef struct {
+ void *arg;
+ pthread_t *thread;
+ const char *tag;
+ void *(*start_routine) (void *);
+ NETDATA_THREAD_OPTIONS options;
+} NETDATA_THREAD;
+
+static __thread NETDATA_THREAD *netdata_thread = NULL;
+
+inline int netdata_thread_tag_exists(void) {
+ return (netdata_thread && netdata_thread->tag && *netdata_thread->tag);
+}
+
+const char *netdata_thread_tag(void) {
+ return (netdata_thread_tag_exists() ? netdata_thread->tag : "MAIN");
+}
+
+// ----------------------------------------------------------------------------
+// compatibility library functions
+
+pid_t gettid(void) {
+#ifdef __FreeBSD__
+
+ return (pid_t)pthread_getthreadid_np();
+
+#elif defined(__APPLE__)
+
+ #if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060)
+ uint64_t curthreadid;
+ pthread_threadid_np(NULL, &curthreadid);
+ return (pid_t)curthreadid;
+ #else /* __MAC_OS_X_VERSION_MIN_REQUIRED */
+ return (pid_t)pthread_self;
+ #endif /* __MAC_OS_X_VERSION_MIN_REQUIRED */
+
+#else /* __APPLE__*/
+
+ return (pid_t)syscall(SYS_gettid);
+
+#endif /* __FreeBSD__, __APPLE__*/
+}
+
+// ----------------------------------------------------------------------------
+// early initialization
+
+size_t netdata_threads_init(void) {
+ int i;
+
+ // --------------------------------------------------------------------
+ // get the required stack size of the threads of netdata
+
+ attr = callocz(1, sizeof(pthread_attr_t));
+ i = pthread_attr_init(attr);
+ if(i != 0)
+ fatal("pthread_attr_init() failed with code %d.", i);
+
+ i = pthread_attr_getstacksize(attr, &default_stacksize);
+ if(i != 0)
+ fatal("pthread_attr_getstacksize() failed with code %d.", i);
+ else
+ debug(D_OPTIONS, "initial pthread stack size is %zu bytes", default_stacksize);
+
+ return default_stacksize;
+}
+
+// ----------------------------------------------------------------------------
+// late initialization
+
+void netdata_threads_init_after_fork(size_t stacksize) {
+ wanted_stacksize = stacksize;
+ int i;
+
+ // ------------------------------------------------------------------------
+ // set default pthread stack size
+
+ if(attr && default_stacksize < wanted_stacksize && wanted_stacksize > 0) {
+ i = pthread_attr_setstacksize(attr, wanted_stacksize);
+ if(i != 0)
+ fatal("pthread_attr_setstacksize() to %zu bytes, failed with code %d.", wanted_stacksize, i);
+ else
+ debug(D_SYSTEM, "Successfully set pthread stacksize to %zu bytes", wanted_stacksize);
+ }
+}
+
+
+// ----------------------------------------------------------------------------
+// netdata_thread_create
+
+static void thread_cleanup(void *ptr) {
+ if(netdata_thread != ptr) {
+ NETDATA_THREAD *info = (NETDATA_THREAD *)ptr;
+ error("THREADS: internal error - thread local variable does not match the one passed to this function. Expected thread '%s', passed thread '%s'", netdata_thread->tag, info->tag);
+ }
+
+ if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP))
+ info("thread with task id %d finished", gettid());
+
+ freez((void *)netdata_thread->tag);
+ netdata_thread->tag = NULL;
+
+ freez(netdata_thread);
+ netdata_thread = NULL;
+}
+
+static void thread_set_name_np(NETDATA_THREAD *nt) {
+
+ if (nt->tag) {
+ int ret = 0;
+
+ char threadname[NETDATA_THREAD_NAME_MAX+1];
+ strncpyz(threadname, nt->tag, NETDATA_THREAD_NAME_MAX);
+
+#if defined(__FreeBSD__)
+ pthread_set_name_np(pthread_self(), threadname);
+#elif defined(__APPLE__)
+ ret = pthread_setname_np(threadname);
+#else
+ ret = pthread_setname_np(pthread_self(), threadname);
+#endif
+
+ if (ret != 0)
+ error("cannot set pthread name of %d to %s. ErrCode: %d", gettid(), threadname, ret);
+ else
+ info("set name of thread %d to %s", gettid(), threadname);
+
+ }
+}
+
+void uv_thread_set_name_np(uv_thread_t ut, const char* name) {
+ int ret = 0;
+
+ char threadname[NETDATA_THREAD_NAME_MAX+1];
+ strncpyz(threadname, name, NETDATA_THREAD_NAME_MAX);
+
+#if defined(__FreeBSD__)
+ pthread_set_name_np(ut, threadname);
+#elif defined(__APPLE__)
+ // Apple can only set its own name
+#else
+ ret = pthread_setname_np(ut, threadname);
+#endif
+
+ if (ret)
+ info("cannot set libuv thread name to %s. Err: %d", threadname, ret);
+}
+
+void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1])
+{
+ threadname[0] = '\0';
+#if defined(__FreeBSD__)
+ pthread_get_name_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
+#elif defined(HAVE_PTHREAD_GETNAME_NP) /* Linux & macOS */
+ (void)pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
+#endif
+}
+
+static void *thread_start(void *ptr) {
+ netdata_thread = (NETDATA_THREAD *)ptr;
+
+ if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_STARTUP))
+ info("thread created with task id %d", gettid());
+
+ if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
+ error("cannot set pthread cancel type to DEFERRED.");
+
+ if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
+ error("cannot set pthread cancel state to ENABLE.");
+
+ thread_set_name_np(ptr);
+
+ void *ret = NULL;
+ pthread_cleanup_push(thread_cleanup, ptr);
+ ret = netdata_thread->start_routine(netdata_thread->arg);
+ pthread_cleanup_pop(1);
+
+ return ret;
+}
+
+int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg) {
+ NETDATA_THREAD *info = mallocz(sizeof(NETDATA_THREAD));
+ info->arg = arg;
+ info->thread = thread;
+ info->tag = strdupz(tag);
+ info->start_routine = start_routine;
+ info->options = options;
+
+ int ret = pthread_create(thread, attr, thread_start, info);
+ if(ret != 0)
+ error("failed to create new thread for %s. pthread_create() failed with code %d", tag, ret);
+
+ else {
+ if (!(options & NETDATA_THREAD_OPTION_JOINABLE)) {
+ int ret2 = pthread_detach(*thread);
+ if (ret2 != 0)
+ error("cannot request detach of newly created %s thread. pthread_detach() failed with code %d", tag, ret2);
+ }
+ }
+
+ return ret;
+}
+
+// ----------------------------------------------------------------------------
+// netdata_thread_cancel
+
+int netdata_thread_cancel(netdata_thread_t thread) {
+ int ret = pthread_cancel(thread);
+ if(ret != 0)
+ error("cannot cancel thread. pthread_cancel() failed with code %d.", ret);
+
+ return ret;
+}
+
+// ----------------------------------------------------------------------------
+// netdata_thread_join
+
+int netdata_thread_join(netdata_thread_t thread, void **retval) {
+ int ret = pthread_join(thread, retval);
+ if(ret != 0)
+ error("cannot join thread. pthread_join() failed with code %d.", ret);
+
+ return ret;
+}
+
+int netdata_thread_detach(pthread_t thread) {
+ int ret = pthread_detach(thread);
+ if(ret != 0)
+ error("cannot detach thread. pthread_detach() failed with code %d.", ret);
+
+ return ret;
+}
diff --git a/libnetdata/threads/threads.h b/libnetdata/threads/threads.h
new file mode 100644
index 0000000..e7d79d3
--- /dev/null
+++ b/libnetdata/threads/threads.h
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_THREADS_H
+#define NETDATA_THREADS_H 1
+
+#include "../libnetdata.h"
+
+extern pid_t gettid(void);
+
+typedef enum {
+ NETDATA_THREAD_OPTION_DEFAULT = 0 << 0,
+ NETDATA_THREAD_OPTION_JOINABLE = 1 << 0,
+ NETDATA_THREAD_OPTION_DONT_LOG_STARTUP = 1 << 1,
+ NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP = 1 << 2,
+ NETDATA_THREAD_OPTION_DONT_LOG = NETDATA_THREAD_OPTION_DONT_LOG_STARTUP|NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP,
+} NETDATA_THREAD_OPTIONS;
+
+#define netdata_thread_cleanup_push(func, arg) pthread_cleanup_push(func, arg)
+#define netdata_thread_cleanup_pop(execute) pthread_cleanup_pop(execute)
+
+typedef pthread_t netdata_thread_t;
+
+#define NETDATA_THREAD_TAG_MAX 100
+extern const char *netdata_thread_tag(void);
+extern int netdata_thread_tag_exists(void);
+
+extern size_t netdata_threads_init(void);
+extern void netdata_threads_init_after_fork(size_t stacksize);
+
+extern int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg);
+extern int netdata_thread_cancel(netdata_thread_t thread);
+extern int netdata_thread_join(netdata_thread_t thread, void **retval);
+extern int netdata_thread_detach(pthread_t thread);
+
+#define NETDATA_THREAD_NAME_MAX 15
+extern void uv_thread_set_name_np(uv_thread_t ut, const char* name);
+extern void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]);
+
+#define netdata_thread_self pthread_self
+#define netdata_thread_testcancel pthread_testcancel
+
+#endif //NETDATA_THREADS_H
diff --git a/libnetdata/url/Makefile.am b/libnetdata/url/Makefile.am
new file mode 100644
index 0000000..161784b
--- /dev/null
+++ b/libnetdata/url/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/url/README.md b/libnetdata/url/README.md
new file mode 100644
index 0000000..4a95547
--- /dev/null
+++ b/libnetdata/url/README.md
@@ -0,0 +1,5 @@
+<!--
+custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/url/README.md
+-->
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Flibnetdata%2Furl%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/libnetdata/url/url.c b/libnetdata/url/url.c
new file mode 100644
index 0000000..3de94fd
--- /dev/null
+++ b/libnetdata/url/url.c
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+
+// ----------------------------------------------------------------------------
+// URL encode / decode
+// code from: http://www.geekhideout.com/urlcode.shtml
+
+/* Converts a hex character to its integer value */
+char from_hex(char ch) {
+ return (char)(isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10);
+}
+
+/* Converts an integer value to its hex character*/
+char to_hex(char code) {
+ static char hex[] = "0123456789abcdef";
+ return hex[code & 15];
+}
+
+/* Returns a url-encoded version of str */
+/* IMPORTANT: be sure to free() the returned string after use */
+char *url_encode(char *str) {
+ char *buf, *pbuf;
+
+ pbuf = buf = mallocz(strlen(str) * 3 + 1);
+
+ while (*str) {
+ if (isalnum(*str) || *str == '-' || *str == '_' || *str == '.' || *str == '~')
+ *pbuf++ = *str;
+
+ else if (*str == ' ')
+ *pbuf++ = '+';
+
+ else{
+ *pbuf++ = '%';
+ *pbuf++ = to_hex(*str >> 4);
+ *pbuf++ = to_hex(*str & 15);
+ }
+
+ str++;
+ }
+ *pbuf = '\0';
+
+ pbuf = strdupz(buf);
+ freez(buf);
+ return pbuf;
+}
+
+/**
+ * Percentage escape decode
+ *
+ * Decode %XX character or return 0 if cannot
+ *
+ * @param s the string to decode
+ *
+ * @return The character decoded on success and 0 otherwise
+ */
+char url_percent_escape_decode(char *s) {
+ if(likely(s[1] && s[2]))
+ return from_hex(s[1]) << 4 | from_hex(s[2]);
+ return 0;
+}
+
+/**
+ * Get byte length
+ *
+ * This (utf8 string related) should be moved in separate file in future
+ *
+ * @param c is the utf8 character
+ * *
+ * @return It reurns the length of the specific character.
+ */
+char url_utf8_get_byte_length(char c) {
+ if(!IS_UTF8_BYTE(c))
+ return 1;
+
+ char length = 0;
+ while(likely(c & 0x80)) {
+ length++;
+ c <<= 1;
+ }
+ //4 byte is max size for UTF-8 char
+ //10XX XXXX is not valid character -> check length == 1
+ if(length > 4 || length == 1)
+ return -1;
+
+ return length;
+}
+
+/**
+ * Decode Multibyte UTF8
+ *
+ * Decode % encoded UTF-8 characters and copy them to *d
+ *
+ * @param s first address
+ * @param d
+ * @param d_end last address
+ *
+ * @return count of bytes written to *d
+ */
+char url_decode_multibyte_utf8(char *s, char *d, char *d_end) {
+ char first_byte = url_percent_escape_decode(s);
+
+ if(unlikely(!first_byte || !IS_UTF8_STARTBYTE(first_byte)))
+ return 0;
+
+ char byte_length = url_utf8_get_byte_length(first_byte);
+
+ if(unlikely(byte_length <= 0 || d+byte_length >= d_end))
+ return 0;
+
+ char to_read = byte_length;
+ while(to_read > 0) {
+ char c = url_percent_escape_decode(s);
+
+ if(unlikely( !IS_UTF8_BYTE(c) ))
+ return 0;
+ if((to_read != byte_length) && IS_UTF8_STARTBYTE(c))
+ return 0;
+
+ *d++ = c;
+ s+=3;
+ to_read--;
+ }
+
+ return byte_length;
+}
+
+/*
+ * The utf8_check() function scans the '\0'-terminated string starting
+ * at s. It returns a pointer to the first byte of the first malformed
+ * or overlong UTF-8 sequence found, or NULL if the string contains
+ * only correct UTF-8. It also spots UTF-8 sequences that could cause
+ * trouble if converted to UTF-16, namely surrogate characters
+ * (U+D800..U+DFFF) and non-Unicode positions (U+FFFE..U+FFFF). This
+ * routine is very likely to find a malformed sequence if the input
+ * uses any other encoding than UTF-8. It therefore can be used as a
+ * very effective heuristic for distinguishing between UTF-8 and other
+ * encodings.
+ *
+ * Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2005-03-30
+ * License: http://www.cl.cam.ac.uk/~mgk25/short-license.html
+ */
+unsigned char *utf8_check(unsigned char *s)
+{
+ while (*s)
+ {
+ if (*s < 0x80)
+ /* 0xxxxxxx */
+ s++;
+ else if ((s[0] & 0xe0) == 0xc0)
+ {
+ /* 110XXXXx 10xxxxxx */
+ if ((s[1] & 0xc0) != 0x80 ||
+ (s[0] & 0xfe) == 0xc0) /* overlong? */
+ return s;
+ else
+ s += 2;
+ }
+ else if ((s[0] & 0xf0) == 0xe0)
+ {
+ /* 1110XXXX 10Xxxxxx 10xxxxxx */
+ if ((s[1] & 0xc0) != 0x80 ||
+ (s[2] & 0xc0) != 0x80 ||
+ (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */
+ (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */
+ (s[0] == 0xef && s[1] == 0xbf &&
+ (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */
+ return s;
+ else
+ s += 3;
+ }
+ else if ((s[0] & 0xf8) == 0xf0)
+ {
+ /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
+ if ((s[1] & 0xc0) != 0x80 ||
+ (s[2] & 0xc0) != 0x80 ||
+ (s[3] & 0xc0) != 0x80 ||
+ (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */
+ (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */
+ return s;
+ else
+ s += 4;
+ }
+ else
+ return s;
+ }
+
+ return NULL;
+}
+
+char *url_decode_r(char *to, char *url, size_t size) {
+ char *s = url, // source
+ *d = to, // destination
+ *e = &to[size - 1]; // destination end
+
+ while(*s && d < e) {
+ if(unlikely(*s == '%')) {
+ char t = url_percent_escape_decode(s);
+ if(IS_UTF8_BYTE(t)) {
+ char bytes_written = url_decode_multibyte_utf8(s, d, e);
+ if(likely(bytes_written)){
+ d += bytes_written;
+ s += (bytes_written * 3)-1;
+ }
+ else {
+ goto fail_cleanup;
+ }
+ }
+ else if(likely(t) && isprint(t)) {
+ // avoid HTTP header injection
+ *d++ = t;
+ s += 2;
+ }
+ else
+ goto fail_cleanup;
+ }
+ else if(unlikely(*s == '+'))
+ *d++ = ' ';
+
+ else
+ *d++ = *s;
+
+ s++;
+ }
+
+ *d = '\0';
+
+ if(unlikely( utf8_check((unsigned char *)to) )) //NULL means sucess here
+ return NULL;
+
+ return to;
+
+fail_cleanup:
+ *d = '\0';
+ return NULL;
+}
+
+/**
+ * Is request complete?
+ *
+ * Check whether the request is complete.
+ * This function cannot check all the requests METHODS, for example, case you are working with POST, it will fail.
+ *
+ * @param begin is the first character of the sequence to analyse.
+ * @param end is the last character of the sequence
+ * @param length is the length of the total of bytes read, it is not the difference between end and begin.
+ *
+ * @return It returns 1 when the request is complete and 0 otherwise.
+ */
+inline int url_is_request_complete(char *begin, char *end, size_t length) {
+
+ if ( begin == end) {
+ //Message cannot be complete when first and last address are the same
+ return 0;
+ }
+
+ //This math to verify the last is valid, because we are discarding the POST
+ if (length > 4) {
+ begin = end - 4;
+ }
+
+ return (strstr(begin, "\r\n\r\n"))?1:0;
+}
+
+/**
+ * Find protocol
+ *
+ * Search for the string ' HTTP/' in the message given.
+ *
+ * @param s is the start of the user request.
+ * @return
+ */
+inline char *url_find_protocol(char *s) {
+ while(*s) {
+ // find the next space
+ while (*s && *s != ' ') s++;
+
+ // is it SPACE + "HTTP/" ?
+ if(*s && !strncmp(s, " HTTP/", 6)) break;
+ else s++;
+ }
+
+ return s;
+}
+
+/**
+ * Map query string
+ *
+ * Map the query string fields that will be decoded.
+ * This functions must be called after to check the presence of query strings,
+ * here we are assuming that you already tested this.
+ *
+ * @param out the pointer to pointers that will be used to map
+ * @param url the input url that we are decoding.
+ *
+ * @return It returns the number of total variables in the query string.
+ */
+int url_map_query_string(char **out, char *url) {
+ (void)out;
+ (void)url;
+ int count = 0;
+
+ //First we try to parse considering that there was not URL encode process
+ char *moveme = url;
+ char *ptr;
+
+ //We always we have at least one here, so I can set this.
+ out[count++] = moveme;
+ while(moveme) {
+ ptr = strchr((moveme+1), '&');
+ if(ptr) {
+ out[count++] = ptr;
+ }
+
+ moveme = ptr;
+ }
+
+ //I could not find any '&', so I am assuming now it is like '%26'
+ if (count == 1) {
+ moveme = url;
+ while(moveme) {
+ ptr = strchr((moveme+1), '%');
+ if(ptr) {
+ char *test = (ptr+1);
+ if (!strncmp(test, "3f", 2) || !strncmp(test, "3F", 2)) {
+ out[count++] = ptr;
+ }
+ }
+ moveme = ptr;
+ }
+ }
+
+ return count;
+}
+
+/**
+ * Parse query string
+ *
+ * Parse the query string mapped and store it inside output.
+ *
+ * @param output is a vector where I will store the string.
+ * @param max is the maximum length of the output
+ * @param map the map done by the function url_map_query_string.
+ * @param total the total number of variables inside map
+ *
+ * @return It returns 0 on success and -1 otherwise
+ */
+int url_parse_query_string(char *output, size_t max, char **map, int total) {
+ if(!total) {
+ return 0;
+ }
+
+ int counter, next;
+ size_t length;
+ char *end;
+ char *begin = map[0];
+ char save;
+ size_t copied = 0;
+ for(counter = 0, next=1 ; next <= total ; ++counter, ++next) {
+ if (next != total) {
+ end = map[next];
+ length = (size_t) (end - begin);
+ save = *end;
+ *end = 0x00;
+ } else {
+ length = strlen(begin);
+ end = NULL;
+ }
+ length++;
+
+ if (length > (max - copied)) {
+ error("Parsing query string: we cannot parse a query string so big");
+ break;
+ }
+
+ if(!url_decode_r(output, begin, length)) {
+ return -1;
+ }
+ length = strlen(output);
+ copied += length;
+ output += length;
+
+ begin = end;
+ if (begin) {
+ *begin = save;
+ }
+ }
+
+ return 0;
+}
diff --git a/libnetdata/url/url.h b/libnetdata/url/url.h
new file mode 100644
index 0000000..10f3fe1
--- /dev/null
+++ b/libnetdata/url/url.h
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_URL_H
+#define NETDATA_URL_H 1
+
+#include "../libnetdata.h"
+
+// ----------------------------------------------------------------------------
+// URL encode / decode
+// code from: http://www.geekhideout.com/urlcode.shtml
+
+/* Converts a hex character to its integer value */
+extern char from_hex(char ch);
+
+/* Converts an integer value to its hex character*/
+extern char to_hex(char code);
+
+/* Returns a url-encoded version of str */
+/* IMPORTANT: be sure to free() the returned string after use */
+extern char *url_encode(char *str);
+
+/* Returns a url-decoded version of str */
+/* IMPORTANT: be sure to free() the returned string after use */
+extern char *url_decode(char *str);
+
+extern char *url_decode_r(char *to, char *url, size_t size);
+
+#define WEB_FIELDS_MAX 400
+extern int url_map_query_string(char **out, char *url);
+extern int url_parse_query_string(char *output, size_t max, char **map, int total);
+
+extern int url_is_request_complete(char *begin,char *end,size_t length);
+extern char *url_find_protocol(char *s);
+
+#endif /* NETDATA_URL_H */