diff options
Diffstat (limited to '')
100 files changed, 16296 insertions, 0 deletions
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am new file mode 100644 index 0000000..598b72f --- /dev/null +++ b/libnetdata/Makefile.am @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +SUBDIRS = \ + adaptive_resortable_list \ + avl \ + buffer \ + clocks \ + config \ + dictionary \ + ebpf \ + eval \ + json \ + health \ + locks \ + log \ + popen \ + procfile \ + simple_pattern \ + socket \ + statistical \ + storage_number \ + threads \ + url \ + tests \ + $(NULL) + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/README.md b/libnetdata/README.md new file mode 100644 index 0000000..cdb199f --- /dev/null +++ b/libnetdata/README.md @@ -0,0 +1,10 @@ +<!-- +title: "libnetdata" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/README.md +--> + +# libnetdata + +`libnetdata` is a collection of library code that is used by all Netdata `C` programs. + +[](<>) diff --git a/libnetdata/adaptive_resortable_list/Makefile.am b/libnetdata/adaptive_resortable_list/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/adaptive_resortable_list/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/adaptive_resortable_list/README.md b/libnetdata/adaptive_resortable_list/README.md new file mode 100644 index 0000000..74e379a --- /dev/null +++ b/libnetdata/adaptive_resortable_list/README.md @@ -0,0 +1,99 @@ +<!-- +title: "Adaptive Re-sortable List (ARL)" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/adaptive_resortable_list/README.md +--> + +# Adaptive Re-sortable List (ARL) + +This library allows Netdata to read a series of `name - value` pairs +in the **fastest possible way**. + +ARLs are used all over Netdata, as they are the most +CPU utilization efficient way to process `/proc` files. They are used to +process both vertical (csv like) and horizontal (one pair per line) `name - value` pairs. + +## How ARL works + +It maintains a linked list of all `NAME` (keywords), sorted in the +order found in the data source. The linked list is kept +sorted at all times - the data source may change at any time, the +linked list will adapt at the next iteration. + +### Initialization + +During initialization (just once), the caller: + +- calls `arl_create()` to create the ARL + +- calls `arl_expect()` multiple times to register the expected keywords + +The library will call the `processor()` function (given to +`arl_create()`), for each expected keyword found. +The default `processor()` expects `dst` to be an `unsigned long long *`. + +Each `name` keyword may have a different `processor()` (by calling +`arl_expect_custom()` instead of `arl_expect()`). + +### Data collection iterations + +For each iteration through the data source, the caller: + +- calls `arl_begin()` to initiate a data collection iteration. + This is to be called just ONCE every time the source is re-evaluated. + +- calls `arl_check()` for each entry read from the file. + +### Cleanup + +When the caller exits: + +- calls `arl_free()` to destroy this and free all memory. + +### Performance + +ARL maintains a list of `name` keywords found in the data source (even the ones +that are not useful for data collection). + +If the data source maintains the same order on the `name-value` pairs, for each +each call to `arl_check()` only an `strcmp()` is executed to verify the +expected order has not changed, a counter is incremented and a pointer is changed. +So, if the data source has 100 `name-value` pairs, and their order remains constant +over time, 100 successful `strcmp()` are executed. + +In the unlikely event that an iteration sees the data source with a different order, +for each out-of-order keyword, a full search of the remaining keywords is made. But +this search uses 32bit hashes, not string comparisons, so it should also be fast. + +When all expectations are satisfied (even in the middle of an iteration), +the call to `arl_check()` will return 1, to signal the caller to stop the loop, +saving valuable CPU resources for the rest of the data source. + +In the following test we used alternative methods to process, **1M times**, +a data source like `/proc/meminfo`, already tokenized, in memory, +to extract the same number of expected metrics: + +|test|code|string comparison|number parsing|duration| +|:--:|:--:|:---------------:|:------------:|:------:| +|1|if-else-if-else-if|`strcmp()`|`strtoull()`|4630.337 ms| +|2|nested loops|inline `simple_hash()` and `strcmp()`|`strtoull()`|1597.481 ms| +|3|nested loops|inline `simple_hash()` and `strcmp()`|`str2ull()`|923.523 ms| +|4|if-else-if-else-if|inline `simple_hash()` and `strcmp()`|`strtoull()`|854.574 ms| +|5|if-else-if-else-if|statement expression `simple_hash()` and `strcmp()`|`strtoull()`|912.013 ms| +|6|if-continue|inline `simple_hash()` and `strcmp()`|`strtoull()`|842.279 ms| +|7|if-else-if-else-if|inline `simple_hash()` and `strcmp()`|`str2ull()`|602.837 ms| +|8|ARL|ARL|`strtoull()`|350.360 ms| +|9|ARL|ARL|`str2ull()`|157.237 ms| + +Compared to unoptimized code (test No 1: 4.6sec): + +- before ARL Netdata was using test No **7** with hashing and a custom `str2ull()` to achieve 602ms. +- the current ARL implementation is test No **9** that needs only 157ms (29 times faster vs unoptimized code, about 4 times faster vs optimized code). + +[Check the source code of this test](https://raw.githubusercontent.com/netdata/netdata/master/tests/profile/benchmark-value-pairs.c). + +## Limitations + +Do not use ARL if the a name/keyword may appear more than once in the +source data. + +[](<>) diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c new file mode 100644 index 0000000..7f4c6c5 --- /dev/null +++ b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +// the default processor() of the ARL +// can be overwritten at arl_create() +inline void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst) { + (void)name; + (void)hash; + + register unsigned long long *d = dst; + *d = str2ull(value); + // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, *d); +} + +inline void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, const char *value, void *dst) { + (void)name; + (void)hash; + + register kernel_uint_t *d = dst; + *d = str2kernel_uint_t(value); + // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, (unsigned long long)*d); +} + +inline void arl_callback_ssize_t(const char *name, uint32_t hash, const char *value, void *dst) { + (void)name; + (void)hash; + + register ssize_t *d = dst; + *d = (ssize_t)str2ll(value, NULL); + // fprintf(stderr, "name '%s' with hash %u and value '%s' is %zd\n", name, hash, value, *d); +} + +// create a new ARL +ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks) { + ARL_BASE *base = callocz(1, sizeof(ARL_BASE)); + + base->name = strdupz(name); + + if(!processor) + base->processor = arl_callback_str2ull; + else + base->processor = processor; + + base->rechecks = rechecks; + + return base; +} + +void arl_free(ARL_BASE *arl_base) { + if(unlikely(!arl_base)) + return; + + while(arl_base->head) { + ARL_ENTRY *e = arl_base->head; + arl_base->head = e->next; + + freez(e->name); +#ifdef NETDATA_INTERNAL_CHECKS + memset(e, 0, sizeof(ARL_ENTRY)); +#endif + freez(e); + } + + freez(arl_base->name); + +#ifdef NETDATA_INTERNAL_CHECKS + memset(arl_base, 0, sizeof(ARL_BASE)); +#endif + + freez(arl_base); +} + +void arl_begin(ARL_BASE *base) { + +#ifdef NETDATA_INTERNAL_CHECKS + if(likely(base->iteration > 10)) { + // do these checks after the ARL has been sorted + + if(unlikely(base->relinkings > (base->expected + base->allocated))) + info("ARL '%s' has %zu relinkings with %zu expected and %zu allocated entries. Is the source changing so fast?" + , base->name, base->relinkings, base->expected, base->allocated); + + if(unlikely(base->slow > base->fast)) + info("ARL '%s' has %zu fast searches and %zu slow searches. Is the source really changing so fast?" + , base->name, base->fast, base->slow); + + /* + if(unlikely(base->iteration % 60 == 0)) { + info("ARL '%s' statistics: iteration %zu, expected %zu, wanted %zu, allocated %zu, fred %zu, relinkings %zu, found %zu, added %zu, fast %zu, slow %zu" + , base->name + , base->iteration + , base->expected + , base->wanted + , base->allocated + , base->fred + , base->relinkings + , base->found + , base->added + , base->fast + , base->slow + ); + // for(e = base->head; e; e = e->next) fprintf(stderr, "%s ", e->name); + // fprintf(stderr, "\n"); + } + */ + } +#endif + + if(unlikely(base->iteration > 0 && (base->added || (base->iteration % base->rechecks) == 0))) { + int wanted_equals_expected = ((base->iteration % base->rechecks) == 0); + + // fprintf(stderr, "\n\narl_begin() rechecking, added %zu, iteration %zu, rechecks %zu, wanted_equals_expected %d\n\n\n", base->added, base->iteration, base->rechecks, wanted_equals_expected); + + base->added = 0; + base->wanted = (wanted_equals_expected)?base->expected:0; + + ARL_ENTRY *e = base->head; + while(e) { + if(e->flags & ARL_ENTRY_FLAG_FOUND) { + + // remove the found flag + e->flags &= ~ARL_ENTRY_FLAG_FOUND; + + // count it in wanted + if(!wanted_equals_expected && e->flags & ARL_ENTRY_FLAG_EXPECTED) + base->wanted++; + + } + else if(e->flags & ARL_ENTRY_FLAG_DYNAMIC && !(base->head == e && !e->next)) { // not last entry + // we can remove this entry + // it is not found, and it was created because + // it was found in the source file + + // remember the next one + ARL_ENTRY *t = e->next; + + // remove it from the list + if(e->next) e->next->prev = e->prev; + if(e->prev) e->prev->next = e->next; + if(base->head == e) base->head = e->next; + + // free it + freez(e->name); + freez(e); + + // count it + base->fred++; + + // continue + e = t; + continue; + } + + e = e->next; + } + } + + if(unlikely(!base->head)) { + // hm... no nodes at all in the list #1700 + // add a fake one to prevent a crash + // this is better than checking for the existence of nodes all the time + arl_expect(base, "a-really-not-existing-source-keyword", NULL); + } + + base->iteration++; + base->next_keyword = base->head; + base->found = 0; + +} + +// register an expected keyword to the ARL +// together with its destination ( i.e. the output of the processor() ) +ARL_ENTRY *arl_expect_custom(ARL_BASE *base, const char *keyword, void (*processor)(const char *name, uint32_t hash, const char *value, void *dst), void *dst) { + ARL_ENTRY *e = callocz(1, sizeof(ARL_ENTRY)); + e->name = strdupz(keyword); + e->hash = simple_hash(e->name); + e->processor = (processor)?processor:base->processor; + e->dst = dst; + e->flags = ARL_ENTRY_FLAG_EXPECTED; + e->prev = NULL; + e->next = base->head; + + if(base->head) base->head->prev = e; + else base->next_keyword = e; + + base->head = e; + base->expected++; + base->allocated++; + + base->wanted = base->expected; + + return e; +} + +int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const char *value) { + ARL_ENTRY *e; + + uint32_t hash = simple_hash(s); + + // find if it already exists in the data + for(e = base->head; e ; e = e->next) + if(e->hash == hash && !strcmp(e->name, s)) + break; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(base->next_keyword && e == base->next_keyword)) + fatal("Internal Error: e == base->last"); +#endif + + if(e) { + // found it in the keywords + + base->relinkings++; + + // run the processor for it + if(unlikely(e->dst)) { + e->processor(e->name, hash, value, e->dst); + base->found++; + } + + // unlink it - we will relink it below + if(e->next) e->next->prev = e->prev; + if(e->prev) e->prev->next = e->next; + + // make sure the head is properly linked + if(base->head == e) + base->head = e->next; + } + else { + // not found + + // create it + e = callocz(1, sizeof(ARL_ENTRY)); + e->name = strdupz(s); + e->hash = hash; + e->flags = ARL_ENTRY_FLAG_DYNAMIC; + + base->allocated++; + base->added++; + } + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely(base->iteration % 60 == 0 && e->flags & ARL_ENTRY_FLAG_FOUND)) + info("ARL '%s': entry '%s' is already found. Did you forget to call arl_begin()?", base->name, s); +#endif + + e->flags |= ARL_ENTRY_FLAG_FOUND; + + // link it here + e->next = base->next_keyword; + if(base->next_keyword) { + e->prev = base->next_keyword->prev; + base->next_keyword->prev = e; + + if(e->prev) + e->prev->next = e; + + if(base->head == base->next_keyword) + base->head = e; + } + else { + e->prev = NULL; + + if(!base->head) + base->head = e; + } + + // prepare the next iteration + base->next_keyword = e->next; + if(unlikely(!base->next_keyword)) + base->next_keyword = base->head; + + if(unlikely(base->found == base->wanted)) { + // fprintf(stderr, "FOUND ALL WANTED 1: found = %zu, wanted = %zu, expected %zu\n", base->found, base->wanted, base->expected); + return 1; + } + + return 0; +} diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h new file mode 100644 index 0000000..011ee73 --- /dev/null +++ b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#ifndef NETDATA_ADAPTIVE_RESORTABLE_LIST_H +#define NETDATA_ADAPTIVE_RESORTABLE_LIST_H 1 + +#define ARL_ENTRY_FLAG_FOUND 0x01 // the entry has been found in the source data +#define ARL_ENTRY_FLAG_EXPECTED 0x02 // the entry is expected by the program +#define ARL_ENTRY_FLAG_DYNAMIC 0x04 // the entry was dynamically allocated, from source data + +typedef struct arl_entry { + char *name; // the keywords + uint32_t hash; // the hash of the keyword + + void *dst; // the dst to pass to the processor + + uint8_t flags; // ARL_ENTRY_FLAG_* + + // the processor to do the job + void (*processor)(const char *name, uint32_t hash, const char *value, void *dst); + + // double linked list for fast re-linkings + struct arl_entry *prev, *next; +} ARL_ENTRY; + +typedef struct arl_base { + char *name; + + size_t iteration; // incremented on each iteration (arl_begin()) + size_t found; // the number of expected keywords found in this iteration + size_t expected; // the number of expected keywords + size_t wanted; // the number of wanted keywords + // i.e. the number of keywords found and expected + + size_t relinkings; // the number of relinkings we have made so far + + size_t allocated; // the number of keywords allocated + size_t fred; // the number of keywords cleaned up + + size_t rechecks; // the number of iterations between re-checks of the + // wanted number of keywords + // this is only needed in cases where the source + // is having less lines over time. + + size_t added; // it is non-zero if new keywords have been added + // this is only needed to detect new lines have + // been added to the file, over time. + +#ifdef NETDATA_INTERNAL_CHECKS + size_t fast; // the number of times we have taken the fast path + size_t slow; // the number of times we have taken the slow path +#endif + + // the processor to do the job + void (*processor)(const char *name, uint32_t hash, const char *value, void *dst); + + // the linked list of the keywords + ARL_ENTRY *head; + + // since we keep the list of keywords sorted (as found in the source data) + // this is next keyword that we expect to find in the source data. + ARL_ENTRY *next_keyword; +} ARL_BASE; + +// create a new ARL +extern ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks); + +// free an ARL +extern void arl_free(ARL_BASE *arl_base); + +// register an expected keyword to the ARL +// together with its destination ( i.e. the output of the processor() ) +extern ARL_ENTRY *arl_expect_custom(ARL_BASE *base, const char *keyword, void (*processor)(const char *name, uint32_t hash, const char *value, void *dst), void *dst); +#define arl_expect(base, keyword, dst) arl_expect_custom(base, keyword, NULL, dst) + +// an internal call to complete the check() call +extern int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const char *value); + +// begin an ARL iteration +extern void arl_begin(ARL_BASE *base); + +extern void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst); +extern void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, const char *value, void *dst); +extern void arl_callback_ssize_t(const char *name, uint32_t hash, const char *value, void *dst); + +// check a keyword against the ARL +// this is to be called for each keyword read from source data +// s = the keyword, as collected +// src = the src data to be passed to the processor +// it is defined in the header file in order to be inlined +static inline int arl_check(ARL_BASE *base, const char *keyword, const char *value) { + ARL_ENTRY *e = base->next_keyword; + +#ifdef NETDATA_INTERNAL_CHECKS + if(unlikely((base->fast + base->slow) % (base->expected + base->allocated) == 0 && (base->fast + base->slow) > (base->expected + base->allocated) * base->iteration)) + info("ARL '%s': Did you forget to call arl_begin()?", base->name); +#endif + + // it should be the first entry (pointed by base->next_keyword) + if(likely(!strcmp(keyword, e->name))) { + // it is + +#ifdef NETDATA_INTERNAL_CHECKS + base->fast++; +#endif + + e->flags |= ARL_ENTRY_FLAG_FOUND; + + // execute the processor + if(unlikely(e->dst)) { + e->processor(e->name, e->hash, value, e->dst); + base->found++; + } + + // be prepared for the next iteration + base->next_keyword = e->next; + if(unlikely(!base->next_keyword)) + base->next_keyword = base->head; + + // stop if we collected all the values for this iteration + if(unlikely(base->found == base->wanted)) { + // fprintf(stderr, "FOUND ALL WANTED 2: found = %zu, wanted = %zu, expected %zu\n", base->found, base->wanted, base->expected); + return 1; + } + + return 0; + } + +#ifdef NETDATA_INTERNAL_CHECKS + base->slow++; +#endif + + // we read from source, a not-expected keyword + return arl_find_or_create_and_relink(base, keyword, value); +} + +#endif //NETDATA_ADAPTIVE_RESORTABLE_LIST_H diff --git a/libnetdata/avl/Makefile.am b/libnetdata/avl/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/avl/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/avl/README.md b/libnetdata/avl/README.md new file mode 100644 index 0000000..2097be5 --- /dev/null +++ b/libnetdata/avl/README.md @@ -0,0 +1,17 @@ +<!-- +title: "AVL" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/avl/README.md +--> + +# AVL + +AVL is a library indexing objects in B-Trees. + +`avl_insert()`, `avl_remove()` and `avl_search()` are adaptations +of the AVL algorithm found in `libavl` v2.0.3, so that they do not +use any memory allocations and their memory footprint is optimized +(by eliminating non-necessary data members). + +In addition to the above, this version of AVL, provides versions using locks +and traversal functions. +[](<>) diff --git a/libnetdata/avl/avl.c b/libnetdata/avl/avl.c new file mode 100644 index 0000000..5219851 --- /dev/null +++ b/libnetdata/avl/avl.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "../libnetdata.h" + +/* ------------------------------------------------------------------------- */ +/* + * avl_insert(), avl_remove() and avl_search() + * are adaptations (by Costa Tsaousis) of the AVL algorithm found in libavl + * v2.0.3, so that they do not use any memory allocations and their memory + * footprint is optimized (by eliminating non-necessary data members). + * + * libavl - library for manipulation of binary trees. + * Copyright (C) 1998, 1999, 2000, 2001, 2002, 2004 Free Software + * Foundation, Inc. +*/ + + +/* Search |tree| for an item matching |item|, and return it if found. + Otherwise return |NULL|. */ +avl *avl_search(avl_tree_type *tree, avl *item) { + avl *p; + + // assert (tree != NULL && item != NULL); + + for (p = tree->root; p != NULL; ) { + int cmp = tree->compar(item, p); + + if (cmp < 0) + p = p->avl_link[0]; + else if (cmp > 0) + p = p->avl_link[1]; + else /* |cmp == 0| */ + return p; + } + + return NULL; +} + +/* Inserts |item| into |tree| and returns a pointer to |item|'s address. + If a duplicate item is found in the tree, + returns a pointer to the duplicate without inserting |item|. + */ +avl *avl_insert(avl_tree_type *tree, avl *item) { + avl *y, *z; /* Top node to update balance factor, and parent. */ + avl *p, *q; /* Iterator, and parent. */ + avl *n; /* Newly inserted node. */ + avl *w; /* New root of rebalanced subtree. */ + unsigned char dir; /* Direction to descend. */ + + unsigned char da[AVL_MAX_HEIGHT]; /* Cached comparison results. */ + int k = 0; /* Number of cached results. */ + + // assert(tree != NULL && item != NULL); + + z = (avl *) &tree->root; + y = tree->root; + dir = 0; + for (q = z, p = y; p != NULL; q = p, p = p->avl_link[dir]) { + int cmp = tree->compar(item, p); + if (cmp == 0) + return p; + + if (p->avl_balance != 0) + z = q, y = p, k = 0; + da[k++] = dir = (unsigned char)(cmp > 0); + } + + n = q->avl_link[dir] = item; + + // tree->avl_count++; + n->avl_link[0] = n->avl_link[1] = NULL; + n->avl_balance = 0; + if (y == NULL) return n; + + for (p = y, k = 0; p != n; p = p->avl_link[da[k]], k++) + if (da[k] == 0) + p->avl_balance--; + else + p->avl_balance++; + + if (y->avl_balance == -2) { + avl *x = y->avl_link[0]; + if (x->avl_balance == -1) { + w = x; + y->avl_link[0] = x->avl_link[1]; + x->avl_link[1] = y; + x->avl_balance = y->avl_balance = 0; + } + else { + // assert (x->avl_balance == +1); + w = x->avl_link[1]; + x->avl_link[1] = w->avl_link[0]; + w->avl_link[0] = x; + y->avl_link[0] = w->avl_link[1]; + w->avl_link[1] = y; + if (w->avl_balance == -1) + x->avl_balance = 0, y->avl_balance = +1; + else if (w->avl_balance == 0) + x->avl_balance = y->avl_balance = 0; + else /* |w->avl_balance == +1| */ + x->avl_balance = -1, y->avl_balance = 0; + w->avl_balance = 0; + } + } + else if (y->avl_balance == +2) { + avl *x = y->avl_link[1]; + if (x->avl_balance == +1) { + w = x; + y->avl_link[1] = x->avl_link[0]; + x->avl_link[0] = y; + x->avl_balance = y->avl_balance = 0; + } + else { + // assert (x->avl_balance == -1); + w = x->avl_link[0]; + x->avl_link[0] = w->avl_link[1]; + w->avl_link[1] = x; + y->avl_link[1] = w->avl_link[0]; + w->avl_link[0] = y; + if (w->avl_balance == +1) + x->avl_balance = 0, y->avl_balance = -1; + else if (w->avl_balance == 0) + x->avl_balance = y->avl_balance = 0; + else /* |w->avl_balance == -1| */ + x->avl_balance = +1, y->avl_balance = 0; + w->avl_balance = 0; + } + } + else return n; + + z->avl_link[y != z->avl_link[0]] = w; + + // tree->avl_generation++; + return n; +} + +/* Deletes from |tree| and returns an item matching |item|. + Returns a null pointer if no matching item found. */ +avl *avl_remove(avl_tree_type *tree, avl *item) { + /* Stack of nodes. */ + avl *pa[AVL_MAX_HEIGHT]; /* Nodes. */ + unsigned char da[AVL_MAX_HEIGHT]; /* |avl_link[]| indexes. */ + int k; /* Stack pointer. */ + + avl *p; /* Traverses tree to find node to delete. */ + int cmp; /* Result of comparison between |item| and |p|. */ + + // assert (tree != NULL && item != NULL); + + k = 0; + p = (avl *) &tree->root; + for(cmp = -1; cmp != 0; cmp = tree->compar(item, p)) { + unsigned char dir = (unsigned char)(cmp > 0); + + pa[k] = p; + da[k++] = dir; + + p = p->avl_link[dir]; + if(p == NULL) return NULL; + } + + item = p; + + if (p->avl_link[1] == NULL) + pa[k - 1]->avl_link[da[k - 1]] = p->avl_link[0]; + else { + avl *r = p->avl_link[1]; + if (r->avl_link[0] == NULL) { + r->avl_link[0] = p->avl_link[0]; + r->avl_balance = p->avl_balance; + pa[k - 1]->avl_link[da[k - 1]] = r; + da[k] = 1; + pa[k++] = r; + } + else { + avl *s; + int j = k++; + + for (;;) { + da[k] = 0; + pa[k++] = r; + s = r->avl_link[0]; + if (s->avl_link[0] == NULL) break; + + r = s; + } + + s->avl_link[0] = p->avl_link[0]; + r->avl_link[0] = s->avl_link[1]; + s->avl_link[1] = p->avl_link[1]; + s->avl_balance = p->avl_balance; + + pa[j - 1]->avl_link[da[j - 1]] = s; + da[j] = 1; + pa[j] = s; + } + } + + // assert (k > 0); + while (--k > 0) { + avl *y = pa[k]; + + if (da[k] == 0) { + y->avl_balance++; + if (y->avl_balance == +1) break; + else if (y->avl_balance == +2) { + avl *x = y->avl_link[1]; + if (x->avl_balance == -1) { + avl *w; + // assert (x->avl_balance == -1); + w = x->avl_link[0]; + x->avl_link[0] = w->avl_link[1]; + w->avl_link[1] = x; + y->avl_link[1] = w->avl_link[0]; + w->avl_link[0] = y; + if (w->avl_balance == +1) + x->avl_balance = 0, y->avl_balance = -1; + else if (w->avl_balance == 0) + x->avl_balance = y->avl_balance = 0; + else /* |w->avl_balance == -1| */ + x->avl_balance = +1, y->avl_balance = 0; + w->avl_balance = 0; + pa[k - 1]->avl_link[da[k - 1]] = w; + } + else { + y->avl_link[1] = x->avl_link[0]; + x->avl_link[0] = y; + pa[k - 1]->avl_link[da[k - 1]] = x; + if (x->avl_balance == 0) { + x->avl_balance = -1; + y->avl_balance = +1; + break; + } + else x->avl_balance = y->avl_balance = 0; + } + } + } + else + { + y->avl_balance--; + if (y->avl_balance == -1) break; + else if (y->avl_balance == -2) { + avl *x = y->avl_link[0]; + if (x->avl_balance == +1) { + avl *w; + // assert (x->avl_balance == +1); + w = x->avl_link[1]; + x->avl_link[1] = w->avl_link[0]; + w->avl_link[0] = x; + y->avl_link[0] = w->avl_link[1]; + w->avl_link[1] = y; + if (w->avl_balance == -1) + x->avl_balance = 0, y->avl_balance = +1; + else if (w->avl_balance == 0) + x->avl_balance = y->avl_balance = 0; + else /* |w->avl_balance == +1| */ + x->avl_balance = -1, y->avl_balance = 0; + w->avl_balance = 0; + pa[k - 1]->avl_link[da[k - 1]] = w; + } + else { + y->avl_link[0] = x->avl_link[1]; + x->avl_link[1] = y; + pa[k - 1]->avl_link[da[k - 1]] = x; + if (x->avl_balance == 0) { + x->avl_balance = +1; + y->avl_balance = -1; + break; + } + else x->avl_balance = y->avl_balance = 0; + } + } + } + } + + // tree->avl_count--; + // tree->avl_generation++; + return item; +} + +/* ------------------------------------------------------------------------- */ +// below are functions by (C) Costa Tsaousis + +// --------------------------- +// traversing + +int avl_walker(avl *node, int (*callback)(void * /*entry*/, void * /*data*/), void *data) { + int total = 0, ret = 0; + + if(node->avl_link[0]) { + ret = avl_walker(node->avl_link[0], callback, data); + if(ret < 0) return ret; + total += ret; + } + + ret = callback(node, data); + if(ret < 0) return ret; + total += ret; + + if(node->avl_link[1]) { + ret = avl_walker(node->avl_link[1], callback, data); + if (ret < 0) return ret; + total += ret; + } + + return total; +} + +int avl_traverse(avl_tree_type *tree, int (*callback)(void * /*entry*/, void * /*data*/), void *data) { + if(tree->root) + return avl_walker(tree->root, callback, data); + else + return 0; +} + +// --------------------------- +// locks + +void avl_read_lock(avl_tree_lock *t) { +#ifndef AVL_WITHOUT_PTHREADS +#ifdef AVL_LOCK_WITH_MUTEX + netdata_mutex_lock(&t->mutex); +#else + netdata_rwlock_rdlock(&t->rwlock); +#endif +#endif /* AVL_WITHOUT_PTHREADS */ +} + +void avl_write_lock(avl_tree_lock *t) { +#ifndef AVL_WITHOUT_PTHREADS +#ifdef AVL_LOCK_WITH_MUTEX + netdata_mutex_lock(&t->mutex); +#else + netdata_rwlock_wrlock(&t->rwlock); +#endif +#endif /* AVL_WITHOUT_PTHREADS */ +} + +void avl_unlock(avl_tree_lock *t) { +#ifndef AVL_WITHOUT_PTHREADS +#ifdef AVL_LOCK_WITH_MUTEX + netdata_mutex_unlock(&t->mutex); +#else + netdata_rwlock_unlock(&t->rwlock); +#endif +#endif /* AVL_WITHOUT_PTHREADS */ +} + +// --------------------------- +// operations with locking + +void avl_init_lock(avl_tree_lock *tree, int (*compar)(void * /*a*/, void * /*b*/)) { + avl_init(&tree->avl_tree, compar); + +#ifndef AVL_WITHOUT_PTHREADS + int lock; + +#ifdef AVL_LOCK_WITH_MUTEX + lock = netdata_mutex_init(&tree->mutex, NULL); +#else + lock = netdata_rwlock_init(&tree->rwlock); +#endif + + if(lock != 0) + fatal("Failed to initialize AVL mutex/rwlock, error: %d", lock); + +#endif /* AVL_WITHOUT_PTHREADS */ +} + +void avl_destroy_lock(avl_tree_lock *tree) { +#ifndef AVL_WITHOUT_PTHREADS + int lock; + +#ifdef AVL_LOCK_WITH_MUTEX + lock = pthread_mutex_destroy(&tree->mutex); +#else + lock = pthread_rwlock_destroy(&tree->rwlock); +#endif + + if(lock != 0) + fatal("Failed to destroy AVL mutex/rwlock, error: %d", lock); + +#endif /* AVL_WITHOUT_PTHREADS */ +} + +avl *avl_search_lock(avl_tree_lock *tree, avl *item) { + avl_read_lock(tree); + avl *ret = avl_search(&tree->avl_tree, item); + avl_unlock(tree); + return ret; +} + +avl * avl_remove_lock(avl_tree_lock *tree, avl *item) { + avl_write_lock(tree); + avl *ret = avl_remove(&tree->avl_tree, item); + avl_unlock(tree); + return ret; +} + +avl *avl_insert_lock(avl_tree_lock *tree, avl *item) { + avl_write_lock(tree); + avl * ret = avl_insert(&tree->avl_tree, item); + avl_unlock(tree); + return ret; +} + +int avl_traverse_lock(avl_tree_lock *tree, int (*callback)(void * /*entry*/, void * /*data*/), void *data) { + int ret; + avl_read_lock(tree); + ret = avl_traverse(&tree->avl_tree, callback, data); + avl_unlock(tree); + return ret; +} + +void avl_init(avl_tree_type *tree, int (*compar)(void * /*a*/, void * /*b*/)) { + tree->root = NULL; + tree->compar = compar; +} + +// ------------------ diff --git a/libnetdata/avl/avl.h b/libnetdata/avl/avl.h new file mode 100644 index 0000000..32e3f27 --- /dev/null +++ b/libnetdata/avl/avl.h @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef _AVL_H +#define _AVL_H 1 + +#include "../libnetdata.h" + +/* Maximum AVL tree height. */ +#ifndef AVL_MAX_HEIGHT +#define AVL_MAX_HEIGHT 92 +#endif + +#ifndef AVL_WITHOUT_PTHREADS +#include <pthread.h> + +// #define AVL_LOCK_WITH_MUTEX 1 + +#ifdef AVL_LOCK_WITH_MUTEX +#define AVL_LOCK_INITIALIZER NETDATA_MUTEX_INITIALIZER +#else /* AVL_LOCK_WITH_MUTEX */ +#define AVL_LOCK_INITIALIZER NETDATA_RWLOCK_INITIALIZER +#endif /* AVL_LOCK_WITH_MUTEX */ + +#else /* AVL_WITHOUT_PTHREADS */ +#define AVL_LOCK_INITIALIZER +#endif /* AVL_WITHOUT_PTHREADS */ + +/* Data structures */ + +/* One element of the AVL tree */ +typedef struct avl { + struct avl *avl_link[2]; /* Subtrees. */ + signed char avl_balance; /* Balance factor. */ +} avl; + +/* An AVL tree */ +typedef struct avl_tree_type { + avl *root; + int (*compar)(void *a, void *b); +} avl_tree_type; + +typedef struct avl_tree_lock { + avl_tree_type avl_tree; + +#ifndef AVL_WITHOUT_PTHREADS +#ifdef AVL_LOCK_WITH_MUTEX + netdata_mutex_t mutex; +#else /* AVL_LOCK_WITH_MUTEX */ + netdata_rwlock_t rwlock; +#endif /* AVL_LOCK_WITH_MUTEX */ +#endif /* AVL_WITHOUT_PTHREADS */ +} avl_tree_lock; + +/* Public methods */ + +/* Insert element a into the AVL tree t + * returns the added element a, or a pointer the + * element that is equal to a (as returned by t->compar()) + * a is linked directly to the tree, so it has to + * be properly allocated by the caller. + */ +avl *avl_insert_lock(avl_tree_lock *tree, avl *item) NEVERNULL WARNUNUSED; +avl *avl_insert(avl_tree_type *tree, avl *item) NEVERNULL WARNUNUSED; + +/* Remove an element a from the AVL tree t + * returns a pointer to the removed element + * or NULL if an element equal to a is not found + * (equal as returned by t->compar()) + */ +avl *avl_remove_lock(avl_tree_lock *tree, avl *item) WARNUNUSED; +avl *avl_remove(avl_tree_type *tree, avl *item) WARNUNUSED; + +/* Find the element into the tree that equal to a + * (equal as returned by t->compar()) + * returns NULL is no element is equal to a + */ +avl *avl_search_lock(avl_tree_lock *tree, avl *item); +avl *avl_search(avl_tree_type *tree, avl *item); + +/* Initialize the avl_tree_lock + */ +void avl_init_lock(avl_tree_lock *tree, int (*compar)(void *a, void *b)); +void avl_init(avl_tree_type *tree, int (*compar)(void *a, void *b)); + +/* Destroy the avl_tree_lock locks + */ +void avl_destroy_lock(avl_tree_lock *tree); + +int avl_traverse_lock(avl_tree_lock *tree, int (*callback)(void *entry, void *data), void *data); +int avl_traverse(avl_tree_type *tree, int (*callback)(void *entry, void *data), void *data); + +#endif /* avl.h */ diff --git a/libnetdata/buffer/Makefile.am b/libnetdata/buffer/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/buffer/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/buffer/README.md b/libnetdata/buffer/README.md new file mode 100644 index 0000000..5a9d7b0 --- /dev/null +++ b/libnetdata/buffer/README.md @@ -0,0 +1,17 @@ +<!-- +title: "BUFFER" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/buffer/README.md +--> + +# BUFFER + +`BUFFER` is a convenience library for working with strings in `C`. +Mainly, `BUFFER`s eliminate the need for tracking the string length, thus providing +a safe alternative for string operations. + +Also, they are super fast in printing and appending data to the string and its `buffer_strlen()` +is just a lookup (it does not traverse the string). + +Netdata uses `BUFFER`s for preparing web responses and buffering data to be sent upstream or +to backend databases. +[](<>) diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c new file mode 100644 index 0000000..c37da21 --- /dev/null +++ b/libnetdata/buffer/buffer.c @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#define BUFFER_OVERFLOW_EOF "EOF" + +static inline void buffer_overflow_init(BUFFER *b) +{ + b->buffer[b->size] = '\0'; + strcpy(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF); +} + +#ifdef NETDATA_INTERNAL_CHECKS +#define buffer_overflow_check(b) _buffer_overflow_check(b, __FILE__, __FUNCTION__, __LINE__) +#else +#define buffer_overflow_check(b) +#endif + +static inline void _buffer_overflow_check(BUFFER *b, const char *file, const char *function, const unsigned long line) +{ + if(b->len > b->size) { + error("BUFFER: length %zu is above size %zu, at line %lu, at function %s() of file '%s'.", b->len, b->size, line, function, file); + b->len = b->size; + } + + if(b->buffer[b->size] != '\0' || strcmp(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF) != 0) { + error("BUFFER: detected overflow at line %lu, at function %s() of file '%s'.", line, function, file); + buffer_overflow_init(b); + } +} + + +void buffer_reset(BUFFER *wb) +{ + buffer_flush(wb); + + wb->contenttype = CT_TEXT_PLAIN; + wb->options = 0; + wb->date = 0; + wb->expires = 0; + + buffer_overflow_check(wb); +} + +const char *buffer_tostring(BUFFER *wb) +{ + buffer_need_bytes(wb, 1); + wb->buffer[wb->len] = '\0'; + + buffer_overflow_check(wb); + + return(wb->buffer); +} + +void buffer_char_replace(BUFFER *wb, char from, char to) +{ + char *s = wb->buffer, *end = &wb->buffer[wb->len]; + + while(s != end) { + if(*s == from) *s = to; + s++; + } + + buffer_overflow_check(wb); +} + +// This trick seems to give an 80% speed increase in 32bit systems +// print_calculated_number_llu_r() will just print the digits up to the +// point the remaining value fits in 32 bits, and then calls +// print_calculated_number_lu_r() to print the rest with 32 bit arithmetic. + +inline char *print_number_lu_r(char *str, unsigned long uvalue) { + char *wstr = str; + + // print each digit + do *wstr++ = (char)('0' + (uvalue % 10)); while(uvalue /= 10); + return wstr; +} + +inline char *print_number_llu_r(char *str, unsigned long long uvalue) { + char *wstr = str; + + // print each digit + do *wstr++ = (char)('0' + (uvalue % 10)); while((uvalue /= 10) && uvalue > (unsigned long long)0xffffffff); + if(uvalue) return print_number_lu_r(wstr, uvalue); + return wstr; +} + +inline char *print_number_llu_r_smart(char *str, unsigned long long uvalue) { +#ifdef ENVIRONMENT32 + if(uvalue > (unsigned long long)0xffffffff) + str = print_number_llu_r(str, uvalue); + else + str = print_number_lu_r(str, uvalue); +#else + do *str++ = (char)('0' + (uvalue % 10)); while(uvalue /= 10); +#endif + + return str; +} + +void buffer_print_llu(BUFFER *wb, unsigned long long uvalue) +{ + buffer_need_bytes(wb, 50); + + char *str = &wb->buffer[wb->len]; + char *wstr = str; + +#ifdef ENVIRONMENT32 + if(uvalue > (unsigned long long)0xffffffff) + wstr = print_number_llu_r(wstr, uvalue); + else + wstr = print_number_lu_r(wstr, uvalue); +#else + do *wstr++ = (char)('0' + (uvalue % 10)); while(uvalue /= 10); +#endif + + // terminate it + *wstr = '\0'; + + // reverse it + char *begin = str, *end = wstr - 1, aux; + while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux; + + // return the buffer length + wb->len += wstr - str; +} + +void buffer_strcat(BUFFER *wb, const char *txt) +{ + // buffer_sprintf(wb, "%s", txt); + + if(unlikely(!txt || !*txt)) return; + + buffer_need_bytes(wb, 1); + + char *s = &wb->buffer[wb->len], *start, *end = &wb->buffer[wb->size]; + size_t len = wb->len; + + start = s; + while(*txt && s != end) + *s++ = *txt++; + + len += s - start; + + wb->len = len; + buffer_overflow_check(wb); + + if(*txt) { + debug(D_WEB_BUFFER, "strcat(): increasing web_buffer at position %zu, size = %zu\n", wb->len, wb->size); + len = strlen(txt); + buffer_increase(wb, len); + buffer_strcat(wb, txt); + } + else { + // terminate the string + // without increasing the length + buffer_need_bytes(wb, (size_t)1); + wb->buffer[wb->len] = '\0'; + } +} + +void buffer_strcat_jsonescape(BUFFER *wb, const char *txt) +{ + while(*txt) { + switch(*txt) { + case '\\': + buffer_need_bytes(wb, 2); + wb->buffer[wb->len++] = '\\'; + wb->buffer[wb->len++] = '\\'; + break; + case '"': + buffer_need_bytes(wb, 2); + wb->buffer[wb->len++] = '\\'; + wb->buffer[wb->len++] = '"'; + break; + default: { + buffer_need_bytes(wb, 1); + wb->buffer[wb->len++] = *txt; + } + } + txt++; + } + + buffer_overflow_check(wb); +} + +void buffer_strcat_htmlescape(BUFFER *wb, const char *txt) +{ + while(*txt) { + switch(*txt) { + case '&': buffer_strcat(wb, "&"); break; + case '<': buffer_strcat(wb, "<"); break; + case '>': buffer_strcat(wb, ">"); break; + case '"': buffer_strcat(wb, """); break; + case '/': buffer_strcat(wb, "/"); break; + case '\'': buffer_strcat(wb, "'"); break; + default: { + buffer_need_bytes(wb, 1); + wb->buffer[wb->len++] = *txt; + } + } + txt++; + } + + buffer_overflow_check(wb); +} + +void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) +{ + if(unlikely(!fmt || !*fmt)) return; + + buffer_need_bytes(wb, len + 1); + + va_list args; + va_start(args, fmt); + wb->len += vsnprintfz(&wb->buffer[wb->len], len, fmt, args); + va_end(args); + + buffer_overflow_check(wb); + + // the buffer is \0 terminated by vsnprintfz +} + +void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args) +{ + if(unlikely(!fmt || !*fmt)) return; + + buffer_need_bytes(wb, 2); + + size_t len = wb->size - wb->len - 1; + + wb->len += vsnprintfz(&wb->buffer[wb->len], len, fmt, args); + + buffer_overflow_check(wb); + + // the buffer is \0 terminated by vsnprintfz +} + +void buffer_sprintf(BUFFER *wb, const char *fmt, ...) +{ + if(unlikely(!fmt || !*fmt)) return; + + va_list args; + size_t wrote = 0, need = 2, multiplier = 0, len; + + do { + need += wrote + multiplier * WEB_DATA_LENGTH_INCREASE_STEP; + multiplier++; + + debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote); + buffer_need_bytes(wb, need); + + len = wb->size - wb->len - 1; + + va_start(args, fmt); + wrote = (size_t) vsnprintfz(&wb->buffer[wb->len], len, fmt, args); + va_end(args); + + } while(wrote >= len); + + wb->len += wrote; + + // the buffer is \0 terminated by vsnprintf +} + + +void buffer_rrd_value(BUFFER *wb, calculated_number value) +{ + buffer_need_bytes(wb, 50); + + if(isnan(value) || isinf(value)) { + buffer_strcat(wb, "null"); + return; + } + else + wb->len += print_calculated_number(&wb->buffer[wb->len], value); + + // terminate it + buffer_need_bytes(wb, 1); + wb->buffer[wb->len] = '\0'; + + buffer_overflow_check(wb); +} + +// generate a javascript date, the fastest possible way... +void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds) +{ + // 10 20 30 = 35 + // 01234567890123456789012345678901234 + // Date(2014,04,01,03,28,20) + + buffer_need_bytes(wb, 30); + + char *b = &wb->buffer[wb->len], *p; + unsigned int *q = (unsigned int *)b; + + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + *q++ = 0x65746144; // "Date" backwards. + #else + *q++ = 0x44617465; // "Date" + #endif + p = (char *)q; + + *p++ = '('; + *p++ = '0' + year / 1000; year %= 1000; + *p++ = '0' + year / 100; year %= 100; + *p++ = '0' + year / 10; + *p++ = '0' + year % 10; + *p++ = ','; + *p = '0' + month / 10; if (*p != '0') p++; + *p++ = '0' + month % 10; + *p++ = ','; + *p = '0' + day / 10; if (*p != '0') p++; + *p++ = '0' + day % 10; + *p++ = ','; + *p = '0' + hours / 10; if (*p != '0') p++; + *p++ = '0' + hours % 10; + *p++ = ','; + *p = '0' + minutes / 10; if (*p != '0') p++; + *p++ = '0' + minutes % 10; + *p++ = ','; + *p = '0' + seconds / 10; if (*p != '0') p++; + *p++ = '0' + seconds % 10; + + unsigned short *r = (unsigned short *)p; + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + *r++ = 0x0029; // ")\0" backwards. + #else + *r++ = 0x2900; // ")\0" + #endif + + wb->len += (size_t)((char *)r - b - 1); + + // terminate it + wb->buffer[wb->len] = '\0'; + buffer_overflow_check(wb); +} + +// generate a date, the fastest possible way... +void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds) +{ + // 10 20 30 = 35 + // 01234567890123456789012345678901234 + // 2014-04-01 03:28:20 + + buffer_need_bytes(wb, 36); + + char *b = &wb->buffer[wb->len]; + char *p = b; + + *p++ = '0' + year / 1000; year %= 1000; + *p++ = '0' + year / 100; year %= 100; + *p++ = '0' + year / 10; + *p++ = '0' + year % 10; + *p++ = '-'; + *p++ = '0' + month / 10; + *p++ = '0' + month % 10; + *p++ = '-'; + *p++ = '0' + day / 10; + *p++ = '0' + day % 10; + *p++ = ' '; + *p++ = '0' + hours / 10; + *p++ = '0' + hours % 10; + *p++ = ':'; + *p++ = '0' + minutes / 10; + *p++ = '0' + minutes % 10; + *p++ = ':'; + *p++ = '0' + seconds / 10; + *p++ = '0' + seconds % 10; + *p = '\0'; + + wb->len += (size_t)(p - b); + + // terminate it + wb->buffer[wb->len] = '\0'; + buffer_overflow_check(wb); +} + +BUFFER *buffer_create(size_t size) +{ + BUFFER *b; + + debug(D_WEB_BUFFER, "Creating new web buffer of size %zu.", size); + + b = callocz(1, sizeof(BUFFER)); + b->buffer = mallocz(size + sizeof(BUFFER_OVERFLOW_EOF) + 2); + b->buffer[0] = '\0'; + b->size = size; + b->contenttype = CT_TEXT_PLAIN; + buffer_overflow_init(b); + buffer_overflow_check(b); + + return(b); +} + +void buffer_free(BUFFER *b) { + if(unlikely(!b)) return; + + buffer_overflow_check(b); + + debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size); + + freez(b->buffer); + freez(b); +} + +void buffer_increase(BUFFER *b, size_t free_size_required) { + buffer_overflow_check(b); + + size_t left = b->size - b->len; + + if(left >= free_size_required) return; + + size_t increase = free_size_required - left; + if(increase < WEB_DATA_LENGTH_INCREASE_STEP) increase = WEB_DATA_LENGTH_INCREASE_STEP; + + debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + increase); + + b->buffer = reallocz(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2); + b->size += increase; + + buffer_overflow_init(b); + buffer_overflow_check(b); +} diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h new file mode 100644 index 0000000..d50910c --- /dev/null +++ b/libnetdata/buffer/buffer.h @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_WEB_BUFFER_H +#define NETDATA_WEB_BUFFER_H 1 + +#include "../libnetdata.h" + +#define WEB_DATA_LENGTH_INCREASE_STEP 1024 + +typedef struct web_buffer { + size_t size; // allocation size of buffer, in bytes + size_t len; // current data length in buffer, in bytes + char *buffer; // the buffer itself + uint8_t contenttype; // the content type of the data in the buffer + uint8_t options; // options related to the content + time_t date; // the timestamp this content has been generated + time_t expires; // the timestamp this content expires +} BUFFER; + +// options +#define WB_CONTENT_CACHEABLE 1 +#define WB_CONTENT_NO_CACHEABLE 2 + +// content-types +#define CT_APPLICATION_JSON 1 +#define CT_TEXT_PLAIN 2 +#define CT_TEXT_HTML 3 +#define CT_APPLICATION_X_JAVASCRIPT 4 +#define CT_TEXT_CSS 5 +#define CT_TEXT_XML 6 +#define CT_APPLICATION_XML 7 +#define CT_TEXT_XSL 8 +#define CT_APPLICATION_OCTET_STREAM 9 +#define CT_APPLICATION_X_FONT_TRUETYPE 10 +#define CT_APPLICATION_X_FONT_OPENTYPE 11 +#define CT_APPLICATION_FONT_WOFF 12 +#define CT_APPLICATION_FONT_WOFF2 13 +#define CT_APPLICATION_VND_MS_FONTOBJ 14 +#define CT_IMAGE_SVG_XML 15 +#define CT_IMAGE_PNG 16 +#define CT_IMAGE_JPG 17 +#define CT_IMAGE_GIF 18 +#define CT_IMAGE_XICON 19 +#define CT_IMAGE_ICNS 20 +#define CT_IMAGE_BMP 21 +#define CT_PROMETHEUS 22 + +#define buffer_cacheable(wb) do { (wb)->options |= WB_CONTENT_CACHEABLE; if((wb)->options & WB_CONTENT_NO_CACHEABLE) (wb)->options &= ~WB_CONTENT_NO_CACHEABLE; } while(0) +#define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE) (wb)->options &= ~WB_CONTENT_CACHEABLE; (wb)->expires = 0; } while(0) + +#define buffer_strlen(wb) ((wb)->len) +extern const char *buffer_tostring(BUFFER *wb); + +#define buffer_flush(wb) wb->buffer[(wb)->len = 0] = '\0' +extern void buffer_reset(BUFFER *wb); + +extern void buffer_strcat(BUFFER *wb, const char *txt); +extern void buffer_rrd_value(BUFFER *wb, calculated_number value); + +extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); +extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); + +extern BUFFER *buffer_create(size_t size); +extern void buffer_free(BUFFER *b); +extern void buffer_increase(BUFFER *b, size_t free_size_required); + +extern void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) PRINTFLIKE(3, 4); +extern void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args); +extern void buffer_sprintf(BUFFER *wb, const char *fmt, ...) PRINTFLIKE(2,3); +extern void buffer_strcat_jsonescape(BUFFER *wb, const char *txt); +extern void buffer_strcat_htmlescape(BUFFER *wb, const char *txt); + +extern void buffer_char_replace(BUFFER *wb, char from, char to); + +extern char *print_number_lu_r(char *str, unsigned long uvalue); +extern char *print_number_llu_r(char *str, unsigned long long uvalue); +extern char *print_number_llu_r_smart(char *str, unsigned long long uvalue); + +extern void buffer_print_llu(BUFFER *wb, unsigned long long uvalue); + +static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) { + if(unlikely(buffer->size - buffer->len < needed_free_size)) + buffer_increase(buffer, needed_free_size); +} + +#endif /* NETDATA_WEB_BUFFER_H */ diff --git a/libnetdata/circular_buffer/Makefile.am b/libnetdata/circular_buffer/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/circular_buffer/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/circular_buffer/README.md b/libnetdata/circular_buffer/README.md new file mode 100644 index 0000000..d36c05f --- /dev/null +++ b/libnetdata/circular_buffer/README.md @@ -0,0 +1,12 @@ +<!-- +title: "circular_buffer" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/circular_buffer/README.md +--> + +# Circular Buffer + +`struct circular_buffer` is an adaptive circular buffer. It will start at an initial size +and grow up to a maximum size as it fills. Two indices within the structure track the current +`read` and `write` position for data. + +[]() diff --git a/libnetdata/circular_buffer/circular_buffer.c b/libnetdata/circular_buffer/circular_buffer.c new file mode 100644 index 0000000..998008d --- /dev/null +++ b/libnetdata/circular_buffer/circular_buffer.c @@ -0,0 +1,85 @@ +#include "../libnetdata.h" + +struct circular_buffer *cbuffer_new(size_t initial, size_t max) { + struct circular_buffer *result = mallocz(sizeof(*result)); + result->size = initial; + result->data = mallocz(initial); + result->write = 0; + result->read = 0; + result->max_size = max; + return result; +} + +void cbuffer_free(struct circular_buffer *buf) { + freez(buf->data); + freez(buf); +} + +static int cbuffer_realloc_unsafe(struct circular_buffer *buf) { + // Check that we can grow + if (buf->size >= buf->max_size) + return 1; + size_t new_size = buf->size * 2; + if (new_size > buf->max_size) + new_size = buf->max_size; + + // We know that: size < new_size <= max_size + // For simplicity align the current data at the bottom of the new buffer + char *new_data = mallocz(new_size); + if (buf->read == buf->write) + buf->write = 0; // buffer is empty + else if (buf->read < buf->write) { + memcpy(new_data, buf->data + buf->read, buf->write - buf->read); + buf->write -= buf->read; + } else { + size_t top_part = buf->size - buf->read; + memcpy(new_data, buf->data + buf->read, top_part); + memcpy(new_data + top_part, buf->data, buf->write); + buf->write = top_part + buf->write; + } + buf->read = 0; + + // Switch buffers + freez(buf->data); + buf->data = new_data; + buf->size = new_size; + return 0; +} + +int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len) { + size_t len = (buf->write >= buf->read) ? (buf->write - buf->read) : (buf->size - buf->read + buf->write); + while (d_len + len >= buf->size) { + if (cbuffer_realloc_unsafe(buf)) { + return 1; + } + } + // Guarantee: write + d_len cannot hit read + if (buf->write + d_len < buf->size) { + memcpy(buf->data + buf->write, d, d_len); + buf->write += d_len; + } + else { + size_t top_part = buf->size - buf->write; + memcpy(buf->data + buf->write, d, top_part); + memcpy(buf->data, d + top_part, d_len - top_part); + buf->write = d_len - top_part; + } + return 0; +} + +// Assume caller does not remove too many bytes (i.e. read will jump over write) +void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num) { + buf->read += num; + // Assume num < size (i.e. caller cannot remove more bytes than are in the buffer) + if (buf->read >= buf->size) + buf->read -= buf->size; +} + +size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start) { + if (start != NULL) + *start = buf->data + buf->read; + if (buf->read <= buf->write) { + return buf->write - buf->read; // Includes empty case + } + return buf->size - buf->read; +} diff --git a/libnetdata/circular_buffer/circular_buffer.h b/libnetdata/circular_buffer/circular_buffer.h new file mode 100644 index 0000000..ba37e0e --- /dev/null +++ b/libnetdata/circular_buffer/circular_buffer.h @@ -0,0 +1,16 @@ +#ifndef CIRCULAR_BUFFER_H +#define CIRCULAR_BUFFER_H 1 + +#include <string.h> + +struct circular_buffer { + size_t size, write, read, max_size; + char *data; +}; + +extern struct circular_buffer *cbuffer_new(size_t initial, size_t max); +extern void cbuffer_free(struct circular_buffer *buf); +extern int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len); +extern void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num); +extern size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start); +#endif diff --git a/libnetdata/clocks/Makefile.am b/libnetdata/clocks/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/clocks/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/clocks/README.md b/libnetdata/clocks/README.md new file mode 100644 index 0000000..e53b86a --- /dev/null +++ b/libnetdata/clocks/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/clocks/README.md +--> + +[](<>) diff --git a/libnetdata/clocks/clocks.c b/libnetdata/clocks/clocks.c new file mode 100644 index 0000000..4ec5fa9 --- /dev/null +++ b/libnetdata/clocks/clocks.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +static int clock_boottime_valid = 1; +static int clock_monotonic_coarse_valid = 1; + +#ifndef HAVE_CLOCK_GETTIME +inline int clock_gettime(clockid_t clk_id, struct timespec *ts) { + struct timeval tv; + if(unlikely(gettimeofday(&tv, NULL) == -1)) { + error("gettimeofday() failed."); + return -1; + } + ts->tv_sec = tv.tv_sec; + ts->tv_nsec = (tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC; + return 0; +} +#endif + +void test_clock_boottime(void) { + struct timespec ts; + if(clock_gettime(CLOCK_BOOTTIME, &ts) == -1 && errno == EINVAL) + clock_boottime_valid = 0; +} + +void test_clock_monotonic_coarse(void) { + struct timespec ts; + if(clock_gettime(CLOCK_MONOTONIC_COARSE, &ts) == -1 && errno == EINVAL) + clock_monotonic_coarse_valid = 0; +} + +static inline time_t now_sec(clockid_t clk_id) { + struct timespec ts; + if(unlikely(clock_gettime(clk_id, &ts) == -1)) { + error("clock_gettime(%d, ×pec) failed.", clk_id); + return 0; + } + return ts.tv_sec; +} + +static inline usec_t now_usec(clockid_t clk_id) { + struct timespec ts; + if(unlikely(clock_gettime(clk_id, &ts) == -1)) { + error("clock_gettime(%d, ×pec) failed.", clk_id); + return 0; + } + return (usec_t)ts.tv_sec * USEC_PER_SEC + (ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC; +} + +static inline int now_timeval(clockid_t clk_id, struct timeval *tv) { + struct timespec ts; + + if(unlikely(clock_gettime(clk_id, &ts) == -1)) { + error("clock_gettime(%d, ×pec) failed.", clk_id); + tv->tv_sec = 0; + tv->tv_usec = 0; + return -1; + } + + tv->tv_sec = ts.tv_sec; + tv->tv_usec = (suseconds_t)((ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC); + return 0; +} + +inline time_t now_realtime_sec(void) { + return now_sec(CLOCK_REALTIME); +} + +inline usec_t now_realtime_usec(void) { + return now_usec(CLOCK_REALTIME); +} + +inline int now_realtime_timeval(struct timeval *tv) { + return now_timeval(CLOCK_REALTIME, tv); +} + +inline time_t now_monotonic_sec(void) { + return now_sec(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC); +} + +inline usec_t now_monotonic_usec(void) { + return now_usec(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC); +} + +inline int now_monotonic_timeval(struct timeval *tv) { + return now_timeval(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC, tv); +} + +inline time_t now_monotonic_high_precision_sec(void) { + return now_sec(CLOCK_MONOTONIC); +} + +inline usec_t now_monotonic_high_precision_usec(void) { + return now_usec(CLOCK_MONOTONIC); +} + +inline int now_monotonic_high_precision_timeval(struct timeval *tv) { + return now_timeval(CLOCK_MONOTONIC, tv); +} + +inline time_t now_boottime_sec(void) { + return now_sec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME : + likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC); +} + +inline usec_t now_boottime_usec(void) { + return now_usec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME : + likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC); +} + +inline int now_boottime_timeval(struct timeval *tv) { + return now_timeval(likely(clock_boottime_valid) ? CLOCK_BOOTTIME : + likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC, + tv); +} + +inline usec_t timeval_usec(struct timeval *tv) { + return (usec_t)tv->tv_sec * USEC_PER_SEC + (tv->tv_usec % USEC_PER_SEC); +} + +inline msec_t timeval_msec(struct timeval *tv) { + return (msec_t)tv->tv_sec * MSEC_PER_SEC + ((tv->tv_usec % USEC_PER_SEC) / MSEC_PER_SEC); +} + +inline susec_t dt_usec_signed(struct timeval *now, struct timeval *old) { + usec_t ts1 = timeval_usec(now); + usec_t ts2 = timeval_usec(old); + + if(likely(ts1 >= ts2)) return (susec_t)(ts1 - ts2); + return -((susec_t)(ts2 - ts1)); +} + +inline usec_t dt_usec(struct timeval *now, struct timeval *old) { + usec_t ts1 = timeval_usec(now); + usec_t ts2 = timeval_usec(old); + return (ts1 > ts2) ? (ts1 - ts2) : (ts2 - ts1); +} + +inline void heartbeat_init(heartbeat_t *hb) +{ + hb->monotonic = hb->realtime = 0ULL; +} + +// waits for the next heartbeat +// it waits using the monotonic clock +// it returns the dt using the realtime clock + +usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) { + heartbeat_t now; + now.monotonic = now_monotonic_usec(); + now.realtime = now_realtime_usec(); + + usec_t next_monotonic = now.monotonic - (now.monotonic % tick) + tick; + + while(now.monotonic < next_monotonic) { + sleep_usec(next_monotonic - now.monotonic); + now.monotonic = now_monotonic_usec(); + now.realtime = now_realtime_usec(); + } + + if(likely(hb->realtime != 0ULL)) { + usec_t dt_monotonic = now.monotonic - hb->monotonic; + usec_t dt_realtime = now.realtime - hb->realtime; + + hb->monotonic = now.monotonic; + hb->realtime = now.realtime; + + if(unlikely(dt_monotonic >= tick + tick / 2)) { + errno = 0; + error("heartbeat missed %llu monotonic microseconds", dt_monotonic - tick); + } + + return dt_realtime; + } + else { + hb->monotonic = now.monotonic; + hb->realtime = now.realtime; + return 0ULL; + } +} + +// returned the elapsed time, since the last heartbeat +// using the monotonic clock + +inline usec_t heartbeat_monotonic_dt_to_now_usec(heartbeat_t *hb) { + if(!hb || !hb->monotonic) return 0ULL; + return now_monotonic_usec() - hb->monotonic; +} + +int sleep_usec(usec_t usec) { + +#ifndef NETDATA_WITH_USLEEP + // we expect microseconds (1.000.000 per second) + // but timespec is nanoseconds (1.000.000.000 per second) + struct timespec rem, req = { + .tv_sec = (time_t) (usec / 1000000), + .tv_nsec = (suseconds_t) ((usec % 1000000) * 1000) + }; + + while (nanosleep(&req, &rem) == -1) { + if (likely(errno == EINTR)) { + debug(D_SYSTEM, "nanosleep() interrupted (while sleeping for %llu microseconds).", usec); + req.tv_sec = rem.tv_sec; + req.tv_nsec = rem.tv_nsec; + } else { + error("Cannot nanosleep() for %llu microseconds.", usec); + break; + } + } + + return 0; +#else + int ret = usleep(usec); + if(unlikely(ret == -1 && errno == EINVAL)) { + // on certain systems, usec has to be up to 999999 + if(usec > 999999) { + int counter = usec / 999999; + while(counter--) + usleep(999999); + + usleep(usec % 999999); + } + else { + error("Cannot usleep() for %llu microseconds.", usec); + return ret; + } + } + + if(ret != 0) + error("usleep() failed for %llu microseconds.", usec); + + return ret; +#endif +} + +static inline collected_number uptime_from_boottime(void) { +#ifdef CLOCK_BOOTTIME_IS_AVAILABLE + return now_boottime_usec() / 1000; +#else + error("uptime cannot be read from CLOCK_BOOTTIME on this system."); + return 0; +#endif +} + +static procfile *read_proc_uptime_ff = NULL; +static inline collected_number read_proc_uptime(char *filename) { + if(unlikely(!read_proc_uptime_ff)) { + read_proc_uptime_ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!read_proc_uptime_ff)) return 0; + } + + read_proc_uptime_ff = procfile_readall(read_proc_uptime_ff); + if(unlikely(!read_proc_uptime_ff)) return 0; + + if(unlikely(procfile_lines(read_proc_uptime_ff) < 1)) { + error("/proc/uptime has no lines."); + return 0; + } + if(unlikely(procfile_linewords(read_proc_uptime_ff, 0) < 1)) { + error("/proc/uptime has less than 1 word in it."); + return 0; + } + + return (collected_number)(strtold(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0); +} + +inline collected_number uptime_msec(char *filename){ + static int use_boottime = -1; + + if(unlikely(use_boottime == -1)) { + collected_number uptime_boottime = uptime_from_boottime(); + collected_number uptime_proc = read_proc_uptime(filename); + + long long delta = (long long)uptime_boottime - (long long)uptime_proc; + if(delta < 0) delta = -delta; + + if(delta <= 1000 && uptime_boottime != 0) { + procfile_close(read_proc_uptime_ff); + info("Using now_boottime_usec() for uptime (dt is %lld ms)", delta); + use_boottime = 1; + } + else if(uptime_proc != 0) { + info("Using /proc/uptime for uptime (dt is %lld ms)", delta); + use_boottime = 0; + } + else { + error("Cannot find any way to read uptime on this system."); + return 1; + } + } + + collected_number uptime; + if(use_boottime) + uptime = uptime_from_boottime(); + else + uptime = read_proc_uptime(filename); + + return uptime; +} diff --git a/libnetdata/clocks/clocks.h b/libnetdata/clocks/clocks.h new file mode 100644 index 0000000..cfe99f5 --- /dev/null +++ b/libnetdata/clocks/clocks.h @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CLOCKS_H +#define NETDATA_CLOCKS_H 1 + +#include "../libnetdata.h" + +#ifndef HAVE_STRUCT_TIMESPEC +struct timespec { + time_t tv_sec; /* seconds */ + long tv_nsec; /* nanoseconds */ +}; +#endif + +#ifndef HAVE_CLOCKID_T +typedef int clockid_t; +#endif + +typedef unsigned long long nsec_t; +typedef unsigned long long msec_t; +typedef unsigned long long usec_t; +typedef long long susec_t; + +typedef struct heartbeat { + usec_t monotonic; + usec_t realtime; +} heartbeat_t; + +/* Linux value is as good as any other */ +#ifndef CLOCK_REALTIME +#define CLOCK_REALTIME 0 +#endif + +#ifndef CLOCK_MONOTONIC +/* fallback to CLOCK_REALTIME if not available */ +#define CLOCK_MONOTONIC CLOCK_REALTIME +#endif + +/* Prefer CLOCK_MONOTONIC_COARSE where available to reduce overhead. It has the same semantics as CLOCK_MONOTONIC */ +#ifndef CLOCK_MONOTONIC_COARSE +/* fallback to CLOCK_MONOTONIC if not available */ +#define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC +#endif + +#ifndef CLOCK_BOOTTIME + +#ifdef CLOCK_UPTIME +/* CLOCK_BOOTTIME falls back to CLOCK_UPTIME on FreeBSD */ +#define CLOCK_BOOTTIME CLOCK_UPTIME +#else // CLOCK_UPTIME +/* CLOCK_BOOTTIME falls back to CLOCK_MONOTONIC */ +#define CLOCK_BOOTTIME CLOCK_MONOTONIC_COARSE +#endif // CLOCK_UPTIME + +#else // CLOCK_BOOTTIME + +#ifdef HAVE_CLOCK_GETTIME +#define CLOCK_BOOTTIME_IS_AVAILABLE 1 // required for /proc/uptime +#endif // HAVE_CLOCK_GETTIME + +#endif // CLOCK_BOOTTIME + +#ifndef NSEC_PER_MSEC +#define NSEC_PER_MSEC 1000000ULL +#endif + +#ifndef NSEC_PER_SEC +#define NSEC_PER_SEC 1000000000ULL +#endif +#ifndef NSEC_PER_USEC +#define NSEC_PER_USEC 1000ULL +#endif + +#ifndef USEC_PER_SEC +#define USEC_PER_SEC 1000000ULL +#endif +#ifndef MSEC_PER_SEC +#define MSEC_PER_SEC 1000ULL +#endif + +#define USEC_PER_MS 1000ULL + +#ifndef HAVE_CLOCK_GETTIME +/* Fallback function for POSIX.1-2001 clock_gettime() function. + * + * We use a realtime clock from gettimeofday(), this will + * make systems without clock_gettime() support sensitive + * to time jumps or hibernation/suspend side effects. + */ +extern int clock_gettime(clockid_t clk_id, struct timespec *ts); +#endif + +/* + * Three clocks are available (cf. man 3 clock_gettime): + * + * REALTIME clock (i.e. wall-clock): + * This clock is affected by discontinuous jumps in the system time + * (e.g., if the system administrator manually changes the clock), and by the incremental adjustments performed by adjtime(3) and NTP. + * + * MONOTONIC clock + * Clock that cannot be set and represents monotonic time since some unspecified starting point. + * This clock is not affected by discontinuous jumps in the system time + * (e.g., if the system administrator manually changes the clock), but is affected by the incremental adjustments performed by adjtime(3) and NTP. + * If not available on the system, this clock falls back to REALTIME clock. + * + * BOOTTIME clock + * Identical to CLOCK_MONOTONIC, except it also includes any time that the system is suspended. + * This allows applications to get a suspend-aware monotonic clock without having to deal with the complications of CLOCK_REALTIME, + * which may have discontinuities if the time is changed using settimeofday(2). + * If not available on the system, this clock falls back to MONOTONIC clock. + * + * All now_*_timeval() functions fill the `struct timeval` with the time from the appropriate clock. + * Those functions return 0 on success, -1 else with errno set appropriately. + * + * All now_*_sec() functions return the time in seconds from the approriate clock, or 0 on error. + * All now_*_usec() functions return the time in microseconds from the approriate clock, or 0 on error. + * + * Most functions will attempt to use CLOCK_MONOTONIC_COARSE if available to reduce contention overhead and improve + * performance scaling. If high precision is required please use one of the available now_*_high_precision_* functions. + */ +extern int now_realtime_timeval(struct timeval *tv); +extern time_t now_realtime_sec(void); +extern usec_t now_realtime_usec(void); + +extern int now_monotonic_timeval(struct timeval *tv); +extern time_t now_monotonic_sec(void); +extern usec_t now_monotonic_usec(void); +extern int now_monotonic_high_precision_timeval(struct timeval *tv); +extern time_t now_monotonic_high_precision_sec(void); +extern usec_t now_monotonic_high_precision_usec(void); + +extern int now_boottime_timeval(struct timeval *tv); +extern time_t now_boottime_sec(void); +extern usec_t now_boottime_usec(void); + + +extern usec_t timeval_usec(struct timeval *tv); +extern msec_t timeval_msec(struct timeval *tv); + +extern usec_t dt_usec(struct timeval *now, struct timeval *old); +extern susec_t dt_usec_signed(struct timeval *now, struct timeval *old); + +extern void heartbeat_init(heartbeat_t *hb); + +/* Sleeps until next multiple of tick using monotonic clock. + * Returns elapsed time in microseconds since previous heartbeat + */ +extern usec_t heartbeat_next(heartbeat_t *hb, usec_t tick); + +/* Returns elapsed time in microseconds since last heartbeat */ +extern usec_t heartbeat_monotonic_dt_to_now_usec(heartbeat_t *hb); + +extern int sleep_usec(usec_t usec); + +/* + * When running a binary with CLOCK_BOOTTIME defined on a system with a linux kernel older than Linux 2.6.39 the + * clock_gettime(2) system call fails with EINVAL. In that case it must fall-back to CLOCK_MONOTONIC. + */ +void test_clock_boottime(void); + +/* + * When running a binary with CLOCK_MONOTONIC_COARSE defined on a system with a linux kernel older than Linux 2.6.32 the + * clock_gettime(2) system call fails with EINVAL. In that case it must fall-back to CLOCK_MONOTONIC. + */ +void test_clock_monotonic_coarse(void); + +extern collected_number uptime_msec(char *filename); + +#endif /* NETDATA_CLOCKS_H */ diff --git a/libnetdata/config/Makefile.am b/libnetdata/config/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/config/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/config/README.md b/libnetdata/config/README.md new file mode 100644 index 0000000..a71f1ee --- /dev/null +++ b/libnetdata/config/README.md @@ -0,0 +1,54 @@ +<!-- +title: "Netdata ini config files" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/config/README.md +--> + +# Netdata ini config files + +Configuration files `netdata.conf` and `stream.conf` are Netdata ini files. + +## Motivation + +The whole idea came up when we were evaluating the documentation involved +in maintaining a complex configuration system. Our intention was to give +configuration options for everything imaginable. But then, documenting all +these options would require a tremendous amount of time, users would have +to search through endless pages for the option they need, etc. + +We concluded then that **configuring software like that is a waste of time +and effort**. Of course there must be plenty of configuration options, but +the implementation itself should require a lot less effort for both the +developers and the users. + +So, we did this: + +1. No configuration is required to run Netdata +2. There are plenty of options to tweak +3. There is minimal documentation (or no at all) + +## Why this works? + +The configuration file is a `name = value` dictionary with `[sections]`. +Write whatever you like there as long as it follows this simple format. + +Netdata loads this dictionary and then when the code needs a value from +it, it just looks up the `name` in the dictionary at the proper `section`. +In all places, in the code, there are both the `names` and their +`default values`, so if something is not found in the configuration +file, the default is used. The lookup is made using B-Trees and hashes +(no string comparisons), so they are super fast. Also the `names` of the +settings can be `my super duper setting that once set to yes, will turn the world upside down = no` + +- so goodbye to most of the documentation involved. + +Next, Netdata can generate a valid configuration for the user to edit. +No need to remember anything or copy and paste settings. Just get the +configuration from the server (`/netdata.conf` on your Netdata server), +edit it and save it. + +Last, what about options you believe you have set, but you misspelled? +When you get the configuration file from the server, there will be a +comment above all `name = value` pairs the server does not use. +So you know that whatever you wrote there, is not used. + +[](<>) diff --git a/libnetdata/config/appconfig.c b/libnetdata/config/appconfig.c new file mode 100644 index 0000000..d9dcde7 --- /dev/null +++ b/libnetdata/config/appconfig.c @@ -0,0 +1,883 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +/* + * @Input: + * Connector / instance to add to an internal structure + * @Return + * The current head of the linked list of connector_instance + * + */ + +_CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance) +{ + static struct _connector_instance *global_connector_instance = NULL; + struct _connector_instance *local_ci, *local_ci_tmp; + + if (unlikely(!connector)) { + if (unlikely(!instance)) + return global_connector_instance; + + local_ci = global_connector_instance; + while (local_ci) { + local_ci_tmp = local_ci->next; + freez(local_ci); + local_ci = local_ci_tmp; + } + global_connector_instance = NULL; + return NULL; + } + + local_ci = callocz(1, sizeof(struct _connector_instance)); + local_ci->instance = instance; + local_ci->connector = connector; + strncpy(local_ci->instance_name, instance->name, CONFIG_MAX_NAME); + strncpy(local_ci->connector_name, connector->name, CONFIG_MAX_NAME); + local_ci->next = global_connector_instance; + global_connector_instance = local_ci; + + return global_connector_instance; +} + +int is_valid_connector(char *type, int check_reserved) +{ + int rc = 1; + + if (unlikely(!type)) + return 0; + + if (!check_reserved) { + if (unlikely(is_valid_connector(type,1))) { + return 0; + } + //if (unlikely(*type == ':') + // return 0; + char *separator = strrchr(type, ':'); + if (likely(separator)) { + *separator = '\0'; + rc = separator - type; + } else + return 0; + } +// else { +// if (unlikely(is_valid_connector(type,1))) { +// error("Section %s invalid -- reserved name", type); +// return 0; +// } +// } + + if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) { + return rc; + } else if (!strcmp(type, "graphite:http") || !strcmp(type, "graphite:https")) { + return rc; + } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) { + return rc; + } else if (!strcmp(type, "json:http") || !strcmp(type, "json:https")) { + return rc; + } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) { + return rc; + } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) { + return rc; + } else if (!strcmp(type, "prometheus_remote_write")) { + return rc; + } else if (!strcmp(type, "prometheus_remote_write:http") || !strcmp(type, "prometheus_remote_write:https")) { + return rc; + } else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) { + return rc; + } else if (!strcmp(type, "pubsub") || !strcmp(type, "pubsub:plaintext")) { + return rc; + } else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) { + return rc; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// locking + +inline void appconfig_wrlock(struct config *root) { + netdata_mutex_lock(&root->mutex); +} + +inline void appconfig_unlock(struct config *root) { + netdata_mutex_unlock(&root->mutex); +} + +inline void config_section_wrlock(struct section *co) { + netdata_mutex_lock(&co->mutex); +} + +inline void config_section_unlock(struct section *co) { + netdata_mutex_unlock(&co->mutex); +} + + +// ---------------------------------------------------------------------------- +// config name-value index + +static int appconfig_option_compare(void *a, void *b) { + if(((struct config_option *)a)->hash < ((struct config_option *)b)->hash) return -1; + else if(((struct config_option *)a)->hash > ((struct config_option *)b)->hash) return 1; + else return strcmp(((struct config_option *)a)->name, ((struct config_option *)b)->name); +} + +#define appconfig_option_index_add(co, cv) (struct config_option *)avl_insert_lock(&((co)->values_index), (avl *)(cv)) +#define appconfig_option_index_del(co, cv) (struct config_option *)avl_remove_lock(&((co)->values_index), (avl *)(cv)) + +static struct config_option *appconfig_option_index_find(struct section *co, const char *name, uint32_t hash) { + struct config_option tmp; + tmp.hash = (hash)?hash:simple_hash(name); + tmp.name = (char *)name; + + return (struct config_option *)avl_search_lock(&(co->values_index), (avl *) &tmp); +} + + +// ---------------------------------------------------------------------------- +// config sections index + +int appconfig_section_compare(void *a, void *b) { + if(((struct section *)a)->hash < ((struct section *)b)->hash) return -1; + else if(((struct section *)a)->hash > ((struct section *)b)->hash) return 1; + else return strcmp(((struct section *)a)->name, ((struct section *)b)->name); +} + +#define appconfig_index_add(root, cfg) (struct section *)avl_insert_lock(&(root)->index, (avl *)(cfg)) +#define appconfig_index_del(root, cfg) (struct section *)avl_remove_lock(&(root)->index, (avl *)(cfg)) + +static struct section *appconfig_index_find(struct config *root, const char *name, uint32_t hash) { + struct section tmp; + tmp.hash = (hash)?hash:simple_hash(name); + tmp.name = (char *)name; + + return (struct section *)avl_search_lock(&root->index, (avl *) &tmp); +} + + +// ---------------------------------------------------------------------------- +// config section methods + +static inline struct section *appconfig_section_find(struct config *root, const char *section) { + return appconfig_index_find(root, section, 0); +} + +static inline struct section *appconfig_section_create(struct config *root, const char *section) { + debug(D_CONFIG, "Creating section '%s'.", section); + + struct section *co = callocz(1, sizeof(struct section)); + co->name = strdupz(section); + co->hash = simple_hash(co->name); + netdata_mutex_init(&co->mutex); + + avl_init_lock(&co->values_index, appconfig_option_compare); + + if(unlikely(appconfig_index_add(root, co) != co)) + error("INTERNAL ERROR: indexing of section '%s', already exists.", co->name); + + appconfig_wrlock(root); + struct section *co2 = root->last_section; + if(co2) { + co2->next = co; + } else { + root->first_section = co; + } + root->last_section = co; + appconfig_unlock(root); + + return co; +} + +void appconfig_section_destroy_non_loaded(struct config *root, const char *section) +{ + struct section *co; + struct config_option *cv, *cv_next; + + debug(D_CONFIG, "Destroying section '%s'.", section); + + co = appconfig_section_find(root, section); + if(!co) { + error("Could not destroy section '%s'. Not found.", section); + return; + } + + config_section_wrlock(co); + for(cv = co->values; cv ; cv = cv->next) { + if (cv->flags & CONFIG_VALUE_LOADED) { + /* Do not destroy values that were loaded from the configuration files. */ + config_section_unlock(co); + return; + } + } + for(cv = co->values ; cv ; cv = cv_next) { + cv_next = cv->next; + if(unlikely(!appconfig_option_index_del(co, cv))) + error("Cannot remove config option '%s' from section '%s'.", cv->name, co->name); + freez(cv->value); + freez(cv->name); + freez(cv); + } + co->values = NULL; + config_section_unlock(co); + + if (unlikely(!appconfig_index_del(root, co))) { + error("Cannot remove section '%s' from config.", section); + return; + } + + avl_destroy_lock(&co->values_index); + freez(co->name); + pthread_mutex_destroy(&co->mutex); + freez(co); +} + + +// ---------------------------------------------------------------------------- +// config name-value methods + +static inline struct config_option *appconfig_value_create(struct section *co, const char *name, const char *value) { + debug(D_CONFIG, "Creating config entry for name '%s', value '%s', in section '%s'.", name, value, co->name); + + struct config_option *cv = callocz(1, sizeof(struct config_option)); + cv->name = strdupz(name); + cv->hash = simple_hash(cv->name); + cv->value = strdupz(value); + + struct config_option *found = appconfig_option_index_add(co, cv); + if(found != cv) { + error("indexing of config '%s' in section '%s': already exists - using the existing one.", cv->name, co->name); + freez(cv->value); + freez(cv->name); + freez(cv); + return found; + } + + config_section_wrlock(co); + struct config_option *cv2 = co->values; + if(cv2) { + while (cv2->next) cv2 = cv2->next; + cv2->next = cv; + } + else co->values = cv; + config_section_unlock(co); + + return cv; +} + +int appconfig_exists(struct config *root, const char *section, const char *name) { + struct config_option *cv; + + debug(D_CONFIG, "request to get config in section '%s', name '%s'", section, name); + + struct section *co = appconfig_section_find(root, section); + if(!co) return 0; + + cv = appconfig_option_index_find(co, name, 0); + if(!cv) return 0; + + return 1; +} + +int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new) { + struct config_option *cv_old, *cv_new; + int ret = -1; + + debug(D_CONFIG, "request to rename config in section '%s', old name '%s', to section '%s', new name '%s'", section_old, name_old, section_new, name_new); + + struct section *co_old = appconfig_section_find(root, section_old); + if(!co_old) return ret; + + struct section *co_new = appconfig_section_find(root, section_new); + if(!co_new) co_new = appconfig_section_create(root, section_new); + + config_section_wrlock(co_old); + if(co_old != co_new) + config_section_wrlock(co_new); + + cv_old = appconfig_option_index_find(co_old, name_old, 0); + if(!cv_old) goto cleanup; + + cv_new = appconfig_option_index_find(co_new, name_new, 0); + if(cv_new) goto cleanup; + + if(unlikely(appconfig_option_index_del(co_old, cv_old) != cv_old)) + error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted tge wrong config entry.", cv_old->name, co_old->name); + + if(co_old->values == cv_old) { + co_old->values = cv_old->next; + } + else { + struct config_option *t; + for(t = co_old->values; t && t->next != cv_old ;t = t->next) ; + if(!t || t->next != cv_old) + error("INTERNAL ERROR: cannot find variable '%s' in section '%s' of the config - but it should be there.", cv_old->name, co_old->name); + else + t->next = cv_old->next; + } + + freez(cv_old->name); + cv_old->name = strdupz(name_new); + cv_old->hash = simple_hash(cv_old->name); + + cv_new = cv_old; + cv_new->next = co_new->values; + co_new->values = cv_new; + + if(unlikely(appconfig_option_index_add(co_new, cv_old) != cv_old)) + error("INTERNAL ERROR: re-indexing of config '%s' in section '%s', already exists.", cv_old->name, co_new->name); + + ret = 0; + +cleanup: + if(co_old != co_new) + config_section_unlock(co_new); + config_section_unlock(co_old); + return ret; +} + +char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value) +{ + struct config_option *cv; + + // Only calls internal to this file check for a NULL result and they do not supply a NULL arg. + // External caller should treat NULL as an error case. + cv = appconfig_option_index_find(co, name, 0); + if (!cv) { + if (!default_value) return NULL; + cv = appconfig_value_create(co, name, default_value); + if (!cv) return NULL; + } + cv->flags |= CONFIG_VALUE_USED; + + if((cv->flags & CONFIG_VALUE_LOADED) || (cv->flags & CONFIG_VALUE_CHANGED)) { + // this is a loaded value from the config file + // if it is different than the default, mark it + if(!(cv->flags & CONFIG_VALUE_CHECKED)) { + if(default_value && strcmp(cv->value, default_value) != 0) cv->flags |= CONFIG_VALUE_CHANGED; + cv->flags |= CONFIG_VALUE_CHECKED; + } + } + + return(cv->value); +} + + +char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value) +{ + if (default_value == NULL) + debug(D_CONFIG, "request to get config in section '%s', name '%s' or fail", section, name); + else + debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value); + + struct section *co = appconfig_section_find(root, section); + if (!co && !default_value) + return NULL; + if(!co) co = appconfig_section_create(root, section); + + return appconfig_get_by_section(co, name, default_value); +} + +long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value) +{ + char buffer[100], *s; + sprintf(buffer, "%lld", value); + + s = appconfig_get(root, section, name, buffer); + if(!s) return value; + + return strtoll(s, NULL, 0); +} + +LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value) +{ + char buffer[100], *s; + sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value); + + s = appconfig_get(root, section, name, buffer); + if(!s) return value; + + return str2ld(s, NULL); +} + +static inline int appconfig_test_boolean_value(char *s) { + if(!strcasecmp(s, "yes") || !strcasecmp(s, "true") || !strcasecmp(s, "on") + || !strcasecmp(s, "auto") || !strcasecmp(s, "on demand")) + return 1; + + return 0; +} + +int appconfig_get_boolean_by_section(struct section *co, const char *name, int value) { + char *s; + + s = appconfig_get_by_section(co, name, (!value)?"no":"yes"); + if(!s) return value; + + return appconfig_test_boolean_value(s); +} + +int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value) +{ + char *s; + if(value) s = "yes"; + else s = "no"; + + s = appconfig_get(root, section, name, s); + if(!s) return value; + + return appconfig_test_boolean_value(s); +} + +int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value) +{ + char *s; + + if(value == CONFIG_BOOLEAN_AUTO) + s = "auto"; + + else if(value == CONFIG_BOOLEAN_NO) + s = "no"; + + else + s = "yes"; + + s = appconfig_get(root, section, name, s); + if(!s) return value; + + if(!strcmp(s, "yes")) + return CONFIG_BOOLEAN_YES; + else if(!strcmp(s, "no")) + return CONFIG_BOOLEAN_NO; + else if(!strcmp(s, "auto") || !strcmp(s, "on demand")) + return CONFIG_BOOLEAN_AUTO; + + return value; +} + +const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value) +{ + struct config_option *cv; + + debug(D_CONFIG, "request to set default config in section '%s', name '%s', value '%s'", section, name, value); + + struct section *co = appconfig_section_find(root, section); + if(!co) return appconfig_set(root, section, name, value); + + cv = appconfig_option_index_find(co, name, 0); + if(!cv) return appconfig_set(root, section, name, value); + + cv->flags |= CONFIG_VALUE_USED; + + if(cv->flags & CONFIG_VALUE_LOADED) + return cv->value; + + if(strcmp(cv->value, value) != 0) { + cv->flags |= CONFIG_VALUE_CHANGED; + + freez(cv->value); + cv->value = strdupz(value); + } + + return cv->value; +} + +const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value) +{ + struct config_option *cv; + + debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value); + + struct section *co = appconfig_section_find(root, section); + if(!co) co = appconfig_section_create(root, section); + + cv = appconfig_option_index_find(co, name, 0); + if(!cv) cv = appconfig_value_create(co, name, value); + cv->flags |= CONFIG_VALUE_USED; + + if(strcmp(cv->value, value) != 0) { + cv->flags |= CONFIG_VALUE_CHANGED; + + freez(cv->value); + cv->value = strdupz(value); + } + + return value; +} + +long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value) +{ + char buffer[100]; + sprintf(buffer, "%lld", value); + + appconfig_set(root, section, name, buffer); + + return value; +} + +LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value) +{ + char buffer[100]; + sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value); + + appconfig_set(root, section, name, buffer); + + return value; +} + +int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value) +{ + char *s; + if(value) s = "yes"; + else s = "no"; + + appconfig_set(root, section, name, s); + + return value; +} + +int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value) +{ + int result = 0; + const char *s; + + s = appconfig_get(root, section, name, value); + if(!s) goto fallback; + + if(!config_parse_duration(s, &result)) { + error("config option '[%s].%s = %s' is configured with an valid duration", section, name, s); + goto fallback; + } + + return result; + + fallback: + if(!config_parse_duration(value, &result)) + error("INTERNAL ERROR: default duration supplied for option '[%s].%s = %s' is not a valid duration", section, name, value); + + return result; +} + +// ---------------------------------------------------------------------------- +// config load/save + +int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name) +{ + int line = 0; + struct section *co = NULL; + int is_exporter_config = 0; + int _backends = 0; // number of backend sections we have + char working_instance[CONFIG_MAX_NAME + 1]; + char working_connector[CONFIG_MAX_NAME + 1]; + struct section *working_connector_section = NULL; + int global_exporting_section = 0; + + char buffer[CONFIG_FILE_LINE_MAX + 1], *s; + + if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME; + + debug(D_CONFIG, "CONFIG: opening config file '%s'", filename); + + FILE *fp = fopen(filename, "r"); + if(!fp) { + // info("CONFIG: cannot open file '%s'. Using internal defaults.", filename); + return 0; + } + + uint32_t section_hash = 0; + if(section_name) { + section_hash = simple_hash(section_name); + } + is_exporter_config = (strstr(filename, EXPORTING_CONF) != NULL); + + while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) { + buffer[CONFIG_FILE_LINE_MAX] = '\0'; + line++; + + s = trim(buffer); + if(!s || *s == '#') { + debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename); + continue; + } + + int len = (int) strlen(s); + if(*s == '[' && s[len - 1] == ']') { + // new section + s[len - 1] = '\0'; + s++; + + if (is_exporter_config) { + global_exporting_section = + !(strcmp(s, CONFIG_SECTION_EXPORTING)) || !(strcmp(s, CONFIG_SECTION_PROMETHEUS)); + if (unlikely(!global_exporting_section)) { + int rc; + rc = is_valid_connector(s, 0); + if (likely(rc)) { + strncpy(working_connector, s, CONFIG_MAX_NAME); + s = s + rc + 1; + if (unlikely(!(*s))) { + _backends++; + sprintf(buffer, "instance_%d", _backends); + s = buffer; + } + strncpy(working_instance, s, CONFIG_MAX_NAME); + working_connector_section = NULL; + if (unlikely(appconfig_section_find(root, working_instance))) { + error("Instance (%s) already exists", working_instance); + co = NULL; + continue; + } + } else { + co = NULL; + error("Section (%s) does not specify a valid connector", s); + continue; + } + } + } + + co = appconfig_section_find(root, s); + if(!co) co = appconfig_section_create(root, s); + + if(co && section_name && overwrite_used && section_hash == co->hash && !strcmp(section_name, co->name)) { + config_section_wrlock(co); + struct config_option *cv2 = co->values; + while (cv2) { + struct config_option *save = cv2->next; + struct config_option *found = appconfig_option_index_del(co, cv2); + if(found != cv2) + error("INTERNAL ERROR: Cannot remove '%s' from section '%s', it was not inserted before.", + cv2->name, co->name); + + freez(cv2->name); + freez(cv2->value); + freez(cv2); + cv2 = save; + } + co->values = NULL; + config_section_unlock(co); + } + + continue; + } + + if(!co) { + // line outside a section + error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename); + continue; + } + + if(section_name && overwrite_used && section_hash != co->hash && strcmp(section_name, co->name)) { + continue; + } + + char *name = s; + char *value = strchr(s, '='); + if(!value) { + error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename); + continue; + } + *value = '\0'; + value++; + + name = trim(name); + value = trim(value); + + if(!name || *name == '#') { + error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename); + continue; + } + + if(!value) value = ""; + + struct config_option *cv = appconfig_option_index_find(co, name, 0); + + if (!cv) { + cv = appconfig_value_create(co, name, value); + if (likely(is_exporter_config) && unlikely(!global_exporting_section)) { + if (unlikely(!working_connector_section)) { + working_connector_section = appconfig_section_find(root, working_connector); + if (!working_connector_section) + working_connector_section = appconfig_section_create(root, working_connector); + if (likely(working_connector_section)) { + add_connector_instance(working_connector_section, co); + } + } + } + } else { + if (((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) { + debug( + D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name); + freez(cv->value); + cv->value = strdupz(value); + } else + debug( + D_CONFIG, + "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.", + line, + filename, + co->name, + cv->name); + } + cv->flags |= CONFIG_VALUE_LOADED; + } + + fclose(fp); + + return 1; +} + +void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) +{ + int i, pri; + struct section *co; + struct config_option *cv; + + for(i = 0; i < 3 ;i++) { + switch(i) { + case 0: + buffer_strcat(wb, + "# netdata configuration\n" + "#\n" + "# You can download the latest version of this file, using:\n" + "#\n" + "# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n" + "# or\n" + "# curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n" + "#\n" + "# You can uncomment and change any of the options below.\n" + "# The value shown in the commented settings, is the default value.\n" + "#\n" + "\n# global netdata configuration\n"); + break; + + case 1: + buffer_strcat(wb, "\n\n# per plugin configuration\n"); + break; + + case 2: + buffer_strcat(wb, "\n\n# per chart configuration\n"); + break; + } + + appconfig_wrlock(root); + for(co = root->first_section; co ; co = co->next) { + if(!strcmp(co->name, CONFIG_SECTION_GLOBAL) + || !strcmp(co->name, CONFIG_SECTION_WEB) + || !strcmp(co->name, CONFIG_SECTION_STATSD) + || !strcmp(co->name, CONFIG_SECTION_PLUGINS) + || !strcmp(co->name, CONFIG_SECTION_CLOUD) + || !strcmp(co->name, CONFIG_SECTION_REGISTRY) + || !strcmp(co->name, CONFIG_SECTION_HEALTH) + || !strcmp(co->name, CONFIG_SECTION_BACKEND) + || !strcmp(co->name, CONFIG_SECTION_STREAM) + || !strcmp(co->name, CONFIG_SECTION_HOST_LABEL) + ) + pri = 0; + else if(!strncmp(co->name, "plugin:", 7)) pri = 1; + else pri = 2; + + if(i == pri) { + int loaded = 0; + int used = 0; + int changed = 0; + int count = 0; + + config_section_wrlock(co); + for(cv = co->values; cv ; cv = cv->next) { + used += (cv->flags & CONFIG_VALUE_USED)?1:0; + loaded += (cv->flags & CONFIG_VALUE_LOADED)?1:0; + changed += (cv->flags & CONFIG_VALUE_CHANGED)?1:0; + count++; + } + config_section_unlock(co); + + if(!count) continue; + if(only_changed && !changed && !loaded) continue; + + if(!used) { + buffer_sprintf(wb, "\n# section '%s' is not used.", co->name); + } + + buffer_sprintf(wb, "\n[%s]\n", co->name); + + config_section_wrlock(co); + for(cv = co->values; cv ; cv = cv->next) { + + if(used && !(cv->flags & CONFIG_VALUE_USED)) { + buffer_sprintf(wb, "\n\t# option '%s' is not used.\n", cv->name); + } + buffer_sprintf(wb, "\t%s%s = %s\n", ((!(cv->flags & CONFIG_VALUE_LOADED)) && (!(cv->flags & CONFIG_VALUE_CHANGED)) && (cv->flags & CONFIG_VALUE_USED))?"# ":"", cv->name, cv->value); + } + config_section_unlock(co); + } + } + appconfig_unlock(root); + } +} + +/** + * Parse Duration + * + * Parse the string setting the result + * + * @param string the timestamp string + * @param result the output variable + * + * @return It returns 1 on success and 0 otherwise + */ +int config_parse_duration(const char* string, int* result) { + while(*string && isspace(*string)) string++; + + if(unlikely(!*string)) goto fallback; + + if(*string == 'n' && !strcmp(string, "never")) { + // this is a valid option + *result = 0; + return 1; + } + + // make sure it is a number + if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback; + + char *e = NULL; + calculated_number n = str2ld(string, &e); + if(e && *e) { + switch (*e) { + case 'Y': + *result = (int) (n * 31536000); + break; + case 'M': + *result = (int) (n * 2592000); + break; + case 'w': + *result = (int) (n * 604800); + break; + case 'd': + *result = (int) (n * 86400); + break; + case 'h': + *result = (int) (n * 3600); + break; + case 'm': + *result = (int) (n * 60); + break; + case 's': + default: + *result = (int) (n); + break; + } + } + else + *result = (int)(n); + + return 1; + + fallback: + *result = 0; + return 0; +} + +struct section *appconfig_get_section(struct config *root, const char *name) +{ + return appconfig_section_find(root, name); +} diff --git a/libnetdata/config/appconfig.h b/libnetdata/config/appconfig.h new file mode 100644 index 0000000..9d02e4a --- /dev/null +++ b/libnetdata/config/appconfig.h @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +/* + * This section manages ini config files, like netdata.conf and stream.conf + * + * It is organized like this: + * + * struct config (i.e. netdata.conf or stream.conf) + * .sections = a linked list of struct section + * .mutex = a mutex to protect the above linked list due to multi-threading + * .index = an AVL tree of struct section + * + * struct section (i.e. [global] or [health] of netdata.conf) + * .value = a linked list of struct config_option + * .mutex = a mutex to protect the above linked list due to multi-threading + * .value_index = an AVL tree of struct config_option + * + * struct config_option (ie. a name-value pair for each ini file option) + * + * The following operations on name-value options are supported: + * SET to set the value of an option + * SET DEFAULT to set the value and the default value of an option + * GET to get the value of an option + * EXISTS to check if an option exists + * MOVE to move an option from a section to another section, and/or rename it + * + * GET and SET operations are provided for the following data types: + * STRING + * NUMBER (long long) + * FLOAT (long double) + * BOOLEAN (false, true) + * BOOLEAN ONDEMAND (false, true, auto) + * + * GET and SET operations create struct config_option, if it is not already present. + * This allows netdata to run even without netdata.conf and stream.conf. The internal + * defaults are used to create the structure that should exist in the ini file and the config + * file can be downloaded from the server. + * + * Also 2 operations are supported for the whole config file: + * + * LOAD To load the ini file from disk + * GENERATE To generate the ini file (this is used to download the ini file from the server) + * + * For each option (name-value pair), the system maintains 4 flags: + * LOADED to indicate that the value has been loaded from the file + * USED to indicate that netdata used the value + * CHANGED to indicate that the value has been changed from the loaded value or the internal default value + * CHECKED is used internally for optimization (to avoid an strcmp() every time GET is called). + * + * TODO: + * 1. The linked lists and the mutexes can be removed and the AVL trees can become DICTIONARY. + * This part of the code was written before we add traversal to AVL. + * + * 2. High level data types could be supported, to simplify the rest of the code: + * MULTIPLE CHOICE to let the user select one of the supported keywords + * this would allow users see in comments the available options + * + * SIMPLE PATTERN to let the user define netdata SIMPLE PATTERNS + * + * 3. Sorting of options should be supported. + * Today, when the ini file is downloaded from the server, the options are shown in the order + * they appear in the linked list (the order they were added, listing changed options first). + * If we remove the linked list, the order they appear in the AVL tree will be used (which is + * random due to simple_hash()). + * Ideally, we support sorting of options when generating the ini file. + * + * 4. There is no free() operation. So, memory is freed on netdata exit. + * + * 5. Avoid memory fragmentation + * Since entries are created from multiple threads and a lot of allocations are required + * for each config_option, fragmentation can be a problem for IoT. + * + * 6. Although this way of managing options is quite flexible and dynamic, it wastes memory + * for the names of the options. Since most of the option names are static, we could provide + * a method to allocate only the dynamic option names. + */ + +#ifndef NETDATA_CONFIG_H +#define NETDATA_CONFIG_H 1 + +#include "../libnetdata.h" + +#define CONFIG_FILENAME "netdata.conf" + +#define CONFIG_SECTION_GLOBAL "global" +#define CONFIG_SECTION_WEB "web" +#define CONFIG_SECTION_STATSD "statsd" +#define CONFIG_SECTION_PLUGINS "plugins" +#define CONFIG_SECTION_CLOUD "cloud" +#define CONFIG_SECTION_REGISTRY "registry" +#define CONFIG_SECTION_HEALTH "health" +#define CONFIG_SECTION_BACKEND "backend" +#define CONFIG_SECTION_STREAM "stream" +#define CONFIG_SECTION_EXPORTING "exporting:global" +#define CONFIG_SECTION_PROMETHEUS "prometheus:exporter" +#define CONFIG_SECTION_HOST_LABEL "host labels" +#define EXPORTING_CONF "exporting.conf" + +// these are used to limit the configuration names and values lengths +// they are not enforced by config.c functions (they will strdup() all strings, no matter of their length) +#define CONFIG_MAX_NAME 1024 +#define CONFIG_MAX_VALUE 2048 + +// ---------------------------------------------------------------------------- +// Config definitions +#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2) + +#define CONFIG_VALUE_LOADED 0x01 // has been loaded from the config +#define CONFIG_VALUE_USED 0x02 // has been accessed from the program +#define CONFIG_VALUE_CHANGED 0x04 // has been changed from the loaded value or the internal default value +#define CONFIG_VALUE_CHECKED 0x08 // has been checked if the value is different from the default + +struct config_option { + avl avl_node; // the index entry of this entry - this has to be first! + + uint8_t flags; + uint32_t hash; // a simple hash to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons + + char *name; + char *value; + + struct config_option *next; // config->mutex protects just this +}; + +struct section { + avl avl_node; // the index entry of this section - this has to be first! + + uint32_t hash; // a simple hash to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons + + char *name; + + struct section *next; // gloabl config_mutex protects just this + + struct config_option *values; + avl_tree_lock values_index; + + netdata_mutex_t mutex; // this locks only the writers, to ensure atomic updates + // readers are protected using the rwlock in avl_tree_lock +}; + +struct config { + struct section *first_section; + struct section *last_section; // optimize inserting at the end + netdata_mutex_t mutex; + avl_tree_lock index; +}; + +#define CONFIG_BOOLEAN_INVALID 100 // an invalid value to check for validity (used as default initialization when needed) + +#define CONFIG_BOOLEAN_NO 0 // disabled +#define CONFIG_BOOLEAN_YES 1 // enabled + +#ifndef CONFIG_BOOLEAN_AUTO +#define CONFIG_BOOLEAN_AUTO 2 // enabled if it has useful info when enabled +#endif + +extern int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name); +extern void config_section_wrlock(struct section *co); +extern void config_section_unlock(struct section *co); + +extern char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value); +extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value); +extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value); +extern LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value); +extern int appconfig_get_boolean_by_section(struct section *co, const char *name, int value); +extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value); +extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value); +extern int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value); + +extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value); +extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value); +extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value); +extern LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value); +extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value); + +extern int appconfig_exists(struct config *root, const char *section, const char *name); +extern int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new); + +extern void appconfig_generate(struct config *root, BUFFER *wb, int only_changed); + +extern int appconfig_section_compare(void *a, void *b); + +extern void appconfig_section_destroy_non_loaded(struct config *root, const char *section); + +extern int config_parse_duration(const char* string, int* result); + +extern struct section *appconfig_get_section(struct config *root, const char *name); + +extern void appconfig_wrlock(struct config *root); +extern void appconfig_unlock(struct config *root); + +struct connector_instance { + char instance_name[CONFIG_MAX_NAME + 1]; + char connector_name[CONFIG_MAX_NAME + 1]; +}; + +typedef struct _connector_instance { + struct section *connector; // actual connector + struct section *instance; // This instance + char instance_name[CONFIG_MAX_NAME + 1]; + char connector_name[CONFIG_MAX_NAME + 1]; + struct _connector_instance *next; // Next instance +} _CONNECTOR_INSTANCE; + +extern _CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance); + +#endif /* NETDATA_CONFIG_H */ diff --git a/libnetdata/dictionary/Makefile.am b/libnetdata/dictionary/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/dictionary/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/dictionary/README.md b/libnetdata/dictionary/README.md new file mode 100644 index 0000000..234ff18 --- /dev/null +++ b/libnetdata/dictionary/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/dictionary/README.md +--> + +[](<>) diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c new file mode 100644 index 0000000..cfcf1fb --- /dev/null +++ b/libnetdata/dictionary/dictionary.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// dictionary statistics + +static inline void NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(DICTIONARY *dict) { + if(likely(dict->stats)) + dict->stats->inserts++; +} +static inline void NETDATA_DICTIONARY_STATS_DELETES_PLUS1(DICTIONARY *dict) { + if(likely(dict->stats)) + dict->stats->deletes++; +} +static inline void NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) { + if(likely(dict->stats)) + dict->stats->searches++; +} +static inline void NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict) { + if(likely(dict->stats)) + dict->stats->entries++; +} +static inline void NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict) { + if(likely(dict->stats)) + dict->stats->entries--; +} + + +// ---------------------------------------------------------------------------- +// dictionary locks + +static inline void dictionary_read_lock(DICTIONARY *dict) { + if(likely(dict->rwlock)) { + // debug(D_DICTIONARY, "Dictionary READ lock"); + netdata_rwlock_rdlock(dict->rwlock); + } +} + +static inline void dictionary_write_lock(DICTIONARY *dict) { + if(likely(dict->rwlock)) { + // debug(D_DICTIONARY, "Dictionary WRITE lock"); + netdata_rwlock_wrlock(dict->rwlock); + } +} + +static inline void dictionary_unlock(DICTIONARY *dict) { + if(likely(dict->rwlock)) { + // debug(D_DICTIONARY, "Dictionary UNLOCK lock"); + netdata_rwlock_unlock(dict->rwlock); + } +} + + +// ---------------------------------------------------------------------------- +// avl index + +static int name_value_compare(void* a, void* b) { + if(((NAME_VALUE *)a)->hash < ((NAME_VALUE *)b)->hash) return -1; + else if(((NAME_VALUE *)a)->hash > ((NAME_VALUE *)b)->hash) return 1; + else return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name); +} + +static inline NAME_VALUE *dictionary_name_value_index_find_nolock(DICTIONARY *dict, const char *name, uint32_t hash) { + NAME_VALUE tmp; + tmp.hash = (hash)?hash:simple_hash(name); + tmp.name = (char *)name; + + NETDATA_DICTIONARY_STATS_SEARCHES_PLUS1(dict); + return (NAME_VALUE *)avl_search(&(dict->values_index), (avl *) &tmp); +} + +// ---------------------------------------------------------------------------- +// internal methods + +static NAME_VALUE *dictionary_name_value_create_nolock(DICTIONARY *dict, const char *name, void *value, size_t value_len, uint32_t hash) { + debug(D_DICTIONARY, "Creating name value entry for name '%s'.", name); + + NAME_VALUE *nv = callocz(1, sizeof(NAME_VALUE)); + + if(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE) + nv->name = (char *)name; + else { + nv->name = strdupz(name); + } + + nv->hash = (hash)?hash:simple_hash(nv->name); + + if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE) + nv->value = value; + else { + nv->value = mallocz(value_len); + memcpy(nv->value, value, value_len); + } + + // index it + NETDATA_DICTIONARY_STATS_INSERTS_PLUS1(dict); + if(unlikely(avl_insert(&((dict)->values_index), (avl *)(nv)) != (avl *)nv)) + error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary."); + + NETDATA_DICTIONARY_STATS_ENTRIES_PLUS1(dict); + + return nv; +} + +static void dictionary_name_value_destroy_nolock(DICTIONARY *dict, NAME_VALUE *nv) { + debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name); + + NETDATA_DICTIONARY_STATS_DELETES_PLUS1(dict); + if(unlikely(avl_remove(&(dict->values_index), (avl *)(nv)) != (avl *)nv)) + error("dictionary: INTERNAL ERROR: dictionary invalid removal of node."); + + NETDATA_DICTIONARY_STATS_ENTRIES_MINUS1(dict); + + if(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) { + debug(D_REGISTRY, "Dictionary freeing value of '%s'", nv->name); + freez(nv->value); + } + + if(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)) { + debug(D_REGISTRY, "Dictionary freeing name '%s'", nv->name); + freez(nv->name); + } + + freez(nv); +} + +// ---------------------------------------------------------------------------- +// API - basic methods + +DICTIONARY *dictionary_create(uint8_t flags) { + debug(D_DICTIONARY, "Creating dictionary."); + + DICTIONARY *dict = callocz(1, sizeof(DICTIONARY)); + + if(flags & DICTIONARY_FLAG_WITH_STATISTICS) + dict->stats = callocz(1, sizeof(struct dictionary_stats)); + + if(!(flags & DICTIONARY_FLAG_SINGLE_THREADED)) { + dict->rwlock = callocz(1, sizeof(netdata_rwlock_t)); + netdata_rwlock_init(dict->rwlock); + } + + avl_init(&dict->values_index, name_value_compare); + dict->flags = flags; + + return dict; +} + +void dictionary_destroy(DICTIONARY *dict) { + debug(D_DICTIONARY, "Destroying dictionary."); + + dictionary_write_lock(dict); + + while(dict->values_index.root) + dictionary_name_value_destroy_nolock(dict, (NAME_VALUE *)dict->values_index.root); + + dictionary_unlock(dict); + + if(dict->stats) + freez(dict->stats); + + if(dict->rwlock) { + netdata_rwlock_destroy(dict->rwlock); + freez(dict->rwlock); + } + + freez(dict); +} + +// ---------------------------------------------------------------------------- + +void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name); + + uint32_t hash = simple_hash(name); + + dictionary_write_lock(dict); + + NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, hash); + if(unlikely(!nv)) { + debug(D_DICTIONARY, "Dictionary entry with name '%s' not found. Creating a new one.", name); + + nv = dictionary_name_value_create_nolock(dict, name, value, value_len, hash); + if(unlikely(!nv)) + fatal("Cannot create name_value."); + } + else { + debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", name); + + if(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE) { + debug(D_REGISTRY, "Dictionary: linking value to '%s'", name); + nv->value = value; + } + else { + debug(D_REGISTRY, "Dictionary: cloning value to '%s'", name); + + // copy the new value without breaking + // any other thread accessing the same entry + void *new = mallocz(value_len), + *old = nv->value; + + memcpy(new, value, value_len); + nv->value = new; + + debug(D_REGISTRY, "Dictionary: freeing old value of '%s'", name); + freez(old); + } + } + + dictionary_unlock(dict); + + return nv->value; +} + +void *dictionary_get(DICTIONARY *dict, const char *name) { + debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name); + + dictionary_read_lock(dict); + NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0); + dictionary_unlock(dict); + + if(unlikely(!nv)) { + debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name); + return NULL; + } + + debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name); + return nv->value; +} + +int dictionary_del(DICTIONARY *dict, const char *name) { + int ret; + + debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name); + + dictionary_write_lock(dict); + + NAME_VALUE *nv = dictionary_name_value_index_find_nolock(dict, name, 0); + if(unlikely(!nv)) { + debug(D_DICTIONARY, "Not found dictionary entry with name '%s'.", name); + ret = -1; + } + else { + debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name); + dictionary_name_value_destroy_nolock(dict, nv); + ret = 0; + } + + dictionary_unlock(dict); + + return ret; +} + + +// ---------------------------------------------------------------------------- +// API - walk through the dictionary +// the dictionary is locked for reading while this happens +// do not user other dictionary calls while walking the dictionary - deadlock! + +static int dictionary_walker(avl *a, int (*callback)(void *entry, void *data), void *data) { + int total = 0, ret = 0; + + if(a->avl_link[0]) { + ret = dictionary_walker(a->avl_link[0], callback, data); + if(ret < 0) return ret; + total += ret; + } + + ret = callback(((NAME_VALUE *)a)->value, data); + if(ret < 0) return ret; + total += ret; + + if(a->avl_link[1]) { + ret = dictionary_walker(a->avl_link[1], callback, data); + if (ret < 0) return ret; + total += ret; + } + + return total; +} + +int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *data), void *data) { + int ret = 0; + + dictionary_read_lock(dict); + + if(likely(dict->values_index.root)) + ret = dictionary_walker(dict->values_index.root, callback, data); + + dictionary_unlock(dict); + + return ret; +} + +static int dictionary_walker_name_value(avl *a, int (*callback)(char *name, void *entry, void *data), void *data) { + int total = 0, ret = 0; + + if(a->avl_link[0]) { + ret = dictionary_walker_name_value(a->avl_link[0], callback, data); + if(ret < 0) return ret; + total += ret; + } + + ret = callback(((NAME_VALUE *)a)->name, ((NAME_VALUE *)a)->value, data); + if(ret < 0) return ret; + total += ret; + + if(a->avl_link[1]) { + ret = dictionary_walker_name_value(a->avl_link[1], callback, data); + if (ret < 0) return ret; + total += ret; + } + + return total; +} + +int dictionary_get_all_name_value(DICTIONARY *dict, int (*callback)(char *name, void *entry, void *data), void *data) { + int ret = 0; + + dictionary_read_lock(dict); + + if(likely(dict->values_index.root)) + ret = dictionary_walker_name_value(dict->values_index.root, callback, data); + + dictionary_unlock(dict); + + return ret; +} diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h new file mode 100644 index 0000000..fc24ec2 --- /dev/null +++ b/libnetdata/dictionary/dictionary.h @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_DICTIONARY_H +#define NETDATA_DICTIONARY_H 1 + +#include "../libnetdata.h" + +struct dictionary_stats { + unsigned long long inserts; + unsigned long long deletes; + unsigned long long searches; + unsigned long long entries; +}; + +typedef struct name_value { + avl avl_node; // the index - this has to be first! + + uint32_t hash; // a simple hash to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons + + char *name; + void *value; +} NAME_VALUE; + +typedef struct dictionary { + avl_tree_type values_index; + + uint8_t flags; + + struct dictionary_stats *stats; + netdata_rwlock_t *rwlock; +} DICTIONARY; + +#define DICTIONARY_FLAG_DEFAULT 0x00000000 +#define DICTIONARY_FLAG_SINGLE_THREADED 0x00000001 +#define DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE 0x00000002 +#define DICTIONARY_FLAG_NAME_LINK_DONT_CLONE 0x00000004 +#define DICTIONARY_FLAG_WITH_STATISTICS 0x00000008 + +extern DICTIONARY *dictionary_create(uint8_t flags); +extern void dictionary_destroy(DICTIONARY *dict); +extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) NEVERNULL; +extern void *dictionary_get(DICTIONARY *dict, const char *name); +extern int dictionary_del(DICTIONARY *dict, const char *name); + +extern int dictionary_get_all(DICTIONARY *dict, int (*callback)(void *entry, void *d), void *data); +extern int dictionary_get_all_name_value(DICTIONARY *dict, int (*callback)(char *name, void *entry, void *d), void *data); + +#endif /* NETDATA_DICTIONARY_H */ diff --git a/libnetdata/ebpf/Makefile.am b/libnetdata/ebpf/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/ebpf/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/ebpf/README.md b/libnetdata/ebpf/README.md new file mode 100644 index 0000000..09c6607 --- /dev/null +++ b/libnetdata/ebpf/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/ebpf/README.md +--> + +[](<>) diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c new file mode 100644 index 0000000..a9ff21f --- /dev/null +++ b/libnetdata/ebpf/ebpf.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <dlfcn.h> +#include <sys/utsname.h> + +#include "../libnetdata.h" + +/* +static int clean_kprobe_event(FILE *out, char *filename, char *father_pid, netdata_ebpf_events_t *ptr) +{ + int fd = open(filename, O_WRONLY | O_APPEND, 0); + if (fd < 0) { + if (out) { + fprintf(out, "Cannot open %s : %s\n", filename, strerror(errno)); + } + return 1; + } + + char cmd[1024]; + int length = snprintf(cmd, 1023, "-:kprobes/%c_netdata_%s_%s", ptr->type, ptr->name, father_pid); + int ret = 0; + if (length > 0) { + ssize_t written = write(fd, cmd, strlen(cmd)); + if (written < 0) { + if (out) { + fprintf( + out, "Cannot remove the event (%d, %d) '%s' from %s : %s\n", getppid(), getpid(), cmd, filename, + strerror((int)errno)); + } + ret = 1; + } + } + + close(fd); + + return ret; +} + +int clean_kprobe_events(FILE *out, int pid, netdata_ebpf_events_t *ptr) +{ + debug(D_EXIT, "Cleaning parent process events."); + char filename[FILENAME_MAX + 1]; + snprintf(filename, FILENAME_MAX, "%s%s", NETDATA_DEBUGFS, "kprobe_events"); + + char removeme[16]; + snprintf(removeme, 15, "%d", pid); + + int i; + for (i = 0; ptr[i].name; i++) { + if (clean_kprobe_event(out, filename, removeme, &ptr[i])) { + break; + } + } + + return 0; +} +*/ + +//---------------------------------------------------------------------------------------------------------------------- + +int get_kernel_version(char *out, int size) +{ + char major[16], minor[16], patch[16]; + char ver[VERSION_STRING_LEN]; + char *version = ver; + + out[0] = '\0'; + int fd = open("/proc/sys/kernel/osrelease", O_RDONLY); + if (fd < 0) + return -1; + + ssize_t len = read(fd, ver, sizeof(ver)); + if (len < 0) { + close(fd); + return -1; + } + + close(fd); + + char *move = major; + while (*version && *version != '.') + *move++ = *version++; + *move = '\0'; + + version++; + move = minor; + while (*version && *version != '.') + *move++ = *version++; + *move = '\0'; + + if (*version) + version++; + else + return -1; + + move = patch; + while (*version && *version != '\n') + *move++ = *version++; + *move = '\0'; + + fd = snprintf(out, (size_t)size, "%s.%s.%s", major, minor, patch); + if (fd > size) + error("The buffer to store kernel version is not smaller than necessary."); + + return ((int)(str2l(major) * 65536) + (int)(str2l(minor) * 256) + (int)str2l(patch)); +} + +int get_redhat_release() +{ + char buffer[VERSION_STRING_LEN + 1]; + int major, minor; + FILE *fp = fopen("/etc/redhat-release", "r"); + + if (fp) { + major = 0; + minor = -1; + size_t length = fread(buffer, sizeof(char), VERSION_STRING_LEN, fp); + if (length > 4) { + buffer[length] = '\0'; + char *end = strchr(buffer, '.'); + char *start; + if (end) { + *end = 0x0; + + if (end > buffer) { + start = end - 1; + + major = strtol(start, NULL, 10); + start = ++end; + + end++; + if (end) { + end = 0x00; + minor = strtol(start, NULL, 10); + } else { + minor = -1; + } + } + } + } + + fclose(fp); + return ((major * 256) + minor); + } else { + return -1; + } +} + +/** + * Check if the kernel is in a list of rejected ones + * + * @return Returns 1 if the kernel is rejected, 0 otherwise. + */ +static int kernel_is_rejected() +{ + // Get kernel version from system + char version_string[VERSION_STRING_LEN + 1]; + int version_string_len = 0; + + if (read_file("/proc/version_signature", version_string, VERSION_STRING_LEN)) { + if (read_file("/proc/version", version_string, VERSION_STRING_LEN)) { + struct utsname uname_buf; + if (!uname(&uname_buf)) { + info("Cannot check kernel version"); + return 0; + } + version_string_len = + snprintfz(version_string, VERSION_STRING_LEN, "%s %s", uname_buf.release, uname_buf.version); + } + } + + if (!version_string_len) + version_string_len = strlen(version_string); + + // Open a file with a list of rejected kernels + char *config_dir = getenv("NETDATA_USER_CONFIG_DIR"); + if (config_dir == NULL) { + config_dir = CONFIG_DIR; + } + + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/%s", config_dir, EBPF_KERNEL_REJECT_LIST_FILE); + FILE *kernel_reject_list = fopen(filename, "r"); + + if (!kernel_reject_list) { + config_dir = getenv("NETDATA_STOCK_CONFIG_DIR"); + if (config_dir == NULL) { + config_dir = LIBCONFIG_DIR; + } + + snprintfz(filename, FILENAME_MAX, "%s/%s", config_dir, EBPF_KERNEL_REJECT_LIST_FILE); + kernel_reject_list = fopen(filename, "r"); + + if (!kernel_reject_list) + return 0; + } + + // Find if the kernel is in the reject list + char *reject_string = NULL; + size_t buf_len = 0; + ssize_t reject_string_len; + while ((reject_string_len = getline(&reject_string, &buf_len, kernel_reject_list) - 1) > 0) { + if (version_string_len >= reject_string_len) { + if (!strncmp(version_string, reject_string, reject_string_len)) { + info("A buggy kernel is detected"); + fclose(kernel_reject_list); + freez(reject_string); + return 1; + } + } + } + + fclose(kernel_reject_list); + freez(reject_string); + + return 0; +} + +static int has_ebpf_kernel_version(int version) +{ + if (kernel_is_rejected()) + return 0; + + // Kernel 4.11.0 or RH > 7.5 + return (version >= NETDATA_MINIMUM_EBPF_KERNEL || get_redhat_release() >= NETDATA_MINIMUM_RH_VERSION); +} + +int has_condition_to_run(int version) +{ + if (!has_ebpf_kernel_version(version)) + return 0; + + return 1; +} + +//---------------------------------------------------------------------------------------------------------------------- + +char *ebpf_kernel_suffix(int version, int isrh) +{ + if (isrh) { + if (version >= NETDATA_EBPF_KERNEL_4_11) + return "4.18"; + else + return "3.10"; + } else { + if (version >= NETDATA_EBPF_KERNEL_5_10) + return "5.10"; + else if (version >= NETDATA_EBPF_KERNEL_4_17) + return "5.4"; + else if (version >= NETDATA_EBPF_KERNEL_4_15) + return "4.16"; + else if (version >= NETDATA_EBPF_KERNEL_4_11) + return "4.14"; + } + + return NULL; +} + +//---------------------------------------------------------------------------------------------------------------------- + +int ebpf_update_kernel(ebpf_data_t *ed) +{ + char *kernel = ebpf_kernel_suffix(ed->running_on_kernel, (ed->isrh < 0) ? 0 : 1); + size_t length = strlen(kernel); + strncpyz(ed->kernel_string, kernel, length); + ed->kernel_string[length] = '\0'; + + return 0; +} + +static int select_file(char *name, const char *program, size_t length, int mode, char *kernel_string) +{ + int ret = -1; + if (!mode) + ret = snprintf(name, length, "rnetdata_ebpf_%s.%s.o", program, kernel_string); + else if (mode == 1) + ret = snprintf(name, length, "dnetdata_ebpf_%s.%s.o", program, kernel_string); + else if (mode == 2) + ret = snprintf(name, length, "pnetdata_ebpf_%s.%s.o", program, kernel_string); + + return ret; +} + +struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, char *kernel_string, struct bpf_object **obj, int *map_fd) +{ + char lpath[4096]; + char lname[128]; + int prog_fd; + + int test = select_file(lname, em->thread_name, (size_t)127, em->mode, kernel_string); + if (test < 0 || test > 127) + return NULL; + + snprintf(lpath, 4096, "%s/%s", plugins_dir, lname); + if (bpf_prog_load(lpath, BPF_PROG_TYPE_KPROBE, obj, &prog_fd)) { + em->enabled = CONFIG_BOOLEAN_NO; + info("Cannot load program: %s", lpath); + return NULL; + } else { + info("The eBPF program %s was loaded with success.", em->thread_name); + } + + struct bpf_map *map; + size_t i = 0; + bpf_map__for_each(map, *obj) + { + map_fd[i] = bpf_map__fd(map); + i++; + } + + struct bpf_program *prog; + struct bpf_link **links = callocz(NETDATA_MAX_PROBES , sizeof(struct bpf_link *)); + i = 0; + bpf_object__for_each_program(prog, *obj) + { + links[i] = bpf_program__attach(prog); + i++; + } + + return links; +} diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h new file mode 100644 index 0000000..d4faccf --- /dev/null +++ b/libnetdata/ebpf/ebpf.h @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_EBPF_H +#define NETDATA_EBPF_H 1 + +#include <bpf/bpf.h> +#include <bpf/libbpf.h> + +#define NETDATA_DEBUGFS "/sys/kernel/debug/tracing/" + +/** + * The next magic number is got doing the following math: + * 294960 = 4*65536 + 11*256 + 0 + * + * For more details, please, read /usr/include/linux/version.h + */ +#define NETDATA_MINIMUM_EBPF_KERNEL 264960 + +/** + * The RedHat magic number was got doing: + * + * 1797 = 7*256 + 5 + * + * For more details, please, read /usr/include/linux/version.h + * in any Red Hat installation. + */ +#define NETDATA_MINIMUM_RH_VERSION 1797 + +/** + * 2048 = 8*256 + 0 + */ +#define NETDATA_RH_8 2048 + +/** + * Kernel 5.10 + * + * 330240 = 5*65536 + 10*256 + */ +#define NETDATA_EBPF_KERNEL_5_10 330240 + +/** + * Kernel 4.17 + * + * 266496 = 4*65536 + 17*256 + */ +#define NETDATA_EBPF_KERNEL_4_17 266496 + +/** + * Kernel 4.15 + * + * 265984 = 4*65536 + 15*256 + */ +#define NETDATA_EBPF_KERNEL_4_15 265984 + +/** + * Kernel 4.11 + * + * 264960 = 4*65536 + 15*256 + */ +#define NETDATA_EBPF_KERNEL_4_11 264960 + +#define VERSION_STRING_LEN 256 +#define EBPF_KERNEL_REJECT_LIST_FILE "ebpf_kernel_reject_list.txt" + +typedef struct ebpf_data { + int *map_fd; + + char *kernel_string; + uint32_t running_on_kernel; + int isrh; +} ebpf_data_t; + +typedef enum { + MODE_RETURN = 0, // This attaches kprobe when the function returns + MODE_DEVMODE, // This stores log given description about the errors raised + MODE_ENTRY // This attaches kprobe when the function is called +} netdata_run_mode_t; + +typedef struct ebpf_module { + const char *thread_name; + const char *config_name; + int enabled; + void *(*start_routine)(void *); + int update_time; + int global_charts; + int apps_charts; + netdata_run_mode_t mode; + uint32_t thread_id; + int optional; +} ebpf_module_t; + +#define NETDATA_MAX_PROBES 64 + +extern int get_kernel_version(char *out, int size); +extern int get_redhat_release(); +extern int has_condition_to_run(int version); +extern char *ebpf_kernel_suffix(int version, int isrh); +extern int ebpf_update_kernel(ebpf_data_t *ef); +extern struct bpf_link **ebpf_load_program(char *plugins_dir, + ebpf_module_t *em, + char *kernel_string, + struct bpf_object **obj, + int *map_fd); + +#endif /* NETDATA_EBPF_H */ diff --git a/libnetdata/eval/Makefile.am b/libnetdata/eval/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/eval/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/eval/README.md b/libnetdata/eval/README.md new file mode 100644 index 0000000..72b4089 --- /dev/null +++ b/libnetdata/eval/README.md @@ -0,0 +1 @@ +[](<>) diff --git a/libnetdata/eval/eval.c b/libnetdata/eval/eval.c new file mode 100644 index 0000000..b53b070 --- /dev/null +++ b/libnetdata/eval/eval.c @@ -0,0 +1,1190 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// data structures for storing the parsed expression in memory + +typedef struct eval_value { + int type; + + union { + calculated_number number; + EVAL_VARIABLE *variable; + struct eval_node *expression; + }; +} EVAL_VALUE; + +typedef struct eval_node { + int id; + unsigned char operator; + int precedence; + + int count; + EVAL_VALUE ops[]; +} EVAL_NODE; + +// these are used for EVAL_NODE.operator +// they are used as internal IDs to identify an operator +// THEY ARE NOT USED FOR PARSING OPERATORS LIKE THAT +#define EVAL_OPERATOR_NOP '\0' +#define EVAL_OPERATOR_EXPRESSION_OPEN '(' +#define EVAL_OPERATOR_EXPRESSION_CLOSE ')' +#define EVAL_OPERATOR_NOT '!' +#define EVAL_OPERATOR_PLUS '+' +#define EVAL_OPERATOR_MINUS '-' +#define EVAL_OPERATOR_AND '&' +#define EVAL_OPERATOR_OR '|' +#define EVAL_OPERATOR_GREATER_THAN_OR_EQUAL 'G' +#define EVAL_OPERATOR_LESS_THAN_OR_EQUAL 'L' +#define EVAL_OPERATOR_NOT_EQUAL '~' +#define EVAL_OPERATOR_EQUAL '=' +#define EVAL_OPERATOR_LESS '<' +#define EVAL_OPERATOR_GREATER '>' +#define EVAL_OPERATOR_MULTIPLY '*' +#define EVAL_OPERATOR_DIVIDE '/' +#define EVAL_OPERATOR_SIGN_PLUS 'P' +#define EVAL_OPERATOR_SIGN_MINUS 'M' +#define EVAL_OPERATOR_ABS 'A' +#define EVAL_OPERATOR_IF_THEN_ELSE '?' + +// ---------------------------------------------------------------------------- +// forward function definitions + +static inline void eval_node_free(EVAL_NODE *op); +static inline EVAL_NODE *parse_full_expression(const char **string, int *error); +static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error); +static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); +static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error); +static inline void print_parsed_as_constant(BUFFER *out, calculated_number n); + +// ---------------------------------------------------------------------------- +// evaluation of expressions + +static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) { + static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0; + calculated_number n; + + if(unlikely(this_hash == 0)) { + this_hash = simple_hash("this"); + now_hash = simple_hash("now"); + after_hash = simple_hash("after"); + before_hash = simple_hash("before"); + status_hash = simple_hash("status"); + removed_hash = simple_hash("REMOVED"); + uninitialized_hash = simple_hash("UNINITIALIZED"); + undefined_hash = simple_hash("UNDEFINED"); + clear_hash = simple_hash("CLEAR"); + warning_hash = simple_hash("WARNING"); + critical_hash = simple_hash("CRITICAL"); + } + + if(unlikely(v->hash == this_hash && !strcmp(v->name, "this"))) { + n = (exp->myself)?*exp->myself:NAN; + buffer_strcat(exp->error_msg, "[ $this = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == after_hash && !strcmp(v->name, "after"))) { + n = (exp->after && *exp->after)?*exp->after:NAN; + buffer_strcat(exp->error_msg, "[ $after = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == before_hash && !strcmp(v->name, "before"))) { + n = (exp->before && *exp->before)?*exp->before:NAN; + buffer_strcat(exp->error_msg, "[ $before = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == now_hash && !strcmp(v->name, "now"))) { + n = now_realtime_sec(); + buffer_strcat(exp->error_msg, "[ $now = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == status_hash && !strcmp(v->name, "status"))) { + n = (exp->status)?*exp->status:RRDCALC_STATUS_UNINITIALIZED; + buffer_strcat(exp->error_msg, "[ $status = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == removed_hash && !strcmp(v->name, "REMOVED"))) { + n = RRDCALC_STATUS_REMOVED; + buffer_strcat(exp->error_msg, "[ $REMOVED = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == uninitialized_hash && !strcmp(v->name, "UNINITIALIZED"))) { + n = RRDCALC_STATUS_UNINITIALIZED; + buffer_strcat(exp->error_msg, "[ $UNINITIALIZED = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == undefined_hash && !strcmp(v->name, "UNDEFINED"))) { + n = RRDCALC_STATUS_UNDEFINED; + buffer_strcat(exp->error_msg, "[ $UNDEFINED = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == clear_hash && !strcmp(v->name, "CLEAR"))) { + n = RRDCALC_STATUS_CLEAR; + buffer_strcat(exp->error_msg, "[ $CLEAR = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == warning_hash && !strcmp(v->name, "WARNING"))) { + n = RRDCALC_STATUS_WARNING; + buffer_strcat(exp->error_msg, "[ $WARNING = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(unlikely(v->hash == critical_hash && !strcmp(v->name, "CRITICAL"))) { + n = RRDCALC_STATUS_CRITICAL; + buffer_strcat(exp->error_msg, "[ $CRITICAL = "); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + if(exp->rrdcalc && health_variable_lookup(v->name, v->hash, exp->rrdcalc, &n)) { + buffer_sprintf(exp->error_msg, "[ ${%s} = ", v->name); + print_parsed_as_constant(exp->error_msg, n); + buffer_strcat(exp->error_msg, " ] "); + return n; + } + + *error = EVAL_ERROR_UNKNOWN_VARIABLE; + buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", v->name); + return NAN; +} + +static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) { + calculated_number n; + + switch(v->type) { + case EVAL_VALUE_EXPRESSION: + n = eval_node(exp, v->expression, error); + break; + + case EVAL_VALUE_NUMBER: + n = v->number; + break; + + case EVAL_VALUE_VARIABLE: + n = eval_variable(exp, v->variable, error); + break; + + default: + *error = EVAL_ERROR_INVALID_VALUE; + n = 0; + break; + } + + return n; +} + +static inline int is_true(calculated_number n) { + if(isnan(n)) return 0; + if(isinf(n)) return 1; + if(n == 0) return 0; + return 1; +} + +calculated_number eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + return is_true(eval_value(exp, &op->ops[0], error)) && is_true(eval_value(exp, &op->ops[1], error)); +} +calculated_number eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + return is_true(eval_value(exp, &op->ops[0], error)) || is_true(eval_value(exp, &op->ops[1], error)); +} +calculated_number eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + return isgreaterequal(n1, n2); +} +calculated_number eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + return islessequal(n1, n2); +} +calculated_number eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + if(isnan(n1) && isnan(n2)) return 1; + if(isinf(n1) && isinf(n2)) return 1; + if(isnan(n1) || isnan(n2)) return 0; + if(isinf(n1) || isinf(n2)) return 0; + return calculated_number_equal(n1, n2); +} +calculated_number eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + return !eval_equal(exp, op, error); +} +calculated_number eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + return isless(n1, n2); +} +calculated_number eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + return isgreater(n1, n2); +} +calculated_number eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + if(isnan(n1) || isnan(n2)) return NAN; + if(isinf(n1) || isinf(n2)) return INFINITY; + return n1 + n2; +} +calculated_number eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + if(isnan(n1) || isnan(n2)) return NAN; + if(isinf(n1) || isinf(n2)) return INFINITY; + return n1 - n2; +} +calculated_number eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + if(isnan(n1) || isnan(n2)) return NAN; + if(isinf(n1) || isinf(n2)) return INFINITY; + return n1 * n2; +} +calculated_number eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + calculated_number n2 = eval_value(exp, &op->ops[1], error); + if(isnan(n1) || isnan(n2)) return NAN; + if(isinf(n1) || isinf(n2)) return INFINITY; + return n1 / n2; +} +calculated_number eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + return eval_value(exp, &op->ops[0], error); +} +calculated_number eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + return !is_true(eval_value(exp, &op->ops[0], error)); +} +calculated_number eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + return eval_value(exp, &op->ops[0], error); +} +calculated_number eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + if(isnan(n1)) return NAN; + if(isinf(n1)) return INFINITY; + return -n1; +} +calculated_number eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + calculated_number n1 = eval_value(exp, &op->ops[0], error); + if(isnan(n1)) return NAN; + if(isinf(n1)) return INFINITY; + return abs(n1); +} +calculated_number eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + if(is_true(eval_value(exp, &op->ops[0], error))) + return eval_value(exp, &op->ops[1], error); + else + return eval_value(exp, &op->ops[2], error); +} + +static struct operator { + const char *print_as; + char precedence; + char parameters; + char isfunction; + calculated_number (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); +} operators[256] = { + // this is a random access array + // we always access it with a known EVAL_OPERATOR_X + + [EVAL_OPERATOR_AND] = { "&&", 2, 2, 0, eval_and }, + [EVAL_OPERATOR_OR] = { "||", 2, 2, 0, eval_or }, + [EVAL_OPERATOR_GREATER_THAN_OR_EQUAL] = { ">=", 3, 2, 0, eval_greater_than_or_equal }, + [EVAL_OPERATOR_LESS_THAN_OR_EQUAL] = { "<=", 3, 2, 0, eval_less_than_or_equal }, + [EVAL_OPERATOR_NOT_EQUAL] = { "!=", 3, 2, 0, eval_not_equal }, + [EVAL_OPERATOR_EQUAL] = { "==", 3, 2, 0, eval_equal }, + [EVAL_OPERATOR_LESS] = { "<", 3, 2, 0, eval_less }, + [EVAL_OPERATOR_GREATER] = { ">", 3, 2, 0, eval_greater }, + [EVAL_OPERATOR_PLUS] = { "+", 4, 2, 0, eval_plus }, + [EVAL_OPERATOR_MINUS] = { "-", 4, 2, 0, eval_minus }, + [EVAL_OPERATOR_MULTIPLY] = { "*", 5, 2, 0, eval_multiply }, + [EVAL_OPERATOR_DIVIDE] = { "/", 5, 2, 0, eval_divide }, + [EVAL_OPERATOR_NOT] = { "!", 6, 1, 0, eval_not }, + [EVAL_OPERATOR_SIGN_PLUS] = { "+", 6, 1, 0, eval_sign_plus }, + [EVAL_OPERATOR_SIGN_MINUS] = { "-", 6, 1, 0, eval_sign_minus }, + [EVAL_OPERATOR_ABS] = { "abs(",6,1, 1, eval_abs }, + [EVAL_OPERATOR_IF_THEN_ELSE] = { "?", 7, 3, 0, eval_if_then_else }, + [EVAL_OPERATOR_NOP] = { NULL, 8, 1, 0, eval_nop }, + [EVAL_OPERATOR_EXPRESSION_OPEN] = { NULL, 8, 1, 0, eval_nop }, + + // this should exist in our evaluation list + [EVAL_OPERATOR_EXPRESSION_CLOSE] = { NULL, 99, 1, 0, eval_nop } +}; + +#define eval_precedence(operator) (operators[(unsigned char)(operator)].precedence) + +static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + if(unlikely(op->count != operators[op->operator].parameters)) { + *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS; + return 0; + } + + calculated_number n = operators[op->operator].eval(exp, op, error); + + return n; +} + +// ---------------------------------------------------------------------------- +// parsed-as generation + +static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *error) { + (void)error; + buffer_sprintf(out, "${%s}", v->name); +} + +static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) { + if(unlikely(isnan(n))) { + buffer_strcat(out, "nan"); + return; + } + + if(unlikely(isinf(n))) { + buffer_strcat(out, "inf"); + return; + } + + char b[100+1], *s; + snprintfz(b, 100, CALCULATED_NUMBER_FORMAT, n); + + s = &b[strlen(b) - 1]; + while(s > b && *s == '0') { + *s ='\0'; + s--; + } + + if(s > b && *s == '.') + *s = '\0'; + + buffer_strcat(out, b); +} + +static inline void print_parsed_as_value(BUFFER *out, EVAL_VALUE *v, int *error) { + switch(v->type) { + case EVAL_VALUE_EXPRESSION: + print_parsed_as_node(out, v->expression, error); + break; + + case EVAL_VALUE_NUMBER: + print_parsed_as_constant(out, v->number); + break; + + case EVAL_VALUE_VARIABLE: + print_parsed_as_variable(out, v->variable, error); + break; + + default: + *error = EVAL_ERROR_INVALID_VALUE; + break; + } +} + +static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error) { + if(unlikely(op->count != operators[op->operator].parameters)) { + *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS; + return; + } + + if(operators[op->operator].parameters == 1) { + + if(operators[op->operator].print_as) + buffer_sprintf(out, "%s", operators[op->operator].print_as); + + //if(op->operator == EVAL_OPERATOR_EXPRESSION_OPEN) + // buffer_strcat(out, "("); + + print_parsed_as_value(out, &op->ops[0], error); + + //if(op->operator == EVAL_OPERATOR_EXPRESSION_OPEN) + // buffer_strcat(out, ")"); + } + + else if(operators[op->operator].parameters == 2) { + buffer_strcat(out, "("); + print_parsed_as_value(out, &op->ops[0], error); + + if(operators[op->operator].print_as) + buffer_sprintf(out, " %s ", operators[op->operator].print_as); + + print_parsed_as_value(out, &op->ops[1], error); + buffer_strcat(out, ")"); + } + else if(op->operator == EVAL_OPERATOR_IF_THEN_ELSE && operators[op->operator].parameters == 3) { + buffer_strcat(out, "("); + print_parsed_as_value(out, &op->ops[0], error); + + if(operators[op->operator].print_as) + buffer_sprintf(out, " %s ", operators[op->operator].print_as); + + print_parsed_as_value(out, &op->ops[1], error); + buffer_strcat(out, " : "); + print_parsed_as_value(out, &op->ops[2], error); + buffer_strcat(out, ")"); + } + + if(operators[op->operator].isfunction) + buffer_strcat(out, ")"); +} + +// ---------------------------------------------------------------------------- +// parsing expressions + +// skip spaces +static inline void skip_spaces(const char **string) { + const char *s = *string; + while(isspace(*s)) s++; + *string = s; +} + +// what character can appear just after an operator keyword +// like NOT AND OR ? +static inline int isoperatorterm_word(const char s) { + if(isspace(s) || s == '(' || s == '$' || s == '!' || s == '-' || s == '+' || isdigit(s) || !s) + return 1; + + return 0; +} + +// what character can appear just after an operator symbol? +static inline int isoperatorterm_symbol(const char s) { + if(isoperatorterm_word(s) || isalpha(s)) + return 1; + + return 0; +} + +// return 1 if the character should never appear in a variable +static inline int isvariableterm(const char s) { + if(isalnum(s) || s == '.' || s == '_') + return 0; + + return 1; +} + +// ---------------------------------------------------------------------------- +// parse operators + +static inline int parse_and(const char **string) { + const char *s = *string; + + // AND + if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') && isoperatorterm_word(s[3])) { + *string = &s[4]; + return 1; + } + + // && + if(s[0] == '&' && s[1] == '&' && isoperatorterm_symbol(s[2])) { + *string = &s[2]; + return 1; + } + + return 0; +} + +static inline int parse_or(const char **string) { + const char *s = *string; + + // OR + if((s[0] == 'O' || s[0] == 'o') && (s[1] == 'R' || s[1] == 'r') && isoperatorterm_word(s[2])) { + *string = &s[3]; + return 1; + } + + // || + if(s[0] == '|' && s[1] == '|' && isoperatorterm_symbol(s[2])) { + *string = &s[2]; + return 1; + } + + return 0; +} + +static inline int parse_greater_than_or_equal(const char **string) { + const char *s = *string; + + // >= + if(s[0] == '>' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + *string = &s[2]; + return 1; + } + + return 0; +} + +static inline int parse_less_than_or_equal(const char **string) { + const char *s = *string; + + // <= + if (s[0] == '<' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + *string = &s[2]; + return 1; + } + + return 0; +} + +static inline int parse_greater(const char **string) { + const char *s = *string; + + // > + if(s[0] == '>' && isoperatorterm_symbol(s[1])) { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_less(const char **string) { + const char *s = *string; + + // < + if(s[0] == '<' && isoperatorterm_symbol(s[1])) { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_equal(const char **string) { + const char *s = *string; + + // == + if(s[0] == '=' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + *string = &s[2]; + return 1; + } + + // = + if(s[0] == '=' && isoperatorterm_symbol(s[1])) { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_not_equal(const char **string) { + const char *s = *string; + + // != + if(s[0] == '!' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + *string = &s[2]; + return 1; + } + + // <> + if(s[0] == '<' && s[1] == '>' && isoperatorterm_symbol(s[2])) { + *string = &s[2]; + } + + return 0; +} + +static inline int parse_not(const char **string) { + const char *s = *string; + + // NOT + if((s[0] == 'N' || s[0] == 'n') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') && isoperatorterm_word(s[3])) { + *string = &s[3]; + return 1; + } + + if(s[0] == '!') { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_multiply(const char **string) { + const char *s = *string; + + // * + if(s[0] == '*' && isoperatorterm_symbol(s[1])) { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_divide(const char **string) { + const char *s = *string; + + // / + if(s[0] == '/' && isoperatorterm_symbol(s[1])) { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_minus(const char **string) { + const char *s = *string; + + // - + if(s[0] == '-' && isoperatorterm_symbol(s[1])) { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_plus(const char **string) { + const char *s = *string; + + // + + if(s[0] == '+' && isoperatorterm_symbol(s[1])) { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_open_subexpression(const char **string) { + const char *s = *string; + + // ( + if(s[0] == '(') { + *string = &s[1]; + return 1; + } + + return 0; +} + +#define parse_close_function(x) parse_close_subexpression(x) + +static inline int parse_close_subexpression(const char **string) { + const char *s = *string; + + // ) + if(s[0] == ')') { + *string = &s[1]; + return 1; + } + + return 0; +} + +static inline int parse_variable(const char **string, char *buffer, size_t len) { + const char *s = *string; + + // $ + if(*s == '$') { + size_t i = 0; + s++; + + if(*s == '{') { + // ${variable_name} + + s++; + while (*s && *s != '}' && i < len) + buffer[i++] = *s++; + + if(*s == '}') + s++; + } + else { + // $variable_name + + while (*s && !isvariableterm(*s) && i < len) + buffer[i++] = *s++; + } + + buffer[i] = '\0'; + + if (buffer[0]) { + *string = s; + return 1; + } + } + + return 0; +} + +static inline int parse_constant(const char **string, calculated_number *number) { + char *end = NULL; + calculated_number n = str2ld(*string, &end); + if(unlikely(!end || *string == end)) { + *number = 0; + return 0; + } + *number = n; + *string = end; + return 1; +} + +static inline int parse_abs(const char **string) { + const char *s = *string; + + // ABS + if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'B' || s[1] == 'b') && (s[2] == 'S' || s[2] == 's') && s[3] == '(') { + *string = &s[3]; + return 1; + } + + return 0; +} + +static inline int parse_if_then_else(const char **string) { + const char *s = *string; + + // ? + if(s[0] == '?') { + *string = &s[1]; + return 1; + } + + return 0; +} + +static struct operator_parser { + unsigned char id; + int (*parse)(const char **); +} operator_parsers[] = { + // the order in this list is important! + // the first matching will be used + // so place the longer of overlapping ones + // at the top + + { EVAL_OPERATOR_AND, parse_and }, + { EVAL_OPERATOR_OR, parse_or }, + { EVAL_OPERATOR_GREATER_THAN_OR_EQUAL, parse_greater_than_or_equal }, + { EVAL_OPERATOR_LESS_THAN_OR_EQUAL, parse_less_than_or_equal }, + { EVAL_OPERATOR_NOT_EQUAL, parse_not_equal }, + { EVAL_OPERATOR_EQUAL, parse_equal }, + { EVAL_OPERATOR_LESS, parse_less }, + { EVAL_OPERATOR_GREATER, parse_greater }, + { EVAL_OPERATOR_PLUS, parse_plus }, + { EVAL_OPERATOR_MINUS, parse_minus }, + { EVAL_OPERATOR_MULTIPLY, parse_multiply }, + { EVAL_OPERATOR_DIVIDE, parse_divide }, + { EVAL_OPERATOR_IF_THEN_ELSE, parse_if_then_else }, + + /* we should not put in this list the following: + * + * - NOT + * - ( + * - ) + * + * these are handled in code + */ + + // termination + { EVAL_OPERATOR_NOP, NULL } +}; + +static inline unsigned char parse_operator(const char **string, int *precedence) { + skip_spaces(string); + + int i; + for(i = 0 ; operator_parsers[i].parse != NULL ; i++) + if(operator_parsers[i].parse(string)) { + if(precedence) *precedence = eval_precedence(operator_parsers[i].id); + return operator_parsers[i].id; + } + + return EVAL_OPERATOR_NOP; +} + +// ---------------------------------------------------------------------------- +// memory management + +static inline EVAL_NODE *eval_node_alloc(int count) { + static int id = 1; + + EVAL_NODE *op = callocz(1, sizeof(EVAL_NODE) + (sizeof(EVAL_VALUE) * count)); + + op->id = id++; + op->operator = EVAL_OPERATOR_NOP; + op->precedence = eval_precedence(EVAL_OPERATOR_NOP); + op->count = count; + return op; +} + +static inline void eval_node_set_value_to_node(EVAL_NODE *op, int pos, EVAL_NODE *value) { + if(pos >= op->count) + fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1); + + op->ops[pos].type = EVAL_VALUE_EXPRESSION; + op->ops[pos].expression = value; +} + +static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, calculated_number value) { + if(pos >= op->count) + fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1); + + op->ops[pos].type = EVAL_VALUE_NUMBER; + op->ops[pos].number = value; +} + +static inline void eval_node_set_value_to_variable(EVAL_NODE *op, int pos, const char *variable) { + if(pos >= op->count) + fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1); + + op->ops[pos].type = EVAL_VALUE_VARIABLE; + op->ops[pos].variable = callocz(1, sizeof(EVAL_VARIABLE)); + op->ops[pos].variable->name = strdupz(variable); + op->ops[pos].variable->hash = simple_hash(op->ops[pos].variable->name); +} + +static inline void eval_variable_free(EVAL_VARIABLE *v) { + freez(v->name); + freez(v); +} + +static inline void eval_value_free(EVAL_VALUE *v) { + switch(v->type) { + case EVAL_VALUE_EXPRESSION: + eval_node_free(v->expression); + break; + + case EVAL_VALUE_VARIABLE: + eval_variable_free(v->variable); + break; + + default: + break; + } +} + +static inline void eval_node_free(EVAL_NODE *op) { + if(op->count) { + int i; + for(i = op->count - 1; i >= 0 ;i--) + eval_value_free(&op->ops[i]); + } + + freez(op); +} + +// ---------------------------------------------------------------------------- +// the parsing logic + +// helper function to avoid allocations all over the place +static inline EVAL_NODE *parse_next_operand_given_its_operator(const char **string, unsigned char operator_type, int *error) { + EVAL_NODE *sub = parse_one_full_operand(string, error); + if(!sub) return NULL; + + EVAL_NODE *op = eval_node_alloc(1); + op->operator = operator_type; + eval_node_set_value_to_node(op, 0, sub); + return op; +} + +// parse a full operand, including its sign or other associative operator (e.g. NOT) +static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error) { + char variable_buffer[EVAL_MAX_VARIABLE_NAME_LENGTH + 1]; + EVAL_NODE *op1 = NULL; + calculated_number number; + + *error = EVAL_ERROR_OK; + + skip_spaces(string); + if(!(**string)) { + *error = EVAL_ERROR_MISSING_OPERAND; + return NULL; + } + + if(parse_not(string)) { + op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_NOT, error); + op1->precedence = eval_precedence(EVAL_OPERATOR_NOT); + } + else if(parse_plus(string)) { + op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_SIGN_PLUS, error); + op1->precedence = eval_precedence(EVAL_OPERATOR_SIGN_PLUS); + } + else if(parse_minus(string)) { + op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_SIGN_MINUS, error); + op1->precedence = eval_precedence(EVAL_OPERATOR_SIGN_MINUS); + } + else if(parse_abs(string)) { + op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_ABS, error); + op1->precedence = eval_precedence(EVAL_OPERATOR_ABS); + } + else if(parse_open_subexpression(string)) { + EVAL_NODE *sub = parse_full_expression(string, error); + if(sub) { + op1 = eval_node_alloc(1); + op1->operator = EVAL_OPERATOR_EXPRESSION_OPEN; + op1->precedence = eval_precedence(EVAL_OPERATOR_EXPRESSION_OPEN); + eval_node_set_value_to_node(op1, 0, sub); + if(!parse_close_subexpression(string)) { + *error = EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION; + eval_node_free(op1); + return NULL; + } + } + } + else if(parse_variable(string, variable_buffer, EVAL_MAX_VARIABLE_NAME_LENGTH)) { + op1 = eval_node_alloc(1); + op1->operator = EVAL_OPERATOR_NOP; + eval_node_set_value_to_variable(op1, 0, variable_buffer); + } + else if(parse_constant(string, &number)) { + op1 = eval_node_alloc(1); + op1->operator = EVAL_OPERATOR_NOP; + eval_node_set_value_to_constant(op1, 0, number); + } + else if(**string) + *error = EVAL_ERROR_UNKNOWN_OPERAND; + else + *error = EVAL_ERROR_MISSING_OPERAND; + + return op1; +} + +// parse an operator and the rest of the expression +// precedence processing is handled here +static inline EVAL_NODE *parse_rest_of_expression(const char **string, int *error, EVAL_NODE *op1) { + EVAL_NODE *op2 = NULL; + unsigned char operator; + int precedence; + + operator = parse_operator(string, &precedence); + skip_spaces(string); + + if(operator != EVAL_OPERATOR_NOP) { + op2 = parse_one_full_operand(string, error); + if(!op2) { + // error is already reported + eval_node_free(op1); + return NULL; + } + + EVAL_NODE *op = eval_node_alloc(operators[operator].parameters); + op->operator = operator; + op->precedence = precedence; + + if(operator == EVAL_OPERATOR_IF_THEN_ELSE && op->count == 3) { + skip_spaces(string); + + if(**string != ':') { + eval_node_free(op); + eval_node_free(op1); + eval_node_free(op2); + *error = EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE; + return NULL; + } + (*string)++; + + skip_spaces(string); + + EVAL_NODE *op3 = parse_one_full_operand(string, error); + if(!op3) { + eval_node_free(op); + eval_node_free(op1); + eval_node_free(op2); + // error is already reported + return NULL; + } + + eval_node_set_value_to_node(op, 2, op3); + } + + eval_node_set_value_to_node(op, 1, op2); + + // precedence processing + // if this operator has a higher precedence compared to its next + // put the next operator on top of us (top = evaluated later) + // function recursion does the rest... + if(op->precedence > op1->precedence && op1->count == 2 && op1->operator != '(' && op1->ops[1].type == EVAL_VALUE_EXPRESSION) { + eval_node_set_value_to_node(op, 0, op1->ops[1].expression); + op1->ops[1].expression = op; + op = op1; + } + else + eval_node_set_value_to_node(op, 0, op1); + + return parse_rest_of_expression(string, error, op); + } + else if(**string == ')') { + ; + } + else if(**string) { + eval_node_free(op1); + op1 = NULL; + *error = EVAL_ERROR_MISSING_OPERATOR; + } + + return op1; +} + +// high level function to parse an expression or a sub-expression +static inline EVAL_NODE *parse_full_expression(const char **string, int *error) { + EVAL_NODE *op1 = parse_one_full_operand(string, error); + if(!op1) { + *error = EVAL_ERROR_MISSING_OPERAND; + return NULL; + } + + return parse_rest_of_expression(string, error, op1); +} + +// ---------------------------------------------------------------------------- +// public API + +int expression_evaluate(EVAL_EXPRESSION *expression) { + expression->error = EVAL_ERROR_OK; + + buffer_reset(expression->error_msg); + expression->result = eval_node(expression, (EVAL_NODE *)expression->nodes, &expression->error); + + if(unlikely(isnan(expression->result))) { + if(expression->error == EVAL_ERROR_OK) + expression->error = EVAL_ERROR_VALUE_IS_NAN; + } + else if(unlikely(isinf(expression->result))) { + if(expression->error == EVAL_ERROR_OK) + expression->error = EVAL_ERROR_VALUE_IS_INFINITE; + } + else if(unlikely(expression->error == EVAL_ERROR_UNKNOWN_VARIABLE)) { + // although there is an unknown variable + // the expression was evaluated successfully + expression->error = EVAL_ERROR_OK; + } + + if(expression->error != EVAL_ERROR_OK) { + expression->result = NAN; + + if(buffer_strlen(expression->error_msg)) + buffer_strcat(expression->error_msg, "; "); + + buffer_sprintf(expression->error_msg, "failed to evaluate expression with error %d (%s)", expression->error, expression_strerror(expression->error)); + return 0; + } + + return 1; +} + +EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error) { + const char *s = string; + int err = EVAL_ERROR_OK; + + EVAL_NODE *op = parse_full_expression(&s, &err); + + if(*s) { + if(op) { + eval_node_free(op); + op = NULL; + } + err = EVAL_ERROR_REMAINING_GARBAGE; + } + + if (failed_at) *failed_at = s; + if (error) *error = err; + + if(!op) { + unsigned long pos = s - string + 1; + error("failed to parse expression '%s': %s at character %lu (i.e.: '%s').", string, expression_strerror(err), pos, s); + return NULL; + } + + BUFFER *out = buffer_create(1024); + print_parsed_as_node(out, op, &err); + if(err != EVAL_ERROR_OK) { + error("failed to re-generate expression '%s' with reason: %s", string, expression_strerror(err)); + eval_node_free(op); + buffer_free(out); + return NULL; + } + + EVAL_EXPRESSION *exp = callocz(1, sizeof(EVAL_EXPRESSION)); + + exp->source = strdupz(string); + exp->parsed_as = strdupz(buffer_tostring(out)); + buffer_free(out); + + exp->error_msg = buffer_create(100); + exp->nodes = (void *)op; + + return exp; +} + +void expression_free(EVAL_EXPRESSION *expression) { + if(!expression) return; + + if(expression->nodes) eval_node_free((EVAL_NODE *)expression->nodes); + freez((void *)expression->source); + freez((void *)expression->parsed_as); + buffer_free(expression->error_msg); + freez(expression); +} + +const char *expression_strerror(int error) { + switch(error) { + case EVAL_ERROR_OK: + return "success"; + + case EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION: + return "missing closing parenthesis"; + + case EVAL_ERROR_UNKNOWN_OPERAND: + return "unknown operand"; + + case EVAL_ERROR_MISSING_OPERAND: + return "expected operand"; + + case EVAL_ERROR_MISSING_OPERATOR: + return "expected operator"; + + case EVAL_ERROR_REMAINING_GARBAGE: + return "remaining characters after expression"; + + case EVAL_ERROR_INVALID_VALUE: + return "invalid value structure - internal error"; + + case EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS: + return "wrong number of operands for operation - internal error"; + + case EVAL_ERROR_VALUE_IS_NAN: + return "value is unset"; + + case EVAL_ERROR_VALUE_IS_INFINITE: + return "computed value is infinite"; + + case EVAL_ERROR_UNKNOWN_VARIABLE: + return "undefined variable"; + + case EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE: + return "missing second sub-expression of inline conditional"; + + default: + return "unknown error"; + } +} diff --git a/libnetdata/eval/eval.h b/libnetdata/eval/eval.h new file mode 100644 index 0000000..fc03d7c --- /dev/null +++ b/libnetdata/eval/eval.h @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_EVAL_H +#define NETDATA_EVAL_H 1 + +#include "../libnetdata.h" + +#define EVAL_MAX_VARIABLE_NAME_LENGTH 300 + +typedef enum rrdcalc_status { + RRDCALC_STATUS_REMOVED = -2, + RRDCALC_STATUS_UNDEFINED = -1, + RRDCALC_STATUS_UNINITIALIZED = 0, + RRDCALC_STATUS_CLEAR = 1, + RRDCALC_STATUS_RAISED = 2, + RRDCALC_STATUS_WARNING = 3, + RRDCALC_STATUS_CRITICAL = 4 +} RRDCALC_STATUS; + +typedef struct eval_variable { + char *name; + uint32_t hash; + struct eval_variable *next; +} EVAL_VARIABLE; + +typedef struct eval_expression { + const char *source; + const char *parsed_as; + + RRDCALC_STATUS *status; + calculated_number *myself; + time_t *after; + time_t *before; + + calculated_number result; + + int error; + BUFFER *error_msg; + + // hidden EVAL_NODE * + void *nodes; + + // custom data to be used for looking up variables + struct rrdcalc *rrdcalc; +} EVAL_EXPRESSION; + +#define EVAL_VALUE_INVALID 0 +#define EVAL_VALUE_NUMBER 1 +#define EVAL_VALUE_VARIABLE 2 +#define EVAL_VALUE_EXPRESSION 3 + +// parsing and evaluation +#define EVAL_ERROR_OK 0 + +// parsing errors +#define EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION 1 +#define EVAL_ERROR_UNKNOWN_OPERAND 2 +#define EVAL_ERROR_MISSING_OPERAND 3 +#define EVAL_ERROR_MISSING_OPERATOR 4 +#define EVAL_ERROR_REMAINING_GARBAGE 5 +#define EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE 6 + +// evaluation errors +#define EVAL_ERROR_INVALID_VALUE 101 +#define EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS 102 +#define EVAL_ERROR_VALUE_IS_NAN 103 +#define EVAL_ERROR_VALUE_IS_INFINITE 104 +#define EVAL_ERROR_UNKNOWN_VARIABLE 105 + +// parse the given string as an expression and return: +// a pointer to an expression if it parsed OK +// NULL in which case the pointer to error has the error code +extern EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error); + +// free all resources allocated for an expression +extern void expression_free(EVAL_EXPRESSION *expression); + +// convert an error code to a message +extern const char *expression_strerror(int error); + +// evaluate an expression and return +// 1 = OK, the result is in: expression->result +// 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg) +extern int expression_evaluate(EVAL_EXPRESSION *expression); + +extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result); + +#endif //NETDATA_EVAL_H diff --git a/libnetdata/health/Makefile.am b/libnetdata/health/Makefile.am new file mode 100644 index 0000000..643458b --- /dev/null +++ b/libnetdata/health/Makefile.am @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + $(NULL) diff --git a/libnetdata/health/health.c b/libnetdata/health/health.c new file mode 100644 index 0000000..74f7a02 --- /dev/null +++ b/libnetdata/health/health.c @@ -0,0 +1,173 @@ +#include "health.h" + +SILENCERS *silencers; + +/** + * Create Silencer + * + * Allocate a new silencer to Netdata. + * + * @return It returns the address off the silencer on success and NULL otherwise + */ +SILENCER *create_silencer(void) { + SILENCER *t = callocz(1, sizeof(SILENCER)); + debug(D_HEALTH, "HEALTH command API: Created empty silencer"); + + return t; +} + +/** + * Health Silencers add + * + * Add more one silencer to the list of silenecers. + * + * @param silencer + */ +void health_silencers_add(SILENCER *silencer) { + // Add the created instance to the linked list in silencers + silencer->next = silencers->silencers; + silencers->silencers = silencer; + debug(D_HEALTH, "HEALTH command API: Added silencer %s:%s:%s:%s:%s", silencer->alarms, + silencer->charts, silencer->contexts, silencer->hosts, silencer->families + ); +} + +/** + * Silencers Add Parameter + * + * Create a new silencer and adjust the variables + * + * @param silencer a pointer to the silencer that will be adjusted + * @param key the key value sent by client + * @param value the value sent to the key + * + * @return It returns the silencer configured on success and NULL otherwise + */ +SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) { + static uint32_t + hash_alarm = 0, + hash_template = 0, + hash_chart = 0, + hash_context = 0, + hash_host = 0, + hash_families = 0; + + if (unlikely(!hash_alarm)) { + hash_alarm = simple_uhash(HEALTH_ALARM_KEY); + hash_template = simple_uhash(HEALTH_TEMPLATE_KEY); + hash_chart = simple_uhash(HEALTH_CHART_KEY); + hash_context = simple_uhash(HEALTH_CONTEXT_KEY); + hash_host = simple_uhash(HEALTH_HOST_KEY); + hash_families = simple_uhash(HEALTH_FAMILIES_KEY); + } + + uint32_t hash = simple_uhash(key); + if (unlikely(silencer == NULL)) { + if ( + (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) || + (hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) || + (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) || + (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) || + (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) || + (hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) + ) { + silencer = create_silencer(); + if(!silencer) { + error("Cannot add a new silencer to Netdata"); + return NULL; + } + } + } + + if (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) { + silencer->alarms = strdupz(value); + silencer->alarms_pattern = simple_pattern_create(silencer->alarms, NULL, SIMPLE_PATTERN_EXACT); + } else if (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) { + silencer->charts = strdupz(value); + silencer->charts_pattern = simple_pattern_create(silencer->charts, NULL, SIMPLE_PATTERN_EXACT); + } else if (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) { + silencer->contexts = strdupz(value); + silencer->contexts_pattern = simple_pattern_create(silencer->contexts, NULL, SIMPLE_PATTERN_EXACT); + } else if (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) { + silencer->hosts = strdupz(value); + silencer->hosts_pattern = simple_pattern_create(silencer->hosts, NULL, SIMPLE_PATTERN_EXACT); + } else if (hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) { + silencer->families = strdupz(value); + silencer->families_pattern = simple_pattern_create(silencer->families, NULL, SIMPLE_PATTERN_EXACT); + } + + return silencer; +} + +/** + * JSON Read Callback + * + * Callback called by netdata to create the silencer. + * + * @param e the main json structure + * + * @return It always return 0. + */ +int health_silencers_json_read_callback(JSON_ENTRY *e) +{ + switch(e->type) { + case JSON_OBJECT: +#ifndef ENABLE_JSONC + e->callback_function = health_silencers_json_read_callback; + if(strcmp(e->name,"")) { + // init silencer + debug(D_HEALTH, "JSON: Got object with a name, initializing new silencer for %s",e->name); +#endif + e->callback_data = create_silencer(); + if(e->callback_data) { + health_silencers_add(e->callback_data); + } +#ifndef ENABLE_JSONC + } +#endif + break; + + case JSON_ARRAY: + e->callback_function = health_silencers_json_read_callback; + break; + + case JSON_STRING: + if(!strcmp(e->name,"type")) { + debug(D_HEALTH, "JSON: Processing type=%s",e->data.string); + if (!strcmp(e->data.string,"SILENCE")) silencers->stype = STYPE_SILENCE_NOTIFICATIONS; + else if (!strcmp(e->data.string,"DISABLE")) silencers->stype = STYPE_DISABLE_ALARMS; + } else { + debug(D_HEALTH, "JSON: Adding %s=%s", e->name, e->data.string); + if (e->callback_data) + (void)health_silencers_addparam(e->callback_data, e->name, e->data.string); + } + break; + + case JSON_BOOLEAN: + debug(D_HEALTH, "JSON: Processing all_alarms"); + silencers->all_alarms=e->data.boolean?1:0; + break; + + case JSON_NUMBER: + case JSON_NULL: + break; + } + + return 0; +} + +/** + * Initialize Global Silencers + * + * Initialize the silencer for the whole netdata system. + * + * @return It returns 0 on success and -1 otherwise + */ +int health_initialize_global_silencers() { + silencers = mallocz(sizeof(SILENCERS)); + silencers->all_alarms=0; + silencers->stype=STYPE_NONE; + silencers->silencers=NULL; + + return 0; +}
\ No newline at end of file diff --git a/libnetdata/health/health.h b/libnetdata/health/health.h new file mode 100644 index 0000000..f7580ed --- /dev/null +++ b/libnetdata/health/health.h @@ -0,0 +1,55 @@ +#ifndef NETDATA_HEALTH_LIB +# define NETDATA_HEALTH_LIB 1 + +# include "../libnetdata.h" + +#define HEALTH_ALARM_KEY "alarm" +#define HEALTH_TEMPLATE_KEY "template" +#define HEALTH_CONTEXT_KEY "context" +#define HEALTH_CHART_KEY "chart" +#define HEALTH_HOST_KEY "hosts" +#define HEALTH_OS_KEY "os" +#define HEALTH_FAMILIES_KEY "families" +#define HEALTH_LOOKUP_KEY "lookup" +#define HEALTH_CALC_KEY "calc" + +typedef struct silencer { + char *alarms; + SIMPLE_PATTERN *alarms_pattern; + + char *hosts; + SIMPLE_PATTERN *hosts_pattern; + + char *contexts; + SIMPLE_PATTERN *contexts_pattern; + + char *charts; + SIMPLE_PATTERN *charts_pattern; + + char *families; + SIMPLE_PATTERN *families_pattern; + + struct silencer *next; +} SILENCER; + +typedef enum silence_type { + STYPE_NONE, + STYPE_DISABLE_ALARMS, + STYPE_SILENCE_NOTIFICATIONS +} SILENCE_TYPE; + +typedef struct silencers { + int all_alarms; + SILENCE_TYPE stype; + SILENCER *silencers; +} SILENCERS; + +extern SILENCERS *silencers; + +extern SILENCER *create_silencer(void); +extern int health_silencers_json_read_callback(JSON_ENTRY *e); +extern void health_silencers_add(SILENCER *silencer); +extern SILENCER * health_silencers_addparam(SILENCER *silencer, char *key, char *value); +extern int health_initialize_global_silencers(); + +#endif diff --git a/libnetdata/inlined.h b/libnetdata/inlined.h new file mode 100644 index 0000000..6f236b8 --- /dev/null +++ b/libnetdata/inlined.h @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_INLINED_H +#define NETDATA_INLINED_H 1 + +#include "libnetdata.h" + +#ifdef KERNEL_32BIT +typedef uint32_t kernel_uint_t; +#define str2kernel_uint_t(string) str2uint32_t(string) +#define KERNEL_UINT_FORMAT "%u" +#else +typedef uint64_t kernel_uint_t; +#define str2kernel_uint_t(string) str2uint64_t(string) +#define KERNEL_UINT_FORMAT "%" PRIu64 +#endif + +#define str2pid_t(string) str2uint32_t(string) + + +// for faster execution, allow the compiler to inline +// these functions that are called thousands of times per second + +static inline uint32_t simple_hash(const char *name) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5; + while (*s) { + hval *= 16777619; + hval ^= (uint32_t) *s++; + } + return hval; +} + +static inline uint32_t simple_uhash(const char *name) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5, c; + while ((c = *s++)) { + if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A'; + hval *= 16777619; + hval ^= c; + } + return hval; +} + +static inline int simple_hash_strcmp(const char *name, const char *b, uint32_t *hash) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5; + int ret = 0; + while (*s) { + if(!ret) ret = *s - *b++; + hval *= 16777619; + hval ^= (uint32_t) *s++; + } + *hash = hval; + return ret; +} + +static inline int str2i(const char *s) { + int n = 0; + char c, negative = (*s == '-'); + + for(c = (negative)?*(++s):*s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + + if(unlikely(negative)) + return -n; + + return n; +} + +static inline long str2l(const char *s) { + long n = 0; + char c, negative = (*s == '-'); + + for(c = (negative)?*(++s):*s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + + if(unlikely(negative)) + return -n; + + return n; +} + +static inline uint32_t str2uint32_t(const char *s) { + uint32_t n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + return n; +} + +static inline uint64_t str2uint64_t(const char *s) { + uint64_t n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + return n; +} + +static inline unsigned long str2ul(const char *s) { + unsigned long n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + return n; +} + +static inline unsigned long long str2ull(const char *s) { + unsigned long long n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + return n; +} + +static inline long long str2ll(const char *s, char **endptr) { + int negative = 0; + + if(unlikely(*s == '-')) { + s++; + negative = 1; + } + else if(unlikely(*s == '+')) + s++; + + long long n = 0; + char c; + for(c = *s; c >= '0' && c <= '9' ; c = *(++s)) { + n *= 10; + n += c - '0'; + } + + if(unlikely(endptr)) + *endptr = (char *)s; + + if(unlikely(negative)) + return -n; + else + return n; +} + +static inline long double str2ld(const char *s, char **endptr) { + int negative = 0; + const char *start = s; + unsigned long long integer_part = 0; + unsigned long decimal_part = 0; + size_t decimal_digits = 0; + + switch(*s) { + case '-': + s++; + negative = 1; + break; + + case '+': + s++; + break; + + case 'n': + if(s[1] == 'a' && s[2] == 'n') { + if(endptr) *endptr = (char *)&s[3]; + return NAN; + } + break; + + case 'i': + if(s[1] == 'n' && s[2] == 'f') { + if(endptr) *endptr = (char *)&s[3]; + return INFINITY; + } + break; + + default: + break; + } + + while (*s >= '0' && *s <= '9') { + integer_part = (integer_part * 10) + (*s - '0'); + s++; + } + + if(unlikely(*s == '.')) { + decimal_part = 0; + s++; + + while (*s >= '0' && *s <= '9') { + decimal_part = (decimal_part * 10) + (*s - '0'); + s++; + decimal_digits++; + } + } + + if(unlikely(*s == 'e' || *s == 'E')) + return strtold(start, endptr); + + if(unlikely(endptr)) + *endptr = (char *)s; + + if(unlikely(negative)) { + if(unlikely(decimal_digits)) + return -((long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits)); + else + return -((long double)integer_part); + } + else { + if(unlikely(decimal_digits)) + return (long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits); + else + return (long double)integer_part; + } +} + +#ifdef NETDATA_STRCMP_OVERRIDE +#ifdef strcmp +#undef strcmp +#endif +#define strcmp(a, b) strsame(a, b) +#endif // NETDATA_STRCMP_OVERRIDE + +static inline int strsame(const char *a, const char *b) { + if(unlikely(a == b)) return 0; + while(*a && *a == *b) { a++; b++; } + return *a - *b; +} + +static inline char *strncpyz(char *dst, const char *src, size_t n) { + char *p = dst; + + while (*src && n--) + *dst++ = *src++; + + *dst = '\0'; + + return p; +} + +static inline void sanitize_json_string(char *dst, char *src, size_t len) { + while (*src != '\0' && len > 1) { + if (*src == '\\' || *src == '\"' || *src < 0x1F) { + if (*src < 0x1F) { + *dst++ = '_'; + src++; + len--; + } else { + *dst++ = '\\'; + *dst++ = *src++; + len -= 2; + } + } else { + *dst++ = *src++; + len--; + } + } + *dst = '\0'; +} + +static inline int read_file(const char *filename, char *buffer, size_t size) { + if(unlikely(!size)) return 3; + + int fd = open(filename, O_RDONLY, 0666); + if(unlikely(fd == -1)) { + buffer[0] = '\0'; + return 1; + } + + ssize_t r = read(fd, buffer, size); + if(unlikely(r == -1)) { + buffer[0] = '\0'; + close(fd); + return 2; + } + buffer[r] = '\0'; + + close(fd); + return 0; +} + +static inline int read_single_number_file(const char *filename, unsigned long long *result) { + char buffer[30 + 1]; + + int ret = read_file(filename, buffer, 30); + if(unlikely(ret)) { + *result = 0; + return ret; + } + + buffer[30] = '\0'; + *result = str2ull(buffer); + return 0; +} + +static inline int read_single_signed_number_file(const char *filename, long long *result) { + char buffer[30 + 1]; + + int ret = read_file(filename, buffer, 30); + if(unlikely(ret)) { + *result = 0; + return ret; + } + + buffer[30] = '\0'; + *result = atoll(buffer); + return 0; +} + +#endif //NETDATA_INLINED_H diff --git a/libnetdata/json/Makefile.am b/libnetdata/json/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/json/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/json/README.md b/libnetdata/json/README.md new file mode 100644 index 0000000..79a48e2 --- /dev/null +++ b/libnetdata/json/README.md @@ -0,0 +1,10 @@ +<!-- +title: "json" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/json/README.md +--> + +# json + +`json` contains a parser for json strings, based on `jsmn` (<https://github.com/zserge/jsmn>), but case you have installed the JSON-C library, the installation script will prefer it, you can also force its use with `--enable-jsonc` in the compilation time. + +[](<>) diff --git a/libnetdata/json/jsmn.c b/libnetdata/json/jsmn.c new file mode 100644 index 0000000..9525358 --- /dev/null +++ b/libnetdata/json/jsmn.c @@ -0,0 +1,328 @@ +#include <stdlib.h> + +#include "jsmn.h" + +/** + * Alloc token + * + * Allocates a fresh unused token from the token pull. + * + * @param parser the controller + * @param tokens the tokens I am working + * @param num_tokens the number total of tokens. + * + * @return it returns the next token to work. + */ +static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, + jsmntok_t *tokens, size_t num_tokens) { + jsmntok_t *tok; + if (parser->toknext >= num_tokens) { + return NULL; + } + tok = &tokens[parser->toknext++]; + tok->start = tok->end = -1; + tok->size = 0; +#ifdef JSMN_PARENT_LINKS + tok->parent = -1; +#endif + return tok; +} + +/** + * Fill Token + * + * Fills token type and boundaries. + * + * @param token the structure to set the values + * @param type is the token type + * @param start is the first position of the value + * @param end is the end of the value + */ +static void jsmn_fill_token(jsmntok_t *token, jsmntype_t type, + int start, int end) { + token->type = type; + token->start = start; + token->end = end; + token->size = 0; +} + +/** + * Parse primitive + * + * Fills next available token with JSON primitive. + * + * @param parser is the control structure + * @param js is the json string + * @param type is the token type + */ +static jsmnerr_t jsmn_parse_primitive(jsmn_parser *parser, const char *js, + size_t len, jsmntok_t *tokens, size_t num_tokens) { + jsmntok_t *token; + int start; + + start = parser->pos; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + switch (js[parser->pos]) { +#ifndef JSMN_STRICT + /* In strict mode primitive must be followed by "," or "}" or "]" */ + case ':': +#endif + case '\t' : case '\r' : case '\n' : case ' ' : + case ',' : case ']' : case '}' : + goto found; + } + if (js[parser->pos] < 32 || js[parser->pos] >= 127) { + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } +#ifdef JSMN_STRICT + /* In strict mode primitive must be followed by a comma/object/array */ + parser->pos = start; + return JSMN_ERROR_PART; +#endif + + found: + if (tokens == NULL) { + parser->pos--; + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + parser->pos--; + return 0; +} + +/** + * Parse string + * + * Fills next token with JSON string. + * + * @param parser is the control structure + * @param js is the json string + * @param len is the js length + * @param tokens is structure with the tokens mapped. + * @param num_tokens is the total number of tokens + * + * @return It returns 0 on success and another integer otherwise + */ +static jsmnerr_t jsmn_parse_string(jsmn_parser *parser, const char *js, + size_t len, jsmntok_t *tokens, size_t num_tokens) { + jsmntok_t *token; + + int start = parser->pos; + + parser->pos++; + + /* Skip starting quote */ + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c = js[parser->pos]; + + /* Quote: end of string */ + if (c == '\"') { + if (tokens == NULL) { + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_STRING, start+1, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + return 0; + } + + /* Backslash: Quoted symbol expected */ + if (c == '\\') { + parser->pos++; + switch (js[parser->pos]) { + /* Allowed escaped symbols */ + case '\"': case '/' : case '\\' : case 'b' : + case 'f' : case 'r' : case 'n' : case 't' : + break; + /* Allows escaped symbol \uXXXX */ + case 'u': + parser->pos++; + int i = 0; + for(; i < 4 && js[parser->pos] != '\0'; i++) { + /* If it isn't a hex character we have an error */ + if(!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ + (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ + (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ + parser->pos = start; + return JSMN_ERROR_INVAL; + } + parser->pos++; + } + parser->pos--; + break; + /* Unexpected symbol */ + default: + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } + } + parser->pos = start; + return JSMN_ERROR_PART; +} + +/** + * JSMN Parse + * + * Parse JSON string and fill tokens. + * + * @param parser the auxiliar vector used to parser + * @param js the string to parse + * @param len the string length + * @param tokens the place to map the tokens + * @param num_tokens the number of tokens present in the tokens structure. + * + * @return It returns the number of tokens present in the string on success or a negative number otherwise + */ +jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, + jsmntok_t *tokens, unsigned int num_tokens) { + jsmnerr_t r; + int i; + jsmntok_t *token; + int count = 0; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c; + jsmntype_t type; + + c = js[parser->pos]; + switch (c) { + case '{': case '[': + count++; + if (tokens == NULL) { + break; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) + return JSMN_ERROR_NOMEM; + if (parser->toksuper != -1) { + tokens[parser->toksuper].size++; +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + } + token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); + token->start = parser->pos; + parser->toksuper = parser->toknext - 1; + break; + case '}': case ']': + if (tokens == NULL) + break; + type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); +#ifdef JSMN_PARENT_LINKS + if (parser->toknext < 1) { + return JSMN_ERROR_INVAL; + } + token = &tokens[parser->toknext - 1]; + for (;;) { + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + token->end = parser->pos + 1; + parser->toksuper = token->parent; + break; + } + if (token->parent == -1) { + break; + } + token = &tokens[token->parent]; + } +#else + for (i = parser->toknext - 1; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + parser->toksuper = -1; + token->end = parser->pos + 1; + break; + } + } + /* Error if unmatched closing bracket */ + if (i == -1) return JSMN_ERROR_INVAL; + for (; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + parser->toksuper = i; + break; + } + } +#endif + break; + case '\"': + r = jsmn_parse_string(parser, js, len, tokens, num_tokens); + if (r < 0) return r; + count++; + if (parser->toksuper != -1 && tokens != NULL) + tokens[parser->toksuper].size++; + break; + case '\t' : case '\r' : case '\n' : case ':' : case ',': case ' ': + break; +#ifdef JSMN_STRICT + /* In strict mode primitives are: numbers and booleans */ + case '-': case '0': case '1' : case '2': case '3' : case '4': + case '5': case '6': case '7' : case '8': case '9': + case 't': case 'f': case 'n' : +#else + /* In non-strict mode every unquoted value is a primitive */ + default: +#endif + r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); + if (r < 0) return r; + count++; + if (parser->toksuper != -1 && tokens != NULL) + tokens[parser->toksuper].size++; + break; + +#ifdef JSMN_STRICT + /* Unexpected char in strict mode */ + default: + return JSMN_ERROR_INVAL; +#endif + } + } + + if (tokens) { + for (i = parser->toknext - 1; i >= 0; i--) { + /* Unmatched opened object or array */ + if (tokens[i].start != -1 && tokens[i].end == -1) { + return JSMN_ERROR_PART; + } + } + } + + return count; +} + +/** + * JSMN Init + * + * Creates a new parser based over a given buffer with an array of tokens + * available. + * + * @param parser is the structure with values to reset + */ +void jsmn_init(jsmn_parser *parser) { + parser->pos = 0; + parser->toknext = 0; + parser->toksuper = -1; +}
\ No newline at end of file diff --git a/libnetdata/json/jsmn.h b/libnetdata/json/jsmn.h new file mode 100644 index 0000000..beff586 --- /dev/null +++ b/libnetdata/json/jsmn.h @@ -0,0 +1,75 @@ +#ifndef __JSMN_H_ +#define __JSMN_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stddef.h> +/** + * JSON type identifier. Basic types are: + * o Object + * o Array + * o String + * o Other primitive: number, boolean (true/false) or null + */ +typedef enum { + JSMN_PRIMITIVE = 0, + JSMN_OBJECT = 1, + JSMN_ARRAY = 2, + JSMN_STRING = 3 +} jsmntype_t; + +typedef enum { + /* Not enough tokens were provided */ + JSMN_ERROR_NOMEM = -1, + /* Invalid character inside JSON string */ + JSMN_ERROR_INVAL = -2, + /* The string is not a full JSON packet, more bytes expected */ + JSMN_ERROR_PART = -3, +} jsmnerr_t; + +/** + * JSON token description. + * + * @param type type (object, array, string etc.) + * @param start start position in JSON data string + * @param end end position in JSON data string + */ +typedef struct { + jsmntype_t type; + int start; + int end; + int size; +#ifdef JSMN_PARENT_LINKS + int parent; +#endif +} jsmntok_t; + +/** + * JSON parser. Contains an array of token blocks available. Also stores + * the string being parsed now and current position in that string + */ +typedef struct { + unsigned int pos; /* offset in the JSON string */ + unsigned int toknext; /* next token to allocate */ + int toksuper; /* superior token node, e.g parent object or array */ +} jsmn_parser; + +/** + * Create JSON parser over an array of tokens + */ +void jsmn_init(jsmn_parser *parser); + +/** + * Run JSON parser. It parses a JSON data string into and array of tokens, each describing + * a single JSON object. + */ +jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len, + jsmntok_t *tokens, unsigned int num_tokens); + +#ifdef __cplusplus +} +#endif + +#endif /* __JSMN_H_ */
\ No newline at end of file diff --git a/libnetdata/json/json.c b/libnetdata/json/json.c new file mode 100644 index 0000000..bd164ae --- /dev/null +++ b/libnetdata/json/json.c @@ -0,0 +1,556 @@ +#include "jsmn.h" +#include "../libnetdata.h" +#include "json.h" +#include "libnetdata/libnetdata.h" +#include "../../health/health.h" + +#define JSON_TOKENS 1024 + +int json_tokens = JSON_TOKENS; + +/** + * Json Tokenise + * + * Map the string given inside tokens. + * + * @param js is the string used to create the tokens + * @param len is the string length + * @param count the number of tokens present in the string + * + * @return it returns the json parsed in tokens + */ +#ifdef ENABLE_JSONC +json_object *json_tokenise(char *js) { + if(!js) { + error("JSON: json string is empty."); + return NULL; + } + + json_object *token = json_tokener_parse(js); + if(!token) { + error("JSON: Invalid json string."); + return NULL; + } + + return token; +} +#else +jsmntok_t *json_tokenise(char *js, size_t len, size_t *count) +{ + int n = json_tokens; + if(!js || !len) { + error("JSON: json string is empty."); + return NULL; + } + + jsmn_parser parser; + jsmn_init(&parser); + + jsmntok_t *tokens = mallocz(sizeof(jsmntok_t) * n); + if(!tokens) return NULL; + + int ret = jsmn_parse(&parser, js, len, tokens, n); + while (ret == JSMN_ERROR_NOMEM) { + n *= 2; + jsmntok_t *new = reallocz(tokens, sizeof(jsmntok_t) * n); + if(!new) { + freez(tokens); + return NULL; + } + tokens = new; + ret = jsmn_parse(&parser, js, len, tokens, n); + } + + if (ret == JSMN_ERROR_INVAL) { + error("JSON: Invalid json string."); + freez(tokens); + return NULL; + } + else if (ret == JSMN_ERROR_PART) { + error("JSON: Truncated JSON string."); + freez(tokens); + return NULL; + } + + if(count) *count = (size_t)ret; + + if(json_tokens < n) json_tokens = n; + return tokens; +} +#endif + +/** + * Callback Print + * + * Set callback print case necesary and wrinte an information inside a buffer to write in the log. + * + * @param e a pointer for a structure that has the complete information about json structure. + * + * @return It always return 0 + */ +int json_callback_print(JSON_ENTRY *e) +{ + BUFFER *wb=buffer_create(300); + + buffer_sprintf(wb,"%s = ", e->name); + char txt[50]; + switch(e->type) { + case JSON_OBJECT: + e->callback_function = json_callback_print; + buffer_strcat(wb,"OBJECT"); + break; + + case JSON_ARRAY: + e->callback_function = json_callback_print; + sprintf(txt,"ARRAY[%lu]", e->data.items); + buffer_strcat(wb, txt); + break; + + case JSON_STRING: + buffer_strcat(wb, e->data.string); + break; + + case JSON_NUMBER: + sprintf(txt,"%Lf", e->data.number); + buffer_strcat(wb,txt); + + break; + + case JSON_BOOLEAN: + buffer_strcat(wb, e->data.boolean?"TRUE":"FALSE"); + break; + + case JSON_NULL: + buffer_strcat(wb,"NULL"); + break; + } + info("JSON: %s", buffer_tostring(wb)); + buffer_free(wb); + return 0; +} + +/** + * JSONC Set String + * + * Set the string value of the structure JSON_ENTRY. + * + * @param e the output structure + */ +static inline void json_jsonc_set_string(JSON_ENTRY *e,char *key,const char *value) { + size_t len = strlen(key); + if(len > JSON_NAME_LEN) + len = JSON_NAME_LEN; + e->type = JSON_STRING; + memcpy(e->name,key,len); + e->name[len] = 0x00; + e->data.string = (char *) value; +} + + +#ifdef ENABLE_JSONC +/** + * JSONC set Boolean + * + * Set the boolean value of the structure JSON_ENTRY + * + * @param e the output structure + * @param value the input value + */ +static inline void json_jsonc_set_boolean(JSON_ENTRY *e,int value) { + e->type = JSON_BOOLEAN; + e->data.boolean = value; +} + +static inline void json_jsonc_set_integer(JSON_ENTRY *e, char *key, int64_t value) { + size_t len = strlen(key); + if(len > JSON_NAME_LEN) + len = JSON_NAME_LEN; + e->type = JSON_NUMBER; + memcpy(e->name, key, len); + e->name[len] = 0; + e->data.number = value; +} + +/** + * Parse Array + * + * Parse the array object. + * + * @param ptr the pointer for the object that we will parse. + * @param callback_data additional data to be used together the callback function + * @param callback_function function used to create a silencer. + */ +static inline void json_jsonc_parse_array(json_object *ptr, void *callback_data,int (*callback_function)(struct json_entry *)) { + int end = json_object_array_length(ptr); + JSON_ENTRY e; + + if(end) { + int i; + i = 0; + + enum json_type type; + do { + json_object *jvalue = json_object_array_get_idx(ptr, i); + if(jvalue) { + e.callback_data = callback_data; + e.type = JSON_OBJECT; + callback_function(&e); + json_object_object_foreach(jvalue, key, val) { + type = json_object_get_type(val); + if (type == json_type_array) { + e.type = JSON_ARRAY; + json_jsonc_parse_array(val, callback_data, callback_function); + } else if (type == json_type_object) { + json_walk(val,callback_data,callback_function); + } else if (type == json_type_string) { + json_jsonc_set_string(&e,key,json_object_get_string(val)); + callback_function(&e); + } else if (type == json_type_boolean) { + json_jsonc_set_boolean(&e,json_object_get_boolean(val)); + callback_function(&e); + } + } + } + + } while (++i < end); + } +} +#else + +/** + * Walk string + * + * Set JSON_ENTRY to string and map the values from jsmntok_t. + * + * @param js the original string + * @param t the tokens + * @param start the first position + * @param e the output structure. + * + * @return It always return 1 + */ +size_t json_walk_string(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e) +{ + char old = js[t[start].end]; + js[t[start].end] = '\0'; + e->original_string = &js[t[start].start]; + + e->type = JSON_STRING; + e->data.string = e->original_string; + if(e->callback_function) e->callback_function(e); + js[t[start].end] = old; + return 1; +} + +/** + * Walk Primitive + * + * Define the data type of the string + * + * @param js the original string + * @param t the tokens + * @param start the first position + * @param e the output structure. + * + * @return It always return 1 + */ +size_t json_walk_primitive(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e) +{ + char old = js[t[start].end]; + js[t[start].end] = '\0'; + e->original_string = &js[t[start].start]; + + switch(e->original_string[0]) { + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': + case '8': case '9': case '-': case '.': + e->type = JSON_NUMBER; + e->data.number = strtold(e->original_string, NULL); + break; + + case 't': case 'T': + e->type = JSON_BOOLEAN; + e->data.boolean = 1; + break; + + case 'f': case 'F': + e->type = JSON_BOOLEAN; + e->data.boolean = 0; + break; + + case 'n': case 'N': + default: + e->type = JSON_NULL; + break; + } + if(e->callback_function) e->callback_function(e); + js[t[start].end] = old; + return 1; +} + +/** + * Array + * + * Measure the array length + * + * @param js the original string + * @param t the tokens + * @param nest the length of structure t + * @param start the first position + * @param e the structure with values and callback to be used inside the function. + * + * @return It returns the array length + */ +size_t json_walk_array(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e) +{ + JSON_ENTRY ne; + + char old = js[t[start].end]; + js[t[start].end] = '\0'; + ne.original_string = &js[t[start].start]; + + memcpy(&ne, e, sizeof(JSON_ENTRY)); + ne.type = JSON_ARRAY; + ne.data.items = t[start].size; + ne.callback_function = e->callback_function; + ne.name[0]='\0'; + ne.fullname[0]='\0'; + if(e->callback_function) e->callback_function(&ne); + js[t[start].end] = old; + + size_t i, init = start, size = t[start].size; + + start++; + for(i = 0; i < size ; i++) { + ne.pos = i; + if (strlen(e->name) > JSON_NAME_LEN - 24 || strlen(e->fullname) > JSON_FULLNAME_LEN -24) { + info("JSON: JSON walk_array ignoring element with name:%s fullname:%s",e->name, e->fullname); + continue; + } + snprintfz(ne.name, JSON_NAME_LEN, "%s[%lu]", e->name, i); + snprintfz(ne.fullname, JSON_FULLNAME_LEN, "%s[%lu]", e->fullname, i); + + switch(t[start].type) { + case JSMN_PRIMITIVE: + start += json_walk_primitive(js, t, start, &ne); + break; + + case JSMN_OBJECT: + start += json_walk_object(js, t, nest + 1, start, &ne); + break; + + case JSMN_ARRAY: + start += json_walk_array(js, t, nest + 1, start, &ne); + break; + + case JSMN_STRING: + start += json_walk_string(js, t, start, &ne); + break; + } + } + return start - init; +} + +/** + * Object + * + * Measure the Object length + * + * @param js the original string + * @param t the tokens + * @param nest the length of structure t + * @param start the first position + * @param e the output structure. + * + * @return It returns the Object length + */ +size_t json_walk_object(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e) +{ + JSON_ENTRY ne = { + .name = "", + .fullname = "", + .callback_data = NULL, + .callback_function = NULL + }; + + char old = js[t[start].end]; + js[t[start].end] = '\0'; + ne.original_string = &js[t[start].start]; + memcpy(&ne, e, sizeof(JSON_ENTRY)); + ne.type = JSON_OBJECT; + ne.callback_function = e->callback_function; + if(e->callback_function) e->callback_function(&ne); + js[t[start].end] = old; + + int key = 1; + size_t i, init = start, size = t[start].size; + + start++; + for(i = 0; i < size ; i++) { + switch(t[start].type) { + case JSMN_PRIMITIVE: + start += json_walk_primitive(js, t, start, &ne); + key = 1; + break; + + case JSMN_OBJECT: + start += json_walk_object(js, t, nest + 1, start, &ne); + key = 1; + break; + + case JSMN_ARRAY: + start += json_walk_array(js, t, nest + 1, start, &ne); + key = 1; + break; + + case JSMN_STRING: + default: + if(key) { + int len = t[start].end - t[start].start; + if (unlikely(len>JSON_NAME_LEN)) len=JSON_NAME_LEN; + strncpy(ne.name, &js[t[start].start], len); + ne.name[len] = '\0'; + len=strlen(e->fullname) + strlen(e->fullname[0]?".":"") + strlen(ne.name); + char *c = mallocz((len+1)*sizeof(char)); + sprintf(c,"%s%s%s", e->fullname, e->fullname[0]?".":"", ne.name); + if (unlikely(len>JSON_FULLNAME_LEN)) len=JSON_FULLNAME_LEN; + strncpy(ne.fullname, c, len); + freez(c); + start++; + key = 0; + } + else { + start += json_walk_string(js, t, start, &ne); + key = 1; + } + break; + } + } + return start - init; +} +#endif + +/** + * Tree + * + * Call the correct walk function according its type. + * + * @param t the json object to work + * @param callback_data additional data to be used together the callback function + * @param callback_function function used to create a silencer. + * + * @return It always return 1 + */ +#ifdef ENABLE_JSONC +size_t json_walk(json_object *t, void *callback_data, int (*callback_function)(struct json_entry *)) { + JSON_ENTRY e; + + e.callback_data = callback_data; + enum json_type type; + json_object_object_foreach(t, key, val) { + type = json_object_get_type(val); + if (type == json_type_array) { + e.type = JSON_ARRAY; + json_jsonc_parse_array(val,NULL,health_silencers_json_read_callback); + } else if (type == json_type_object) { + e.type = JSON_OBJECT; + } else if (type == json_type_string) { + json_jsonc_set_string(&e,key,json_object_get_string(val)); + callback_function(&e); + } else if (type == json_type_boolean) { + json_jsonc_set_boolean(&e,json_object_get_boolean(val)); + callback_function(&e); + } else if (type == json_type_int) { + json_jsonc_set_integer(&e,key,json_object_get_int64(val)); + callback_function(&e); + } + } + + return 1; +} +#else +/** + * Tree + * + * Call the correct walk function according its type. + * + * @param js the original string + * @param t the tokens + * @param callback_data additional data to be used together the callback function + * @param callback_function function used to create a silencer. + * + * @return It always return 1 + */ +size_t json_walk_tree(char *js, jsmntok_t *t, void *callback_data, int (*callback_function)(struct json_entry *)) +{ + JSON_ENTRY e = { + .name = "", + .fullname = "", + .callback_data = callback_data, + .callback_function = callback_function + }; + + switch (t[0].type) { + case JSMN_OBJECT: + e.type = JSON_OBJECT; + json_walk_object(js, t, 0, 0, &e); + break; + + case JSMN_ARRAY: + e.type = JSON_ARRAY; + json_walk_array(js, t, 0, 0, &e); + break; + + case JSMN_PRIMITIVE: + case JSMN_STRING: + break; + } + + return 1; +} +#endif + +/** + * JSON Parse + * + * Parse the json message with the callback function + * + * @param js the string that the callback function will parse + * @param callback_data additional data to be used together the callback function + * @param callback_function function used to create a silencer. + * + * @return JSON_OK case everything happend as expected, JSON_CANNOT_PARSE case there were errors in the + * parsing procces and JSON_CANNOT_DOWNLOAD case the string given(js) is NULL. + */ +int json_parse(char *js, void *callback_data, int (*callback_function)(JSON_ENTRY *)) +{ + if(js) { +#ifdef ENABLE_JSONC + json_object *tokens = json_tokenise(js); +#else + size_t count; + jsmntok_t *tokens = json_tokenise(js, strlen(js), &count); +#endif + + if(tokens) { +#ifdef ENABLE_JSONC + json_walk(tokens, callback_data, callback_function); + json_object_put(tokens); +#else + json_walk_tree(js, tokens, callback_data, callback_function); + freez(tokens); +#endif + return JSON_OK; + } + + return JSON_CANNOT_PARSE; + } + + return JSON_CANNOT_DOWNLOAD; +} + +/* +int json_test(char *str) +{ + return json_parse(str, NULL, json_callback_print); +} + */
\ No newline at end of file diff --git a/libnetdata/json/json.h b/libnetdata/json/json.h new file mode 100644 index 0000000..79b58b1 --- /dev/null +++ b/libnetdata/json/json.h @@ -0,0 +1,72 @@ +#ifndef CHECKIN_JSON_H +#define CHECKIN_JSON_H 1 + + +#if ENABLE_JSONC +# include <json-c/json.h> +#endif + +#include "jsmn.h" + +//https://www.ibm.com/support/knowledgecenter/en/SS9H2Y_7.6.0/com.ibm.dp.doc/json_parserlimits.html +#define JSON_NAME_LEN 256 +#define JSON_FULLNAME_LEN 1024 + +typedef enum { + JSON_OBJECT = 0, + JSON_ARRAY = 1, + JSON_STRING = 2, + JSON_NUMBER = 3, + JSON_BOOLEAN = 4, + JSON_NULL = 5, +} JSON_ENTRY_TYPE; + +typedef struct json_entry { + JSON_ENTRY_TYPE type; + char name[JSON_NAME_LEN + 1]; + char fullname[JSON_FULLNAME_LEN + 1]; + union { + char *string; // type == JSON_STRING + long double number; // type == JSON_NUMBER + int boolean; // type == JSON_BOOLEAN + size_t items; // type == JSON_ARRAY + } data; + size_t pos; // the position of this item in its parent + + char *original_string; + + void *callback_data; + int (*callback_function)(struct json_entry *); +} JSON_ENTRY; + +// ---------------------------------------------------------------------------- +// public functions + +#define JSON_OK 0 +#define JSON_CANNOT_DOWNLOAD 1 +#define JSON_CANNOT_PARSE 2 + +int json_parse(char *js, void *callback_data, int (*callback_function)(JSON_ENTRY *)); + + +// ---------------------------------------------------------------------------- +// private functions + +#ifdef ENABLE_JSONC +json_object *json_tokenise(char *js); +size_t json_walk(json_object *t, void *callback_data, int (*callback_function)(struct json_entry *)); +#else +jsmntok_t *json_tokenise(char *js, size_t len, size_t *count); +size_t json_walk_tree(char *js, jsmntok_t *t, void *callback_data, int (*callback_function)(struct json_entry *)); +#endif + +size_t json_walk_object(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e); +size_t json_walk_array(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e); +size_t json_walk_string(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e); +size_t json_walk_primitive(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e); + +int json_callback_print(JSON_ENTRY *e); + + + +#endif
\ No newline at end of file diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c new file mode 100644 index 0000000..325df3f --- /dev/null +++ b/libnetdata/libnetdata.c @@ -0,0 +1,1493 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata.h" + +#ifdef __APPLE__ +#define INHERIT_NONE 0 +#endif /* __APPLE__ */ +#if defined(__FreeBSD__) || defined(__APPLE__) +# define O_NOATIME 0 +# define MADV_DONTFORK INHERIT_NONE +#endif /* __FreeBSD__ || __APPLE__*/ + +struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 }; +int enable_ksm = 1; + +volatile sig_atomic_t netdata_exit = 0; +const char *os_type = NETDATA_OS_TYPE; +const char *program_version = VERSION; + +// ---------------------------------------------------------------------------- +// memory allocation functions that handle failures + +// although netdata does not use memory allocations too often (netdata tries to +// maintain its memory footprint stable during runtime, i.e. all buffers are +// allocated during initialization and are adapted to current use throughout +// its lifetime), these can be used to override the default system allocation +// routines. + +#ifdef NETDATA_LOG_ALLOCATIONS +static __thread struct memory_statistics { + volatile ssize_t malloc_calls_made; + volatile ssize_t calloc_calls_made; + volatile ssize_t realloc_calls_made; + volatile ssize_t strdup_calls_made; + volatile ssize_t free_calls_made; + volatile ssize_t memory_calls_made; + volatile ssize_t allocated_memory; + volatile ssize_t mmapped_memory; +} memory_statistics = { 0, 0, 0, 0, 0, 0, 0, 0 }; + +__thread size_t log_thread_memory_allocations = 0; + +static inline void print_allocations(const char *file, const char *function, const unsigned long line, const char *type, size_t size) { + static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 }; + + fprintf(stderr, "%s iteration %zu MEMORY TRACE: %lu@%s : %s : %s : %zu\n", + netdata_thread_tag(), + log_thread_memory_allocations, + line, file, function, + type, size + ); + + fprintf(stderr, "%s iteration %zu MEMORY ALLOCATIONS: (%04lu@%-40.40s:%-40.40s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): %s : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n", + netdata_thread_tag(), + log_thread_memory_allocations, + line, file, function, + (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory, + (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory, + type, + memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made, + memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made, + memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made, + memory_statistics.strdup_calls_made, memory_statistics.strdup_calls_made - old.strdup_calls_made, + memory_statistics.free_calls_made, memory_statistics.free_calls_made - old.free_calls_made + ); + + memcpy(&old, &memory_statistics, sizeof(struct memory_statistics)); +} + +static inline void mmap_accounting(size_t size) { + if(log_thread_memory_allocations) { + memory_statistics.memory_calls_made++; + memory_statistics.mmapped_memory += size; + } +} + +void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) { + if(log_thread_memory_allocations) { + memory_statistics.memory_calls_made++; + memory_statistics.malloc_calls_made++; + memory_statistics.allocated_memory += size; + print_allocations(file, function, line, "malloc()", size); + } + + size_t *n = (size_t *)malloc(sizeof(size_t) + size); + if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size); + *n = size; + return (void *)&n[1]; +} + +void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) { + size = nmemb * size; + + if(log_thread_memory_allocations) { + memory_statistics.memory_calls_made++; + memory_statistics.calloc_calls_made++; + memory_statistics.allocated_memory += size; + print_allocations(file, function, line, "calloc()", size); + } + + size_t *n = (size_t *)calloc(1, sizeof(size_t) + size); + if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size); + *n = size; + return (void *)&n[1]; +} + +void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size) { + if(!ptr) return mallocz_int(file, function, line, size); + + size_t *n = (size_t *)ptr; + n--; + size_t old_size = *n; + + n = realloc(n, sizeof(size_t) + size); + if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size); + + if(log_thread_memory_allocations) { + memory_statistics.memory_calls_made++; + memory_statistics.realloc_calls_made++; + memory_statistics.allocated_memory += (size - old_size); + print_allocations(file, function, line, "realloc()", size - old_size); + } + + *n = size; + return (void *)&n[1]; +} + +char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) { + size_t size = strlen(s) + 1; + + if(log_thread_memory_allocations) { + memory_statistics.memory_calls_made++; + memory_statistics.strdup_calls_made++; + memory_statistics.allocated_memory += size; + print_allocations(file, function, line, "strdup()", size); + } + + size_t *n = (size_t *)malloc(sizeof(size_t) + size); + if (unlikely(!n)) fatal("strdupz() cannot allocate %zu bytes of memory.", size); + + *n = size; + char *t = (char *)&n[1]; + strcpy(t, s); + return t; +} + +void freez_int(const char *file, const char *function, const unsigned long line, void *ptr) { + if(unlikely(!ptr)) return; + + size_t *n = (size_t *)ptr; + n--; + size_t size = *n; + + if(log_thread_memory_allocations) { + memory_statistics.memory_calls_made++; + memory_statistics.free_calls_made++; + memory_statistics.allocated_memory -= size; + print_allocations(file, function, line, "free()", size); + } + + free(n); +} +#else + +char *strdupz(const char *s) { + char *t = strdup(s); + if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s); + return t; +} + +// If ptr is NULL, no operation is performed. +void freez(void *ptr) { + free(ptr); +} + +void *mallocz(size_t size) { + void *p = malloc(size); + if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size); + return p; +} + +void *callocz(size_t nmemb, size_t size) { + void *p = calloc(nmemb, size); + if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size); + return p; +} + +void *reallocz(void *ptr, size_t size) { + void *p = realloc(ptr, size); + if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size); + return p; +} + +#endif + +// -------------------------------------------------------------------------------------------------------------------- + +void json_escape_string(char *dst, const char *src, size_t size) { + const char *t; + char *d = dst, *e = &dst[size - 1]; + + for(t = src; *t && d < e ;t++) { + if(unlikely(*t == '\\' || *t == '"')) { + if(unlikely(d + 1 >= e)) break; + *d++ = '\\'; + } + *d++ = *t; + } + + *d = '\0'; +} + +void json_fix_string(char *s) { + unsigned char c; + while((c = (unsigned char)*s)) { + if(unlikely(c == '\\')) + *s++ = '/'; + else if(unlikely(c == '"')) + *s++ = '\''; + else if(unlikely(isspace(c) || iscntrl(c))) + *s++ = ' '; + else if(unlikely(!isprint(c) || c > 127)) + *s++ = '_'; + else + s++; + } +} + +unsigned char netdata_map_chart_names[256] = { + [0] = '\0', // + [1] = '_', // + [2] = '_', // + [3] = '_', // + [4] = '_', // + [5] = '_', // + [6] = '_', // + [7] = '_', // + [8] = '_', // + [9] = '_', // + [10] = '_', // + [11] = '_', // + [12] = '_', // + [13] = '_', // + [14] = '_', // + [15] = '_', // + [16] = '_', // + [17] = '_', // + [18] = '_', // + [19] = '_', // + [20] = '_', // + [21] = '_', // + [22] = '_', // + [23] = '_', // + [24] = '_', // + [25] = '_', // + [26] = '_', // + [27] = '_', // + [28] = '_', // + [29] = '_', // + [30] = '_', // + [31] = '_', // + [32] = '_', // + [33] = '_', // ! + [34] = '_', // " + [35] = '_', // # + [36] = '_', // $ + [37] = '_', // % + [38] = '_', // & + [39] = '_', // ' + [40] = '_', // ( + [41] = '_', // ) + [42] = '_', // * + [43] = '_', // + + [44] = '.', // , + [45] = '-', // - + [46] = '.', // . + [47] = '/', // / + [48] = '0', // 0 + [49] = '1', // 1 + [50] = '2', // 2 + [51] = '3', // 3 + [52] = '4', // 4 + [53] = '5', // 5 + [54] = '6', // 6 + [55] = '7', // 7 + [56] = '8', // 8 + [57] = '9', // 9 + [58] = '_', // : + [59] = '_', // ; + [60] = '_', // < + [61] = '_', // = + [62] = '_', // > + [63] = '_', // ? + [64] = '_', // @ + [65] = 'a', // A + [66] = 'b', // B + [67] = 'c', // C + [68] = 'd', // D + [69] = 'e', // E + [70] = 'f', // F + [71] = 'g', // G + [72] = 'h', // H + [73] = 'i', // I + [74] = 'j', // J + [75] = 'k', // K + [76] = 'l', // L + [77] = 'm', // M + [78] = 'n', // N + [79] = 'o', // O + [80] = 'p', // P + [81] = 'q', // Q + [82] = 'r', // R + [83] = 's', // S + [84] = 't', // T + [85] = 'u', // U + [86] = 'v', // V + [87] = 'w', // W + [88] = 'x', // X + [89] = 'y', // Y + [90] = 'z', // Z + [91] = '_', // [ + [92] = '/', // backslash + [93] = '_', // ] + [94] = '_', // ^ + [95] = '_', // _ + [96] = '_', // ` + [97] = 'a', // a + [98] = 'b', // b + [99] = 'c', // c + [100] = 'd', // d + [101] = 'e', // e + [102] = 'f', // f + [103] = 'g', // g + [104] = 'h', // h + [105] = 'i', // i + [106] = 'j', // j + [107] = 'k', // k + [108] = 'l', // l + [109] = 'm', // m + [110] = 'n', // n + [111] = 'o', // o + [112] = 'p', // p + [113] = 'q', // q + [114] = 'r', // r + [115] = 's', // s + [116] = 't', // t + [117] = 'u', // u + [118] = 'v', // v + [119] = 'w', // w + [120] = 'x', // x + [121] = 'y', // y + [122] = 'z', // z + [123] = '_', // { + [124] = '_', // | + [125] = '_', // } + [126] = '_', // ~ + [127] = '_', // + [128] = '_', // + [129] = '_', // + [130] = '_', // + [131] = '_', // + [132] = '_', // + [133] = '_', // + [134] = '_', // + [135] = '_', // + [136] = '_', // + [137] = '_', // + [138] = '_', // + [139] = '_', // + [140] = '_', // + [141] = '_', // + [142] = '_', // + [143] = '_', // + [144] = '_', // + [145] = '_', // + [146] = '_', // + [147] = '_', // + [148] = '_', // + [149] = '_', // + [150] = '_', // + [151] = '_', // + [152] = '_', // + [153] = '_', // + [154] = '_', // + [155] = '_', // + [156] = '_', // + [157] = '_', // + [158] = '_', // + [159] = '_', // + [160] = '_', // + [161] = '_', // + [162] = '_', // + [163] = '_', // + [164] = '_', // + [165] = '_', // + [166] = '_', // + [167] = '_', // + [168] = '_', // + [169] = '_', // + [170] = '_', // + [171] = '_', // + [172] = '_', // + [173] = '_', // + [174] = '_', // + [175] = '_', // + [176] = '_', // + [177] = '_', // + [178] = '_', // + [179] = '_', // + [180] = '_', // + [181] = '_', // + [182] = '_', // + [183] = '_', // + [184] = '_', // + [185] = '_', // + [186] = '_', // + [187] = '_', // + [188] = '_', // + [189] = '_', // + [190] = '_', // + [191] = '_', // + [192] = '_', // + [193] = '_', // + [194] = '_', // + [195] = '_', // + [196] = '_', // + [197] = '_', // + [198] = '_', // + [199] = '_', // + [200] = '_', // + [201] = '_', // + [202] = '_', // + [203] = '_', // + [204] = '_', // + [205] = '_', // + [206] = '_', // + [207] = '_', // + [208] = '_', // + [209] = '_', // + [210] = '_', // + [211] = '_', // + [212] = '_', // + [213] = '_', // + [214] = '_', // + [215] = '_', // + [216] = '_', // + [217] = '_', // + [218] = '_', // + [219] = '_', // + [220] = '_', // + [221] = '_', // + [222] = '_', // + [223] = '_', // + [224] = '_', // + [225] = '_', // + [226] = '_', // + [227] = '_', // + [228] = '_', // + [229] = '_', // + [230] = '_', // + [231] = '_', // + [232] = '_', // + [233] = '_', // + [234] = '_', // + [235] = '_', // + [236] = '_', // + [237] = '_', // + [238] = '_', // + [239] = '_', // + [240] = '_', // + [241] = '_', // + [242] = '_', // + [243] = '_', // + [244] = '_', // + [245] = '_', // + [246] = '_', // + [247] = '_', // + [248] = '_', // + [249] = '_', // + [250] = '_', // + [251] = '_', // + [252] = '_', // + [253] = '_', // + [254] = '_', // + [255] = '_' // +}; + +// make sure the supplied string +// is good for a netdata chart/dimension ID/NAME +void netdata_fix_chart_name(char *s) { + while ((*s = netdata_map_chart_names[(unsigned char) *s])) s++; +} + +unsigned char netdata_map_chart_ids[256] = { + [0] = '\0', // + [1] = '_', // + [2] = '_', // + [3] = '_', // + [4] = '_', // + [5] = '_', // + [6] = '_', // + [7] = '_', // + [8] = '_', // + [9] = '_', // + [10] = '_', // + [11] = '_', // + [12] = '_', // + [13] = '_', // + [14] = '_', // + [15] = '_', // + [16] = '_', // + [17] = '_', // + [18] = '_', // + [19] = '_', // + [20] = '_', // + [21] = '_', // + [22] = '_', // + [23] = '_', // + [24] = '_', // + [25] = '_', // + [26] = '_', // + [27] = '_', // + [28] = '_', // + [29] = '_', // + [30] = '_', // + [31] = '_', // + [32] = '_', // + [33] = '_', // ! + [34] = '_', // " + [35] = '_', // # + [36] = '_', // $ + [37] = '_', // % + [38] = '_', // & + [39] = '_', // ' + [40] = '_', // ( + [41] = '_', // ) + [42] = '_', // * + [43] = '_', // + + [44] = '.', // , + [45] = '-', // - + [46] = '.', // . + [47] = '_', // / + [48] = '0', // 0 + [49] = '1', // 1 + [50] = '2', // 2 + [51] = '3', // 3 + [52] = '4', // 4 + [53] = '5', // 5 + [54] = '6', // 6 + [55] = '7', // 7 + [56] = '8', // 8 + [57] = '9', // 9 + [58] = '_', // : + [59] = '_', // ; + [60] = '_', // < + [61] = '_', // = + [62] = '_', // > + [63] = '_', // ? + [64] = '_', // @ + [65] = 'a', // A + [66] = 'b', // B + [67] = 'c', // C + [68] = 'd', // D + [69] = 'e', // E + [70] = 'f', // F + [71] = 'g', // G + [72] = 'h', // H + [73] = 'i', // I + [74] = 'j', // J + [75] = 'k', // K + [76] = 'l', // L + [77] = 'm', // M + [78] = 'n', // N + [79] = 'o', // O + [80] = 'p', // P + [81] = 'q', // Q + [82] = 'r', // R + [83] = 's', // S + [84] = 't', // T + [85] = 'u', // U + [86] = 'v', // V + [87] = 'w', // W + [88] = 'x', // X + [89] = 'y', // Y + [90] = 'z', // Z + [91] = '_', // [ + [92] = '/', // backslash + [93] = '_', // ] + [94] = '_', // ^ + [95] = '_', // _ + [96] = '_', // ` + [97] = 'a', // a + [98] = 'b', // b + [99] = 'c', // c + [100] = 'd', // d + [101] = 'e', // e + [102] = 'f', // f + [103] = 'g', // g + [104] = 'h', // h + [105] = 'i', // i + [106] = 'j', // j + [107] = 'k', // k + [108] = 'l', // l + [109] = 'm', // m + [110] = 'n', // n + [111] = 'o', // o + [112] = 'p', // p + [113] = 'q', // q + [114] = 'r', // r + [115] = 's', // s + [116] = 't', // t + [117] = 'u', // u + [118] = 'v', // v + [119] = 'w', // w + [120] = 'x', // x + [121] = 'y', // y + [122] = 'z', // z + [123] = '_', // { + [124] = '_', // | + [125] = '_', // } + [126] = '_', // ~ + [127] = '_', // + [128] = '_', // + [129] = '_', // + [130] = '_', // + [131] = '_', // + [132] = '_', // + [133] = '_', // + [134] = '_', // + [135] = '_', // + [136] = '_', // + [137] = '_', // + [138] = '_', // + [139] = '_', // + [140] = '_', // + [141] = '_', // + [142] = '_', // + [143] = '_', // + [144] = '_', // + [145] = '_', // + [146] = '_', // + [147] = '_', // + [148] = '_', // + [149] = '_', // + [150] = '_', // + [151] = '_', // + [152] = '_', // + [153] = '_', // + [154] = '_', // + [155] = '_', // + [156] = '_', // + [157] = '_', // + [158] = '_', // + [159] = '_', // + [160] = '_', // + [161] = '_', // + [162] = '_', // + [163] = '_', // + [164] = '_', // + [165] = '_', // + [166] = '_', // + [167] = '_', // + [168] = '_', // + [169] = '_', // + [170] = '_', // + [171] = '_', // + [172] = '_', // + [173] = '_', // + [174] = '_', // + [175] = '_', // + [176] = '_', // + [177] = '_', // + [178] = '_', // + [179] = '_', // + [180] = '_', // + [181] = '_', // + [182] = '_', // + [183] = '_', // + [184] = '_', // + [185] = '_', // + [186] = '_', // + [187] = '_', // + [188] = '_', // + [189] = '_', // + [190] = '_', // + [191] = '_', // + [192] = '_', // + [193] = '_', // + [194] = '_', // + [195] = '_', // + [196] = '_', // + [197] = '_', // + [198] = '_', // + [199] = '_', // + [200] = '_', // + [201] = '_', // + [202] = '_', // + [203] = '_', // + [204] = '_', // + [205] = '_', // + [206] = '_', // + [207] = '_', // + [208] = '_', // + [209] = '_', // + [210] = '_', // + [211] = '_', // + [212] = '_', // + [213] = '_', // + [214] = '_', // + [215] = '_', // + [216] = '_', // + [217] = '_', // + [218] = '_', // + [219] = '_', // + [220] = '_', // + [221] = '_', // + [222] = '_', // + [223] = '_', // + [224] = '_', // + [225] = '_', // + [226] = '_', // + [227] = '_', // + [228] = '_', // + [229] = '_', // + [230] = '_', // + [231] = '_', // + [232] = '_', // + [233] = '_', // + [234] = '_', // + [235] = '_', // + [236] = '_', // + [237] = '_', // + [238] = '_', // + [239] = '_', // + [240] = '_', // + [241] = '_', // + [242] = '_', // + [243] = '_', // + [244] = '_', // + [245] = '_', // + [246] = '_', // + [247] = '_', // + [248] = '_', // + [249] = '_', // + [250] = '_', // + [251] = '_', // + [252] = '_', // + [253] = '_', // + [254] = '_', // + [255] = '_' // +}; + +// make sure the supplied string +// is good for a netdata chart/dimension ID/NAME +void netdata_fix_chart_id(char *s) { + while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++; +} + +/* +// http://stackoverflow.com/questions/7666509/hash-function-for-string +uint32_t simple_hash(const char *name) +{ + const char *s = name; + uint32_t hash = 5381; + int i; + + while((i = *s++)) hash = ((hash << 5) + hash) + i; + + // fprintf(stderr, "HASH: %lu %s\n", hash, name); + + return hash; +} +*/ + +/* +// http://isthe.com/chongo/tech/comp/fnv/#FNV-1a +uint32_t simple_hash(const char *name) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5; + + // FNV-1a algorithm + while (*s) { + // multiply by the 32 bit FNV magic prime mod 2^32 + // NOTE: No need to optimize with left shifts. + // GCC will use imul instruction anyway. + // Tested with 'gcc -O3 -S' + //hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24); + hval *= 16777619; + + // xor the bottom with the current octet + hval ^= (uint32_t) *s++; + } + + // fprintf(stderr, "HASH: %u = %s\n", hval, name); + return hval; +} + +uint32_t simple_uhash(const char *name) { + unsigned char *s = (unsigned char *) name; + uint32_t hval = 0x811c9dc5, c; + + // FNV-1a algorithm + while ((c = *s++)) { + if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A'; + hval *= 16777619; + hval ^= c; + } + return hval; +} +*/ + +/* +// http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx +// one at a time hash +uint32_t simple_hash(const char *name) { + unsigned char *s = (unsigned char *)name; + uint32_t h = 0; + + while(*s) { + h += *s++; + h += (h << 10); + h ^= (h >> 6); + } + + h += (h << 3); + h ^= (h >> 11); + h += (h << 15); + + // fprintf(stderr, "HASH: %u = %s\n", h, name); + + return h; +} +*/ + +void strreverse(char *begin, char *end) { + while (end > begin) { + // clearer code. + char aux = *end; + *end-- = *begin; + *begin++ = aux; + } +} + +char *strsep_on_1char(char **ptr, char c) { + if(unlikely(!ptr || !*ptr)) + return NULL; + + // remember the position we started + char *s = *ptr; + + // skip separators in front + while(*s == c) s++; + char *ret = s; + + // find the next separator + while(*s++) { + if(unlikely(*s == c)) { + *s++ = '\0'; + *ptr = s; + return ret; + } + } + + *ptr = NULL; + return ret; +} + +char *mystrsep(char **ptr, char *s) { + char *p = ""; + while (p && !p[0] && *ptr) p = strsep(ptr, s); + return (p); +} + +char *trim(char *s) { + // skip leading spaces + while (*s && isspace(*s)) s++; + if (!*s) return NULL; + + // skip tailing spaces + // this way is way faster. Writes only one NUL char. + ssize_t l = strlen(s); + if (--l >= 0) { + char *p = s + l; + while (p > s && isspace(*p)) p--; + *++p = '\0'; + } + + if (!*s) return NULL; + + return s; +} + +inline char *trim_all(char *buffer) { + char *d = buffer, *s = buffer; + + // skip spaces + while(isspace(*s)) s++; + + while(*s) { + // copy the non-space part + while(*s && !isspace(*s)) *d++ = *s++; + + // add a space if we have to + if(*s && isspace(*s)) { + *d++ = ' '; + s++; + } + + // skip spaces + while(isspace(*s)) s++; + } + + *d = '\0'; + + if(d > buffer) { + d--; + if(isspace(*d)) *d = '\0'; + } + + if(!buffer[0]) return NULL; + return buffer; +} + +static int memory_file_open(const char *filename, size_t size) { + // info("memory_file_open('%s', %zu", filename, size); + + int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664); + if (fd != -1) { + if (lseek(fd, size, SEEK_SET) == (off_t) size) { + if (write(fd, "", 1) == 1) { + if (ftruncate(fd, size)) + error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size); + } + else error("Cannot write to file '%s' at position %zu.", filename, size); + } + else error("Cannot seek file '%s' to size %zu.", filename, size); + } + else error("Cannot create/open file '%s'.", filename); + + return fd; +} + +// mmap_shared is used for memory mode = map +static void *memory_file_mmap(const char *filename, size_t size, int flags) { + // info("memory_file_mmap('%s', %zu", filename, size); + static int log_madvise = 1; + + int fd = -1; + if(filename) { + fd = memory_file_open(filename, size); + if(fd == -1) return MAP_FAILED; + } + + void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0); + if (mem != MAP_FAILED) { +#ifdef NETDATA_LOG_ALLOCATIONS + mmap_accounting(size); +#endif + int advise = MADV_SEQUENTIAL | MADV_DONTFORK; + if (flags & MAP_SHARED) advise |= MADV_WILLNEED; + + if (madvise(mem, size, advise) != 0 && log_madvise) { + error("Cannot advise the kernel about shared memory usage."); + log_madvise--; + } + } + + if(fd != -1) + close(fd); + + return mem; +} + +#ifdef MADV_MERGEABLE +static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) { + // info("memory_file_mmap_ksm('%s', %zu", filename, size); + static int log_madvise_2 = 1, log_madvise_3 = 1; + + int fd = -1; + if(filename) { + fd = memory_file_open(filename, size); + if(fd == -1) return MAP_FAILED; + } + + void *mem = mmap(NULL, size, PROT_READ | PROT_WRITE, flags | MAP_ANONYMOUS, -1, 0); + if (mem != MAP_FAILED) { +#ifdef NETDATA_LOG_ALLOCATIONS + mmap_accounting(size); +#endif + if(fd != -1) { + if (lseek(fd, 0, SEEK_SET) == 0) { + if (read(fd, mem, size) != (ssize_t) size) + error("Cannot read from file '%s'", filename); + } + else error("Cannot seek to beginning of file '%s'.", filename); + } + + // don't use MADV_SEQUENTIAL|MADV_DONTFORK, they disable MADV_MERGEABLE + if (madvise(mem, size, MADV_SEQUENTIAL | MADV_DONTFORK) != 0 && log_madvise_2) { + error("Cannot advise the kernel about the memory usage (MADV_SEQUENTIAL|MADV_DONTFORK) of file '%s'.", filename); + log_madvise_2--; + } + + if (madvise(mem, size, MADV_MERGEABLE) != 0 && log_madvise_3) { + error("Cannot advise the kernel about the memory usage (MADV_MERGEABLE) of file '%s'.", filename); + log_madvise_3--; + } + } + + if(fd != -1) + close(fd); + + return mem; +} +#else +static void *memory_file_mmap_ksm(const char *filename, size_t size, int flags) { + // info("memory_file_mmap_ksm FALLBACK ('%s', %zu", filename, size); + + if(filename) + return memory_file_mmap(filename, size, flags); + + // when KSM is not available and no filename is given (memory mode = ram), + // we just report failure + return MAP_FAILED; +} +#endif + +void *mymmap(const char *filename, size_t size, int flags, int ksm) { + void *mem = NULL; + + if (filename && (flags & MAP_SHARED || !enable_ksm || !ksm)) + // memory mode = map | save + // when KSM is not enabled + // MAP_SHARED is used for memory mode = map (no KSM possible) + mem = memory_file_mmap(filename, size, flags); + + else + // memory mode = save | ram + // when KSM is enabled + // for memory mode = ram, the filename is NULL + mem = memory_file_mmap_ksm(filename, size, flags); + + if(mem == MAP_FAILED) return NULL; + + errno = 0; + return mem; +} + +int memory_file_save(const char *filename, void *mem, size_t size) { + char tmpfilename[FILENAME_MAX + 1]; + + snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid()); + + int fd = open(tmpfilename, O_RDWR | O_CREAT | O_NOATIME, 0664); + if (fd < 0) { + error("Cannot create/open file '%s'.", filename); + return -1; + } + + if (write(fd, mem, size) != (ssize_t) size) { + error("Cannot write to file '%s' %ld bytes.", filename, (long) size); + close(fd); + return -1; + } + + close(fd); + + if (rename(tmpfilename, filename)) { + error("Cannot rename '%s' to '%s'", tmpfilename, filename); + return -1; + } + + return 0; +} + +int fd_is_valid(int fd) { + return fcntl(fd, F_GETFD) != -1 || errno != EBADF; +} + +char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) { + char *s = fgets(buf, (int)buf_size, fp); + if (!s) return NULL; + + char *t = s; + if (*t != '\0') { + // find the string end + while (*++t != '\0'); + + // trim trailing spaces/newlines/tabs + while (--t > s && *t == '\n') + *t = '\0'; + } + + if (len) + *len = t - s + 1; + + return s; +} + +int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args) { + int size = vsnprintf(dst, n, fmt, args); + + if (unlikely((size_t) size > n)) { + // truncated + size = (int)n; + } + + dst[size] = '\0'; + return size; +} + +int snprintfz(char *dst, size_t n, const char *fmt, ...) { + va_list args; + + va_start(args, fmt); + int ret = vsnprintfz(dst, n, fmt, args); + va_end(args); + + return ret; +} + +/* +// poor man cycle counting +static unsigned long tsc; +void begin_tsc(void) { + unsigned long a, d; + asm volatile ("cpuid\nrdtsc" : "=a" (a), "=d" (d) : "0" (0) : "ebx", "ecx"); + tsc = ((unsigned long)d << 32) | (unsigned long)a; +} +unsigned long end_tsc(void) { + unsigned long a, d; + asm volatile ("rdtscp" : "=a" (a), "=d" (d) : : "ecx"); + return (((unsigned long)d << 32) | (unsigned long)a) - tsc; +} +*/ + +int recursively_delete_dir(const char *path, const char *reason) { + DIR *dir = opendir(path); + if(!dir) { + error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path); + return -1; + } + + int ret = 0; + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR + && ( + (de->d_name[0] == '.' && de->d_name[1] == '\0') + || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + )) + continue; + + char fullpath[FILENAME_MAX + 1]; + snprintfz(fullpath, FILENAME_MAX, "%s/%s", path, de->d_name); + + if(de->d_type == DT_DIR) { + int r = recursively_delete_dir(fullpath, reason); + if(r > 0) ret += r; + continue; + } + + info("Deleting %s file '%s'", reason?reason:"", fullpath); + if(unlikely(unlink(fullpath) == -1)) + error("Cannot delete %s file '%s'", reason?reason:"", fullpath); + else + ret++; + } + + info("Deleting empty directory '%s'", path); + if(unlikely(rmdir(path) == -1)) + error("Cannot delete empty directory '%s'", path); + else + ret++; + + closedir(dir); + + return ret; +} + +static int is_virtual_filesystem(const char *path, char **reason) { + +#if defined(__APPLE__) || defined(__FreeBSD__) + (void)path; + (void)reason; +#else + struct statfs stat; + // stat.f_fsid.__val[0] is a file system id + // stat.f_fsid.__val[1] is the inode + // so their combination uniquely identifies the file/dir + + if (statfs(path, &stat) == -1) { + if(reason) *reason = "failed to statfs()"; + return -1; + } + + if(stat.f_fsid.__val[0] != 0 || stat.f_fsid.__val[1] != 0) { + errno = EINVAL; + if(reason) *reason = "is not a virtual file system"; + return -1; + } +#endif + + return 0; +} + +int verify_netdata_host_prefix() { + if(!netdata_configured_host_prefix) + netdata_configured_host_prefix = ""; + + if(!*netdata_configured_host_prefix) + return 0; + + char buffer[FILENAME_MAX + 1]; + char *path = netdata_configured_host_prefix; + char *reason = "unknown reason"; + errno = 0; + + struct stat sb; + if (stat(path, &sb) == -1) { + reason = "failed to stat()"; + goto failed; + } + + if((sb.st_mode & S_IFMT) != S_IFDIR) { + errno = EINVAL; + reason = "is not a directory"; + goto failed; + } + + path = buffer; + snprintfz(path, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix); + if(is_virtual_filesystem(path, &reason) == -1) + goto failed; + + snprintfz(path, FILENAME_MAX, "%s/sys", netdata_configured_host_prefix); + if(is_virtual_filesystem(path, &reason) == -1) + goto failed; + + if(netdata_configured_host_prefix && *netdata_configured_host_prefix) + info("Using host prefix directory '%s'", netdata_configured_host_prefix); + + return 0; + +failed: + error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason); + netdata_configured_host_prefix = ""; + return -1; +} + +char *strdupz_path_subpath(const char *path, const char *subpath) { + if(unlikely(!path || !*path)) path = "."; + if(unlikely(!subpath)) subpath = ""; + + // skip trailing slashes in path + size_t len = strlen(path); + while(len > 0 && path[len - 1] == '/') len--; + + // skip leading slashes in subpath + while(subpath[0] == '/') subpath++; + + // if the last character in path is / and (there is a subpath or path is now empty) + // keep the trailing slash in path and remove the additional slash + char *slash = "/"; + if(path[len] == '/' && (*subpath || len == 0)) { + slash = ""; + len++; + } + else if(!*subpath) { + // there is no subpath + // no need for trailing slash + slash = ""; + } + + char buffer[FILENAME_MAX + 1]; + snprintfz(buffer, FILENAME_MAX, "%.*s%s%s", (int)len, path, slash, subpath); + return strdupz(buffer); +} + +int path_is_dir(const char *path, const char *subpath) { + char *s = strdupz_path_subpath(path, subpath); + + size_t max_links = 100; + + int is_dir = 0; + struct stat statbuf; + while(max_links-- && stat(s, &statbuf) == 0) { + if((statbuf.st_mode & S_IFMT) == S_IFDIR) { + is_dir = 1; + break; + } + else if((statbuf.st_mode & S_IFMT) == S_IFLNK) { + char buffer[FILENAME_MAX + 1]; + ssize_t l = readlink(s, buffer, FILENAME_MAX); + if(l > 0) { + buffer[l] = '\0'; + freez(s); + s = strdupz(buffer); + continue; + } + else { + is_dir = 0; + break; + } + } + else { + is_dir = 0; + break; + } + } + + freez(s); + return is_dir; +} + +int path_is_file(const char *path, const char *subpath) { + char *s = strdupz_path_subpath(path, subpath); + + size_t max_links = 100; + + int is_file = 0; + struct stat statbuf; + while(max_links-- && stat(s, &statbuf) == 0) { + if((statbuf.st_mode & S_IFMT) == S_IFREG) { + is_file = 1; + break; + } + else if((statbuf.st_mode & S_IFMT) == S_IFLNK) { + char buffer[FILENAME_MAX + 1]; + ssize_t l = readlink(s, buffer, FILENAME_MAX); + if(l > 0) { + buffer[l] = '\0'; + freez(s); + s = strdupz(buffer); + continue; + } + else { + is_file = 0; + break; + } + } + else { + is_file = 0; + break; + } + } + + freez(s); + return is_file; +} + +void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data, size_t depth) { + if(depth > 3) { + error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath); + return; + } + + char *udir = strdupz_path_subpath(user_path, subpath); + char *sdir = strdupz_path_subpath(stock_path, subpath); + + debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + + DIR *dir = opendir(udir); + if (!dir) { + error("CONFIG cannot open user-config directory '%s'.", udir); + } + else { + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR || de->d_type == DT_LNK) { + if( !de->d_name[0] || + (de->d_name[0] == '.' && de->d_name[1] == '\0') || + (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + ) { + debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name); + continue; + } + + if(path_is_dir(udir, de->d_name)) { + recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); + continue; + } + } + + if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { + size_t len = strlen(de->d_name); + if(path_is_file(udir, de->d_name) && + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { + char *filename = strdupz_path_subpath(udir, de->d_name); + debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename); + callback(filename, data); + freez(filename); + continue; + } + } + + debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + } + + closedir(dir); + } + + debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir); + + dir = opendir(sdir); + if (!dir) { + error("CONFIG cannot open stock config directory '%s'.", sdir); + } + else if (strcmp(udir, sdir)) { + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR || de->d_type == DT_LNK) { + if( !de->d_name[0] || + (de->d_name[0] == '.' && de->d_name[1] == '\0') || + (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + ) { + debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name); + continue; + } + + if(path_is_dir(sdir, de->d_name)) { + // we recurse in stock subdirectory, only when there is no corresponding + // user subdirectory - to avoid reading the files twice + + if(!path_is_dir(udir, de->d_name)) + recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); + + continue; + } + } + + if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { + size_t len = strlen(de->d_name); + if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) && + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { + char *filename = strdupz_path_subpath(sdir, de->d_name); + debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename); + callback(filename, data); + freez(filename); + continue; + } + + } + + debug(D_HEALTH, "CONFIG ignoring stock-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + } + + closedir(dir); + } + + debug(D_HEALTH, "CONFIG done traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + + freez(udir); + freez(sdir); +} + +// Returns the number of bytes read from the file if file_size is not NULL. +// The actual buffer has an extra byte set to zero (not included in the count). +char *read_by_filename(char *filename, long *file_size) +{ + FILE *f = fopen(filename, "r"); + if (!f) + return NULL; + if (fseek(f, 0, SEEK_END) < 0) { + fclose(f); + return NULL; + } + long size = ftell(f); + if (size <= 0 || fseek(f, 0, SEEK_END) < 0) { + fclose(f); + return NULL; + } + char *contents = callocz(size + 1, 1); + if (!contents) { + fclose(f); + return NULL; + } + if (fseek(f, 0, SEEK_SET) < 0) { + fclose(f); + freez(contents); + return NULL; + } + size_t res = fread(contents, 1, size, f); + if ( res != (size_t)size) { + freez(contents); + fclose(f); + return NULL; + } + fclose(f); + if (file_size) + *file_size = size; + return contents; +} diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h new file mode 100644 index 0000000..50568b5 --- /dev/null +++ b/libnetdata/libnetdata.h @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_LIB_H +#define NETDATA_LIB_H 1 + +# ifdef __cplusplus +extern "C" { +# endif + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#define OS_LINUX 1 +#define OS_FREEBSD 2 +#define OS_MACOS 3 + + +// ---------------------------------------------------------------------------- +// system include files for all netdata C programs + +/* select the memory allocator, based on autoconf findings */ +#if defined(ENABLE_JEMALLOC) + +#if defined(HAVE_JEMALLOC_JEMALLOC_H) +#include <jemalloc/jemalloc.h> +#else // !defined(HAVE_JEMALLOC_JEMALLOC_H) +#include <malloc.h> +#endif // !defined(HAVE_JEMALLOC_JEMALLOC_H) + +#elif defined(ENABLE_TCMALLOC) + +#include <google/tcmalloc.h> + +#else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ + +#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#include <malloc.h> +#endif /* __FreeBSD__ || __APPLE__ */ + +#endif /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ + +// ---------------------------------------------------------------------------- + +#if defined(__FreeBSD__) +#include <pthread_np.h> +#define NETDATA_OS_TYPE "freebsd" +#elif defined(__APPLE__) +#define NETDATA_OS_TYPE "macos" +#else +#define NETDATA_OS_TYPE "linux" +#endif /* __FreeBSD__, __APPLE__*/ + +#include <pthread.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <stddef.h> +#include <ctype.h> +#include <string.h> +#include <strings.h> +#include <arpa/inet.h> +#include <netinet/tcp.h> +#include <sys/ioctl.h> +#include <libgen.h> +#include <dirent.h> +#include <fcntl.h> +#include <getopt.h> +#include <grp.h> +#include <pwd.h> +#include <limits.h> +#include <locale.h> +#include <net/if.h> +#include <poll.h> +#include <signal.h> +#include <syslog.h> +#include <sys/mman.h> +#include <sys/resource.h> +#include <sys/socket.h> +#include <sys/syscall.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/un.h> +#include <time.h> +#include <unistd.h> +#include <uuid/uuid.h> +#include <spawn.h> +#include <uv.h> +#include <assert.h> + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#ifdef HAVE_RESOLV_H +#include <resolv.h> +#endif + +#ifdef HAVE_NETDB_H +#include <netdb.h> +#endif + +#ifdef HAVE_SYS_PRCTL_H +#include <sys/prctl.h> +#endif + +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif + +#ifdef HAVE_SYS_VFS_H +#include <sys/vfs.h> +#endif + +#ifdef HAVE_SYS_STATFS_H +#include <sys/statfs.h> +#endif + +#ifdef HAVE_SYS_MOUNT_H +#include <sys/mount.h> +#endif + +#ifdef HAVE_SYS_STATVFS_H +#include <sys/statvfs.h> +#endif + +// #1408 +#ifdef MAJOR_IN_MKDEV +#include <sys/mkdev.h> +#endif +#ifdef MAJOR_IN_SYSMACROS +#include <sys/sysmacros.h> +#endif + +#ifdef STORAGE_WITH_MATH +#include <math.h> +#include <float.h> +#endif + +#if defined(HAVE_INTTYPES_H) +#include <inttypes.h> +#elif defined(HAVE_STDINT_H) +#include <stdint.h> +#endif + +#ifdef NETDATA_WITH_ZLIB +#include <zlib.h> +#endif + +#ifdef HAVE_CAPABILITY +#include <sys/capability.h> +#endif + + +// ---------------------------------------------------------------------------- +// netdata common definitions + +#if (SIZEOF_VOID_P == 8) +#define ENVIRONMENT64 +#elif (SIZEOF_VOID_P == 4) +#define ENVIRONMENT32 +#else +#error "Cannot detect if this is a 32 or 64 bit CPU" +#endif + +#ifdef __GNUC__ +#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#endif // __GNUC__ + +#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL +#define NEVERNULL __attribute__((returns_nonnull)) +#else +#define NEVERNULL +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_NOINLINE +#define NOINLINE __attribute__((noinline)) +#else +#define NOINLINE +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC +#define MALLOCLIKE __attribute__((malloc)) +#else +#define MALLOCLIKE +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_FORMAT +#define PRINTFLIKE(f, a) __attribute__ ((format(__printf__, f, a))) +#else +#define PRINTFLIKE(f, a) +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN +#define NORETURN __attribute__ ((noreturn)) +#else +#define NORETURN +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT +#define WARNUNUSED __attribute__ ((warn_unused_result)) +#else +#define WARNUNUSED +#endif + +#ifdef abs +#undef abs +#endif +#define abs(x) (((x) < 0)? (-(x)) : (x)) + +#define MIN(a,b) (((a)<(b))?(a):(b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) + +#define GUID_LEN 36 + +extern void netdata_fix_chart_id(char *s); +extern void netdata_fix_chart_name(char *s); + +extern void strreverse(char* begin, char* end); +extern char *mystrsep(char **ptr, char *s); +extern char *trim(char *s); // remove leading and trailing spaces; may return NULL +extern char *trim_all(char *buffer); // like trim(), but also remove duplicate spaces inside the string; may return NULL + +extern int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args); +extern int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4); + +// memory allocation functions that handle failures +#ifdef NETDATA_LOG_ALLOCATIONS +extern __thread size_t log_thread_memory_allocations; +#define strdupz(s) strdupz_int(__FILE__, __FUNCTION__, __LINE__, s) +#define callocz(nmemb, size) callocz_int(__FILE__, __FUNCTION__, __LINE__, nmemb, size) +#define mallocz(size) mallocz_int(__FILE__, __FUNCTION__, __LINE__, size) +#define reallocz(ptr, size) reallocz_int(__FILE__, __FUNCTION__, __LINE__, ptr, size) +#define freez(ptr) freez_int(__FILE__, __FUNCTION__, __LINE__, ptr) + +extern char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s); +extern void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size); +extern void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size); +extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size); +extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr); +#else // NETDATA_LOG_ALLOCATIONS +extern char *strdupz(const char *s) MALLOCLIKE NEVERNULL; +extern void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL; +extern void *mallocz(size_t size) MALLOCLIKE NEVERNULL; +extern void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL; +extern void freez(void *ptr); +#endif // NETDATA_LOG_ALLOCATIONS + +extern void json_escape_string(char *dst, const char *src, size_t size); +extern void json_fix_string(char *s); + +extern void *mymmap(const char *filename, size_t size, int flags, int ksm); +extern int memory_file_save(const char *filename, void *mem, size_t size); + +extern int fd_is_valid(int fd); + +extern struct rlimit rlimit_nofile; + +extern int enable_ksm; + +extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len); + +extern int verify_netdata_host_prefix(); + +extern int recursively_delete_dir(const char *path, const char *reason); + +extern volatile sig_atomic_t netdata_exit; +extern const char *os_type; + +extern const char *program_version; + +extern char *strdupz_path_subpath(const char *path, const char *subpath); +extern int path_is_dir(const char *path, const char *subpath); +extern int path_is_file(const char *path, const char *subpath); +extern void recursive_config_double_dir_load( + const char *user_path + , const char *stock_path + , const char *subpath + , int (*callback)(const char *filename, void *data) + , void *data + , size_t depth +); +extern char *read_by_filename(char *filename, long *file_size); + +/* fix for alpine linux */ +#ifndef RUSAGE_THREAD +#ifdef RUSAGE_CHILDREN +#define RUSAGE_THREAD RUSAGE_CHILDREN +#endif +#endif + +#define BITS_IN_A_KILOBIT 1000 + +/* misc. */ +#define UNUSED(x) (void)(x) +#define error_report(x, args...) do { errno = 0; error(x, ##args); } while(0) + +extern void netdata_cleanup_and_exit(int ret) NORETURN; +extern void send_statistics(const char *action, const char *action_result, const char *action_data); +extern char *netdata_configured_host_prefix; +#include "os.h" +#include "storage_number/storage_number.h" +#include "threads/threads.h" +#include "buffer/buffer.h" +#include "locks/locks.h" +#include "circular_buffer/circular_buffer.h" +#include "avl/avl.h" +#include "inlined.h" +#include "clocks/clocks.h" +#include "popen/popen.h" +#include "simple_pattern/simple_pattern.h" +#ifdef ENABLE_HTTPS +# include "socket/security.h" +#endif +#include "socket/socket.h" +#include "config/appconfig.h" +#include "log/log.h" +#include "procfile/procfile.h" +#include "dictionary/dictionary.h" +#ifdef HAVE_LIBBPF +#include "ebpf/ebpf.h" +#endif +#include "eval/eval.h" +#include "statistical/statistical.h" +#include "adaptive_resortable_list/adaptive_resortable_list.h" +#include "url/url.h" +#include "json/json.h" +#include "health/health.h" +#include "string/utf8.h" + +// BEWARE: Outside of the C code this also exists in alarm-notify.sh +#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" + +# ifdef __cplusplus +} +# endif + +#endif // NETDATA_LIB_H diff --git a/libnetdata/locks/Makefile.am b/libnetdata/locks/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/locks/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/locks/README.md b/libnetdata/locks/README.md new file mode 100644 index 0000000..b525c97 --- /dev/null +++ b/libnetdata/locks/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/locks/README.md +--> + +[](<>) diff --git a/libnetdata/locks/locks.c b/libnetdata/locks/locks.c new file mode 100644 index 0000000..ca9a5ae --- /dev/null +++ b/libnetdata/locks/locks.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// automatic thread cancelability management, based on locks + +static __thread int netdata_thread_first_cancelability = 0; +static __thread int netdata_thread_lock_cancelability = 0; + +inline void netdata_thread_disable_cancelability(void) { + int old; + int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old); + if(ret != 0) + error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret); + else { + if(!netdata_thread_lock_cancelability) + netdata_thread_first_cancelability = old; + + netdata_thread_lock_cancelability++; + } +} + +inline void netdata_thread_enable_cancelability(void) { + if(netdata_thread_lock_cancelability < 1) { + error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d on thread %s - results will be undefined - please report this!", netdata_thread_lock_cancelability, netdata_thread_tag()); + } + else if(netdata_thread_lock_cancelability == 1) { + int old = 1; + int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old); + if(ret != 0) + error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret); + else { + if(old != PTHREAD_CANCEL_DISABLE) + error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, (old == PTHREAD_CANCEL_ENABLE)?"ENABLED":"UNKNOWN", old); + } + + netdata_thread_lock_cancelability = 0; + } + else + netdata_thread_lock_cancelability--; +} + +// ---------------------------------------------------------------------------- +// mutex + +int __netdata_mutex_init(netdata_mutex_t *mutex) { + int ret = pthread_mutex_init(mutex, NULL); + if(unlikely(ret != 0)) + error("MUTEX_LOCK: failed to initialize (code %d).", ret); + return ret; +} + +int __netdata_mutex_lock(netdata_mutex_t *mutex) { + netdata_thread_disable_cancelability(); + + int ret = pthread_mutex_lock(mutex); + if(unlikely(ret != 0)) { + netdata_thread_enable_cancelability(); + error("MUTEX_LOCK: failed to get lock (code %d)", ret); + } + return ret; +} + +int __netdata_mutex_trylock(netdata_mutex_t *mutex) { + netdata_thread_disable_cancelability(); + + int ret = pthread_mutex_trylock(mutex); + if(ret != 0) + netdata_thread_enable_cancelability(); + + return ret; +} + +int __netdata_mutex_unlock(netdata_mutex_t *mutex) { + int ret = pthread_mutex_unlock(mutex); + if(unlikely(ret != 0)) + error("MUTEX_LOCK: failed to unlock (code %d).", ret); + else + netdata_thread_enable_cancelability(); + + return ret; +} + +int netdata_mutex_init_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) from %lu@%s, %s()", mutex, line, file, function); + } + + int ret = __netdata_mutex_init(mutex); + + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_mutex_lock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) from %lu@%s, %s()", mutex, line, file, function); + } + + int ret = __netdata_mutex_lock(mutex); + + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_mutex_trylock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) from %lu@%s, %s()", mutex, line, file, function); + } + + int ret = __netdata_mutex_trylock(mutex); + + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) from %lu@%s, %s()", mutex, line, file, function); + } + + int ret = __netdata_mutex_unlock(mutex); + + debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + + +// ---------------------------------------------------------------------------- +// r/w lock + +int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) { + int ret = pthread_rwlock_destroy(rwlock); + if(unlikely(ret != 0)) + error("RW_LOCK: failed to destroy lock (code %d)", ret); + return ret; +} + +int __netdata_rwlock_init(netdata_rwlock_t *rwlock) { + int ret = pthread_rwlock_init(rwlock, NULL); + if(unlikely(ret != 0)) + error("RW_LOCK: failed to initialize lock (code %d)", ret); + return ret; +} + +int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) { + netdata_thread_disable_cancelability(); + + int ret = pthread_rwlock_rdlock(rwlock); + if(unlikely(ret != 0)) { + netdata_thread_enable_cancelability(); + error("RW_LOCK: failed to obtain read lock (code %d)", ret); + } + + return ret; +} + +int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) { + netdata_thread_disable_cancelability(); + + int ret = pthread_rwlock_wrlock(rwlock); + if(unlikely(ret != 0)) { + error("RW_LOCK: failed to obtain write lock (code %d)", ret); + netdata_thread_enable_cancelability(); + } + + return ret; +} + +int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) { + int ret = pthread_rwlock_unlock(rwlock); + if(unlikely(ret != 0)) + error("RW_LOCK: failed to release lock (code %d)", ret); + else + netdata_thread_enable_cancelability(); + + return ret; +} + +int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) { + netdata_thread_disable_cancelability(); + + int ret = pthread_rwlock_tryrdlock(rwlock); + if(ret != 0) + netdata_thread_enable_cancelability(); + + return ret; +} + +int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) { + netdata_thread_disable_cancelability(); + + int ret = pthread_rwlock_trywrlock(rwlock); + if(ret != 0) + netdata_thread_enable_cancelability(); + + return ret; +} + + +int netdata_rwlock_destroy_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) from %lu@%s, %s()", rwlock, line, file, function); + } + + int ret = __netdata_rwlock_destroy(rwlock); + + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_rwlock_init_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) from %lu@%s, %s()", rwlock, line, file, function); + } + + int ret = __netdata_rwlock_init(rwlock); + + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_rwlock_rdlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function); + } + + int ret = __netdata_rwlock_rdlock(rwlock); + + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_rwlock_wrlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function); + } + + int ret = __netdata_rwlock_wrlock(rwlock); + + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_rwlock_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function); + } + + int ret = __netdata_rwlock_unlock(rwlock); + + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_rwlock_tryrdlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function); + } + + int ret = __netdata_rwlock_tryrdlock(rwlock); + + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} + +int netdata_rwlock_trywrlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, + const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) { + usec_t start = 0; + (void)start; + + if(unlikely(debug_flags & D_LOCKS)) { + start = now_boottime_usec(); + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function); + } + + int ret = __netdata_rwlock_trywrlock(rwlock); + + debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function); + + return ret; +} diff --git a/libnetdata/locks/locks.h b/libnetdata/locks/locks.h new file mode 100644 index 0000000..850dd7e --- /dev/null +++ b/libnetdata/locks/locks.h @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_LOCKS_H +#define NETDATA_LOCKS_H 1 + +#include "../libnetdata.h" + +typedef pthread_mutex_t netdata_mutex_t; +#define NETDATA_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +typedef pthread_rwlock_t netdata_rwlock_t; +#define NETDATA_RWLOCK_INITIALIZER PTHREAD_RWLOCK_INITIALIZER + +extern int __netdata_mutex_init(netdata_mutex_t *mutex); +extern int __netdata_mutex_lock(netdata_mutex_t *mutex); +extern int __netdata_mutex_trylock(netdata_mutex_t *mutex); +extern int __netdata_mutex_unlock(netdata_mutex_t *mutex); + +extern int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock); +extern int __netdata_rwlock_init(netdata_rwlock_t *rwlock); +extern int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock); +extern int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock); +extern int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock); +extern int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock); +extern int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock); + +extern int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex); +extern int netdata_mutex_lock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex); +extern int netdata_mutex_trylock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex); +extern int netdata_mutex_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex); + +extern int netdata_rwlock_destroy_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +extern int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +extern int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +extern int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +extern int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +extern int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); +extern int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock); + +extern void netdata_thread_disable_cancelability(void); +extern void netdata_thread_enable_cancelability(void); + +#ifdef NETDATA_INTERNAL_CHECKS + +#define netdata_mutex_init(mutex) netdata_mutex_init_debug(__FILE__, __FUNCTION__, __LINE__, mutex) +#define netdata_mutex_lock(mutex) netdata_mutex_lock_debug(__FILE__, __FUNCTION__, __LINE__, mutex) +#define netdata_mutex_trylock(mutex) netdata_mutex_trylock_debug(__FILE__, __FUNCTION__, __LINE__, mutex) +#define netdata_mutex_unlock(mutex) netdata_mutex_unlock_debug(__FILE__, __FUNCTION__, __LINE__, mutex) + +#define netdata_rwlock_destroy(rwlock) netdata_rwlock_destroy_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_init(rwlock) netdata_rwlock_init_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_rdlock(rwlock) netdata_rwlock_rdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_wrlock(rwlock) netdata_rwlock_wrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_unlock(rwlock) netdata_rwlock_unlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_tryrdlock(rwlock) netdata_rwlock_tryrdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) +#define netdata_rwlock_trywrlock(rwlock) netdata_rwlock_trywrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock) + +#else // !NETDATA_INTERNAL_CHECKS + +#define netdata_mutex_init(mutex) __netdata_mutex_init(mutex) +#define netdata_mutex_lock(mutex) __netdata_mutex_lock(mutex) +#define netdata_mutex_trylock(mutex) __netdata_mutex_trylock(mutex) +#define netdata_mutex_unlock(mutex) __netdata_mutex_unlock(mutex) + +#define netdata_rwlock_destroy(rwlock) __netdata_rwlock_destroy(rwlock) +#define netdata_rwlock_init(rwlock) __netdata_rwlock_init(rwlock) +#define netdata_rwlock_rdlock(rwlock) __netdata_rwlock_rdlock(rwlock) +#define netdata_rwlock_wrlock(rwlock) __netdata_rwlock_wrlock(rwlock) +#define netdata_rwlock_unlock(rwlock) __netdata_rwlock_unlock(rwlock) +#define netdata_rwlock_tryrdlock(rwlock) __netdata_rwlock_tryrdlock(rwlock) +#define netdata_rwlock_trywrlock(rwlock) __netdata_rwlock_trywrlock(rwlock) + +#endif // NETDATA_INTERNAL_CHECKS + +#endif //NETDATA_LOCKS_H diff --git a/libnetdata/log/Makefile.am b/libnetdata/log/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/log/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/log/README.md b/libnetdata/log/README.md new file mode 100644 index 0000000..c0d7a34 --- /dev/null +++ b/libnetdata/log/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/log/README.md +--> + +[](<>) diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c new file mode 100644 index 0000000..95d3d98 --- /dev/null +++ b/libnetdata/log/log.c @@ -0,0 +1,872 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <daemon/main.h> +#include "../libnetdata.h" + +int web_server_is_multithreaded = 1; + +const char *program_name = ""; +uint64_t debug_flags = DEBUG; + +int access_log_syslog = 1; +int error_log_syslog = 1; +int output_log_syslog = 1; // debug log + +int stdaccess_fd = -1; +FILE *stdaccess = NULL; + +const char *stdaccess_filename = NULL; +const char *stderr_filename = NULL; +const char *stdout_filename = NULL; +const char *facility_log = NULL; + +// ---------------------------------------------------------------------------- +// Log facility(https://tools.ietf.org/html/rfc5424) +// +// The facilities accepted in the Netdata are in according with the following +// header files for their respective operate system: +// sys/syslog.h (Linux ) +// sys/sys/syslog.h (FreeBSD) +// bsd/sys/syslog.h (darwin-xnu) + +#define LOG_AUTH_KEY "auth" +#define LOG_AUTHPRIV_KEY "authpriv" +#ifdef __FreeBSD__ +# define LOG_CONSOLE_KEY "console" +#endif +#define LOG_CRON_KEY "cron" +#define LOG_DAEMON_KEY "daemon" +#define LOG_FTP_KEY "ftp" +#ifdef __APPLE__ +# define LOG_INSTALL_KEY "install" +#endif +#define LOG_KERN_KEY "kern" +#define LOG_LPR_KEY "lpr" +#define LOG_MAIL_KEY "mail" +//#define LOG_INTERNAL_MARK_KEY "mark" +#ifdef __APPLE__ +# define LOG_NETINFO_KEY "netinfo" +# define LOG_RAS_KEY "ras" +# define LOG_REMOTEAUTH_KEY "remoteauth" +#endif +#define LOG_NEWS_KEY "news" +#ifdef __FreeBSD__ +# define LOG_NTP_KEY "ntp" +#endif +#define LOG_SECURITY_KEY "security" +#define LOG_SYSLOG_KEY "syslog" +#define LOG_USER_KEY "user" +#define LOG_UUCP_KEY "uucp" +#ifdef __APPLE__ +# define LOG_LAUNCHD_KEY "launchd" +#endif +#define LOG_LOCAL0_KEY "local0" +#define LOG_LOCAL1_KEY "local1" +#define LOG_LOCAL2_KEY "local2" +#define LOG_LOCAL3_KEY "local3" +#define LOG_LOCAL4_KEY "local4" +#define LOG_LOCAL5_KEY "local5" +#define LOG_LOCAL6_KEY "local6" +#define LOG_LOCAL7_KEY "local7" + +static int log_facility_id(const char *facility_name) +{ + static int + hash_auth = 0, + hash_authpriv = 0, +#ifdef __FreeBSD__ + hash_console = 0, +#endif + hash_cron = 0, + hash_daemon = 0, + hash_ftp = 0, +#ifdef __APPLE__ + hash_install = 0, +#endif + hash_kern = 0, + hash_lpr = 0, + hash_mail = 0, +// hash_mark = 0, +#ifdef __APPLE__ + hash_netinfo = 0, + hash_ras = 0, + hash_remoteauth = 0, +#endif + hash_news = 0, +#ifdef __FreeBSD__ + hash_ntp = 0, +#endif + hash_security = 0, + hash_syslog = 0, + hash_user = 0, + hash_uucp = 0, +#ifdef __APPLE__ + hash_launchd = 0, +#endif + hash_local0 = 0, + hash_local1 = 0, + hash_local2 = 0, + hash_local3 = 0, + hash_local4 = 0, + hash_local5 = 0, + hash_local6 = 0, + hash_local7 = 0; + + if(unlikely(!hash_auth)) + { + hash_auth = simple_hash(LOG_AUTH_KEY); + hash_authpriv = simple_hash(LOG_AUTHPRIV_KEY); +#ifdef __FreeBSD__ + hash_console = simple_hash(LOG_CONSOLE_KEY); +#endif + hash_cron = simple_hash(LOG_CRON_KEY); + hash_daemon = simple_hash(LOG_DAEMON_KEY); + hash_ftp = simple_hash(LOG_FTP_KEY); +#ifdef __APPLE__ + hash_install = simple_hash(LOG_INSTALL_KEY); +#endif + hash_kern = simple_hash(LOG_KERN_KEY); + hash_lpr = simple_hash(LOG_LPR_KEY); + hash_mail = simple_hash(LOG_MAIL_KEY); +// hash_mark = simple_uhash(); +#ifdef __APPLE__ + hash_netinfo = simple_hash(LOG_NETINFO_KEY); + hash_ras = simple_hash(LOG_RAS_KEY); + hash_remoteauth = simple_hash(LOG_REMOTEAUTH_KEY); +#endif + hash_news = simple_hash(LOG_NEWS_KEY); +#ifdef __FreeBSD__ + hash_ntp = simple_hash(LOG_NTP_KEY); +#endif + hash_security = simple_hash(LOG_SECURITY_KEY); + hash_syslog = simple_hash(LOG_SYSLOG_KEY); + hash_user = simple_hash(LOG_USER_KEY); + hash_uucp = simple_hash(LOG_UUCP_KEY); +#ifdef __APPLE__ + hash_launchd = simple_hash(LOG_LAUNCHD_KEY); +#endif + hash_local0 = simple_hash(LOG_LOCAL0_KEY); + hash_local1 = simple_hash(LOG_LOCAL1_KEY); + hash_local2 = simple_hash(LOG_LOCAL2_KEY); + hash_local3 = simple_hash(LOG_LOCAL3_KEY); + hash_local4 = simple_hash(LOG_LOCAL4_KEY); + hash_local5 = simple_hash(LOG_LOCAL5_KEY); + hash_local6 = simple_hash(LOG_LOCAL6_KEY); + hash_local7 = simple_hash(LOG_LOCAL7_KEY); + } + + int hash = simple_hash(facility_name); + if ( hash == hash_auth ) + { + return LOG_AUTH; + } + else if ( hash == hash_authpriv ) + { + return LOG_AUTHPRIV; + } +#ifdef __FreeBSD__ + else if ( hash == hash_console ) + { + return LOG_CONSOLE; + } +#endif + else if ( hash == hash_cron ) + { + return LOG_CRON; + } + else if ( hash == hash_daemon ) + { + return LOG_DAEMON; + } + else if ( hash == hash_ftp ) + { + return LOG_FTP; + } +#ifdef __APPLE__ + else if ( hash == hash_install ) + { + return LOG_INSTALL; + } +#endif + else if ( hash == hash_kern ) + { + return LOG_KERN; + } + else if ( hash == hash_lpr ) + { + return LOG_LPR; + } + else if ( hash == hash_mail ) + { + return LOG_MAIL; + } + /* + else if ( hash == hash_mark ) + { + //this is internal for all OS + return INTERNAL_MARK; + } + */ +#ifdef __APPLE__ + else if ( hash == hash_netinfo ) + { + return LOG_NETINFO; + } + else if ( hash == hash_ras ) + { + return LOG_RAS; + } + else if ( hash == hash_remoteauth ) + { + return LOG_REMOTEAUTH; + } +#endif + else if ( hash == hash_news ) + { + return LOG_NEWS; + } +#ifdef __FreeBSD__ + else if ( hash == hash_ntp ) + { + return LOG_NTP; + } +#endif + else if ( hash == hash_security ) + { + //FreeBSD is the unique that does not consider + //this facility deprecated. We are keeping + //it for other OS while they are kept in their headers. +#ifdef __FreeBSD__ + return LOG_SECURITY; +#else + return LOG_AUTH; +#endif + } + else if ( hash == hash_syslog ) + { + return LOG_SYSLOG; + } + else if ( hash == hash_user ) + { + return LOG_USER; + } + else if ( hash == hash_uucp ) + { + return LOG_UUCP; + } + else if ( hash == hash_local0 ) + { + return LOG_LOCAL0; + } + else if ( hash == hash_local1 ) + { + return LOG_LOCAL1; + } + else if ( hash == hash_local2 ) + { + return LOG_LOCAL2; + } + else if ( hash == hash_local3 ) + { + return LOG_LOCAL3; + } + else if ( hash == hash_local4 ) + { + return LOG_LOCAL4; + } + else if ( hash == hash_local5 ) + { + return LOG_LOCAL5; + } + else if ( hash == hash_local6 ) + { + return LOG_LOCAL6; + } + else if ( hash == hash_local7 ) + { + return LOG_LOCAL7; + } +#ifdef __APPLE__ + else if ( hash == hash_launchd ) + { + return LOG_LAUNCHD; + } +#endif + + return LOG_DAEMON; +} + +//we do not need to use this now, but I already created this function to be +//used case necessary. +/* +char *log_facility_name(int code) +{ + char *defvalue = { "daemon" }; + switch(code) + { + case LOG_AUTH: + { + return "auth"; + } + case LOG_AUTHPRIV: + { + return "authpriv"; + } +#ifdef __FreeBSD__ + case LOG_CONSOLE: + { + return "console"; + } +#endif + case LOG_CRON: + { + return "cron"; + } + case LOG_DAEMON: + { + return defvalue; + } + case LOG_FTP: + { + return "ftp"; + } +#ifdef __APPLE__ + case LOG_INSTALL: + { + return "install"; + } +#endif + case LOG_KERN: + { + return "kern"; + } + case LOG_LPR: + { + return "lpr"; + } + case LOG_MAIL: + { + return "mail"; + } +#ifdef __APPLE__ + case LOG_NETINFO: + { + return "netinfo" ; + } + case LOG_RAS: + { + return "ras"; + } + case LOG_REMOTEAUTH: + { + return "remoteauth"; + } +#endif + case LOG_NEWS: + { + return "news"; + } +#ifdef __FreeBSD__ + case LOG_NTP: + { + return "ntp" ; + } + case LOG_SECURITY: + { + return "security"; + } +#endif + case LOG_SYSLOG: + { + return "syslog"; + } + case LOG_USER: + { + return "user"; + } + case LOG_UUCP: + { + return "uucp"; + } + case LOG_LOCAL0: + { + return "local0"; + } + case LOG_LOCAL1: + { + return "local1"; + } + case LOG_LOCAL2: + { + return "local2"; + } + case LOG_LOCAL3: + { + return "local3"; + } + case LOG_LOCAL4: + { + return "local4" ; + } + case LOG_LOCAL5: + { + return "local5"; + } + case LOG_LOCAL6: + { + return "local6"; + } + case LOG_LOCAL7: + { + return "local7" ; + } +#ifdef __APPLE__ + case LOG_LAUNCHD: + { + return "launchd"; + } +#endif + } + + return defvalue; +} +*/ + +// ---------------------------------------------------------------------------- + +void syslog_init() { + static int i = 0; + + if(!i) { + openlog(program_name, LOG_PID,log_facility_id(facility_log)); + i = 1; + } +} + +#define LOG_DATE_LENGTH 26 + +static inline void log_date(char *buffer, size_t len) { + if(unlikely(!buffer || !len)) + return; + + time_t t; + struct tm *tmp, tmbuf; + + t = now_realtime_sec(); + tmp = localtime_r(&t, &tmbuf); + + if (tmp == NULL) { + buffer[0] = '\0'; + return; + } + + if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0)) + buffer[0] = '\0'; + + buffer[len - 1] = '\0'; +} + +static netdata_mutex_t log_mutex = NETDATA_MUTEX_INITIALIZER; +static inline void log_lock() { + netdata_mutex_lock(&log_mutex); +} +static inline void log_unlock() { + netdata_mutex_unlock(&log_mutex); +} + +static FILE *open_log_file(int fd, FILE *fp, const char *filename, int *enabled_syslog, int is_stdaccess, int *fd_ptr) { + int f, devnull = 0; + + if(!filename || !*filename || !strcmp(filename, "none") || !strcmp(filename, "/dev/null")) { + filename = "/dev/null"; + devnull = 1; + } + + if(!strcmp(filename, "syslog")) { + filename = "/dev/null"; + devnull = 1; + + syslog_init(); + if(enabled_syslog) *enabled_syslog = 1; + } + else if(enabled_syslog) *enabled_syslog = 0; + + // don't do anything if the user is willing + // to have the standard one + if(!strcmp(filename, "system")) { + if(fd != -1 && !is_stdaccess) { + if(fd_ptr) *fd_ptr = fd; + return fp; + } + + filename = "stderr"; + } + + if(!strcmp(filename, "stdout")) + f = STDOUT_FILENO; + + else if(!strcmp(filename, "stderr")) + f = STDERR_FILENO; + + else { + f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664); + if(f == -1) { + error("Cannot open file '%s'. Leaving %d to its default.", filename, fd); + if(fd_ptr) *fd_ptr = fd; + return fp; + } + } + + // if there is a level-2 file pointer + // flush it before switching the level-1 fds + if(fp) + fflush(fp); + + if(devnull && is_stdaccess) { + fd = -1; + fp = NULL; + } + + if(fd != f && fd != -1) { + // it automatically closes + int t = dup2(f, fd); + if (t == -1) { + error("Cannot dup2() new fd %d to old fd %d for '%s'", f, fd, filename); + close(f); + if(fd_ptr) *fd_ptr = fd; + return fp; + } + // info("dup2() new fd %d to old fd %d for '%s'", f, fd, filename); + close(f); + } + else fd = f; + + if(!fp) { + fp = fdopen(fd, "a"); + if (!fp) + error("Cannot fdopen() fd %d ('%s')", fd, filename); + else { + if (setvbuf(fp, NULL, _IOLBF, 0) != 0) + error("Cannot set line buffering on fd %d ('%s')", fd, filename); + } + } + + if(fd_ptr) *fd_ptr = fd; + return fp; +} + +void reopen_all_log_files() { + if(stdout_filename) + open_log_file(STDOUT_FILENO, stdout, stdout_filename, &output_log_syslog, 0, NULL); + + if(stderr_filename) + open_log_file(STDERR_FILENO, stderr, stderr_filename, &error_log_syslog, 0, NULL); + + if(stdaccess_filename) + stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd); +} + +void open_all_log_files() { + // disable stdin + open_log_file(STDIN_FILENO, stdin, "/dev/null", NULL, 0, NULL); + + open_log_file(STDOUT_FILENO, stdout, stdout_filename, &output_log_syslog, 0, NULL); + open_log_file(STDERR_FILENO, stderr, stderr_filename, &error_log_syslog, 0, NULL); + stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd); +} + +// ---------------------------------------------------------------------------- +// error log throttling + +time_t error_log_throttle_period = 1200; +unsigned long error_log_errors_per_period = 200; +unsigned long error_log_errors_per_period_backup = 0; + +int error_log_limit(int reset) { + static time_t start = 0; + static unsigned long counter = 0, prevented = 0; + + // fprintf(stderr, "FLOOD: counter=%lu, allowed=%lu, backup=%lu, period=%llu\n", counter, error_log_errors_per_period, error_log_errors_per_period_backup, (unsigned long long)error_log_throttle_period); + + // do not throttle if the period is 0 + if(error_log_throttle_period == 0) + return 0; + + // prevent all logs if the errors per period is 0 + if(error_log_errors_per_period == 0) +#ifdef NETDATA_INTERNAL_CHECKS + return 0; +#else + return 1; +#endif + + time_t now = now_monotonic_sec(); + if(!start) start = now; + + if(reset) { + if(prevented) { + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stderr, "%s: %s LOG FLOOD PROTECTION reset for process '%s' (prevented %lu logs in the last %ld seconds).\n" + , date + , program_name + , program_name + , prevented + , now - start + ); + } + + start = now; + counter = 0; + prevented = 0; + } + + // detect if we log too much + counter++; + + if(now - start > error_log_throttle_period) { + if(prevented) { + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stderr, "%s: %s LOG FLOOD PROTECTION resuming logging from process '%s' (prevented %lu logs in the last %ld seconds).\n" + , date + , program_name + , program_name + , prevented + , error_log_throttle_period + ); + } + + // restart the period accounting + start = now; + counter = 1; + prevented = 0; + + // log this error + return 0; + } + + if(counter > error_log_errors_per_period) { + if(!prevented) { + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stderr, "%s: %s LOG FLOOD PROTECTION too many logs (%lu logs in %ld seconds, threshold is set to %lu logs in %ld seconds). Preventing more logs from process '%s' for %ld seconds.\n" + , date + , program_name + , counter + , now - start + , error_log_errors_per_period + , error_log_throttle_period + , program_name + , start + error_log_throttle_period - now); + } + + prevented++; + + // prevent logging this error +#ifdef NETDATA_INTERNAL_CHECKS + return 0; +#else + return 1; +#endif + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// debug log + +void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { + va_list args; + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + + va_start( args, fmt ); + printf("%s: %s DEBUG : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function); + vprintf(fmt, args); + va_end( args ); + putchar('\n'); + + if(output_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_ERR, fmt, args ); + va_end( args ); + } + + fflush(stdout); +} + +// ---------------------------------------------------------------------------- +// info log + +void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) +{ + va_list args; + + // prevent logging too much + if(error_log_limit(0)) return; + + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_INFO, fmt, args ); + va_end( args ); + } + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + + log_lock(); + + va_start( args, fmt ); + if(debug_flags) fprintf(stderr, "%s: %s INFO : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function); + else fprintf(stderr, "%s: %s INFO : %s : ", date, program_name, netdata_thread_tag()); + vfprintf( stderr, fmt, args ); + va_end( args ); + + fputc('\n', stderr); + + log_unlock(); +} + +// ---------------------------------------------------------------------------- +// error log + +#if defined(STRERROR_R_CHAR_P) +// GLIBC version of strerror_r +static const char *strerror_result(const char *a, const char *b) { (void)b; return a; } +#elif defined(HAVE_STRERROR_R) +// POSIX version of strerror_r +static const char *strerror_result(int a, const char *b) { (void)a; return b; } +#elif defined(HAVE_C__GENERIC) + +// what a trick! +// http://stackoverflow.com/questions/479207/function-overloading-in-c +static const char *strerror_result_int(int a, const char *b) { (void)a; return b; } +static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; } + +#define strerror_result(a, b) _Generic((a), \ + int: strerror_result_int, \ + char *: strerror_result_string \ + )(a, b) + +#else +#error "cannot detect the format of function strerror_r()" +#endif + +void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { + // save a copy of errno - just in case this function generates a new error + int __errno = errno; + + va_list args; + + // prevent logging too much + if(error_log_limit(0)) return; + + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_ERR, fmt, args ); + va_end( args ); + } + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + + log_lock(); + + va_start( args, fmt ); + if(debug_flags) fprintf(stderr, "%s: %s %-5.5s : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, prefix, netdata_thread_tag(), line, file, function); + else fprintf(stderr, "%s: %s %-5.5s : %s : ", date, program_name, prefix, netdata_thread_tag()); + vfprintf( stderr, fmt, args ); + va_end( args ); + + if(__errno) { + char buf[1024]; + fprintf(stderr, " (errno %d, %s)\n", __errno, strerror_result(strerror_r(__errno, buf, 1023), buf)); + errno = 0; + } + else + fputc('\n', stderr); + + log_unlock(); +} + +void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { + // save a copy of errno - just in case this function generates a new error + int __errno = errno; + va_list args; + const char *thread_tag; + char os_threadname[NETDATA_THREAD_NAME_MAX + 1]; + + if(error_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_CRIT, fmt, args ); + va_end( args ); + } + + thread_tag = netdata_thread_tag(); + if (!netdata_thread_tag_exists()) { + os_thread_get_current_name_np(os_threadname); + if ('\0' != os_threadname[0]) { /* If it is not an empty string replace "MAIN" thread_tag */ + thread_tag = os_threadname; + } + } + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + + log_lock(); + + va_start( args, fmt ); + if(debug_flags) fprintf(stderr, "%s: %s FATAL : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, thread_tag, line, file, function); + else fprintf(stderr, "%s: %s FATAL : %s : ", date, program_name, thread_tag); + vfprintf( stderr, fmt, args ); + va_end( args ); + + perror(" # "); + fputc('\n', stderr); + + log_unlock(); + + char action_data[70+1]; + snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, __errno); + char action_result[60+1]; + + snprintfz(action_result, 60, "%s:%s", program_name, strncmp(thread_tag, "STREAM_RECEIVER", strlen("STREAM_RECEIVER")) ? thread_tag : "[x]"); + send_statistics("FATAL", action_result, action_data); + + netdata_cleanup_and_exit(1); +} + +// ---------------------------------------------------------------------------- +// access log + +void log_access( const char *fmt, ... ) { + va_list args; + + if(access_log_syslog) { + va_start( args, fmt ); + vsyslog(LOG_INFO, fmt, args ); + va_end( args ); + } + + if(stdaccess) { + static netdata_mutex_t access_mutex = NETDATA_MUTEX_INITIALIZER; + + if(web_server_is_multithreaded) + netdata_mutex_lock(&access_mutex); + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(stdaccess, "%s: ", date); + + va_start( args, fmt ); + vfprintf( stdaccess, fmt, args ); + va_end( args ); + fputc('\n', stdaccess); + + if(web_server_is_multithreaded) + netdata_mutex_unlock(&access_mutex); + } +} diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h new file mode 100644 index 0000000..c99c151 --- /dev/null +++ b/libnetdata/log/log.h @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_LOG_H +#define NETDATA_LOG_H 1 + +# ifdef __cplusplus +extern "C" { +# endif + +#include "../libnetdata.h" + +#define D_WEB_BUFFER 0x0000000000000001 +#define D_WEB_CLIENT 0x0000000000000002 +#define D_LISTENER 0x0000000000000004 +#define D_WEB_DATA 0x0000000000000008 +#define D_OPTIONS 0x0000000000000010 +#define D_PROCNETDEV_LOOP 0x0000000000000020 +#define D_RRD_STATS 0x0000000000000040 +#define D_WEB_CLIENT_ACCESS 0x0000000000000080 +#define D_TC_LOOP 0x0000000000000100 +#define D_DEFLATE 0x0000000000000200 +#define D_CONFIG 0x0000000000000400 +#define D_PLUGINSD 0x0000000000000800 +#define D_CHILDS 0x0000000000001000 +#define D_EXIT 0x0000000000002000 +#define D_CHECKS 0x0000000000004000 +#define D_NFACCT_LOOP 0x0000000000008000 +#define D_PROCFILE 0x0000000000010000 +#define D_RRD_CALLS 0x0000000000020000 +#define D_DICTIONARY 0x0000000000040000 +#define D_MEMORY 0x0000000000080000 +#define D_CGROUP 0x0000000000100000 +#define D_REGISTRY 0x0000000000200000 +#define D_VARIABLES 0x0000000000400000 +#define D_HEALTH 0x0000000000800000 +#define D_CONNECT_TO 0x0000000001000000 +#define D_RRDHOST 0x0000000002000000 +#define D_LOCKS 0x0000000004000000 +#define D_BACKEND 0x0000000008000000 +#define D_STATSD 0x0000000010000000 +#define D_POLLFD 0x0000000020000000 +#define D_STREAM 0x0000000040000000 +#define D_RRDENGINE 0x0000000100000000 +#define D_ACLK 0x0000000200000000 +#define D_METADATALOG 0x0000000400000000 +#define D_GUIDLOG 0x0000000800000000 +#define D_SYSTEM 0x8000000000000000 + +//#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS) +//#define DEBUG 0xffffffff +#define DEBUG (0) + +extern int web_server_is_multithreaded; + +extern uint64_t debug_flags; + +extern const char *program_name; + +extern int stdaccess_fd; +extern FILE *stdaccess; + +extern const char *stdaccess_filename; +extern const char *stderr_filename; +extern const char *stdout_filename; +extern const char *facility_log; + +extern int access_log_syslog; +extern int error_log_syslog; +extern int output_log_syslog; + +extern time_t error_log_throttle_period; +extern unsigned long error_log_errors_per_period, error_log_errors_per_period_backup; +extern int error_log_limit(int reset); + +extern void open_all_log_files(); +extern void reopen_all_log_files(); + +static inline void debug_dummy(void) {} + +#define error_log_limit_reset() do { error_log_errors_per_period = error_log_errors_per_period_backup; error_log_limit(1); } while(0) +#define error_log_limit_unlimited() do { \ + error_log_limit_reset(); \ + error_log_errors_per_period = ((error_log_errors_per_period_backup * 10) < 10000) ? 10000 : (error_log_errors_per_period_backup * 10); \ + } while(0) + +#ifdef NETDATA_INTERNAL_CHECKS +#define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0) +#else +#define debug(type, args...) debug_dummy() +#endif + +#define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args) +#define infoerr(args...) error_int("INFO", __FILE__, __FUNCTION__, __LINE__, ##args) +#define error(args...) error_int("ERROR", __FILE__, __FUNCTION__, __LINE__, ##args) +#define fatal(args...) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args) +#define fatal_assert(expr) ((expr) ? (void)(0) : fatal_int(__FILE__, __FUNCTION__, __LINE__, "Assertion `%s' failed", #expr)) + +extern void send_statistics(const char *action, const char *action_result, const char *action_data); +extern void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5); +extern void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(4, 5); +extern void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(5, 6); +extern void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5); +extern void log_access( const char *fmt, ... ) PRINTFLIKE(1, 2); + +# ifdef __cplusplus +} +# endif + +#endif /* NETDATA_LOG_H */ diff --git a/libnetdata/os.c b/libnetdata/os.c new file mode 100644 index 0000000..4271a91 --- /dev/null +++ b/libnetdata/os.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "os.h" + +// ---------------------------------------------------------------------------- +// system functions +// to retrieve settings of the system + +int processors = 1; +long get_system_cpus(void) { + processors = 1; + +#ifdef __APPLE__ + int32_t tmp_processors; + + if (unlikely(GETSYSCTL_BY_NAME("hw.logicalcpu", tmp_processors))) { + error("Assuming system has %d processors.", processors); + } else { + processors = tmp_processors; + } + + return processors; +#elif __FreeBSD__ + int32_t tmp_processors; + + if (unlikely(GETSYSCTL_BY_NAME("hw.ncpu", tmp_processors))) { + error("Assuming system has %d processors.", processors); + } else { + processors = tmp_processors; + } + + return processors; +#else + + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/proc/stat", netdata_configured_host_prefix); + + procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); + if(!ff) { + error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors); + return processors; + } + + ff = procfile_readall(ff); + if(!ff) { + error("Cannot open file '%s'. Assuming system has %d processors.", filename, processors); + return processors; + } + + processors = 0; + unsigned int i; + for(i = 0; i < procfile_lines(ff); i++) { + if(!procfile_linewords(ff, i)) continue; + + if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) processors++; + } + processors--; + if(processors < 1) processors = 1; + + procfile_close(ff); + + debug(D_SYSTEM, "System has %d processors.", processors); + return processors; + +#endif /* __APPLE__, __FreeBSD__ */ +} + +pid_t pid_max = 32768; +pid_t get_system_pid_max(void) { +#ifdef __APPLE__ + // As we currently do not know a solution to query pid_max from the os + // we use the number defined in bsd/sys/proc_internal.h in XNU sources + pid_max = 99999; + return pid_max; +#elif __FreeBSD__ + int32_t tmp_pid_max; + + if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) { + pid_max = 99999; + error("Assuming system's maximum pid is %d.", pid_max); + } else { + pid_max = tmp_pid_max; + } + + return pid_max; +#else + + static char read = 0; + if(unlikely(read)) return pid_max; + read = 1; + + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix); + + unsigned long long max = 0; + if(read_single_number_file(filename, &max) != 0) { + error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max); + return pid_max; + } + + if(!max) { + error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max); + return pid_max; + } + + pid_max = (pid_t) max; + return pid_max; + +#endif /* __APPLE__, __FreeBSD__ */ +} + +unsigned int system_hz; +void get_system_HZ(void) { + long ticks; + + if ((ticks = sysconf(_SC_CLK_TCK)) == -1) { + error("Cannot get system clock ticks"); + } + + system_hz = (unsigned int) ticks; +} + +// ===================================================================================================================== +// FreeBSD + +#if (TARGET_OS == OS_FREEBSD) + +int getsysctl_by_name(const char *name, void *ptr, size_t len) { + size_t nlen = len; + + if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { + error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} + +int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len) { + size_t nlen = len; + + if (unlikely(!mib[0])) + if (unlikely(getsysctl_mib(name, mib, miblen))) + return 1; + + if (unlikely(sysctl(mib, miblen, ptr, &nlen, NULL, 0) == -1)) { + error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + + return 0; +} + +int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len) { + size_t nlen = *len; + + if (unlikely(!mib[0])) + if (unlikely(getsysctl_mib(name, mib, miblen))) + return 1; + + if (unlikely(sysctl(mib, miblen, ptr, len, NULL, 0) == -1)) { + error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(ptr != NULL && nlen != *len)) { + error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)*len, (unsigned long)nlen); + return 1; + } + + return 0; +} + +int getsysctl_mib(const char *name, int *mib, size_t len) { + size_t nlen = len; + + if (unlikely(sysctlnametomib(name, mib, &nlen) == -1)) { + error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} + + +#endif + + +// ===================================================================================================================== +// MacOS + +#if (TARGET_OS == OS_MACOS) + +int getsysctl_by_name(const char *name, void *ptr, size_t len) { + size_t nlen = len; + + if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { + error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno)); + return 1; + } + if (unlikely(nlen != len)) { + error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + return 1; + } + return 0; +} + +#endif // (TARGET_OS == OS_MACOS) diff --git a/libnetdata/os.h b/libnetdata/os.h new file mode 100644 index 0000000..2494174 --- /dev/null +++ b/libnetdata/os.h @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_OS_H +#define NETDATA_OS_H + +#include "libnetdata.h" + +// ===================================================================================================================== +// Linux + +#if (TARGET_OS == OS_LINUX) + + +// ===================================================================================================================== +// FreeBSD + +#elif (TARGET_OS == OS_FREEBSD) + +#include <sys/sysctl.h> + +#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var)) +extern int getsysctl_by_name(const char *name, void *ptr, size_t len); + +#define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int)) + +extern int getsysctl_mib(const char *name, int *mib, size_t len); + +#define GETSYSCTL_SIMPLE(name, mib, var) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), &(var), sizeof(var)) +#define GETSYSCTL_WSIZE(name, mib, var, size) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), var, size) + +extern int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len); + +#define GETSYSCTL_SIZE(name, mib, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), NULL, &(size)) +#define GETSYSCTL(name, mib, var, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), &(var), &(size)) + +extern int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len); + + +// ===================================================================================================================== +// MacOS + +#elif (TARGET_OS == OS_MACOS) + +#include <sys/sysctl.h> + +#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var)) +extern int getsysctl_by_name(const char *name, void *ptr, size_t len); + + +// ===================================================================================================================== +// unknown O/S + +#else +#error unsupported operating system +#endif + + +// ===================================================================================================================== +// common for all O/S + +extern int processors; +extern long get_system_cpus(void); + +extern pid_t pid_max; +extern pid_t get_system_pid_max(void); + +extern unsigned int system_hz; +extern void get_system_HZ(void); + +#endif //NETDATA_OS_H diff --git a/libnetdata/popen/Makefile.am b/libnetdata/popen/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/popen/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/popen/README.md b/libnetdata/popen/README.md new file mode 100644 index 0000000..e51e205 --- /dev/null +++ b/libnetdata/popen/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/popen/README.md +--> + +[](<>) diff --git a/libnetdata/popen/popen.c b/libnetdata/popen/popen.c new file mode 100644 index 0000000..c0135cf --- /dev/null +++ b/libnetdata/popen/popen.c @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +static pthread_mutex_t myp_lock; +static int myp_tracking = 0; + +struct mypopen { + pid_t pid; + struct mypopen *next; + struct mypopen *prev; +}; + +static struct mypopen *mypopen_root = NULL; + +// myp_add_lock takes the lock if we're tracking. +static void myp_add_lock(void) { + if (myp_tracking == 0) + return; + + netdata_mutex_lock(&myp_lock); +} + +// myp_add_unlock release the lock if we're tracking. +static void myp_add_unlock(void) { + if (myp_tracking == 0) + return; + + netdata_mutex_unlock(&myp_lock); +} + +// myp_add_locked adds pid if we're tracking. +// myp_add_lock must have been called previously. +static void myp_add_locked(pid_t pid) { + struct mypopen *mp; + + if (myp_tracking == 0) + return; + + mp = mallocz(sizeof(struct mypopen)); + mp->pid = pid; + + mp->next = mypopen_root; + mp->prev = NULL; + if (mypopen_root != NULL) + mypopen_root->prev = mp; + mypopen_root = mp; + netdata_mutex_unlock(&myp_lock); +} + +// myp_del deletes pid if we're tracking. +static void myp_del(pid_t pid) { + struct mypopen *mp; + + if (myp_tracking == 0) + return; + + netdata_mutex_lock(&myp_lock); + for (mp = mypopen_root; mp != NULL; mp = mp->next) { + if (mp->pid == pid) { + if (mp->next != NULL) + mp->next->prev = mp->prev; + if (mp->prev != NULL) + mp->prev->next = mp->next; + if (mypopen_root == mp) + mypopen_root = mp->next; + freez(mp); + break; + } + } + + if (mp == NULL) + error("Cannot find pid %d.", pid); + + netdata_mutex_unlock(&myp_lock); +} + +#define PIPE_READ 0 +#define PIPE_WRITE 1 + +/* custom_popene flag definitions */ +#define FLAG_CREATE_PIPE 1 // Create a pipe like popen() when set, otherwise set stdout to /dev/null +#define FLAG_CLOSE_FD 2 // Close all file descriptors other than STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO + +/* + * Returns -1 on failure, 0 on success. When FLAG_CREATE_PIPE is set, on success set the FILE *fp pointer. + */ +static inline int custom_popene(const char *command, volatile pid_t *pidptr, char **env, uint8_t flags, FILE **fpp) { + FILE *fp = NULL; + int ret = 0; // success by default + int pipefd[2], error; + pid_t pid; + char *const spawn_argv[] = { + "sh", + "-c", + (char *)command, + NULL + }; + posix_spawnattr_t attr; + posix_spawn_file_actions_t fa; + + if (flags & FLAG_CREATE_PIPE) { + if (pipe(pipefd) == -1) + return -1; + if ((fp = fdopen(pipefd[PIPE_READ], "r")) == NULL) { + goto error_after_pipe; + } + } + + if (flags & FLAG_CLOSE_FD) { + // Mark all files to be closed by the exec() stage of posix_spawn() + int i; + for (i = (int) (sysconf(_SC_OPEN_MAX) - 1); i >= 0; i--) { + if (i != STDIN_FILENO && i != STDERR_FILENO) + (void) fcntl(i, F_SETFD, FD_CLOEXEC); + } + } + + if (!posix_spawn_file_actions_init(&fa)) { + if (flags & FLAG_CREATE_PIPE) { + // move the pipe to stdout in the child + if (posix_spawn_file_actions_adddup2(&fa, pipefd[PIPE_WRITE], STDOUT_FILENO)) { + error("posix_spawn_file_actions_adddup2() failed"); + goto error_after_posix_spawn_file_actions_init; + } + } else { + // set stdout to /dev/null + if (posix_spawn_file_actions_addopen(&fa, STDOUT_FILENO, "/dev/null", O_WRONLY, 0)) { + error("posix_spawn_file_actions_addopen() failed"); + // this is not a fatal error + } + } + } else { + error("posix_spawn_file_actions_init() failed."); + goto error_after_pipe; + } + if (!(error = posix_spawnattr_init(&attr))) { + // reset all signals in the child + sigset_t mask; + + if (posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF)) + error("posix_spawnattr_setflags() failed."); + sigemptyset(&mask); + if (posix_spawnattr_setsigmask(&attr, &mask)) + error("posix_spawnattr_setsigmask() failed."); + } else { + error("posix_spawnattr_init() failed."); + } + + // Take the lock while we fork to ensure we don't race with SIGCHLD + // delivery on a process which exits quickly. + myp_add_lock(); + if (!posix_spawn(&pid, "/bin/sh", &fa, &attr, spawn_argv, env)) { + *pidptr = pid; + myp_add_locked(pid); + debug(D_CHILDS, "Spawned command: '%s' on pid %d from parent pid %d.", command, pid, getpid()); + } else { + myp_add_unlock(); + error("Failed to spawn command: '%s' from parent pid %d.", command, getpid()); + if (flags & FLAG_CREATE_PIPE) { + fclose(fp); + } + ret = -1; + } + if (flags & FLAG_CREATE_PIPE) { + close(pipefd[PIPE_WRITE]); + if (0 == ret) // on success set FILE * pointer + *fpp = fp; + } + + if (!error) { + // posix_spawnattr_init() succeeded + if (posix_spawnattr_destroy(&attr)) + error("posix_spawnattr_destroy"); + } + if (posix_spawn_file_actions_destroy(&fa)) + error("posix_spawn_file_actions_destroy"); + + return ret; + +error_after_posix_spawn_file_actions_init: + if (posix_spawn_file_actions_destroy(&fa)) + error("posix_spawn_file_actions_destroy"); +error_after_pipe: + if (flags & FLAG_CREATE_PIPE) { + if (fp) + fclose(fp); + else + close(pipefd[PIPE_READ]); + + close(pipefd[PIPE_WRITE]); + } + return -1; +} + +// See man environ +extern char **environ; + +// myp_init should be called by apps which act as init +// (pid 1) so that processes created by mypopen and mypopene +// are tracked. This enables the reaper to ignore processes +// which will be handled internally, by calling myp_reap, to +// avoid issues with already reaped processes during wait calls. +// +// Callers should call myp_free() to clean up resources. +void myp_init(void) { + info("process tracking enabled."); + myp_tracking = 1; + + if (netdata_mutex_init(&myp_lock) != 0) { + fatal("myp_init() mutex init failed."); + } +} + +// myp_free cleans up any resources allocated for process +// tracking. +void myp_free(void) { + struct mypopen *mp, *next; + + if (myp_tracking == 0) + return; + + netdata_mutex_lock(&myp_lock); + for (mp = mypopen_root; mp != NULL; mp = next) { + next = mp->next; + freez(mp); + } + + mypopen_root = NULL; + myp_tracking = 0; + netdata_mutex_unlock(&myp_lock); +} + +// myp_reap returns 1 if pid should be reaped, 0 otherwise. +int myp_reap(pid_t pid) { + struct mypopen *mp; + + if (myp_tracking == 0) + return 0; + + netdata_mutex_lock(&myp_lock); + for (mp = mypopen_root; mp != NULL; mp = mp->next) { + if (mp->pid == pid) { + netdata_mutex_unlock(&myp_lock); + return 0; + } + } + netdata_mutex_unlock(&myp_lock); + + return 1; +} + +FILE *mypopen(const char *command, volatile pid_t *pidptr) { + FILE *fp = NULL; + (void)custom_popene(command, pidptr, environ, FLAG_CREATE_PIPE | FLAG_CLOSE_FD, &fp); + return fp; +} + +FILE *mypopene(const char *command, volatile pid_t *pidptr, char **env) { + FILE *fp = NULL; + (void)custom_popene(command, pidptr, env, FLAG_CREATE_PIPE | FLAG_CLOSE_FD, &fp); + return fp; +} + +// returns 0 on success, -1 on failure +int netdata_spawn(const char *command, volatile pid_t *pidptr) { + return custom_popene(command, pidptr, environ, 0, NULL); +} + +int custom_pclose(FILE *fp, pid_t pid) { + int ret; + siginfo_t info; + + debug(D_EXIT, "Request to mypclose() on pid %d", pid); + + if (fp) { + // close the pipe fd + // this is required in musl + // without it the childs do not exit + close(fileno(fp)); + + // close the pipe file pointer + fclose(fp); + } + + errno = 0; + + ret = waitid(P_PID, (id_t) pid, &info, WEXITED); + myp_del(pid); + + if (ret != -1) { + switch (info.si_code) { + case CLD_EXITED: + if(info.si_status) + error("child pid %d exited with code %d.", info.si_pid, info.si_status); + return(info.si_status); + + case CLD_KILLED: + error("child pid %d killed by signal %d.", info.si_pid, info.si_status); + return(-1); + + case CLD_DUMPED: + error("child pid %d core dumped by signal %d.", info.si_pid, info.si_status); + return(-2); + + case CLD_STOPPED: + error("child pid %d stopped by signal %d.", info.si_pid, info.si_status); + return(0); + + case CLD_TRAPPED: + error("child pid %d trapped by signal %d.", info.si_pid, info.si_status); + return(-4); + + case CLD_CONTINUED: + error("child pid %d continued by signal %d.", info.si_pid, info.si_status); + return(0); + + default: + error("child pid %d gave us a SIGCHLD with code %d and status %d.", info.si_pid, info.si_code, info.si_status); + return(-5); + } + } + else + error("Cannot waitid() for pid %d", pid); + + return 0; +} + +int mypclose(FILE *fp, pid_t pid) +{ + return custom_pclose(fp, pid); +} + +int netdata_spawn_waitpid(pid_t pid) +{ + return custom_pclose(NULL, pid); +}
\ No newline at end of file diff --git a/libnetdata/popen/popen.h b/libnetdata/popen/popen.h new file mode 100644 index 0000000..f387cff --- /dev/null +++ b/libnetdata/popen/popen.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_POPEN_H +#define NETDATA_POPEN_H 1 + +#include "../libnetdata.h" + +#define PIPE_READ 0 +#define PIPE_WRITE 1 + +extern FILE *mypopen(const char *command, volatile pid_t *pidptr); +extern FILE *mypopene(const char *command, volatile pid_t *pidptr, char **env); +extern int mypclose(FILE *fp, pid_t pid); +extern int netdata_spawn(const char *command, volatile pid_t *pidptr); +extern int netdata_spawn_waitpid(pid_t pid); +extern void myp_init(void); +extern void myp_free(void); +extern int myp_reap(pid_t pid); + +extern void signals_unblock(void); +extern void signals_reset(void); + +#endif /* NETDATA_POPEN_H */ diff --git a/libnetdata/procfile/Makefile.am b/libnetdata/procfile/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/procfile/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/procfile/README.md b/libnetdata/procfile/README.md new file mode 100644 index 0000000..3eeb1e2 --- /dev/null +++ b/libnetdata/procfile/README.md @@ -0,0 +1,67 @@ +<!-- +title: "PROCFILE" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/procfile/README.md +--> + +# PROCFILE + +procfile is a library for reading text data files (i.e `/proc` files) in the fastest possible way. + +## How it works + +The library automatically adapts (through the iterations) its memory so that each file +is read with single `read()` call. + +Then the library splits the file into words, using the supplied separators. +The library also supported quoted words (i.e. strings within of which the separators are ignored). + +### Initialization + +Initially the caller: + +- calls `procfile_open()` to open the file and allocate the structures needed. + +### Iterations + +For each iteration, the caller: + +- calls `procfile_readall()` to read updated contents. + This call also rewinds (`lseek()` to 0) before reading it. + + For every file, a [BUFFER](/libnetdata/buffer/README.md) is used that is automatically adjusted to fit the entire + file contents of the file. So the file is read with a single `read()` call (providing atomicity / consistency when + the data are read from the kernel). + + Once the data are read, 2 arrays of pointers are updated: + + - a `words` array, pointing to each word in the data read + - a `lines` array, pointing to the first word for each line + + This is highly optimized. Both arrays are automatically adjusted to + fit all contents and are updated in a single pass on the data. + + The library provides a number of macros: + + - `procfile_lines()` returns the # of lines read + - `procfile_linewords()` returns the # of words in the given line + - `procfile_word()` returns a pointer the given word # + - `procfile_line()` returns a pointer to the first word of the given line # + - `procfile_lineword()` returns a pointer to the given word # of the given line # + +### Cleanup + +When the caller exits: + +- calls `procfile_free()` to close the file and free all memory used. + +### Performance + +- a **raspberry Pi 1** (the oldest single core one) can process 5.000+ `/proc` files per second. +- a **J1900 Celeron** processor can process 23.000+ `/proc` files per second per core. + +To achieve this kind of performance, the library tries to work in batches so that the code +and the data are inside the processor's caches. + +This library is extensively used in Netdata and its plugins. + +[](<>) diff --git a/libnetdata/procfile/procfile.c b/libnetdata/procfile/procfile.c new file mode 100644 index 0000000..4a812ba --- /dev/null +++ b/libnetdata/procfile/procfile.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#define PF_PREFIX "PROCFILE" + +#define PFWORDS_INCREASE_STEP 200 +#define PFLINES_INCREASE_STEP 10 +#define PROCFILE_INCREMENT_BUFFER 512 + +int procfile_open_flags = O_RDONLY; + +int procfile_adaptive_initial_allocation = 0; + +// if adaptive allocation is set, these store the +// max values we have seen so far +size_t procfile_max_lines = PFLINES_INCREASE_STEP; +size_t procfile_max_words = PFWORDS_INCREASE_STEP; +size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER; + + +// ---------------------------------------------------------------------------- + +char *procfile_filename(procfile *ff) { + if(ff->filename[0]) return ff->filename; + + char buffer[FILENAME_MAX + 1]; + snprintfz(buffer, FILENAME_MAX, "/proc/self/fd/%d", ff->fd); + + ssize_t l = readlink(buffer, ff->filename, FILENAME_MAX); + if(unlikely(l == -1)) + snprintfz(ff->filename, FILENAME_MAX, "unknown filename for fd %d", ff->fd); + else + ff->filename[l] = '\0'; + + // on non-linux systems, something like this will be needed + // fcntl(ff->fd, F_GETPATH, ff->filename) + + return ff->filename; +} + +// ---------------------------------------------------------------------------- +// An array of words + +static inline void pfwords_add(procfile *ff, char *str) { + // debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str); + + pfwords *fw = ff->words; + if(unlikely(fw->len == fw->size)) { + // debug(D_PROCFILE, PF_PREFIX ": expanding words"); + + ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + PFWORDS_INCREASE_STEP) * sizeof(char *)); + fw->size += PFWORDS_INCREASE_STEP; + } + + fw->words[fw->len++] = str; +} + +NEVERNULL +static inline pfwords *pfwords_new(void) { + // debug(D_PROCFILE, PF_PREFIX ": initializing words"); + + size_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP; + + pfwords *new = mallocz(sizeof(pfwords) + size * sizeof(char *)); + new->len = 0; + new->size = size; + return new; +} + +static inline void pfwords_reset(pfwords *fw) { + // debug(D_PROCFILE, PF_PREFIX ": reseting words"); + fw->len = 0; +} + +static inline void pfwords_free(pfwords *fw) { + // debug(D_PROCFILE, PF_PREFIX ": freeing words"); + + freez(fw); +} + + +// ---------------------------------------------------------------------------- +// An array of lines + +NEVERNULL +static inline size_t *pflines_add(procfile *ff) { + // debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word); + + pflines *fl = ff->lines; + if(unlikely(fl->len == fl->size)) { + // debug(D_PROCFILE, PF_PREFIX ": expanding lines"); + + ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + PFLINES_INCREASE_STEP) * sizeof(ffline)); + fl->size += PFLINES_INCREASE_STEP; + } + + ffline *ffl = &fl->lines[fl->len++]; + ffl->words = 0; + ffl->first = ff->words->len; + + return &ffl->words; +} + +NEVERNULL +static inline pflines *pflines_new(void) { + // debug(D_PROCFILE, PF_PREFIX ": initializing lines"); + + size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_words : PFLINES_INCREASE_STEP; + + pflines *new = mallocz(sizeof(pflines) + size * sizeof(ffline)); + new->len = 0; + new->size = size; + return new; +} + +static inline void pflines_reset(pflines *fl) { + // debug(D_PROCFILE, PF_PREFIX ": reseting lines"); + + fl->len = 0; +} + +static inline void pflines_free(pflines *fl) { + // debug(D_PROCFILE, PF_PREFIX ": freeing lines"); + + freez(fl); +} + + +// ---------------------------------------------------------------------------- +// The procfile + +void procfile_close(procfile *ff) { + if(unlikely(!ff)) return; + + debug(D_PROCFILE, PF_PREFIX ": Closing file '%s'", procfile_filename(ff)); + + if(likely(ff->lines)) pflines_free(ff->lines); + if(likely(ff->words)) pfwords_free(ff->words); + + if(likely(ff->fd != -1)) close(ff->fd); + freez(ff); +} + +NOINLINE +static void procfile_parser(procfile *ff) { + // debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename); + + char *s = ff->data // our current position + , *e = &ff->data[ff->len] // the terminating null + , *t = ff->data; // the first character of a word (or quoted / parenthesized string) + + // the look up array to find our type of character + PF_CHAR_TYPE *separators = ff->separators; + + char quote = 0; // the quote character - only when in quoted string + size_t opened = 0; // counts the number of open parenthesis + + size_t *line_words = pflines_add(ff); + + while(s < e) { + PF_CHAR_TYPE ct = separators[(unsigned char)(*s)]; + + // this is faster than a switch() + // read more here: http://lazarenko.me/switch/ + if(likely(ct == PF_CHAR_IS_WORD)) { + s++; + } + else if(likely(ct == PF_CHAR_IS_SEPARATOR)) { + if(!quote && !opened) { + if (s != t) { + // separator, but we have word before it + *s = '\0'; + pfwords_add(ff, t); + (*line_words)++; + t = ++s; + } + else { + // separator at the beginning + // skip it + t = ++s; + } + } + else { + // we are inside a quote or parenthesized string + s++; + } + } + else if(likely(ct == PF_CHAR_IS_NEWLINE)) { + // end of line + + *s = '\0'; + pfwords_add(ff, t); + (*line_words)++; + t = ++s; + + // debug(D_PROCFILE, PF_PREFIX ": ended line %d with %d words", l, ff->lines->lines[l].words); + + line_words = pflines_add(ff); + } + else if(likely(ct == PF_CHAR_IS_QUOTE)) { + if(unlikely(!quote && s == t)) { + // quote opened at the beginning + quote = *s; + t = ++s; + } + else if(unlikely(quote && quote == *s)) { + // quote closed + quote = 0; + + *s = '\0'; + pfwords_add(ff, t); + (*line_words)++; + t = ++s; + } + else + s++; + } + else if(likely(ct == PF_CHAR_IS_OPEN)) { + if(s == t) { + opened++; + t = ++s; + } + else if(opened) { + opened++; + s++; + } + else + s++; + } + else if(likely(ct == PF_CHAR_IS_CLOSE)) { + if(opened) { + opened--; + + if(!opened) { + *s = '\0'; + pfwords_add(ff, t); + (*line_words)++; + t = ++s; + } + else + s++; + } + else + s++; + } + else + fatal("Internal Error: procfile_readall() does not handle all the cases."); + } + + if(likely(s > t && t < e)) { + // the last word + if(unlikely(ff->len >= ff->size)) { + // we are going to loose the last byte + s = &ff->data[ff->size - 1]; + } + + *s = '\0'; + pfwords_add(ff, t); + (*line_words)++; + // t = ++s; + } +} + +procfile *procfile_readall(procfile *ff) { + // debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename); + + ff->len = 0; // zero the used size + ssize_t r = 1; // read at least once + while(r > 0) { + ssize_t s = ff->len; + ssize_t x = ff->size - s; + + if(unlikely(!x)) { + debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", procfile_filename(ff)); + ff = reallocz(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER); + ff->size += PROCFILE_INCREMENT_BUFFER; + } + + debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s)); + r = read(ff->fd, &ff->data[s], ff->size - s); + if(unlikely(r == -1)) { + if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd); + procfile_close(ff); + return NULL; + } + + ff->len += r; + } + + // debug(D_PROCFILE, "Rewinding file '%s'", ff->filename); + if(unlikely(lseek(ff->fd, 0, SEEK_SET) == -1)) { + if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot rewind on file '%s'.", procfile_filename(ff)); + procfile_close(ff); + return NULL; + } + + pflines_reset(ff->lines); + pfwords_reset(ff->words); + procfile_parser(ff); + + if(unlikely(procfile_adaptive_initial_allocation)) { + if(unlikely(ff->len > procfile_max_allocation)) procfile_max_allocation = ff->len; + if(unlikely(ff->lines->len > procfile_max_lines)) procfile_max_lines = ff->lines->len; + if(unlikely(ff->words->len > procfile_max_words)) procfile_max_words = ff->words->len; + } + + // debug(D_PROCFILE, "File '%s' updated.", ff->filename); + return ff; +} + +NOINLINE +static void procfile_set_separators(procfile *ff, const char *separators) { + static PF_CHAR_TYPE def[256]; + static char initilized = 0; + + if(unlikely(!initilized)) { + // this is thread safe + // if initialized is zero, multiple threads may be executing + // this code at the same time, setting in def[] the exact same values + int i = 256; + while(i--) { + if(unlikely(i == '\n' || i == '\r')) + def[i] = PF_CHAR_IS_NEWLINE; + + else if(unlikely(isspace(i) || !isprint(i))) + def[i] = PF_CHAR_IS_SEPARATOR; + + else + def[i] = PF_CHAR_IS_WORD; + } + + initilized = 1; + } + + // copy the default + PF_CHAR_TYPE *ffs = ff->separators, *ffd = def, *ffe = &def[256]; + while(ffd != ffe) + *ffs++ = *ffd++; + + // set the separators + if(unlikely(!separators)) + separators = " \t=|"; + + ffs = ff->separators; + const char *s = separators; + while(*s) + ffs[(int)*s++] = PF_CHAR_IS_SEPARATOR; +} + +void procfile_set_quotes(procfile *ff, const char *quotes) { + PF_CHAR_TYPE *ffs = ff->separators; + + // remove all quotes + int i = 256; + while(i--) + if(unlikely(ffs[i] == PF_CHAR_IS_QUOTE)) + ffs[i] = PF_CHAR_IS_WORD; + + // if nothing given, return + if(unlikely(!quotes || !*quotes)) + return; + + // set the quotes + const char *s = quotes; + while(*s) + ffs[(int)*s++] = PF_CHAR_IS_QUOTE; +} + +void procfile_set_open_close(procfile *ff, const char *open, const char *close) { + PF_CHAR_TYPE *ffs = ff->separators; + + // remove all open/close + int i = 256; + while(i--) + if(unlikely(ffs[i] == PF_CHAR_IS_OPEN || ffs[i] == PF_CHAR_IS_CLOSE)) + ffs[i] = PF_CHAR_IS_WORD; + + // if nothing given, return + if(unlikely(!open || !*open || !close || !*close)) + return; + + // set the openings + const char *s = open; + while(*s) + ffs[(int)*s++] = PF_CHAR_IS_OPEN; + + // set the closings + s = close; + while(*s) + ffs[(int)*s++] = PF_CHAR_IS_CLOSE; +} + +procfile *procfile_open(const char *filename, const char *separators, uint32_t flags) { + debug(D_PROCFILE, PF_PREFIX ": Opening file '%s'", filename); + + int fd = open(filename, procfile_open_flags, 0666); + if(unlikely(fd == -1)) { + if(unlikely(!(flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) error(PF_PREFIX ": Cannot open file '%s'", filename); + return NULL; + } + + // info("PROCFILE: opened '%s' on fd %d", filename, fd); + + size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_allocation : PROCFILE_INCREMENT_BUFFER; + procfile *ff = mallocz(sizeof(procfile) + size); + + //strncpyz(ff->filename, filename, FILENAME_MAX); + ff->filename[0] = '\0'; + + ff->fd = fd; + ff->size = size; + ff->len = 0; + ff->flags = flags; + + ff->lines = pflines_new(); + ff->words = pfwords_new(); + + procfile_set_separators(ff, separators); + + debug(D_PROCFILE, "File '%s' opened.", filename); + return ff; +} + +procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags) { + if(unlikely(!ff)) return procfile_open(filename, separators, flags); + + if(likely(ff->fd != -1)) { + // info("PROCFILE: closing fd %d", ff->fd); + close(ff->fd); + } + + ff->fd = open(filename, procfile_open_flags, 0666); + if(unlikely(ff->fd == -1)) { + procfile_close(ff); + return NULL; + } + + // info("PROCFILE: opened '%s' on fd %d", filename, ff->fd); + + //strncpyz(ff->filename, filename, FILENAME_MAX); + ff->filename[0] = '\0'; + ff->flags = flags; + + // do not do the separators again if NULL is given + if(likely(separators)) procfile_set_separators(ff, separators); + + return ff; +} + +// ---------------------------------------------------------------------------- +// example parsing of procfile data + +void procfile_print(procfile *ff) { + size_t lines = procfile_lines(ff), l; + char *s; + (void)s; + + debug(D_PROCFILE, "File '%s' with %zu lines and %zu words", procfile_filename(ff), ff->lines->len, ff->words->len); + + for(l = 0; likely(l < lines) ;l++) { + size_t words = procfile_linewords(ff, l); + + debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); + + size_t w; + for(w = 0; likely(w < words) ;w++) { + s = procfile_lineword(ff, l, w); + debug(D_PROCFILE, " [%zu.%zu] '%s'", l, w, s); + } + } +} diff --git a/libnetdata/procfile/procfile.h b/libnetdata/procfile/procfile.h new file mode 100644 index 0000000..b107358 --- /dev/null +++ b/libnetdata/procfile/procfile.h @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PROCFILE_H +#define NETDATA_PROCFILE_H 1 + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// An array of words + +typedef struct { + size_t len; // used entries + size_t size; // capacity + char *words[]; // array of pointers +} pfwords; + + +// ---------------------------------------------------------------------------- +// An array of lines + +typedef struct { + size_t words; // how many words this line has + size_t first; // the id of the first word of this line + // in the words array +} ffline; + +typedef struct { + size_t len; // used entries + size_t size; // capacity + ffline lines[]; // array of lines +} pflines; + + +// ---------------------------------------------------------------------------- +// The procfile + +#define PROCFILE_FLAG_DEFAULT 0x00000000 +#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001 + +typedef enum procfile_separator { + PF_CHAR_IS_SEPARATOR, + PF_CHAR_IS_NEWLINE, + PF_CHAR_IS_WORD, + PF_CHAR_IS_QUOTE, + PF_CHAR_IS_OPEN, + PF_CHAR_IS_CLOSE +} PF_CHAR_TYPE; + +typedef struct { + char filename[FILENAME_MAX + 1]; // not populated until profile_filename() is called + + uint32_t flags; + int fd; // the file desriptor + size_t len; // the bytes we have placed into data + size_t size; // the bytes we have allocated for data + pflines *lines; + pfwords *words; + PF_CHAR_TYPE separators[256]; + char data[]; // allocated buffer to keep file contents +} procfile; + +// close the proc file and free all related memory +extern void procfile_close(procfile *ff); + +// (re)read and parse the proc file +extern procfile *procfile_readall(procfile *ff); + +// open a /proc or /sys file +extern procfile *procfile_open(const char *filename, const char *separators, uint32_t flags); + +// re-open a file +// if separators == NULL, the last separators are used +extern procfile *procfile_reopen(procfile *ff, const char *filename, const char *separators, uint32_t flags); + +// example walk-through a procfile parsed file +extern void procfile_print(procfile *ff); + +extern void procfile_set_quotes(procfile *ff, const char *quotes); +extern void procfile_set_open_close(procfile *ff, const char *open, const char *close); + +extern char *procfile_filename(procfile *ff); + +// ---------------------------------------------------------------------------- + +// set to the O_XXXX flags, to have procfile_open and procfile_reopen use them when opening proc files +extern int procfile_open_flags; + +// set this to 1, to have procfile adapt its initial buffer allocation to the max allocation used so far +extern int procfile_adaptive_initial_allocation; + +// return the number of lines present +#define procfile_lines(ff) ((ff)->lines->len) + +// return the number of words of the Nth line +#define procfile_linewords(ff, line) (((line) < procfile_lines(ff)) ? (ff)->lines->lines[(line)].words : 0) + +// return the Nth word of the file, or empty string +#define procfile_word(ff, word) (((word) < (ff)->words->len) ? (ff)->words->words[(word)] : "") + +// return the first word of the Nth line, or empty string +#define procfile_line(ff, line) (((line) < procfile_lines(ff)) ? procfile_word((ff), (ff)->lines->lines[(line)].first) : "") + +// return the Nth word of the current line +#define procfile_lineword(ff, line, word) (((line) < procfile_lines(ff) && (word) < procfile_linewords((ff), (line))) ? procfile_word((ff), (ff)->lines->lines[(line)].first + (word)) : "") + +#endif /* NETDATA_PROCFILE_H */ diff --git a/libnetdata/required_dummies.h b/libnetdata/required_dummies.h new file mode 100644 index 0000000..aa87e39 --- /dev/null +++ b/libnetdata/required_dummies.h @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_LIB_DUMMIES_H +#define NETDATA_LIB_DUMMIES_H 1 + +// callback required by fatal() +void netdata_cleanup_and_exit(int ret) +{ + exit(ret); +} + +void send_statistics(const char *action, const char *action_result, const char *action_data) +{ + (void)action; + (void)action_result; + (void)action_data; + return; +} + +// callbacks required by popen() +void signals_block(void){}; +void signals_unblock(void){}; +void signals_reset(void){}; + +// callback required by eval() +int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result) +{ + (void)variable; + (void)hash; + (void)rc; + (void)result; + return 0; +}; + +// required by get_system_cpus() +char *netdata_configured_host_prefix = ""; + +#endif // NETDATA_LIB_DUMMIES_H diff --git a/libnetdata/simple_pattern/Makefile.am b/libnetdata/simple_pattern/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/simple_pattern/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/simple_pattern/README.md b/libnetdata/simple_pattern/README.md new file mode 100644 index 0000000..a90759c --- /dev/null +++ b/libnetdata/simple_pattern/README.md @@ -0,0 +1,43 @@ +<!-- +title: "Netdata simple patterns" +description: "Netdata supports simple patterns, which are less cryptic versions of regular expressions. Use familiar notation for powerful results." +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/simple_pattern/README.md +--> + +# Netdata simple patterns + +Unix prefers regular expressions. But they are just too hard, too cryptic +to use, write and understand. + +So, Netdata supports **simple patterns**. + +Simple patterns are a space separated list of words, that can have `*` +as a wildcard. Each world may use any number of `*`. Simple patterns +allow **negative** matches by prefixing a word with `!`. + +So, `pattern = !*bad* *` will match anything, except all those that +contain the word `bad`. + +Simple patterns are quite powerful: `pattern = *foobar* !foo* !*bar *` +matches everything containing `foobar`, except strings that start +with `foo` or end with `bar`. + +You can use the Netdata command line to check simple patterns, +like this: + +```sh +# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world' +RESULT: MATCHED - pattern '*foobar* !foo* !*bar *' matches 'hello world' + +# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world bar' +RESULT: NOT MATCHED - pattern '*foobar* !foo* !*bar *' does not match 'hello world bar' + +# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world foobar' +RESULT: MATCHED - pattern '*foobar* !foo* !*bar *' matches 'hello world foobar' +``` + +Netdata stops processing to the first positive or negative match +(left to right). If it is not matched by either positive or negative +patterns, it is denied at the end. + +[](<>) diff --git a/libnetdata/simple_pattern/simple_pattern.c b/libnetdata/simple_pattern/simple_pattern.c new file mode 100644 index 0000000..70b06a2 --- /dev/null +++ b/libnetdata/simple_pattern/simple_pattern.c @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +struct simple_pattern { + const char *match; + size_t len; + + SIMPLE_PREFIX_MODE mode; + char negative; + + struct simple_pattern *child; + + struct simple_pattern *next; +}; + +static inline struct simple_pattern *parse_pattern(char *str, SIMPLE_PREFIX_MODE default_mode) { + // fprintf(stderr, "PARSING PATTERN: '%s'\n", str); + + SIMPLE_PREFIX_MODE mode; + struct simple_pattern *child = NULL; + + char *s = str, *c = str; + + // skip asterisks in front + while(*c == '*') c++; + + // find the next asterisk + while(*c && *c != '*') c++; + + // do we have an asterisk in the middle? + if(*c == '*' && c[1] != '\0') { + // yes, we have + child = parse_pattern(c, default_mode); + c[1] = '\0'; + } + + // check what this one matches + + size_t len = strlen(s); + if(len >= 2 && *s == '*' && s[len - 1] == '*') { + s[len - 1] = '\0'; + s++; + mode = SIMPLE_PATTERN_SUBSTRING; + } + else if(len >= 1 && *s == '*') { + s++; + mode = SIMPLE_PATTERN_SUFFIX; + } + else if(len >= 1 && s[len - 1] == '*') { + s[len - 1] = '\0'; + mode = SIMPLE_PATTERN_PREFIX; + } + else + mode = default_mode; + + // allocate the structure + struct simple_pattern *m = callocz(1, sizeof(struct simple_pattern)); + if(*s) { + m->match = strdupz(s); + m->len = strlen(m->match); + m->mode = mode; + } + else { + m->mode = SIMPLE_PATTERN_SUBSTRING; + } + + m->child = child; + + return m; +} + +SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode) { + struct simple_pattern *root = NULL, *last = NULL; + + if(unlikely(!list || !*list)) return root; + + int isseparator[256] = { + [' '] = 1 // space + , ['\t'] = 1 // tab + , ['\r'] = 1 // carriage return + , ['\n'] = 1 // new line + , ['\f'] = 1 // form feed + , ['\v'] = 1 // vertical tab + }; + + if (unlikely(separators && *separators)) { + memset(&isseparator[0], 0, sizeof(isseparator)); + while(*separators) isseparator[(unsigned char)*separators++] = 1; + } + + char *buf = mallocz(strlen(list) + 1); + const char *s = list; + + while(s && *s) { + buf[0] = '\0'; + char *c = buf; + + char negative = 0; + + // skip all spaces + while(isseparator[(unsigned char)*s]) + s++; + + if(*s == '!') { + negative = 1; + s++; + } + + // empty string + if(unlikely(!*s)) + break; + + // find the next space + char escape = 0; + while(*s) { + if(*s == '\\' && !escape) { + escape = 1; + s++; + } + else { + if (isseparator[(unsigned char)*s] && !escape) { + s++; + break; + } + + *c++ = *s++; + escape = 0; + } + } + + // terminate our string + *c = '\0'; + + // if we matched the empty string, skip it + if(unlikely(!*buf)) + continue; + + // fprintf(stderr, "FOUND PATTERN: '%s'\n", buf); + struct simple_pattern *m = parse_pattern(buf, default_mode); + m->negative = negative; + + // link it at the end + if(unlikely(!root)) + root = last = m; + else { + last->next = m; + last = m; + } + } + + freez(buf); + return (SIMPLE_PATTERN *)root; +} + +static inline char *add_wildcarded(const char *matched, size_t matched_size, char *wildcarded, size_t *wildcarded_size) { + //if(matched_size) { + // char buf[matched_size + 1]; + // strncpyz(buf, matched, matched_size); + // fprintf(stderr, "ADD WILDCARDED '%s' of length %zu\n", buf, matched_size); + //} + + if(unlikely(wildcarded && *wildcarded_size && matched && *matched && matched_size)) { + size_t wss = *wildcarded_size - 1; + size_t len = (matched_size < wss)?matched_size:wss; + if(likely(len)) { + strncpyz(wildcarded, matched, len); + + *wildcarded_size -= len; + return &wildcarded[len]; + } + } + + return wildcarded; +} + +static inline int match_pattern(struct simple_pattern *m, const char *str, size_t len, char *wildcarded, size_t *wildcarded_size) { + char *s; + + if(m->len <= len) { + switch(m->mode) { + case SIMPLE_PATTERN_SUBSTRING: + if(!m->len) return 1; + if((s = strstr(str, m->match))) { + wildcarded = add_wildcarded(str, s - str, wildcarded, wildcarded_size); + if(!m->child) { + wildcarded = add_wildcarded(&s[m->len], len - (&s[m->len] - str), wildcarded, wildcarded_size); + return 1; + } + return match_pattern(m->child, &s[m->len], len - (s - str) - m->len, wildcarded, wildcarded_size); + } + break; + + case SIMPLE_PATTERN_PREFIX: + if(unlikely(strncmp(str, m->match, m->len) == 0)) { + if(!m->child) { + wildcarded = add_wildcarded(&str[m->len], len - m->len, wildcarded, wildcarded_size); + return 1; + } + return match_pattern(m->child, &str[m->len], len - m->len, wildcarded, wildcarded_size); + } + break; + + case SIMPLE_PATTERN_SUFFIX: + if(unlikely(strcmp(&str[len - m->len], m->match) == 0)) { + wildcarded = add_wildcarded(str, len - m->len, wildcarded, wildcarded_size); + if(!m->child) return 1; + return 0; + } + break; + + case SIMPLE_PATTERN_EXACT: + default: + if(unlikely(strcmp(str, m->match) == 0)) { + if(!m->child) return 1; + return 0; + } + break; + } + } + + return 0; +} + +int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size) { + struct simple_pattern *m, *root = (struct simple_pattern *)list; + + if(unlikely(!root || !str || !*str)) return 0; + + size_t len = strlen(str); + for(m = root; m ; m = m->next) { + char *ws = wildcarded; + size_t wss = wildcarded_size; + if(unlikely(ws)) *ws = '\0'; + + if (match_pattern(m, str, len, ws, &wss)) { + + //if(ws && wss) + // fprintf(stderr, "FINAL WILDCARDED '%s' of length %zu\n", ws, strlen(ws)); + + if (m->negative) return 0; + return 1; + } + } + + return 0; +} + +static inline void free_pattern(struct simple_pattern *m) { + if(!m) return; + + free_pattern(m->child); + free_pattern(m->next); + freez((void *)m->match); + freez(m); +} + +void simple_pattern_free(SIMPLE_PATTERN *list) { + if(!list) return; + + free_pattern(((struct simple_pattern *)list)); +} + +/* Debugging patterns + + This code should be dead - it is useful for debugging but should not be called by production code. + Feel free to comment it out, but please leave it in the file. +*/ +extern void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p) +{ + struct simple_pattern *root = (struct simple_pattern *)p; + if(root==NULL) { + debug(debug_type,"dump_pattern(NULL)"); + return; + } + debug(debug_type,"dump_pattern(%p) child=%p next=%p mode=%u match=%s", root, root->child, root->next, root->mode, + root->match); + if(root->child!=NULL) + simple_pattern_dump(debug_type, (SIMPLE_PATTERN*)root->child); + if(root->next!=NULL) + simple_pattern_dump(debug_type, (SIMPLE_PATTERN*)root->next); +} + +/* Heuristic: decide if the pattern could match a DNS name. + + Although this functionality is used directly by socket.c:connection_allowed() it must be in this file + because of the SIMPLE_PATTERN/simple_pattern structure hiding. + Based on RFC952 / RFC1123. We need to decide if the pattern may match a DNS name, or not. For the negative + cases we need to be sure that it can only match an ipv4 or ipv6 address: + * IPv6 addresses contain ':', which are illegal characters in DNS. + * IPv4 addresses cannot contain alpha- characters. + * DNS TLDs must be alphanumeric to distinguish from IPv4. + Some patterns (e.g. "*a*" ) could match multiple cases (i.e. DNS or IPv6). + Some patterns will be awkward (e.g. "192.168.*") as they look like they are intended to match IPv4-only + but could match DNS (i.e. "192.168.com" is a valid name). +*/ +static void scan_is_potential_name(struct simple_pattern *p, int *alpha, int *colon, int *wildcards) +{ + while (p) { + if (p->match) { + if(p->mode == SIMPLE_PATTERN_EXACT && !strcmp("localhost", p->match)) { + p = p->child; + continue; + } + char const *scan = p->match; + while (*scan != 0) { + if ((*scan >= 'a' && *scan <= 'z') || (*scan >= 'A' && *scan <= 'Z')) + *alpha = 1; + if (*scan == ':') + *colon = 1; + scan++; + } + if (p->mode != SIMPLE_PATTERN_EXACT) + *wildcards = 1; + p = p->child; + } + } +} + +extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p) +{ + int alpha=0, colon=0, wildcards=0; + struct simple_pattern *root = (struct simple_pattern*)p; + while (root != NULL) { + if (root->match != NULL) { + scan_is_potential_name(root, &alpha, &colon, &wildcards); + } + if (root->mode != SIMPLE_PATTERN_EXACT) + wildcards = 1; + root = root->next; + } + return (alpha || wildcards) && !colon; +} + +char *simple_pattern_trim_around_equal(char *src) { + char *store = mallocz(strlen(src) +1); + if(!store) + return NULL; + + char *dst = store; + while (*src) { + if (*src == '=') { + if (*(dst -1) == ' ') + dst--; + + *dst++ = *src++; + if (*src == ' ') + src++; + } + + *dst++ = *src++; + } + *dst = 0x00; + + return store; +} + +char *simple_pattern_iterate(SIMPLE_PATTERN **p) +{ + struct simple_pattern *root = (struct simple_pattern *) *p; + struct simple_pattern **Proot = (struct simple_pattern **)p; + + (*Proot) = (*Proot)->next; + return (char *) root->match; +} diff --git a/libnetdata/simple_pattern/simple_pattern.h b/libnetdata/simple_pattern/simple_pattern.h new file mode 100644 index 0000000..36fbbde --- /dev/null +++ b/libnetdata/simple_pattern/simple_pattern.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SIMPLE_PATTERN_H +#define NETDATA_SIMPLE_PATTERN_H + +#include "../libnetdata.h" + + +typedef enum { + SIMPLE_PATTERN_EXACT, + SIMPLE_PATTERN_PREFIX, + SIMPLE_PATTERN_SUFFIX, + SIMPLE_PATTERN_SUBSTRING +} SIMPLE_PREFIX_MODE; + +typedef void SIMPLE_PATTERN; + +// create a simple_pattern from the string given +// default_mode is used in cases where EXACT matches, without an asterisk, +// should be considered PREFIX matches. +extern SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, SIMPLE_PREFIX_MODE default_mode); + +// test if string str is matched from the pattern and fill 'wildcarded' with the parts matched by '*' +extern int simple_pattern_matches_extract(SIMPLE_PATTERN *list, const char *str, char *wildcarded, size_t wildcarded_size); + +// test if string str is matched from the pattern +#define simple_pattern_matches(list, str) simple_pattern_matches_extract(list, str, NULL, 0) + +// free a simple_pattern that was created with simple_pattern_create() +// list can be NULL, in which case, this does nothing. +extern void simple_pattern_free(SIMPLE_PATTERN *list); + +extern void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p) ; +extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p) ; +extern char *simple_pattern_iterate(SIMPLE_PATTERN **p); + +//Auxiliary function to create a pattern +char *simple_pattern_trim_around_equal(char *src); + +#endif //NETDATA_SIMPLE_PATTERN_H diff --git a/libnetdata/socket/Makefile.am b/libnetdata/socket/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/socket/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/socket/README.md b/libnetdata/socket/README.md new file mode 100644 index 0000000..6736f13 --- /dev/null +++ b/libnetdata/socket/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/socket/README.md +--> + +[](<>) diff --git a/libnetdata/socket/security.c b/libnetdata/socket/security.c new file mode 100644 index 0000000..53366c4 --- /dev/null +++ b/libnetdata/socket/security.c @@ -0,0 +1,385 @@ +#include "../libnetdata.h" + +#ifdef ENABLE_HTTPS + +SSL_CTX *netdata_exporting_ctx=NULL; +SSL_CTX *netdata_client_ctx=NULL; +SSL_CTX *netdata_srv_ctx=NULL; +const char *security_key=NULL; +const char *security_cert=NULL; +const char *tls_version=NULL; +const char *tls_ciphers=NULL; +int netdata_validate_server = NETDATA_SSL_VALID_CERTIFICATE; + +/** + * Info Callback + * + * Function used as callback for the OpenSSL Library + * + * @param ssl a pointer to the SSL structure of the client + * @param where the variable with the flags set. + * @param ret the return of the caller + */ +static void security_info_callback(const SSL *ssl, int where, int ret __maybe_unused) { + (void)ssl; + if (where & SSL_CB_ALERT) { + debug(D_WEB_CLIENT,"SSL INFO CALLBACK %s %s", SSL_alert_type_string(ret), SSL_alert_desc_string_long(ret)); + } +} + +/** + * OpenSSL Library + * + * Starts the openssl library for the Netdata. + */ +void security_openssl_library() +{ +#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 +# if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) + OPENSSL_config(NULL); +# endif + + SSL_load_error_strings(); + + SSL_library_init(); +#else + if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) != 1) { + error("SSL library cannot be initialized."); + } +#endif +} + +#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_110 +/** + * TLS version + * + * Returns the TLS version depending of the user input. + * + * @param lversion is the user input. + * + * @return it returns the version number. + */ +int tls_select_version(const char *lversion) { + if (!strcmp(lversion, "1") || !strcmp(lversion, "1.0")) + return TLS1_VERSION; + else if (!strcmp(lversion, "1.1")) + return TLS1_1_VERSION; + else if (!strcmp(lversion, "1.2")) + return TLS1_2_VERSION; +#if defined(TLS1_3_VERSION) + else if (!strcmp(lversion, "1.3")) + return TLS1_3_VERSION; +#endif + +#if defined(TLS_MAX_VERSION) + return TLS_MAX_VERSION; +#else + return TLS1_2_VERSION; +#endif +} +#endif + +/** + * OpenSSL common options + * + * Clients and SERVER have common options, this function is responsible to set them in the context. + * + * @param ctx the initialized SSL context. + * @param side 0 means server, and 1 client. + */ +void security_openssl_common_options(SSL_CTX *ctx, int side) { +#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_110 + if (!side) { + int version = tls_select_version(tls_version) ; +#endif +#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 + SSL_CTX_set_options (ctx,SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION); +#else + SSL_CTX_set_min_proto_version(ctx, TLS1_VERSION); + SSL_CTX_set_max_proto_version(ctx, version); + + if(tls_ciphers && strcmp(tls_ciphers, "none") != 0) { + if (!SSL_CTX_set_cipher_list(ctx, tls_ciphers)) { + error("SSL error. cannot set the cipher list"); + } + } + } +#endif + + SSL_CTX_set_mode(ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); +} + +/** + * Initialize Openssl Client + * + * Starts the client context with TLS 1.2. + * + * @return It returns the context on success or NULL otherwise + */ +SSL_CTX * security_initialize_openssl_client() { + SSL_CTX *ctx; +#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 + ctx = SSL_CTX_new(SSLv23_client_method()); +#else + ctx = SSL_CTX_new(TLS_client_method()); +#endif + if(ctx) { +#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 + SSL_CTX_set_options (ctx,SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION); +#else + SSL_CTX_set_min_proto_version(ctx, TLS1_VERSION); +# if defined(TLS_MAX_VERSION) + SSL_CTX_set_max_proto_version(ctx, TLS_MAX_VERSION); +# elif defined(TLS1_3_VERSION) + SSL_CTX_set_max_proto_version(ctx, TLS1_3_VERSION); +# elif defined(TLS1_2_VERSION) + SSL_CTX_set_max_proto_version(ctx, TLS1_2_VERSION); +# endif +#endif + } + + return ctx; +} + +/** + * Initialize OpenSSL server + * + * Starts the server context with TLS 1.2 and load the certificate. + * + * @return It returns the context on success or NULL otherwise + */ +static SSL_CTX * security_initialize_openssl_server() { + SSL_CTX *ctx; + char lerror[512]; + static int netdata_id_context = 1; + + //TO DO: Confirm the necessity to check return for other OPENSSL function +#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 + ctx = SSL_CTX_new(SSLv23_server_method()); + if (!ctx) { + error("Cannot create a new SSL context, netdata won't encrypt communication"); + return NULL; + } + + SSL_CTX_use_certificate_file(ctx, security_cert, SSL_FILETYPE_PEM); +#else + ctx = SSL_CTX_new(TLS_server_method()); + if (!ctx) { + error("Cannot create a new SSL context, netdata won't encrypt communication"); + return NULL; + } + + SSL_CTX_use_certificate_chain_file(ctx, security_cert); +#endif + security_openssl_common_options(ctx, 0); + + SSL_CTX_use_PrivateKey_file(ctx,security_key,SSL_FILETYPE_PEM); + + if (!SSL_CTX_check_private_key(ctx)) { + ERR_error_string_n(ERR_get_error(),lerror,sizeof(lerror)); + error("SSL cannot check the private key: %s",lerror); + SSL_CTX_free(ctx); + return NULL; + } + + SSL_CTX_set_session_id_context(ctx,(void*)&netdata_id_context,(unsigned int)sizeof(netdata_id_context)); + SSL_CTX_set_info_callback(ctx,security_info_callback); + +#if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_095) + SSL_CTX_set_verify_depth(ctx,1); +#endif + debug(D_WEB_CLIENT,"SSL GLOBAL CONTEXT STARTED\n"); + + return ctx; +} + +/** + * Start SSL + * + * Call the correct function to start the SSL context. + * + * @param selector informs the context that must be initialized, the following list has the valid values: + * NETDATA_SSL_CONTEXT_SERVER - the server context + * NETDATA_SSL_CONTEXT_STREAMING - Starts the streaming context. + * NETDATA_SSL_CONTEXT_EXPORTING - Starts the OpenTSDB contextv + */ +void security_start_ssl(int selector) { + switch (selector) { + case NETDATA_SSL_CONTEXT_SERVER: { + struct stat statbuf; + if (stat(security_key, &statbuf) || stat(security_cert, &statbuf)) { + info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n"); + return; + } + + netdata_srv_ctx = security_initialize_openssl_server(); + break; + } + case NETDATA_SSL_CONTEXT_STREAMING: { + netdata_client_ctx = security_initialize_openssl_client(); + //This is necessary for the stream, because it is working sometimes with nonblock socket. + //It returns the bitmask afte to change, there is not any description of errors in the documentation + SSL_CTX_set_mode(netdata_client_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE |SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER |SSL_MODE_AUTO_RETRY); + break; + } + case NETDATA_SSL_CONTEXT_EXPORTING: { + netdata_exporting_ctx = security_initialize_openssl_client(); + break; + } + } +} + +/** + * Clean Open SSL + * + * Clean all the allocated contexts from netdata. + */ +void security_clean_openssl() +{ + if (netdata_srv_ctx) { + SSL_CTX_free(netdata_srv_ctx); + } + + if (netdata_client_ctx) { + SSL_CTX_free(netdata_client_ctx); + } + + if (netdata_exporting_ctx) { + SSL_CTX_free(netdata_exporting_ctx); + } + +#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 + ERR_free_strings(); +#endif +} + +/** + * Process accept + * + * Process the SSL handshake with the client case it is necessary. + * + * @param ssl is a pointer for the SSL structure + * @param msg is a copy of the first 8 bytes of the initial message received + * + * @return it returns 0 case it performs the handshake, 8 case it is clean connection + * and another integer power of 2 otherwise. + */ +int security_process_accept(SSL *ssl,int msg) { + int sock = SSL_get_fd(ssl); + int test; + if (msg > 0x17) + { + return NETDATA_SSL_NO_HANDSHAKE; + } + + ERR_clear_error(); + if ((test = SSL_accept(ssl)) <= 0) { + int sslerrno = SSL_get_error(ssl, test); + switch(sslerrno) { + case SSL_ERROR_WANT_READ: + { + error("SSL handshake did not finish and it wanna read on socket %d!", sock); + return NETDATA_SSL_WANT_READ; + } + case SSL_ERROR_WANT_WRITE: + { + error("SSL handshake did not finish and it wanna read on socket %d!", sock); + return NETDATA_SSL_WANT_WRITE; + } + case SSL_ERROR_NONE: + case SSL_ERROR_SSL: + case SSL_ERROR_SYSCALL: + default: + { + u_long err; + char buf[256]; + int counter = 0; + while ((err = ERR_get_error()) != 0) { + ERR_error_string_n(err, buf, sizeof(buf)); + info("%d SSL Handshake error (%s) on socket %d ", counter++, ERR_error_string((long)SSL_get_error(ssl, test), NULL), sock); + } + return NETDATA_SSL_NO_HANDSHAKE; + } + } + } + + if (SSL_is_init_finished(ssl)) + { + debug(D_WEB_CLIENT_ACCESS,"SSL Handshake finished %s errno %d on socket fd %d", ERR_error_string((long)SSL_get_error(ssl, test), NULL), errno, sock); + } + + return NETDATA_SSL_HANDSHAKE_COMPLETE; +} + +/** + * Test Certificate + * + * Check the certificate of Netdata parent + * + * @param ssl is the connection structure + * + * @return It returns 0 on success and -1 otherwise + */ +int security_test_certificate(SSL *ssl) { + X509* cert = SSL_get_peer_certificate(ssl); + int ret; + long status; + if (!cert) { + return -1; + } + + status = SSL_get_verify_result(ssl); + if((X509_V_OK != status)) + { + char error[512]; + ERR_error_string_n(ERR_get_error(), error, sizeof(error)); + error("SSL RFC4158 check: We have a invalid certificate, the tests result with %ld and message %s", status, error); + ret = -1; + } else { + ret = 0; + } + + return ret; +} + +/** + * Location for context + * + * Case the user give us a directory with the certificates available and + * the Netdata parent certificate, we use this function to validate the certificate. + * + * @param ctx the context where the path will be set. + * @param file the file with Netdata parent certificate. + * @param path the directory where the certificates are stored. + * + * @return It returns 0 on success and -1 otherwise. + */ +int security_location_for_context(SSL_CTX *ctx, char *file, char *path) { + struct stat statbuf; + if (stat(file, &statbuf)) { + info("Netdata does not have the parent's SSL certificate, so it will use the default OpenSSL configuration to validate certificates!"); + return 0; + } + + ERR_clear_error(); + u_long err; + char buf[256]; + if(!SSL_CTX_load_verify_locations(ctx, file, path)) { + goto slfc; + } + + if(!SSL_CTX_set_default_verify_paths(ctx)) { + goto slfc; + } + + return 0; + +slfc: + while ((err = ERR_get_error()) != 0) { + ERR_error_string_n(err, buf, sizeof(buf)); + error("Cannot set the directory for the certificates and the parent SSL certificate: %s",buf); + } + return -1; +} + +#endif diff --git a/libnetdata/socket/security.h b/libnetdata/socket/security.h new file mode 100644 index 0000000..17ecc6d --- /dev/null +++ b/libnetdata/socket/security.h @@ -0,0 +1,55 @@ +#ifndef NETDATA_SECURITY_H +# define NETDATA_SECURITY_H + +# define NETDATA_SSL_HANDSHAKE_COMPLETE 0 //All the steps were successful +# define NETDATA_SSL_START 1 //Starting handshake, conn variable is NULL +# define NETDATA_SSL_WANT_READ 2 //The connection wanna read from socket +# define NETDATA_SSL_WANT_WRITE 4 //The connection wanna write on socket +# define NETDATA_SSL_NO_HANDSHAKE 8 //Continue without encrypt connection. +# define NETDATA_SSL_OPTIONAL 16 //Flag to define the HTTP request +# define NETDATA_SSL_FORCE 32 //We only accepts HTTPS request +# define NETDATA_SSL_INVALID_CERTIFICATE 64 //Accepts invalid certificate +# define NETDATA_SSL_VALID_CERTIFICATE 128 //Accepts invalid certificate +# define NETDATA_SSL_PROXY_HTTPS 256 //Proxy is using HTTPS + +#define NETDATA_SSL_CONTEXT_SERVER 0 +#define NETDATA_SSL_CONTEXT_STREAMING 1 +#define NETDATA_SSL_CONTEXT_EXPORTING 2 + +# ifdef ENABLE_HTTPS + +#define OPENSSL_VERSION_095 0x00905100L +#define OPENSSL_VERSION_097 0x0907000L +#define OPENSSL_VERSION_110 0x10100000L +#define OPENSSL_VERSION_111 0x10101000L + +# include <openssl/ssl.h> +# include <openssl/err.h> +# if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) && (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) +# include <openssl/conf.h> +# endif + +struct netdata_ssl{ + SSL *conn; //SSL connection + uint32_t flags; //The flags for SSL connection +}; + +extern SSL_CTX *netdata_exporting_ctx; +extern SSL_CTX *netdata_client_ctx; +extern SSL_CTX *netdata_srv_ctx; +extern const char *security_key; +extern const char *security_cert; +extern const char *tls_version; +extern const char *tls_ciphers; +extern int netdata_validate_server; +extern int security_location_for_context(SSL_CTX *ctx,char *file,char *path); + +void security_openssl_library(); +void security_clean_openssl(); +void security_start_ssl(int selector); +int security_process_accept(SSL *ssl,int msg); +int security_test_certificate(SSL *ssl); +SSL_CTX * security_initialize_openssl_client(); + +# endif //ENABLE_HTTPS +#endif //NETDATA_SECURITY_H diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c new file mode 100644 index 0000000..73eb8e6 --- /dev/null +++ b/libnetdata/socket/socket.c @@ -0,0 +1,1733 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +// -------------------------------------------------------------------------------------------------------------------- +// various library calls + +#ifdef __gnu_linux__ +#define LARGE_SOCK_SIZE 33554431 // don't ask why - I found it at brubeck source - I guess it is just a large number +#else +#define LARGE_SOCK_SIZE 4096 +#endif + +int sock_setnonblock(int fd) { + int flags; + + flags = fcntl(fd, F_GETFL); + flags |= O_NONBLOCK; + + int ret = fcntl(fd, F_SETFL, flags); + if(ret < 0) + error("Failed to set O_NONBLOCK on socket %d", fd); + + return ret; +} + +int sock_delnonblock(int fd) { + int flags; + + flags = fcntl(fd, F_GETFL); + flags &= ~O_NONBLOCK; + + int ret = fcntl(fd, F_SETFL, flags); + if(ret < 0) + error("Failed to remove O_NONBLOCK on socket %d", fd); + + return ret; +} + +int sock_setreuse(int fd, int reuse) { + int ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + + if(ret == -1) + error("Failed to set SO_REUSEADDR on socket %d", fd); + + return ret; +} + +int sock_setreuse_port(int fd, int reuse) { + int ret; + +#ifdef SO_REUSEPORT + ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &reuse, sizeof(reuse)); + if(ret == -1 && errno != ENOPROTOOPT) + error("failed to set SO_REUSEPORT on socket %d", fd); +#else + ret = -1; +#endif + + return ret; +} + +int sock_enlarge_in(int fd) { + int ret, bs = LARGE_SOCK_SIZE; + + ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs)); + + if(ret == -1) + error("Failed to set SO_RCVBUF on socket %d", fd); + + return ret; +} + +int sock_enlarge_out(int fd) { + int ret, bs = LARGE_SOCK_SIZE; + ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs)); + + if(ret == -1) + error("Failed to set SO_SNDBUF on socket %d", fd); + + return ret; +} + + +// -------------------------------------------------------------------------------------------------------------------- + +char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port) { + char buffer[100 + 1]; + + switch(family) { + case AF_INET: + snprintfz(buffer, 100, "%s:%s:%d", protocol, ip, port); + break; + + case AF_INET6: + default: + snprintfz(buffer, 100, "%s:[%s]:%d", protocol, ip, port); + break; + + case AF_UNIX: + snprintfz(buffer, 100, "%s:%s", protocol, ip); + break; + } + + return strdupz(buffer); +} + +// -------------------------------------------------------------------------------------------------------------------- +// listening sockets + +int create_listen_socket_unix(const char *path, int listen_backlog) { + int sock; + + debug(D_LISTENER, "LISTENER: UNIX creating new listening socket on path '%s'", path); + + sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock < 0) { + error("LISTENER: UNIX socket() on path '%s' failed.", path); + return -1; + } + + sock_setnonblock(sock); + sock_enlarge_in(sock); + + struct sockaddr_un name; + memset(&name, 0, sizeof(struct sockaddr_un)); + name.sun_family = AF_UNIX; + strncpy(name.sun_path, path, sizeof(name.sun_path)-1); + + errno = 0; + if (unlink(path) == -1 && errno != ENOENT) + error("LISTENER: failed to remove existing (probably obsolete or left-over) file on UNIX socket path '%s'.", path); + + if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { + close(sock); + error("LISTENER: UNIX bind() on path '%s' failed.", path); + return -1; + } + + // we have to chmod this to 0777 so that the client will be able + // to read from and write to this socket. + if(chmod(path, 0777) == -1) + error("LISTENER: failed to chmod() socket file '%s'.", path); + + if(listen(sock, listen_backlog) < 0) { + close(sock); + error("LISTENER: UNIX listen() on path '%s' failed.", path); + return -1; + } + + debug(D_LISTENER, "LISTENER: Listening on UNIX path '%s'", path); + return sock; +} + +int create_listen_socket4(int socktype, const char *ip, uint16_t port, int listen_backlog) { + int sock; + + debug(D_LISTENER, "LISTENER: IPv4 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); + + sock = socket(AF_INET, socktype, 0); + if(sock < 0) { + error("LISTENER: IPv4 socket() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + sock_setreuse(sock, 1); + sock_setreuse_port(sock, 1); + sock_setnonblock(sock); + sock_enlarge_in(sock); + + struct sockaddr_in name; + memset(&name, 0, sizeof(struct sockaddr_in)); + name.sin_family = AF_INET; + name.sin_port = htons (port); + + int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr); + if(ret != 1) { + error("LISTENER: Failed to convert IP '%s' to a valid IPv4 address.", ip); + close(sock); + return -1; + } + + if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { + close(sock); + error("LISTENER: IPv4 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + if(socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) { + close(sock); + error("LISTENER: IPv4 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + debug(D_LISTENER, "LISTENER: Listening on IPv4 ip '%s' port %d, socktype %d", ip, port, socktype); + return sock; +} + +int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int port, int listen_backlog) { + int sock; + int ipv6only = 1; + + debug(D_LISTENER, "LISTENER: IPv6 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); + + sock = socket(AF_INET6, socktype, 0); + if (sock < 0) { + error("LISTENER: IPv6 socket() on ip '%s' port %d, socktype %d, failed.", ip, port, socktype); + return -1; + } + + sock_setreuse(sock, 1); + sock_setreuse_port(sock, 1); + sock_setnonblock(sock); + sock_enlarge_in(sock); + + /* IPv6 only */ + if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0) + error("LISTENER: Cannot set IPV6_V6ONLY on ip '%s' port %d, socktype %d.", ip, port, socktype); + + struct sockaddr_in6 name; + memset(&name, 0, sizeof(struct sockaddr_in6)); + name.sin6_family = AF_INET6; + name.sin6_port = htons ((uint16_t) port); + name.sin6_scope_id = scope_id; + + int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr); + if(ret != 1) { + error("LISTENER: Failed to convert IP '%s' to a valid IPv6 address.", ip); + close(sock); + return -1; + } + + name.sin6_scope_id = scope_id; + + if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { + close(sock); + error("LISTENER: IPv6 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + if (socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) { + close(sock); + error("LISTENER: IPv6 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + return -1; + } + + debug(D_LISTENER, "LISTENER: Listening on IPv6 ip '%s' port %d, socktype %d", ip, port, socktype); + return sock; +} + +static inline int listen_sockets_add(LISTEN_SOCKETS *sockets, int fd, int family, int socktype, const char *protocol, const char *ip, uint16_t port, int acl_flags) { + if(sockets->opened >= MAX_LISTEN_FDS) { + error("LISTENER: Too many listening sockets. Failed to add listening %s socket at ip '%s' port %d, protocol %s, socktype %d", protocol, ip, port, protocol, socktype); + close(fd); + return -1; + } + + sockets->fds[sockets->opened] = fd; + sockets->fds_types[sockets->opened] = socktype; + sockets->fds_families[sockets->opened] = family; + sockets->fds_names[sockets->opened] = strdup_client_description(family, protocol, ip, port); + sockets->fds_acl_flags[sockets->opened] = acl_flags; + + sockets->opened++; + return 0; +} + +int listen_sockets_check_is_member(LISTEN_SOCKETS *sockets, int fd) { + size_t i; + for(i = 0; i < sockets->opened ;i++) + if(sockets->fds[i] == fd) return 1; + + return 0; +} + +static inline void listen_sockets_init(LISTEN_SOCKETS *sockets) { + size_t i; + for(i = 0; i < MAX_LISTEN_FDS ;i++) { + sockets->fds[i] = -1; + sockets->fds_names[i] = NULL; + sockets->fds_types[i] = -1; + } + + sockets->opened = 0; + sockets->failed = 0; +} + +void listen_sockets_close(LISTEN_SOCKETS *sockets) { + size_t i; + for(i = 0; i < sockets->opened ;i++) { + close(sockets->fds[i]); + sockets->fds[i] = -1; + + freez(sockets->fds_names[i]); + sockets->fds_names[i] = NULL; + + sockets->fds_types[i] = -1; + } + + sockets->opened = 0; + sockets->failed = 0; +} + +/* + * SSL ACL + * + * Search the SSL acl and apply it case it is set. + * + * @param acl is the acl given by the user. + */ +WEB_CLIENT_ACL socket_ssl_acl(char *acl) { + char *ssl = strchr(acl,'^'); + if(ssl) { + //Due the format of the SSL command it is always the last command, + //we finish it here to avoid problems with the ACLs + *ssl = '\0'; +#ifdef ENABLE_HTTPS + ssl++; + if (!strncmp("SSL=",ssl,4)) { + ssl += 4; + if (!strcmp(ssl,"optional")) { + return WEB_CLIENT_ACL_SSL_OPTIONAL; + } + else if (!strcmp(ssl,"force")) { + return WEB_CLIENT_ACL_SSL_FORCE; + } + } +#endif + } + + return WEB_CLIENT_ACL_NONE; +} + +WEB_CLIENT_ACL read_acl(char *st) { + WEB_CLIENT_ACL ret = socket_ssl_acl(st); + + if (!strcmp(st,"dashboard")) ret |= WEB_CLIENT_ACL_DASHBOARD; + if (!strcmp(st,"registry")) ret |= WEB_CLIENT_ACL_REGISTRY; + if (!strcmp(st,"badges")) ret |= WEB_CLIENT_ACL_BADGE; + if (!strcmp(st,"management")) ret |= WEB_CLIENT_ACL_MGMT; + if (!strcmp(st,"streaming")) ret |= WEB_CLIENT_ACL_STREAMING; + if (!strcmp(st,"netdata.conf")) ret |= WEB_CLIENT_ACL_NETDATACONF; + + return ret; +} + +static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, uint16_t default_port, int listen_backlog) { + int added = 0; + WEB_CLIENT_ACL acl_flags = WEB_CLIENT_ACL_NONE; + + struct addrinfo hints; + struct addrinfo *result = NULL, *rp = NULL; + + char buffer[strlen(definition) + 1]; + strcpy(buffer, definition); + + char buffer2[10 + 1]; + snprintfz(buffer2, 10, "%d", default_port); + + char *ip = buffer, *port = buffer2, *interface = "", *portconfig;; + + int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; + const char *protocol_str = "tcp"; + + if(strncmp(ip, "tcp:", 4) == 0) { + ip += 4; + protocol = IPPROTO_TCP; + socktype = SOCK_STREAM; + protocol_str = "tcp"; + } + else if(strncmp(ip, "udp:", 4) == 0) { + ip += 4; + protocol = IPPROTO_UDP; + socktype = SOCK_DGRAM; + protocol_str = "udp"; + } + else if(strncmp(ip, "unix:", 5) == 0) { + char *path = ip + 5; + socktype = SOCK_STREAM; + protocol_str = "unix"; + int fd = create_listen_socket_unix(path, listen_backlog); + if (fd == -1) { + error("LISTENER: Cannot create unix socket '%s'", path); + sockets->failed++; + } else { + acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT; + listen_sockets_add(sockets, fd, AF_UNIX, socktype, protocol_str, path, 0, acl_flags); + added++; + } + return added; + } + + char *e = ip; + if(*e == '[') { + e = ++ip; + while(*e && *e != ']') e++; + if(*e == ']') { + *e = '\0'; + e++; + } + } + else { + while(*e && *e != ':' && *e != '%' && *e != '=') e++; + } + + if(*e == '%') { + *e = '\0'; + e++; + interface = e; + while(*e && *e != ':' && *e != '=') e++; + } + + if(*e == ':') { + port = e + 1; + *e = '\0'; + e++; + while(*e && *e != '=') e++; + } + + if(*e == '=') { + *e='\0'; + e++; + portconfig = e; + while (*e != '\0') { + if (*e == '|') { + *e = '\0'; + acl_flags |= read_acl(portconfig); + e++; + portconfig = e; + continue; + } + e++; + } + acl_flags |= read_acl(portconfig); + } else { + acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT; + } + + //Case the user does not set the option SSL in the "bind to", but he has + //the certificates, I must redirect, so I am assuming here the default option + if(!(acl_flags & WEB_CLIENT_ACL_SSL_OPTIONAL) && !(acl_flags & WEB_CLIENT_ACL_SSL_FORCE)) { + acl_flags |= WEB_CLIENT_ACL_SSL_DEFAULT; + } + + uint32_t scope_id = 0; + if(*interface) { + scope_id = if_nametoindex(interface); + if(!scope_id) + error("LISTENER: Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + } + + if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all")) + ip = NULL; + + if(!*port) + port = buffer2; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = socktype; + hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ + hints.ai_protocol = protocol; + hints.ai_canonname = NULL; + hints.ai_addr = NULL; + hints.ai_next = NULL; + + int r = getaddrinfo(ip, port, &hints, &result); + if (r != 0) { + error("LISTENER: getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r)); + return -1; + } + + for (rp = result; rp != NULL; rp = rp->ai_next) { + int fd = -1; + int family; + + char rip[INET_ADDRSTRLEN + INET6_ADDRSTRLEN] = "INVALID"; + uint16_t rport = default_port; + + family = rp->ai_addr->sa_family; + switch (family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr; + inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN); + rport = ntohs(sin->sin_port); + // info("Attempting to listen on IPv4 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); + fd = create_listen_socket4(socktype, rip, rport, listen_backlog); + break; + } + + case AF_INET6: { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr; + inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN); + rport = ntohs(sin6->sin6_port); + // info("Attempting to listen on IPv6 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); + fd = create_listen_socket6(socktype, scope_id, rip, rport, listen_backlog); + break; + } + + default: + debug(D_LISTENER, "LISTENER: Unknown socket family %d", family); + break; + } + + if (fd == -1) { + error("LISTENER: Cannot bind to ip '%s', port %d", rip, rport); + sockets->failed++; + } + else { + listen_sockets_add(sockets, fd, family, socktype, protocol_str, rip, rport, acl_flags); + added++; + } + } + + freeaddrinfo(result); + + return added; +} + +int listen_sockets_setup(LISTEN_SOCKETS *sockets) { + listen_sockets_init(sockets); + + sockets->backlog = (int) appconfig_get_number(sockets->config, sockets->config_section, "listen backlog", sockets->backlog); + + long long int old_port = sockets->default_port; + long long int new_port = appconfig_get_number(sockets->config, sockets->config_section, "default port", sockets->default_port); + if(new_port < 1 || new_port > 65535) { + error("LISTENER: Invalid listen port %lld given. Defaulting to %lld.", new_port, old_port); + sockets->default_port = (uint16_t) appconfig_set_number(sockets->config, sockets->config_section, "default port", old_port); + } + else sockets->default_port = (uint16_t)new_port; + + debug(D_OPTIONS, "LISTENER: Default listen port set to %d.", sockets->default_port); + + char *s = appconfig_get(sockets->config, sockets->config_section, "bind to", sockets->default_bind_to); + while(*s) { + char *e = s; + + // skip separators, moving both s(tart) and e(nd) + while(isspace(*e) || *e == ',') s = ++e; + + // move e(nd) to the first separator + while(*e && !isspace(*e) && *e != ',') e++; + + // is there anything? + if(!*s || s == e) break; + + char buf[e - s + 1]; + strncpyz(buf, s, e - s); + bind_to_this(sockets, buf, sockets->default_port, sockets->backlog); + + s = e; + } + + if(sockets->failed) { + size_t i; + for(i = 0; i < sockets->opened ;i++) + info("LISTENER: Listen socket %s opened successfully.", sockets->fds_names[i]); + } + + return (int)sockets->opened; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// connect to another host/port + +// connect_to_this_unix() +// path the path of the unix socket +// timeout the timeout for establishing a connection + +static inline int connect_to_unix(const char *path, struct timeval *timeout) { + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if(fd == -1) { + error("Failed to create UNIX socket() for '%s'", path); + return -1; + } + + if(timeout) { + if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0) + error("Failed to set timeout on UNIX socket '%s'", path); + } + + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, path, sizeof(addr.sun_path)-1); + + if (connect(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { + error("Cannot connect to UNIX socket on path '%s'.", path); + close(fd); + return -1; + } + + debug(D_CONNECT_TO, "Connected to UNIX socket on path '%s'.", path); + + return fd; +} + +// connect_to_this_ip46() +// protocol IPPROTO_TCP, IPPROTO_UDP +// socktype SOCK_STREAM, SOCK_DGRAM +// host the destination hostname or IP address (IPv4 or IPv6) to connect to +// if it resolves to many IPs, all are tried (IPv4 and IPv6) +// scope_id the if_index id of the interface to use for connecting (0 = any) +// (used only under IPv6) +// service the service name or port to connect to +// timeout the timeout for establishing a connection + +int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout) { + struct addrinfo hints; + struct addrinfo *ai_head = NULL, *ai = NULL; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = socktype; + hints.ai_protocol = protocol; + + int ai_err = getaddrinfo(host, service, &hints, &ai_head); + if (ai_err != 0) { + error("Cannot resolve host '%s', port '%s': %s", host, service, gai_strerror(ai_err)); + return -1; + } + + int fd = -1; + for (ai = ai_head; ai != NULL && fd == -1; ai = ai->ai_next) { + + if (ai->ai_family == PF_INET6) { + struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr; + if(pSadrIn6->sin6_scope_id == 0) { + pSadrIn6->sin6_scope_id = scope_id; + } + } + + char hostBfr[NI_MAXHOST + 1]; + char servBfr[NI_MAXSERV + 1]; + + getnameinfo(ai->ai_addr, + ai->ai_addrlen, + hostBfr, + sizeof(hostBfr), + servBfr, + sizeof(servBfr), + NI_NUMERICHOST | NI_NUMERICSERV); + + debug(D_CONNECT_TO, "Address info: host = '%s', service = '%s', ai_flags = 0x%02X, ai_family = %d (PF_INET = %d, PF_INET6 = %d), ai_socktype = %d (SOCK_STREAM = %d, SOCK_DGRAM = %d), ai_protocol = %d (IPPROTO_TCP = %d, IPPROTO_UDP = %d), ai_addrlen = %lu (sockaddr_in = %lu, sockaddr_in6 = %lu)", + hostBfr, + servBfr, + (unsigned int)ai->ai_flags, + ai->ai_family, + PF_INET, + PF_INET6, + ai->ai_socktype, + SOCK_STREAM, + SOCK_DGRAM, + ai->ai_protocol, + IPPROTO_TCP, + IPPROTO_UDP, + (unsigned long)ai->ai_addrlen, + (unsigned long)sizeof(struct sockaddr_in), + (unsigned long)sizeof(struct sockaddr_in6)); + + switch (ai->ai_addr->sa_family) { + case PF_INET: { + struct sockaddr_in *pSadrIn = (struct sockaddr_in *)ai->ai_addr; + (void)pSadrIn; + + debug(D_CONNECT_TO, "ai_addr = sin_family: %d (AF_INET = %d, AF_INET6 = %d), sin_addr: '%s', sin_port: '%s'", + pSadrIn->sin_family, + AF_INET, + AF_INET6, + hostBfr, + servBfr); + break; + } + + case PF_INET6: { + struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr; + (void)pSadrIn6; + + debug(D_CONNECT_TO,"ai_addr = sin6_family: %d (AF_INET = %d, AF_INET6 = %d), sin6_addr: '%s', sin6_port: '%s', sin6_flowinfo: %u, sin6_scope_id: %u", + pSadrIn6->sin6_family, + AF_INET, + AF_INET6, + hostBfr, + servBfr, + pSadrIn6->sin6_flowinfo, + pSadrIn6->sin6_scope_id); + break; + } + + default: { + debug(D_CONNECT_TO, "Unknown protocol family %d.", ai->ai_family); + continue; + } + } + + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if(fd != -1) { + if(timeout) { + if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0) + error("Failed to set timeout on the socket to ip '%s' port '%s'", hostBfr, servBfr); + } + + errno = 0; + if(connect(fd, ai->ai_addr, ai->ai_addrlen) < 0) { + if(errno == EALREADY || errno == EINPROGRESS) { + info("Waiting for connection to ip %s port %s to be established", hostBfr, servBfr); + + fd_set fds; + FD_ZERO(&fds); + FD_SET(0, &fds); + int rc = select (1, NULL, &fds, NULL, timeout); + + if(rc > 0 && FD_ISSET(fd, &fds)) { + info("connect() to ip %s port %s completed successfully", hostBfr, servBfr); + } + else if(rc == -1) { + error("Failed to connect to '%s', port '%s'. select() returned %d", hostBfr, servBfr, rc); + close(fd); + fd = -1; + } + else { + error("Timed out while connecting to '%s', port '%s'. select() returned %d", hostBfr, servBfr, rc); + close(fd); + fd = -1; + } + } + else { + error("Failed to connect to '%s', port '%s'", hostBfr, servBfr); + close(fd); + fd = -1; + } + } + + if(fd != -1) + debug(D_CONNECT_TO, "Connected to '%s' on port '%s'.", hostBfr, servBfr); + } + } + + freeaddrinfo(ai_head); + + return fd; +} + +// connect_to_this() +// +// definition format: +// +// [PROTOCOL:]IP[%INTERFACE][:PORT] +// +// PROTOCOL = tcp or udp +// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6) +// INTERFACE = for IPv6 only, the network interface to use +// PORT = port number or service name + +int connect_to_this(const char *definition, int default_port, struct timeval *timeout) { + char buffer[strlen(definition) + 1]; + strcpy(buffer, definition); + + char default_service[10 + 1]; + snprintfz(default_service, 10, "%d", default_port); + + char *host = buffer, *service = default_service, *interface = ""; + int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; + uint32_t scope_id = 0; + + if(strncmp(host, "tcp:", 4) == 0) { + host += 4; + protocol = IPPROTO_TCP; + socktype = SOCK_STREAM; + } + else if(strncmp(host, "udp:", 4) == 0) { + host += 4; + protocol = IPPROTO_UDP; + socktype = SOCK_DGRAM; + } + else if(strncmp(host, "unix:", 5) == 0) { + char *path = host + 5; + return connect_to_unix(path, timeout); + } + + char *e = host; + if(*e == '[') { + e = ++host; + while(*e && *e != ']') e++; + if(*e == ']') { + *e = '\0'; + e++; + } + } + else { + while(*e && *e != ':' && *e != '%') e++; + } + + if(*e == '%') { + *e = '\0'; + e++; + interface = e; + while(*e && *e != ':') e++; + } + + if(*e == ':') { + *e = '\0'; + e++; + service = e; + } + + debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP); + + if(!*host) { + error("Definition '%s' does not specify a host.", definition); + return -1; + } + + if(*interface) { + scope_id = if_nametoindex(interface); + if(!scope_id) + error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + } + + if(!*service) + service = default_service; + + + return connect_to_this_ip46(protocol, socktype, host, scope_id, service, timeout); +} + +int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) { + int sock = -1; + + const char *s = destination; + while(*s) { + const char *e = s; + + // skip path, moving both s(tart) and e(nd) + if(*e == '/') + while(!isspace(*e) && *e != ',') s = ++e; + + // skip separators, moving both s(tart) and e(nd) + while(isspace(*e) || *e == ',') s = ++e; + + // move e(nd) to the first separator + while(*e && !isspace(*e) && *e != ',' && *e != '/') e++; + + // is there anything? + if(!*s || s == e) break; + + char buf[e - s + 1]; + strncpyz(buf, s, e - s); + if(reconnects_counter) *reconnects_counter += 1; + sock = connect_to_this(buf, default_port, timeout); + if(sock != -1) { + if(connected_to && connected_to_size) { + strncpy(connected_to, buf, connected_to_size); + connected_to[connected_to_size - 1] = '\0'; + } + break; + } + s = e; + } + + return sock; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// helpers to send/receive data in one call, in blocking mode, with a timeout + +#ifdef ENABLE_HTTPS +ssize_t recv_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { +#else +ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) { +#endif + + for(;;) { + struct pollfd fd = { + .fd = sockfd, + .events = POLLIN, + .revents = 0 + }; + + errno = 0; + int retval = poll(&fd, 1, timeout * 1000); + + if(retval == -1) { + // failed + + if(errno == EINTR || errno == EAGAIN) + continue; + + return -1; + } + + if(!retval) { + // timeout + return 0; + } + + if(fd.events & POLLIN) break; + } + +#ifdef ENABLE_HTTPS + if (ssl->conn) { + if (!ssl->flags) { + return SSL_read(ssl->conn,buf,len); + } + } +#endif + return recv(sockfd, buf, len, flags); +} + +#ifdef ENABLE_HTTPS +ssize_t send_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { +#else +ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) { +#endif + + for(;;) { + struct pollfd fd = { + .fd = sockfd, + .events = POLLOUT, + .revents = 0 + }; + + errno = 0; + int retval = poll(&fd, 1, timeout * 1000); + + if(retval == -1) { + // failed + + if(errno == EINTR || errno == EAGAIN) + continue; + + return -1; + } + + if(!retval) { + // timeout + return 0; + } + + if(fd.events & POLLOUT) break; + } + +#ifdef ENABLE_HTTPS + if(ssl->conn) { + if (!ssl->flags) { + return SSL_write(ssl->conn, buf, len); + } + } +#endif + return send(sockfd, buf, len, flags); +} + + +// -------------------------------------------------------------------------------------------------------------------- +// accept4() replacement for systems that do not have one + +#ifndef HAVE_ACCEPT4 +int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) { + int fd = accept(sock, addr, addrlen); + int newflags = 0; + + if (fd < 0) return fd; + + if (flags & SOCK_NONBLOCK) { + newflags |= O_NONBLOCK; + flags &= ~SOCK_NONBLOCK; + } + +#ifdef SOCK_CLOEXEC +#ifdef O_CLOEXEC + if (flags & SOCK_CLOEXEC) { + newflags |= O_CLOEXEC; + flags &= ~SOCK_CLOEXEC; + } +#endif +#endif + + if (flags) { + close(fd); + errno = EINVAL; + return -1; + } + + if (fcntl(fd, F_SETFL, newflags) < 0) { + int saved_errno = errno; + close(fd); + errno = saved_errno; + return -1; + } + + return fd; +} +#endif + +/* + * --------------------------------------------------------------------------------------------------------------------- + * connection_allowed() - if there is an access list then check the connection matches a pattern. + * Numeric patterns are checked against the IP address first, only if they + * do not match is the hostname resolved (reverse-DNS) and checked. If the + * hostname matches then we perform forward DNS resolution to check the IP + * is really associated with the DNS record. This call is repeatable: the + * web server may check more refined matches against the connection. Will + * update the client_host if uninitialized - ensure the hostsize is the number + * of *writable* bytes (i.e. be aware of the strdup used to compact the pollinfo). + */ +extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, + const char *patname, int allow_dns) +{ + debug(D_LISTENER,"checking %s... (allow_dns=%d)", patname, allow_dns); + if (!access_list) + return 1; + if (simple_pattern_matches(access_list, client_ip)) + return 1; + // If the hostname is unresolved (and needed) then attempt the DNS lookups. + //if (client_host[0]==0 && simple_pattern_is_potential_name(access_list)) + if (client_host[0]==0 && allow_dns) + { + struct sockaddr_storage sadr; + socklen_t addrlen = sizeof(sadr); + int err = getpeername(fd, (struct sockaddr*)&sadr, &addrlen); + if (err != 0 || + (err = getnameinfo((struct sockaddr *)&sadr, addrlen, client_host, (socklen_t)hostsize, + NULL, 0, NI_NAMEREQD)) != 0) { + error("Incoming %s on '%s' does not match a numeric pattern, and host could not be resolved (err=%s)", + patname, client_ip, gai_strerror(err)); + if (hostsize >= 8) + strcpy(client_host,"UNKNOWN"); + return 0; + } + struct addrinfo *addr_infos = NULL; + if (getaddrinfo(client_host, NULL, NULL, &addr_infos) !=0 ) { + error("LISTENER: cannot validate hostname '%s' from '%s' by resolving it", + client_host, client_ip); + if (hostsize >= 8) + strcpy(client_host,"UNKNOWN"); + return 0; + } + struct addrinfo *scan = addr_infos; + int validated = 0; + while (scan) { + char address[INET6_ADDRSTRLEN]; + address[0] = 0; + switch (scan->ai_addr->sa_family) { + case AF_INET: + inet_ntop(AF_INET, &((struct sockaddr_in*)(scan->ai_addr))->sin_addr, address, INET6_ADDRSTRLEN); + break; + case AF_INET6: + inet_ntop(AF_INET6, &((struct sockaddr_in6*)(scan->ai_addr))->sin6_addr, address, INET6_ADDRSTRLEN); + break; + } + debug(D_LISTENER, "Incoming ip %s rev-resolved onto %s, validating against forward-resolution %s", + client_ip, client_host, address); + if (!strcmp(client_ip, address)) { + validated = 1; + break; + } + scan = scan->ai_next; + } + if (!validated) { + error("LISTENER: Cannot validate '%s' as ip of '%s', not listed in DNS", client_ip, client_host); + if (hostsize >= 8) + strcpy(client_host,"UNKNOWN"); + } + if (addr_infos!=NULL) + freeaddrinfo(addr_infos); + } + if (!simple_pattern_matches(access_list, client_host)) { + debug(D_LISTENER, "Incoming connection on '%s' (%s) does not match allowed pattern for %s", + client_ip, client_host, patname); + return 0; + } + return 1; +} + +// -------------------------------------------------------------------------------------------------------------------- +// accept_socket() - accept a socket and store client IP and port +int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize, + char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns) { + struct sockaddr_storage sadr; + socklen_t addrlen = sizeof(sadr); + + int nfd = accept4(fd, (struct sockaddr *)&sadr, &addrlen, flags); + if (likely(nfd >= 0)) { + if (getnameinfo((struct sockaddr *)&sadr, addrlen, client_ip, (socklen_t)ipsize, + client_port, (socklen_t)portsize, NI_NUMERICHOST | NI_NUMERICSERV) != 0) { + error("LISTENER: cannot getnameinfo() on received client connection."); + strncpyz(client_ip, "UNKNOWN", ipsize - 1); + strncpyz(client_port, "UNKNOWN", portsize - 1); + } + if (!strcmp(client_ip, "127.0.0.1") || !strcmp(client_ip, "::1")) { + strncpy(client_ip, "localhost", ipsize); + client_ip[ipsize - 1] = '\0'; + } + +#ifdef __FreeBSD__ + if(((struct sockaddr *)&sadr)->sa_family == AF_LOCAL) + strncpyz(client_ip, "localhost", ipsize); +#endif + + client_ip[ipsize - 1] = '\0'; + client_port[portsize - 1] = '\0'; + + switch (((struct sockaddr *)&sadr)->sa_family) { + case AF_UNIX: + debug(D_LISTENER, "New UNIX domain web client from %s on socket %d.", client_ip, fd); + // set the port - certain versions of libc return garbage on unix sockets + strncpy(client_port, "UNIX", portsize); + client_port[portsize - 1] = '\0'; + break; + + case AF_INET: + debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); + break; + + case AF_INET6: + if (strncmp(client_ip, "::ffff:", 7) == 0) { + memmove(client_ip, &client_ip[7], strlen(&client_ip[7]) + 1); + debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); + } + else + debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd); + break; + + default: + debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd); + break; + } + if (!connection_allowed(nfd, client_ip, client_host, hostsize, access_list, "connection", allow_dns)) { + errno = 0; + error("Permission denied for client '%s', port '%s'", client_ip, client_port); + close(nfd); + nfd = -1; + errno = EPERM; + } + } +#ifdef HAVE_ACCEPT4 + else if (errno == ENOSYS) + error("netdata has been compiled with the assumption that the system has the accept4() call, but it is not here. Recompile netdata like this: ./configure --disable-accept4 ..."); +#endif + + return nfd; +} + + +// -------------------------------------------------------------------------------------------------------------------- +// poll() based listener +// this should be the fastest possible listener for up to 100 sockets +// above 100, an epoll() interface is needed on Linux + +#define POLL_FDS_INCREASE_STEP 10 + +inline POLLINFO *poll_add_fd(POLLJOB *p + , int fd + , int socktype + , WEB_CLIENT_ACL port_acl + , uint32_t flags + , const char *client_ip + , const char *client_port + , const char *client_host + , void *(*add_callback)(POLLINFO * /*pi*/, short int * /*events*/, void * /*data*/) + , void (*del_callback)(POLLINFO * /*pi*/) + , int (*rcv_callback)(POLLINFO * /*pi*/, short int * /*events*/) + , int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/) + , void *data +) { + debug(D_POLLFD, "POLLFD: ADD: request to add fd %d, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", fd, p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + if(unlikely(fd < 0)) return NULL; + + //if(p->limit && p->used >= p->limit) { + // info("Max sockets limit reached (%zu sockets), dropping connection", p->used); + // close(fd); + // return NULL; + //} + + if(unlikely(!p->first_free)) { + size_t new_slots = p->slots + POLL_FDS_INCREASE_STEP; + debug(D_POLLFD, "POLLFD: ADD: increasing size (current = %zu, new = %zu, used = %zu, min = %zu, max = %zu)", p->slots, new_slots, p->used, p->min, p->max); + + p->fds = reallocz(p->fds, sizeof(struct pollfd) * new_slots); + p->inf = reallocz(p->inf, sizeof(POLLINFO) * new_slots); + + // reset all the newly added slots + ssize_t i; + for(i = new_slots - 1; i >= (ssize_t)p->slots ; i--) { + debug(D_POLLFD, "POLLFD: ADD: resetting new slot %zd", i); + p->fds[i].fd = -1; + p->fds[i].events = 0; + p->fds[i].revents = 0; + + p->inf[i].p = p; + p->inf[i].slot = (size_t)i; + p->inf[i].flags = 0; + p->inf[i].socktype = -1; + p->inf[i].port_acl = -1; + + p->inf[i].client_ip = NULL; + p->inf[i].client_port = NULL; + p->inf[i].client_host = NULL; + p->inf[i].del_callback = p->del_callback; + p->inf[i].rcv_callback = p->rcv_callback; + p->inf[i].snd_callback = p->snd_callback; + p->inf[i].data = NULL; + + // link them so that the first free will be earlier in the array + // (we loop decrementing i) + p->inf[i].next = p->first_free; + p->first_free = &p->inf[i]; + } + + p->slots = new_slots; + } + + POLLINFO *pi = p->first_free; + p->first_free = p->first_free->next; + + debug(D_POLLFD, "POLLFD: ADD: selected slot %zu, next free is %zd", pi->slot, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + struct pollfd *pf = &p->fds[pi->slot]; + pf->fd = fd; + pf->events = POLLIN; + pf->revents = 0; + + pi->fd = fd; + pi->p = p; + pi->socktype = socktype; + pi->port_acl = port_acl; + pi->flags = flags; + pi->next = NULL; + pi->client_ip = strdupz(client_ip); + pi->client_port = strdupz(client_port); + pi->client_host = strdupz(client_host); + + pi->del_callback = del_callback; + pi->rcv_callback = rcv_callback; + pi->snd_callback = snd_callback; + + pi->connected_t = now_boottime_sec(); + pi->last_received_t = 0; + pi->last_sent_t = 0; + pi->last_sent_t = 0; + pi->recv_count = 0; + pi->send_count = 0; + + netdata_thread_disable_cancelability(); + p->used++; + if(unlikely(pi->slot > p->max)) + p->max = pi->slot; + + if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) { + pi->data = add_callback(pi, &pf->events, data); + } + + if(pi->flags & POLLINFO_FLAG_SERVER_SOCKET) { + p->min = pi->slot; + } + netdata_thread_enable_cancelability(); + + debug(D_POLLFD, "POLLFD: ADD: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + return pi; +} + +inline void poll_close_fd(POLLINFO *pi) { + POLLJOB *p = pi->p; + + struct pollfd *pf = &p->fds[pi->slot]; + debug(D_POLLFD, "POLLFD: DEL: request to clear slot %zu (fd %d), old next free was %zd", pi->slot, pf->fd, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + + if(unlikely(pf->fd == -1)) return; + + netdata_thread_disable_cancelability(); + + if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) { + pi->del_callback(pi); + + if(likely(!(pi->flags & POLLINFO_FLAG_DONT_CLOSE))) { + if(close(pf->fd) == -1) + error("Failed to close() poll_events() socket %d", pf->fd); + } + } + + pf->fd = -1; + pf->events = 0; + pf->revents = 0; + + pi->fd = -1; + pi->socktype = -1; + pi->flags = 0; + pi->data = NULL; + + pi->del_callback = NULL; + pi->rcv_callback = NULL; + pi->snd_callback = NULL; + + freez(pi->client_ip); + pi->client_ip = NULL; + + freez(pi->client_port); + pi->client_port = NULL; + + freez(pi->client_host); + pi->client_host = NULL; + + pi->next = p->first_free; + p->first_free = pi; + + p->used--; + if(unlikely(p->max == pi->slot)) { + p->max = p->min; + ssize_t i; + for(i = (ssize_t)pi->slot; i > (ssize_t)p->min ;i--) { + if (unlikely(p->fds[i].fd != -1)) { + p->max = (size_t)i; + break; + } + } + } + netdata_thread_enable_cancelability(); + + debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); +} + +void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data) { + (void)pi; + (void)events; + (void)data; + + // error("POLLFD: internal error: poll_default_add_callback() called"); + + return NULL; +} + +void poll_default_del_callback(POLLINFO *pi) { + if(pi->data) + error("POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak"); +} + +int poll_default_rcv_callback(POLLINFO *pi, short int *events) { + *events |= POLLIN; + + char buffer[1024 + 1]; + + ssize_t rc; + do { + rc = recv(pi->fd, buffer, 1024, MSG_DONTWAIT); + if (rc < 0) { + // read failed + if (errno != EWOULDBLOCK && errno != EAGAIN) { + error("POLLFD: poll_default_rcv_callback(): recv() failed with %zd.", rc); + return -1; + } + } else if (rc) { + // data received + info("POLLFD: internal error: poll_default_rcv_callback() is discarding %zd bytes received on socket %d", rc, pi->fd); + } + } while (rc != -1); + + return 0; +} + +int poll_default_snd_callback(POLLINFO *pi, short int *events) { + *events &= ~POLLOUT; + + info("POLLFD: internal error: poll_default_snd_callback(): nothing to send on socket %d", pi->fd); + return 0; +} + +void poll_default_tmr_callback(void *timer_data) { + (void)timer_data; +} + +static void poll_events_cleanup(void *data) { + POLLJOB *p = (POLLJOB *)data; + + size_t i; + for(i = 0 ; i <= p->max ; i++) { + POLLINFO *pi = &p->inf[i]; + poll_close_fd(pi); + } + + freez(p->fds); + freez(p->inf); +} + +static void poll_events_process(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, short int revents, time_t now) { + short int events = pf->events; + int fd = pf->fd; + pf->revents = 0; + size_t i = pi->slot; + + if(unlikely(fd == -1)) { + debug(D_POLLFD, "POLLFD: LISTENER: ignoring slot %zu, it does not have an fd", i); + return; + } + + debug(D_POLLFD, "POLLFD: LISTENER: processing events for slot %zu (events = %d, revents = %d)", i, events, revents); + + if(revents & POLLIN || revents & POLLPRI) { + // receiving data + + pi->last_received_t = now; + pi->recv_count++; + + if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) { + // read data from client TCP socket + debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", i, fd); + + pf->events = 0; + if (pi->rcv_callback(pi, &pf->events) == -1) { + poll_close_fd(&p->inf[i]); + return; + } + pf = &p->fds[i]; + pi = &p->inf[i]; + +#ifdef NETDATA_INTERNAL_CHECKS + // this is common - it is used for web server file copies + if(unlikely(!(pf->events & (POLLIN|POLLOUT)))) { + error("POLLFD: LISTENER: after reading, client slot %zu (fd %d) from %s port %s was left without expecting input or output. ", i, fd, pi->client_ip?pi->client_ip:"<undefined-ip>", pi->client_port?pi->client_port:"<undefined-port>"); + //poll_close_fd(pi); + //return; + } +#endif + } + else if(likely(pi->flags & POLLINFO_FLAG_SERVER_SOCKET)) { + // new connection + // debug(D_POLLFD, "POLLFD: LISTENER: accepting connections from slot %zu (fd %d)", i, fd); + + switch(pi->socktype) { + case SOCK_STREAM: { + // a TCP socket + // we accept the connection + + int nfd; + do { + char client_ip[INET6_ADDRSTRLEN]; + char client_port[NI_MAXSERV]; + char client_host[NI_MAXHOST]; + client_host[0] = 0; + client_ip[0] = 0; + client_port[0] = 0; + + debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", i, fd); + nfd = accept_socket(fd, SOCK_NONBLOCK, client_ip, INET6_ADDRSTRLEN, client_port, NI_MAXSERV, + client_host, NI_MAXHOST, p->access_list, p->allow_dns); + if (unlikely(nfd < 0)) { + // accept failed + + debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", i, fd); + + if(unlikely(errno == EMFILE)) { + error("POLLFD: LISTENER: too many open files - sleeping for 1ms - used by this thread %zu, max for this thread %zu", p->used, p->limit); + usleep(1000); // 10ms + } + else if(unlikely(errno != EWOULDBLOCK && errno != EAGAIN)) + error("POLLFD: LISTENER: accept() failed."); + + break; + } + else { + // accept ok + // info("POLLFD: LISTENER: client '[%s]:%s' connected to '%s' on fd %d", client_ip, client_port, sockets->fds_names[i], nfd); + poll_add_fd(p + , nfd + , SOCK_STREAM + , pi->port_acl + , POLLINFO_FLAG_CLIENT_SOCKET + , client_ip + , client_port + , client_host + , p->add_callback + , p->del_callback + , p->rcv_callback + , p->snd_callback + , NULL + ); + + // it may have reallocated them, so refresh our pointers + pf = &p->fds[i]; + pi = &p->inf[i]; + } + } while (nfd >= 0 && (!p->limit || p->used < p->limit)); + break; + } + + case SOCK_DGRAM: { + // a UDP socket + // we read data from the server socket + + debug(D_POLLFD, "POLLFD: LISTENER: reading data from UDP slot %zu (fd %d)", i, fd); + + // TODO: access_list is not applied to UDP + // but checking the access list on every UDP packet will destroy + // performance, especially for statsd. + + pf->events = 0; + pi->rcv_callback(pi, &pf->events); + break; + } + + default: { + error("POLLFD: LISTENER: Unknown socktype %d on slot %zu", pi->socktype, pi->slot); + break; + } + } + } + } + + if(unlikely(revents & POLLOUT)) { + // sending data + debug(D_POLLFD, "POLLFD: LISTENER: sending data to socket on slot %zu (fd %d)", i, fd); + + pi->last_sent_t = now; + pi->send_count++; + + pf->events = 0; + if (pi->snd_callback(pi, &pf->events) == -1) { + poll_close_fd(&p->inf[i]); + return; + } + pf = &p->fds[i]; + pi = &p->inf[i]; + +#ifdef NETDATA_INTERNAL_CHECKS + // this is common - it is used for streaming + if(unlikely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET && !(pf->events & (POLLIN|POLLOUT)))) { + error("POLLFD: LISTENER: after sending, client slot %zu (fd %d) from %s port %s was left without expecting input or output. ", i, fd, pi->client_ip?pi->client_ip:"<undefined-ip>", pi->client_port?pi->client_port:"<undefined-port>"); + //poll_close_fd(pi); + //return; + } +#endif + } + + if(unlikely(revents & POLLERR)) { + error("POLLFD: LISTENER: processing POLLERR events for slot %zu fd %d (events = %d, revents = %d)", i, events, revents, fd); + pf->events = 0; + poll_close_fd(pi); + return; + } + + if(unlikely(revents & POLLHUP)) { + error("POLLFD: LISTENER: processing POLLHUP events for slot %zu fd %d (events = %d, revents = %d)", i, events, revents, fd); + pf->events = 0; + poll_close_fd(pi); + return; + } + + if(unlikely(revents & POLLNVAL)) { + error("POLLFD: LISTENER: processing POLLNVAL events for slot %zu fd %d (events = %d, revents = %d)", i, events, revents, fd); + pf->events = 0; + poll_close_fd(pi); + return; + } +} + +void poll_events(LISTEN_SOCKETS *sockets + , void *(*add_callback)(POLLINFO * /*pi*/, short int * /*events*/, void * /*data*/) + , void (*del_callback)(POLLINFO * /*pi*/) + , int (*rcv_callback)(POLLINFO * /*pi*/, short int * /*events*/) + , int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/) + , void (*tmr_callback)(void * /*timer_data*/) + , SIMPLE_PATTERN *access_list + , int allow_dns + , void *data + , time_t tcp_request_timeout_seconds + , time_t tcp_idle_timeout_seconds + , time_t timer_milliseconds + , void *timer_data + , size_t max_tcp_sockets +) { + if(!sockets || !sockets->opened) { + error("POLLFD: internal error: no listening sockets are opened"); + return; + } + + if(timer_milliseconds <= 0) timer_milliseconds = 0; + + int retval; + + POLLJOB p = { + .slots = 0, + .used = 0, + .max = 0, + .limit = max_tcp_sockets, + .fds = NULL, + .inf = NULL, + .first_free = NULL, + + .complete_request_timeout = tcp_request_timeout_seconds, + .idle_timeout = tcp_idle_timeout_seconds, + .checks_every = (tcp_idle_timeout_seconds / 3) + 1, + + .access_list = access_list, + .allow_dns = allow_dns, + + .timer_milliseconds = timer_milliseconds, + .timer_data = timer_data, + + .add_callback = add_callback?add_callback:poll_default_add_callback, + .del_callback = del_callback?del_callback:poll_default_del_callback, + .rcv_callback = rcv_callback?rcv_callback:poll_default_rcv_callback, + .snd_callback = snd_callback?snd_callback:poll_default_snd_callback, + .tmr_callback = tmr_callback?tmr_callback:poll_default_tmr_callback + }; + + size_t i; + for(i = 0; i < sockets->opened ;i++) { + + POLLINFO *pi = poll_add_fd(&p + , sockets->fds[i] + , sockets->fds_types[i] + , sockets->fds_acl_flags[i] + , POLLINFO_FLAG_SERVER_SOCKET + , (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN" + , "" + , "" + , p.add_callback + , p.del_callback + , p.rcv_callback + , p.snd_callback + , NULL + ); + + pi->data = data; + info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN"); + } + + int listen_sockets_active = 1; + + int timeout_ms = 1000; // in milliseconds + time_t last_check = now_boottime_sec(); + + usec_t timer_usec = timer_milliseconds * USEC_PER_MS; + usec_t now_usec = 0, next_timer_usec = 0, last_timer_usec = 0; + (void)last_timer_usec; + + if(unlikely(timer_usec)) { + now_usec = now_boottime_usec(); + next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec; + } + + netdata_thread_cleanup_push(poll_events_cleanup, &p); + + while(!netdata_exit) { + if(unlikely(timer_usec)) { + now_usec = now_boottime_usec(); + + if(unlikely(timer_usec && now_usec >= next_timer_usec)) { + debug(D_POLLFD, "Calling timer callback after %zu usec", (size_t)(now_usec - last_timer_usec)); + last_timer_usec = now_usec; + p.tmr_callback(p.timer_data); + now_usec = now_boottime_usec(); + next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec; + } + + usec_t dt_usec = next_timer_usec - now_usec; + if(dt_usec < 1000 * USEC_PER_MS) + timeout_ms = 1000; + else + timeout_ms = (int)(dt_usec / USEC_PER_MS); + } + + // enable or disable the TCP listening sockets, based on the current number of sockets used and the limit set + if((listen_sockets_active && (p.limit && p.used >= p.limit)) || (!listen_sockets_active && (!p.limit || p.used < p.limit))) { + listen_sockets_active = !listen_sockets_active; + info("%s listening sockets (used TCP sockets %zu, max allowed for this worker %zu)", (listen_sockets_active)?"ENABLING":"DISABLING", p.used, p.limit); + for (i = 0; i <= p.max; i++) { + if(p.inf[i].flags & POLLINFO_FLAG_SERVER_SOCKET && p.inf[i].socktype == SOCK_STREAM) { + p.fds[i].events = (short int) ((listen_sockets_active) ? POLLIN : 0); + } + } + } + + debug(D_POLLFD, "POLLFD: LISTENER: Waiting on %zu sockets for %zu ms...", p.max + 1, (size_t)timeout_ms); + retval = poll(p.fds, p.max + 1, timeout_ms); + time_t now = now_boottime_sec(); + + if(unlikely(retval == -1)) { + error("POLLFD: LISTENER: poll() failed while waiting on %zu sockets.", p.max + 1); + break; + } + else if(unlikely(!retval)) { + debug(D_POLLFD, "POLLFD: LISTENER: poll() timeout."); + } + else { + for (i = 0; i <= p.max; i++) { + struct pollfd *pf = &p.fds[i]; + short int revents = pf->revents; + if (unlikely(revents)) + poll_events_process(&p, &p.inf[i], pf, revents, now); + } + } + + if(unlikely(p.checks_every > 0 && now - last_check > p.checks_every)) { + last_check = now; + + // security checks + for(i = 0; i <= p.max; i++) { + POLLINFO *pi = &p.inf[i]; + + if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) { + if (unlikely(pi->send_count == 0 && p.complete_request_timeout > 0 && (now - pi->connected_t) >= p.complete_request_timeout)) { + info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s has not sent a complete request in %zu seconds - closing it. " + , i + , pi->fd + , pi->client_ip ? pi->client_ip : "<undefined-ip>" + , pi->client_port ? pi->client_port : "<undefined-port>" + , (size_t) p.complete_request_timeout + ); + poll_close_fd(pi); + } + else if(unlikely(pi->recv_count && p.idle_timeout > 0 && now - ((pi->last_received_t > pi->last_sent_t) ? pi->last_received_t : pi->last_sent_t) >= p.idle_timeout )) { + info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s is idle for more than %zu seconds - closing it. " + , i + , pi->fd + , pi->client_ip ? pi->client_ip : "<undefined-ip>" + , pi->client_port ? pi->client_port : "<undefined-port>" + , (size_t) p.idle_timeout + ); + poll_close_fd(pi); + } + } + } + } + } + + netdata_thread_cleanup_pop(1); + debug(D_POLLFD, "POLLFD: LISTENER: cleanup completed"); +} diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h new file mode 100644 index 0000000..a40d801 --- /dev/null +++ b/libnetdata/socket/socket.h @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SOCKET_H +#define NETDATA_SOCKET_H + +#include "../libnetdata.h" + +#ifndef MAX_LISTEN_FDS +#define MAX_LISTEN_FDS 50 +#endif + +typedef enum web_client_acl { + WEB_CLIENT_ACL_NONE = 0, + WEB_CLIENT_ACL_NOCHECK = 0, + WEB_CLIENT_ACL_DASHBOARD = 1 << 0, + WEB_CLIENT_ACL_REGISTRY = 1 << 1, + WEB_CLIENT_ACL_BADGE = 1 << 2, + WEB_CLIENT_ACL_MGMT = 1 << 3, + WEB_CLIENT_ACL_STREAMING = 1 << 4, + WEB_CLIENT_ACL_NETDATACONF = 1 << 5, + WEB_CLIENT_ACL_SSL_OPTIONAL = 1 << 6, + WEB_CLIENT_ACL_SSL_FORCE = 1 << 7, + WEB_CLIENT_ACL_SSL_DEFAULT = 1 << 8 +} WEB_CLIENT_ACL; + +#define web_client_can_access_dashboard(w) ((w)->acl & WEB_CLIENT_ACL_DASHBOARD) +#define web_client_can_access_registry(w) ((w)->acl & WEB_CLIENT_ACL_REGISTRY) +#define web_client_can_access_badges(w) ((w)->acl & WEB_CLIENT_ACL_BADGE) +#define web_client_can_access_mgmt(w) ((w)->acl & WEB_CLIENT_ACL_MGMT) +#define web_client_can_access_stream(w) ((w)->acl & WEB_CLIENT_ACL_STREAMING) +#define web_client_can_access_netdataconf(w) ((w)->acl & WEB_CLIENT_ACL_NETDATACONF) +#define web_client_is_using_ssl_optional(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_OPTIONAL) +#define web_client_is_using_ssl_force(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_FORCE) +#define web_client_is_using_ssl_default(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_DEFAULT) + +typedef struct listen_sockets { + struct config *config; // the config file to use + const char *config_section; // the netdata configuration section to read settings from + const char *default_bind_to; // the default bind to configuration string + uint16_t default_port; // the default port to use + int backlog; // the default listen backlog to use + + size_t opened; // the number of sockets opened + size_t failed; // the number of sockets attempted to open, but failed + int fds[MAX_LISTEN_FDS]; // the open sockets + char *fds_names[MAX_LISTEN_FDS]; // descriptions for the open sockets + int fds_types[MAX_LISTEN_FDS]; // the socktype for the open sockets (SOCK_STREAM, SOCK_DGRAM) + int fds_families[MAX_LISTEN_FDS]; // the family of the open sockets (AF_UNIX, AF_INET, AF_INET6) + WEB_CLIENT_ACL fds_acl_flags[MAX_LISTEN_FDS]; // the acl to apply to the open sockets (dashboard, badges, streaming, netdata.conf, management) +} LISTEN_SOCKETS; + +extern char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port); + +extern int listen_sockets_setup(LISTEN_SOCKETS *sockets); +extern void listen_sockets_close(LISTEN_SOCKETS *sockets); + +extern int connect_to_this(const char *definition, int default_port, struct timeval *timeout); +extern int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size); +int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout); + +#ifdef ENABLE_HTTPS +extern ssize_t recv_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); +extern ssize_t send_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); +#else +extern ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); +extern ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); +#endif + +extern int sock_setnonblock(int fd); +extern int sock_delnonblock(int fd); +extern int sock_setreuse(int fd, int reuse); +extern int sock_setreuse_port(int fd, int reuse); +extern int sock_enlarge_in(int fd); +extern int sock_enlarge_out(int fd); + +extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, + SIMPLE_PATTERN *access_list, const char *patname, int allow_dns); +extern int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize, + char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns); + +#ifndef HAVE_ACCEPT4 +extern int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags); + +#ifndef SOCK_NONBLOCK +#define SOCK_NONBLOCK 00004000 +#endif /* #ifndef SOCK_NONBLOCK */ + +#ifndef SOCK_CLOEXEC +#define SOCK_CLOEXEC 02000000 +#endif /* #ifndef SOCK_CLOEXEC */ + +#endif /* #ifndef HAVE_ACCEPT4 */ + + +// ---------------------------------------------------------------------------- +// poll() based listener + +#define POLLINFO_FLAG_SERVER_SOCKET 0x00000001 +#define POLLINFO_FLAG_CLIENT_SOCKET 0x00000002 +#define POLLINFO_FLAG_DONT_CLOSE 0x00000004 + +typedef struct poll POLLJOB; + +typedef struct pollinfo { + POLLJOB *p; // the parent + size_t slot; // the slot id + + int fd; // the file descriptor + int socktype; // the client socket type + WEB_CLIENT_ACL port_acl; // the access lists permitted on this web server port (it's -1 for client sockets) + char *client_ip; // Max INET6_ADDRSTRLEN bytes + char *client_port; // Max NI_MAXSERV bytes + char *client_host; // Max NI_MAXHOST bytes + + time_t connected_t; // the time the socket connected + time_t last_received_t; // the time the socket last received data + time_t last_sent_t; // the time the socket last sent data + + size_t recv_count; // the number of times the socket was ready for inbound traffic + size_t send_count; // the number of times the socket was ready for outbound traffic + + uint32_t flags; // internal flags + + // callbacks for this socket + void (*del_callback)(struct pollinfo *pi); + int (*rcv_callback)(struct pollinfo *pi, short int *events); + int (*snd_callback)(struct pollinfo *pi, short int *events); + + // the user data + void *data; + + // linking of free pollinfo structures + // for quickly finding the next available + // this is like a stack, it grows and shrinks + // (with gaps - lower empty slots are preferred) + struct pollinfo *next; +} POLLINFO; + +struct poll { + size_t slots; + size_t used; + size_t min; + size_t max; + + size_t limit; + + time_t complete_request_timeout; + time_t idle_timeout; + time_t checks_every; + + time_t timer_milliseconds; + void *timer_data; + + struct pollfd *fds; + struct pollinfo *inf; + struct pollinfo *first_free; + + SIMPLE_PATTERN *access_list; + int allow_dns; + + void *(*add_callback)(POLLINFO *pi, short int *events, void *data); + void (*del_callback)(POLLINFO *pi); + int (*rcv_callback)(POLLINFO *pi, short int *events); + int (*snd_callback)(POLLINFO *pi, short int *events); + void (*tmr_callback)(void *timer_data); +}; + +#define pollinfo_from_slot(p, slot) (&((p)->inf[(slot)])) + +extern int poll_default_snd_callback(POLLINFO *pi, short int *events); +extern int poll_default_rcv_callback(POLLINFO *pi, short int *events); +extern void poll_default_del_callback(POLLINFO *pi); +extern void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data); + +extern POLLINFO *poll_add_fd(POLLJOB *p + , int fd + , int socktype + , WEB_CLIENT_ACL port_acl + , uint32_t flags + , const char *client_ip + , const char *client_port + , const char *client_host + , void *(*add_callback)(POLLINFO *pi, short int *events, void *data) + , void (*del_callback)(POLLINFO *pi) + , int (*rcv_callback)(POLLINFO *pi, short int *events) + , int (*snd_callback)(POLLINFO *pi, short int *events) + , void *data +); +extern void poll_close_fd(POLLINFO *pi); + +extern void poll_events(LISTEN_SOCKETS *sockets + , void *(*add_callback)(POLLINFO *pi, short int *events, void *data) + , void (*del_callback)(POLLINFO *pi) + , int (*rcv_callback)(POLLINFO *pi, short int *events) + , int (*snd_callback)(POLLINFO *pi, short int *events) + , void (*tmr_callback)(void *timer_data) + , SIMPLE_PATTERN *access_list + , int allow_dns + , void *data + , time_t tcp_request_timeout_seconds + , time_t tcp_idle_timeout_seconds + , time_t timer_milliseconds + , void *timer_data + , size_t max_tcp_sockets +); + +#endif //NETDATA_SOCKET_H diff --git a/libnetdata/statistical/Makefile.am b/libnetdata/statistical/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/statistical/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/statistical/README.md b/libnetdata/statistical/README.md new file mode 100644 index 0000000..ee1faba --- /dev/null +++ b/libnetdata/statistical/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/statistical/README.md +--> + +[](<>) diff --git a/libnetdata/statistical/statistical.c b/libnetdata/statistical/statistical.c new file mode 100644 index 0000000..ba8f0c4 --- /dev/null +++ b/libnetdata/statistical/statistical.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +LONG_DOUBLE default_single_exponential_smoothing_alpha = 0.1; + +void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg) { + const LONG_DOUBLE *value, *end = &series[entries]; + + fprintf(stderr, "%s of %zu entries [ ", msg, entries); + for(value = series; value < end ;value++) { + if(value != series) fprintf(stderr, ", "); + fprintf(stderr, "%" LONG_DOUBLE_MODIFIER, *value); + } + fprintf(stderr, " ] results in " CALCULATED_NUMBER_FORMAT "\n", result); +} + +// -------------------------------------------------------------------------------------------------------------------- + +inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count) { + const LONG_DOUBLE *value, *end = &series[entries]; + LONG_DOUBLE sum = 0; + size_t c = 0; + + for(value = series; value < end ; value++) { + if(calculated_number_isnumber(*value)) { + sum += *value; + c++; + } + } + + if(unlikely(!c)) sum = NAN; + if(likely(count)) *count = c; + + return sum; +} + +inline LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries) { + return sum_and_count(series, entries, NULL); +} + +inline LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries) { + size_t count = 0; + LONG_DOUBLE sum = sum_and_count(series, entries, &count); + + if(unlikely(!count)) return NAN; + return sum / (LONG_DOUBLE)count; +} + +// -------------------------------------------------------------------------------------------------------------------- + +LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period) { + if(unlikely(period <= 0)) + return 0.0; + + size_t i, count; + LONG_DOUBLE sum = 0, avg = 0; + LONG_DOUBLE p[period]; + + for(count = 0; count < period ; count++) + p[count] = 0.0; + + for(i = 0, count = 0; i < entries; i++) { + LONG_DOUBLE value = series[i]; + if(unlikely(!calculated_number_isnumber(value))) continue; + + if(unlikely(count < period)) { + sum += value; + avg = (count == period - 1) ? sum / (LONG_DOUBLE)period : 0; + } + else { + sum = sum - p[count % period] + value; + avg = sum / (LONG_DOUBLE)period; + } + + p[count % period] = value; + count++; + } + + return avg; +} + +// -------------------------------------------------------------------------------------------------------------------- + +static int qsort_compare(const void *a, const void *b) { + LONG_DOUBLE *p1 = (LONG_DOUBLE *)a, *p2 = (LONG_DOUBLE *)b; + LONG_DOUBLE n1 = *p1, n2 = *p2; + + if(unlikely(isnan(n1) || isnan(n2))) { + if(isnan(n1) && !isnan(n2)) return -1; + if(!isnan(n1) && isnan(n2)) return 1; + return 0; + } + if(unlikely(isinf(n1) || isinf(n2))) { + if(!isinf(n1) && isinf(n2)) return -1; + if(isinf(n1) && !isinf(n2)) return 1; + return 0; + } + + if(unlikely(n1 < n2)) return -1; + if(unlikely(n1 > n2)) return 1; + return 0; +} + +inline void sort_series(LONG_DOUBLE *series, size_t entries) { + qsort(series, entries, sizeof(LONG_DOUBLE), qsort_compare); +} + +inline LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries) { + LONG_DOUBLE *copy = mallocz(sizeof(LONG_DOUBLE) * entries); + memcpy(copy, series, sizeof(LONG_DOUBLE) * entries); + return copy; +} + +LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) { + if(unlikely(entries == 0)) return NAN; + if(unlikely(entries == 1)) return series[0]; + if(unlikely(entries == 2)) return (series[0] + series[1]) / 2; + + LONG_DOUBLE average; + if(entries % 2 == 0) { + size_t m = entries / 2; + average = (series[m] + series[m + 1]) / 2; + } + else { + average = series[entries / 2]; + } + + return average; +} + +LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) { + if(unlikely(entries == 0)) return NAN; + if(unlikely(entries == 1)) return series[0]; + + if(unlikely(entries == 2)) + return (series[0] + series[1]) / 2; + + LONG_DOUBLE *copy = copy_series(series, entries); + sort_series(copy, entries); + + LONG_DOUBLE avg = median_on_sorted_series(copy, entries); + + freez(copy); + return avg; +} + +// -------------------------------------------------------------------------------------------------------------------- + +LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period) { + if(entries <= period) + return median(series, entries); + + LONG_DOUBLE *data = copy_series(series, entries); + + size_t i; + for(i = period; i < entries; i++) { + data[i - period] = median(&series[i - period], period); + } + + LONG_DOUBLE avg = median(data, entries - period); + freez(data); + return avg; +} + +// -------------------------------------------------------------------------------------------------------------------- + +// http://stackoverflow.com/a/15150143/4525767 +LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) { + LONG_DOUBLE median = 0.0f; + LONG_DOUBLE average = 0.0f; + size_t i; + + for(i = 0; i < entries ; i++) { + LONG_DOUBLE value = series[i]; + if(unlikely(!calculated_number_isnumber(value))) continue; + + average += ( value - average ) * 0.1f; // rough running average. + median += copysignl( average * 0.01, value - median ); + } + + return median; +} + +// -------------------------------------------------------------------------------------------------------------------- + +LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) { + if(unlikely(entries == 0)) return NAN; + if(unlikely(entries == 1)) return series[0]; + + const LONG_DOUBLE *value, *end = &series[entries]; + size_t count; + LONG_DOUBLE sum; + + for(count = 0, sum = 0, value = series ; value < end ;value++) { + if(likely(calculated_number_isnumber(*value))) { + count++; + sum += *value; + } + } + + if(unlikely(count == 0)) return NAN; + if(unlikely(count == 1)) return sum; + + LONG_DOUBLE average = sum / (LONG_DOUBLE)count; + + for(count = 0, sum = 0, value = series ; value < end ;value++) { + if(calculated_number_isnumber(*value)) { + count++; + sum += powl(*value - average, 2); + } + } + + if(unlikely(count == 0)) return NAN; + if(unlikely(count == 1)) return average; + + LONG_DOUBLE variance = sum / (LONG_DOUBLE)(count); // remove -1 from count to have a population stddev + LONG_DOUBLE stddev = sqrtl(variance); + return stddev; +} + +// -------------------------------------------------------------------------------------------------------------------- + +LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) { + if(unlikely(entries == 0)) + return NAN; + + if(unlikely(isnan(alpha))) + alpha = default_single_exponential_smoothing_alpha; + + const LONG_DOUBLE *value = series, *end = &series[entries]; + LONG_DOUBLE level = (1.0 - alpha) * (*value); + + for(value++ ; value < end; value++) { + if(likely(calculated_number_isnumber(*value))) + level = alpha * (*value) + (1.0 - alpha) * level; + } + + return level; +} + +LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) { + if(unlikely(entries == 0)) + return NAN; + + if(unlikely(isnan(alpha))) + alpha = default_single_exponential_smoothing_alpha; + + const LONG_DOUBLE *value = &series[entries -1]; + LONG_DOUBLE level = (1.0 - alpha) * (*value); + + for(value++ ; value >= series; value--) { + if(likely(calculated_number_isnumber(*value))) + level = alpha * (*value) + (1.0 - alpha) * level; + } + + return level; +} + +// -------------------------------------------------------------------------------------------------------------------- + +// http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/ +LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast) { + if(unlikely(entries == 0)) + return NAN; + + LONG_DOUBLE level, trend; + + if(unlikely(isnan(alpha))) + alpha = 0.3; + + if(unlikely(isnan(beta))) + beta = 0.05; + + level = series[0]; + + if(likely(entries > 1)) + trend = series[1] - series[0]; + else + trend = 0; + + const LONG_DOUBLE *value = series; + for(value++ ; value >= series; value--) { + if(likely(calculated_number_isnumber(*value))) { + + LONG_DOUBLE last_level = level; + level = alpha * *value + (1.0 - alpha) * (level + trend); + trend = beta * (level - last_level) + (1.0 - beta) * trend; + + } + } + + if(forecast) + *forecast = level + trend; + + return level; +} + +// -------------------------------------------------------------------------------------------------------------------- + +/* + * Based on th R implementation + * + * a: level component + * b: trend component + * s: seasonal component + * + * Additive: + * + * Yhat[t+h] = a[t] + h * b[t] + s[t + 1 + (h - 1) mod p], + * a[t] = α (Y[t] - s[t-p]) + (1-α) (a[t-1] + b[t-1]) + * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1] + * s[t] = γ (Y[t] - a[t]) + (1-γ) s[t-p] + * + * Multiplicative: + * + * Yhat[t+h] = (a[t] + h * b[t]) * s[t + 1 + (h - 1) mod p], + * a[t] = α (Y[t] / s[t-p]) + (1-α) (a[t-1] + b[t-1]) + * b[t] = β (a[t] - a[t-1]) + (1-β) b[t-1] + * s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p] + */ +static int __HoltWinters( + const LONG_DOUBLE *series, + int entries, // start_time + h + + LONG_DOUBLE alpha, // alpha parameter of Holt-Winters Filter. + LONG_DOUBLE beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing. + LONG_DOUBLE gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted. + + const int *seasonal, + const int *period, + const LONG_DOUBLE *a, // Start value for level (a[0]). + const LONG_DOUBLE *b, // Start value for trend (b[0]). + LONG_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0]) + + /* return values */ + LONG_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing + LONG_DOUBLE *level, // Estimated values for the level component (size entries - t + 2) + LONG_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2) + LONG_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2) +) +{ + if(unlikely(entries < 4)) + return 0; + + int start_time = 2; + + LONG_DOUBLE res = 0, xhat = 0, stmp = 0; + int i, i0, s0; + + /* copy start values to the beginning of the vectors */ + level[0] = *a; + if(beta > 0) trend[0] = *b; + if(gamma > 0) memcpy(season, s, *period * sizeof(LONG_DOUBLE)); + + for(i = start_time - 1; i < entries; i++) { + /* indices for period i */ + i0 = i - start_time + 2; + s0 = i0 + *period - 1; + + /* forecast *for* period i */ + xhat = level[i0 - 1] + (beta > 0 ? trend[i0 - 1] : 0); + stmp = gamma > 0 ? season[s0 - *period] : (*seasonal != 1); + if (*seasonal == 1) + xhat += stmp; + else + xhat *= stmp; + + /* Sum of Squared Errors */ + res = series[i] - xhat; + *SSE += res * res; + + /* estimate of level *in* period i */ + if (*seasonal == 1) + level[i0] = alpha * (series[i] - stmp) + + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]); + else + level[i0] = alpha * (series[i] / stmp) + + (1 - alpha) * (level[i0 - 1] + trend[i0 - 1]); + + /* estimate of trend *in* period i */ + if (beta > 0) + trend[i0] = beta * (level[i0] - level[i0 - 1]) + + (1 - beta) * trend[i0 - 1]; + + /* estimate of seasonal component *in* period i */ + if (gamma > 0) { + if (*seasonal == 1) + season[s0] = gamma * (series[i] - level[i0]) + + (1 - gamma) * stmp; + else + season[s0] = gamma * (series[i] / level[i0]) + + (1 - gamma) * stmp; + } + } + + return 1; +} + +LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast) { + if(unlikely(isnan(alpha))) + alpha = 0.3; + + if(unlikely(isnan(beta))) + beta = 0.05; + + if(unlikely(isnan(gamma))) + gamma = 0; + + int seasonal = 0; + int period = 0; + LONG_DOUBLE a0 = series[0]; + LONG_DOUBLE b0 = 0; + LONG_DOUBLE s[] = {}; + + LONG_DOUBLE errors = 0.0; + size_t nb_computations = entries; + LONG_DOUBLE *estimated_level = callocz(nb_computations, sizeof(LONG_DOUBLE)); + LONG_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(LONG_DOUBLE)); + LONG_DOUBLE *estimated_season = callocz(nb_computations, sizeof(LONG_DOUBLE)); + + int ret = __HoltWinters( + series, + (int)entries, + alpha, + beta, + gamma, + &seasonal, + &period, + &a0, + &b0, + s, + &errors, + estimated_level, + estimated_trend, + estimated_season + ); + + LONG_DOUBLE value = estimated_level[nb_computations - 1]; + + if(forecast) + *forecast = 0.0; + + freez(estimated_level); + freez(estimated_trend); + freez(estimated_season); + + if(!ret) + return 0.0; + + return value; +} diff --git a/libnetdata/statistical/statistical.h b/libnetdata/statistical/statistical.h new file mode 100644 index 0000000..bbf241f --- /dev/null +++ b/libnetdata/statistical/statistical.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_STATISTICAL_H +#define NETDATA_STATISTICAL_H 1 + +#include "../libnetdata.h" + +extern void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg); + +extern LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries); +extern LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period); +extern LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries); +extern LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period); +extern LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries); +extern LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries); +extern LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha); +extern LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha); +extern LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast); +extern LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast); +extern LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count); +extern LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries); +extern LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries); +extern LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries); +extern void sort_series(LONG_DOUBLE *series, size_t entries); + +#endif //NETDATA_STATISTICAL_H diff --git a/libnetdata/storage_number/Makefile.am b/libnetdata/storage_number/Makefile.am new file mode 100644 index 0000000..c5f8450 --- /dev/null +++ b/libnetdata/storage_number/Makefile.am @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +SUBDIRS = \ + tests \ + $(NULL) + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/storage_number/README.md b/libnetdata/storage_number/README.md new file mode 100644 index 0000000..295b3d6 --- /dev/null +++ b/libnetdata/storage_number/README.md @@ -0,0 +1,17 @@ +<!-- +title: "Netdata storage number" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/storage_number/README.md +--> + +# Netdata storage number + +Although `netdata` does all its calculations using `long double`, it stores all values using +a **custom-made 32-bit number**. + +This custom-made number can store in 29 bits values from `-167772150000000.0` to `167772150000000.0` +with a precision of 0.00001 (yes, it's a floating point number, meaning that higher integer values +have less decimal precision) and 3 bits for flags. + +This provides an extremely optimized memory footprint with just 0.0001% max accuracy loss. + +[](<>) diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c new file mode 100644 index 0000000..8ef1353 --- /dev/null +++ b/libnetdata/storage_number/storage_number.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +storage_number pack_storage_number(calculated_number value, uint32_t flags) { + // bit 32 = sign 0:positive, 1:negative + // bit 31 = 0:divide, 1:multiply + // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) + // bit 27 SN_EXISTS_100 + // bit 26 SN_EXISTS_RESET + // bit 25 SN_EXISTS + // bit 24 to bit 1 = the value + + storage_number r = get_storage_number_flags(flags); + if(!value) return r; + + int m = 0; + calculated_number n = value, factor = 10; + + // if the value is negative + // add the sign bit and make it positive + if(n < 0) { + r += (1 << 31); // the sign bit 32 + n = -n; + } + + if(n / 10000000.0 > 0x00ffffff) { + factor = 100; + r |= SN_EXISTS_100; + } + + // make its integer part fit in 0x00ffffff + // by dividing it by 10 up to 7 times + // and increasing the multiplier + while(m < 7 && n > (calculated_number)0x00ffffff) { + n /= factor; + m++; + } + + if(m) { + // the value was too big and we divided it + // so we add a multiplier to unpack it + r += (1 << 30) + (m << 27); // the multiplier m + + if(n > (calculated_number)0x00ffffff) { + #ifdef NETDATA_INTERNAL_CHECKS + error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); + #endif + r += 0x00ffffff; + return r; + } + } + else { + // 0x0019999e is the number that can be multiplied + // by 10 to give 0x00ffffff + // while the value is below 0x0019999e we can + // multiply it by 10, up to 7 times, increasing + // the multiplier + while(m < 7 && n < (calculated_number)0x0019999e) { + n *= 10; + m++; + } + + if (unlikely(n > (calculated_number) (0x00ffffff))) { + n /= 10; + m--; + } + // the value was small enough and we multiplied it + // so we add a divider to unpack it + r += (0 << 30) + (m << 27); // the divider m + } + +#ifdef STORAGE_WITH_MATH + // without this there are rounding problems + // example: 0.9 becomes 0.89 + r += lrint((double) n); +#else + r += (storage_number)n; +#endif + + return r; +} + +calculated_number unpack_storage_number(storage_number value) { + if(!value) return 0; + + int sign = 0, exp = 0; + int factor = 10; + + // bit 32 = 0:positive, 1:negative + if(unlikely(value & (1 << 31))) + sign = 1; + + // bit 31 = 0:divide, 1:multiply + if(unlikely(value & (1 << 30))) + exp = 1; + + // bit 27 SN_EXISTS_100 + if(unlikely(value & (1 << 26))) + factor = 100; + + // bit 26 SN_EXISTS_RESET + // bit 25 SN_EXISTS + + // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) + int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27; + + // bit 24 to bit 1 = the value, so remove all other bits + value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24)); + + calculated_number n = value; + + // fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, factor = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, factor, n); + + if(exp) { + for(; mul; mul--) + n *= factor; + } + else { + for( ; mul ; mul--) + n /= 10; + } + + if(sign) n = -n; + return n; +} + +/* +int print_calculated_number(char *str, calculated_number value) +{ + char *wstr = str; + + int sign = (value < 0) ? 1 : 0; + if(sign) value = -value; + +#ifdef STORAGE_WITH_MATH + // without llrintl() there are rounding problems + // for example 0.9 becomes 0.89 + unsigned long long uvalue = (unsigned long long int) llrintl(value * (calculated_number)100000); +#else + unsigned long long uvalue = value * (calculated_number)100000; +#endif + + wstr = print_number_llu_r_smart(str, uvalue); + + // make sure we have 6 bytes at least + while((wstr - str) < 6) *wstr++ = '0'; + + // put the sign back + if(sign) *wstr++ = '-'; + + // reverse it + char *begin = str, *end = --wstr, aux; + while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux; + // wstr--; + // strreverse(str, wstr); + + // remove trailing zeros + int decimal = 5; + while(decimal > 0 && *wstr == '0') { + *wstr-- = '\0'; + decimal--; + } + + // terminate it, one position to the right + // to let space for a dot + wstr[2] = '\0'; + + // make space for the dot + int i; + for(i = 0; i < decimal ;i++) { + wstr[1] = wstr[0]; + wstr--; + } + + // put the dot + if(wstr[2] == '\0') { wstr[1] = '\0'; decimal--; } + else wstr[1] = '.'; + + // return the buffer length + return (int) ((wstr - str) + 2 + decimal ); +} +*/ + +int print_calculated_number(char *str, calculated_number value) { + // info("printing number " CALCULATED_NUMBER_FORMAT, value); + char integral_str[50], fractional_str[50]; + + char *wstr = str; + + if(unlikely(value < 0)) { + *wstr++ = '-'; + value = -value; + } + + calculated_number integral, fractional; + +#ifdef STORAGE_WITH_MATH + fractional = calculated_number_modf(value, &integral) * 10000000.0; +#else + fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL); +#endif + + unsigned long long integral_int = (unsigned long long)integral; + unsigned long long fractional_int = (unsigned long long)calculated_number_llrint(fractional); + if(unlikely(fractional_int >= 10000000)) { + integral_int += 1; + fractional_int -= 10000000; + } + + // info("integral " CALCULATED_NUMBER_FORMAT " (%llu), fractional " CALCULATED_NUMBER_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int); + + char *istre; + if(unlikely(integral_int == 0)) { + integral_str[0] = '0'; + istre = &integral_str[1]; + } + else + // convert the integral part to string (reversed) + istre = print_number_llu_r_smart(integral_str, integral_int); + + // copy reversed the integral string + istre--; + while( istre >= integral_str ) *wstr++ = *istre--; + + if(likely(fractional_int != 0)) { + // add a dot + *wstr++ = '.'; + + // convert the fractional part to string (reversed) + char *fstre = print_number_llu_r_smart(fractional_str, fractional_int); + + // prepend zeros to reach 7 digits length + int decimal = 7; + int len = (int)(fstre - fractional_str); + while(len < decimal) { + *wstr++ = '0'; + len++; + } + + char *begin = fractional_str; + while(begin < fstre && *begin == '0') begin++; + + // copy reversed the fractional string + fstre--; + while( fstre >= begin ) *wstr++ = *fstre--; + } + + *wstr = '\0'; + // info("printed number '%s'", str); + return (int)(wstr - str); +} diff --git a/libnetdata/storage_number/storage_number.h b/libnetdata/storage_number/storage_number.h new file mode 100644 index 0000000..28b7f26 --- /dev/null +++ b/libnetdata/storage_number/storage_number.h @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_STORAGE_NUMBER_H +#define NETDATA_STORAGE_NUMBER_H 1 + +#include "../libnetdata.h" + +#ifdef NETDATA_WITHOUT_LONG_DOUBLE + +#define powl pow +#define modfl modf +#define llrintl llrint +#define roundl round +#define sqrtl sqrt +#define copysignl copysign +#define strtold strtod + +typedef double calculated_number; +#define CALCULATED_NUMBER_FORMAT "%0.7f" +#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0f" +#define CALCULATED_NUMBER_FORMAT_AUTO "%f" + +#define LONG_DOUBLE_MODIFIER "f" +typedef double LONG_DOUBLE; + +#else // NETDATA_WITHOUT_LONG_DOUBLE + +typedef long double calculated_number; +#define CALCULATED_NUMBER_FORMAT "%0.7Lf" +#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0Lf" +#define CALCULATED_NUMBER_FORMAT_AUTO "%Lf" + +#define LONG_DOUBLE_MODIFIER "Lf" +typedef long double LONG_DOUBLE; + +#endif // NETDATA_WITHOUT_LONG_DOUBLE + +//typedef long long calculated_number; +//#define CALCULATED_NUMBER_FORMAT "%lld" + +typedef long long collected_number; +#define COLLECTED_NUMBER_FORMAT "%lld" + +/* +typedef long double collected_number; +#define COLLECTED_NUMBER_FORMAT "%0.7Lf" +*/ + +#define calculated_number_modf(x, y) modfl(x, y) +#define calculated_number_llrint(x) llrintl(x) +#define calculated_number_round(x) roundl(x) +#define calculated_number_fabs(x) fabsl(x) +#define calculated_number_pow(x, y) powl(x, y) +#define calculated_number_epsilon (calculated_number)0.0000001 + +#define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon) + +#define calculated_number_isnumber(a) (!(fpclassify(a) & (FP_NAN|FP_INFINITE))) + +typedef uint32_t storage_number; +#define STORAGE_NUMBER_FORMAT "%u" + +#define SN_EXISTS (1 << 24) // the value exists +#define SN_EXISTS_RESET (1 << 25) // the value has been overflown +#define SN_EXISTS_100 (1 << 26) // very large value (multipler is 100 instead of 10) + +// extract the flags +#define get_storage_number_flags(value) ((((storage_number)(value)) & (1 << 24)) | (((storage_number)(value)) & (1 << 25)) | (((storage_number)(value)) & (1 << 26))) +#define SN_EMPTY_SLOT 0x00000000 + +// checks +#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0) +#define did_storage_number_reset(value) ((get_storage_number_flags(value) == SN_EXISTS_RESET)?1:0) + +storage_number pack_storage_number(calculated_number value, uint32_t flags); +calculated_number unpack_storage_number(storage_number value); + +int print_calculated_number(char *str, calculated_number value); + +// sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE +#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff ) +#define STORAGE_NUMBER_POSITIVE_MIN_RAW (storage_number)( (0 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 ) +#define STORAGE_NUMBER_NEGATIVE_MAX_RAW (storage_number)( (1 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 ) +#define STORAGE_NUMBER_NEGATIVE_MIN_RAW (storage_number)( (1 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff ) + +// accepted accuracy loss +#define ACCURACY_LOSS_ACCEPTED_PERCENT 0.0001 +#define accuracy_loss(t1, t2) (((t1) == (t2) || (t1) == 0.0 || (t2) == 0.0) ? 0.0 : (100.0 - (((t1) > (t2)) ? ((t2) * 100.0 / (t1) ) : ((t1) * 100.0 / (t2))))) + +// Maximum acceptable rate of increase for counters. With a rate of 10% netdata can safely detect overflows with a +// period of at least every other 10 samples. +#define MAX_INCREMENTAL_PERCENT_RATE 10 + +#endif /* NETDATA_STORAGE_NUMBER_H */ diff --git a/libnetdata/storage_number/tests/Makefile.am b/libnetdata/storage_number/tests/Makefile.am new file mode 100644 index 0000000..babdcf0 --- /dev/null +++ b/libnetdata/storage_number/tests/Makefile.am @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in diff --git a/libnetdata/storage_number/tests/test_storage_number.c b/libnetdata/storage_number/tests/test_storage_number.c new file mode 100644 index 0000000..0e13208 --- /dev/null +++ b/libnetdata/storage_number/tests/test_storage_number.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../../libnetdata.h" +#include "../../required_dummies.h" +#include <setjmp.h> +#include <cmocka.h> + +static void test_number_pinting(void **state) +{ + (void)state; + + char value[50]; + + print_calculated_number(value, 0); + assert_string_equal(value, "0"); + + print_calculated_number(value, 0.0000001); + assert_string_equal(value, "0.0000001"); + + print_calculated_number(value, 0.00000009); + assert_string_equal(value, "0.0000001"); + + print_calculated_number(value, 0.000000001); + assert_string_equal(value, "0"); + + print_calculated_number(value, 99.99999999999999999); + assert_string_equal(value, "100"); + + print_calculated_number(value, -99.99999999999999999); + assert_string_equal(value, "-100"); + + print_calculated_number(value, 123.4567890123456789); + assert_string_equal(value, "123.456789"); + + print_calculated_number(value, 9999.9999999); + assert_string_equal(value, "9999.9999999"); + + print_calculated_number(value, -9999.9999999); + assert_string_equal(value, "-9999.9999999"); + + print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_EXISTS))); + assert_string_equal(value, "16.77722"); +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_number_pinting) + }; + + return cmocka_run_group_tests_name("storage_number", tests, NULL, NULL); +} diff --git a/libnetdata/string/utf8.h b/libnetdata/string/utf8.h new file mode 100644 index 0000000..133ec71 --- /dev/null +++ b/libnetdata/string/utf8.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_STRING_UTF8_H +#define NETDATA_STRING_UTF8_H 1 + +#define IS_UTF8_BYTE(x) (x & 0x80) +#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x)&&(x & 0x40)) + +#endif /* NETDATA_STRING_UTF8_H */ diff --git a/libnetdata/tests/Makefile.am b/libnetdata/tests/Makefile.am new file mode 100644 index 0000000..babdcf0 --- /dev/null +++ b/libnetdata/tests/Makefile.am @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in diff --git a/libnetdata/tests/test_str2ld.c b/libnetdata/tests/test_str2ld.c new file mode 100644 index 0000000..9d59f6c --- /dev/null +++ b/libnetdata/tests/test_str2ld.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" +#include "../required_dummies.h" +#include <setjmp.h> +#include <cmocka.h> + +static void test_str2ld(void **state) +{ + (void)state; + char *values[] = { + "1.2345678", + "-35.6", + "0.00123", + "23842384234234.2", + ".1", + "1.2e-10", + "hello", + "1wrong", + "nan", + "inf", + NULL + }; + + for (int i = 0; values[i]; i++) { + char *e_mine = "hello", *e_sys = "world"; + LONG_DOUBLE mine = str2ld(values[i], &e_mine); + LONG_DOUBLE sys = strtold(values[i], &e_sys); + + if (isnan(mine)) + assert_true(isnan(sys)); + else if (isinf(mine)) + assert_true(isinf(sys)); + else if (mine != sys) + assert_false(abs(mine - sys) > 0.000001); + + assert_ptr_equal(e_mine, e_sys); + } +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_str2ld) + }; + + return cmocka_run_group_tests_name("str2ld", tests, NULL, NULL); +} diff --git a/libnetdata/threads/Makefile.am b/libnetdata/threads/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/threads/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/threads/README.md b/libnetdata/threads/README.md new file mode 100644 index 0000000..377da5e --- /dev/null +++ b/libnetdata/threads/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/threads/README.md +--> + +[](<>) diff --git a/libnetdata/threads/threads.c b/libnetdata/threads/threads.c new file mode 100644 index 0000000..d763694 --- /dev/null +++ b/libnetdata/threads/threads.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +static size_t default_stacksize = 0, wanted_stacksize = 0; +static pthread_attr_t *attr = NULL; + +// ---------------------------------------------------------------------------- +// per thread data + +typedef struct { + void *arg; + pthread_t *thread; + const char *tag; + void *(*start_routine) (void *); + NETDATA_THREAD_OPTIONS options; +} NETDATA_THREAD; + +static __thread NETDATA_THREAD *netdata_thread = NULL; + +inline int netdata_thread_tag_exists(void) { + return (netdata_thread && netdata_thread->tag && *netdata_thread->tag); +} + +const char *netdata_thread_tag(void) { + return (netdata_thread_tag_exists() ? netdata_thread->tag : "MAIN"); +} + +// ---------------------------------------------------------------------------- +// compatibility library functions + +pid_t gettid(void) { +#ifdef __FreeBSD__ + + return (pid_t)pthread_getthreadid_np(); + +#elif defined(__APPLE__) + + #if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060) + uint64_t curthreadid; + pthread_threadid_np(NULL, &curthreadid); + return (pid_t)curthreadid; + #else /* __MAC_OS_X_VERSION_MIN_REQUIRED */ + return (pid_t)pthread_self; + #endif /* __MAC_OS_X_VERSION_MIN_REQUIRED */ + +#else /* __APPLE__*/ + + return (pid_t)syscall(SYS_gettid); + +#endif /* __FreeBSD__, __APPLE__*/ +} + +// ---------------------------------------------------------------------------- +// early initialization + +size_t netdata_threads_init(void) { + int i; + + // -------------------------------------------------------------------- + // get the required stack size of the threads of netdata + + attr = callocz(1, sizeof(pthread_attr_t)); + i = pthread_attr_init(attr); + if(i != 0) + fatal("pthread_attr_init() failed with code %d.", i); + + i = pthread_attr_getstacksize(attr, &default_stacksize); + if(i != 0) + fatal("pthread_attr_getstacksize() failed with code %d.", i); + else + debug(D_OPTIONS, "initial pthread stack size is %zu bytes", default_stacksize); + + return default_stacksize; +} + +// ---------------------------------------------------------------------------- +// late initialization + +void netdata_threads_init_after_fork(size_t stacksize) { + wanted_stacksize = stacksize; + int i; + + // ------------------------------------------------------------------------ + // set default pthread stack size + + if(attr && default_stacksize < wanted_stacksize && wanted_stacksize > 0) { + i = pthread_attr_setstacksize(attr, wanted_stacksize); + if(i != 0) + fatal("pthread_attr_setstacksize() to %zu bytes, failed with code %d.", wanted_stacksize, i); + else + debug(D_SYSTEM, "Successfully set pthread stacksize to %zu bytes", wanted_stacksize); + } +} + + +// ---------------------------------------------------------------------------- +// netdata_thread_create + +static void thread_cleanup(void *ptr) { + if(netdata_thread != ptr) { + NETDATA_THREAD *info = (NETDATA_THREAD *)ptr; + error("THREADS: internal error - thread local variable does not match the one passed to this function. Expected thread '%s', passed thread '%s'", netdata_thread->tag, info->tag); + } + + if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP)) + info("thread with task id %d finished", gettid()); + + freez((void *)netdata_thread->tag); + netdata_thread->tag = NULL; + + freez(netdata_thread); + netdata_thread = NULL; +} + +static void thread_set_name_np(NETDATA_THREAD *nt) { + + if (nt->tag) { + int ret = 0; + + char threadname[NETDATA_THREAD_NAME_MAX+1]; + strncpyz(threadname, nt->tag, NETDATA_THREAD_NAME_MAX); + +#if defined(__FreeBSD__) + pthread_set_name_np(pthread_self(), threadname); +#elif defined(__APPLE__) + ret = pthread_setname_np(threadname); +#else + ret = pthread_setname_np(pthread_self(), threadname); +#endif + + if (ret != 0) + error("cannot set pthread name of %d to %s. ErrCode: %d", gettid(), threadname, ret); + else + info("set name of thread %d to %s", gettid(), threadname); + + } +} + +void uv_thread_set_name_np(uv_thread_t ut, const char* name) { + int ret = 0; + + char threadname[NETDATA_THREAD_NAME_MAX+1]; + strncpyz(threadname, name, NETDATA_THREAD_NAME_MAX); + +#if defined(__FreeBSD__) + pthread_set_name_np(ut, threadname); +#elif defined(__APPLE__) + // Apple can only set its own name +#else + ret = pthread_setname_np(ut, threadname); +#endif + + if (ret) + info("cannot set libuv thread name to %s. Err: %d", threadname, ret); +} + +void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]) +{ + threadname[0] = '\0'; +#if defined(__FreeBSD__) + pthread_get_name_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1); +#elif defined(HAVE_PTHREAD_GETNAME_NP) /* Linux & macOS */ + (void)pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1); +#endif +} + +static void *thread_start(void *ptr) { + netdata_thread = (NETDATA_THREAD *)ptr; + + if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_STARTUP)) + info("thread created with task id %d", gettid()); + + if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) + error("cannot set pthread cancel type to DEFERRED."); + + if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) + error("cannot set pthread cancel state to ENABLE."); + + thread_set_name_np(ptr); + + void *ret = NULL; + pthread_cleanup_push(thread_cleanup, ptr); + ret = netdata_thread->start_routine(netdata_thread->arg); + pthread_cleanup_pop(1); + + return ret; +} + +int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg) { + NETDATA_THREAD *info = mallocz(sizeof(NETDATA_THREAD)); + info->arg = arg; + info->thread = thread; + info->tag = strdupz(tag); + info->start_routine = start_routine; + info->options = options; + + int ret = pthread_create(thread, attr, thread_start, info); + if(ret != 0) + error("failed to create new thread for %s. pthread_create() failed with code %d", tag, ret); + + else { + if (!(options & NETDATA_THREAD_OPTION_JOINABLE)) { + int ret2 = pthread_detach(*thread); + if (ret2 != 0) + error("cannot request detach of newly created %s thread. pthread_detach() failed with code %d", tag, ret2); + } + } + + return ret; +} + +// ---------------------------------------------------------------------------- +// netdata_thread_cancel + +int netdata_thread_cancel(netdata_thread_t thread) { + int ret = pthread_cancel(thread); + if(ret != 0) + error("cannot cancel thread. pthread_cancel() failed with code %d.", ret); + + return ret; +} + +// ---------------------------------------------------------------------------- +// netdata_thread_join + +int netdata_thread_join(netdata_thread_t thread, void **retval) { + int ret = pthread_join(thread, retval); + if(ret != 0) + error("cannot join thread. pthread_join() failed with code %d.", ret); + + return ret; +} + +int netdata_thread_detach(pthread_t thread) { + int ret = pthread_detach(thread); + if(ret != 0) + error("cannot detach thread. pthread_detach() failed with code %d.", ret); + + return ret; +} diff --git a/libnetdata/threads/threads.h b/libnetdata/threads/threads.h new file mode 100644 index 0000000..e7d79d3 --- /dev/null +++ b/libnetdata/threads/threads.h @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_THREADS_H +#define NETDATA_THREADS_H 1 + +#include "../libnetdata.h" + +extern pid_t gettid(void); + +typedef enum { + NETDATA_THREAD_OPTION_DEFAULT = 0 << 0, + NETDATA_THREAD_OPTION_JOINABLE = 1 << 0, + NETDATA_THREAD_OPTION_DONT_LOG_STARTUP = 1 << 1, + NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP = 1 << 2, + NETDATA_THREAD_OPTION_DONT_LOG = NETDATA_THREAD_OPTION_DONT_LOG_STARTUP|NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP, +} NETDATA_THREAD_OPTIONS; + +#define netdata_thread_cleanup_push(func, arg) pthread_cleanup_push(func, arg) +#define netdata_thread_cleanup_pop(execute) pthread_cleanup_pop(execute) + +typedef pthread_t netdata_thread_t; + +#define NETDATA_THREAD_TAG_MAX 100 +extern const char *netdata_thread_tag(void); +extern int netdata_thread_tag_exists(void); + +extern size_t netdata_threads_init(void); +extern void netdata_threads_init_after_fork(size_t stacksize); + +extern int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg); +extern int netdata_thread_cancel(netdata_thread_t thread); +extern int netdata_thread_join(netdata_thread_t thread, void **retval); +extern int netdata_thread_detach(pthread_t thread); + +#define NETDATA_THREAD_NAME_MAX 15 +extern void uv_thread_set_name_np(uv_thread_t ut, const char* name); +extern void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]); + +#define netdata_thread_self pthread_self +#define netdata_thread_testcancel pthread_testcancel + +#endif //NETDATA_THREADS_H diff --git a/libnetdata/url/Makefile.am b/libnetdata/url/Makefile.am new file mode 100644 index 0000000..161784b --- /dev/null +++ b/libnetdata/url/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/url/README.md b/libnetdata/url/README.md new file mode 100644 index 0000000..4a95547 --- /dev/null +++ b/libnetdata/url/README.md @@ -0,0 +1,5 @@ +<!-- +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/url/README.md +--> + +[](<>) diff --git a/libnetdata/url/url.c b/libnetdata/url/url.c new file mode 100644 index 0000000..3de94fd --- /dev/null +++ b/libnetdata/url/url.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// URL encode / decode +// code from: http://www.geekhideout.com/urlcode.shtml + +/* Converts a hex character to its integer value */ +char from_hex(char ch) { + return (char)(isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10); +} + +/* Converts an integer value to its hex character*/ +char to_hex(char code) { + static char hex[] = "0123456789abcdef"; + return hex[code & 15]; +} + +/* Returns a url-encoded version of str */ +/* IMPORTANT: be sure to free() the returned string after use */ +char *url_encode(char *str) { + char *buf, *pbuf; + + pbuf = buf = mallocz(strlen(str) * 3 + 1); + + while (*str) { + if (isalnum(*str) || *str == '-' || *str == '_' || *str == '.' || *str == '~') + *pbuf++ = *str; + + else if (*str == ' ') + *pbuf++ = '+'; + + else{ + *pbuf++ = '%'; + *pbuf++ = to_hex(*str >> 4); + *pbuf++ = to_hex(*str & 15); + } + + str++; + } + *pbuf = '\0'; + + pbuf = strdupz(buf); + freez(buf); + return pbuf; +} + +/** + * Percentage escape decode + * + * Decode %XX character or return 0 if cannot + * + * @param s the string to decode + * + * @return The character decoded on success and 0 otherwise + */ +char url_percent_escape_decode(char *s) { + if(likely(s[1] && s[2])) + return from_hex(s[1]) << 4 | from_hex(s[2]); + return 0; +} + +/** + * Get byte length + * + * This (utf8 string related) should be moved in separate file in future + * + * @param c is the utf8 character + * * + * @return It reurns the length of the specific character. + */ +char url_utf8_get_byte_length(char c) { + if(!IS_UTF8_BYTE(c)) + return 1; + + char length = 0; + while(likely(c & 0x80)) { + length++; + c <<= 1; + } + //4 byte is max size for UTF-8 char + //10XX XXXX is not valid character -> check length == 1 + if(length > 4 || length == 1) + return -1; + + return length; +} + +/** + * Decode Multibyte UTF8 + * + * Decode % encoded UTF-8 characters and copy them to *d + * + * @param s first address + * @param d + * @param d_end last address + * + * @return count of bytes written to *d + */ +char url_decode_multibyte_utf8(char *s, char *d, char *d_end) { + char first_byte = url_percent_escape_decode(s); + + if(unlikely(!first_byte || !IS_UTF8_STARTBYTE(first_byte))) + return 0; + + char byte_length = url_utf8_get_byte_length(first_byte); + + if(unlikely(byte_length <= 0 || d+byte_length >= d_end)) + return 0; + + char to_read = byte_length; + while(to_read > 0) { + char c = url_percent_escape_decode(s); + + if(unlikely( !IS_UTF8_BYTE(c) )) + return 0; + if((to_read != byte_length) && IS_UTF8_STARTBYTE(c)) + return 0; + + *d++ = c; + s+=3; + to_read--; + } + + return byte_length; +} + +/* + * The utf8_check() function scans the '\0'-terminated string starting + * at s. It returns a pointer to the first byte of the first malformed + * or overlong UTF-8 sequence found, or NULL if the string contains + * only correct UTF-8. It also spots UTF-8 sequences that could cause + * trouble if converted to UTF-16, namely surrogate characters + * (U+D800..U+DFFF) and non-Unicode positions (U+FFFE..U+FFFF). This + * routine is very likely to find a malformed sequence if the input + * uses any other encoding than UTF-8. It therefore can be used as a + * very effective heuristic for distinguishing between UTF-8 and other + * encodings. + * + * Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2005-03-30 + * License: http://www.cl.cam.ac.uk/~mgk25/short-license.html + */ +unsigned char *utf8_check(unsigned char *s) +{ + while (*s) + { + if (*s < 0x80) + /* 0xxxxxxx */ + s++; + else if ((s[0] & 0xe0) == 0xc0) + { + /* 110XXXXx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[0] & 0xfe) == 0xc0) /* overlong? */ + return s; + else + s += 2; + } + else if ((s[0] & 0xf0) == 0xe0) + { + /* 1110XXXX 10Xxxxxx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */ + (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */ + (s[0] == 0xef && s[1] == 0xbf && + (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */ + return s; + else + s += 3; + } + else if ((s[0] & 0xf8) == 0xf0) + { + /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */ + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[3] & 0xc0) != 0x80 || + (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */ + (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */ + return s; + else + s += 4; + } + else + return s; + } + + return NULL; +} + +char *url_decode_r(char *to, char *url, size_t size) { + char *s = url, // source + *d = to, // destination + *e = &to[size - 1]; // destination end + + while(*s && d < e) { + if(unlikely(*s == '%')) { + char t = url_percent_escape_decode(s); + if(IS_UTF8_BYTE(t)) { + char bytes_written = url_decode_multibyte_utf8(s, d, e); + if(likely(bytes_written)){ + d += bytes_written; + s += (bytes_written * 3)-1; + } + else { + goto fail_cleanup; + } + } + else if(likely(t) && isprint(t)) { + // avoid HTTP header injection + *d++ = t; + s += 2; + } + else + goto fail_cleanup; + } + else if(unlikely(*s == '+')) + *d++ = ' '; + + else + *d++ = *s; + + s++; + } + + *d = '\0'; + + if(unlikely( utf8_check((unsigned char *)to) )) //NULL means sucess here + return NULL; + + return to; + +fail_cleanup: + *d = '\0'; + return NULL; +} + +/** + * Is request complete? + * + * Check whether the request is complete. + * This function cannot check all the requests METHODS, for example, case you are working with POST, it will fail. + * + * @param begin is the first character of the sequence to analyse. + * @param end is the last character of the sequence + * @param length is the length of the total of bytes read, it is not the difference between end and begin. + * + * @return It returns 1 when the request is complete and 0 otherwise. + */ +inline int url_is_request_complete(char *begin, char *end, size_t length) { + + if ( begin == end) { + //Message cannot be complete when first and last address are the same + return 0; + } + + //This math to verify the last is valid, because we are discarding the POST + if (length > 4) { + begin = end - 4; + } + + return (strstr(begin, "\r\n\r\n"))?1:0; +} + +/** + * Find protocol + * + * Search for the string ' HTTP/' in the message given. + * + * @param s is the start of the user request. + * @return + */ +inline char *url_find_protocol(char *s) { + while(*s) { + // find the next space + while (*s && *s != ' ') s++; + + // is it SPACE + "HTTP/" ? + if(*s && !strncmp(s, " HTTP/", 6)) break; + else s++; + } + + return s; +} + +/** + * Map query string + * + * Map the query string fields that will be decoded. + * This functions must be called after to check the presence of query strings, + * here we are assuming that you already tested this. + * + * @param out the pointer to pointers that will be used to map + * @param url the input url that we are decoding. + * + * @return It returns the number of total variables in the query string. + */ +int url_map_query_string(char **out, char *url) { + (void)out; + (void)url; + int count = 0; + + //First we try to parse considering that there was not URL encode process + char *moveme = url; + char *ptr; + + //We always we have at least one here, so I can set this. + out[count++] = moveme; + while(moveme) { + ptr = strchr((moveme+1), '&'); + if(ptr) { + out[count++] = ptr; + } + + moveme = ptr; + } + + //I could not find any '&', so I am assuming now it is like '%26' + if (count == 1) { + moveme = url; + while(moveme) { + ptr = strchr((moveme+1), '%'); + if(ptr) { + char *test = (ptr+1); + if (!strncmp(test, "3f", 2) || !strncmp(test, "3F", 2)) { + out[count++] = ptr; + } + } + moveme = ptr; + } + } + + return count; +} + +/** + * Parse query string + * + * Parse the query string mapped and store it inside output. + * + * @param output is a vector where I will store the string. + * @param max is the maximum length of the output + * @param map the map done by the function url_map_query_string. + * @param total the total number of variables inside map + * + * @return It returns 0 on success and -1 otherwise + */ +int url_parse_query_string(char *output, size_t max, char **map, int total) { + if(!total) { + return 0; + } + + int counter, next; + size_t length; + char *end; + char *begin = map[0]; + char save; + size_t copied = 0; + for(counter = 0, next=1 ; next <= total ; ++counter, ++next) { + if (next != total) { + end = map[next]; + length = (size_t) (end - begin); + save = *end; + *end = 0x00; + } else { + length = strlen(begin); + end = NULL; + } + length++; + + if (length > (max - copied)) { + error("Parsing query string: we cannot parse a query string so big"); + break; + } + + if(!url_decode_r(output, begin, length)) { + return -1; + } + length = strlen(output); + copied += length; + output += length; + + begin = end; + if (begin) { + *begin = save; + } + } + + return 0; +} diff --git a/libnetdata/url/url.h b/libnetdata/url/url.h new file mode 100644 index 0000000..10f3fe1 --- /dev/null +++ b/libnetdata/url/url.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_URL_H +#define NETDATA_URL_H 1 + +#include "../libnetdata.h" + +// ---------------------------------------------------------------------------- +// URL encode / decode +// code from: http://www.geekhideout.com/urlcode.shtml + +/* Converts a hex character to its integer value */ +extern char from_hex(char ch); + +/* Converts an integer value to its hex character*/ +extern char to_hex(char code); + +/* Returns a url-encoded version of str */ +/* IMPORTANT: be sure to free() the returned string after use */ +extern char *url_encode(char *str); + +/* Returns a url-decoded version of str */ +/* IMPORTANT: be sure to free() the returned string after use */ +extern char *url_decode(char *str); + +extern char *url_decode_r(char *to, char *url, size_t size); + +#define WEB_FIELDS_MAX 400 +extern int url_map_query_string(char **out, char *url); +extern int url_parse_query_string(char *output, size_t max, char **map, int total); + +extern int url_is_request_complete(char *begin,char *end,size_t length); +extern char *url_find_protocol(char *s); + +#endif /* NETDATA_URL_H */ |